From ee146b835e4cb394c95de20e13adb23592f811ff Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Mon, 21 Oct 2024 23:21:29 +0200 Subject: [PATCH 01/38] add base version of #1890 --- api/views.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/api/views.py b/api/views.py index 8e2391f92..00823b767 100644 --- a/api/views.py +++ b/api/views.py @@ -30,6 +30,10 @@ from omi.dialects.oep.compiler import JSONCompiler from omi.structure import OEPMetadata from rest_framework import generics, status +from rest_framework.permissions import IsAuthenticated + +# views.py +from rest_framework.response import Response from rest_framework.views import APIView import api.parser @@ -42,12 +46,15 @@ from api.serializers import ( EnergyframeworkSerializer, EnergymodelSerializer, + ScenarioBundleScenarioDatasetSerializer, ScenarioDataTablesSerializer, ) +from api.sparql_helpers import add_datasets_to_scenario, remove_datasets_from_scenario from dataedit.models import Embargo from dataedit.models import Schema as DBSchema from dataedit.models import Table as DBTable from dataedit.views import get_tag_keywords_synchronized_metadata, schema_whitelist +from factsheet.permission_decorator import only_if_user_is_owner_of_scenario_bundle from modelview.models import Energyframework, Energymodel from oeplatform.settings import PLAYGROUNDS, UNVERSIONED_SCHEMAS, USE_LOEP, USE_ONTOP @@ -1571,3 +1578,58 @@ class ScenarioDataTablesListAPIView(generics.ListAPIView): topic = "scenario" queryset = DBTable.objects.filter(schema__name=topic) serializer_class = ScenarioDataTablesSerializer + + +class ManageScenarioDatasets(APIView): + permission_classes = [IsAuthenticated] # Require authentication + + @only_if_user_is_owner_of_scenario_bundle + def post(self, request): + serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data) + if serializer.is_valid(): + scenario_uuid = serializer.validated_data["scenario"] + datasets = serializer.validated_data["dataset"] + dataset_type = serializer.validated_data["type"] + + # Add datasets to the scenario in the bundle (implementation depends + # on your model) + success = add_datasets_to_scenario(scenario_uuid, datasets, dataset_type) + + if success: + return Response( + {"message": "Datasets added successfully"}, + status=status.HTTP_200_OK, + ) + else: + return Response( + {"error": "Failed to add datasets"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + @only_if_user_is_owner_of_scenario_bundle + def delete(self, request): + serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data) + if serializer.is_valid(): + scenario_uuid = serializer.validated_data["scenario"] + datasets = serializer.validated_data["dataset"] + dataset_type = serializer.validated_data["type"] + + # Remove datasets from the scenario in the bundle + success = remove_datasets_from_scenario( + scenario_uuid, datasets, dataset_type + ) + + if success: + return Response( + {"message": "Datasets removed successfully"}, + status=status.HTTP_200_OK, + ) + else: + return Response( + {"error": "Failed to remove datasets"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) From 454ac9aaacc320ceae65e4787269ca33e50ace4c Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Mon, 21 Oct 2024 23:47:07 +0200 Subject: [PATCH 02/38] change to post method as this seems to be the correct way of doing things --- sparql_query/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sparql_query/views.py b/sparql_query/views.py index b5d9b0968..4ca9757d9 100644 --- a/sparql_query/views.py +++ b/sparql_query/views.py @@ -2,7 +2,7 @@ from django.core.exceptions import SuspiciousOperation from django.http import HttpResponseBadRequest, JsonResponse from django.shortcuts import render -from django.views.decorators.http import require_GET +from django.views.decorators.http import require_POST from oeplatform.settings import OEKG_SPARQL_ENDPOINT_URL from sparql_query.utils import validate_sparql_query @@ -14,9 +14,9 @@ def main_view(request): return response -@require_GET +@require_POST def sparql_endpoint(request): - sparql_query = request.GET.get("query", "") + sparql_query = request.POST.get("query", "") if not sparql_query: return HttpResponseBadRequest("Missing 'query' parameter.") @@ -28,7 +28,7 @@ def sparql_endpoint(request): headers = {"Accept": "application/sparql-results+json"} - response = requests.get( + response = requests.post( endpoint_url, params={"query": sparql_query}, headers=headers ) From 0a90ca07044a6141b79af114e0ef1954990a4ef3 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Mon, 21 Oct 2024 23:47:43 +0200 Subject: [PATCH 03/38] add developer specific directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bf77ea202..f7fac0ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ venv*/ 0_env/ /envs /node_env +/oep-django-5 .DS_Store From 4ade232e826046a44363b791a86aa627b54d18e8 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Mon, 21 Oct 2024 23:48:27 +0200 Subject: [PATCH 04/38] remove deprecated comments --- dataedit/views.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dataedit/views.py b/dataedit/views.py index 0eb9976a7..673fa1578 100644 --- a/dataedit/views.py +++ b/dataedit/views.py @@ -2352,9 +2352,6 @@ def post(self, request, schema, table, review_id): Handle POST requests for contributor's review. Merges and updates the review data in the PeerReview table. - Missing parts: - - merge contributor field review and reviewer field review - Args: request (HttpRequest): The incoming HTTP POST request. schema (str): The schema of the table. @@ -2364,9 +2361,6 @@ def post(self, request, schema, table, review_id): Returns: HttpResponse: Rendered HTML response for contributor review. - Note: - This method has some missing parts regarding the merging of contributor - and reviewer field review. """ context = {} From 9f0815ad298706d38757141a28d4badbc94d6ff1 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Mon, 21 Oct 2024 23:52:28 +0200 Subject: [PATCH 05/38] add base serializer for scenario dataset post payload #1890 --- api/serializers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/api/serializers.py b/api/serializers.py index 0f80d4faa..eb5a2110f 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -53,3 +53,18 @@ class Meta: model = Table # fields = ["id", "model_name", "acronym", "url"] fields = ["id", "name", "human_readable_name", "url"] + + +class ScenarioBundleScenarioDatasetSerializer(serializers.Serializer): + scenario = serializers.UUIDField(required=True) # Validate the scenario UUID + dataset = serializers.ListField( + child=serializers.CharField(max_length=255), required=True + ) # List of dataset table names + type = serializers.ChoiceField( + choices=["input", "output"], required=True + ) # Type: input or output + + def __getitem__(self, items): + print(type(items), items) + if items is None: + return None From 5da67ce976d75a7160cebd7e195cfb67ff5212d0 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Mon, 21 Oct 2024 23:53:47 +0200 Subject: [PATCH 06/38] base implementation of sparql endpoint methods to add or remove datasets #1890 (remove would be part of the delete method). --- api/sparql_helpers.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 api/sparql_helpers.py diff --git a/api/sparql_helpers.py b/api/sparql_helpers.py new file mode 100644 index 000000000..71b2a34b7 --- /dev/null +++ b/api/sparql_helpers.py @@ -0,0 +1,50 @@ +import requests + +from factsheet.oekg.connection import update_endpoint + + +def add_datasets_to_scenario(scenario_uuid, datasets, dataset_type): + """ + Function to add datasets to a scenario bundle in Jena Fuseki. + """ + for dataset in datasets: + sparql_query = f""" + PREFIX oeo: + INSERT DATA {{ + GRAPH {{ + oeo:{dataset} a oeo:{dataset_type}Dataset . + }} + }} + """ + response = send_sparql_update(sparql_query) + if not response.ok: + return False # Return False if any query fails + return True + + +def remove_datasets_from_scenario(scenario_uuid, datasets, dataset_type): + """ + Function to remove datasets from a scenario bundle in Jena Fuseki. + """ + for dataset in datasets: + sparql_query = f""" + PREFIX oeo: + DELETE DATA {{ + GRAPH {{ + oeo:{dataset} a oeo:{dataset_type}Dataset . + }} + }} + """ + response = send_sparql_update(sparql_query) + if not response.ok: + return False # Return False if any query fails + return True + + +def send_sparql_update(query): + """ + Helper function to send a SPARQL update query to Fuseki. + """ + headers = {"Content-Type": "application/sparql-update"} + response = requests.post(update_endpoint, data=query, headers=headers) + return response From ccfa1cb5039a0fff792101b322955ab505bb3cea Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 22 Oct 2024 00:14:12 +0200 Subject: [PATCH 07/38] adapt serializer to new payload structure & implement base validation for all fields #1890 --- api/serializers.py | 53 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index eb5a2110f..fe7565cf7 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -1,3 +1,6 @@ +from re import match +from uuid import UUID + from django.urls import reverse from rest_framework import serializers @@ -55,16 +58,48 @@ class Meta: fields = ["id", "name", "human_readable_name", "url"] -class ScenarioBundleScenarioDatasetSerializer(serializers.Serializer): - scenario = serializers.UUIDField(required=True) # Validate the scenario UUID - dataset = serializers.ListField( - child=serializers.CharField(max_length=255), required=True - ) # List of dataset table names +class DatasetSerializer(serializers.Serializer): + name = serializers.CharField(max_length=255, required=True) # Dataset table name type = serializers.ChoiceField( choices=["input", "output"], required=True ) # Type: input or output - def __getitem__(self, items): - print(type(items), items) - if items is None: - return None + # Custom validation for 'name' + def validate_name(self, value): + # Use regex to allow alphanumeric characters and underscores + if not match(r"^[\w]+$", value): + raise serializers.ValidationError( + "Dataset name should contain only" + "alphanumeric characters and underscores." + ) + # Add any additional custom validation logic here + return value + + +class ScenarioBundleScenarioDatasetSerializer(serializers.Serializer): + scenario = serializers.UUIDField(required=True) # Validate the scenario UUID + dataset = serializers.ListField( + child=DatasetSerializer(), required=True + ) # List of datasets with 'name' and 'type' + + # Custom validation for 'scenario' + def validate_scenario(self, value): + try: + UUID(str(value)) + except ValueError: + raise serializers.ValidationError("Invalid UUID format for scenario.") + # Add any additional custom validation logic here + return value + + # Custom validation for the entire dataset list + def validate_dataset(self, value): + if not value: + raise serializers.ValidationError("The dataset list cannot be empty.") + + # Check for duplicates in dataset names + dataset_names = [dataset["name"] for dataset in value] + if len(dataset_names) != len(set(dataset_names)): + raise serializers.ValidationError("Dataset names must be unique.") + + # Add any additional custom validation logic here + return value From 8dbd4e9919bd78d711eb1a68b974a3a338229cca Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 22 Oct 2024 00:25:35 +0200 Subject: [PATCH 08/38] enhance delete endpoint #1890 --- api/views.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/api/views.py b/api/views.py index 00823b767..ac8c9b21b 100644 --- a/api/views.py +++ b/api/views.py @@ -1614,22 +1614,26 @@ def delete(self, request): if serializer.is_valid(): scenario_uuid = serializer.validated_data["scenario"] datasets = serializer.validated_data["dataset"] - dataset_type = serializer.validated_data["type"] - # Remove datasets from the scenario in the bundle - success = remove_datasets_from_scenario( - scenario_uuid, datasets, dataset_type - ) + # Iterate over each dataset to process it properly + for dataset in datasets: + dataset_name = dataset["name"] + dataset_type = dataset["type"] - if success: - return Response( - {"message": "Datasets removed successfully"}, - status=status.HTTP_200_OK, - ) - else: - return Response( - {"error": "Failed to remove datasets"}, - status=status.HTTP_400_BAD_REQUEST, + # Remove the dataset from the scenario in the bundle + success = remove_datasets_from_scenario( + scenario_uuid, dataset_name, dataset_type ) + if not success: + return Response( + {"error": f"Failed to remove dataset {dataset_name}"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + return Response( + {"message": "Datasets removed successfully"}, + status=status.HTTP_200_OK, + ) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) From 8d82338ff15e5f88e45a88b31be38abc9dbd0a6e Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 22 Oct 2024 00:26:09 +0200 Subject: [PATCH 09/38] enhance post endpoint #1890 --- api/views.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/api/views.py b/api/views.py index ac8c9b21b..51035947c 100644 --- a/api/views.py +++ b/api/views.py @@ -1589,22 +1589,27 @@ def post(self, request): if serializer.is_valid(): scenario_uuid = serializer.validated_data["scenario"] datasets = serializer.validated_data["dataset"] - dataset_type = serializer.validated_data["type"] - # Add datasets to the scenario in the bundle (implementation depends - # on your model) - success = add_datasets_to_scenario(scenario_uuid, datasets, dataset_type) + # Iterate over each dataset to process it properly + for dataset in datasets: + dataset_name = dataset["name"] + dataset_type = dataset["type"] - if success: - return Response( - {"message": "Datasets added successfully"}, - status=status.HTTP_200_OK, - ) - else: - return Response( - {"error": "Failed to add datasets"}, - status=status.HTTP_400_BAD_REQUEST, + # Add datasets to the scenario in the bundle (implementation depends + # on your model) + success = add_datasets_to_scenario( + scenario_uuid, dataset_name, dataset_type ) + if not success: + return Response( + {"error": "Failed to add datasets"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + return Response( + {"message": "Datasets added successfully"}, + status=status.HTTP_200_OK, + ) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) From 4b5dce591dc3666925b8a6fd80ef7d3d2ca489d9 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 22 Oct 2024 00:27:10 +0200 Subject: [PATCH 10/38] change behavior to only handle one data resource at a time #1890 --- api/sparql_helpers.py | 47 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/api/sparql_helpers.py b/api/sparql_helpers.py index 71b2a34b7..c45cc8c45 100644 --- a/api/sparql_helpers.py +++ b/api/sparql_helpers.py @@ -3,41 +3,40 @@ from factsheet.oekg.connection import update_endpoint -def add_datasets_to_scenario(scenario_uuid, datasets, dataset_type): +def add_datasets_to_scenario(scenario_uuid, dataset_name, dataset_type): """ Function to add datasets to a scenario bundle in Jena Fuseki. """ - for dataset in datasets: - sparql_query = f""" - PREFIX oeo: - INSERT DATA {{ - GRAPH {{ - oeo:{dataset} a oeo:{dataset_type}Dataset . - }} + + sparql_query = f""" + PREFIX oeo: + INSERT DATA {{ + GRAPH {{ + oeo:{dataset_name} a oeo:{dataset_type}Dataset . }} - """ - response = send_sparql_update(sparql_query) - if not response.ok: - return False # Return False if any query fails + }} + """ + response = send_sparql_update(sparql_query) + if not response.ok: + return False # Return False if any query fails return True -def remove_datasets_from_scenario(scenario_uuid, datasets, dataset_type): +def remove_datasets_from_scenario(scenario_uuid, dataset_name, dataset_type): """ Function to remove datasets from a scenario bundle in Jena Fuseki. """ - for dataset in datasets: - sparql_query = f""" - PREFIX oeo: - DELETE DATA {{ - GRAPH {{ - oeo:{dataset} a oeo:{dataset_type}Dataset . - }} + sparql_query = f""" + PREFIX oeo: + DELETE DATA {{ + GRAPH {{ + oeo:{dataset_name} a oeo:{dataset_type}Dataset . }} - """ - response = send_sparql_update(sparql_query) - if not response.ok: - return False # Return False if any query fails + }} + """ + response = send_sparql_update(sparql_query) + if not response.ok: + return False # Return False if any query fails return True From 1fbf3332197783f93b8044b1c2ea48033295eb50 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 23 Jan 2025 12:57:31 +0100 Subject: [PATCH 11/38] add api endpoint #1890 --- api/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/urls.py b/api/urls.py index 726fd8abe..aa4234e46 100644 --- a/api/urls.py +++ b/api/urls.py @@ -203,4 +203,9 @@ views.ScenarioDataTablesListAPIView.as_view(), name="list-scenario-datasets", ), + re_path( + r"^v0/scenario-bundle/scenario/manage-datasets/?$", + views.ManageScenarioDatasets.as_view(), + name="add-scenario-datasets", + ), ] From 54237381d2c2d14ade19f58c081badcb90f19623 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 29 Jan 2025 21:10:59 +0100 Subject: [PATCH 12/38] remove comment --- api/serializers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index fe7565cf7..e83dfe0de 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -72,7 +72,7 @@ def validate_name(self, value): "Dataset name should contain only" "alphanumeric characters and underscores." ) - # Add any additional custom validation logic here + return value @@ -88,7 +88,7 @@ def validate_scenario(self, value): UUID(str(value)) except ValueError: raise serializers.ValidationError("Invalid UUID format for scenario.") - # Add any additional custom validation logic here + return value # Custom validation for the entire dataset list @@ -101,5 +101,4 @@ def validate_dataset(self, value): if len(dataset_names) != len(set(dataset_names)): raise serializers.ValidationError("Dataset names must be unique.") - # Add any additional custom validation logic here return value From 84318714f85ead6450c6072652f375d96a5c237a Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Mon, 3 Feb 2025 14:32:25 +0100 Subject: [PATCH 13/38] add a readme to clarify the intended usage of the oekg app --- oekg/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 oekg/README.md diff --git a/oekg/README.md b/oekg/README.md new file mode 100644 index 000000000..aa285f205 --- /dev/null +++ b/oekg/README.md @@ -0,0 +1,5 @@ +# What is this app used for? + +The OEKG django app is used to encapsulate functionality to interact with the OEKG within the OEP. If one needs such functionality in another django app like `api` then the oekg app should be imported there. New functionality should also extend the oekg app. + +This includes variables and functions to connect to databases (like jenna fuseki) and to access or edit its content. The main libraries used here are rdfLib (broadly used in the facthseet app to create scenario bundles) and the SPARQLWrapper to formulate a Query as a string. The latter approach is more efficient as it avoids parsing data (like the Graph) to python data types. From 82b5a6359497331c2e0c40d4722369b530fb2876 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:31:50 +0100 Subject: [PATCH 14/38] move oekg sparql query definition file to oekg app --- api/sparql_helpers.py | 49 ---------------------------- oekg/sparqlQuery.py | 76 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 49 deletions(-) delete mode 100644 api/sparql_helpers.py create mode 100644 oekg/sparqlQuery.py diff --git a/api/sparql_helpers.py b/api/sparql_helpers.py deleted file mode 100644 index c45cc8c45..000000000 --- a/api/sparql_helpers.py +++ /dev/null @@ -1,49 +0,0 @@ -import requests - -from factsheet.oekg.connection import update_endpoint - - -def add_datasets_to_scenario(scenario_uuid, dataset_name, dataset_type): - """ - Function to add datasets to a scenario bundle in Jena Fuseki. - """ - - sparql_query = f""" - PREFIX oeo: - INSERT DATA {{ - GRAPH {{ - oeo:{dataset_name} a oeo:{dataset_type}Dataset . - }} - }} - """ - response = send_sparql_update(sparql_query) - if not response.ok: - return False # Return False if any query fails - return True - - -def remove_datasets_from_scenario(scenario_uuid, dataset_name, dataset_type): - """ - Function to remove datasets from a scenario bundle in Jena Fuseki. - """ - sparql_query = f""" - PREFIX oeo: - DELETE DATA {{ - GRAPH {{ - oeo:{dataset_name} a oeo:{dataset_type}Dataset . - }} - }} - """ - response = send_sparql_update(sparql_query) - if not response.ok: - return False # Return False if any query fails - return True - - -def send_sparql_update(query): - """ - Helper function to send a SPARQL update query to Fuseki. - """ - headers = {"Content-Type": "application/sparql-update"} - response = requests.post(update_endpoint, data=query, headers=headers) - return response diff --git a/oekg/sparqlQuery.py b/oekg/sparqlQuery.py new file mode 100644 index 000000000..9dcba0b3e --- /dev/null +++ b/oekg/sparqlQuery.py @@ -0,0 +1,76 @@ +from uuid import UUID + +import requests +from sparqlModels import DatasetConfig +from SPARQLWrapper import JSON, POST + +from factsheet.oekg.connection import sparql_wrapper_update, update_endpoint + + +def add_datasets_to_scenario(oekgDatasetConfig: DatasetConfig): + """ + Function to add datasets to a scenario bundle in Jena Fuseki. + """ + + new_dataset_uid = UUID() + + if oekgDatasetConfig.dataset_type == "input": + rel_property = "RO_0002233" + elif oekgDatasetConfig.dataset_type == "output": + rel_property = "RO_0002234" + + sparql_query = f""" + PREFIX oeo: + PREFIX rdfs: + + INSERT DATA {{ + GRAPH {{ + a oeo:OEO_00030030 ; + rdfs:label "{oekgDatasetConfig.dataset_label}" ; + oeo:has_iri "{oekgDatasetConfig.dataset_url}" ; + oeo:has_id "{oekgDatasetConfig.dataset_id}" ; + oeo:has_key "{oekgDatasetConfig.dataset_key}" . + + oeo:{rel_property} + . + }} + }} + """ # noqa + + print(sparql_query) + # response = send_sparql_update(sparql_query) + sparql_wrapper_update.setQuery(sparql_query) + sparql_wrapper_update.setMethod(POST) + sparql_wrapper_update.setReturnFormat(JSON) + response = sparql_wrapper_update.query() + http_response = response.response + if not http_response.status == 200: + return False # Return False if any query fails + return True + + +def remove_datasets_from_scenario(scenario_uuid, dataset_name, dataset_type): + """ + Function to remove datasets from a scenario bundle in Jena Fuseki. + """ + sparql_query = f""" + PREFIX oeo: + DELETE DATA {{ + GRAPH {{ + oeo:{dataset_name} a oeo:{dataset_type}Dataset . + }} + }} + """ + response = send_sparql_update(sparql_query) + if not response.ok: + return False # Return False if any query fails + return True + + +def send_sparql_update(query): + """ + Helper function to send a SPARQL update query to Fuseki. + """ + headers = {"Content-Type": "application/sparql-update"} + response = requests.post(update_endpoint, data=query, headers=headers) + return response From 9d98aa9e95a8ddb869b45451d00f9093a158f1ee Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:33:04 +0100 Subject: [PATCH 15/38] define interface for add oekg dataset: add datacalss based models to represent complex oekg objects like datasets and ease the parameter management --- oekg/sparqlModels.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 oekg/sparqlModels.py diff --git a/oekg/sparqlModels.py b/oekg/sparqlModels.py new file mode 100644 index 000000000..aa47ec4a6 --- /dev/null +++ b/oekg/sparqlModels.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass +from uuid import UUID + + +@dataclass +class DatasetConfig: + bundle_uuid: UUID + scenario_uuid: UUID + dataset_label: str + dataset_url: str + dataset_id: int + dataset_key: bool + dataset_type: str From 07ddf39eb3c220c4ba72335b58c593a5f1525b0f Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:34:32 +0100 Subject: [PATCH 16/38] add scenario bundle uid field to the serializer as it is part of the requests payload --- api/serializers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/serializers.py b/api/serializers.py index e83dfe0de..f0c797d60 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -77,6 +77,9 @@ def validate_name(self, value): class ScenarioBundleScenarioDatasetSerializer(serializers.Serializer): + scenario_bundle = serializers.UUIDField( + required=True + ) # Validate the scenario bundle UUID scenario = serializers.UUIDField(required=True) # Validate the scenario UUID dataset = serializers.ListField( child=DatasetSerializer(), required=True From a7887b0e1745ea03e1b3da7df32f4785a04330f7 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:37:16 +0100 Subject: [PATCH 17/38] adapt view code to changes in imported functionality --- api/views.py | 49 +++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/api/views.py b/api/views.py index 39affbacc..335921785 100644 --- a/api/views.py +++ b/api/views.py @@ -50,13 +50,13 @@ ScenarioBundleScenarioDatasetSerializer, ScenarioDataTablesSerializer, ) -from api.sparql_helpers import add_datasets_to_scenario, remove_datasets_from_scenario from dataedit.models import Embargo from dataedit.models import Schema as DBSchema from dataedit.models import Table as DBTable from dataedit.views import get_tag_keywords_synchronized_metadata, schema_whitelist -from factsheet.permission_decorator import only_if_user_is_owner_of_scenario_bundle +from factsheet.permission_decorator import post_only_if_user_is_owner_of_scenario_bundle from modelview.models import Energyframework, Energymodel +from oekg.sparqlQuery import add_datasets_to_scenario, remove_datasets_from_scenario from oeplatform.settings import PLAYGROUNDS, UNVERSIONED_SCHEMAS, USE_LOEP, USE_ONTOP if USE_LOEP: @@ -251,11 +251,11 @@ class Sequence(APIView): @api_exception def put(self, request, schema, sequence): if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS: - raise APIError('Schema is not in allowed set of schemes for upload') + raise APIError("Schema is not in allowed set of schemes for upload") if schema.startswith("_"): - raise APIError('Schema starts with _, which is not allowed') + raise APIError("Schema starts with _, which is not allowed") if request.user.is_anonymous: - raise APIError('User is anonymous', 401) + raise APIError("User is anonymous", 401) if actions.has_table(dict(schema=schema, sequence_name=sequence), {}): raise APIError("Sequence already exists", 409) return self.__create_sequence(request, schema, sequence, request.data) @@ -264,11 +264,11 @@ def put(self, request, schema, sequence): @require_delete_permission def delete(self, request, schema, sequence): if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS: - raise APIError('Schema is not in allowed set of schemes for upload') + raise APIError("Schema is not in allowed set of schemes for upload") if schema.startswith("_"): - raise APIError('Schema starts with _, which is not allowed') + raise APIError("Schema starts with _, which is not allowed") if request.user.is_anonymous: - raise APIError('User is anonymous', 401) + raise APIError("User is anonymous", 401) return self.__delete_sequence(request, schema, sequence, request.data) @load_cursor() @@ -378,9 +378,9 @@ def post(self, request, schema, table): :return: """ if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS: - raise APIError('Schema is not in allowed set of schemes for upload') + raise APIError("Schema is not in allowed set of schemes for upload") if schema.startswith("_"): - raise APIError('Schema starts with _, which is not allowed') + raise APIError("Schema starts with _, which is not allowed") json_data = request.data if "column" in json_data["type"]: @@ -430,11 +430,11 @@ def put(self, request, schema, table): :return: """ if schema not in PLAYGROUNDS and schema not in UNVERSIONED_SCHEMAS: - raise APIError('Schema is not in allowed set of schemes for upload') + raise APIError("Schema is not in allowed set of schemes for upload") if schema.startswith("_"): - raise APIError('Schema starts with _, which is not allowed') + raise APIError("Schema starts with _, which is not allowed") if request.user.is_anonymous: - raise APIError('User is anonymous', 401) + raise APIError("User is anonymous", 401) if actions.has_table(dict(schema=schema, table=table), {}): raise APIError("Table already exists", 409) json_data = request.data.get("query", {}) @@ -974,10 +974,10 @@ def get(self, request, schema, table, row_id=None): content_type="text/csv", session=session, ) - response["Content-Disposition"] = ( - 'attachment; filename="{schema}__{table}.csv"'.format( - schema=schema, table=table - ) + response[ + "Content-Disposition" + ] = 'attachment; filename="{schema}__{table}.csv"'.format( + schema=schema, table=table ) return response elif format == "datapackage": @@ -1005,10 +1005,10 @@ def get(self, request, schema, table, row_id=None): content_type="application/zip", session=session, ) - response["Content-Disposition"] = ( - 'attachment; filename="{schema}__{table}.zip"'.format( - schema=schema, table=table - ) + response[ + "Content-Disposition" + ] = 'attachment; filename="{schema}__{table}.zip"'.format( + schema=schema, table=table ) return response else: @@ -1586,10 +1586,11 @@ class ScenarioDataTablesListAPIView(generics.ListAPIView): class ManageScenarioDatasets(APIView): permission_classes = [IsAuthenticated] # Require authentication - @only_if_user_is_owner_of_scenario_bundle + @post_only_if_user_is_owner_of_scenario_bundle def post(self, request): serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data) if serializer.is_valid(): + bundle_uuid = serializer.validated_data["scenario_bundle"] scenario_uuid = serializer.validated_data["scenario"] datasets = serializer.validated_data["dataset"] @@ -1601,7 +1602,7 @@ def post(self, request): # Add datasets to the scenario in the bundle (implementation depends # on your model) success = add_datasets_to_scenario( - scenario_uuid, dataset_name, dataset_type + bundle_uuid, scenario_uuid, dataset_name, dataset_type ) if not success: return Response( @@ -1616,7 +1617,7 @@ def post(self, request): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - @only_if_user_is_owner_of_scenario_bundle + @post_only_if_user_is_owner_of_scenario_bundle def delete(self, request): serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data) if serializer.is_valid(): From 81bb8dfca3d0a190eb6b815e15056a9a6b3a1403 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:37:44 +0100 Subject: [PATCH 18/38] mark method as static since no self is used --- dataedit/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dataedit/models.py b/dataedit/models.py index 1ea828af4..8e01fde54 100644 --- a/dataedit/models.py +++ b/dataedit/models.py @@ -719,5 +719,6 @@ def filter_opr_by_table(schema, table): """ return PeerReview.objects.filter(schema=schema, table=table) + @staticmethod def filter_opr_by_id(opr_id): return PeerReview.objects.filter(id=opr_id).first() From cbb889c4ad1242840a4b5df17e11e6677c4650f5 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:41:40 +0100 Subject: [PATCH 19/38] add oekg:scenarioBundle access permission check to handle post requests (handling differs; requires other functionality) --- factsheet/permission_decorator.py | 63 ++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/factsheet/permission_decorator.py b/factsheet/permission_decorator.py index 773e6854b..b1df1228b 100644 --- a/factsheet/permission_decorator.py +++ b/factsheet/permission_decorator.py @@ -1,12 +1,21 @@ import json from functools import wraps -from django.http import HttpResponseForbidden +from django.http import HttpResponse, HttpResponseForbidden from factsheet.models import ScenarioBundleAccessControl def only_if_user_is_owner_of_scenario_bundle(view_func): + """ + Wrapper that checks if the current user is the owner of + the Scenario bundle. + + It determines the owner of the Scenario bundle by checking + the ScenarioBundleEditAccess model. The uid of the scenario + bundle is passed as a URL parameter or in the request body. + """ + @wraps(view_func) def _wrapped_view(request, *args, **kwargs): # Get the uid from the URL parameters or any other source. @@ -15,6 +24,7 @@ def _wrapped_view(request, *args, **kwargs): kwargs.get("uid") or json.loads(request.body).get("uid") or json.loads(request.body).get("id") + or request.GET.get("id") ) except Exception: uid = request.GET.get("id") @@ -38,3 +48,54 @@ def _wrapped_view(request, *args, **kwargs): return HttpResponseForbidden("Access Denied") return _wrapped_view + + +def post_only_if_user_is_owner_of_scenario_bundle(view_func): + """ + Wrapper that checks if the current user is the owner of + the Scenario bundle. This is a decorator for POST requests. + + It differs from the only_if_user_is_owner_of_scenario_bundle + as it depends on data from the request body instead of URL parameters. + + It determines the owner of the Scenario bundle by checking + the ScenarioBundleEditAccess model. The uid of the scenario + bundle is passed as a URL parameter or in the request body. + """ + + @wraps(view_func) + def _wrapped_view(view_instance, request, *args, **kwargs): + # Get the uid from the URL parameters or any other source. + + bundle_uid = kwargs.get("uid") or request.data.get("scenario_bundle") + if not bundle_uid: + return HttpResponse( + "The bundle_uid (scenario bundle) was not found in" + "the request body or URL parameters", + ) + + user_id = request.user + if not user_id: + return HttpResponse( + "The user id was not found in the request body or URL parameters", + ) + + try: + # Retrieve the ScenarioBundleEditAccess object based on the uid. + scenario_bundle_access = ScenarioBundleAccessControl.objects.get( + bundle_id=bundle_uid + ) + except ScenarioBundleAccessControl.DoesNotExist: + # Handle the case where the ScenarioBundleEditAccess with the + # provided uid is not found. + return HttpResponseForbidden( + "UID not available or scenario bundle does not exist. Access denied" + ) + + # Check if the current user is the owner (creator) of the Scenario bundle. + if request.user == scenario_bundle_access.owner_user: + return view_func(request, *args, **kwargs) + else: + return HttpResponseForbidden("Access Denied") + + return _wrapped_view From e483d0054b6024246f85cda7d1842d9f16742336 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:43:45 +0100 Subject: [PATCH 20/38] =?UTF-8?q?add=20constant=20to=20simplify=20usage=20?= =?UTF-8?q?of=20sparqlWrapper=C2=B4s=20raw=20sparql=20query=20execution=20?= =?UTF-8?q?capability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- factsheet/oekg/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/factsheet/oekg/connection.py b/factsheet/oekg/connection.py index c43bf48f6..b96759078 100644 --- a/factsheet/oekg/connection.py +++ b/factsheet/oekg/connection.py @@ -39,6 +39,7 @@ update_endpoint = "http://%(host)s:%(port)s/%(name)s/update" % rdfdb sparql = SPARQLWrapper(query_endpoint) +sparql_wrapper_update = SPARQLWrapper(update_endpoint) store = sparqlstore.SPARQLUpdateStore() From 76ca79643fcedff60d4ad9c41dfe83cf032900c0 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:45:17 +0100 Subject: [PATCH 21/38] add docsting description to make usage more clear: multiple possibilities to send SparqlQueries to the oeplatform, this one differs from the API route. --- oekg/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/oekg/views.py b/oekg/views.py index 27c49fbf6..e5681ae65 100644 --- a/oekg/views.py +++ b/oekg/views.py @@ -4,8 +4,8 @@ from django.shortcuts import render from django.views.decorators.http import require_POST -from oeplatform.settings import OEKG_SPARQL_ENDPOINT_URL from oekg.utils import validate_sparql_query +from oeplatform.settings import OEKG_SPARQL_ENDPOINT_URL def main_view(request): @@ -16,6 +16,9 @@ def main_view(request): @require_POST def sparql_endpoint(request): + """ + Public SPARQL endpoint. Must only allow read queries. + """ sparql_query = request.POST.get("query", "") if not sparql_query: From 37b6dfe8528abc296f8cbd75e9ce534fe354c518 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 14:49:23 +0100 Subject: [PATCH 22/38] remove parameters from dependency array to avoid loop causing memory overflow --- factsheet/frontend/src/components/scenarioBundle.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/factsheet/frontend/src/components/scenarioBundle.js b/factsheet/frontend/src/components/scenarioBundle.js index 5186d7418..72bd0c9c8 100644 --- a/factsheet/frontend/src/components/scenarioBundle.js +++ b/factsheet/frontend/src/components/scenarioBundle.js @@ -410,7 +410,8 @@ function Factsheet(props) { const filteredResult = filterByValue(selectedTechnologies, technologies); - setSelectedTechnologiesTree(filteredResult[0]["children"]); + // setSelectedTechnologiesTree(filteredResult[0]); + // setSelectedTechnologies(s) function getAllNodeIds(nodes) { let ids = []; @@ -426,13 +427,9 @@ function Factsheet(props) { const allIds = getAllNodeIds(filteredResult[0]["children"]); setAllNodeIds(allIds); - }, []); - - - - + }, []); - }, [selectedTechnologies, technologies]); + }, []); // Todo: check if the empty dependency array raises errors const handleSaveFactsheet = () => { setOpenBackDrop(true); @@ -2010,6 +2007,7 @@ function Factsheet(props) { ] } const scenario_count = 'Scenarios' + ' (' + scenarios.length + ')'; + console.log(scenarios); const renderScenariosOverview = () => ( { From 0f799d8fe4f336496ddb2688596fde871ff3a7c9 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Feb 2025 16:23:48 +0100 Subject: [PATCH 23/38] fix import path --- oekg/sparqlQuery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oekg/sparqlQuery.py b/oekg/sparqlQuery.py index 9dcba0b3e..08ab94a28 100644 --- a/oekg/sparqlQuery.py +++ b/oekg/sparqlQuery.py @@ -1,10 +1,10 @@ from uuid import UUID import requests -from sparqlModels import DatasetConfig from SPARQLWrapper import JSON, POST from factsheet.oekg.connection import sparql_wrapper_update, update_endpoint +from oekg.sparqlModels import DatasetConfig def add_datasets_to_scenario(oekgDatasetConfig: DatasetConfig): From e7d3084ad5514c5623303d463d21810f54685047 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:24:37 +0100 Subject: [PATCH 24/38] add documentation page for new oekg-scenario-dataset api --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 62a2de40c..fb82ad5aa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -116,6 +116,7 @@ nav: # - Draft open-API schema: oeplatform-code/web-api/oedb-rest-api/swagger-ui.html - OEKG API: - oeplatform-code/web-api/oekg-api/index.md + - Edit scenario datasets: oeplatform-code/web-api/oekg-api/scenario-dataset.md - Features: - oeplatform-code/features/index.md - metaBuilder Metadata creation: From 098c4a6909504977184a857b5777f5674f2d80f0 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:25:09 +0100 Subject: [PATCH 25/38] add documentation for oekg-scenario-dataset api --- .../web-api/oekg-api/scenario-dataset.md | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 docs/oeplatform-code/web-api/oekg-api/scenario-dataset.md diff --git a/docs/oeplatform-code/web-api/oekg-api/scenario-dataset.md b/docs/oeplatform-code/web-api/oekg-api/scenario-dataset.md new file mode 100644 index 000000000..4976f0d82 --- /dev/null +++ b/docs/oeplatform-code/web-api/oekg-api/scenario-dataset.md @@ -0,0 +1,86 @@ +# API to manipulate dataset for in a scenario + +## Basics + +This functionality is part of the oeplatform web api and can be accessed sending POST requests to this endpoint: + +- `https://openenergyplatform.org/api/v0/scenario-bundle/scenario/manage-datasets/` + +You need a client to send http requests. + +- Python: requests +- linux: curl +- Client software: HTTPie +- and more + +For authorization you must use you API Token which can be optioned form the profile page on the OEP. In case you leaked it you can also reset the token. See section Access restrictions and future consideration. + +The post request must contain a body with payload: + +``` json +{ + "scenario_bundle": "1970ba29-155b-6e70-7c22-c12a33244a24", + "scenario": "5d95247d-df75-a95b-7286-dd4b3bc1c92a", + "datasets": [ + { + "name": "eu_leg_data_2017_eio_ir_article23_t3", + "type": "input" + }, + { + "name": "testetstetst", + "type": "output" + }, + { + "name": "WS_23_24_B665_2025_01_23", + "external_url": "https://databus.openenergyplatform.org/koubaa/LLEC_Dataset/WS_23_24_B665_2025_01_23/WS_23_24_B665_2025_01_23", + "type": "output" + }, + ] +} +``` + +- scenario_bundle: can be obtained from the scenario bundle website (copy from url) +- scenario: can also be obtained from the website; In the scenario tab there is a button to copy each scenario UID +- datasets: Is a list of all datasets you want to add +- name: you can lookup a table name that is available on the OEP and published in the scenario topic. The technical name is required here. +- type: Chose either "input" or "output" here, the dataset will be added to the related section in the scenario +- external_url: This parameter is OPTIONAL to be precise you dont have to use it if you are adding a dataset that is available on the OEP. You can use it to link external datasets but it requires you to first register them on the databus to get a persistent id. The databus offers a Publishing page. After the dataset is registered you can copy the file or version URL and add it to the external_url field. + +- +- The databus also offers a API in case you want to register in bulk + +## Example using curl + +``` bash +curl --request POST \ + --url https://openenergyplatform.org/api/v0/scenario-bundle/scenario/manage-datasets/ \ + --header 'Authorization: Token ' \ + --header 'Content-Type: application/json' \ + --data '{ + "scenario_bundle": "1970ba29-155b-6e70-7c22-c12a33244a24", + "scenario": "5d95247d-df75-a95b-7286-dd4b3bc1c92a", + "datasets": [ + { + "name": "eu_leg_data_2017_eio_ir_article23_t3", + "type": "input" + }, + { + "name": "testetstetst", + "type": "output" + }, + { + "name": "WS_23_24_B665_2025_01_23", + "external_url": "https://databus.openenergyplatform.org/koubaa/LLEC_Dataset/WS_23_24_B665_2025_01_23/WS_23_24_B665_2025_01_23", + "type": "output" + }, + { + "name": "first_test_table", + "type": "output" + } + ] +}' +``` + +## Access restrictions and future consideration + +Currently only the person who created a scenario bundle is able to edit its content. Soon this will change and users will be able to assign a group to a bundle. Groups are also used to manage access to dataset resources on the OEP here we will use the same groups. Once this is implemented you will have to create/assign a group to you bundle and then you can collaborate on the editing. From 41a6d98175d25e7dc259406d7a77c3b874a9348d Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:26:43 +0100 Subject: [PATCH 26/38] extend the serializer to validate user data fields: - also check if datasets are availabe in database - check if a valid external url was used (databus url required) --- api/serializers.py | 76 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index f0c797d60..4689c5a4b 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -4,8 +4,10 @@ from django.urls import reverse from rest_framework import serializers +from dataedit.helper import get_readable_table_name from dataedit.models import Table from modelview.models import Energyframework, Energymodel +from oeplatform.settings import URL class EnergyframeworkSerializer(serializers.ModelSerializer): @@ -59,21 +61,73 @@ class Meta: class DatasetSerializer(serializers.Serializer): - name = serializers.CharField(max_length=255, required=True) # Dataset table name - type = serializers.ChoiceField( - choices=["input", "output"], required=True - ) # Type: input or output - - # Custom validation for 'name' + name = serializers.CharField(max_length=255, required=True) + external_url = serializers.URLField( + max_length=1000, required=False, allow_null=True + ) + type = serializers.ChoiceField(choices=["input", "output"], required=True) + # title = serializers.SerializerMethodField() + + # ✅ Basic validation for 'name' (regex check only) def validate_name(self, value): - # Use regex to allow alphanumeric characters and underscores if not match(r"^[\w]+$", value): raise serializers.ValidationError( - "Dataset name should contain only" - "alphanumeric characters and underscores." + "Dataset name should contain only alphanumeric characters " + "and underscores." + ) + return value # Don't check DB here, do it in validate() + + # ✅ Main validation logic (includes db check for object existence) + def validate(self, data): + name = data.get("name") + external_url = data.get("external_url") + + if external_url: + # ✅ External URL provided → Skip DB check for 'name' + if not external_url.startswith("https://databus.openenergyplatform.org"): + raise serializers.ValidationError( + { + "external_url": ( + "If you want to link distributions stored outside the OEP, " + "please use the Databus: " + "https://databus.openenergyplatform.org/app/publish-wizard " + "to register your data and use the file or version URI as " + "a persistent identifier." + ) + } + ) + data["name"] = f"{name} (external dataset)" + else: + # ✅ No external URL → Validate 'name' in the database + if not Table.objects.filter(name=name).exists(): + raise serializers.ValidationError( + { + "name": f"Dataset '{name}' does not exist in the database." + "If you want to add links to external distributions please " + "add 'external_url' to the request body." + } + ) + full_label = self.get_title(data) + if full_label: + data["name"] = full_label + + # ✅ Generate internal distribution URL + reversed_url = reverse( + "dataedit:view", + kwargs={"schema": "scenario", "table": name}, ) + data["external_url"] = f"{URL}{reversed_url}" - return value + return data # Return updated data with 'distribution_url' if applicable + + def get_title(self, data): + name = data.get("name") + # ✅ Generate internal distribution label + full_label = get_readable_table_name(table_obj=Table.objects.get(name=name)) + if full_label: + return full_label + else: + return None class ScenarioBundleScenarioDatasetSerializer(serializers.Serializer): @@ -81,7 +135,7 @@ class ScenarioBundleScenarioDatasetSerializer(serializers.Serializer): required=True ) # Validate the scenario bundle UUID scenario = serializers.UUIDField(required=True) # Validate the scenario UUID - dataset = serializers.ListField( + datasets = serializers.ListField( child=DatasetSerializer(), required=True ) # List of datasets with 'name' and 'type' From 55d1cf5344b0442915b7548791f3fc4437793cf9 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:26:56 +0100 Subject: [PATCH 27/38] rename view --- api/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/urls.py b/api/urls.py index aa4234e46..d66f8075c 100644 --- a/api/urls.py +++ b/api/urls.py @@ -205,7 +205,7 @@ ), re_path( r"^v0/scenario-bundle/scenario/manage-datasets/?$", - views.ManageScenarioDatasets.as_view(), + views.ManageOekgScenarioDatasets.as_view(), name="add-scenario-datasets", ), ] From f12bb445e7466c7d3e37a0c30b62ad07c7138831 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:27:52 +0100 Subject: [PATCH 28/38] add module: utility for api functions with function that generates a config object based oin user request data --- api/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 api/utils.py diff --git a/api/utils.py b/api/utils.py new file mode 100644 index 000000000..de18f06d0 --- /dev/null +++ b/api/utils.py @@ -0,0 +1,14 @@ +""" +Collection of utility functions for the API used to define various action +like processing steps. +""" + +from oekg.sparqlModels import DatasetConfig + + +def get_dataset_configs(validated_data) -> list[DatasetConfig]: + """Converts validated serializer data into a list of DatasetConfig objects.""" + return [ + DatasetConfig.from_serializer_data(validated_data, dataset_entry) + for dataset_entry in validated_data["datasets"] + ] From 8d1b41e4b22dbab1c28d411737e33aaf95fac2ee Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:30:52 +0100 Subject: [PATCH 29/38] post view now is more lean and keeps functionality to its responsibilities - comment delete view which is not yet implemented --- api/views.py | 107 +++++++++++++++++++++++---------------------------- 1 file changed, 49 insertions(+), 58 deletions(-) diff --git a/api/views.py b/api/views.py index 335921785..9db97c07b 100644 --- a/api/views.py +++ b/api/views.py @@ -50,13 +50,16 @@ ScenarioBundleScenarioDatasetSerializer, ScenarioDataTablesSerializer, ) +from api.utils import get_dataset_configs from dataedit.models import Embargo from dataedit.models import Schema as DBSchema from dataedit.models import Table as DBTable from dataedit.views import get_tag_keywords_synchronized_metadata, schema_whitelist from factsheet.permission_decorator import post_only_if_user_is_owner_of_scenario_bundle from modelview.models import Energyframework, Energymodel -from oekg.sparqlQuery import add_datasets_to_scenario, remove_datasets_from_scenario + +# from oekg.sparqlQuery import remove_datasets_from_scenario +from oekg.utils import process_datasets_sparql_query from oeplatform.settings import PLAYGROUNDS, UNVERSIONED_SCHEMAS, USE_LOEP, USE_ONTOP if USE_LOEP: @@ -1583,66 +1586,54 @@ class ScenarioDataTablesListAPIView(generics.ListAPIView): serializer_class = ScenarioDataTablesSerializer -class ManageScenarioDatasets(APIView): +class ManageOekgScenarioDatasets(APIView): permission_classes = [IsAuthenticated] # Require authentication @post_only_if_user_is_owner_of_scenario_bundle def post(self, request): serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data) - if serializer.is_valid(): - bundle_uuid = serializer.validated_data["scenario_bundle"] - scenario_uuid = serializer.validated_data["scenario"] - datasets = serializer.validated_data["dataset"] - - # Iterate over each dataset to process it properly - for dataset in datasets: - dataset_name = dataset["name"] - dataset_type = dataset["type"] - - # Add datasets to the scenario in the bundle (implementation depends - # on your model) - success = add_datasets_to_scenario( - bundle_uuid, scenario_uuid, dataset_name, dataset_type - ) - if not success: - return Response( - {"error": "Failed to add datasets"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - return Response( - {"message": "Datasets added successfully"}, - status=status.HTTP_200_OK, - ) - - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - @post_only_if_user_is_owner_of_scenario_bundle - def delete(self, request): - serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data) - if serializer.is_valid(): - scenario_uuid = serializer.validated_data["scenario"] - datasets = serializer.validated_data["dataset"] - - # Iterate over each dataset to process it properly - for dataset in datasets: - dataset_name = dataset["name"] - dataset_type = dataset["type"] - - # Remove the dataset from the scenario in the bundle - success = remove_datasets_from_scenario( - scenario_uuid, dataset_name, dataset_type - ) - - if not success: - return Response( - {"error": f"Failed to remove dataset {dataset_name}"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - return Response( - {"message": "Datasets removed successfully"}, - status=status.HTTP_200_OK, - ) - - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + try: + dataset_configs = get_dataset_configs(serializer.validated_data) + response_data = process_datasets_sparql_query(dataset_configs) + except APIError as e: + return Response({"error": str(e)}, status=e.status) + except Exception: + return Response({"error": "An unexpected error occurred."}, status=500) + + if "error" in response_data: + return Response(response_data, status=status.HTTP_400_BAD_REQUEST) + + return Response(response_data, status=status.HTTP_200_OK) + + # @post_only_if_user_is_owner_of_scenario_bundle + # def delete(self, request): + # serializer = ScenarioBundleScenarioDatasetSerializer(data=request.data) + # if serializer.is_valid(): + # scenario_uuid = serializer.validated_data["scenario"] + # datasets = serializer.validated_data["datasets"] + + # # Iterate over each dataset to process it properly + # for dataset in datasets: + # dataset_name = dataset["name"] + # dataset_type = dataset["type"] + + # # Remove the dataset from the scenario in the bundle + # success = remove_datasets_from_scenario( + # scenario_uuid, dataset_name, dataset_type + # ) + + # if not success: + # return Response( + # {"error": f"Failed to remove dataset {dataset_name}"}, + # status=status.HTTP_400_BAD_REQUEST, + # ) + + # return Response( + # {"message": "Datasets removed successfully"}, + # status=status.HTTP_200_OK, + # ) + + # return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) From a2dcfdf71caca939c264640b7a4c59c380083cb6 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:31:39 +0100 Subject: [PATCH 30/38] implement method to get object url --- dataedit/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dataedit/models.py b/dataedit/models.py index 8e01fde54..e80747154 100644 --- a/dataedit/models.py +++ b/dataedit/models.py @@ -16,6 +16,7 @@ IntegerField, JSONField, ) +from django.urls import reverse from django.utils import timezone # Create your models here. @@ -77,6 +78,9 @@ class Table(Tagable): is_publish = BooleanField(null=False, default=False) human_readable_name = CharField(max_length=1000, null=True) + def get_absolute_url(self): + return reverse("dataedit:view", kwargs={"pk": self.pk}) + @classmethod def load(cls, schema, table): """ From 4f1bdda5d83a9f2f3a2d200a5c5ecc9c637f4ec9 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:32:02 +0100 Subject: [PATCH 31/38] fix missing parameter in permission decorator --- factsheet/permission_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/factsheet/permission_decorator.py b/factsheet/permission_decorator.py index b1df1228b..e24514e66 100644 --- a/factsheet/permission_decorator.py +++ b/factsheet/permission_decorator.py @@ -94,7 +94,7 @@ def _wrapped_view(view_instance, request, *args, **kwargs): # Check if the current user is the owner (creator) of the Scenario bundle. if request.user == scenario_bundle_access.owner_user: - return view_func(request, *args, **kwargs) + return view_func(view_instance, request, *args, **kwargs) else: return HttpResponseForbidden("Access Denied") From 28049698c4df0b32e8658621ee33ad9ff23eaacd Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:32:32 +0100 Subject: [PATCH 32/38] implement button to copy a scenario UUID --- .../frontend/src/components/scenarioBundle.js | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/factsheet/frontend/src/components/scenarioBundle.js b/factsheet/frontend/src/components/scenarioBundle.js index 72bd0c9c8..70e6e5680 100644 --- a/factsheet/frontend/src/components/scenarioBundle.js +++ b/factsheet/frontend/src/components/scenarioBundle.js @@ -26,6 +26,7 @@ import Tabs from '@mui/material/Tabs'; import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; import Fab from '@mui/material/Fab'; +import ContentCopyOutlinedIcon from '@mui/icons-material/ContentCopyOutlined'; import AddIcon from '@mui/icons-material/Add.js'; import Checkbox from '@mui/material/Checkbox'; import FormGroup from '@mui/material/FormGroup'; @@ -2007,12 +2008,30 @@ function Factsheet(props) { ] } const scenario_count = 'Scenarios' + ' (' + scenarios.length + ')'; - console.log(scenarios); - const renderScenariosOverview = () => ( - - { - scenarios.map((v, i) => - v.acronym !== '' && +console.log(scenarios); + +const renderScenariosOverview = () => ( + + {scenarios.map((v, i) => + v.acronym !== '' ? ( + + + @@ -2226,10 +2245,12 @@ function Factsheet(props) {
- ) - } - - ) + + ) : null + )} + +); + const renderPublicationOverview = () => ( From 36cbb58b52e02da012abbb2d4c76e5d846ab4630 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:33:20 +0100 Subject: [PATCH 33/38] add dataset UUID field to config dataclass and implement method to read the config from serialized user data --- oekg/sparqlModels.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/oekg/sparqlModels.py b/oekg/sparqlModels.py index aa47ec4a6..df4ba35f1 100644 --- a/oekg/sparqlModels.py +++ b/oekg/sparqlModels.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from uuid import UUID +from uuid import UUID, uuid4 @dataclass @@ -8,6 +8,17 @@ class DatasetConfig: scenario_uuid: UUID dataset_label: str dataset_url: str - dataset_id: int - dataset_key: bool + dataset_id: UUID dataset_type: str + + @classmethod + def from_serializer_data(cls, validated_data: dict, dataset_entry: dict): + """Converts validated serializer data into a DatasetConfig object.""" + return cls( + bundle_uuid=validated_data["scenario_bundle"], + scenario_uuid=validated_data["scenario"], + dataset_label=dataset_entry["name"], + dataset_url=dataset_entry["external_url"], + dataset_id=uuid4(), + dataset_type=dataset_entry["type"], # "input" or "output" + ) From 910c5fe9827fd6f88d8120eb8327f5da16ba8236 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:34:56 +0100 Subject: [PATCH 34/38] new oekg check: sparql query to check if scenario is part of the scenario bundle (both values from user) --- oekg/sparqlQuery.py | 51 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/oekg/sparqlQuery.py b/oekg/sparqlQuery.py index 08ab94a28..0d4a64313 100644 --- a/oekg/sparqlQuery.py +++ b/oekg/sparqlQuery.py @@ -1,11 +1,60 @@ +import logging from uuid import UUID import requests from SPARQLWrapper import JSON, POST -from factsheet.oekg.connection import sparql_wrapper_update, update_endpoint +from factsheet.oekg.connection import sparql, sparql_wrapper_update, update_endpoint from oekg.sparqlModels import DatasetConfig +logger = logging.getLogger("oeplatform") + + +def scenario_in_bundle(bundle_uuid: UUID, scenario_uuid: UUID) -> bool: + """ + Check if a scenario is part of a scenario bundle in the KG. + """ + sparql_query = f""" + PREFIX oeo: + + ASK {{ + ?p + . + }} + """ + sparql.setQuery(sparql_query) + sparql.setMethod(POST) + sparql.setReturnFormat(JSON) + response = sparql.query().convert() + + return response.get( + "boolean", False + ) # Returns True if scenario is part of the bundle + + +def dataset_exists(scenario_uuid: UUID, dataset_url: str) -> bool: + """ + Check if a dataset with the same label already exists. + """ + + sparql_query = f""" + PREFIX oeo: + PREFIX rdfs: + + ASK {{ + ?p ?dataset . + ?dataset oeo:has_iri "{dataset_url}" . + }} + + """ # noqa + + sparql.setQuery(sparql_query) + sparql.setMethod(POST) + sparql.setReturnFormat(JSON) + response = sparql.query().convert() + + return response.get("boolean", False) # Returns True if dataset exists + def add_datasets_to_scenario(oekgDatasetConfig: DatasetConfig): """ From 98aebe0abb60183a630238406f4b390fbcbfbb05 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:36:29 +0100 Subject: [PATCH 35/38] enhance function: adding datasets now works correctly and includes checks. Additionally errors are handled better. --- oekg/sparqlQuery.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/oekg/sparqlQuery.py b/oekg/sparqlQuery.py index 0d4a64313..0e1812f2c 100644 --- a/oekg/sparqlQuery.py +++ b/oekg/sparqlQuery.py @@ -61,28 +61,33 @@ def add_datasets_to_scenario(oekgDatasetConfig: DatasetConfig): Function to add datasets to a scenario bundle in Jena Fuseki. """ - new_dataset_uid = UUID() + # Check if a dataset with the same label exists + if dataset_exists(oekgDatasetConfig.scenario_uuid, oekgDatasetConfig.dataset_url): + return False # Skip insertion + # Check: used constant string values here. Get ids from oeo + # graph to make sure ids still exists? if oekgDatasetConfig.dataset_type == "input": rel_property = "RO_0002233" + type_entity = "OEO_00030029" elif oekgDatasetConfig.dataset_type == "output": rel_property = "RO_0002234" + type_entity = "OEO_00030030" + # oeo:has_id "{oekgDatasetConfig.dataset_id}" ; + # The above seems to be deprecated in the OEKG sparql_query = f""" PREFIX oeo: PREFIX rdfs: INSERT DATA {{ - GRAPH {{ - a oeo:OEO_00030030 ; - rdfs:label "{oekgDatasetConfig.dataset_label}" ; - oeo:has_iri "{oekgDatasetConfig.dataset_url}" ; - oeo:has_id "{oekgDatasetConfig.dataset_id}" ; - oeo:has_key "{oekgDatasetConfig.dataset_key}" . - - oeo:{rel_property} - . - }} + a oeo:{type_entity} ; + rdfs:label "{oekgDatasetConfig.dataset_label}" ; + oeo:has_iri "{oekgDatasetConfig.dataset_url}" ; + oeo:has_key "{oekgDatasetConfig.dataset_id}" . + + oeo:{rel_property} + . }} """ # noqa @@ -91,10 +96,15 @@ def add_datasets_to_scenario(oekgDatasetConfig: DatasetConfig): sparql_wrapper_update.setQuery(sparql_query) sparql_wrapper_update.setMethod(POST) sparql_wrapper_update.setReturnFormat(JSON) - response = sparql_wrapper_update.query() - http_response = response.response - if not http_response.status == 200: - return False # Return False if any query fails + try: + response = sparql_wrapper_update.query() + http_response = response.response + if not http_response.status == 200: + return False # Return False if any query fails + except Exception as e: + logger.error(f"Failed to update datasets in OEKG: {e}") + return False + return True From 945456f68c35ada24712a0c6a63a50b62cca9954 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:39:11 +0100 Subject: [PATCH 36/38] enhancement: add better error and success massages and execute per dataset to handle multiple datasets in request --- oekg/utils.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/oekg/utils.py b/oekg/utils.py index 45efc56c2..6df506b58 100644 --- a/oekg/utils.py +++ b/oekg/utils.py @@ -1,7 +1,10 @@ import re +from oekg.sparqlModels import DatasetConfig +from oekg.sparqlQuery import add_datasets_to_scenario, scenario_in_bundle -def validate_sparql_query(query): + +def validate_public_sparql_query(query): """ Validate the SPARQL query to prevent injection attacks. """ @@ -25,3 +28,46 @@ def validate_sparql_query(query): return False return True + + +def process_datasets_sparql_query(dataset_configs: list[DatasetConfig]): + """ + Attempts to add each dataset to the scenario. + Returns a count of added datasets and a list of skipped ones. + """ + total_datasets = len(dataset_configs) + added_count = 0 + skipped_datasets = [] + + for dataset_config in dataset_configs: + # Check if scenario is part of the scenario bundle + + if not scenario_in_bundle( + dataset_config.bundle_uuid, dataset_config.scenario_uuid + ): + response: dict = {} + response["error"] = ( + f"Scenario {dataset_config.scenario_uuid} is not part" + f"of bundle {dataset_config.bundle_uuid}" + ) + return response + + success = add_datasets_to_scenario(dataset_config) + + if success: + added_count += 1 + else: + skipped_datasets.append(dataset_config.dataset_label) + + # Construct a clear response + response: dict = { + "info": "successfully processed your request", + "added_count": f"{added_count} / {total_datasets}", + } + + if skipped_datasets: + # TODO: Add return a reason from add_datasets_to_scenario if needed + response["reason"] = "Dataset already exists in the scenario." + response["skipped"] = skipped_datasets + + return response From a230aaf57088d9408ed1b36800422ebe47aae593 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:39:23 +0100 Subject: [PATCH 37/38] adapt renamed function --- oekg/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oekg/views.py b/oekg/views.py index e5681ae65..c7328b4d4 100644 --- a/oekg/views.py +++ b/oekg/views.py @@ -4,7 +4,7 @@ from django.shortcuts import render from django.views.decorators.http import require_POST -from oekg.utils import validate_sparql_query +from oekg.utils import validate_public_sparql_query from oeplatform.settings import OEKG_SPARQL_ENDPOINT_URL @@ -24,7 +24,7 @@ def sparql_endpoint(request): if not sparql_query: return HttpResponseBadRequest("Missing 'query' parameter.") - if not validate_sparql_query(sparql_query): + if not validate_public_sparql_query(sparql_query): raise SuspiciousOperation("Invalid SPARQL query.") endpoint_url = OEKG_SPARQL_ENDPOINT_URL From f46efeb9d85f8a5093f06a46cdd2385e90c94901 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Feb 2025 17:47:07 +0100 Subject: [PATCH 38/38] update changelog #1890 --- versions/changelogs/current.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/versions/changelogs/current.md b/versions/changelogs/current.md index a90e38c5a..f15851996 100644 --- a/versions/changelogs/current.md +++ b/versions/changelogs/current.md @@ -8,12 +8,16 @@ - Removed the outdated & unmaintained references module that was intended to handle bibtex files and store them in a django model [(#1913)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1913). -- Change sparql endpoint for OEKG to use the http post method to match the expected usage [(#1913)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1913). +- Change sparql endpoint for OEKG to use the http post method to match the expected usage [(#1913)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1913). - Extract header/footer template [(#1914)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1914) ## Features +- Implement new API Endpoint to add new datasets to a scenario bundle -> scenario -> input or output datasets. This eases bulk adding datasets. The API provides extensive error messages. Datasets listed in the scenario topic on the OEP and external datasets registered on the databus.openenergyplatform.org can be used. [(#1914)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1894) + ## Bugs ## Documentation updates + +- Provide documentation for the OEKG:Scenario Bundle dataset management as described in #1890 [(#1914)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1894)