From ff305ecb3c45242e7af28eccff0097802a9a05aa Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 22 Oct 2025 13:06:35 -0400 Subject: [PATCH 1/4] replace keywords with types --- marble_node_registry/migrations.py | 46 ++++++++++++++++++++++++++++++ marble_node_registry/update.py | 11 +++++++ node_registry.schema.json | 25 ++++++++++++++-- pyproject.toml | 2 +- tests/test_registry.py | 1 + tests/test_update.py | 33 +++++++++++++++++---- 6 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 marble_node_registry/migrations.py diff --git a/marble_node_registry/migrations.py b/marble_node_registry/migrations.py new file mode 100644 index 0000000..1d8bf7e --- /dev/null +++ b/marble_node_registry/migrations.py @@ -0,0 +1,46 @@ +# Migrations are used to ensure that data provided by nodes that are using an +# older version of the schema are updated automatically to comply with the newest +# version of the schema. +# +# If a backwards incompatible change is introduced in the schema, please make a +# new migration function here to ensure that older data is properly updated. +# +# Migration functions will be applied to each node's data in the order that they +# appear in the MIGRATIONS variable at the bottom of this file. +# +# All migration function should take a single argument which contains the node's +# data and modify that data in place. + +def convert_keywords_to_types(data: dict) -> None: + """ + Add service types if they don't exist. + + Since version 1.3.0 service "types" are now required. If they don't exist + then they can be derived from service "keywords" which were used in place + of "types" prior to this version. + """ + + keyword2type = { + "catalog": "catalog", + "data": "data", + "jupyterhub": "jupyterhub", + "other": "other", + "service-wps": "wps", + "service-wms": "wms", + "service-wfs": "wfs", + "service-wcs": "wcs", + "service-ogcapi_processes": "ogcapi_processes" + } + for service in data["services"]: + if "types" not in service: + service["types"] = [] + for keyword in service["keywords"]: + if (type_ := keyword2type.get(keyword)): + service["types"].append(type_) + if not service["types"]: + service["types"].append("other") + + +MIGRATIONS = ( + convert_keywords_to_types, +) diff --git a/marble_node_registry/update.py b/marble_node_registry/update.py index 52a9554..f9bfaab 100644 --- a/marble_node_registry/update.py +++ b/marble_node_registry/update.py @@ -6,6 +6,8 @@ import datetime from copy import deepcopy +from migrations import MIGRATIONS + THIS_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.dirname(THIS_DIR) SCHEMA_FILE = os.path.join(ROOT_DIR, "node_registry.schema.json") @@ -87,6 +89,15 @@ def update_registry() -> None: ) continue + try: + for migration in MIGRATIONS: + migration(data) + except Exception as e: + registry[name] = org_data + registry[name]["status"] = "invalid_configuration" + sys.stderr.write(f"unable to apply migrations for Node named {name}: {e}.") + continue + try: jsonschema.validate(instance=registry, schema=schema) except jsonschema.exceptions.ValidationError as e: diff --git a/node_registry.schema.json b/node_registry.schema.json index 7859e5a..93662d3 100644 --- a/node_registry.schema.json +++ b/node_registry.schema.json @@ -148,6 +148,7 @@ "type": "object", "required": [ "name", + "types", "keywords", "description", "links" @@ -157,12 +158,32 @@ "type": "string", "minLength": 1 }, - "keywords": { + "types": { "type": "array", "minItems": 1, "items": { "type": "string", - "pattern": "^catalog|data|jupyterhub|other|service-(wps|wms|wfs|wcs|ogcapi_processes)$" + "enum": [ + "auth", + "management", + "catalog", + "data", + "jupyterhub", + "other", + "wps", + "wms", + "wfs", + "wcs", + "ogcapi_processes", + "ogcapi_dggs" + ] + } + }, + "keywords": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 } }, "description": { diff --git a/pyproject.toml b/pyproject.toml index 7650700..35aba7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,4 +2,4 @@ addopts = [ "--import-mode=importlib", ] -pythonpath = "." +pythonpath = "./marble_node_registry" diff --git a/tests/test_registry.py b/tests/test_registry.py index b94105e..9605641 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -35,6 +35,7 @@ def registry_content_with_services(registry_content): { "name": "test-service", "keywords": ["other"], + "types": ["other"], "description": "test service", "version": "1.2.3", "links": [ diff --git a/tests/test_update.py b/tests/test_update.py index 1a7a9ac..32de517 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -4,12 +4,13 @@ import pytest -from marble_node_registry import update +import update # type: ignore GOOD_SERVICES = { "services": [ { "name": "geoserver", + "types": ["data", "wps", "wms", "wfs"], "keywords": ["data", "service-wps", "service-wms", "service-wfs"], "description": "GeoServer is a server that allows users to view and edit geospatial data.", "links": [ @@ -19,7 +20,8 @@ }, { "name": "weaver", - "keywords": ["service-ogcapi_processes"], + "types": ["ogcapi_processes"], + "keywords": ["service-ogcapi_processes", "some-other-keyword"], "description": "An OGC-API flavored Execution Management Service", "links": [ {"rel": "service", "type": "application/json", "href": "https://daccs-uoft.example.com/weaver/"}, @@ -339,11 +341,30 @@ class TestOnlineNodeUpdateWithInvalidServices(InvalidResponseTests, NonInitialTe services = {"services": [{"bad_key": "some_value"}]} -class TestOnlineNodeUpdateWithInvalidServiceKeywords(InvalidResponseTests, NonInitialTests): - """Test when updates have previously been run and the reported services keywords are not valid""" +class TestOnlineNodeUpdateWithInvalidServiceTypes(InvalidResponseTests, NonInitialTests): + """Test when updates have previously been run and the reported services types are not valid""" services = deepcopy(GOOD_SERVICES) @pytest.fixture(scope="class", autouse=True) - def bad_keywords(self): - self.services["services"][0]["keywords"] = ["something-bad"] + def bad_types(self): + self.services["services"][0]["types"] = ["something-bad"] + + +class TestOnlineNodeUpdateWithNoTypes(ValidResponseTests, NonInitialTests): + """ + Test when updates have previously been run and there are no services types + + This ensures that service types are updated as expected from the provided keywords + """ + + services = deepcopy(GOOD_SERVICES) + + @pytest.fixture(scope="class", autouse=True) + def no_types(self): + for service in self.services["services"]: + service.pop("types") + + def test_services_updated(self, example_node_name, updated_registry): + """Test that the services values are updated""" + assert updated_registry.call_args.args[0][example_node_name]["services"] == GOOD_SERVICES["services"] From 9612185e0239d22396c8e594a111fc08d1c57f46 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:46:12 -0400 Subject: [PATCH 2/4] review updates --- node_registry.schema.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/node_registry.schema.json b/node_registry.schema.json index 93662d3..cc08036 100644 --- a/node_registry.schema.json +++ b/node_registry.schema.json @@ -1,5 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/refs/tags/1.2.0/node_registry.schema.json", "patternProperties": { "^[a-zA-Z0-9]+$": { "type": "object", @@ -167,6 +168,7 @@ "auth", "management", "catalog", + "stac", "data", "jupyterhub", "other", @@ -175,7 +177,10 @@ "wfs", "wcs", "ogcapi_processes", - "ogcapi_dggs" + "ogcapi_dggs", + "ogcapi_coverages", + "ogcapi_features", + "ogcapi_records" ] } }, From 218af8484dbf1b04137ff7c08e32dbf7776123b9 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 27 Oct 2025 13:09:08 -0400 Subject: [PATCH 3/4] update tests --- doc/node_registry.example.json | 9 +++++++++ tests/test_update.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/node_registry.example.json b/doc/node_registry.example.json index adfe802..67df818 100644 --- a/doc/node_registry.example.json +++ b/doc/node_registry.example.json @@ -41,6 +41,12 @@ "service-wms", "service-wfs" ], + "types": [ + "data", + "wps", + "wms", + "wfs" + ], "description": "GeoServer is a server that allows users to view and edit geospatial data.", "links": [ { @@ -60,6 +66,9 @@ "keywords": [ "service-ogcapi_processes" ], + "types": [ + "ogcapi_processes" + ], "description": "An OGC-API flavored Execution Management Service", "links": [ { diff --git a/tests/test_update.py b/tests/test_update.py index 67ab3d1..6b2b560 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -267,7 +267,7 @@ def test_last_updated_updated(self, example_node_name, example_registry_content, example_node_name ].get("last_updated") - def test_final_registry_valid(self, updated_registry, node_registry_schema): + def test_final_registry_validity(self, updated_registry, node_registry_schema): jsonschema.validate(instance=updated_registry.call_args.args[0], schema=node_registry_schema) @@ -302,7 +302,7 @@ def test_last_updated_no_change(self, example_node_name, example_registry_conten example_node_name ].get("last_updated") - def test_final_registry_valid(self, updated_registry, node_registry_schema): + def test_final_registry_validity(self, updated_registry, node_registry_schema): jsonschema.validate(instance=updated_registry.call_args.args[0], schema=node_registry_schema) From 3e1f652708f50c2046b50ff584142653e704b82d Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 28 Oct 2025 09:21:54 -0400 Subject: [PATCH 4/4] bump version to 1.3.0 --- node_registry.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_registry.schema.json b/node_registry.schema.json index 1ea52d6..9486c07 100644 --- a/node_registry.schema.json +++ b/node_registry.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/refs/tags/1.2.0/node_registry.schema.json", + "$id": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/refs/tags/1.3.0/node_registry.schema.json", "patternProperties": { "^[a-zA-Z0-9]+$": { "type": "object",