diff --git a/.github/workflows/unittest-workflow.yaml b/.github/workflows/unittest-workflow.yaml index 5ed40a3..d83721a 100644 --- a/.github/workflows/unittest-workflow.yaml +++ b/.github/workflows/unittest-workflow.yaml @@ -21,7 +21,12 @@ jobs: shell: bash -l {0} run: | cd /home/runner/work/deep-code/deep-code - pip install .[dev] + pip install -e ".[dev]" + + - name: Lint with ruff + run: | + pip install ruff + ruff check - name: Run unit tests shell: bash -l {0} diff --git a/README.md b/README.md index 02f2eed..4c7b856 100644 --- a/README.md +++ b/README.md @@ -56,18 +56,10 @@ pytest --cov-report html --cov=deep-code providing different utility functions. Use the --help option with these subcommands to get more details on usage. -The CLI retrieves the Git username and personal access token from a hidden file named .gitaccess. Ensure this file is located in the same directory where you execute the CLI +The CLI retrieves the Git username and personal access token from a hidden file named +.gitaccess. Ensure this file is located in the same directory where you execute the CLI command. -### deep-code publish-product - -Publish a dataset which is a result of an experiment to the EarthCODE -open-science catalog. - -```commandline - deep-code publish-dataset /path/to/dataset-config.yaml - ``` - #### .gitaccess example ``` @@ -75,6 +67,15 @@ github-username: your-git-user github-token: personal access token ``` +### deep-code publish + +Publish the experiment, workflow and dataset which is a result of an experiment to +the EarthCODE open-science catalog. + +```commandline + deep-code publish /path/to/dataset-config.yaml /path/to/workflow-config.yaml + ``` + #### dataset-config.yaml example ``` @@ -92,42 +93,26 @@ cf_parameter: - name: hydrology ``` -dataset-id has to be a valid dataset-id from `deep-esdl-public` s3 or your team bucket. - -### deep-code publish-workflow +dataset-id has to be a valid dataset-id from `deep-esdl-public` s3 bucket or your team +bucket. -Publish a workflow/experiment to the EarthCODE open-science catalog. - -```commandline -deep-code publish-workflow /path/to/workflow-config.yaml - ``` #### workflow-config.yaml example ``` -workflow_id: "4D Med hydrology cube generation" +workflow_id: "esa-cci-permafrost" properties: - title: "Hydrology cube generation recipe" - description: "4D Med cube generation" + title: "ESA CCI permafrost" + description: "cube generation workflow for esa-cci-permafrost" keywords: - Earth Science themes: - - Atmosphere - - Ocean - - Evaporation + - cryosphere license: proprietary jupyter_kernel_info: - name: deepesdl-xcube-1.7.1 + name: deepesdl-xcube-1.8.3 python_version: 3.11 - env_file: https://git/env.yml -links: - - rel: "documentation" - type: "application/json" - title: "4DMed Hydrology Cube Generation Recipe" - href: "https://github.com/deepesdl/cube-gen/tree/main/hydrology/README.md" - - rel: "jupyter-notebook" - type: "application/json" - title: "Workflow Jupyter Notebook" - href: "https://github.com/deepesdl/cube-gen/blob/main/hydrology/notebooks/reading_hydrology.ipynb" + env_file: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/environment.yml" +jupyter_notebook_url: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb" contact: - name: Tejas Morbagal Harish organization: Brockmann Consult GmbH diff --git a/deep_code/cli/main.py b/deep_code/cli/main.py index af140a4..e4f5380 100644 --- a/deep_code/cli/main.py +++ b/deep_code/cli/main.py @@ -6,7 +6,7 @@ import click -from deep_code.cli.publish import publish_dataset, publish_workflow +from deep_code.cli.publish import publish @click.group() @@ -15,8 +15,7 @@ def main(): pass -main.add_command(publish_dataset) -main.add_command(publish_workflow) +main.add_command(publish) if __name__ == "__main__": main() diff --git a/deep_code/cli/publish.py b/deep_code/cli/publish.py index a3d81d0..47e34fb 100644 --- a/deep_code/cli/publish.py +++ b/deep_code/cli/publish.py @@ -6,21 +6,16 @@ import click -from deep_code.tools.publish import DatasetPublisher, WorkflowPublisher +from deep_code.tools.publish import Publisher -@click.command(name="publish-dataset") +@click.command(name="publish") @click.argument("dataset_config", type=click.Path(exists=True)) -def publish_dataset(dataset_config): +@click.argument("workflow_config", type=click.Path(exists=True)) +def publish(dataset_config, workflow_config): """Request publishing a dataset to the open science catalogue. """ - publisher = DatasetPublisher() - publisher.publish_dataset(dataset_config_path=dataset_config) - - -@click.command(name="publish-workflow") -@click.argument("workflow_metadata", type=click.Path(exists=True)) -def publish_workflow(workflow_metadata): - - workflow_publisher = WorkflowPublisher() - workflow_publisher.publish_workflow(workflow_config_path=workflow_metadata) + publisher = Publisher( + dataset_config_path=dataset_config, workflow_config_path=workflow_config + ) + publisher.publish_all() diff --git a/deep_code/constants.py b/deep_code/constants.py index 992ddf4..814a03f 100644 --- a/deep_code/constants.py +++ b/deep_code/constants.py @@ -4,8 +4,10 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. -OSC_SCHEMA_URI = "https://stac-extensions.github.io/osc/v1.0.0-rc.3/schema.json" +OSC_SCHEMA_URI = "https://stac-extensions.github.io/osc/v1.0.0/schema.json" CF_SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json" +THEMES_SCHEMA_URI = "https://stac-extensions.github.io/themes/v1.0.0/schema.json" +OSC_THEME_SCHEME = "https://github.com/stac-extensions/osc#theme" OSC_REPO_OWNER = "ESA-EarthCODE" OSC_REPO_NAME = "open-science-catalog-metadata-testing" OSC_BRANCH_NAME = "add-new-collection" @@ -14,3 +16,16 @@ ) OGC_API_RECORD_SPEC = "http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core" WF_BRANCH_NAME = "add-new-workflow-from-deepesdl" +VARIABLE_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/variables/catalog.json" +PRODUCT_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/products/catalog.json" +DEEPESDL_COLLECTION_SELF_HREF = ( + "https://esa-earthcode.github.io/open-science-catalog-metadata/projects/deepesdl" + "/collection.json" +) +BASE_URL_OSC = "https://esa-earthcode.github.io/open-science-catalog-metadata" +EXPERIMENT_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/experiments/catalog.json" +WORKFLOW_BASE_CATALOG_SELF_HREF = ( + "https://esa-earthcode.github.io/open-science-catalog-metadata/workflows/catalog" + ".json" +) +PROJECT_COLLECTION_NAME = "deep-earth-system-data-lab" diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 3e0f5e8..f355cb5 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -1,120 +1,108 @@ -from unittest.mock import MagicMock, mock_open, patch - -import pytest - -from deep_code.tools.publish import DatasetPublisher - - -class TestDatasetPublisher: - @patch("deep_code.tools.publish.fsspec.open") - def test_init_missing_credentials(self, mock_fsspec_open): - mock_fsspec_open.return_value.__enter__.return_value = mock_open( - read_data="{}" - )() - - with pytest.raises( - ValueError, match="GitHub credentials are missing in the `.gitaccess` file." - ): - DatasetPublisher() - - @patch("deep_code.tools.publish.fsspec.open") - def test_publish_dataset_missing_ids(self, mock_fsspec_open): - git_yaml_content = """ - github-username: test-user - github-token: test-token - """ - dataset_yaml_content = """ - collection-id: test-collection - """ - mock_fsspec_open.side_effect = [ - mock_open(read_data=git_yaml_content)(), - mock_open(read_data=dataset_yaml_content)(), - ] +import unittest +from unittest.mock import patch, mock_open, MagicMock +import json +import yaml +from pathlib import Path +import tempfile +from pystac import Catalog + +from deep_code.tools.publish import Publisher - publisher = DatasetPublisher() - - with pytest.raises( - ValueError, match="Dataset ID or Collection ID missing in the config." - ): - publisher.publish_dataset("/path/to/dataset-config.yaml") - - @patch("deep_code.utils.github_automation.os.chdir") - @patch("deep_code.utils.github_automation.subprocess.run") - @patch("deep_code.utils.github_automation.os.path.expanduser", return_value="/tmp") - @patch("requests.post") - @patch("deep_code.utils.github_automation.GitHubAutomation") - @patch("deep_code.tools.publish.fsspec.open") - def test_publish_dataset_success( - self, - mock_fsspec_open, - mock_github_automation, - mock_requests_post, - mock_expanduser, - mock_subprocess_run, - mock_chdir, - ): - # Mock the YAML reads - git_yaml_content = """ - github-username: test-user - github-token: test-token - """ - dataset_yaml_content = """ - dataset_id: test-dataset - collection_id: test-collection - documentation_link: http://example.com/doc - access_link: http://example.com/access - dataset_status: ongoing - dataset_region: Global - osc_theme: ["climate"] - cf_parameter: [] - """ - mock_fsspec_open.side_effect = [ - mock_open(read_data=git_yaml_content)(), - mock_open(read_data=dataset_yaml_content)(), - ] - # Mock GitHubAutomation methods - mock_git = mock_github_automation.return_value - mock_git.fork_repository.return_value = None - mock_git.clone_repository.return_value = None - mock_git.create_branch.return_value = None - mock_git.add_file.return_value = None - mock_git.commit_and_push.return_value = None - mock_git.create_pull_request.return_value = "http://example.com/pr" - mock_git.clean_up.return_value = None - - # Mock subprocess.run & os.chdir - mock_subprocess_run.return_value = None - mock_chdir.return_value = None - - # Mock STAC generator - mock_collection = MagicMock() - mock_collection.to_dict.return_value = { - "type": "Collection", - "id": "test-collection", - "description": "A test STAC collection", - "extent": { - "spatial": {"bbox": [[-180.0, -90.0, 180.0, 90.0]]}, - "temporal": {"interval": [["2023-01-01T00:00:00Z", None]]}, - }, - "links": [], - "stac_version": "1.0.0", +class TestPublisher(unittest.TestCase): + @patch("fsspec.open") + @patch("deep_code.tools.publish.GitHubPublisher") + def setUp(self, mock_github_publisher, mock_fsspec_open): + # Mock GitHubPublisher to avoid reading .gitaccess + self.mock_github_publisher_instance = MagicMock() + mock_github_publisher.return_value = self.mock_github_publisher_instance + + # Mock dataset and workflow config files + self.dataset_config = { + "collection_id": "test-collection", + "dataset_id": "test-dataset", + } + self.workflow_config = { + "properties": {"title": "Test Workflow"}, + "workflow_id": "test-workflow", } - with patch("deep_code.tools.publish.OscDatasetStacGenerator") as mock_generator: - mock_generator.return_value.build_dataset_stac_collection.return_value = ( - mock_collection - ) - # Instantiate & publish - publisher = DatasetPublisher() - publisher.publish_dataset("/fake/path/to/dataset-config.yaml") + # Mock fsspec.open for config files + self.mock_fsspec_open = mock_fsspec_open + self.mock_fsspec_open.side_effect = [ + mock_open(read_data=yaml.dump(self.dataset_config)).return_value, + mock_open(read_data=yaml.dump(self.workflow_config)).return_value, + ] - # Assert that we called git clone with /tmp/temp_repo - # Because expanduser("~") is now patched to /tmp, the actual path is /tmp/temp_repo - auth_url = "https://test-user:test-token@github.com/test-user/open-science-catalog-metadata-testing.git" - mock_subprocess_run.assert_any_call( - ["git", "clone", auth_url, "/tmp/temp_repo"], check=True + # Initialize Publisher + self.publisher = Publisher( + dataset_config_path="test-dataset-config.yaml", + workflow_config_path="test-workflow-config.yaml", ) - # Also confirm we changed directories to /tmp/temp_repo - mock_chdir.assert_any_call("/tmp/temp_repo") + def test_normalize_name(self): + self.assertEqual(Publisher._normalize_name("Test Name"), "test-name") + self.assertEqual(Publisher._normalize_name("Test Name"), "test---name") + self.assertIsNone(Publisher._normalize_name("")) + self.assertIsNone(Publisher._normalize_name(None)) + + def test_write_to_file(self): + # Create a temporary file + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + file_path = temp_file.name + + # Test data + data = {"key": "value"} + + # Call the method + Publisher._write_to_file(file_path, data) + + # Read the file and verify its content + with open(file_path, "r") as f: + content = json.load(f) + self.assertEqual(content, data) + + # Clean up + Path(file_path).unlink() + + def test_update_base_catalog(self): + # Create a mock Catalog + catalog = Catalog(id="test-catalog", description="Test Catalog") + + # Mock file path and item ID + catalog_path = "test-catalog.json" + item_id = "test-item" + self_href = "https://example.com/catalog.json" + + self.publisher.workflow_title = "Test Workflow" + + # Mock the Catalog.from_file method + with patch("pystac.Catalog.from_file", return_value=catalog): + updated_catalog = self.publisher._update_base_catalog( + catalog_path, item_id, self_href + ) + + # Assertions + self.assertEqual(updated_catalog.get_self_href(), self_href) + self.assertIsInstance(updated_catalog, Catalog) + + def test_read_config_files(self): + # Mock dataset and workflow config files + dataset_config = { + "collection_id": "test-collection", + "dataset_id": "test-dataset", + } + workflow_config = { + "properties": {"title": "Test Workflow"}, + "workflow_id": "test-workflow", + } + + # Mock fsspec.open for config files + self.mock_fsspec_open.side_effect = [ + mock_open(read_data=yaml.dump(dataset_config)).return_value, + mock_open(read_data=yaml.dump(workflow_config)).return_value, + ] + + # Assertions + self.assertEqual(self.publisher.dataset_config, dataset_config) + self.assertEqual(self.publisher.workflow_config, workflow_config) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 64285a7..e8864b1 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -217,3 +217,61 @@ def test_open_dataset_failure(self, mock_logger, mock_new_data_store): ) self.assertIn("Public store, Authenticated store", str(context.exception)) self.assertEqual(mock_new_data_store.call_count, 2) + + +class TestFormatString(unittest.TestCase): + def test_single_word(self): + self.assertEqual( + OscDatasetStacGenerator.format_string("temperature"), "Temperature" + ) + self.assertEqual(OscDatasetStacGenerator.format_string("temp"), "Temp") + self.assertEqual(OscDatasetStacGenerator.format_string("hello"), "Hello") + + def test_multiple_words_with_spaces(self): + self.assertEqual( + OscDatasetStacGenerator.format_string("surface temp"), "Surface Temp" + ) + self.assertEqual( + OscDatasetStacGenerator.format_string("this is a test"), "This Is A Test" + ) + + def test_multiple_words_with_underscores(self): + self.assertEqual( + OscDatasetStacGenerator.format_string("surface_temp"), "Surface Temp" + ) + self.assertEqual( + OscDatasetStacGenerator.format_string("this_is_a_test"), "This Is A Test" + ) + + def test_mixed_spaces_and_underscores(self): + self.assertEqual( + OscDatasetStacGenerator.format_string("surface_temp and_more"), + "Surface Temp And More", + ) + self.assertEqual( + OscDatasetStacGenerator.format_string( + "mixed_case_with_underscores_and spaces" + ), + "Mixed Case With Underscores And Spaces", + ) + + def test_edge_cases(self): + # Empty string + self.assertEqual(OscDatasetStacGenerator.format_string(""), "") + # Single word with trailing underscore + self.assertEqual( + OscDatasetStacGenerator.format_string("temperature_"), "Temperature" + ) + # Single word with leading underscore + self.assertEqual(OscDatasetStacGenerator.format_string("_temp"), "Temp") + # Single word with leading/trailing spaces + self.assertEqual(OscDatasetStacGenerator.format_string(" hello "), "Hello") + # Multiple spaces or underscores + self.assertEqual( + OscDatasetStacGenerator.format_string("too___many___underscores"), + "Too Many Underscores", + ) + self.assertEqual( + OscDatasetStacGenerator.format_string("too many spaces"), + "Too Many Spaces", + ) diff --git a/deep_code/tests/utils/test_github_automation.py b/deep_code/tests/utils/test_github_automation.py index 6a66868..efa284c 100644 --- a/deep_code/tests/utils/test_github_automation.py +++ b/deep_code/tests/utils/test_github_automation.py @@ -1,4 +1,4 @@ -import json +import logging import unittest from pathlib import Path from unittest.mock import MagicMock, patch @@ -8,113 +8,164 @@ class TestGitHubAutomation(unittest.TestCase): def setUp(self): - self.github = GitHubAutomation( - username="test-user", - token="test-token", - repo_owner="test-owner", - repo_name="test-repo", + # Set up test data + self.username = "testuser" + self.token = "testtoken" + self.repo_owner = "testowner" + self.repo_name = "testrepo" + self.github_automation = GitHubAutomation( + self.username, self.token, self.repo_owner, self.repo_name ) + logging.disable(logging.CRITICAL) # Disable logging during tests + + def tearDown(self): + logging.disable(logging.NOTSET) # Re-enable logging after tests @patch("requests.post") def test_fork_repository(self, mock_post): - """Test the fork_repository method.""" + # Mock the response from GitHub API mock_response = MagicMock() mock_response.raise_for_status.return_value = None mock_post.return_value = mock_response - self.github.fork_repository() + # Call the method + self.github_automation.fork_repository() + # Assertions mock_post.assert_called_once_with( - "https://api.github.com/repos/test-owner/test-repo/forks", - headers={"Authorization": "token test-token"}, + f"https://api.github.com/repos/{self.repo_owner}/{self.repo_name}/forks", + headers={"Authorization": f"token {self.token}"}, ) @patch("subprocess.run") - @patch("os.chdir") - def test_clone_repository(self, mock_chdir, mock_run): - """Test the clone_repository method.""" - self.github.clone_repository() + def test_clone_repository_new(self, mock_run): + # Mock the subprocess.run method + mock_run.return_value = MagicMock() + + # Mock os.path.exists to return False (directory does not exist) + with patch("os.path.exists", return_value=False): + self.github_automation.clone_sync_repository() + # Assertions mock_run.assert_called_once_with( - ["git", "clone", self.github.fork_repo_url, self.github.local_clone_dir], + [ + "git", + "clone", + f"https://{self.username}:{self.token}@github.com/{self.username}/{self.repo_name}.git", + self.github_automation.local_clone_dir, + ], check=True, ) - mock_chdir.assert_called_once_with(self.github.local_clone_dir) + + @patch("subprocess.run") + def test_clone_repository_existing(self, mock_run): + # Mock the subprocess.run method + mock_run.return_value = MagicMock() + + # Mock os.path.exists to return True (directory exists) + with patch("os.path.exists", return_value=True): + with patch("os.chdir"): + self.github_automation.clone_sync_repository() + + # Assertions + mock_run.assert_called_once_with(["git", "pull"], check=True) @patch("subprocess.run") def test_create_branch(self, mock_run): - """Test the create_branch method.""" - branch_name = "test-branch" - self.github.create_branch(branch_name) + # Mock the subprocess.run method + mock_run.return_value = MagicMock() + + # Mock os.chdir + with patch("os.chdir"): + self.github_automation.create_branch("test-branch") + # Assertions mock_run.assert_called_once_with( - ["git", "checkout", "-b", branch_name], check=True + ["git", "checkout", "-b", "test-branch"], check=True ) @patch("subprocess.run") - @patch("builtins.open", new_callable=unittest.mock.mock_open) - @patch("pathlib.Path.mkdir") - def test_add_file(self, mock_mkdir, mock_open, mock_run): - """Test the add_file method.""" - file_path = "test-dir/test-file.json" - content = {"key": "value"} - - self.github.add_file(file_path, content) - - mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) - mock_open.assert_called_once_with( - Path(self.github.local_clone_dir) / file_path, "w" - ) - mock_open().write.assert_called_once_with(json.dumps(content, indent=2)) + def test_add_file(self, mock_run): + # Mock the subprocess.run method + mock_run.return_value = MagicMock() + + # Mock os.chdir and Path + with patch("os.chdir"), patch("pathlib.Path.mkdir"), patch( + "builtins.open", unittest.mock.mock_open() + ): + self.github_automation.add_file("test/file.json", {"key": "value"}) + + # Assertions mock_run.assert_called_once_with( - ["git", "add", str(Path(self.github.local_clone_dir) / file_path)], + [ + "git", + "add", + str(Path(self.github_automation.local_clone_dir) / "test/file.json"), + ], check=True, ) @patch("subprocess.run") def test_commit_and_push(self, mock_run): - """Test the commit_and_push method.""" - branch_name = "test-branch" - commit_message = "Test commit message" + # Mock the subprocess.run method + mock_run.return_value = MagicMock() - self.github.commit_and_push(branch_name, commit_message) + # Mock os.chdir + with patch("os.chdir"): + self.github_automation.commit_and_push("test-branch", "Test commit message") - mock_run.assert_any_call(["git", "commit", "-m", commit_message], check=True) + # Assertions + mock_run.assert_any_call( + ["git", "commit", "-m", "Test commit message"], check=True + ) mock_run.assert_any_call( - ["git", "push", "-u", "origin", branch_name], check=True + ["git", "push", "-u", "origin", "test-branch"], check=True ) @patch("requests.post") def test_create_pull_request(self, mock_post): - """Test the create_pull_request method.""" - branch_name = "test-branch" - pr_title = "Test PR" - pr_body = "This is a test PR" - base_branch = "main" - + # Mock the response from GitHub API mock_response = MagicMock() - mock_response.json.return_value = {"html_url": "https://github.com/test-pr"} mock_response.raise_for_status.return_value = None + mock_response.json.return_value = {"html_url": "https://github.com/test/pull/1"} mock_post.return_value = mock_response - self.github.create_pull_request(branch_name, pr_title, pr_body, base_branch) + # Mock os.chdir + with patch("os.chdir"): + self.github_automation.create_pull_request( + "test-branch", "Test PR", "Test body" + ) + # Assertions mock_post.assert_called_once_with( - "https://api.github.com/repos/test-owner/test-repo/pulls", - headers={"Authorization": "token test-token"}, + f"https://api.github.com/repos/{self.repo_owner}/{self.repo_name}/pulls", + headers={"Authorization": f"token {self.token}"}, json={ - "title": pr_title, - "head": f"test-user:{branch_name}", - "base": base_branch, - "body": pr_body, + "title": "Test PR", + "head": f"{self.username}:test-branch", + "base": "main", + "body": "Test body", }, ) @patch("subprocess.run") - @patch("os.chdir") - def test_clean_up(self, mock_chdir, mock_run): - """Test the clean_up method.""" - self.github.clean_up() + def test_clean_up(self, mock_run): + # Mock the subprocess.run method + mock_run.return_value = MagicMock() + + # Mock os.chdir + with patch("os.chdir"): + self.github_automation.clean_up() + + # Assertions + mock_run.assert_called_once_with( + ["rm", "-rf", self.github_automation.local_clone_dir] + ) + + def test_file_exists(self): + # Mock os.path.isfile + with patch("os.path.isfile", return_value=True): + result = self.github_automation.file_exists("test/file.json") - mock_chdir.assert_called_once_with("..") - mock_run.assert_called_once_with(["rm", "-rf", self.github.local_clone_dir]) + # Assertions + self.assertTrue(result) diff --git a/deep_code/tests/utils/test_ogc_api_record.py b/deep_code/tests/utils/test_ogc_api_record.py index 52640fe..236ed4c 100644 --- a/deep_code/tests/utils/test_ogc_api_record.py +++ b/deep_code/tests/utils/test_ogc_api_record.py @@ -3,111 +3,241 @@ from deep_code.constants import OGC_API_RECORD_SPEC from deep_code.utils.ogc_api_record import ( Contact, + ExperimentAsOgcRecord, JupyterKernelInfo, - OgcRecord, + LinksBuilder, RecordProperties, Theme, ThemeConcept, + WorkflowAsOgcRecord, ) -class TestClasses(unittest.TestCase): +class TestContact(unittest.TestCase): def test_contact_initialization(self): contact = Contact( - name="Person-X", - organization="Organization X", + name="John Doe", + organization="DeepESDL", position="Researcher", - links=[{"url": "http://example.com", "type": "website"}], + links=[{"href": "https://example.com"}], contactInstructions="Contact via email", - roles=["developer", "reviewer"], + roles=["principal investigator"], ) - self.assertEqual(contact.name, "Person-X") - self.assertEqual(contact.organization, "Organization X") + self.assertEqual(contact.name, "John Doe") + self.assertEqual(contact.organization, "DeepESDL") self.assertEqual(contact.position, "Researcher") - self.assertEqual(len(contact.links), 1) + self.assertEqual(contact.links, [{"href": "https://example.com"}]) self.assertEqual(contact.contactInstructions, "Contact via email") - self.assertIn("developer", contact.roles) + self.assertEqual(contact.roles, ["principal investigator"]) + def test_contact_default_values(self): + contact = Contact(name="Jane Doe", organization="DeepESDL") + + self.assertEqual(contact.position, "") + self.assertEqual(contact.links, []) + self.assertEqual(contact.contactInstructions, "") + self.assertEqual(contact.roles, ["principal investigator"]) + + +class TestThemeConcept(unittest.TestCase): def test_theme_concept_initialization(self): - theme_concept = ThemeConcept(id="concept1") - self.assertEqual(theme_concept.id, "concept1") + theme_concept = ThemeConcept(id="climate") + + self.assertEqual(theme_concept.id, "climate") + +class TestTheme(unittest.TestCase): def test_theme_initialization(self): - theme_concepts = [ThemeConcept(id="concept1"), ThemeConcept(id="concept2")] - theme = Theme(concepts=theme_concepts, scheme="http://example.com/scheme") + theme_concept = ThemeConcept(id="climate") + theme = Theme(concepts=[theme_concept], scheme="https://example.com") - self.assertEqual(len(theme.concepts), 2) - self.assertEqual(theme.scheme, "http://example.com/scheme") + self.assertEqual(theme.concepts, [theme_concept]) + self.assertEqual(theme.scheme, "https://example.com") + +class TestJupyterKernelInfo(unittest.TestCase): def test_jupyter_kernel_info_initialization(self): kernel_info = JupyterKernelInfo( - name="Python", python_version=3.9, env_file="env.yml" + name="python3", python_version=3.9, env_file="environment.yml" ) - self.assertEqual(kernel_info.name, "Python") + self.assertEqual(kernel_info.name, "python3") self.assertEqual(kernel_info.python_version, 3.9) - self.assertEqual(kernel_info.env_file, "env.yml") + self.assertEqual(kernel_info.env_file, "environment.yml") + +class TestRecordProperties(unittest.TestCase): def test_record_properties_initialization(self): kernel_info = JupyterKernelInfo( - name="Python", python_version=3.9, env_file="env.yml" + name="python3", python_version=3.9, env_file="environment.yml" + ) + contact = Contact(name="John Doe", organization="DeepESDL") + theme = Theme( + concepts=[ThemeConcept(id="climate")], scheme="https://example.com" + ) + + record_properties = RecordProperties( + created="2023-01-01", + type="workflow", + title="Test Workflow", + description="A test workflow", + jupyter_kernel_info=kernel_info, + osc_project="DeepESDL", + osc_workflow="test-workflow", + updated="2023-01-02", + contacts=[contact], + themes=[theme], + keywords=["test", "workflow"], + formats=[{"type": "application/json"}], + license="MIT", + ) + + self.assertEqual(record_properties.created, "2023-01-01") + self.assertEqual(record_properties.updated, "2023-01-02") + self.assertEqual(record_properties.type, "workflow") + self.assertEqual(record_properties.title, "Test Workflow") + self.assertEqual(record_properties.description, "A test workflow") + self.assertEqual(record_properties.jupyter_kernel_info, kernel_info) + self.assertEqual(record_properties.osc_project, "DeepESDL") + self.assertEqual(record_properties.osc_workflow, "test-workflow") + self.assertEqual(record_properties.keywords, ["test", "workflow"]) + self.assertEqual(record_properties.contacts, [contact]) + self.assertEqual(record_properties.themes, [theme]) + self.assertEqual(record_properties.formats, [{"type": "application/json"}]) + self.assertEqual(record_properties.license, "MIT") + + def test_record_properties_to_dict(self): + kernel_info = JupyterKernelInfo( + name="python3", python_version=3.9, env_file="environment.yml" + ) + record_properties = RecordProperties( + created="2023-01-01", + type="workflow", + title="Test Workflow", + description="A test workflow", + jupyter_kernel_info=kernel_info, + osc_project="DeepESDL", + osc_workflow="test-workflow", ) - contacts = [Contact(name="Jane Doe", organization="Org Y")] - themes = [Theme(concepts=[ThemeConcept(id="concept1")], scheme="scheme1")] + result = record_properties.to_dict() + + self.assertEqual(result["created"], "2023-01-01") + self.assertEqual(result["type"], "workflow") + self.assertEqual(result["title"], "Test Workflow") + self.assertEqual(result["description"], "A test workflow") + self.assertEqual(result["jupyter_kernel_info"], kernel_info.to_dict()) + self.assertEqual(result["osc:project"], "DeepESDL") + self.assertEqual(result["osc:workflow"], "test-workflow") + self.assertNotIn("osc_project", result) + self.assertNotIn("osc_workflow", result) + + +class TestLinksBuilder(unittest.TestCase): + def test_build_theme_links_for_records(self): + links_builder = LinksBuilder(themes=["climate", "ocean"]) + theme_links = links_builder.build_theme_links_for_records() + + expected_links = [ + { + "rel": "related", + "href": "../../themes/climate/catalog.json", + "type": "application/json", + "title": "Theme: Climate", + }, + { + "rel": "related", + "href": "../../themes/ocean/catalog.json", + "type": "application/json", + "title": "Theme: Ocean", + }, + ] + + self.assertEqual(theme_links, expected_links) + + def test_build_link_to_dataset(self): + link = LinksBuilder.build_link_to_dataset("test-collection") + + expected_link = [ + { + "rel": "child", + "href": "../../products/test-collection/collection.json", + "type": "application/json", + "title": "test-collection", + } + ] + + self.assertEqual(link, expected_link) + + +class TestWorkflowAsOgcRecord(unittest.TestCase): + def test_workflow_as_ogc_record_initialization(self): + kernel_info = JupyterKernelInfo( + name="python3", python_version=3.9, env_file="environment.yml" + ) record_properties = RecordProperties( - created="2025-01-01", - type="dataset", - title="Sample Dataset", - description="A sample dataset", + created="2023-01-01", + type="workflow", + title="Test Workflow", + description="A test workflow", jupyter_kernel_info=kernel_info, - updated="2025-01-02", - contacts=contacts, - themes=themes, - keywords=["sample", "test"], - formats=[{"format": "JSON"}], - license="CC-BY", + osc_project="DeepESDL", ) - self.assertEqual(record_properties.created, "2025-01-01") - self.assertEqual(record_properties.updated, "2025-01-02") - self.assertEqual(record_properties.type, "dataset") - self.assertEqual(record_properties.title, "Sample Dataset") - self.assertEqual(record_properties.description, "A sample dataset") - self.assertEqual(record_properties.jupyter_kernel_info.name, "Python") - self.assertEqual(len(record_properties.contacts), 1) - self.assertEqual(len(record_properties.themes), 1) - self.assertIn("sample", record_properties.keywords) - self.assertEqual(record_properties.license, "CC-BY") - - def test_ogc_record_initialization(self): + workflow_record = WorkflowAsOgcRecord( + id="test-workflow", + type="workflow", + title="Test Workflow", + jupyter_notebook_url="https://example.com/notebook.ipynb", + properties=record_properties, + links=[{"rel": "self", "href": "https://example.com"}], + ) + + self.assertEqual(workflow_record.id, "test-workflow") + self.assertEqual(workflow_record.type, "workflow") + self.assertEqual(workflow_record.title, "Test Workflow") + self.assertEqual( + workflow_record.jupyter_notebook_url, "https://example.com/notebook.ipynb" + ) + self.assertEqual(workflow_record.properties, record_properties) + self.assertEqual(workflow_record.conformsTo, [OGC_API_RECORD_SPEC]) + self.assertEqual(workflow_record.links[0]["rel"], "root") + self.assertEqual(workflow_record.links[-1]["rel"], "self") + + +class TestExperimentAsOgcRecord(unittest.TestCase): + def test_experiment_as_ogc_record_initialization(self): kernel_info = JupyterKernelInfo( - name="Python", python_version=3.9, env_file="env.yml" + name="python3", python_version=3.12, env_file="environment.yml" ) - properties = RecordProperties( - created="2025-01-01", - type="dataset", - title="Sample Dataset", - description="A sample dataset", + record_properties = RecordProperties( + created="2023-01-01", + type="experiment", + title="Test Experiment", + description="A test experiment", jupyter_kernel_info=kernel_info, + osc_project="DeepESDL", ) - ogc_record = OgcRecord( - id="record1", - type="Feature", - time={"start": "2025-01-01T00:00:00Z", "end": "2025-01-02T00:00:00Z"}, - properties=properties, - links=[{"href": "http://example.com", "rel": "self"}], - linkTemplates=[{"template": "http://example.com/{id}"}], + experiment_record = ExperimentAsOgcRecord( + id="test-experiment", + title="Test Experiment", + type="experiment", + jupyter_notebook_url="https://example.com/notebook.ipynb", + collection_id="test-collection", + properties=record_properties, + links=[{"rel": "self", "href": "https://example.com"}], ) - self.assertEqual(ogc_record.id, "record1") - self.assertEqual(ogc_record.type, "Feature") - self.assertEqual(ogc_record.time["start"], "2025-01-01T00:00:00Z") - self.assertEqual(ogc_record.properties.title, "Sample Dataset") - self.assertEqual(len(ogc_record.links), 1) + self.assertEqual(experiment_record.id, "test-experiment") + self.assertEqual(experiment_record.title, "Test Experiment") + self.assertEqual(experiment_record.type, "experiment") self.assertEqual( - ogc_record.linkTemplates[0]["template"], "http://example.com/{id}" + experiment_record.jupyter_notebook_url, "https://example.com/notebook.ipynb" ) - self.assertEqual(ogc_record.conformsTo[0], OGC_API_RECORD_SPEC) + self.assertEqual(experiment_record.collection_id, "test-collection") + self.assertEqual(experiment_record.properties, record_properties) + self.assertEqual(experiment_record.conformsTo, [OGC_API_RECORD_SPEC]) + self.assertEqual(experiment_record.links[0]["rel"], "root") + self.assertEqual(experiment_record.links[-1]["rel"], "self") diff --git a/deep_code/tests/utils/test_ogc_record_generator.py b/deep_code/tests/utils/test_ogc_record_generator.py index f4fe372..d56cf7d 100644 --- a/deep_code/tests/utils/test_ogc_record_generator.py +++ b/deep_code/tests/utils/test_ogc_record_generator.py @@ -1,7 +1,6 @@ import unittest -from datetime import datetime, timezone -from deep_code.constants import DEFAULT_THEME_SCHEME +from deep_code.constants import OSC_THEME_SCHEME from deep_code.utils.ogc_record_generator import OSCWorkflowOGCApiRecordGenerator @@ -30,7 +29,7 @@ def test_build_theme(self): self.assertEqual(len(theme.concepts), 2) self.assertEqual(theme.concepts[0].id, "theme1") self.assertEqual(theme.concepts[1].id, "theme2") - self.assertEqual(theme.scheme, DEFAULT_THEME_SCHEME) + self.assertEqual(theme.scheme, OSC_THEME_SCHEME) def test_build_record_properties(self): generator = OSCWorkflowOGCApiRecordGenerator() @@ -50,8 +49,6 @@ def test_build_record_properties(self): record_properties = generator.build_record_properties(properties, contacts) - now_iso = datetime.now(timezone.utc).isoformat() - self.assertEqual(record_properties.title, "Test Workflow") self.assertEqual(record_properties.description, "A test description") self.assertEqual(len(record_properties.contacts), 1) diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index ba762e1..c17264f 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -1,24 +1,33 @@ #!/usr/bin/env python3 - # Copyright (c) 2025 by Brockmann Consult GmbH # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. +import copy +import json import logging +from datetime import datetime from pathlib import Path import fsspec import yaml +from pystac import Catalog, Link from deep_code.constants import ( + EXPERIMENT_BASE_CATALOG_SELF_HREF, OSC_BRANCH_NAME, OSC_REPO_NAME, OSC_REPO_OWNER, - WF_BRANCH_NAME, + WORKFLOW_BASE_CATALOG_SELF_HREF, ) from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator from deep_code.utils.github_automation import GitHubAutomation -from deep_code.utils.ogc_api_record import OgcRecord +from deep_code.utils.helper import serialize +from deep_code.utils.ogc_api_record import ( + ExperimentAsOgcRecord, + LinksBuilder, + WorkflowAsOgcRecord, +) from deep_code.utils.ogc_record_generator import OSCWorkflowOGCApiRecordGenerator logger = logging.getLogger(__name__) @@ -43,6 +52,8 @@ def __init__(self): self.github_automation = GitHubAutomation( self.github_username, self.github_token, OSC_REPO_OWNER, OSC_REPO_NAME ) + self.github_automation.fork_repository() + self.github_automation.clone_sync_repository() def publish_files( self, @@ -65,9 +76,6 @@ def publish_files( URL of the created pull request. """ try: - logger.info("Forking and cloning repository...") - self.github_automation.fork_repository() - self.github_automation.clone_repository() self.github_automation.create_branch(branch_name) # Add each file to the branch @@ -90,54 +98,77 @@ def publish_files( self.github_automation.clean_up() -class DatasetPublisher: +class Publisher: """Publishes products (datasets) to the OSC GitHub repository. - Inherits from BasePublisher for GitHub publishing logic. """ - def __init__(self): + def __init__(self, dataset_config_path: str, workflow_config_path: str): # Composition self.gh_publisher = GitHubPublisher() + self.collection_id = "" + self.workflow_title = "" - def publish_dataset(self, dataset_config_path: str): - """Publish a product collection to the specified GitHub repository.""" - with fsspec.open(dataset_config_path, "r") as file: - dataset_config = yaml.safe_load(file) or {} - - dataset_id = dataset_config.get("dataset_id") - collection_id = dataset_config.get("collection_id") - documentation_link = dataset_config.get("documentation_link") - access_link = dataset_config.get("access_link") - dataset_status = dataset_config.get("dataset_status") - osc_region = dataset_config.get("osc_region") - osc_themes = dataset_config.get("osc_themes") - cf_params = dataset_config.get("cf_parameter") - - if not dataset_id or not collection_id: - raise ValueError("Dataset ID or Collection ID missing in the config.") + # Paths to configuration files + self.dataset_config_path = dataset_config_path + self.workflow_config_path = workflow_config_path - logger.info("Generating STAC collection...") + # Load configuration files + self._read_config_files() + self.collection_id = self.dataset_config.get("collection_id") + self.workflow_title = self.workflow_config.get("properties", {}).get("title") - generator = OscDatasetStacGenerator( - dataset_id=dataset_id, - collection_id=collection_id, - documentation_link=documentation_link, - access_link=access_link, - osc_status=dataset_status, - osc_region=osc_region, - osc_themes=osc_themes, - cf_params=cf_params, - ) + if not self.collection_id: + raise ValueError("collection_id is missing in dataset config.") - variable_ids = generator.get_variable_ids() - ds_collection = generator.build_dataset_stac_collection() + def _read_config_files(self) -> None: + with fsspec.open(self.dataset_config_path, "r") as file: + self.dataset_config = yaml.safe_load(file) or {} + with fsspec.open(self.workflow_config_path, "r") as file: + self.workflow_config = yaml.safe_load(file) or {} - # Prepare a dictionary of file paths and content - file_dict = {} - product_path = f"products/{collection_id}/collection.json" - file_dict[product_path] = ds_collection.to_dict() + @staticmethod + def _write_to_file(file_path: str, data: dict): + """Write a dictionary to a JSON file. - # Add or update variable files + Args: + file_path (str): The path to the file. + data (dict): The data to write. + """ + # Create the directory if it doesn't exist + Path(file_path).parent.mkdir(parents=True, exist_ok=True) + try: + json_content = json.dumps(data, indent=2, default=serialize) + except TypeError as e: + raise RuntimeError(f"JSON serialization failed: {e}") + + with open(file_path, "w") as f: + f.write(json_content) + + def _update_and_add_to_file_dict( + self, file_dict, catalog_path, update_method, *args + ): + """Update a catalog using the specified method and add it to file_dict. + + Args: + file_dict: The dictionary to which the updated catalog will be added. + catalog_path: The path to the catalog file. + update_method: The method to call for updating the catalog. + *args: Additional arguments to pass to the update method. + """ + full_path = ( + Path(self.gh_publisher.github_automation.local_clone_dir) / catalog_path + ) + updated_catalog = update_method(full_path, *args) + file_dict[full_path] = updated_catalog.to_dict() + + def _update_variable_catalogs(self, generator, file_dict, variable_ids): + """Update or create variable catalogs and add them to file_dict. + + Args: + generator: The generator object. + file_dict: The dictionary to which the updated catalogs will be added. + variable_ids: A list of variable IDs. + """ for var_id in variable_ids: var_file_path = f"variables/{var_id}/catalog.json" if not self.gh_publisher.github_automation.file_exists(var_file_path): @@ -160,74 +191,229 @@ def publish_dataset(self, dataset_config_path: str): ) file_dict[var_file_path] = updated_catalog.to_dict() - # Create branch name, commit message, PR info - branch_name = f"{OSC_BRANCH_NAME}-{collection_id}" - commit_message = f"Add new dataset collection: {collection_id}" - pr_title = "Add new dataset collection" - pr_body = "This PR adds a new dataset collection to the repository." - - # Publish all files in one go - pr_url = self.gh_publisher.publish_files( - branch_name=branch_name, - file_dict=file_dict, - commit_message=commit_message, - pr_title=pr_title, - pr_body=pr_body, + def publish_dataset(self, write_to_file: bool = False): + """Prepare dataset/product collection for publishing to the specified GitHub + repository.""" + + dataset_id = self.dataset_config.get("dataset_id") + self.collection_id = self.dataset_config.get("collection_id") + documentation_link = self.dataset_config.get("documentation_link") + access_link = self.dataset_config.get("access_link") + dataset_status = self.dataset_config.get("dataset_status") + osc_region = self.dataset_config.get("osc_region") + osc_themes = self.dataset_config.get("osc_themes") + cf_params = self.dataset_config.get("cf_parameter") + + if not dataset_id or not self.collection_id: + raise ValueError("Dataset ID or Collection ID missing in the config.") + + logger.info("Generating STAC collection...") + + generator = OscDatasetStacGenerator( + dataset_id=dataset_id, + collection_id=self.collection_id, + documentation_link=documentation_link, + access_link=access_link, + osc_status=dataset_status, + osc_region=osc_region, + osc_themes=osc_themes, + cf_params=cf_params, ) - logger.info(f"Pull request created: {pr_url}") + variable_ids = generator.get_variable_ids() + ds_collection = generator.build_dataset_stac_collection() + # Prepare a dictionary of file paths and content + file_dict = {} + product_path = f"products/{self.collection_id}/collection.json" + file_dict[product_path] = ds_collection.to_dict() -class WorkflowPublisher: - """Publishes workflows to the OSC GitHub repository.""" + # Update or create variable catalogs for each osc:variable + self._update_variable_catalogs(generator, file_dict, variable_ids) - def __init__(self): - self.gh_publisher = GitHubPublisher() + # Update variable base catalog + variable_base_catalog_path = "variables/catalog.json" + self._update_and_add_to_file_dict( + file_dict, + variable_base_catalog_path, + generator.update_variable_base_catalog, + variable_ids, + ) + + # Update product base catalog + product_catalog_path = "products/catalog.json" + self._update_and_add_to_file_dict( + file_dict, product_catalog_path, generator.update_product_base_catalog + ) + + # Update DeepESDL collection + deepesdl_collection_path = "projects/deep-earth-system-data-lab/collection.json" + self._update_and_add_to_file_dict( + file_dict, deepesdl_collection_path, generator.update_deepesdl_collection + ) + + # Write to files if testing + if write_to_file: + for file_path, data in file_dict.items(): + self._write_to_file(file_path, data) # Pass file_path and data + return {} + return file_dict @staticmethod def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").lower() if name else None - def publish_workflow(self, workflow_config_path: str): - with fsspec.open(workflow_config_path, "r") as file: - workflow_config = yaml.safe_load(file) or {} + def _update_base_catalog( + self, catalog_path: str, item_id: str, self_href: str + ) -> Catalog: + """Update a base catalog by adding a link to a new item. - workflow_id = self._normalize_name(workflow_config.get("workflow_id")) + Args: + catalog_path: Path to the base catalog JSON file. + item_id: ID of the new item (experiment or workflow). + self_href: Self-href for the base catalog. + + Returns: + Updated Catalog object. + """ + # Load the base catalog + base_catalog = Catalog.from_file( + Path(self.gh_publisher.github_automation.local_clone_dir) / catalog_path + ) + + # Add a link to the new item + base_catalog.add_link( + Link( + rel="item", + target=f"./{item_id}/record.json", + media_type="application/json", + title=f"{self.workflow_title}", + ) + ) + + # Set the self-href for the base catalog + base_catalog.set_self_href(self_href) + + return base_catalog + + def publish_workflow_experiment(self, write_to_file: bool = False): + """prepare workflow and experiment as ogc api record to publish it to the + specified GitHub repository.""" + workflow_id = self._normalize_name(self.workflow_config.get("workflow_id")) if not workflow_id: raise ValueError("workflow_id is missing in workflow config.") - properties_list = workflow_config.get("properties", []) - contacts = workflow_config.get("contact", []) - links = workflow_config.get("links", []) + properties_list = self.workflow_config.get("properties", {}) + osc_themes = properties_list.get("themes") + contacts = self.workflow_config.get("contact", []) + links = self.workflow_config.get("links", []) + jupyter_notebook_url = self.workflow_config.get("jupyter_notebook_url") logger.info("Generating OGC API Record for the workflow...") rg = OSCWorkflowOGCApiRecordGenerator() wf_record_properties = rg.build_record_properties(properties_list, contacts) + # make a copy for experiment record + exp_record_properties = copy.deepcopy(wf_record_properties) - ogc_record = OgcRecord( + link_builder = LinksBuilder(osc_themes) + theme_links = link_builder.build_theme_links_for_records() + + workflow_record = WorkflowAsOgcRecord( id=workflow_id, type="Feature", - time={}, + title=self.workflow_title, properties=wf_record_properties, - links=links, + links=links + theme_links, + jupyter_notebook_url=jupyter_notebook_url, + themes=osc_themes, + ) + # Convert to dictionary and cleanup + workflow_dict = workflow_record.to_dict() + if "jupyter_notebook_url" in workflow_dict: + del workflow_dict["jupyter_notebook_url"] + if "osc_workflow" in workflow_dict["properties"]: + del workflow_dict["properties"]["osc_workflow"] + wf_file_path = f"workflows/{workflow_id}/record.json" + file_dict = {wf_file_path: workflow_dict} + + # Build properties for the experiment record + exp_record_properties.type = "experiment" + exp_record_properties.osc_workflow = workflow_id + + experiment_record = ExperimentAsOgcRecord( + id=workflow_id, + title=self.workflow_title, + type="Feature", + jupyter_notebook_url=jupyter_notebook_url, + collection_id=self.collection_id, + properties=exp_record_properties, + links=links + theme_links, + ) + # Convert to dictionary and cleanup + experiment_dict = experiment_record.to_dict() + if "jupyter_notebook_url" in experiment_dict: + del experiment_dict["jupyter_notebook_url"] + if "collection_id" in experiment_dict: + del experiment_dict["collection_id"] + if "osc:project" in experiment_dict["properties"]: + del experiment_dict["properties"]["osc:project"] + exp_file_path = f"experiments/{workflow_id}/record.json" + file_dict[exp_file_path] = experiment_dict + + # Update base catalogs of experiments and workflows with links + file_dict["experiments/catalog.json"] = self._update_base_catalog( + catalog_path="experiments/catalog.json", + item_id=workflow_id, + self_href=EXPERIMENT_BASE_CATALOG_SELF_HREF, ) - file_path = f"workflow/{workflow_id}/collection.json" - - # Prepare the single file dict - file_dict = {file_path: ogc_record.to_dict()} - - branch_name = f"{WF_BRANCH_NAME}-{workflow_id}" - commit_message = f"Add new workflow: {workflow_id}" - pr_title = "Add new workflow" - pr_body = "This PR adds a new workflow to the OSC repository." - - pr_url = self.gh_publisher.publish_files( - branch_name=branch_name, - file_dict=file_dict, - commit_message=commit_message, - pr_title=pr_title, - pr_body=pr_body, + file_dict["workflows/catalog.json"] = self._update_base_catalog( + catalog_path="workflows/catalog.json", + item_id=workflow_id, + self_href=WORKFLOW_BASE_CATALOG_SELF_HREF, ) + # Write to files if testing + if write_to_file: + for file_path, data in file_dict.items(): + self._write_to_file(file_path, data) + return {} + return file_dict + + def publish_all(self, write_to_file: bool = False): + """Publish both dataset and workflow/experiment in a single PR.""" + # Get file dictionaries from both methods + dataset_files = self.publish_dataset(write_to_file=write_to_file) + workflow_files = self.publish_workflow_experiment(write_to_file=write_to_file) + + # Combine the file dictionaries + combined_files = {**dataset_files, **workflow_files} + + if not write_to_file: + # Create branch name, commit message, PR info + branch_name = ( + f"{OSC_BRANCH_NAME}-{self.collection_id}" + f"-{datetime.now().strftime('%Y%m%d%H%M%S')}" + ) + commit_message = ( + f"Add new dataset collection: {self.collection_id} and " + f"workflow/experiment: {self.workflow_config.get('workflow_id')}" + ) + pr_title = ( + f"Add new dataset collection: {self.collection_id} and " + f"workflow/experiment: {self.workflow_config.get('workflow_id')}" + ) + pr_body = ( + f"This PR adds a new dataset collection: {self.collection_id} and " + f"its corresponding workflow/experiment to the repository." + ) - logger.info(f"Pull request created: {pr_url}") + # Publish all files in one go + pr_url = self.gh_publisher.publish_files( + branch_name=branch_name, + file_dict=combined_files, + commit_message=commit_message, + pr_title=pr_title, + pr_body=pr_body, + ) + + logger.info(f"Pull request created: {pr_url}") diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 3d4da00..9b6b703 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - # Copyright (c) 2025 by Brockmann Consult GmbH # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. @@ -12,6 +11,13 @@ from pystac import Catalog, Collection, Extent, Link, SpatialExtent, TemporalExtent from xcube.core.store import new_data_store +from deep_code.constants import ( + DEEPESDL_COLLECTION_SELF_HREF, + OSC_THEME_SCHEME, + PRODUCT_BASE_CATALOG_SELF_HREF, + VARIABLE_BASE_CATALOG_SELF_HREF, +) +from deep_code.utils.ogc_api_record import Theme, ThemeConcept from deep_code.utils.osc_extension import OscExtension @@ -169,7 +175,10 @@ def _get_temporal_extent(self) -> TemporalExtent: @staticmethod def _normalize_name(name: str | None) -> str | None: - return name.replace(" ", "-").lower() if name else None + if name: + return (name.replace(" ", "-"). + replace("_", "-").lower()) + return None def _get_general_metadata(self) -> dict: return { @@ -193,7 +202,11 @@ def extract_metadata_for_variable(self, variable_data) -> dict: def get_variable_ids(self) -> list[str]: """Get variable IDs for all variables in the dataset.""" - return list(self.variables_metadata.keys()) + variable_ids = list(self.variables_metadata.keys()) + # Remove 'crs' and 'spatial_ref' from the list if they exist, note that + # spatial_ref will be normalized to spatial-ref in variable_ids and skipped. + return [var_id for var_id in variable_ids if var_id not in ["crs", + "spatial-ref"]] def get_variables_metadata(self) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" @@ -217,11 +230,9 @@ def _add_gcmd_link_to_var_catalog( """ gcmd_keyword_url = var_metadata.get("gcmd_keyword_url") if not gcmd_keyword_url: - self.logger.debug( - f"No gcmd_keyword_url in var_metadata. Skipping adding GCMD link in " - f'the {var_metadata.get("variable_id")} catalog' - ) - return + gcmd_keyword_url = input( + f"Enter GCMD keyword URL or a similar url for" + f" {var_metadata.get("variable_id")}: ").strip() var_catalog.add_link( Link( rel="via", @@ -257,7 +268,7 @@ def build_variable_catalog(self, var_metadata) -> Catalog: var_catalog = Catalog( id=var_id, description=var_metadata.get("description"), - title=var_id, + title=self.format_string(var_id), stac_extensions=[ "https://stac-extensions.github.io/themes/v1.0.0/schema.json" ], @@ -303,6 +314,8 @@ def build_variable_catalog(self, var_metadata) -> Catalog: # Add gcmd link for the variable definition self._add_gcmd_link_to_var_catalog(var_catalog, var_metadata) + self.add_themes_as_related_links_var_catalog(var_catalog) + self_href = ( f"https://esa-earthcode.github.io/open-science-catalog-metadata/variables" f"/{var_id}/catalog.json" @@ -312,6 +325,74 @@ def build_variable_catalog(self, var_metadata) -> Catalog: return var_catalog + def update_product_base_catalog(self, product_catalog_path) -> Catalog: + """Link product to base product catalog""" + product_base_catalog = Catalog.from_file(product_catalog_path) + product_base_catalog.add_link( + Link( + rel="child", + target=f"./{self.collection_id}/collection.json", + media_type="application/json", + title=self.collection_id, + ) + ) + # 'self' link: the direct URL where this JSON is hosted + product_base_catalog.set_self_href(PRODUCT_BASE_CATALOG_SELF_HREF) + return product_base_catalog + + def update_variable_base_catalog(self, variable_base_catalog_path, variable_ids) \ + -> ( + Catalog): + """Link product to base product catalog""" + variable_base_catalog = Catalog.from_file(variable_base_catalog_path) + for var_id in variable_ids: + variable_base_catalog.add_link( + Link( + rel="child", + target=f"./{var_id}/catalog.json", + media_type="application/json", + title=self.format_string(var_id), + ) + ) + # 'self' link: the direct URL where this JSON is hosted + variable_base_catalog.set_self_href(VARIABLE_BASE_CATALOG_SELF_HREF) + return variable_base_catalog + + def add_themes_as_related_links_var_catalog(self, var_catalog): + """Add themes as related links to variable catalog""" + for theme in self.osc_themes: + var_catalog.add_link( + Link( + rel="related", + target=f"../../themes/{theme}/catalog.json", + media_type="application/json", + title=f"Theme: {self.format_string(theme)}", + ) + ) + + def update_deepesdl_collection(self, deepesdl_collection_full_path): + deepesdl_collection = Collection.from_file(deepesdl_collection_full_path) + deepesdl_collection.add_link( + Link( + rel="child", + target=f"../../products/{self.collection_id}/collection.json", + media_type="application/json", + title=self.collection_id, + ) + ) + # add themes to deepesdl + for theme in self.osc_themes: + deepesdl_collection.add_link( + Link( + rel="related", + target=f"../../themes/{theme}/catalog.json", + media_type="application/json", + title=f"Theme: {self.format_string(theme)}" + ) + ) + deepesdl_collection.set_self_href(DEEPESDL_COLLECTION_SELF_HREF) + return deepesdl_collection + def update_existing_variable_catalog(self, var_file_path, var_id) -> Catalog: existing_catalog = Catalog.from_file(var_file_path) now_iso = datetime.now(timezone.utc).isoformat() @@ -326,6 +407,7 @@ def update_existing_variable_catalog(self, var_file_path, var_id) -> Catalog: title=self.collection_id, ) ) + self.add_themes_as_related_links_var_catalog(existing_catalog) self_href = ( f"https://esa-earthcode.github.io/open-science-catalog-metadata/variables" f"/{var_id}/catalog.json" @@ -335,6 +417,20 @@ def update_existing_variable_catalog(self, var_file_path, var_id) -> Catalog: return existing_catalog + @staticmethod + def format_string(s: str) -> str: + # Strip leading/trailing spaces/underscores and replace underscores with spaces + words = s.strip(" _").replace("_", " ").replace("-", " ").split() + # Capitalize each word and join them with a space + return " ".join(word.capitalize() for word in words) + + @staticmethod + def build_theme(osc_themes: list[str]) -> Theme: + """Convert each string into a ThemeConcept + """ + concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] + return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) + def build_dataset_stac_collection(self) -> Collection: """Build an OSC STAC Collection for the dataset. @@ -363,7 +459,6 @@ def build_dataset_stac_collection(self) -> Collection: osc_extension.osc_type = "product" osc_extension.osc_status = self.osc_status osc_extension.osc_region = self.osc_region - osc_extension.osc_themes = self.osc_themes osc_extension.osc_variables = variables osc_extension.osc_missions = self.osc_missions if self.cf_params: @@ -400,23 +495,49 @@ def build_dataset_stac_collection(self) -> Collection: title="Products", ) ) + # Add variables ref for var in variables: collection.add_link( Link( rel="related", - target=f"../../varibales/{var}/catalog.json", + target=f"../../variables/{var}/catalog.json", media_type="application/json", - title="Variable: " + var, + title="Variable: " + self.format_string(var), ) ) self_href = ( "https://esa-earthcode.github.io/" - "open-science-catalog-metadata/products/deepesdl/collection.json" + f"open-science-catalog-metadata/products/{self.collection_id}/collection.json" ) collection.set_self_href(self_href) + # align with themes instead of osc:themes + if self.osc_themes: + theme_obj = self.build_theme(self.osc_themes) + collection.extra_fields["themes"] = [theme_obj] + + for theme in self.osc_themes: + formatted_theme = self.format_string(theme) + collection.add_link( + Link( + rel="related", + target=f"../../themes/{theme}/catalog.json", + media_type="application/json", + title=f"Theme: {formatted_theme}", + ) + ) + + collection.add_link( + Link( + rel="related", + target="../../projects/deep-earth-system-data-lab/collection.json", + media_type="application/json", + title="Project: DeepESDL" + ) + ) + # Validate OSC extension fields try: osc_extension.validate_extension() diff --git a/deep_code/utils/github_automation.py b/deep_code/utils/github_automation.py index dbecfe4..bdbae38 100644 --- a/deep_code/utils/github_automation.py +++ b/deep_code/utils/github_automation.py @@ -12,6 +12,8 @@ import requests +from deep_code.utils.helper import serialize + class GitHubAutomation: """Automates GitHub operations needed to create a Pull Request. @@ -43,22 +45,33 @@ def fork_repository(self): response.raise_for_status() logging.info(f"Repository forked to {self.username}/{self.repo_name}") - def clone_repository(self): - """Clone the forked repository locally.""" - logging.info("Cloning forked repository...") - try: - subprocess.run( - ["git", "clone", self.fork_repo_url, self.local_clone_dir], check=True - ) - os.chdir(self.local_clone_dir) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"Failed to clone repository: {e}") + def clone_sync_repository(self): + """Clone the forked repository locally if it doesn't exist, or pull updates if it does.""" + logging.info("Checking local repository...") + if not os.path.exists(self.local_clone_dir): + logging.info("Cloning forked repository...") + try: + subprocess.run( + ["git", "clone", self.fork_repo_url, self.local_clone_dir], + check=True, + ) + logging.info(f"Repository cloned to {self.local_clone_dir}") + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to clone repository: {e}") + else: + logging.info("Local repository already exists. Pulling latest changes...") + try: + os.chdir(self.local_clone_dir) + subprocess.run(["git", "pull"], check=True) + logging.info("Repository updated with latest changes.") + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to pull latest changes: {e}") - @staticmethod - def create_branch(branch_name: str): + def create_branch(self, branch_name: str): """Create a new branch in the local repository.""" logging.info(f"Creating new branch: {branch_name}...") try: + os.chdir(self.local_clone_dir) subprocess.run(["git", "checkout", "-b", branch_name], check=True) except subprocess.CalledProcessError as e: raise RuntimeError(f"Failed Creating branch: '{branch_name}': {e}") @@ -66,22 +79,29 @@ def create_branch(branch_name: str): def add_file(self, file_path: str, content): """Add a new file to the local repository.""" logging.info(f"Adding new file: {file_path}...") + os.chdir(self.local_clone_dir) full_path = Path(self.local_clone_dir) / file_path full_path.parent.mkdir(parents=True, exist_ok=True) + # Ensure content is serializable + if hasattr(content, "to_dict"): + content = content.to_dict() + if not isinstance(content, (dict, list, str, int, float, bool, type(None))): + raise TypeError(f"Cannot serialize content of type {type(content)}") + try: + json_content = json.dumps(content, indent=2, default=serialize) + except TypeError as e: + raise RuntimeError(f"JSON serialization failed: {e}") with open(full_path, "w") as f: - # Convert content to dictionary if it's a PySTAC object - if hasattr(content, "to_dict"): - content = content.to_dict() - f.write(json.dumps(content, indent=2)) + f.write(json_content) try: subprocess.run(["git", "add", str(full_path)], check=True) except subprocess.CalledProcessError as e: raise RuntimeError(f"Failed to add file '{file_path}': {e}") - @staticmethod - def commit_and_push(branch_name: str, commit_message: str): + def commit_and_push(self, branch_name: str, commit_message: str): """Commit changes and push to the forked repository.""" logging.info("Committing and pushing changes...") + os.chdir(self.local_clone_dir) try: subprocess.run(["git", "commit", "-m", commit_message], check=True) subprocess.run(["git", "push", "-u", "origin", branch_name], check=True) @@ -93,6 +113,7 @@ def create_pull_request( ): """Create a pull request from the forked repository to the base repository.""" logging.info("Creating a pull request...") + os.chdir(self.local_clone_dir) url = f"https://api.github.com/repos/{self.repo_owner}/{self.repo_name}/pulls" headers = {"Authorization": f"token {self.token}"} data = { diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py new file mode 100644 index 0000000..cca6b75 --- /dev/null +++ b/deep_code/utils/helper.py @@ -0,0 +1,14 @@ +def serialize(obj): + """Convert non-serializable objects to JSON-compatible formats. + Args: + obj: The object to serialize. + Returns: + A JSON-compatible representation of the object. + Raises: + TypeError: If the object cannot be serialized. + """ + if isinstance(obj, set): + return list(obj) + if hasattr(obj, "__dict__"): + return obj.__dict__ + raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 437c2c8..fd417ca 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -1,9 +1,13 @@ from typing import Any, Optional from xrlint.util.constructible import MappingConstructible -from xrlint.util.serializable import JsonSerializable +from xrlint.util.serializable import JsonSerializable, JsonValue -from deep_code.constants import OGC_API_RECORD_SPEC +from deep_code.constants import ( + BASE_URL_OSC, + OGC_API_RECORD_SPEC, + PROJECT_COLLECTION_NAME, +) class Contact(MappingConstructible["Contact"], JsonSerializable): @@ -30,7 +34,7 @@ def __init__(self, id: str): class Theme(MappingConstructible["Theme"], JsonSerializable): - def __init__(self, concepts: list[ThemeConcept], scheme: str): + def __init__(self, concepts: list, scheme: str): self.concepts = concepts self.scheme = scheme @@ -50,6 +54,8 @@ def __init__( title: str, description: str, jupyter_kernel_info: JupyterKernelInfo, + osc_project: str, + osc_workflow: str = None, updated: str = None, contacts: list[Contact] = None, themes: list[Theme] = None, @@ -63,32 +69,200 @@ def __init__( self.title = title self.description = description self.jupyter_kernel_info = jupyter_kernel_info + self.osc_project = osc_project + self.osc_workflow = osc_workflow self.keywords = keywords or [] self.contacts = contacts self.themes = themes self.formats = formats or [] self.license = license + def to_dict(self, value_name: str | None = None) -> dict[str, JsonValue]: + """Convert this object into a JSON-serializable dictionary.""" + data = super().to_dict(value_name) + if self.osc_workflow is not None: + data["osc:workflow"] = self.osc_workflow + del data["osc_workflow"] # Remove the original key as it has been renamed + if self.osc_project is not None: + data["osc:project"] = self.osc_project + del data["osc_project"] + return data -class OgcRecord(MappingConstructible["OgcRecord"], JsonSerializable): + +class LinksBuilder: + def __init__(self, themes: list[str]): + self.themes = themes + self.theme_links = [] + + def build_theme_links_for_records(self): + for theme in self.themes: + formatted_theme = theme.capitalize() + link = { + "rel": "related", + "href": f"../../themes/{theme}/catalog.json", + "type": "application/json", + "title": f"Theme: {formatted_theme}", + } + self.theme_links.append(link) + return self.theme_links + + @staticmethod + def build_link_to_dataset(collection_id): + return [ + { + "rel": "child", + "href": f"../../products/{collection_id}/collection.json", + "type": "application/json", + "title": f"{collection_id}", + } + ] + + +class WorkflowAsOgcRecord(MappingConstructible["OgcRecord"], JsonSerializable): def __init__( self, id: str, type: str, - time: dict, + title: str, + jupyter_notebook_url: str, properties: RecordProperties, links: list[dict], linkTemplates: list = [], conformsTo: list[str] = None, geometry: Optional[Any] = None, + themes: Optional[Any] = None, + ): + if conformsTo is None: + conformsTo = [OGC_API_RECORD_SPEC] + self.id = id + self.type = type + self.title = title + self.jupyter_notebook_url = jupyter_notebook_url + self.geometry = geometry + self.properties = properties + self.linkTemplates = linkTemplates + self.conformsTo = conformsTo + self.themes = themes + self.links = self._generate_static_links() + links + + def _generate_static_links(self): + """Generates static links (root and parent) for the record.""" + return [ + { + "rel": "root", + "href": "../../catalog.json", + "type": "application/json", + "title": "Open Science Catalog", + }, + { + "rel": "parent", + "href": "../catalog.json", + "type": "application/json", + "title": "Workflows", + }, + { + "rel": "child", + "href": f"../../experiments/{self.id}/record.json", + "type": "application/json", + "title": f"{self.title}", + }, + { + "rel": "jupyter-notebook", + "type": "application/json", + "title": "Jupyter Notebook", + "href": f"{self.jupyter_notebook_url}", + }, + { + "rel": "related", + "href": f"../../projects/{PROJECT_COLLECTION_NAME}/collection.json", + "type": "application/json", + "title": "Project: DeepESDL", + }, + { + "rel": "self", + "href": f"{BASE_URL_OSC}/workflows/{self.id}/record.json", + "type": "application/json", + }, + ] + + +class ExperimentAsOgcRecord(MappingConstructible["OgcRecord"], JsonSerializable): + def __init__( + self, + id: str, + title: str, + type: str, + jupyter_notebook_url: str, + collection_id: str, + properties: RecordProperties, + links: list[dict], + linkTemplates=None, + conformsTo: list[str] = None, + geometry: Optional[Any] = None, ): + if linkTemplates is None: + linkTemplates = [] if conformsTo is None: conformsTo = [OGC_API_RECORD_SPEC] self.id = id + self.title = title self.type = type self.conformsTo = conformsTo - self.time = time + self.jupyter_notebook_url = jupyter_notebook_url + self.collection_id = collection_id self.geometry = geometry self.properties = properties self.linkTemplates = linkTemplates - self.links = links + self.links = self._generate_static_links() + links + + def _generate_static_links(self): + """Generates static links (root and parent) for the record.""" + return [ + { + "rel": "root", + "href": "../../catalog.json", + "type": "application/json", + "title": "Open Science Catalog", + }, + { + "rel": "parent", + "href": "../catalog.json", + "type": "application/json", + "title": "Experiments", + }, + { + "rel": "related", + "href": f"../../workflows/{self.id}/record.json", + "type": "application/json", + "title": f"Workflow: {self.title}", + }, + { + "rel": "child", + "href": f"../../products/{self.collection_id}/collection.json", + "type": "application/json", + "title": f"{self.collection_id}", + }, + { + "rel": "related", + "href": f"../../projects/{PROJECT_COLLECTION_NAME}/collection.json", + "type": "application/json", + "title": "Project: DeepESDL", + }, + { + "rel": "input", + "href": "./input.yaml", + "type": "application/yaml", + "title": "Input parameters", + }, + { + "rel": "environment", + "href": "./environment.yaml", + "type": "application/yaml", + "title": "Execution environment", + }, + { + "rel": "self", + "href": f"{BASE_URL_OSC}/experiments/{self.id}/record.json", + "type": "application/json", + }, + ] diff --git a/deep_code/utils/ogc_record_generator.py b/deep_code/utils/ogc_record_generator.py index 481663f..83bd08a 100644 --- a/deep_code/utils/ogc_record_generator.py +++ b/deep_code/utils/ogc_record_generator.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone -from deep_code.constants import DEFAULT_THEME_SCHEME +from deep_code.constants import OSC_THEME_SCHEME from deep_code.utils.ogc_api_record import ( Contact, RecordProperties, @@ -37,18 +37,33 @@ def build_theme(osc_themes: list[str]) -> Theme: """Convert each string into a ThemeConcept """ concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] - return Theme(concepts=concepts, scheme=DEFAULT_THEME_SCHEME) + return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) - def build_record_properties(self, properties, contacts) -> RecordProperties: - """Build a RecordProperties object from a list of single-key property dicts + def build_record_properties( + self, properties: dict, contacts: list + ) -> RecordProperties: + """Build a RecordProperties object from a properties dictionary. + + Args: + properties: A dictionary containing properties (e.g., title, description, themes). + contacts: A list of contact dictionaries. + + Returns: + A RecordProperties object. """ now_iso = datetime.now(timezone.utc).isoformat() properties.update({"created": now_iso}) properties.update({"updated": now_iso}) + themes_list = properties.get("themes", []) + properties.update({"contacts": self.build_contact_objects(contacts)}) + if themes_list: theme_obj = self.build_theme(themes_list) properties.update({"themes": [theme_obj]}) + properties.setdefault("type", "workflow") + properties.setdefault("osc_project", "deep-earth-system-data-lab") + return RecordProperties.from_value(properties) diff --git a/deep_code/utils/osc_extension.py b/deep_code/utils/osc_extension.py index 8a777de..48b68e6 100644 --- a/deep_code/utils/osc_extension.py +++ b/deep_code/utils/osc_extension.py @@ -10,7 +10,7 @@ from pystac import Extent, SpatialExtent, TemporalExtent from pystac.extensions.base import ExtensionManagementMixin, PropertiesExtension -from deep_code.constants import CF_SCHEMA_URI, OSC_SCHEMA_URI +from deep_code.constants import CF_SCHEMA_URI, OSC_SCHEMA_URI, THEMES_SCHEMA_URI class OscExtension( @@ -63,18 +63,6 @@ def osc_project(self) -> str | None: def osc_project(self, v: str) -> None: self._set_property("osc:project", v, pop_if_none=False) - @property - def osc_themes(self) -> list[str] | None: - return self._get_property("osc:themes", list) - - @osc_themes.setter - def osc_themes(self, value: list[str]) -> None: - if not isinstance(value, list) or not all( - isinstance(item, str) for item in value - ): - raise ValueError("osc:themes must be a list of strings") - self._set_property("osc:themes", value, pop_if_none=False) - @property def osc_region(self) -> str | None: return self._get_property("osc:region", str) @@ -150,7 +138,7 @@ def updated(self, value: str) -> None: @classmethod def get_schema_uri(cls) -> list[str]: - return [OSC_SCHEMA_URI, CF_SCHEMA_URI] + return [OSC_SCHEMA_URI, CF_SCHEMA_URI, THEMES_SCHEMA_URI] @classmethod def ext( diff --git a/deep_code/version.py b/deep_code/version.py index e2b6e0b..30dc845 100644 --- a/deep_code/version.py +++ b/deep_code/version.py @@ -19,4 +19,4 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -version = "0.0.1.dev0" +version = "0.0.1.dev1"