Implement publish product feature

.github/workflows/unittest-workflow.yaml

@@ -0,0 +1,36 @@
+name: Unittest deep-code
+
+on:
+  push:
+  release:
+    types: [published]
+
+jobs:
+  unittest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout deep-code
+        uses: actions/checkout@v4
+
+      - name: Set up MicroMamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+
+      - name: Install deep-code in editable mode
+        shell: bash -l {0}
+        run: |
+          cd /home/runner/work/deep-code/deep-code
+          pip install -e .
+
+      - name: Run unit tests
+        shell: bash -l {0}
+        run: |
+          cd /home/runner/work/deep-code/deep-code
+          pytest --cov=deep_code --cov-report=xml
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: deepesdl/deep-code

.gitignore

@@ -160,3 +160,7 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Exclude sensitive configuration files from version control
+.gitaccess
+dataset-config.yaml

README.md

@@ -1 +1,87 @@
-# deep-code
+# deep-code
+
+[![Build Status](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml/badge.svg)](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml)
+[![codecov](https://codecov.io/gh/deepesdl/deep-code/graph/badge.svg?token=47MQXOXWOK)](https://codecov.io/gh/deepesdl/deep-code)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![License](https://img.shields.io/github/license/dcs4cop/xcube-smos)](https://github.com/deepesdl/deep-code/blob/main/LICENSE)
+
+`deep-code` is a lightweight python tool that comprises a command line interface(CLI) 
+and Python API providing utilities that aid integration of DeepESDL datasets, 
+experiments with EarthCODE.
+
+## Setup
+
+## Install
+`deep-code` will be available in PyPI and conda-forge. Till the stable release,
+developers/contributors can follow the below steps to install deep-code.
+
+## Installing from the repository for Developer
+
+To install deep-code directly from the git repository, clone the repository, and execute the steps below:
+
+```commandline
+conda env create
+conda activate deep-code
+pip install -e .
+```
+
+This installs all the dependencies of `deep-code` into a fresh conda environment, 
+and installs deep-code from the repository into the same environment.
+
+## Testing
+
+To run the unit test suite:
+
+```commandline
+pytest
+```
+
+To analyze test coverage
+```shell
+pytest --cov=deep-code
+```
+
+To produce an HTML coverage report
+
+```commandline
+pytest --cov-report html --cov=deep-code
+```
+
+## deep_code usage
+
+`deep_code` provides a command-line tool called deep-code, which has several subcommands 
+providing different utility functions.
+Use the --help option with these subcommands to get more details on usage.
+
+###  deep-code publish-product
+
+Publish a dataset which is a result of an experiment to the EarthCODE 
+open-science catalog.
+
+```commandline
+ deep-code publish-dataset /path/to/dataset-config.yaml
+ ```
+
+#### .gitaccess example
+
+```
+github-username: your-git-user
+github-token: personal access token
+```
+
+#### dataset-config.yaml example
+
+```
+dataset-id: hydrology-1D-0.009deg-100x60x60-3.0.2.zarr 
+collection-id: hydrology
+
+#non-mandatory
+documentation-link: https://deepesdl.readthedocs.io/en/latest/datasets/hydrology-1D-0-009deg-100x60x60-3-0-2-zarr/
+access-link: s3://test
+dataset-status: completed
+dataset-region: global
+dataset-theme: ["ocean", "environment"]
+cf-parameter: [{"Name" : "hydrology"}]
+```
+
+dataset-id has to be a valid dataset-id from `deep-esdl-public` s3 or your team bucket.

deep_code/__init__.py

@@ -1,5 +1,5 @@
 # The MIT License (MIT)
-# Copyright (c) 2024 by the xcube development team and contributors
+# Copyright (c) 2024 by DeepESDL and Brockmann Consult GmbH
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -21,4 +21,4 @@
 
 from .version import version
 
-__version__ = version
+__version__ = version

deep_code/cli/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.

deep_code/cli/main.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+import click
+
+from deep_code.cli.publish import publish_dataset
+
+
+@click.group()
+def main():
+    """Deep Code CLI."""
+    pass
+
+
+main.add_command(publish_dataset)
+if __name__ == "__main__":
+    main()

deep_code/cli/publish.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+import click
+
+from deep_code.tools.publish import DatasetPublisher
+
+
+@click.command(name="publish-dataset")
+@click.argument(
+    "dataset_config",
+    type=click.Path(exists=True)
+)
+def publish_dataset(dataset_config):
+    """Request publishing a dataset to the open science catalogue.
+    """
+    publisher = DatasetPublisher()
+    publisher.publish_dataset(dataset_config_path=dataset_config)

deep_code/constants.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+OSC_SCHEMA_URI = "https://stac-extensions.github.io/osc/v1.0.0-rc.3/schema.json"
+CF_SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json"
+OSC_REPO_OWNER = "ESA-EarthCODE"
+OSC_REPO_NAME = "open-science-catalog-metadata-testing"
+OSC_BRANCH_NAME = "add-new-collection"

deep_code/tests/tools/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.

deep_code/tests/tools/test_publish.py

@@ -0,0 +1,122 @@
+import pytest
+from unittest.mock import patch, MagicMock, mock_open
+
+from deep_code.tools.publish import DatasetPublisher
+
+
+class TestDatasetPublisher:
+    @patch("deep_code.tools.publish.fsspec.open")
+    def test_init_missing_credentials(self, mock_fsspec_open):
+        mock_fsspec_open.return_value.__enter__.return_value = mock_open(
+            read_data="{}"
+        )()
+
+        with pytest.raises(
+            ValueError, match="GitHub credentials are missing in the `.gitaccess` file."
+        ):
+            DatasetPublisher()
+
+    @patch("deep_code.tools.publish.fsspec.open")
+    def test_publish_dataset_missing_ids(self, mock_fsspec_open):
+        git_yaml_content = """
+        github-username: test-user
+        github-token: test-token
+        """
+        dataset_yaml_content = """
+        collection-id: test-collection
+        """
+        mock_fsspec_open.side_effect = [
+            mock_open(read_data=git_yaml_content)(),
+            mock_open(read_data=dataset_yaml_content)(),
+        ]
+
+        publisher = DatasetPublisher()
+
+        with pytest.raises(
+            ValueError,
+            match="Dataset ID or Collection ID is missing in the "
+            "dataset-config.yaml file.",
+        ):
+            publisher.publish_dataset("/path/to/dataset-config.yaml")
+
+    @patch("deep_code.utils.github_automation.os.chdir")
+    @patch("deep_code.utils.github_automation.subprocess.run")
+    @patch("deep_code.utils.github_automation.os.path.expanduser", return_value="/tmp")
+    @patch("requests.post")
+    @patch("deep_code.utils.github_automation.GitHubAutomation")
+    @patch("deep_code.tools.publish.fsspec.open")
+    def test_publish_dataset_success(
+        self,
+        mock_fsspec_open,
+        mock_github_automation,
+        mock_requests_post,
+        mock_expanduser,
+        mock_subprocess_run,
+        mock_chdir,
+    ):
+
+        #  Mock the YAML reads
+        git_yaml_content = """
+          github-username: test-user
+          github-token: test-token
+          """
+        dataset_yaml_content = """
+          dataset-id: test-dataset
+          collection-id: test-collection
+          documentation-link: http://example.com/doc
+          access-link: http://example.com/access
+          dataset-status: ongoing
+          dataset-region: Global
+          dataset-theme: ["climate"]
+          cf-parameter: []
+          """
+        mock_fsspec_open.side_effect = [
+            mock_open(read_data=git_yaml_content)(),
+            mock_open(read_data=dataset_yaml_content)(),
+        ]
+
+        # Mock GitHubAutomation methods
+        mock_git = mock_github_automation.return_value
+        mock_git.fork_repository.return_value = None
+        mock_git.clone_repository.return_value = None
+        mock_git.create_branch.return_value = None
+        mock_git.add_file.return_value = None
+        mock_git.commit_and_push.return_value = None
+        mock_git.create_pull_request.return_value = "http://example.com/pr"
+        mock_git.clean_up.return_value = None
+
+        # Mock subprocess.run & os.chdir
+        mock_subprocess_run.return_value = None
+        mock_chdir.return_value = None
+
+        # Mock STAC generator
+        mock_collection = MagicMock()
+        mock_collection.to_dict.return_value = {
+            "type": "Collection",
+            "id": "test-collection",
+            "description": "A test STAC collection",
+            "extent": {
+                "spatial": {"bbox": [[-180.0, -90.0, 180.0, 90.0]]},
+                "temporal": {"interval": [["2023-01-01T00:00:00Z", None]]},
+            },
+            "links": [],
+            "stac_version": "1.0.0",
+        }
+        with patch("deep_code.tools.publish.OSCProductSTACGenerator") as mock_generator:
+            mock_generator.return_value.build_stac_collection.return_value = (
+                mock_collection
+            )
+
+            # Instantiate & publish
+            publisher = DatasetPublisher()
+            publisher.publish_dataset("/fake/path/to/dataset-config.yaml")
+
+        # 6Assert that we called git clone with /tmp/temp_repo
+        # Because expanduser("~") is now patched to /tmp, the actual path is /tmp/temp_repo
+        auth_url = "https://test-user:test-token@github.com/test-user/open-science-catalog-metadata-testing.git"
+        mock_subprocess_run.assert_any_call(
+            ["git", "clone", auth_url, "/tmp/temp_repo"], check=True
+        )
+
+        # Also confirm we changed directories to /tmp/temp_repo
+        mock_chdir.assert_any_call("/tmp/temp_repo")

deep_code/tests/utils/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.