From 4b1282ac3ebc4fd29e8b27c8f791e39fb6211f89 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 9 Apr 2025 10:51:49 +0200 Subject: [PATCH 1/4] support publish to testing and staging osc forks --- deep_code/cli/publish.py | 13 +++++++++++-- deep_code/tools/publish.py | 25 +++++++++++++++++++++---- deep_code/version.py | 2 +- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/deep_code/cli/publish.py b/deep_code/cli/publish.py index 47e34fb..11e4f91 100644 --- a/deep_code/cli/publish.py +++ b/deep_code/cli/publish.py @@ -12,10 +12,19 @@ @click.command(name="publish") @click.argument("dataset_config", type=click.Path(exists=True)) @click.argument("workflow_config", type=click.Path(exists=True)) -def publish(dataset_config, workflow_config): +@click.option( + "--environment", + type=click.Choice(["production", "staging", "testing"], + case_sensitive=False), + default="production", + help="Target environment for publishing (production, staging, testing)", +) +def publish(dataset_config, workflow_config, environment): """Request publishing a dataset to the open science catalogue. """ publisher = Publisher( - dataset_config_path=dataset_config, workflow_config_path=workflow_config + dataset_config_path=dataset_config, + workflow_config_path=workflow_config, + environment=environment.lower(), ) publisher.publish_all() diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index c17264f..22cc5c5 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -41,7 +41,7 @@ class GitHubPublisher: - Common GitHub automation steps (fork, clone, branch, file commit, pull request) """ - def __init__(self): + def __init__(self, repo_name: str = OSC_REPO_NAME): with fsspec.open(".gitaccess", "r") as file: git_config = yaml.safe_load(file) or {} self.github_username = git_config.get("github-username") @@ -50,7 +50,10 @@ def __init__(self): raise ValueError("GitHub credentials are missing in the `.gitaccess` file.") self.github_automation = GitHubAutomation( - self.github_username, self.github_token, OSC_REPO_OWNER, OSC_REPO_NAME + self.github_username, + self.github_token, + OSC_REPO_OWNER, + repo_name ) self.github_automation.fork_repository() self.github_automation.clone_sync_repository() @@ -102,9 +105,23 @@ class Publisher: """Publishes products (datasets) to the OSC GitHub repository. """ - def __init__(self, dataset_config_path: str, workflow_config_path: str): + def __init__( + self, + dataset_config_path: str, + workflow_config_path: str, + environment: str = "production", + ): + self.environment = environment + # Determine repo name based on environment + repo_name = "open-science-catalog-metadata" + + if environment == "staging": + repo_name = "open-science-catalog-metadata-staging" + elif environment == "testing": + repo_name = "open-science-catalog-metadata-testing" + # Composition - self.gh_publisher = GitHubPublisher() + self.gh_publisher = GitHubPublisher(repo_name=repo_name) self.collection_id = "" self.workflow_title = "" diff --git a/deep_code/version.py b/deep_code/version.py index 3104c7c..f3eb35c 100644 --- a/deep_code/version.py +++ b/deep_code/version.py @@ -19,4 +19,4 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -version = "0.1.1" +version = "0.1.2.dev0" From 143e6bde857316f7cf6d222794a9f55287fe4616 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 9 Apr 2025 10:53:17 +0200 Subject: [PATCH 2/4] black and ruff formatted --- deep_code/cli/publish.py | 3 +-- deep_code/tests/tools/test_publish.py | 9 +++++---- deep_code/tools/new.py | 2 -- deep_code/tools/publish.py | 5 +---- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/deep_code/cli/publish.py b/deep_code/cli/publish.py index 11e4f91..b1cef4e 100644 --- a/deep_code/cli/publish.py +++ b/deep_code/cli/publish.py @@ -14,8 +14,7 @@ @click.argument("workflow_config", type=click.Path(exists=True)) @click.option( "--environment", - type=click.Choice(["production", "staging", "testing"], - case_sensitive=False), + type=click.Choice(["production", "staging", "testing"], case_sensitive=False), default="production", help="Target environment for publishing (production, staging, testing)", ) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index f355cb5..f62db2c 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -1,9 +1,10 @@ -import unittest -from unittest.mock import patch, mock_open, MagicMock import json -import yaml -from pathlib import Path import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, mock_open, patch + +import yaml from pystac import Catalog from deep_code.tools.publish import Publisher diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index f54d0b2..89e62c5 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -9,5 +9,3 @@ workflow notebook template (e.g. workflow.ipynb), a template Python package (code and pyproject.toml), and a template setup for documentation (e.g., using mkdocs), setup of the build pipeline""" - - diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 22cc5c5..9918847 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -50,10 +50,7 @@ def __init__(self, repo_name: str = OSC_REPO_NAME): raise ValueError("GitHub credentials are missing in the `.gitaccess` file.") self.github_automation = GitHubAutomation( - self.github_username, - self.github_token, - OSC_REPO_OWNER, - repo_name + self.github_username, self.github_token, OSC_REPO_OWNER, repo_name ) self.github_automation.fork_repository() self.github_automation.clone_sync_repository() From e114db9a344436a6c120705711d41c118a79f773 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 9 Apr 2025 11:48:45 +0200 Subject: [PATCH 3/4] updated README.md --- README.md | 36 +++++++++++++++++++++++++++++++----- deep_code/cli/publish.py | 3 ++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c1ab255..1661314 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ experiments/workflow as OGC API record and Datasets as an OSC stac collection. ## Setup ## Install -`deep-code` will be available in PyPI and conda-forge. Till the stable release, +`deep-code` will be available in PyPI for now and will be available in conda-forge +in the near future. Till the stable release, developers/contributors can follow the below steps to install deep-code. ## Installing from the repository for Developers/Contributors @@ -69,13 +70,38 @@ github-token: personal access token ### deep-code publish -Publish the experiment, workflow and dataset which is a result of an experiment to -the EarthCODE open-science catalog. +Publishes metadata of experiment, workflow and dataset to the EarthCODE open-science +catalog -```commandline - deep-code publish /path/to/dataset-config.yaml /path/to/workflow-config.yaml +### Usage +``` +deep-code publish DATASET_CONFIG WORKFLOW_CONFIG [--environment ENVIRONMENT] ``` +#### Arguments + DATASET_CONFIG - Path to the dataset configuration YAML file + (e.g., dataset-config.yaml) + + WORKFLOW_CONFIG - Path to the workflow configuration YAML file + (e.g., workflow-config.yaml) + +#### Options + --environment, -e - Target catalog environment: + production (default) | staging | testing + +#### Examples: +1. Publish to staging catalog +``` +deep-code publish dataset-config.yaml workflow-config.yaml --environment=staging +``` +2. Publish to testing catalog +``` +deep-code publish dataset-config.yaml workflow-config.yaml -e testing +``` +3. Publish to production catalog +``` +deep-code publish dataset-config.yaml workflow-config.yaml +``` #### dataset-config.yaml example ``` diff --git a/deep_code/cli/publish.py b/deep_code/cli/publish.py index b1cef4e..935c168 100644 --- a/deep_code/cli/publish.py +++ b/deep_code/cli/publish.py @@ -19,7 +19,8 @@ help="Target environment for publishing (production, staging, testing)", ) def publish(dataset_config, workflow_config, environment): - """Request publishing a dataset to the open science catalogue. + """Request publishing a dataset along with experiment and workflow metadata to the + open science catalogue. """ publisher = Publisher( dataset_config_path=dataset_config, From dac6ea70284f6fb7c850aa71a163b5efacb246c0 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 9 Apr 2025 12:16:38 +0200 Subject: [PATCH 4/4] updated cli publish to include shorthand param -e --- deep_code/cli/publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deep_code/cli/publish.py b/deep_code/cli/publish.py index 935c168..39db233 100644 --- a/deep_code/cli/publish.py +++ b/deep_code/cli/publish.py @@ -13,7 +13,7 @@ @click.argument("dataset_config", type=click.Path(exists=True)) @click.argument("workflow_config", type=click.Path(exists=True)) @click.option( - "--environment", + "--environment", "-e", type=click.Choice(["production", "staging", "testing"], case_sensitive=False), default="production", help="Target environment for publishing (production, staging, testing)",