diff --git a/deep_code/tests/tools/test_new.py b/deep_code/tests/tools/test_new.py new file mode 100644 index 0000000..2065443 --- /dev/null +++ b/deep_code/tests/tools/test_new.py @@ -0,0 +1,59 @@ +import unittest +import shutil +from pathlib import Path +from unittest.mock import patch, MagicMock + +from deep_code.tools.new import RepositoryInitializer + + +class TestRepositoryInitializer(unittest.TestCase): + def setUp(self): + """Set up a temporary directory for testing.""" + self.test_repo_name = "test_repo" + self.test_repo_dir = Path(self.test_repo_name).absolute() + self.templates_dir = Path(__file__).parent.parent / "deep_code" / "templates" + + # Mock GitHub credentials + self.github_username = "test_user" + self.github_token = "test_token" + + # Ensure the repository directory does not exist before each test + if self.test_repo_dir.exists(): + shutil.rmtree(self.test_repo_dir) + + def tearDown(self): + """Clean up the temporary directory after each test.""" + if self.test_repo_dir.exists(): + shutil.rmtree(self.test_repo_dir) + + @patch("deep_code.tools.new.read_git_access_file") + def test_missing_github_credentials(self, mock_read_git_access_file): + """Test that an error is raised if GitHub credentials are missing.""" + # Mock the Git access file with missing credentials + mock_read_git_access_file.return_value = {} + + # Verify that an error is raised + with self.assertRaises(ValueError) as context: + RepositoryInitializer(self.test_repo_name) + self.assertIn("GitHub credentials are missing", str(context.exception)) + + @patch("deep_code.tools.new.read_git_access_file") + def test_template_not_found(self, mock_read_git_access_file): + """Test that an error is raised if a template file is missing.""" + # Mock the Git access file + mock_read_git_access_file.return_value = { + "github-username": self.github_username, + "github-token": self.github_token, + } + + # Initialize the repository with a non-existent template + initializer = RepositoryInitializer(self.test_repo_name) + initializer.templates_dir = Path("/non/existent/path") + + # Verify that an error is raised + with self.assertRaises(FileNotFoundError): + initializer.initialize() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 3d1ed1e..3d5ebe0 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -1,5 +1,144 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2025 by Brockmann Consult GmbH +# Permissions are hereby granted under the terms of the MIT License: +# https://opensource.org/licenses/MIT. + """Logic for initializing repositories Initialize a GitHub repository with the proposed configurations files, an initial workflow notebook template (e.g. workflow.ipynb), a template Python package (code and pyproject.toml), and a template setup for documentation (e.g., using mkdocs), -setup of thebuild pipeline""" +setup of the build pipeline""" + +import subprocess +from pathlib import Path +import logging + +import requests + +from deep_code.utils.helper import read_git_access_file + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class RepositoryInitializer: + """ + A utility class to initialize a GitHub repository with configuration files, + a workflow notebook template, and a Python package template for DeepESDL experiment + """ + + def __init__(self, repo_name: str): + """ + Initialize the RepositoryInitializer. + """ + self.repo_name = repo_name + self.repo_dir = Path(repo_name).absolute() + self.templates_dir = Path(__file__).parent / "templates" + git_config = read_git_access_file() + self.github_username = git_config.get("github-username") + self.github_token = git_config.get("github-token") + if not self.github_username or not self.github_token: + raise ValueError("GitHub credentials are missing in the `.gitaccess` file.") + + def create_local_repo(self) -> None: + """Create a local directory for the repository and initialize it as a Git repository.""" + logger.info(f"Creating local repository: {self.repo_dir}") + self.repo_dir.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "init"], cwd=self.repo_dir, check=True) + + def _load_template(self, template_name: str) -> str: + """Load a template file from the templates directory.""" + template_path = self.templates_dir / template_name + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + with open(template_path, "r") as file: + return file.read() + + def create_config_files(self) -> None: + """Create configuration files in the repository.""" + logger.info("Creating configuration files...") + + # Create README.md + readme_content = (f"# {self.repo_name}\n\nThis repository contains workflows " + f"and Python code for a DeepESDL Experiment.") + (self.repo_dir / "README.md").write_text(readme_content) + + # Create .gitignore + gitignore_content = self._load_template(".gitignore") + (self.repo_dir / ".gitignore").write_text(gitignore_content) + + def create_jupyter_notebook_template(self) -> None: + """Create a workflow notebook template (workflow.ipynb).""" + logger.info("Creating workflow notebook template...") + workflow_content = self._load_template("workflow.ipynb") + (self.repo_dir / "workflow.ipynb").write_text(workflow_content) + + def create_python_package(self) -> None: + """Create a Python package template with a pyproject.toml file.""" + logger.info("Creating Python package template...") + + # Create package directory + package_dir = self.repo_dir / self.repo_name + package_dir.mkdir(exist_ok=True) + + # Create __init__.py + (package_dir / "__init__.py").write_text("# Package initialization\n") + + # Create pyproject.toml + pyproject_content = self._load_template("pyproject.toml") + pyproject_content = pyproject_content.replace("{repo_name}", self.repo_name) + (self.repo_dir / "pyproject.toml").write_text(pyproject_content) + + def create_github_repo(self) -> None: + """Create a remote GitHub repository and push the local repository.""" + if not self.github_username or not self.github_token: + logger.warning("GitHub credentials not provided. Skipping remote repository creation.") + return + + logger.info("Creating remote GitHub repository...") + + repo_url = "https://api.github.com/user/repos" + repo_data = { + "name": self.repo_name, + "description": "Repository for DeepESDL workflows and Python code.", + "private": True, + } + headers = { + "Authorization": f"token {self.github_token}", + "Accept": "application/vnd.github.v3+json", + } + response = requests.post(repo_url, json=repo_data, headers=headers) + response.raise_for_status() + + remote_url = f"https://github.com/{self.github_username}/{self.repo_name}.git" + subprocess.run(["git", "remote", "add", "origin", remote_url], cwd=self.repo_dir, check=True) + subprocess.run(["git", "add", "."], cwd=self.repo_dir, check=True) + subprocess.run(["git", "commit", "-m", "Initial commit"], cwd=self.repo_dir, check=True) + subprocess.run(["git", "push", "-u", "origin", "main"], cwd=self.repo_dir, check=True) + + logger.info(f"Remote repository created: {remote_url}") + + def create_github_actions_workflow(self) -> None: + """Create a GitHub Actions workflow for running unit tests.""" + logger.info("Creating GitHub Actions workflow...") + + workflows_dir = self.repo_dir / ".github" / "workflows" + workflows_dir.mkdir(parents=True, exist_ok=True) + + workflow_content = self._load_template("unit-tests.yml") + (workflows_dir / "unit-tests.yml").write_text(workflow_content) + + def initialize(self) -> None: + """Initialize the repository with all templates and configurations.""" + self.create_local_repo() + self.create_config_files() + self.create_jupyter_notebook_template() + self.create_python_package() + self.create_github_repo() + logger.info(f"Repository '{self.repo_name}' initialized successfully!") + + +if __name__ == '__main__': + r = RepositoryInitializer("deepesdl-test-experiment") + r.initialize() \ No newline at end of file diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index c17264f..74e1769 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -22,7 +22,7 @@ ) from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator from deep_code.utils.github_automation import GitHubAutomation -from deep_code.utils.helper import serialize +from deep_code.utils.helper import serialize, read_git_access_file from deep_code.utils.ogc_api_record import ( ExperimentAsOgcRecord, LinksBuilder, @@ -42,8 +42,7 @@ class GitHubPublisher: """ def __init__(self): - with fsspec.open(".gitaccess", "r") as file: - git_config = yaml.safe_load(file) or {} + git_config = read_git_access_file() self.github_username = git_config.get("github-username") self.github_token = git_config.get("github-token") if not self.github_username or not self.github_token: diff --git a/deep_code/tools/templates/.gitignore b/deep_code/tools/templates/.gitignore new file mode 100644 index 0000000..605f546 --- /dev/null +++ b/deep_code/tools/templates/.gitignore @@ -0,0 +1,14 @@ +# Ignore Python compiled files +__pycache__/ +*.pyc +*.pyo +*.pyd + +# Ignore Jupyter notebook checkpoints +.ipynb_checkpoints/ + +# Ignore virtual environments +venv/ +env/ + +.gitaccess \ No newline at end of file diff --git a/deep_code/tools/templates/pyproject.toml b/deep_code/tools/templates/pyproject.toml new file mode 100644 index 0000000..83648af --- /dev/null +++ b/deep_code/tools/templates/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ['setuptools>=61.2.0', 'wheel', 'build'] +build-backend = 'setuptools.build_meta' + +[project] +name = '{repo_name}' +version = '0.1.0' +description = 'A Python package for DeepESDL workflows.' +authors = [ + {name = 'Your Name', email = 'your.email@example.com'} +] +dependencies = [ + 'numpy', + 'pandas', +] \ No newline at end of file diff --git a/deep_code/tools/templates/workflow-template.yml b/deep_code/tools/templates/workflow-template.yml new file mode 100644 index 0000000..4f0af0a --- /dev/null +++ b/deep_code/tools/templates/workflow-template.yml @@ -0,0 +1,31 @@ +name: Unit Tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev] + + - name: Run unit tests + run: | + pytest tests/ \ No newline at end of file diff --git a/deep_code/tools/templates/workflow.ipynb b/deep_code/tools/templates/workflow.ipynb new file mode 100644 index 0000000..feb7b8a --- /dev/null +++ b/deep_code/tools/templates/workflow.ipynb @@ -0,0 +1,47 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflow Notebook\n\n", + "This notebook provides a template for running DeepESDL workflows.\n", + "Modify this notebook to implement your specific workflow." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import required libraries\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "# Add your workflow code here\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index cca6b75..35d7f33 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -1,3 +1,7 @@ +import fsspec +import yaml + + def serialize(obj): """Convert non-serializable objects to JSON-compatible formats. Args: @@ -12,3 +16,7 @@ def serialize(obj): if hasattr(obj, "__dict__"): return obj.__dict__ raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") + +def read_git_access_file(): + with fsspec.open(".gitaccess", "r") as file: + return yaml.safe_load(file) or {} \ No newline at end of file