diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index dd8ec867e9..789f1e5090 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,6 @@ { "name": "eclipse-s-core", "image": "ghcr.io/eclipse-score/devcontainer:v1.1.0", + "postCreateCommand": "bash .devcontainer/prepare_workspace.sh", "postStartCommand": "ssh-keygen -f '/home/vscode/.ssh/known_hosts' -R '[localhost]:2222' || true" } diff --git a/.devcontainer/prepare_workspace.sh b/.devcontainer/prepare_workspace.sh new file mode 100755 index 0000000000..2a2e5ffdef --- /dev/null +++ b/.devcontainer/prepare_workspace.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euo pipefail + +# Install pipx +sudo apt update +sudo apt install -y pipx + +# Install gita +pipx install gita + +# Enable bash autocompletion for gita +echo "eval \"\$(register-python-argcomplete gita -s bash)\"" >> ~/.bashrc + +# Set GITA_PROJECT_HOME environment variable +echo "export GITA_PROJECT_HOME=$(pwd)/.gita" >> ~/.bashrc +GITA_PROJECT_HOME=$(pwd)/.gita +mkdir -p "$GITA_PROJECT_HOME" +export GITA_PROJECT_HOME + +# Generate workspace metadata files from known_good.json: +# - .gita-workspace.csv +python3 tools/known_good_to_workspace_metadata.py --known-good known_good.json --gita-workspace .gita-workspace.csv diff --git a/.gitignore b/.gitignore index ca2081e493..fc5e8eb6a8 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ __pycache__/ /_build /docs/ubproject.toml /docs/_collections + +# Workspace files +/score_*/ +/.gita/ +/.gita-workspace.csv diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000000..5234f5b4f6 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,47 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Update workspace metadata from known good", + "type": "shell", + "command": "python3", + "args": [ + "tools/known_good_to_workspace_metadata.py" + ], + "problemMatcher": [] + }, + { + "label": "Switch Bazel modules to local_path_overrides", + "type": "shell", + "command": "python3", + "args": [ + "tools/update_module_from_known_good.py", + "--override-type", + "local_path" + ], + "problemMatcher": [] + }, + { + "label": "Switch Bazel modules to git_overrides", + "type": "shell", + "command": "python3", + "args": [ + "tools/update_module_from_known_good.py", + "--override-type", + "git" + ] + }, + { + "label": "Gita: Generate workspace", + "type": "shell", + "command": "gita", + "args": [ + "clone", + "--preserve-path", + "--from-file", + ".gita-workspace.csv" + ], + "problemMatcher": [] + } + ] +} diff --git a/README.md b/README.md index 07821af1a8..2946b1237e 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,47 @@ Execute `bazel query //feature_showcase/...` to obtain list of targets that You bazel build --config bl-x86_64-linux @score_orchestrator//src/... --verbose_failures ``` +## Workspace support + +You can obtain a complete S-CORE workspace, i.e. a git checkout of all modules from `known_good.json`, on the specific branches / commits, integrated into one Bazel build. +This helps with cross-module development, debugging, and generally "trying out things". + +> [!NOTE] +> The startup of the [S-CORE devcontainer](https://github.com/eclipse-score/devcontainer) [integrated in this repository](.devcontainer/) already installs supported workspace managers and generates the required metadata. +> You can do this manually as well, of course (e.g. if you do not use the devcontainer). +> Take a look at `.devcontainer/prepare_workspace.sh`, which contains the setup script. + +> [!NOTE] +> Not all Bazel targets are supported yet. +> Running `./scripts/integration_test.sh` will work, though. +> Take a look at the [Known Issues](#known-issues-️) below to see which Bazel targets are available and working. + +The supported workspace managers are: + +| Name | Description | +|------|-------------| +| [Gita](https://github.com/nosarthur/gita) | "a command-line tool to manage multiple git repos" | + +A description of how to use these workspace managers, together with their advantages and drawbacks, is beyond the scope of this document. +In case of doubt, choose the first. + +### Initialization of the workspace + +> [!WARNING] +> This will change the file `score_modules.MODULE.bazel`. +> Do **not** commit these changes! + +1. Switch to local path overrides, using the VSCode Task (`Terminal`->`Run Task...`) "Switch Bazel modules to `local_path_overrides`". + Note that you can switch back to `git_overrides` (the default) using the task "Switch Bazel modules to `git_overrides`" + +2. Run VSCode Task "<Name>: Generate workspace", e.g. "Gita: Generate workspace". + This will clone all modules using the chosen workspace manager. + The modules will be in sub-directories starting with `score_`. + Note that the usage of different workspace managers is mutually exclusive. + +When you now run Bazel, it will use the local working copies of all modules and not download them from git remotes. +You can make local changes to each module, which will be directly reflected in the next Bazel run. + ## Known Issues ⚠️ ### Orchestrator diff --git a/tools/get_module_info.py b/tools/get_module_info.py old mode 100644 new mode 100755 diff --git a/tools/known_good_to_workspace_metadata.py b/tools/known_good_to_workspace_metadata.py new file mode 100644 index 0000000000..145c17614e --- /dev/null +++ b/tools/known_good_to_workspace_metadata.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +import argparse +import json +import csv + +MODULES_CSV_HEADER = [ + "repo_url", + "name", + "workspace_path", + "version", + "hash", + "branch" +] + +def main(): + parser = argparse.ArgumentParser(description="Convert known_good.json to workspace metadata files for gita and git submodules.") + + parser.add_argument("--known-good", dest="known_good", default="known_good.json", help="Path to known_good.json") + parser.add_argument("--gita-workspace", dest="gita_workspace", default=".gita-workspace.csv", help="File to output gita workspace metadata") + args = parser.parse_args() + + with open(args.known_good, "r") as f: + data = json.load(f) + + modules = data.get("modules", {}) + + gita_metadata = [] + for name, info in modules.items(): + repo_url = info.get("repo", "") + if not repo_url: + raise RuntimeError("repo must not be empty") + + # default branch: main + branch = info.get("branch", "main") + + # if no hash is given, use branch + hash_ = info.get("hash", branch) + + # workspace_path is not available in known_good.json, default to name of repository + workspace_path = name + + # gita format: {url},{name},{path},{prop['type']},{repo_flags},{branch} + row = [repo_url, name, workspace_path, "", "", hash_] + gita_metadata.append(row) + + with open(args.gita_workspace, "w", newline="") as f: + writer = csv.writer(f) + for row in gita_metadata: + writer.writerow(row) + +if __name__ == "__main__": + main() diff --git a/tools/update_module_from_known_good.py b/tools/update_module_from_known_good.py old mode 100644 new mode 100755 index 67820ecf0b..138590bfa9 --- a/tools/update_module_from_known_good.py +++ b/tools/update_module_from_known_good.py @@ -90,8 +90,24 @@ def generate_git_override_blocks(modules_dict: Dict[str, Any], repo_commit_dict: return blocks +def generate_local_override_blocks(modules_dict: Dict[str, Any]) -> List[str]: + """Generate bazel_dep and local_path_override blocks for each module.""" + blocks = [] + + for name, module in modules_dict.items(): + block = ( + f'bazel_dep(name = "{name}")\n' + 'local_path_override(\n' + f' module_name = "{name}",\n' + f' path = "{name}",\n' + ')\n' + ) + + blocks.append(block) + + return blocks -def generate_file_content(modules: Dict[str, Any], repo_commit_dict: Dict[str, str], timestamp: Optional[str] = None) -> str: +def generate_file_content(args: argparse.Namespace, modules: Dict[str, Any], repo_commit_dict: Dict[str, str], timestamp: Optional[str] = None) -> str: """Generate the complete content for score_modules.MODULE.bazel.""" # License header assembled with parenthesis grouping (no indentation preserved in output). header = ( @@ -117,7 +133,15 @@ def generate_file_content(modules: Dict[str, Any], repo_commit_dict: Dict[str, s "\n" ) - blocks = generate_git_override_blocks(modules, repo_commit_dict) + if args.override_type == "git": + blocks = generate_git_override_blocks(modules, repo_commit_dict) + else: + header += ( + "# Note: This file uses local_path overrides. Ensure that local paths are set up correctly.\n" + "\n" + ) + blocks = generate_local_override_blocks(modules) + if not blocks: raise SystemExit("No valid modules to generate git_override blocks") @@ -149,6 +173,12 @@ def main() -> None: action="append", help="Override commit for a specific repo (format: @)" ) + parser.add_argument( + "--override-type", + choices=["local_path", "git"], + default="git", + help="Type of override to use (default: git)" + ) args = parser.parse_args() @@ -180,7 +210,7 @@ def main() -> None: # Generate file content timestamp = data.get("timestamp") or datetime.now().isoformat() - content = generate_file_content(modules, repo_commit_dict, timestamp) + content = generate_file_content(args, modules, repo_commit_dict, timestamp) if args.dry_run: print(f"Dry run: would write to {output_path}\n") @@ -191,7 +221,7 @@ def main() -> None: else: with open(output_path, "w", encoding="utf-8") as f: f.write(content) - print(f"Generated {output_path} with {len(modules)} git_override entries") + print(f"Generated {output_path} with {len(modules)} {args.override_type}_override entries") if __name__ == "__main__": diff --git a/tools/update_module_latest.py b/tools/update_module_latest.py old mode 100644 new mode 100755 index 77fdee311b..210f847c46 --- a/tools/update_module_latest.py +++ b/tools/update_module_latest.py @@ -119,7 +119,7 @@ def load_known_good(path: str) -> dict: def write_known_good(path: str, original: dict, modules: list[Module]) -> None: out = dict(original) # shallow copy - out["timestamp"] = dt.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + out["timestamp"] = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat() + "Z" out["modules"] = {} for m in modules: mod_dict = {"repo": m.repo, "hash": m.hash}