From 83e658e402de4c472f79dd40231f50d6480db750 Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Thu, 4 Dec 2025 11:35:49 +0000 Subject: [PATCH 1/8] Generate Workspace using Gita This change adds the capability to generate a workspace directly from the known_good.json file. To achieve this, the known_good.json file is transformed into a configuration file for a git workspace management tool called gita, which is then used to check out all mentioned modules on the specific commit/branch that is given in known_good.json. In addition, the score_modules.MODULE.bazel is updated with local_path_overrides to the checked out modules. Hence, a bazel build will immediately work as expected. # Conflicts: # .gitignore # Conflicts: # .devcontainer/devcontainer.json --- .devcontainer/devcontainer.json | 1 + .devcontainer/prepare_workspace.sh | 29 +++++++++++++++ .gitignore | 5 +++ tools/known_good_to_gita_workspace.py | 50 ++++++++++++++++++++++++++ tools/update_module_from_known_good.py | 38 +++++++++++++++++--- 5 files changed, 119 insertions(+), 4 deletions(-) create mode 100755 .devcontainer/prepare_workspace.sh create mode 100644 tools/known_good_to_gita_workspace.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index dd8ec867e9..789f1e5090 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,6 @@ { "name": "eclipse-s-core", "image": "ghcr.io/eclipse-score/devcontainer:v1.1.0", + "postCreateCommand": "bash .devcontainer/prepare_workspace.sh", "postStartCommand": "ssh-keygen -f '/home/vscode/.ssh/known_hosts' -R '[localhost]:2222' || true" } diff --git a/.devcontainer/prepare_workspace.sh b/.devcontainer/prepare_workspace.sh new file mode 100755 index 0000000000..ba2aa6f6fa --- /dev/null +++ b/.devcontainer/prepare_workspace.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +# Install pipx +sudo apt update +sudo apt install -y pipx + +# Install gita +pipx install gita + +# Enable bash autocompletion for gita +echo "eval \"\$(register-python-argcomplete gita -s bash)\"" >> ~/.bashrc + +# Set GITA_PROJECT_HOME environment variable +echo "export GITA_PROJECT_HOME=$(pwd)/.gita" >> ~/.bashrc +GITA_PROJECT_HOME=$(pwd)/.gita +mkdir -p "$GITA_PROJECT_HOME" +export GITA_PROJECT_HOME + +# Generate .gita-workspace.csv from known_good.json +python3 tools/known_good_to_gita_workspace.py known_good.json .gita-workspace.csv +python3 tools/update_module_from_known_good.py --override-type local_path + +# Automagically clone repositories listed in .gita-workspace.csv +echo "Cloning repositories listed in .gita-workspace.csv..." +gita clone --preserve-path --from-file .gita-workspace.csv +# Output the status of all cloned repositories +echo "Status of all cloned repositories:" +gita ll diff --git a/.gitignore b/.gitignore index ca2081e493..fc5e8eb6a8 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ __pycache__/ /_build /docs/ubproject.toml /docs/_collections + +# Workspace files +/score_*/ +/.gita/ +/.gita-workspace.csv diff --git a/tools/known_good_to_gita_workspace.py b/tools/known_good_to_gita_workspace.py new file mode 100644 index 0000000000..10ebb86691 --- /dev/null +++ b/tools/known_good_to_gita_workspace.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import argparse +import json +import csv + +MODULES_CSV_HEADER = [ + "repo_url", + "name", + "workspace_path", + "version", + "hash", + "branch" +] + +def main(): + parser = argparse.ArgumentParser(description="Convert known_good.json to baseline.csv format.") + parser.add_argument("input_json", help="Path to known_good.json") + parser.add_argument("output_csv", help="Path to output baseline.csv") + args = parser.parse_args() + + with open(args.input_json, "r") as f: + data = json.load(f) + + modules = data.get("modules", {}) + rows = [] + for name, info in modules.items(): + repo_url = info.get("repo", "") + if not repo_url: + raise RuntimeError("repo must not be empty") + + # default branch: main + branch = info.get("branch", "main") + + # if no hash is given, use branch + hash_ = info.get("hash", branch) + + # workspace_path is not available in known_good.json, default to name of repository + workspace_path = name + + # gita format: {url},{name},{path},{prop['type']},{repo_flags},{branch} + row = [repo_url, name, workspace_path, "", "", hash_] + rows.append(row) + + with open(args.output_csv, "w", newline="") as f: + writer = csv.writer(f) + for row in rows: + writer.writerow(row) + +if __name__ == "__main__": + main() diff --git a/tools/update_module_from_known_good.py b/tools/update_module_from_known_good.py index 67820ecf0b..138590bfa9 100644 --- a/tools/update_module_from_known_good.py +++ b/tools/update_module_from_known_good.py @@ -90,8 +90,24 @@ def generate_git_override_blocks(modules_dict: Dict[str, Any], repo_commit_dict: return blocks +def generate_local_override_blocks(modules_dict: Dict[str, Any]) -> List[str]: + """Generate bazel_dep and local_path_override blocks for each module.""" + blocks = [] + + for name, module in modules_dict.items(): + block = ( + f'bazel_dep(name = "{name}")\n' + 'local_path_override(\n' + f' module_name = "{name}",\n' + f' path = "{name}",\n' + ')\n' + ) + + blocks.append(block) + + return blocks -def generate_file_content(modules: Dict[str, Any], repo_commit_dict: Dict[str, str], timestamp: Optional[str] = None) -> str: +def generate_file_content(args: argparse.Namespace, modules: Dict[str, Any], repo_commit_dict: Dict[str, str], timestamp: Optional[str] = None) -> str: """Generate the complete content for score_modules.MODULE.bazel.""" # License header assembled with parenthesis grouping (no indentation preserved in output). header = ( @@ -117,7 +133,15 @@ def generate_file_content(modules: Dict[str, Any], repo_commit_dict: Dict[str, s "\n" ) - blocks = generate_git_override_blocks(modules, repo_commit_dict) + if args.override_type == "git": + blocks = generate_git_override_blocks(modules, repo_commit_dict) + else: + header += ( + "# Note: This file uses local_path overrides. Ensure that local paths are set up correctly.\n" + "\n" + ) + blocks = generate_local_override_blocks(modules) + if not blocks: raise SystemExit("No valid modules to generate git_override blocks") @@ -149,6 +173,12 @@ def main() -> None: action="append", help="Override commit for a specific repo (format: @)" ) + parser.add_argument( + "--override-type", + choices=["local_path", "git"], + default="git", + help="Type of override to use (default: git)" + ) args = parser.parse_args() @@ -180,7 +210,7 @@ def main() -> None: # Generate file content timestamp = data.get("timestamp") or datetime.now().isoformat() - content = generate_file_content(modules, repo_commit_dict, timestamp) + content = generate_file_content(args, modules, repo_commit_dict, timestamp) if args.dry_run: print(f"Dry run: would write to {output_path}\n") @@ -191,7 +221,7 @@ def main() -> None: else: with open(output_path, "w", encoding="utf-8") as f: f.write(content) - print(f"Generated {output_path} with {len(modules)} git_override entries") + print(f"Generated {output_path} with {len(modules)} {args.override_type}_override entries") if __name__ == "__main__": From d3a232349d521c89993ea579d3435e009df139c6 Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Thu, 4 Dec 2025 14:09:37 +0100 Subject: [PATCH 2/8] also generate .gitmodules metadata --- .devcontainer/prepare_workspace.sh | 8 +++-- .gitignore | 1 + ...py => known_good_to_workspace_metadata.py} | 29 ++++++++++++++----- 3 files changed, 28 insertions(+), 10 deletions(-) rename tools/{known_good_to_gita_workspace.py => known_good_to_workspace_metadata.py} (50%) diff --git a/.devcontainer/prepare_workspace.sh b/.devcontainer/prepare_workspace.sh index ba2aa6f6fa..28d7deb5d0 100755 --- a/.devcontainer/prepare_workspace.sh +++ b/.devcontainer/prepare_workspace.sh @@ -17,8 +17,12 @@ GITA_PROJECT_HOME=$(pwd)/.gita mkdir -p "$GITA_PROJECT_HOME" export GITA_PROJECT_HOME -# Generate .gita-workspace.csv from known_good.json -python3 tools/known_good_to_gita_workspace.py known_good.json .gita-workspace.csv +# Generate a few workspace metadata files from known_good.json: +# - .gita-workspace.csv +# - .gitmodules +python3 tools/known_good_to_workspace_metadata.py --known-good known_good.json --gita-workspace .gita-workspace.csv --git-submodules .gitmodules + +# Replace git_overrides with local_path_overrides for Bazel python3 tools/update_module_from_known_good.py --override-type local_path # Automagically clone repositories listed in .gita-workspace.csv diff --git a/.gitignore b/.gitignore index fc5e8eb6a8..10ad921221 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ __pycache__/ /score_*/ /.gita/ /.gita-workspace.csv +/.gitmodules diff --git a/tools/known_good_to_gita_workspace.py b/tools/known_good_to_workspace_metadata.py similarity index 50% rename from tools/known_good_to_gita_workspace.py rename to tools/known_good_to_workspace_metadata.py index 10ebb86691..0658d26123 100644 --- a/tools/known_good_to_gita_workspace.py +++ b/tools/known_good_to_workspace_metadata.py @@ -13,16 +13,19 @@ ] def main(): - parser = argparse.ArgumentParser(description="Convert known_good.json to baseline.csv format.") - parser.add_argument("input_json", help="Path to known_good.json") - parser.add_argument("output_csv", help="Path to output baseline.csv") + parser = argparse.ArgumentParser(description="Convert known_good.json to workspace metadata files for gita and git submodules.") + + parser.add_argument("--known-good", dest="known_good", default="known_good.json", help="Path to known_good.json") + parser.add_argument("--gita-workspace", dest="gita_workspace", default=".gita-workspace.csv", help="File to output gita workspace metadata") + parser.add_argument("--git-submodules", dest="git_submodules", default=".gitmodules", help="File to output git submodules metadata") args = parser.parse_args() - with open(args.input_json, "r") as f: + with open(args.known_good, "r") as f: data = json.load(f) modules = data.get("modules", {}) - rows = [] + + gita_metadata = [] for name, info in modules.items(): repo_url = info.get("repo", "") if not repo_url: @@ -39,12 +42,22 @@ def main(): # gita format: {url},{name},{path},{prop['type']},{repo_flags},{branch} row = [repo_url, name, workspace_path, "", "", hash_] - rows.append(row) + gita_metadata.append(row) - with open(args.output_csv, "w", newline="") as f: + with open(args.gita_workspace, "w", newline="") as f: writer = csv.writer(f) - for row in rows: + for row in gita_metadata: writer.writerow(row) + with open(args.git_submodules, "w") as f: + for name, info in modules.items(): + repo_url = info.get("repo", "") + branch = info.get("branch", "main") + workspace_path = name + f.write(f"[submodule \"{name}\"]\n") + f.write(f"\tpath = {workspace_path}\n") + f.write(f"\turl = {repo_url}\n") + f.write(f"\tbranch = {branch}\n") + if __name__ == "__main__": main() From bc3f3c43ac198b3220dea88a41370681a3c46557 Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Thu, 4 Dec 2025 14:14:24 +0100 Subject: [PATCH 3/8] Remove git submodules generation This does not work, since submodules require also commit tracking in the top-level repository. This would by default pollute reference_integration, which is certainly not what we want. --- .devcontainer/prepare_workspace.sh | 3 +-- .gitignore | 1 - tools/known_good_to_workspace_metadata.py | 11 ----------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.devcontainer/prepare_workspace.sh b/.devcontainer/prepare_workspace.sh index 28d7deb5d0..f97076b192 100755 --- a/.devcontainer/prepare_workspace.sh +++ b/.devcontainer/prepare_workspace.sh @@ -19,8 +19,7 @@ export GITA_PROJECT_HOME # Generate a few workspace metadata files from known_good.json: # - .gita-workspace.csv -# - .gitmodules -python3 tools/known_good_to_workspace_metadata.py --known-good known_good.json --gita-workspace .gita-workspace.csv --git-submodules .gitmodules +python3 tools/known_good_to_workspace_metadata.py --known-good known_good.json --gita-workspace .gita-workspace.csv # Replace git_overrides with local_path_overrides for Bazel python3 tools/update_module_from_known_good.py --override-type local_path diff --git a/.gitignore b/.gitignore index 10ad921221..fc5e8eb6a8 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,3 @@ __pycache__/ /score_*/ /.gita/ /.gita-workspace.csv -/.gitmodules diff --git a/tools/known_good_to_workspace_metadata.py b/tools/known_good_to_workspace_metadata.py index 0658d26123..145c17614e 100644 --- a/tools/known_good_to_workspace_metadata.py +++ b/tools/known_good_to_workspace_metadata.py @@ -17,7 +17,6 @@ def main(): parser.add_argument("--known-good", dest="known_good", default="known_good.json", help="Path to known_good.json") parser.add_argument("--gita-workspace", dest="gita_workspace", default=".gita-workspace.csv", help="File to output gita workspace metadata") - parser.add_argument("--git-submodules", dest="git_submodules", default=".gitmodules", help="File to output git submodules metadata") args = parser.parse_args() with open(args.known_good, "r") as f: @@ -49,15 +48,5 @@ def main(): for row in gita_metadata: writer.writerow(row) - with open(args.git_submodules, "w") as f: - for name, info in modules.items(): - repo_url = info.get("repo", "") - branch = info.get("branch", "main") - workspace_path = name - f.write(f"[submodule \"{name}\"]\n") - f.write(f"\tpath = {workspace_path}\n") - f.write(f"\turl = {repo_url}\n") - f.write(f"\tbranch = {branch}\n") - if __name__ == "__main__": main() From b4401265c1e7d39568be13659f3ac6a233903822 Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Thu, 4 Dec 2025 14:31:26 +0100 Subject: [PATCH 4/8] do not create a workspace by default --- .devcontainer/prepare_workspace.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.devcontainer/prepare_workspace.sh b/.devcontainer/prepare_workspace.sh index f97076b192..918b8e4ff5 100755 --- a/.devcontainer/prepare_workspace.sh +++ b/.devcontainer/prepare_workspace.sh @@ -20,13 +20,3 @@ export GITA_PROJECT_HOME # Generate a few workspace metadata files from known_good.json: # - .gita-workspace.csv python3 tools/known_good_to_workspace_metadata.py --known-good known_good.json --gita-workspace .gita-workspace.csv - -# Replace git_overrides with local_path_overrides for Bazel -python3 tools/update_module_from_known_good.py --override-type local_path - -# Automagically clone repositories listed in .gita-workspace.csv -echo "Cloning repositories listed in .gita-workspace.csv..." -gita clone --preserve-path --from-file .gita-workspace.csv -# Output the status of all cloned repositories -echo "Status of all cloned repositories:" -gita ll From ad8ec26cd6ea3fdf6225b51b7a8b009ac7112962 Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Thu, 4 Dec 2025 14:30:52 +0100 Subject: [PATCH 5/8] Add VSCode Tasks and documentation --- .devcontainer/prepare_workspace.sh | 2 +- .vscode/tasks.json | 47 ++++++++++++++++++++++++++++++ README.md | 32 ++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .vscode/tasks.json diff --git a/.devcontainer/prepare_workspace.sh b/.devcontainer/prepare_workspace.sh index 918b8e4ff5..2a2e5ffdef 100755 --- a/.devcontainer/prepare_workspace.sh +++ b/.devcontainer/prepare_workspace.sh @@ -17,6 +17,6 @@ GITA_PROJECT_HOME=$(pwd)/.gita mkdir -p "$GITA_PROJECT_HOME" export GITA_PROJECT_HOME -# Generate a few workspace metadata files from known_good.json: +# Generate workspace metadata files from known_good.json: # - .gita-workspace.csv python3 tools/known_good_to_workspace_metadata.py --known-good known_good.json --gita-workspace .gita-workspace.csv diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000000..5234f5b4f6 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,47 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Update workspace metadata from known good", + "type": "shell", + "command": "python3", + "args": [ + "tools/known_good_to_workspace_metadata.py" + ], + "problemMatcher": [] + }, + { + "label": "Switch Bazel modules to local_path_overrides", + "type": "shell", + "command": "python3", + "args": [ + "tools/update_module_from_known_good.py", + "--override-type", + "local_path" + ], + "problemMatcher": [] + }, + { + "label": "Switch Bazel modules to git_overrides", + "type": "shell", + "command": "python3", + "args": [ + "tools/update_module_from_known_good.py", + "--override-type", + "git" + ] + }, + { + "label": "Gita: Generate workspace", + "type": "shell", + "command": "gita", + "args": [ + "clone", + "--preserve-path", + "--from-file", + ".gita-workspace.csv" + ], + "problemMatcher": [] + } + ] +} diff --git a/README.md b/README.md index 07821af1a8..d38033cbbd 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,38 @@ Execute `bazel query //feature_showcase/...` to obtain list of targets that You bazel build --config bl-x86_64-linux @score_orchestrator//src/... --verbose_failures ``` +## Workspace support + +You can obtain a complete S-CORE workspace, i.e. a git checkout of all modules from `known_good.json`, on the specific branches / commits, integrated into one Bazel build. +This helps with cross-module development, debugging, and generally "trying out things". +The startup of the supplied devcontainer already generates the required metadata. + +The supported workspace managers are: + +| Name | Description | +|------|-------------| +| [Gita](https://github.com/nosarthur/gita) | "a command-line tool to manage multiple git repos" | + +A description of how to use these workspace managers, together with their advantages and drawbacks, is beyond the scope of this document. +In case of doubt, choose the first. + +### Initialization of the workspace + +> [!WARNING] +> This will change the file `score_modules.MODULE.bazel`. +> Do **not** commit these changes! + +1. Switch to local path overrides, using the VSCode Task (`Terminal`->`Run Task...`) "Switch Bazel modules to `local_path_overrides`". + Note that you can switch back to `git_overrides` (the default) using the task "Switch Bazel modules to `git_overrides`" + +2. Run VSCode Task "<Name>: Generate workspace", e.g. "Gita: Generate workspace". + This will clone all modules using the chosen workspace manager. + The modules will be in sub-directories starting with `score_`. + Note that the usage of different workspace managers is mutually exclusive. + +When you now run Bazel, it will use the local working copies of all modules and not download them from git remotes. +You can make local changes to each module, which will be directly reflected in the next Bazel run. + ## Known Issues ⚠️ ### Orchestrator From 9d5c9fd27a2ddf36c6a704a5f0fa0b459a58aed7 Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Wed, 17 Dec 2025 17:05:05 +0100 Subject: [PATCH 6/8] Fix deprecation warning --- tools/update_module_latest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/update_module_latest.py b/tools/update_module_latest.py index 77fdee311b..210f847c46 100644 --- a/tools/update_module_latest.py +++ b/tools/update_module_latest.py @@ -119,7 +119,7 @@ def load_known_good(path: str) -> dict: def write_known_good(path: str, original: dict, modules: list[Module]) -> None: out = dict(original) # shallow copy - out["timestamp"] = dt.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + out["timestamp"] = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat() + "Z" out["modules"] = {} for m in modules: mod_dict = {"repo": m.repo, "hash": m.hash} From 3115a5a256800c461a21752ae38d6a98fb05b62a Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Thu, 18 Dec 2025 09:43:18 +0100 Subject: [PATCH 7/8] mark as executable --- tools/get_module_info.py | 0 tools/update_module_from_known_good.py | 0 tools/update_module_latest.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/get_module_info.py mode change 100644 => 100755 tools/update_module_from_known_good.py mode change 100644 => 100755 tools/update_module_latest.py diff --git a/tools/get_module_info.py b/tools/get_module_info.py old mode 100644 new mode 100755 diff --git a/tools/update_module_from_known_good.py b/tools/update_module_from_known_good.py old mode 100644 new mode 100755 diff --git a/tools/update_module_latest.py b/tools/update_module_latest.py old mode 100644 new mode 100755 From 0bad9562d6f67b1635a6117d4f68956520e882dc Mon Sep 17 00:00:00 2001 From: Oliver Pajonk Date: Mon, 22 Dec 2025 09:10:17 +0000 Subject: [PATCH 8/8] clarify the devcontainer relation and which build targets should be working --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d38033cbbd..2946b1237e 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,16 @@ bazel build --config bl-x86_64-linux @score_orchestrator//src/... --verbose_fail You can obtain a complete S-CORE workspace, i.e. a git checkout of all modules from `known_good.json`, on the specific branches / commits, integrated into one Bazel build. This helps with cross-module development, debugging, and generally "trying out things". -The startup of the supplied devcontainer already generates the required metadata. + +> [!NOTE] +> The startup of the [S-CORE devcontainer](https://github.com/eclipse-score/devcontainer) [integrated in this repository](.devcontainer/) already installs supported workspace managers and generates the required metadata. +> You can do this manually as well, of course (e.g. if you do not use the devcontainer). +> Take a look at `.devcontainer/prepare_workspace.sh`, which contains the setup script. + +> [!NOTE] +> Not all Bazel targets are supported yet. +> Running `./scripts/integration_test.sh` will work, though. +> Take a look at the [Known Issues](#known-issues-️) below to see which Bazel targets are available and working. The supported workspace managers are: