From 4ff69fcad9626803dc9fec1ab2fa9b29647fc26a Mon Sep 17 00:00:00 2001 From: Jean-Pierre Busch Date: Tue, 10 Oct 2023 18:13:11 +0200 Subject: [PATCH 1/6] enable non-recursive vcs import --- .gitlab-ci/docker-ros.yml | 1 + README.md | 3 +++ action.yml | 3 +++ docker/Dockerfile | 7 ++++++- scripts/build.sh | 1 + scripts/ci.sh | 1 + templates/docker-compose.template.yml | 1 + 7 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci/docker-ros.yml b/.gitlab-ci/docker-ros.yml index 84078a4..e2012c8 100644 --- a/.gitlab-ci/docker-ros.yml +++ b/.gitlab-ci/docker-ros.yml @@ -37,6 +37,7 @@ variables: ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file` CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file` + ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `additional-repos-file` # ----- DOCKER_ROS_GIT_REF: main diff --git a/README.md b/README.md index 4009a47..f6b8a8a 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,9 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe - **`enable-recursive-custom-script` | `ENABLE_RECURSIVE_CUSTOM_SCRIPT`** Enable recursive discovery of files named `custom-script-file` *default:* `false` +- **`enable-recursive-vcs-import` | `ENABLE_RECURSIVE_VCS_IMPORT`** + Enable recursive discovery of files named `additional-repo-file` + *default:* `true` - **`git-ssh-known-host-keys` | `GIT_SSH_KNOWN_HOST_KEYS`** Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`) - **`git-ssh-private-key` | `GIT_SSH_PRIVATE_KEY`** diff --git a/action.yml b/action.yml index 1377565..187905d 100644 --- a/action.yml +++ b/action.yml @@ -174,6 +174,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} + ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} - name: Set up industrial_ci if: ${{ inputs.enable-industrial-ci == 'true' }} @@ -227,6 +228,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} + ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} @@ -258,5 +260,6 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} + ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 8eff386..920f1f7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -60,7 +60,12 @@ COPY docker/docker-ros/docker/recursive_vcs_import.py /usr/local/bin RUN apt-get update && \ apt-get install -y python-is-python3 && \ rm -rf /var/lib/apt/lists/* -RUN /usr/local/bin/recursive_vcs_import.py src src/upstream +ARG ENABLE_RECURSIVE_VCS_IMPORT="true" +RUN if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \ + /usr/local/bin/recursive_vcs_import.py src src/upstream ; \ + else \ + vcs import src/upstream < src/target/.repos ; \ + fi # create install script with list of rosdep dependencies RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \ diff --git a/scripts/build.sh b/scripts/build.sh index 23a94ad..d71f874 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -30,6 +30,7 @@ build_image() { $(if [[ -n "${ENABLE_RECURSIVE_ADDITIONAL_PIP}" ]]; then echo "--build-arg ENABLE_RECURSIVE_ADDITIONAL_PIP=${ENABLE_RECURSIVE_ADDITIONAL_PIP}"; fi) \ $(if [[ -n "${CUSTOM_SCRIPT_FILE}" ]]; then echo "--build-arg CUSTOM_SCRIPT_FILE=${CUSTOM_SCRIPT_FILE}"; fi) \ $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \ + $(if [[ -n "${ENABLE_RECURSIVE_VCS_IMPORT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_VCS_IMPORT=${ENABLE_RECURSIVE_VCS_IMPORT}"; fi) \ . echo "Successfully built stage '${TARGET}' for platform '${PLATFORM}' as '${IMAGE}'" } diff --git a/scripts/ci.sh b/scripts/ci.sh index 7e05820..5a3c82a 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -33,6 +33,7 @@ ADDITIONAL_PIP_FILE="${ADDITIONAL_PIP_FILE:-}" ENABLE_RECURSIVE_ADDITIONAL_PIP="${ENABLE_RECURSIVE_ADDITIONAL_PIP:-}" CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}" ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}" +ENABLE_RECURSIVE_VCS_IMPORT="${ENABLE_RECURSIVE_VCS_IMPORT:-}" _ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}" _IMAGE_POSTFIX="${_IMAGE_POSTFIX:-""}" diff --git a/templates/docker-compose.template.yml b/templates/docker-compose.template.yml index 87780f4..4c780d3 100644 --- a/templates/docker-compose.template.yml +++ b/templates/docker-compose.template.yml @@ -18,6 +18,7 @@ x-build: &build ENABLE_RECURSIVE_ADDITIONAL_DEBS: $ENABLE_RECURSIVE_ADDITIONAL_DEBS ENABLE_RECURSIVE_ADDITIONAL_PIP: $ENABLE_RECURSIVE_ADDITIONAL_PIP ENABLE_RECURSIVE_CUSTOM_SCRIPT: $ENABLE_RECURSIVE_CUSTOM_SCRIPT + ENABLE_RECURSIVE_VCS_IMPORT: $ENABLE_RECURSIVE_VCS_IMPORT GIT_HTTPS_PASSWORD: $GIT_HTTPS_PASSWORD GIT_HTTPS_SERVER: $GIT_HTTPS_SERVER GIT_HTTPS_USER: $GIT_HTTPS_USER From 37a86f280ecfe939c84114e03866d32e535a819d Mon Sep 17 00:00:00 2001 From: Jean-Pierre Busch Date: Tue, 10 Oct 2023 18:20:57 +0200 Subject: [PATCH 2/6] enable custom `.repos` filename --- .gitlab-ci/docker-ros.yml | 1 + README.md | 5 ++++- action.yml | 3 +++ docker/Dockerfile | 6 ++++-- docker/recursive_vcs_import.py | 9 ++++++--- scripts/build.sh | 1 + scripts/ci.sh | 1 + templates/docker-compose.template.yml | 1 + 8 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci/docker-ros.yml b/.gitlab-ci/docker-ros.yml index e2012c8..b74480b 100644 --- a/.gitlab-ci/docker-ros.yml +++ b/.gitlab-ci/docker-ros.yml @@ -37,6 +37,7 @@ variables: ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file` CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file` + ADDITIONAL_REPOS_FILE: .repos # Relative filepath to script containing custom installation commands ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `additional-repos-file` # ----- DOCKER_ROS_GIT_REF: main diff --git a/README.md b/README.md index f6b8a8a..6fafb15 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,10 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe *default:* `docker/additional-files` - **`additional-pip-file` | `ADDITIONAL_PIP_FILE`** Relative filepath to file containing additional pip packages to install - *default:* `docker/additional-pip-requirements.txt` + *default:* `docker/additional-pip-requirements.txt` +- **`additional-repos-file` | `ADDITIONAL_REPOS_FILE`** + Relative filepath to file containing additional repos to install via vcstools + *default:* `.repos` - **`base-image` | `BASE_IMAGE`** Base image `name:tag` *required* diff --git a/action.yml b/action.yml index 187905d..af3d886 100644 --- a/action.yml +++ b/action.yml @@ -174,6 +174,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} + ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} - name: Set up industrial_ci @@ -228,6 +229,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} + ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} @@ -260,6 +262,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} + ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 920f1f7..de996bd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -60,11 +60,13 @@ COPY docker/docker-ros/docker/recursive_vcs_import.py /usr/local/bin RUN apt-get update && \ apt-get install -y python-is-python3 && \ rm -rf /var/lib/apt/lists/* + +ARG ADDITIONAL_REPOS_FILE=".repos" ARG ENABLE_RECURSIVE_VCS_IMPORT="true" RUN if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \ - /usr/local/bin/recursive_vcs_import.py src src/upstream ; \ + /usr/local/bin/recursive_vcs_import.py src/target src/upstream ${ADDITIONAL_REPOS_FILE}; \ else \ - vcs import src/upstream < src/target/.repos ; \ + vcs import src/upstream < src/target/${ADDITIONAL_REPOS_FILE} ; \ fi # create install script with list of rosdep dependencies diff --git a/docker/recursive_vcs_import.py b/docker/recursive_vcs_import.py index e8cda6a..438301c 100755 --- a/docker/recursive_vcs_import.py +++ b/docker/recursive_vcs_import.py @@ -6,22 +6,25 @@ from typing import List, Optional -def findDotRepos(search_path: str, clone_path: Optional[str] = None) -> List[pathlib.Path]: +def findDotRepos(search_path: str, clone_path: Optional[str] = None, repos_file: str = ".repos") -> List[pathlib.Path]: repos = list(pathlib.Path(search_path).glob("**/*.repos")) if clone_path is not None: - repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos"))) + repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos"))) # find .repos in upstream dependencies + else: + repos = list(pathlib.Path(search_path).glob(f"**/*{repos_file}")) # find .repos in target repo return repos def main(): search_path = sys.argv[1] if len(sys.argv) > 1 else "." clone_path = sys.argv[2] if len(sys.argv) > 2 else "." + repos_file = sys.argv[3] if len(sys.argv) > 3 else ".repos" cloned_repos = [] while True: - found_repos = findDotRepos(search_path, clone_path) + found_repos = findDotRepos(search_path, clone_path, repos_file) remaining_repos = set(found_repos) - set(cloned_repos) if not remaining_repos: diff --git a/scripts/build.sh b/scripts/build.sh index d71f874..901b599 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -30,6 +30,7 @@ build_image() { $(if [[ -n "${ENABLE_RECURSIVE_ADDITIONAL_PIP}" ]]; then echo "--build-arg ENABLE_RECURSIVE_ADDITIONAL_PIP=${ENABLE_RECURSIVE_ADDITIONAL_PIP}"; fi) \ $(if [[ -n "${CUSTOM_SCRIPT_FILE}" ]]; then echo "--build-arg CUSTOM_SCRIPT_FILE=${CUSTOM_SCRIPT_FILE}"; fi) \ $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \ + $(if [[ -n "${ADDITIONAL_REPOS_FILE}" ]]; then echo "--build-arg ADDITIONAL_REPOS_FILE=${ADDITIONAL_REPOS_FILE}"; fi) \ $(if [[ -n "${ENABLE_RECURSIVE_VCS_IMPORT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_VCS_IMPORT=${ENABLE_RECURSIVE_VCS_IMPORT}"; fi) \ . echo "Successfully built stage '${TARGET}' for platform '${PLATFORM}' as '${IMAGE}'" diff --git a/scripts/ci.sh b/scripts/ci.sh index 5a3c82a..719bcc9 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -33,6 +33,7 @@ ADDITIONAL_PIP_FILE="${ADDITIONAL_PIP_FILE:-}" ENABLE_RECURSIVE_ADDITIONAL_PIP="${ENABLE_RECURSIVE_ADDITIONAL_PIP:-}" CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}" ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}" +ADDITIONAL_REPOS_FILE="${ADDITIONAL_REPOS_FILE:-}" ENABLE_RECURSIVE_VCS_IMPORT="${ENABLE_RECURSIVE_VCS_IMPORT:-}" _ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}" _IMAGE_POSTFIX="${_IMAGE_POSTFIX:-""}" diff --git a/templates/docker-compose.template.yml b/templates/docker-compose.template.yml index 4c780d3..fc1afc1 100644 --- a/templates/docker-compose.template.yml +++ b/templates/docker-compose.template.yml @@ -14,6 +14,7 @@ x-build: &build ADDITIONAL_DEBS_FILE: $ADDITIONAL_DEBS_FILE ADDITIONAL_FILES_DIR: $ADDITIONAL_FILES_DIR ADDITIONAL_PIP_FILE: $ADDITIONAL_PIP_FILE + ADDITIONAL_REPOS_FILE: $ADDITIONAL_REPOS_FILE CUSTOM_SCRIPT_FILE: $CUSTOM_SCRIPT_FILE ENABLE_RECURSIVE_ADDITIONAL_DEBS: $ENABLE_RECURSIVE_ADDITIONAL_DEBS ENABLE_RECURSIVE_ADDITIONAL_PIP: $ENABLE_RECURSIVE_ADDITIONAL_PIP From 19e2b4df8d722e10969e348412b05ed76295ca51 Mon Sep 17 00:00:00 2001 From: Jean-Pierre Busch Date: Wed, 11 Oct 2023 12:44:35 +0200 Subject: [PATCH 3/6] fix handling of recursive additional deps file --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index de996bd..68f046b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -88,7 +88,7 @@ ARG ENABLE_RECURSIVE_ADDITIONAL_DEBS="false" RUN echo "apt-get install -y \\" >> $WORKSPACE/.install-dependencies.sh && \ set -o pipefail && \ if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \ - find . -type f -name $(basename {ADDITIONAL_DEBS_FILE}) -exec sed '$a\' {} \; | awk '{print " " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \ + find . -type f -name $(basename ${ADDITIONAL_DEBS_FILE}) -exec sed '$a\' {} \; | awk '{print " " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \ elif [[ -f src/target/${ADDITIONAL_DEBS_FILE} ]]; then \ cat src/target/${ADDITIONAL_DEBS_FILE} | awk '{print " " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \ fi && \ From ccb8665066949b3146fa444d16c504a6d7a82e09 Mon Sep 17 00:00:00 2001 From: Jean-Pierre Busch Date: Wed, 11 Oct 2023 13:38:33 +0200 Subject: [PATCH 4/6] revert changes in recursive_vcs_import --- docker/Dockerfile | 2 +- docker/recursive_vcs_import.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 68f046b..e3c79e8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && \ ARG ADDITIONAL_REPOS_FILE=".repos" ARG ENABLE_RECURSIVE_VCS_IMPORT="true" RUN if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \ - /usr/local/bin/recursive_vcs_import.py src/target src/upstream ${ADDITIONAL_REPOS_FILE}; \ + /usr/local/bin/recursive_vcs_import.py src src/upstream ; \ else \ vcs import src/upstream < src/target/${ADDITIONAL_REPOS_FILE} ; \ fi diff --git a/docker/recursive_vcs_import.py b/docker/recursive_vcs_import.py index 438301c..e8cda6a 100755 --- a/docker/recursive_vcs_import.py +++ b/docker/recursive_vcs_import.py @@ -6,25 +6,22 @@ from typing import List, Optional -def findDotRepos(search_path: str, clone_path: Optional[str] = None, repos_file: str = ".repos") -> List[pathlib.Path]: +def findDotRepos(search_path: str, clone_path: Optional[str] = None) -> List[pathlib.Path]: repos = list(pathlib.Path(search_path).glob("**/*.repos")) if clone_path is not None: - repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos"))) # find .repos in upstream dependencies - else: - repos = list(pathlib.Path(search_path).glob(f"**/*{repos_file}")) # find .repos in target repo + repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos"))) return repos def main(): search_path = sys.argv[1] if len(sys.argv) > 1 else "." clone_path = sys.argv[2] if len(sys.argv) > 2 else "." - repos_file = sys.argv[3] if len(sys.argv) > 3 else ".repos" cloned_repos = [] while True: - found_repos = findDotRepos(search_path, clone_path, repos_file) + found_repos = findDotRepos(search_path, clone_path) remaining_repos = set(found_repos) - set(cloned_repos) if not remaining_repos: From 4fe94ad6e39d6ee6d2e8d7b07a365dc5fb2167d7 Mon Sep 17 00:00:00 2001 From: Jean-Pierre Busch Date: Wed, 11 Oct 2023 13:41:55 +0200 Subject: [PATCH 5/6] rename `VCS_IMPORT_FILE` --- .gitlab-ci/docker-ros.yml | 4 ++-- README.md | 2 +- action.yml | 6 +++--- docker/Dockerfile | 4 ++-- scripts/build.sh | 2 +- scripts/ci.sh | 2 +- templates/docker-compose.template.yml | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci/docker-ros.yml b/.gitlab-ci/docker-ros.yml index b74480b..bb5903a 100644 --- a/.gitlab-ci/docker-ros.yml +++ b/.gitlab-ci/docker-ros.yml @@ -37,8 +37,8 @@ variables: ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file` CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file` - ADDITIONAL_REPOS_FILE: .repos # Relative filepath to script containing custom installation commands - ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `additional-repos-file` + VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools + ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `vcs-import-file` # ----- DOCKER_ROS_GIT_REF: main diff --git a/README.md b/README.md index 6fafb15..463f956 100644 --- a/README.md +++ b/README.md @@ -325,7 +325,7 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe - **`additional-pip-file` | `ADDITIONAL_PIP_FILE`** Relative filepath to file containing additional pip packages to install *default:* `docker/additional-pip-requirements.txt` -- **`additional-repos-file` | `ADDITIONAL_REPOS_FILE`** +- **`vcs-import-file` | `VCS_IMPORT_FILE`** Relative filepath to file containing additional repos to install via vcstools *default:* `.repos` - **`base-image` | `BASE_IMAGE`** diff --git a/action.yml b/action.yml index af3d886..7b89ac3 100644 --- a/action.yml +++ b/action.yml @@ -174,7 +174,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} - ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }} + VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} - name: Set up industrial_ci @@ -229,7 +229,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} - ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }} + VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} @@ -262,7 +262,7 @@ runs: ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }} CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }} ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} - ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }} + VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index e3c79e8..9b89414 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -61,12 +61,12 @@ RUN apt-get update && \ apt-get install -y python-is-python3 && \ rm -rf /var/lib/apt/lists/* -ARG ADDITIONAL_REPOS_FILE=".repos" +ARG VCS_IMPORT_FILE=".repos" ARG ENABLE_RECURSIVE_VCS_IMPORT="true" RUN if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \ /usr/local/bin/recursive_vcs_import.py src src/upstream ; \ else \ - vcs import src/upstream < src/target/${ADDITIONAL_REPOS_FILE} ; \ + vcs import src/upstream < src/target/${VCS_IMPORT_FILE} ; \ fi # create install script with list of rosdep dependencies diff --git a/scripts/build.sh b/scripts/build.sh index 901b599..6d33dad 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -30,7 +30,7 @@ build_image() { $(if [[ -n "${ENABLE_RECURSIVE_ADDITIONAL_PIP}" ]]; then echo "--build-arg ENABLE_RECURSIVE_ADDITIONAL_PIP=${ENABLE_RECURSIVE_ADDITIONAL_PIP}"; fi) \ $(if [[ -n "${CUSTOM_SCRIPT_FILE}" ]]; then echo "--build-arg CUSTOM_SCRIPT_FILE=${CUSTOM_SCRIPT_FILE}"; fi) \ $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \ - $(if [[ -n "${ADDITIONAL_REPOS_FILE}" ]]; then echo "--build-arg ADDITIONAL_REPOS_FILE=${ADDITIONAL_REPOS_FILE}"; fi) \ + $(if [[ -n "${VCS_IMPORT_FILE}" ]]; then echo "--build-arg VCS_IMPORT_FILE=${VCS_IMPORT_FILE}"; fi) \ $(if [[ -n "${ENABLE_RECURSIVE_VCS_IMPORT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_VCS_IMPORT=${ENABLE_RECURSIVE_VCS_IMPORT}"; fi) \ . echo "Successfully built stage '${TARGET}' for platform '${PLATFORM}' as '${IMAGE}'" diff --git a/scripts/ci.sh b/scripts/ci.sh index 719bcc9..81ebfff 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -33,7 +33,7 @@ ADDITIONAL_PIP_FILE="${ADDITIONAL_PIP_FILE:-}" ENABLE_RECURSIVE_ADDITIONAL_PIP="${ENABLE_RECURSIVE_ADDITIONAL_PIP:-}" CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}" ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}" -ADDITIONAL_REPOS_FILE="${ADDITIONAL_REPOS_FILE:-}" +VCS_IMPORT_FILE="${VCS_IMPORT_FILE:-}" ENABLE_RECURSIVE_VCS_IMPORT="${ENABLE_RECURSIVE_VCS_IMPORT:-}" _ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}" _IMAGE_POSTFIX="${_IMAGE_POSTFIX:-""}" diff --git a/templates/docker-compose.template.yml b/templates/docker-compose.template.yml index fc1afc1..3449bb4 100644 --- a/templates/docker-compose.template.yml +++ b/templates/docker-compose.template.yml @@ -14,7 +14,7 @@ x-build: &build ADDITIONAL_DEBS_FILE: $ADDITIONAL_DEBS_FILE ADDITIONAL_FILES_DIR: $ADDITIONAL_FILES_DIR ADDITIONAL_PIP_FILE: $ADDITIONAL_PIP_FILE - ADDITIONAL_REPOS_FILE: $ADDITIONAL_REPOS_FILE + VCS_IMPORT_FILE: $VCS_IMPORT_FILE CUSTOM_SCRIPT_FILE: $CUSTOM_SCRIPT_FILE ENABLE_RECURSIVE_ADDITIONAL_DEBS: $ENABLE_RECURSIVE_ADDITIONAL_DEBS ENABLE_RECURSIVE_ADDITIONAL_PIP: $ENABLE_RECURSIVE_ADDITIONAL_PIP From c1ec673ba3ca41a36cc3c1f3096ddab268377437 Mon Sep 17 00:00:00 2001 From: Jean-Pierre Busch Date: Wed, 11 Oct 2023 13:50:40 +0200 Subject: [PATCH 6/6] small readme improvements --- .gitlab-ci/docker-ros.yml | 4 ++-- README.md | 4 ++-- docker/recursive_vcs_import.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci/docker-ros.yml b/.gitlab-ci/docker-ros.yml index bb5903a..b838dce 100644 --- a/.gitlab-ci/docker-ros.yml +++ b/.gitlab-ci/docker-ros.yml @@ -37,8 +37,8 @@ variables: ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file` CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file` - VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools - ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `vcs-import-file` + VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false) + ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `*.repos` # ----- DOCKER_ROS_GIT_REF: main diff --git a/README.md b/README.md index 463f956..1f70aa5 100644 --- a/README.md +++ b/README.md @@ -326,7 +326,7 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe Relative filepath to file containing additional pip packages to install *default:* `docker/additional-pip-requirements.txt` - **`vcs-import-file` | `VCS_IMPORT_FILE`** - Relative filepath to file containing additional repos to install via vcstools + Relative filepath to file containing additional repos to install via vcstools (only relevant if `enable-recursive-vcs-import=false`) *default:* `.repos` - **`base-image` | `BASE_IMAGE`** Base image `name:tag` @@ -377,7 +377,7 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe Enable recursive discovery of files named `custom-script-file` *default:* `false` - **`enable-recursive-vcs-import` | `ENABLE_RECURSIVE_VCS_IMPORT`** - Enable recursive discovery of files named `additional-repo-file` + Enable recursive discovery of files named `*.repos` *default:* `true` - **`git-ssh-known-host-keys` | `GIT_SSH_KNOWN_HOST_KEYS`** Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`) diff --git a/docker/recursive_vcs_import.py b/docker/recursive_vcs_import.py index e8cda6a..4d0bf95 100755 --- a/docker/recursive_vcs_import.py +++ b/docker/recursive_vcs_import.py @@ -13,6 +13,7 @@ def findDotRepos(search_path: str, clone_path: Optional[str] = None) -> List[pat repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos"))) return repos + def main(): search_path = sys.argv[1] if len(sys.argv) > 1 else "." @@ -39,4 +40,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main()