Improve dependency installation via vcstools

.gitlab-ci/docker-ros.yml

@@ -37,6 +37,8 @@ variables:
   ENABLE_RECURSIVE_ADDITIONAL_PIP:   'false'                          # Enable recursive discovery of files named `additional-pip-file`
   CUSTOM_SCRIPT_FILE:         docker/custom.sh                        # Relative filepath to script containing custom installation commands
   ENABLE_RECURSIVE_CUSTOM_SCRIPT:    'false'                          # Enable recursive discovery of files named `custom-script-file`
+  ADDITIONAL_REPOS_FILE:      .repos                                  # Relative filepath to script containing custom installation commands
+  ENABLE_RECURSIVE_VCS_IMPORT:       'true'                           # Enable recursive discovery of files named `additional-repos-file`
   # -----
   DOCKER_ROS_GIT_REF:     main
 

README.md

@@ -324,7 +324,10 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
   *default:* `docker/additional-files`  
 - **`additional-pip-file` | `ADDITIONAL_PIP_FILE`**  
   Relative filepath to file containing additional pip packages to install  
-  *default:* `docker/additional-pip-requirements.txt`  
+  *default:* `docker/additional-pip-requirements.txt`
+- **`additional-repos-file` | `ADDITIONAL_REPOS_FILE`**  
+  Relative filepath to file containing additional repos to install via vcstools
+  *default:* `.repos`
 - **`base-image` | `BASE_IMAGE`**  
   Base image `name:tag`  
   *required*  
@@ -373,6 +376,9 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
 - **`enable-recursive-custom-script` | `ENABLE_RECURSIVE_CUSTOM_SCRIPT`**  
   Enable recursive discovery of files named `custom-script-file`  
   *default:* `false`
+- **`enable-recursive-vcs-import` | `ENABLE_RECURSIVE_VCS_IMPORT`**  
+  Enable recursive discovery of files named `additional-repo-file`  
+  *default:* `true`
 - **`git-ssh-known-host-keys` | `GIT_SSH_KNOWN_HOST_KEYS`**  
   Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`)  
 - **`git-ssh-private-key` | `GIT_SSH_PRIVATE_KEY`**  

action.yml

@@ -174,6 +174,8 @@ runs:
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
+        ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }}
+        ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
 
     - name: Set up industrial_ci
       if: ${{ inputs.enable-industrial-ci == 'true' }}
@@ -227,6 +229,8 @@ runs:
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
+        ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }}
+        ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
         _ENABLE_IMAGE_PUSH: true
         _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }}
 
@@ -258,5 +262,7 @@ runs:
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
+        ADDITIONAL_REPOS_FILE: ${{ inputs.additional-repos-file }}
+        ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
         _ENABLE_IMAGE_PUSH: true
         _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }}

docker/Dockerfile

@@ -60,7 +60,14 @@ COPY docker/docker-ros/docker/recursive_vcs_import.py /usr/local/bin
 RUN apt-get update && \
     apt-get install -y python-is-python3 && \
     rm -rf /var/lib/apt/lists/*
-RUN /usr/local/bin/recursive_vcs_import.py src src/upstream
+
+ARG ADDITIONAL_REPOS_FILE=".repos"
+ARG ENABLE_RECURSIVE_VCS_IMPORT="true"
+RUN if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \
+        /usr/local/bin/recursive_vcs_import.py src/target src/upstream ${ADDITIONAL_REPOS_FILE}; \
+    else \
+        vcs import src/upstream < src/target/${ADDITIONAL_REPOS_FILE} ; \
+    fi
 
 # create install script with list of rosdep dependencies
 RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \
@@ -81,7 +88,7 @@ ARG ENABLE_RECURSIVE_ADDITIONAL_DEBS="false"
 RUN echo "apt-get install -y \\" >> $WORKSPACE/.install-dependencies.sh && \
     set -o pipefail && \
     if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \
-        find . -type f -name $(basename {ADDITIONAL_DEBS_FILE}) -exec sed '$a\' {} \; | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \
+        find . -type f -name $(basename ${ADDITIONAL_DEBS_FILE}) -exec sed '$a\' {} \; | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \
     elif [[ -f src/target/${ADDITIONAL_DEBS_FILE} ]]; then \
         cat src/target/${ADDITIONAL_DEBS_FILE} | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \
     fi && \

docker/recursive_vcs_import.py

@@ -6,22 +6,25 @@
 from typing import List, Optional
 
 
-def findDotRepos(search_path: str, clone_path: Optional[str] = None) -> List[pathlib.Path]:
+def findDotRepos(search_path: str, clone_path: Optional[str] = None, repos_file: str = ".repos") -> List[pathlib.Path]:
 
     repos = list(pathlib.Path(search_path).glob("**/*.repos"))
     if clone_path is not None:
-        repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos")))
+        repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos"))) # find .repos in upstream dependencies
+    else:
+        repos = list(pathlib.Path(search_path).glob(f"**/*{repos_file}"))      # find .repos in target repo
     return repos
 
 def main():
 
     search_path = sys.argv[1] if len(sys.argv) > 1 else "."
     clone_path = sys.argv[2] if len(sys.argv) > 2 else "."
+    repos_file = sys.argv[3] if len(sys.argv) > 3 else ".repos"
     cloned_repos = []
 
     while True:
 
-        found_repos = findDotRepos(search_path, clone_path)
+        found_repos = findDotRepos(search_path, clone_path, repos_file)
         remaining_repos = set(found_repos) - set(cloned_repos)
 
         if not remaining_repos:

scripts/build.sh

@@ -30,6 +30,8 @@ build_image() {
         $(if [[ -n "${ENABLE_RECURSIVE_ADDITIONAL_PIP}" ]]; then echo "--build-arg ENABLE_RECURSIVE_ADDITIONAL_PIP=${ENABLE_RECURSIVE_ADDITIONAL_PIP}"; fi) \
         $(if [[ -n "${CUSTOM_SCRIPT_FILE}" ]]; then echo "--build-arg CUSTOM_SCRIPT_FILE=${CUSTOM_SCRIPT_FILE}"; fi) \
         $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \
+        $(if [[ -n "${ADDITIONAL_REPOS_FILE}" ]]; then echo "--build-arg ADDITIONAL_REPOS_FILE=${ADDITIONAL_REPOS_FILE}"; fi) \
+        $(if [[ -n "${ENABLE_RECURSIVE_VCS_IMPORT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_VCS_IMPORT=${ENABLE_RECURSIVE_VCS_IMPORT}"; fi) \
         .
     echo "Successfully built stage '${TARGET}' for platform '${PLATFORM}' as '${IMAGE}'"
 }

scripts/ci.sh

@@ -33,6 +33,8 @@ ADDITIONAL_PIP_FILE="${ADDITIONAL_PIP_FILE:-}"
 ENABLE_RECURSIVE_ADDITIONAL_PIP="${ENABLE_RECURSIVE_ADDITIONAL_PIP:-}"
 CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}"
 ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}"
+ADDITIONAL_REPOS_FILE="${ADDITIONAL_REPOS_FILE:-}"
+ENABLE_RECURSIVE_VCS_IMPORT="${ENABLE_RECURSIVE_VCS_IMPORT:-}"
 _ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}"
 _IMAGE_POSTFIX="${_IMAGE_POSTFIX:-""}"
 

templates/docker-compose.template.yml

@@ -14,10 +14,12 @@ x-build: &build
     ADDITIONAL_DEBS_FILE: $ADDITIONAL_DEBS_FILE
     ADDITIONAL_FILES_DIR: $ADDITIONAL_FILES_DIR
     ADDITIONAL_PIP_FILE: $ADDITIONAL_PIP_FILE
+    ADDITIONAL_REPOS_FILE: $ADDITIONAL_REPOS_FILE
     CUSTOM_SCRIPT_FILE: $CUSTOM_SCRIPT_FILE
     ENABLE_RECURSIVE_ADDITIONAL_DEBS: $ENABLE_RECURSIVE_ADDITIONAL_DEBS
     ENABLE_RECURSIVE_ADDITIONAL_PIP: $ENABLE_RECURSIVE_ADDITIONAL_PIP
     ENABLE_RECURSIVE_CUSTOM_SCRIPT: $ENABLE_RECURSIVE_CUSTOM_SCRIPT
+    ENABLE_RECURSIVE_VCS_IMPORT: $ENABLE_RECURSIVE_VCS_IMPORT
     GIT_HTTPS_PASSWORD: $GIT_HTTPS_PASSWORD
     GIT_HTTPS_SERVER: $GIT_HTTPS_SERVER
     GIT_HTTPS_USER: $GIT_HTTPS_USER