Merge pull request #13 from ika-rwth-aachen/improvement/vcs-import

Improve dependency installation via vcstools

.gitlab-ci/docker-ros.yml

@@ -37,6 +37,8 @@ variables:
   ENABLE_RECURSIVE_ADDITIONAL_PIP:   'false'                          # Enable recursive discovery of files named `additional-pip-file`
   CUSTOM_SCRIPT_FILE:         docker/custom.sh                        # Relative filepath to script containing custom installation commands
   ENABLE_RECURSIVE_CUSTOM_SCRIPT:    'false'                          # Enable recursive discovery of files named `custom-script-file`
+  VCS_IMPORT_FILE:            .repos                                  # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false)
+  ENABLE_RECURSIVE_VCS_IMPORT:       'true'                           # Enable recursive discovery of files named `*.repos`
   # -----
   DOCKER_ROS_GIT_REF:     main
 

README.md

@@ -324,7 +324,10 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
   *default:* `docker/additional-files`  
 - **`additional-pip-file` | `ADDITIONAL_PIP_FILE`**  
   Relative filepath to file containing additional pip packages to install  
-  *default:* `docker/additional-pip-requirements.txt`  
+  *default:* `docker/additional-pip-requirements.txt`
+- **`vcs-import-file` | `VCS_IMPORT_FILE`**  
+  Relative filepath to file containing additional repos to install via vcstools (only relevant if `enable-recursive-vcs-import=false`)
+  *default:* `.repos`
 - **`base-image` | `BASE_IMAGE`**  
   Base image `name:tag`  
   *required*  
@@ -373,6 +376,9 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
 - **`enable-recursive-custom-script` | `ENABLE_RECURSIVE_CUSTOM_SCRIPT`**  
   Enable recursive discovery of files named `custom-script-file`  
   *default:* `false`
+- **`enable-recursive-vcs-import` | `ENABLE_RECURSIVE_VCS_IMPORT`**  
+  Enable recursive discovery of files named `*.repos`  
+  *default:* `true`
 - **`git-ssh-known-host-keys` | `GIT_SSH_KNOWN_HOST_KEYS`**  
   Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`)  
 - **`git-ssh-private-key` | `GIT_SSH_PRIVATE_KEY`**  

action.yml

@@ -174,6 +174,8 @@ runs:
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
+        VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
+        ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
 
     - name: Set up industrial_ci
       if: ${{ inputs.enable-industrial-ci == 'true' }}
@@ -227,6 +229,8 @@ runs:
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
+        VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
+        ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
         _ENABLE_IMAGE_PUSH: true
         _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }}
 
@@ -258,5 +262,7 @@ runs:
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
+        VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
+        ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
         _ENABLE_IMAGE_PUSH: true
         _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }}

docker/Dockerfile

@@ -60,7 +60,14 @@ COPY docker/docker-ros/docker/recursive_vcs_import.py /usr/local/bin
 RUN apt-get update && \
     apt-get install -y python-is-python3 && \
     rm -rf /var/lib/apt/lists/*
-RUN /usr/local/bin/recursive_vcs_import.py src src/upstream
+
+ARG VCS_IMPORT_FILE=".repos"
+ARG ENABLE_RECURSIVE_VCS_IMPORT="true"
+RUN if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \
+        /usr/local/bin/recursive_vcs_import.py src src/upstream ; \
+    else \
+        vcs import src/upstream < src/target/${VCS_IMPORT_FILE} ; \
+    fi
 
 # create install script with list of rosdep dependencies
 RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \
@@ -81,7 +88,7 @@ ARG ENABLE_RECURSIVE_ADDITIONAL_DEBS="false"
 RUN echo "apt-get install -y \\" >> $WORKSPACE/.install-dependencies.sh && \
     set -o pipefail && \
     if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \
-        find . -type f -name $(basename {ADDITIONAL_DEBS_FILE}) -exec sed '$a\' {} \; | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \
+        find . -type f -name $(basename ${ADDITIONAL_DEBS_FILE}) -exec sed '$a\' {} \; | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \
     elif [[ -f src/target/${ADDITIONAL_DEBS_FILE} ]]; then \
         cat src/target/${ADDITIONAL_DEBS_FILE} | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.install-dependencies.sh ; \
     fi && \

docker/recursive_vcs_import.py

@@ -13,6 +13,7 @@ def findDotRepos(search_path: str, clone_path: Optional[str] = None) -> List[pat
         repos.extend(list(pathlib.Path(clone_path).glob("**/*.repos")))
     return repos
 
+
 def main():
 
     search_path = sys.argv[1] if len(sys.argv) > 1 else "."
@@ -39,4 +40,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

scripts/build.sh

@@ -30,6 +30,8 @@ build_image() {
         $(if [[ -n "${ENABLE_RECURSIVE_ADDITIONAL_PIP}" ]]; then echo "--build-arg ENABLE_RECURSIVE_ADDITIONAL_PIP=${ENABLE_RECURSIVE_ADDITIONAL_PIP}"; fi) \
         $(if [[ -n "${CUSTOM_SCRIPT_FILE}" ]]; then echo "--build-arg CUSTOM_SCRIPT_FILE=${CUSTOM_SCRIPT_FILE}"; fi) \
         $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \
+        $(if [[ -n "${VCS_IMPORT_FILE}" ]]; then echo "--build-arg VCS_IMPORT_FILE=${VCS_IMPORT_FILE}"; fi) \
+        $(if [[ -n "${ENABLE_RECURSIVE_VCS_IMPORT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_VCS_IMPORT=${ENABLE_RECURSIVE_VCS_IMPORT}"; fi) \
         .
     echo "Successfully built stage '${TARGET}' for platform '${PLATFORM}' as '${IMAGE}'"
 }

scripts/ci.sh

@@ -33,6 +33,8 @@ ADDITIONAL_PIP_FILE="${ADDITIONAL_PIP_FILE:-}"
 ENABLE_RECURSIVE_ADDITIONAL_PIP="${ENABLE_RECURSIVE_ADDITIONAL_PIP:-}"
 CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}"
 ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}"
+VCS_IMPORT_FILE="${VCS_IMPORT_FILE:-}"
+ENABLE_RECURSIVE_VCS_IMPORT="${ENABLE_RECURSIVE_VCS_IMPORT:-}"
 _ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}"
 _IMAGE_POSTFIX="${_IMAGE_POSTFIX:-""}"
 

templates/docker-compose.template.yml

@@ -14,10 +14,12 @@ x-build: &build
     ADDITIONAL_DEBS_FILE: $ADDITIONAL_DEBS_FILE
     ADDITIONAL_FILES_DIR: $ADDITIONAL_FILES_DIR
     ADDITIONAL_PIP_FILE: $ADDITIONAL_PIP_FILE
+    VCS_IMPORT_FILE: $VCS_IMPORT_FILE
     CUSTOM_SCRIPT_FILE: $CUSTOM_SCRIPT_FILE
     ENABLE_RECURSIVE_ADDITIONAL_DEBS: $ENABLE_RECURSIVE_ADDITIONAL_DEBS
     ENABLE_RECURSIVE_ADDITIONAL_PIP: $ENABLE_RECURSIVE_ADDITIONAL_PIP
     ENABLE_RECURSIVE_CUSTOM_SCRIPT: $ENABLE_RECURSIVE_CUSTOM_SCRIPT
+    ENABLE_RECURSIVE_VCS_IMPORT: $ENABLE_RECURSIVE_VCS_IMPORT
     GIT_HTTPS_PASSWORD: $GIT_HTTPS_PASSWORD
     GIT_HTTPS_SERVER: $GIT_HTTPS_SERVER
     GIT_HTTPS_USER: $GIT_HTTPS_USER