add first version of blacklisting packages in docker-ros workspace

ika-rwth-aachen · Sep 19, 2023 · c2a7ae2 · c2a7ae2
1 parent 77705e3
commit c2a7ae2
Show file tree

Hide file tree

Showing 7 changed files with 45 additions and 0 deletions.
diff --git a/.gitlab-ci/docker-ros.yml b/.gitlab-ci/docker-ros.yml
@@ -35,6 +35,8 @@ variables:
   ADDITIONAL_FILES_DIR:       docker/additional-files                 # Relative path to directory containing additional files to copy into image"
   ADDITIONAL_PIP_FILE:        docker/additional-pip-requirements.txt  # Relative filepath to file containing additional pip packages to install
   ENABLE_RECURSIVE_ADDITIONAL_PIP:   'false'                          # Enable recursive discovery of files named `additional-pip-file`
+  BLACKLISTED_PACKAGES_FILE:  docker/blacklisted-packages.txt         # Relative filepath to file containing the blacklisted packages
+  ENABLE_RECURSIVE_BLACKLISTED_PACKAGES:  'false'                     # Enable recursive discovery of files named `blacklisted-packages-file`
   CUSTOM_SCRIPT_FILE:         docker/custom.sh                        # Relative filepath to script containing custom installation commands
   ENABLE_RECURSIVE_CUSTOM_SCRIPT:    'false'                          # Enable recursive discovery of files named `custom-script-file`
   # -----

diff --git a/README.md b/README.md
@@ -298,6 +298,8 @@ If your ROS-based repository requires Python dependencies that cannot be install
 
 Create a file `additional-pip-requirements.txt` in your `docker` folder (or configure a different `ADDITIONAL_PIP_FILE`) and list any other Python dependencies that need to be installed via *pip*.
 
+### TODO: Blacklist/Whitelist packages
+
 ### Custom Installation Script
 
 If your ROS-based repository requires to execute any other installation or pre-/post-installation steps, you can use a special `custom.sh` script.
@@ -325,6 +327,9 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
 - **`additional-pip-file` | `ADDITIONAL_PIP_FILE`**  
   Relative filepath to file containing additional pip packages to install  
   *default:* `docker/additional-pip-requirements.txt`  
+- **`blacklisted-packages-file` | `BLACKLISTED_PACKAGES_FILE`**  
+  Relative filepath to file containing blacklisted packages  
+  *default:* `docker/blacklisted-packages.txt`
 - **`base-image` | `BASE_IMAGE`**  
   Base image `name:tag`  
   *required*  
@@ -370,6 +375,9 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
 - **`enable-recursive-additional-pip` | `ENABLE_RECURSIVE_ADDITIONAL_PIP`**  
   Enable recursive discovery of files named `additional-pip-file`  
   *default:* `false`
+- **`enable-recursive-blacklisted-packages` | `ENABLE_RECURSIVE_BLACKLISTED_PACKAGES`**  
+  Enable recursive discovery of files named `blacklisted-packages-file`  
+  *default:* `false`
 - **`enable-recursive-custom-script` | `ENABLE_RECURSIVE_CUSTOM_SCRIPT`**  
   Enable recursive discovery of files named `custom-script-file`  
   *default:* `false`

diff --git a/action.yml b/action.yml
@@ -88,6 +88,14 @@ inputs:
     description: "Enable recursive discovery of files named `additional-pip-file`"
     default: false
 
+  blacklisted-packages-file:
+    description: "Relative filepath to file containing blacklisted packages to remove from workspace"
+    default: docker/blacklisted-packages.txt
+
+  enable-recursive-blacklisted-packages:
+    description: "Enable recursive discovery of files named `blacklisted-packages-file`"
+    default: false
+
   custom-script-file:
     description: "Relative filepath to script containing custom installation commands"
     default: docker/custom.sh
@@ -172,6 +180,8 @@ runs:
         ADDITIONAL_FILES_DIR: ${{ inputs.additional-files-dir }}
         ADDITIONAL_PIP_FILE: ${{ inputs.additional-pip-file }}
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
+        BLACKLISTED_PACKAGES_FILE: ${{ inputs.blacklisted-packages-file }}
+        ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: ${{ inputs.enable-recursive-blacklisted-packages }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
 
@@ -225,6 +235,8 @@ runs:
         ADDITIONAL_FILES_DIR: ${{ inputs.additional-files-dir }}
         ADDITIONAL_PIP_FILE: ${{ inputs.additional-pip-file }}
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
+        BLACKLISTED_PACKAGES_FILE: ${{ inputs.blacklisted-packages-file }}
+        ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: ${{ inputs.enable-recursive-blacklisted-packages }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
         _ENABLE_IMAGE_PUSH: true
@@ -256,6 +268,8 @@ runs:
         ADDITIONAL_FILES_DIR: ${{ inputs.additional-files-dir }}
         ADDITIONAL_PIP_FILE: ${{ inputs.additional-pip-file }}
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
+        BLACKLISTED_PACKAGES_FILE: ${{ inputs.blacklisted-packages-file }}
+        ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: ${{ inputs.enable-recursive-blacklisted-packages }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
         _ENABLE_IMAGE_PUSH: true

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -62,6 +62,21 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 RUN /usr/local/bin/recursive_vcs_import.py src src/upstream
 
+# remove blacklisted packages from workspace
+ARG BLACKLISTED_PACKAGES_FILE="docker/blacklisted-packages.txt"
+ARG ENABLE_RECURSIVE_BLACKLISTED_PACKAGES="false"
+RUN echo "colcon list -p --base-paths src/ --packages-select \\" >> $WORKSPACE/.remove-packages.sh && \
+    if [[ $ENABLE_RECURSIVE_BLACKLISTED_PACKAGES == 'true' ]]; then \
+        find . -type f -name $(basename ${BLACKLISTED_PACKAGES_FILE}) -exec sed '$a\' {} \; | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.remove-packages.sh ; \
+    elif [[ -f src/target/${BLACKLISTED_PACKAGES_FILE} ]]; then \
+        cat src/target/${BLACKLISTED_PACKAGES_FILE} | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.remove-packages.sh ; \
+    fi && \
+    echo ";" >> $WORKSPACE/.remove-packages.sh && \
+    chmod +x $WORKSPACE/.remove-packages.sh && \
+    $WORKSPACE/.remove-packages.sh | xargs rm -rf
+
+# TODO: only keep whitelist packages in workspace
+
 # create install script with list of rosdep dependencies
 RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \
     source /opt/ros/$ROS_DISTRO/setup.bash && \

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -28,6 +28,8 @@ build_image() {
         $(if [[ -n "${ADDITIONAL_FILES_DIR}" ]]; then echo "--build-arg ADDITIONAL_FILES_DIR=${ADDITIONAL_FILES_DIR}"; fi) \
         $(if [[ -n "${ADDITIONAL_PIP_FILE}" ]]; then echo "--build-arg ADDITIONAL_PIP_FILE=${ADDITIONAL_PIP_FILE}"; fi) \
         $(if [[ -n "${ENABLE_RECURSIVE_ADDITIONAL_PIP}" ]]; then echo "--build-arg ENABLE_RECURSIVE_ADDITIONAL_PIP=${ENABLE_RECURSIVE_ADDITIONAL_PIP}"; fi) \
+        $(if [[ -n "${BLACKLISTED_PACKAGES_FILE}" ]]; then echo "--build-arg BLACKLISTED_PACKAGES_FILE=${BLACKLISTED_PACKAGES_FILE}"; fi) \
+        $(if [[ -n "${ENABLE_RECURSIVE_BLACKLISTED_PACKAGES}" ]]; then echo "--build-arg ENABLE_RECURSIVE_BLACKLISTED_PACKAGES=${ENABLE_RECURSIVE_BLACKLISTED_PACKAGES}"; fi) \
         $(if [[ -n "${CUSTOM_SCRIPT_FILE}" ]]; then echo "--build-arg CUSTOM_SCRIPT_FILE=${CUSTOM_SCRIPT_FILE}"; fi) \
         $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \
         .

diff --git a/scripts/ci.sh b/scripts/ci.sh
@@ -31,6 +31,8 @@ ENABLE_RECURSIVE_ADDITIONAL_DEBS="${ENABLE_RECURSIVE_ADDITIONAL_DEBS:-}"
 ADDITIONAL_FILES_DIR="${ADDITIONAL_FILES_DIR:-}"
 ADDITIONAL_PIP_FILE="${ADDITIONAL_PIP_FILE:-}"
 ENABLE_RECURSIVE_ADDITIONAL_PIP="${ENABLE_RECURSIVE_ADDITIONAL_PIP:-}"
+BLACKLISTED_PACKAGES_FILE="${BLACKLISTED_PACKAGES_FILE:-}"
+ENABLE_RECURSIVE_BLACKLISTED_PACKAGES="${ENABLE_RECURSIVE_BLACKLISTED_PACKAGES:-}"
 CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}"
 ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}"
 _ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}"

diff --git a/templates/docker-compose.template.yml b/templates/docker-compose.template.yml
@@ -14,9 +14,11 @@ x-build: &build
     ADDITIONAL_DEBS_FILE: $ADDITIONAL_DEBS_FILE
     ADDITIONAL_FILES_DIR: $ADDITIONAL_FILES_DIR
     ADDITIONAL_PIP_FILE: $ADDITIONAL_PIP_FILE
+    BLACKLISTED_PACKAGES_FILE: $BLACKLISTED_PACKAGES_FILE
     CUSTOM_SCRIPT_FILE: $CUSTOM_SCRIPT_FILE
     ENABLE_RECURSIVE_ADDITIONAL_DEBS: $ENABLE_RECURSIVE_ADDITIONAL_DEBS
     ENABLE_RECURSIVE_ADDITIONAL_PIP: $ENABLE_RECURSIVE_ADDITIONAL_PIP
+    ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: $ENABLE_RECURSIVE_BLACKLISTED_PACKAGES
     ENABLE_RECURSIVE_CUSTOM_SCRIPT: $ENABLE_RECURSIVE_CUSTOM_SCRIPT
     GIT_HTTPS_PASSWORD: $GIT_HTTPS_PASSWORD
     GIT_HTTPS_SERVER: $GIT_HTTPS_SERVER