diff --git a/.dockerignore b/.dockerignore index ac26bde50a1..679e11b25ef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ # Ignore Docker files docker !docker/etc +!docker/scripts # Ignore a part of files under src src/**/.* diff --git a/.github/actions/docker-build-and-push-cuda/action.yaml b/.github/actions/docker-build-and-push-cuda/action.yaml index 59f5ccc3569..fc28683d892 100644 --- a/.github/actions/docker-build-and-push-cuda/action.yaml +++ b/.github/actions/docker-build-and-push-cuda/action.yaml @@ -1,20 +1,33 @@ name: docker-build-and-push-cuda -description: Workflow do build and push CUDA images to registry. +description: Composite action to build and push CUDA images to registry. inputs: platform: - description: "" + description: Target platform. required: true - bake-target: - description: "" + target-image: + description: Target docker image name in the registry. required: true build-args: - description: "" + description: Additional build args. required: false runs: using: composite steps: + - name: Install jq and vcstool + run: | + sudo apt-get -y update + sudo apt-get -y install jq python3-pip + pip install --no-cache-dir vcstool + shell: bash + + - name: Run vcs import + run: | + mkdir src + vcs import src < autoware.repos + shell: bash + - name: Setup Docker Buildx uses: docker/setup-buildx-action@v3 @@ -51,11 +64,23 @@ runs: run: echo "date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT shell: bash + - name: Docker meta for autoware:base-cuda + id: meta-base-cuda + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} + tags: | + type=raw,value=base-cuda-${{ inputs.platform }} + type=raw,value=base-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }} + bake-target: docker-metadata-action-base-cuda + flavor: | + latest=false + - name: Docker meta for autoware:universe-sensing-perception-devel-cuda id: meta-universe-sensing-perception-devel-cuda uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-sensing-perception-devel-cuda-${{ inputs.platform }} type=raw,value=universe-sensing-perception-devel-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -67,7 +92,7 @@ runs: id: meta-universe-sensing-perception-cuda uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-sensing-perception-cuda-${{ inputs.platform }} type=raw,value=universe-sensing-perception-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -79,7 +104,7 @@ runs: id: meta-universe-devel-cuda uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-devel-cuda-${{ inputs.platform }} type=raw,value=universe-devel-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -91,7 +116,7 @@ runs: id: meta-universe-cuda uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-cuda-${{ inputs.platform }} type=raw,value=universe-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -112,6 +137,7 @@ runs: push: true files: | docker/docker-bake-cuda.hcl + ${{ steps.meta-base-cuda.outputs.bake-file }} ${{ steps.meta-universe-sensing-perception-devel-cuda.outputs.bake-file }} ${{ steps.meta-universe-sensing-perception-cuda.outputs.bake-file }} ${{ steps.meta-universe-devel-cuda.outputs.bake-file }} diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 0a064ee1acf..3e3bc857fea 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -1,15 +1,15 @@ name: docker-build-and-push -description: "" +description: Composite action to build and push non CUDA images to registry. inputs: platform: - description: "" + description: Target platform. required: true - bake-target: - description: "" + target-image: + description: Target docker image name in the registry. required: true build-args: - description: "" + description: Additional build args. required: false runs: @@ -68,7 +68,7 @@ runs: id: meta-base uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=base-${{ inputs.platform }} type=raw,value=base-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -80,7 +80,7 @@ runs: id: meta-core-devel uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=core-devel-${{ inputs.platform }} type=raw,value=core-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -92,7 +92,7 @@ runs: id: meta-universe-sensing-perception-devel uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-sensing-perception-devel-${{ inputs.platform }} type=raw,value=universe-sensing-perception-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -104,7 +104,7 @@ runs: id: meta-universe-sensing-perception uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-sensing-perception-${{ inputs.platform }} type=raw,value=universe-sensing-perception-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -116,7 +116,7 @@ runs: id: meta-universe-localization-mapping-devel uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-localization-mapping-devel-${{ inputs.platform }} type=raw,value=universe-localization-mapping-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -128,7 +128,7 @@ runs: id: meta-universe-localization-mapping uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-localization-mapping-${{ inputs.platform }} type=raw,value=universe-localization-mapping-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -140,7 +140,7 @@ runs: id: meta-universe-planning-control-devel uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-planning-control-devel-${{ inputs.platform }} type=raw,value=universe-planning-control-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -152,7 +152,7 @@ runs: id: meta-universe-planning-control uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-planning-control-${{ inputs.platform }} type=raw,value=universe-planning-control-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -164,7 +164,7 @@ runs: id: meta-universe-vehicle-system-devel uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-vehicle-system-devel-${{ inputs.platform }} type=raw,value=universe-vehicle-system-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -176,7 +176,7 @@ runs: id: meta-universe-vehicle-system uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-vehicle-system-${{ inputs.platform }} type=raw,value=universe-vehicle-system-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -188,7 +188,7 @@ runs: id: meta-universe-devel uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-devel-${{ inputs.platform }} type=raw,value=universe-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }} @@ -200,7 +200,7 @@ runs: id: meta-universe uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} tags: | type=raw,value=universe-${{ inputs.platform }} type=raw,value=universe-${{ steps.date.outputs.date }}-${{ inputs.platform }} diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index bfb6383c208..9a6a6e7e140 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -1,16 +1,16 @@ name: docker-build -description: "" +description: Composite action to build images only. inputs: platform: - description: "" + description: Target platform. required: true cache-tag-suffix: - description: "" + description: Suffix of the target cache tag. required: true build-args: - description: "" - required: true + description: Additional build args. + required: false runs: using: composite diff --git a/.github/workflows/docker-build-and-push-arm64.yaml b/.github/workflows/docker-build-and-push-arm64.yaml index 46d3280d6d3..e6da14db8b6 100644 --- a/.github/workflows/docker-build-and-push-arm64.yaml +++ b/.github/workflows/docker-build-and-push-arm64.yaml @@ -57,7 +57,7 @@ jobs: uses: ./.github/actions/docker-build-and-push with: platform: arm64 - bake-target: autoware + target-image: autoware build-args: | *.platform=linux/arm64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} @@ -67,6 +67,36 @@ jobs: *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:arm64-main *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:arm64-${{ github.ref_name }},mode=max + - name: Show disk space + if: always() + run: | + df -h + + docker-build-and-push-cuda: + needs: [load-env, docker-build-and-push] + runs-on: buildjet-16vcpu-ubuntu-2204-arm + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + *.env + *.repos + .github/actions/docker-build-and-push/action.yaml + .github/workflows/docker-build-and-push*.yaml + ansible-galaxy-requirements.yaml + ansible/** + docker/** + - name: Build 'Autoware' with CUDA if: ${{ steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' || @@ -74,7 +104,7 @@ jobs: uses: ./.github/actions/docker-build-and-push-cuda with: platform: arm64 - bake-target: autoware + target-image: autoware build-args: | *.platform=linux/arm64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml index ccb5f123326..b4fcf4787e0 100644 --- a/.github/workflows/docker-build-and-push.yaml +++ b/.github/workflows/docker-build-and-push.yaml @@ -57,7 +57,7 @@ jobs: uses: ./.github/actions/docker-build-and-push with: platform: amd64 - bake-target: autoware + target-image: autoware build-args: | *.platform=linux/amd64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} @@ -67,6 +67,36 @@ jobs: *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:amd64-main *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:amd64-${{ github.ref_name }},mode=max + - name: Show disk space + if: always() + run: | + df -h + + docker-build-and-push-cuda: + needs: [load-env, docker-build-and-push] + runs-on: buildjet-16vcpu-ubuntu-2204 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + *.env + *.repos + .github/actions/docker-build-and-push/action.yaml + .github/workflows/docker-build-and-push*.yaml + ansible-galaxy-requirements.yaml + ansible/** + docker/** + - name: Build 'Autoware' with CUDA if: ${{ steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' || @@ -74,7 +104,7 @@ jobs: uses: ./.github/actions/docker-build-and-push-cuda with: platform: amd64 - bake-target: autoware + target-image: autoware build-args: | *.platform=linux/amd64 *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} diff --git a/ansible/roles/artifacts/tasks/main.yaml b/ansible/roles/artifacts/tasks/main.yaml index 02825a0d041..c4f7ede0089 100644 --- a/ansible/roles/artifacts/tasks/main.yaml +++ b/ansible/roles/artifacts/tasks/main.yaml @@ -402,29 +402,3 @@ dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_labels.txt" mode: "644" checksum: sha256:a2a91f5fe9c2e68e3e3647a272bb9bb25cd07631a1990a3fb15efddce7691131 - -# tvm_utility -- name: Create tvm_utility/models directory inside {{ data_dir }} - ansible.builtin.file: - path: "{{ data_dir }}/tvm_utility/models" - mode: "755" - state: directory - -- name: Download yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz - become: true - ansible.builtin.get_url: - url: https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/3.0.0-20221221/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz - dest: "{{ data_dir }}/tvm_utility/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz" - mode: "644" - checksum: sha256:66b3ca668e577393b657fbe1ed626538d89ca3adccd5862de6c7fa190238dbca - -- name: Create yolo_v2_tiny folder in tvm_utility/models of {{ data_dir }} - ansible.builtin.file: - path: "{{ data_dir }}/tvm_utility/models/yolo_v2_tiny" - mode: "755" - state: directory - -- name: Extract yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz - ansible.builtin.unarchive: - src: "{{ data_dir }}/tvm_utility/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz" - dest: "{{ data_dir }}/tvm_utility/models/yolo_v2_tiny" diff --git a/docker/Dockerfile b/docker/Dockerfile index 19bb14cf97d..784351331b3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,6 +19,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ # Copy files COPY setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/ COPY ansible/ /autoware/ansible/ +COPY docker/scripts/cleanup_system.sh /autoware/cleanup_system.sh +RUN chmod +x /autoware/cleanup_system.sh WORKDIR /autoware # Set up base environment @@ -35,6 +37,16 @@ RUN chmod +x /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] +FROM base AS base-cuda +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Set up CUDA runtime environment and artifacts +# hadolint ignore=SC2002 +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y --module base --download-artifacts --no-cuda-drivers --runtime openadkit \ + && pip uninstall -y ansible ansible-core \ + && apt-get autoremove -y && rm -rf "$HOME"/.cache + # hadolint ignore=DL3006 FROM $BASE_IMAGE AS rosdep-depend SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -42,6 +54,8 @@ ARG ROS_DISTRO COPY setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/ COPY ansible/ /autoware/ansible/ +COPY docker/scripts/resolve_rosdep_keys.sh /autoware/resolve_rosdep_keys.sh +RUN chmod +x /autoware/resolve_rosdep_keys.sh WORKDIR /autoware RUN rm -f /etc/apt/apt.conf.d/docker-clean \ @@ -54,21 +68,13 @@ RUN --mount=type=ssh \ # Generate install package lists COPY src/core /autoware/src/core -RUN rosdep update && rosdep keys --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN rosdep update && /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-core-depend-packages.txt \ && cat /rosdep-core-depend-packages.txt COPY src/universe/external /autoware/src/universe/external COPY src/universe/autoware.universe/common /autoware/src/universe/autoware.universe/common -RUN rosdep keys --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-common-depend-packages.txt \ && cat /rosdep-universe-common-depend-packages.txt @@ -78,18 +84,10 @@ ARG ROS_DISTRO COPY src/universe/autoware.universe/perception /autoware/src/universe/autoware.universe/perception COPY src/universe/autoware.universe/sensing /autoware/src/universe/autoware.universe/sensing -RUN rosdep keys --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-sensing-perception-depend-packages.txt \ && cat /rosdep-universe-sensing-perception-depend-packages.txt -RUN rosdep keys --dependency-types=exec --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-sensing-perception-exec-depend-packages.txt \ && cat /rosdep-universe-sensing-perception-exec-depend-packages.txt @@ -99,18 +97,10 @@ ARG ROS_DISTRO COPY src/universe/autoware.universe/localization /autoware/src/universe/autoware.universe/localization COPY src/universe/autoware.universe/map /autoware/src/universe/autoware.universe/map -RUN rosdep keys --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-localization-mapping-depend-packages.txt \ && cat /rosdep-universe-localization-mapping-depend-packages.txt -RUN rosdep keys --dependency-types=exec --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-localization-mapping-exec-depend-packages.txt \ && cat /rosdep-universe-localization-mapping-exec-depend-packages.txt @@ -125,18 +115,10 @@ COPY src/universe/autoware.universe/map/map_loader /autoware/src/universe/autowa COPY src/universe/autoware.universe/map/autoware_map_projection_loader /autoware/src/universe/autoware.universe/map/autoware_map_projection_loader COPY src/universe/autoware.universe/sensing/autoware_pcl_extensions /autoware/src/universe/autoware.universe/sensing/autoware_pcl_extensions COPY src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor /autoware/src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor -RUN rosdep keys --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-planning-control-depend-packages.txt \ && cat /rosdep-universe-planning-control-depend-packages.txt -RUN rosdep keys --dependency-types=exec --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-planning-control-exec-depend-packages.txt \ && cat /rosdep-universe-planning-control-exec-depend-packages.txt @@ -148,18 +130,10 @@ COPY src/universe/autoware.universe/vehicle /autoware/src/universe/autoware.univ COPY src/universe/autoware.universe/system /autoware/src/universe/autoware.universe/system COPY src/universe/autoware.universe/map/autoware_map_height_fitter /autoware/src/universe/autoware.universe/map/autoware_map_height_fitter COPY src/universe/autoware.universe/localization/autoware_pose2twist /autoware/src/universe/autoware.universe/localization/autoware_pose2twist -RUN rosdep keys --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-vehicle-system-depend-packages.txt \ && cat /rosdep-universe-vehicle-system-depend-packages.txt -RUN rosdep keys --dependency-types=exec --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-vehicle-system-exec-depend-packages.txt \ && cat /rosdep-universe-vehicle-system-exec-depend-packages.txt @@ -173,19 +147,11 @@ COPY src/sensor_component /autoware/src/sensor_component COPY src/sensor_kit /autoware/src/sensor_kit COPY src/universe /autoware/src/universe COPY src/vehicle /autoware/src/vehicle -RUN rosdep keys --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-universe-depend-packages.txt \ && cat /rosdep-universe-depend-packages.txt -RUN rosdep keys --dependency-types=exec --ignore-src --from-paths src \ - | xargs rosdep resolve --rosdistro ${ROS_DISTRO} \ - | grep -v '^#' \ - | sed 's/ \+/\n/g'\ - | sort \ +RUN /autoware/resolve_rosdep_keys.sh /autoware/src ${ROS_DISTRO} \ > /rosdep-exec-depend-packages.txt \ && cat /rosdep-exec-depend-packages.txt @@ -194,6 +160,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ENV CCACHE_DIR="/root/.ccache" +COPY docker/scripts/build_and_clean.sh /autoware/build_and_clean.sh +RUN chmod +x /autoware/build_and_clean.sh + # Set up development environment and tools RUN --mount=type=ssh \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ @@ -213,15 +182,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-depend,source=/autoware/src/core,target=/autoware/src/core \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build /autoware/log + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -245,15 +206,20 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-depend,source=/autoware/src/universe/external,target=/autoware/src/universe/external \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && source /opt/autoware/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build /autoware/log + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware + +ENTRYPOINT ["/ros_entrypoint.sh"] +CMD ["/bin/bash"] + +FROM universe-common-devel AS universe-common-devel-cuda +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Set up CUDA development environment +# hadolint ignore=SC2002 +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y --module all --no-cuda-drivers --runtime openadkit \ + && pip uninstall -y ansible ansible-core \ + && apt-get autoremove -y && rm -rf "$HOME"/.cache ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -277,30 +243,23 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-universe-sensing-perception-depend,source=/autoware/src/universe/autoware.universe/sensing,target=/autoware/src/universe/autoware.universe/sensing \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && source /opt/autoware/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build /autoware/log + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] -FROM universe-sensing-perception-devel AS universe-sensing-perception-devel-cuda +FROM universe-common-devel-cuda AS universe-sensing-perception-devel-cuda SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ENV CCACHE_DIR="/root/.ccache" -# Set up CUDA development environment +# Install rosdep dependencies +COPY --from=rosdep-universe-sensing-perception-depend /rosdep-universe-sensing-perception-depend-packages.txt /tmp/rosdep-universe-sensing-perception-depend-packages.txt +# hadolint ignore=SC2002 RUN --mount=type=ssh \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - ./setup-dev-env.sh -y --module all --no-cuda-drivers openadkit \ - && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache + apt-get update \ + && cat /tmp/rosdep-universe-sensing-perception-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ @@ -318,15 +277,9 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-universe-sensing-perception-depend,source=/autoware/src/universe/autoware.universe/sensing/autoware_cuda_utils,target=/autoware/src/universe/autoware.universe/sensing/autoware_cuda_utils \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && source /opt/autoware/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware + +COPY --from=universe-sensing-perception-devel /opt/autoware /opt/autoware ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -350,15 +303,7 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-universe-localization-mapping-depend,source=/autoware/src/universe/autoware.universe/map,target=/autoware/src/universe/autoware.universe/map \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && source /opt/autoware/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build /autoware/log + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware FROM universe-common-devel AS universe-planning-control-devel SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -384,15 +329,7 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor,target=/autoware/src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && source /opt/autoware/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build /autoware/log + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -413,20 +350,12 @@ RUN --mount=type=ssh \ # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-universe-vehicle-system-depend,source=/autoware/src/universe/autoware.universe/vehicle,target=/autoware/src/universe/autoware.universe/vehicle \ - --mount=type=bind,from=rosdep-universe-vehicle-system-depend,source=/autoware/src/universe/autoware.universe/system,target=/autoware/src/universe/autoware.universe/system \ + --mount=type=bind,from=rosdep-universe-vehicle-system-depend,source=/autoware/src/universe/autoware.universe/system,target=/autoware/xsrc/universe/autoware.universe/system \ --mount=type=bind,from=rosdep-universe-vehicle-system-depend,source=/autoware/src/universe/autoware.universe/map/autoware_map_height_fitter,target=/autoware/src/universe/autoware.universe/map/autoware_map_height_fitter \ --mount=type=bind,from=rosdep-universe-vehicle-system-depend,source=/autoware/src/universe/autoware.universe/localization/autoware_pose2twist,target=/autoware/src/universe/autoware.universe/localization/autoware_pose2twist \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && source /opt/autoware/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build /autoware/log + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -461,30 +390,24 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-universe-depend,source=/autoware/src/vehicle,target=/autoware/src/vehicle \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && source /opt/autoware/setup.bash \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && colcon build --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --merge-install \ - --install-base /opt/autoware \ - --mixin release compile-commands ccache \ - && du -sh ${CCACHE_DIR} && ccache -s \ - && rm -rf /autoware/build /autoware/log + && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] -FROM universe-devel AS universe-devel-cuda +FROM universe-common-devel-cuda AS universe-devel-cuda SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# Set up CUDA development environment +# Install rosdep dependencies +COPY --from=rosdep-universe-depend /rosdep-universe-depend-packages.txt /tmp/rosdep-universe-depend-packages.txt +# hadolint ignore=SC2002 RUN --mount=type=ssh \ - --mount=type=cache,target=/var/cache/apt,sharing=locked \ - ./setup-dev-env.sh -y --module all --no-cuda-drivers openadkit \ - && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache + apt-get update \ + && cat /tmp/rosdep-universe-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache COPY --from=universe-sensing-perception-devel-cuda /opt/autoware /opt/autoware +COPY --from=universe-devel /opt/autoware /opt/autoware ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -503,14 +426,7 @@ RUN --mount=type=ssh \ && pip uninstall -y ansible ansible-core \ && apt-get update \ && cat /tmp/rosdep-universe-sensing-perception-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ - && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ - && find / -name "*.o" -type f -delete \ - && find / -name "*.h" -type f -delete \ - && find / -name "*.hpp" -type f -delete \ - && rm -rf /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ - /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ - /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + && /autoware/cleanup_system.sh $LIB_DIR $ROS_DISTRO COPY --from=universe-sensing-perception-devel /opt/autoware /opt/autoware @@ -521,26 +437,21 @@ RUN echo "source /opt/autoware/setup.bash" > /etc/bash.bashrc ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] -FROM universe-sensing-perception AS universe-sensing-perception-cuda +FROM base-cuda AS universe-sensing-perception-cuda SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR -# Set up CUDA runtime environment and artifacts +# Set up runtime environment +COPY --from=rosdep-universe-sensing-perception-depend /rosdep-universe-sensing-perception-exec-depend-packages.txt /tmp/rosdep-universe-sensing-perception-exec-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ - ./setup-dev-env.sh -y --module all --download-artifacts --no-cuda-drivers --runtime openadkit \ + ./setup-dev-env.sh -y --module all --no-nvidia --no-cuda-drivers --runtime openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ - && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ - && find / -name "*.o" -type f -delete \ - && find / -name "*.h" -type f -delete \ - && find / -name "*.hpp" -type f -delete \ - && rm -rf /autoware/ansible /autoware/ansible-galaxy-requirements.yaml /autoware/setup-dev-env.sh /autoware/*.env \ - /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ - /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ - /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + && apt-get update \ + && cat /tmp/rosdep-universe-sensing-perception-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ + && /autoware/cleanup_system.sh $LIB_DIR $ROS_DISTRO COPY --from=universe-sensing-perception-devel-cuda /opt/autoware /opt/autoware @@ -561,15 +472,7 @@ RUN --mount=type=ssh \ && pip uninstall -y ansible ansible-core \ && apt-get update \ && cat /tmp/rosdep-universe-localization-mapping-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ - && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ - && find / -name "*.o" -type f -delete \ - && find / -name "*.h" -type f -delete \ - && find / -name "*.hpp" -type f -delete \ - && rm -rf /autoware/ansible /autoware/ansible-galaxy-requirements.yaml /autoware/setup-dev-env.sh /autoware/*.env \ - /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ - /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ - /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + && /autoware/cleanup_system.sh $LIB_DIR $ROS_DISTRO COPY --from=universe-localization-mapping-devel /opt/autoware /opt/autoware @@ -594,15 +497,7 @@ RUN --mount=type=ssh \ && pip uninstall -y ansible ansible-core \ && apt-get update \ && cat /tmp/rosdep-universe-planning-control-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ - && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ - && find / -name "*.o" -type f -delete \ - && find / -name "*.h" -type f -delete \ - && find / -name "*.hpp" -type f -delete \ - && rm -rf /autoware/ansible /autoware/ansible-galaxy-requirements.yaml /autoware/setup-dev-env.sh /autoware/*.env \ - /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ - /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ - /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + && /autoware/cleanup_system.sh $LIB_DIR $ROS_DISTRO COPY --from=universe-planning-control-devel /opt/autoware /opt/autoware @@ -627,15 +522,7 @@ RUN --mount=type=ssh \ && pip uninstall -y ansible ansible-core \ && apt-get update \ && cat /tmp/rosdep-universe-vehicle-system-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ - && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ - && find / -name "*.o" -type f -delete \ - && find / -name "*.h" -type f -delete \ - && find / -name "*.hpp" -type f -delete \ - && rm -rf /autoware/ansible /autoware/ansible-galaxy-requirements.yaml /autoware/setup-dev-env.sh /autoware/*.env \ - /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ - /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ - /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + && /autoware/cleanup_system.sh $LIB_DIR $ROS_DISTRO COPY --from=universe-vehicle-system-devel /opt/autoware /opt/autoware @@ -660,14 +547,7 @@ RUN --mount=type=ssh \ && pip uninstall -y ansible ansible-core \ && apt-get update \ && cat /tmp/rosdep-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ - && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ - && find / -name "*.o" -type f -delete \ - && find / -name "*.h" -type f -delete \ - && find / -name "*.hpp" -type f -delete \ - && rm -rf /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ - /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ - /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + && /autoware/cleanup_system.sh $LIB_DIR $ROS_DISTRO COPY --from=universe-devel /opt/autoware /opt/autoware @@ -678,26 +558,21 @@ RUN echo "source /opt/autoware/setup.bash" > /etc/bash.bashrc ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] -FROM universe AS universe-cuda +FROM base-cuda AS universe-cuda SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR -# Set up CUDA runtime environment and artifacts +# Set up runtime environment +COPY --from=rosdep-universe-depend /rosdep-exec-depend-packages.txt /tmp/rosdep-exec-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ - ./setup-dev-env.sh -y --module all --download-artifacts --no-cuda-drivers --runtime openadkit \ + ./setup-dev-env.sh -y --module all --no-nvidia --no-cuda-drivers --runtime openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ - && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ - && find / -name "*.o" -type f -delete \ - && find / -name "*.h" -type f -delete \ - && find / -name "*.hpp" -type f -delete \ - && rm -rf /autoware/ansible /autoware/ansible-galaxy-requirements.yaml /autoware/setup-dev-env.sh /autoware/*.env \ - /root/.local/pipx /opt/ros/"$ROS_DISTRO"/include /etc/apt/sources.list.d/cuda*.list \ - /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ - /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* + && apt-get update \ + && cat /tmp/rosdep-exec-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ + && /autoware/cleanup_system.sh $LIB_DIR $ROS_DISTRO COPY --from=universe-devel-cuda /opt/autoware /opt/autoware diff --git a/docker/Dockerfile.svg b/docker/Dockerfile.svg index 5d983ab1436..7ad00076c97 100644 --- a/docker/Dockerfile.svg +++ b/docker/Dockerfile.svg @@ -4,482 +4,538 @@ - - + + G - + cluster_legend - + key -FROM ...  -COPY --from=...  -RUN --mount=(.*)from=...  +FROM ...  +COPY --from=...  +RUN --mount=(.*)from=...  key2 -  -  -  +  +  +  key:e->key2:w - - + + key:e->key2:w - - + + key:e->key2:w - - + + external_image_0 - -$BASE_IMAGE + +$BASE_IMAGE stage_0 - -base + +base external_image_0->stage_0 - - + + - + -stage_1 - -rosdep-depend +stage_2 + +rosdep-depend - + -external_image_0->stage_1 - - +external_image_0->stage_2 + + - + -stage_7 - -core-devel +stage_1 + +base-cuda - + -stage_0->stage_7 - - +stage_0->stage_1 + + - + -stage_16 - -universe-sensing-perception +stage_8 + +core-devel - + -stage_0->stage_16 - - +stage_0->stage_8 + + stage_18 - -universe-localization-mapping + +universe-sensing-perception stage_0->stage_18 - - - - - -stage_19 - -universe-planning-control - - - -stage_0->stage_19 - - + + + - + stage_20 - -universe-vehicle-system + +universe-localization-mapping - + stage_0->stage_20 - - + - + stage_21 - -universe + +universe-planning-control - + stage_0->stage_21 - - - - - + + - - -stage_1->stage_7 - - + + +stage_22 + +universe-vehicle-system - + + +stage_0->stage_22 + + + + -stage_2 - -rosdep-universe-sensing-perception-depend +stage_23 + +universe - + -stage_1->stage_2 - - +stage_0->stage_23 + + + + + + +stage_2->stage_8 + + - + stage_3 - -rosdep-universe-localization-mapping-depend + +rosdep-universe-sensing-perception-depend - - -stage_1->stage_3 - - - + + +stage_2->stage_3 + + - + stage_4 - -rosdep-universe-planning-control-depend + +rosdep-universe-localization-mapping-depend - - -stage_1->stage_4 - - + + +stage_2->stage_4 + + - + stage_5 - -rosdep-universe-vehicle-system-depend + +rosdep-universe-planning-control-depend - - -stage_1->stage_5 - - + + +stage_2->stage_5 + + - + stage_6 - -rosdep-universe-depend + +rosdep-universe-vehicle-system-depend - - -stage_1->stage_6 - - + + +stage_2->stage_6 + + - - -stage_8 - -universe-common-devel + + +stage_7 + +rosdep-universe-depend - + -stage_1->stage_8 - - - - - -stage_7->stage_8 - - - - - -stage_17 - -universe-sensing-perception-cuda - - - -stage_16->stage_17 - - - - - -stage_22 - -universe-cuda - - - -stage_21->stage_22 - - - - - -stage_2->stage_16 - - +stage_2->stage_7 + + + - + stage_9 - -universe-sensing-perception-devel + +universe-common-devel stage_2->stage_9 - - + + - - -stage_10 - -universe-sensing-perception-devel-cuda + + +stage_19 + +universe-sensing-perception-cuda - - -stage_2->stage_10 - - - + + +stage_1->stage_19 + + + + + +stage_24 + +universe-cuda + + + +stage_1->stage_24 + + + + + +stage_8->stage_9 + + stage_3->stage_18 - - + + + + + +stage_3->stage_19 + + - + stage_11 - -universe-localization-mapping-devel + +universe-sensing-perception-devel - + stage_3->stage_11 - - - - - -stage_4->stage_19 - - + + - + stage_12 - -universe-planning-control-devel - - - -stage_4->stage_12 - - - - - -stage_5->stage_20 - - + +universe-sensing-perception-devel-cuda + + + +stage_3->stage_12 + + + + + + +stage_4->stage_20 + + + - + stage_13 - -universe-vehicle-system-devel + +universe-localization-mapping-devel - - -stage_5->stage_13 - - + + +stage_4->stage_13 + + - - -stage_6->stage_21 - + + +stage_5->stage_21 + + - + stage_14 - -universe-devel + +universe-planning-control-devel + + + +stage_5->stage_14 + + + + + +stage_6->stage_22 + + + - - -stage_6->stage_14 - - + + +stage_15 + +universe-vehicle-system-devel - - -stage_8->stage_9 - - - - - -stage_8->stage_11 - - - - - -stage_8->stage_12 - - - - - -stage_8->stage_13 - - + + +stage_6->stage_15 + + - + -stage_8->stage_14 - - +stage_7->stage_23 + + - - -stage_9->stage_16 - - - - + -stage_9->stage_10 - - +stage_7->stage_24 + + + - - -stage_9->stage_14 - - + + +stage_16 + +universe-devel - - -stage_15 - -universe-devel-cuda + + +stage_7->stage_16 + + - + + +stage_17 + +universe-devel-cuda + + + +stage_7->stage_17 + + + + -stage_10->stage_15 - - +stage_9->stage_11 + + - + -stage_10->stage_17 - - +stage_9->stage_13 + + - + + +stage_9->stage_14 + + + + +stage_9->stage_15 + + + + + +stage_9->stage_16 + + + + + +stage_10 + +universe-common-devel-cuda + + + +stage_9->stage_10 + + + + + stage_11->stage_18 - - + + - - -stage_11->stage_14 - - + + +stage_11->stage_12 + + + + + +stage_11->stage_16 + + - + stage_12->stage_19 - - + + - - -stage_12->stage_14 - - + + +stage_12->stage_17 + + - + stage_13->stage_20 - - + + - - -stage_13->stage_14 - - + + +stage_13->stage_16 + + - + stage_14->stage_21 - - + + - - -stage_14->stage_15 - - + + +stage_14->stage_16 + + - + stage_15->stage_22 - - + + + + + +stage_15->stage_16 + + + + + +stage_16->stage_23 + + + + + +stage_16->stage_17 + + + + + +stage_17->stage_24 + + + + + +stage_10->stage_12 + + + + + +stage_10->stage_17 + + diff --git a/docker/docker-bake-cuda.hcl b/docker/docker-bake-cuda.hcl index 87aa27df879..44ec1142178 100644 --- a/docker/docker-bake-cuda.hcl +++ b/docker/docker-bake-cuda.hcl @@ -1,5 +1,6 @@ group "default" { targets = [ + "base-cuda", "universe-sensing-perception-devel-cuda", "universe-sensing-perception-cuda", "universe-devel-cuda", @@ -8,11 +9,18 @@ group "default" { } // For docker/metadata-action +target "docker-metadata-action-base-cuda" {} target "docker-metadata-action-universe-sensing-perception-devel-cuda" {} target "docker-metadata-action-universe-sensing-perception-cuda" {} target "docker-metadata-action-universe-devel-cuda" {} target "docker-metadata-action-universe-cuda" {} +target "base-cuda" { + inherits = ["docker-metadata-action-base-cuda"] + dockerfile = "docker/Dockerfile" + target = "base-cuda" +} + target "universe-sensing-perception-devel-cuda" { inherits = ["docker-metadata-action-universe-sensing-perception-devel-cuda"] dockerfile = "docker/Dockerfile" diff --git a/docker/scripts/build_and_clean.sh b/docker/scripts/build_and_clean.sh new file mode 100755 index 00000000000..53c968ef722 --- /dev/null +++ b/docker/scripts/build_and_clean.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +function build_and_clean() { + local ccache_dir=$1 + local install_base=$2 + + du -sh "$ccache_dir" && ccache -s && + colcon build --cmake-args \ + " -Wno-dev" \ + " --no-warn-unused-cli" \ + --merge-install \ + --install-base "$install_base" \ + --mixin release compile-commands ccache && + du -sh "$ccache_dir" && ccache -s && + rm -rf /autoware/build /autoware/log +} + +build_and_clean "$@" diff --git a/docker/scripts/cleanup_system.sh b/docker/scripts/cleanup_system.sh new file mode 100755 index 00000000000..1e8d911becc --- /dev/null +++ b/docker/scripts/cleanup_system.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +function cleanup_system() { + local lib_dir=$1 + local ros_distro=$2 + + apt-get autoremove -y && rm -rf "$HOME"/.cache && + find /usr/lib/"$lib_dir"-linux-gnu -name "*.a" -type f -delete && + find / -name "*.o" -type f -delete && + find / -name "*.h" -type f -delete && + find / -name "*.hpp" -type f -delete && + rm -rf /autoware/ansible /autoware/ansible-galaxy-requirements.yaml /autoware/setup-dev-env.sh /autoware/*.env \ + /root/.local/pipx /opt/ros/"$ros_distro"/include /etc/apt/sources.list.d/cuda*.list \ + /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/nvidia-docker.list \ + /usr/include /usr/share/doc /usr/lib/gcc /usr/lib/jvm /usr/lib/llvm* +} + +cleanup_system "$@" diff --git a/docker/scripts/resolve_rosdep_keys.sh b/docker/scripts/resolve_rosdep_keys.sh new file mode 100755 index 00000000000..1615e9f68fe --- /dev/null +++ b/docker/scripts/resolve_rosdep_keys.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +function resolve_rosdep_keys() { + local src_path=$1 + local ros_distro=$2 + + rosdep keys --ignore-src --from-paths "$src_path" | + xargs rosdep resolve --rosdistro "$ros_distro" | + grep -v '^#' | + sed 's/ \+/\n/g' | + sort +} + +resolve_rosdep_keys "$@"