Merge branch 'branch-24.10' into add_back_some_nb_tsts

rapidsai · Aug 27, 2024 · 773f5c1 · 773f5c1
2 parents 3e13d02 + b007a92
commit 773f5c1
Show file tree

Hide file tree

Showing 11 changed files with 178 additions and 213 deletions.
diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
@@ -39,6 +39,9 @@ on:
       RAFT_ANN_BENCH_CPU_TAG:
         required: true
         type: string
+      BUILD_RAFT_ANN_BENCH_CPU_IMAGE:
+        required: true
+        type: boolean
 
 jobs:
   build:
@@ -146,7 +149,7 @@ jobs:
             RAPIDS_VER=${{ inputs.RAPIDS_VER }}
           tags: ${{ inputs.RAFT_ANN_BENCH_DATASETS_TAG }}-${{ matrix.ARCH }}
       - name: Build RAFT ANN Benchmarks CPU image
-        if: inputs.CUDA_VER == '12.5.1' # we don't need to build CPU packages for different CUDA versions.
+        if: inputs.BUILD_RAFT_ANN_BENCH_CPU_IMAGE
         uses: docker/build-push-action@v6
         with:
           context: context

diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml
@@ -31,6 +31,25 @@ permissions:
   statuses: none
 
 jobs:
+  pr-builder:
+    if: ${{ !cancelled() && inputs.build_type == 'pull-request' }}
+    needs:
+      - checks
+      - compute-matrix
+      - build
+      - build-multiarch-manifest
+      - test
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit run --all-files
   compute-matrix:
     runs-on: ubuntu-latest
     container:
@@ -119,7 +138,7 @@ jobs:
 
           echo "TEST_MATRIX=$(yq -n -o json 'env(TEST_MATRIX)' | jq -c '{include: .}')" | tee --append "${GITHUB_OUTPUT}"
   build:
-    needs: compute-matrix
+    needs: [checks, compute-matrix]
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
       fail-fast: false
@@ -133,6 +152,7 @@ jobs:
       LINUX_VER: ${{ matrix.LINUX_VER }}
       PYTHON_VER: ${{ matrix.PYTHON_VER }}
       RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+      BUILD_RAFT_ANN_BENCH_CPU_IMAGE: ${{ matrix.BUILD_RAFT_ANN_BENCH_CPU_IMAGE }}
       BASE_TAG:
         "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
         ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\
@@ -167,8 +187,46 @@ jobs:
         ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
         ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
         py${{ matrix.PYTHON_VER }}"
-  test:
+  build-multiarch-manifest:
     needs: [build, compute-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Create multiarch manifest
+        shell: bash
+        env:
+          RAFT_ANN_BENCH_CPU_IMAGE_BUILT: ${{ matrix.BUILD_RAFT_ANN_BENCH_CPU_IMAGE }}
+          BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}
+          BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
+          RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+          ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }}
+          CUDA_TAG: ${{ matrix.CUDA_TAG }}
+          PYTHON_VER: ${{ matrix.PYTHON_VER }}
+          NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}
+          NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
+          RAFT_ANN_BENCH_IMAGE_REPO: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}
+          RAFT_ANN_BENCH_TAG_PREFIX: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}
+          RAFT_ANN_BENCH_DATASETS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}
+          RAFT_ANN_BENCH_DATASETS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}
+          RAFT_ANN_BENCH_CPU_IMAGE_REPO: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}
+          RAFT_ANN_BENCH_CPU_TAG_PREFIX: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}
+          GPUCIBOT_DOCKERHUB_USER: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+          ARCHES: ${{ toJSON(matrix.ARCHES) }}
+        run: ci/create-multiarch-manifest.sh
+  test:
+    needs: [compute-matrix, build]
     if: inputs.run_tests
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
@@ -190,184 +248,3 @@ jobs:
         cuda${{ matrix.CUDA_VER }}-\
         py${{ matrix.PYTHON_VER }}-\
         ${{ matrix.ARCH }}"
-  build-multiarch-manifest:
-    if: inputs.build_type == 'branch'
-    needs: [build, compute-matrix]
-    strategy:
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
-          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
-      - name: Create multiarch manifest
-        run: |
-          base_source_tags=()
-          notebooks_source_tags=()
-          raft_ann_bench_source_tags=()
-          raft_ann_bench_dataset_source_tags=()
-          raft_ann_bench_cpu_source_tags=()
-
-          base_tag_array=(
-            rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:
-            ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          notebooks_tag_array=(
-            rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:
-            ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          raft_ann_bench_tag_array=(
-            rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}:
-            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          raft_ann_bench_tag="$(printf %s "${raft_ann_bench_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          raft_ann_bench_datasets_tag_array=(
-            rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}:
-            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          raft_ann_bench_datasets_tag="$(printf %s "${raft_ann_bench_datasets_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          raft_ann_bench_cpu_tag_array=(
-            rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}:
-            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          raft_ann_bench_cpu_tag="$(printf %s "${raft_ann_bench_cpu_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
-            base_source_tags+=("${base_tag}-${arch}")
-            notebooks_source_tags+=("${notebooks_tag}-${arch}")
-            raft_ann_bench_source_tags+=("${raft_ann_bench_tag}-${arch}")
-            raft_ann_bench_datasets_source_tags+=("${raft_ann_bench_datasets_tag}-${arch}")
-            raft_ann_bench_cpu_source_tags+=("${raft_ann_bench_cpu_tag}-${arch}")
-          done
-
-          docker manifest create ${base_tag} ${base_source_tags[@]}
-          docker manifest push ${base_tag}
-          docker manifest create ${notebooks_tag} ${notebooks_source_tags[@]}
-          docker manifest push ${notebooks_tag}
-          docker manifest create ${raft_ann_bench_tag} ${raft_ann_bench_source_tags[@]}
-          docker manifest push ${raft_ann_bench_tag}
-          docker manifest create ${raft_ann_bench_datasets_tag} ${raft_ann_bench_datasets_source_tags[@]}
-          docker manifest push ${raft_ann_bench_datasets_tag}
-          docker manifest create ${raft_ann_bench_cpu_tag} ${raft_ann_bench_cpu_source_tags[@]}
-          docker manifest push ${raft_ann_bench_cpu_tag}
-  delete-temp-images:
-    if: always()
-    needs: [compute-matrix, build, test, build-multiarch-manifest]
-    strategy:
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Remove temporary images
-        run: |
-          HUB_TOKEN=$(
-            curl -s -H "Content-Type: application/json" \
-              -X POST \
-              -d "{\"username\": \"${{ secrets.GPUCIBOT_DOCKERHUB_USER }}\", \"password\": \"${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}\"}" \
-              https://hub.docker.com/v2/users/login/ | jq -r .token \
-          )
-          echo "::add-mask::${HUB_TOKEN}"
-
-          org="rapidsai"
-          base_repo="${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}"
-          base_tag_array=(
-            ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-
-          notebooks_repo="${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}"
-          notebooks_tag_array=(
-            ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          raft_ann_bench_repo="${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}"
-          raft_ann_bench_tag_array=(
-            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          raft_ann_bench_tag="$(printf %s "${raft_ann_bench_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          raft_ann_bench_datasets_repo="${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}"
-          raft_ann_bench_datasets_tag_array=(
-            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          raft_ann_bench_datasets_tag="$(printf %s "${raft_ann_bench_datasets_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          raft_ann_bench_cpu_repo="${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}"
-          raft_ann_bench_cpu_tag_array=(
-            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}
-            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            py${{ matrix.PYTHON_VER }}
-          )
-          raft_ann_bench_cpu_tag="$(printf %s "${raft_ann_bench_cpu_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
-
-          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
-            curl -i -X DELETE \
-              -H "Accept: application/json" \
-              -H "Authorization: JWT $HUB_TOKEN" \
-              "https://hub.docker.com/v2/repositories/$org/$base_repo/tags/$base_tag-$arch/"
-            curl -i -X DELETE \
-              -H "Accept: application/json" \
-              -H "Authorization: JWT $HUB_TOKEN" \
-              "https://hub.docker.com/v2/repositories/$org/$notebooks_repo/tags/$notebooks_tag-$arch/"
-            curl -i -X DELETE \
-              -H "Accept: application/json" \
-              -H "Authorization: JWT $HUB_TOKEN" \
-              "https://hub.docker.com/v2/repositories/$org/$raft_ann_bench_repo/tags/$raft_ann_bench_tag-$arch/"
-            curl -i -X DELETE \
-              -H "Accept: application/json" \
-              -H "Authorization: JWT $HUB_TOKEN" \
-              "https://hub.docker.com/v2/repositories/$org/$raft_ann_bench_datasets_repo/tags/$raft_ann_bench_datasets_tag-$arch/"
-            curl -i -X DELETE \
-              -H "Accept: application/json" \
-              -H "Authorization: JWT $HUB_TOKEN" \
-              "https://hub.docker.com/v2/repositories/$org/$raft_ann_bench_cpu_repo/tags/$raft_ann_bench_cpu_tag-$arch/"
-          done
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -10,26 +10,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pr-builder:
-    needs:
-      - checks
-      - docker
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
-  checks:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-      - name: Run pre-commit
-        run: |
-          pip install pre-commit
-          pre-commit run --all-files
   docker:
-    needs: [checks]
     uses: ./.github/workflows/build-test-publish-images.yml
     with:
       build_type: pull-request

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -15,3 +15,14 @@ To build just the `base` image with default arguments: `docker buildx build --pu
 - `CUDA_VER` - Version of CUDA to use. Should be `major.minor.patch`
 - `PYTHON_VER` - Version of Python to use. Should be `major.minor`
 - `RAPIDS_VER` - Version of RAPIDS to use. Should be `YY.MM`
+
+## Cleaning Up
+
+Every build first writes images to the https://hub.docker.com/r/rapidsai/staging repo on DockerHub,
+then pushes them on to the individual repos like `rapidsai/base`, `rapidsai/notebooks`, etc.
+
+A scheduled job regularly deletes old images from that `rapidsai/staging` repo.
+See https://github.com/rapidsai/workflows/blob/main/.github/workflows/cleanup_staging.yaml for details.
+
+If you come back to a pull requests here after more than a few days and find that jobs are failing with errors
+that suggest that some necessary images don't exist, re-run all of CI on that pull request to produce new images.
diff --git a/ci/compute-matrix.jq b/ci/compute-matrix.jq
@@ -14,6 +14,12 @@ def compute_ubuntu_version($x):
 def compute_cuda_tag($x):
   $x + {CUDA_TAG: $x.CUDA_VER | split(".") | [.[0], .[1]] | join(".") };
 
+def latest_cuda_version($cuda_versions):
+  $cuda_versions | max_by(. | split(".") | map(tonumber));
+
+def compute_build_raft_ann_bench_cpu_image($x; $latest_cuda_version):
+  $x + {BUILD_RAFT_ANN_BENCH_CPU_IMAGE: ($x.CUDA_VER == $latest_cuda_version)}; # we don't need to build CPU packages for different CUDA versions
+
 # Checks the current entry to see if it matches the given exclude
 def matches($entry; $exclude):
   all($exclude | to_entries | .[]; $entry[.key] == .value);
@@ -32,11 +38,13 @@ def compute_matrix($input):
   keys_unsorted as $matrix_keys |
   to_entries |
   map(.value) |
+  latest_cuda_version($input.CUDA_VER) as $latest_cuda_version |
   [
     combinations |
     lists2dict($matrix_keys; .) |
     compute_ubuntu_version(.) |
     compute_cuda_tag(.) |
+    compute_build_raft_ann_bench_cpu_image(.; $latest_cuda_version) |
     filter_excludes(.; $excludes) |
     compute_arch(.)
   ] |