From 1eb989536d9f00ca3e3c7ee01ec778959052b7ea Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 5 Feb 2025 12:58:56 -0500 Subject: [PATCH] Use `rapids-pip-retry` in CI jobs that might need retries (#1814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uses a retry wrapper for `pip` commands to try to alleviate CI failures due to hash mismatches that result from network hiccups xref rapidsai/build-planning#148 This will retry failures that show up in CI like: ``` Collecting nvidia-cublas-cu12 (from libraft-cu12==25.2.*,>=0.0.0a0) Downloading https://pypi.nvidia.com/nvidia-cublas-cu12/nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_aarch64.whl (604.9 MB) ━━━━━━━━━━━━━━━━━━━━━ 350.2/604.9 MB 229.2 MB/s eta 0:00:02 ERROR: THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE. If you have updated the package versions, please update the hashes. Otherwise, examine the package contents carefully; someone may have tampered with them. nvidia-cublas-cu12 from https://pypi.nvidia.com/nvidia-cublas-cu12/nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_aarch64.whl#sha256=93a4e0e386cc7f6e56c822531396de8170ed17068a1e18f987574895044cd8c3 (from libraft-cu12==25.2.*,>=0.0.0a0): Expected sha256 93a4e0e386cc7f6e56c822531396de8170ed17068a1e18f987574895044cd8c3 Got 849c88d155cb4b4a3fdfebff9270fb367c58370b4243a2bdbcb1b9e7e940b7be ``` Authors: - Gil Forsyth (https://github.com/gforsyth) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/rmm/pull/1814 --- .github/workflows/build.yaml | 16 ++++++++-------- .github/workflows/pr.yaml | 24 ++++++++++++------------ .github/workflows/test.yaml | 6 +++--- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel.sh | 2 +- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 786597402..b5a88c995 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -56,7 +56,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -68,7 +68,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build-cpp: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} @@ -79,7 +79,7 @@ jobs: wheel-build-python: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-publish-cpp: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: wheel-publish-python: needs: wheel-build-python secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 35a7228ae..699efc636 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,7 +26,7 @@ jobs: - devcontainer - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@nvks-runners if: always() with: needs: ${{ toJSON(needs) }} @@ -57,7 +57,7 @@ jobs: needs: - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@nvks-runners with: files_yaml: | test_cpp: @@ -79,40 +79,40 @@ jobs: secrets: inherit needs: - telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@nvks-runners with: enable_check_generated_files: false ignored_pr_jobs: telemetry-summarize conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@nvks-runners with: build_type: pull-request conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners with: build_type: pull-request conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -122,7 +122,7 @@ jobs: wheel-build-cpp: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request @@ -130,14 +130,14 @@ jobs: wheel-build-python: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: build_type: pull-request script: ci/build_wheel_python.sh wheel-tests: needs: [wheel-build-python, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@nvks-runners if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request @@ -146,7 +146,7 @@ jobs: secrets: inherit needs: - telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@nvks-runners with: arch: '["amd64"]' cuda: '["12.8"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d3dd9d2e..241680b99 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,7 +19,7 @@ on: jobs: cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -27,7 +27,7 @@ jobs: sha: ${{ inputs.sha }} python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -35,7 +35,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@use-nightly-matrix-for-branch-tests + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@nvks-runners with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index a05a44b61..9396e9cc5 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -16,7 +16,7 @@ cd "${package_dir}" sccache --zero-stats -python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check +rapids-pip-retry wheel . -w dist -v --no-deps --disable-pip-version-check sccache --show-adv-stats diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index c2aef4a1a..c61397fe2 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -25,7 +25,7 @@ echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${CPP_WHEELHOUSE}"/librmm sccache --zero-stats PIP_CONSTRAINT="${PWD}/build-constraints.txt" \ - python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check + rapids-pip-retry wheel . -w dist -v --no-deps --disable-pip-version-check sccache --show-adv-stats diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 700f44dcc..f3807afdd 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -11,7 +11,7 @@ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from- rapids-generate-pip-constraints test_python ./constraints.txt # echo to expand wildcard before adding '[extra]' requires for pip -python -m pip install \ +rapids-pip-retry install \ -v \ --constraint ./constraints.txt \ "$(echo "${WHEELHOUSE}"/rmm_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]"