From 5bc3762ef96c154df0c8310e9ea429f194c0d989 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Feb 2025 15:18:05 -0500 Subject: [PATCH 1/6] Create Conda CI test env in one step Issue: https://github.com/rapidsai/build-planning/issues/22 --- ci/test_python.sh | 73 +++++++++++++++++++++-------------------------- dependencies.yaml | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 40 deletions(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 15949b40..99a9b9b7 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -14,15 +14,6 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -rapids-logger "Generate Python testing dependencies" -rapids-dependency-file-generator \ - --output conda \ - --file-key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ - --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${PYTHON_CHANNEL}" \ -| tee env.yaml - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" @@ -50,6 +41,19 @@ set +e # bulk sampler IO tests (hangs in CI) if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-logger "Generate Python testing dependencies" + rapids-dependency-file-generator \ + --output conda \ + --file-key test_cugraph_dgl \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + --prepend-channel pytorch \ + --prepend-channel conda-forge \ + --prepend-channel "${DGL_CHANNEL}" \ + --prepend-channel nvidia \ + | tee env.yaml + rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_dgl # activate test_cugraph_dgl environment for dgl @@ -64,18 +68,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then fi - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - --channel pytorch \ - --channel conda-forge \ - --channel "${DGL_CHANNEL}" \ - --channel nvidia \ - "pylibwholegraph=${RAPIDS_VERSION}" \ - "cugraph-dgl=${RAPIDS_VERSION}" \ - 'pytorch>=2.3' \ - "ogb" - rapids-print-env rapids-logger "Check GPU usage" @@ -98,6 +90,16 @@ else fi if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-logger "Generate Python testing dependencies" + rapids-dependency-file-generator \ + --output conda \ + --file-key test_cugraph_pyg \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + --prepend-channel pytorch \ + | tee env.yaml + rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_pyg # Temporarily allow unbound variables for conda activation. @@ -105,15 +107,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then conda activate test_cugraph_pyg set -u - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - --channel pytorch \ - "pylibwholegraph=${RAPIDS_VERSION}" \ - "cugraph-pyg=${RAPIDS_VERSION}" \ - 'pytorch>=2.3' \ - 'ogb' - rapids-print-env rapids-logger "Check GPU usage" @@ -136,6 +129,16 @@ else fi if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-logger "Generate Python testing dependencies" + rapids-dependency-file-generator \ + --output conda \ + --file-key test_pylibwholegraph \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + --prepend-channel pytorch \ + | tee env.yaml + rapids-mamba-retry env create --yes -f env.yaml -n test_pylibwholegraph # Temporarily allow unbound variables for conda activation. @@ -143,16 +146,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then conda activate test_pylibwholegraph set -u - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - --channel pytorch \ - 'mkl<2024.1.0' \ - "pylibwholegraph=${RAPIDS_VERSION}" \ - 'pytorch>=2.3' \ - 'pytest-forked' \ - 'ogb' - rapids-print-env rapids-logger "Check GPU usage" diff --git a/dependencies.yaml b/dependencies.yaml index ce9f4578..8a8f4265 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -68,6 +68,46 @@ files: - depends_on_ogb - py_version - test_python_common + test_cugraph_dgl: + output: none + includes: + - cuda_version + - depends_on_cugraph + - depends_on_cudf + - depends_on_dgl + - depends_on_pytorch + - depends_on_ogb + - py_version + - test_python_common + - depends_on_pylibwholegraph + - depends_on_cugraph_dgl + test_cugraph_pyg: + output: none + includes: + - cuda_version + - depends_on_cugraph + - depends_on_cudf + - depends_on_dgl + - depends_on_pytorch + - depends_on_ogb + - py_version + - test_python_common + - depends_on_pylibwholegraph + - depends_on_cugraph_pyg + test_pylibwholegraph: + output: none + includes: + - cuda_version + - depends_on_cugraph + - depends_on_cudf + - depends_on_dgl + - depends_on_pytorch + - depends_on_ogb + - py_version + - test_python_common + - depends_on_mkl + - depends_on_pylibwholegraph + - test_python_pylibwholegraph py_build_pylibwholegraph: output: pyproject @@ -393,6 +433,7 @@ dependencies: - output_types: [conda, pyproject, requirements] packages: - pytest-forked + - output_types: [pyproject, requirements] - scipy cugraph_dgl_dev: common: @@ -658,3 +699,13 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=13.2.0 - {matrix: null, packages: *cupy_packages_cu11} + depends_on_cugraph_pyg: + common: + - output_types: conda + packages: + - cugraph-pyg==25.4.*,>=0.0.0a0 + depends_on_mkl: + common: + - output_types: conda + packages: + - mkl<2024.1.0 From 477da456b3d95eca2dd83b6ce7f26bf2605616b1 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Feb 2025 15:23:47 -0500 Subject: [PATCH 2/6] Oops --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index 8a8f4265..bf0ffd72 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -434,6 +434,7 @@ dependencies: packages: - pytest-forked - output_types: [pyproject, requirements] + packages: - scipy cugraph_dgl_dev: common: From cbf1e4aa63a9c4ce93ef274a71999a8dafd23683 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Feb 2025 15:27:26 -0500 Subject: [PATCH 3/6] Put scipy back --- dependencies.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index bf0ffd72..36dc8ea4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -433,8 +433,6 @@ dependencies: - output_types: [conda, pyproject, requirements] packages: - pytest-forked - - output_types: [pyproject, requirements] - packages: - scipy cugraph_dgl_dev: common: From 93ce71ea720daf285ac28239834cdd01feb51f79 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Feb 2025 15:32:36 -0500 Subject: [PATCH 4/6] Copyright --- ci/test_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 99a9b9b7..41ce694a 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail From f90fcd759e73fd28670f7f64076696152ad104ec Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 14 Feb 2025 14:16:00 -0500 Subject: [PATCH 5/6] Consolidate build_docs.sh and test_cpp.sh --- ci/build_docs.sh | 16 +++++++--------- ci/release/update-version.sh | 1 + ci/test_cpp.sh | 11 ++--------- dependencies.yaml | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 6c4e8366..3b70e068 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -3,16 +3,21 @@ set -euo pipefail +rapids-logger "Downloading artifacts from previous jobs" + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) + rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" rapids-dependency-file-generator \ --output conda \ --file-key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" \ + | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n docs @@ -23,15 +28,8 @@ set -u rapids-print-env -rapids-logger "Downloading artifacts from previous jobs" - -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) export RAPIDS_DOCS_DIR="$(mktemp -d)" -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - "libwholegraph=${RAPIDS_VERSION}" - rapids-logger "Build C++ docs" pushd cpp doxygen Doxyfile diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 2eb0dfda..ce4b0ca4 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -57,6 +57,7 @@ DEPENDENCIES=( libraft libraft-headers librmm + libwholegraph pylibcugraph pylibwholegraph rmm diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index c61fb1ad..9b733581 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -8,8 +8,6 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" - CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) rapids-logger "Generate C++ testing dependencies" @@ -18,7 +16,7 @@ rapids-dependency-file-generator \ --file-key test_cpp \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ --prepend-channel "${CPP_CHANNEL}" \ -| tee env.yaml + | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test @@ -32,11 +30,6 @@ mkdir -p "${RAPIDS_TESTS_DIR}" rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - "libwholegraph=${RAPIDS_VERSION}" \ - "libwholegraph-tests=${RAPIDS_VERSION}" - rapids-logger "Check GPU usage" nvidia-smi diff --git a/dependencies.yaml b/dependencies.yaml index 36dc8ea4..0bc1ea8e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -43,11 +43,14 @@ files: - cuda_version - docs - py_version + - depends_on_libwholegraph test_cpp: output: none includes: - cuda_version - test_cpp + - depends_on_libwholegraph + - depends_on_libwholegraph_tests test_notebooks: output: none includes: @@ -549,6 +552,18 @@ dependencies: - pylibwholegraph-cu11==25.4.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} + depends_on_libwholegraph: + common: + - output_types: conda + packages: + - libwholegraph==25.4.*,>=0.0.0a0 + + depends_on_libwholegraph_tests: + common: + - output_types: conda + packages: + - libwholegraph-tests==25.4.*,>=0.0.0a0 + depends_on_rmm: common: - output_types: conda From 9f40411a07f11b16c090d12f473e39e4c1f27674 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 14 Feb 2025 17:34:22 -0500 Subject: [PATCH 6/6] Review feedback --- ci/test_python.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 41ce694a..73127d27 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -6,6 +6,12 @@ set -euo pipefail # Support invoking test_python.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +if [[ "${RAPIDS_CUDA_VERSION%%.*}" == "11" ]]; then + DGL_CHANNEL="dglteam/label/th23_cu118" +else + DGL_CHANNEL="dglteam/label/th23_cu121" +fi + . /opt/conda/etc/profile.d/conda.sh RAPIDS_VERSION="$(rapids-version)" @@ -41,7 +47,7 @@ set +e # bulk sampler IO tests (hangs in CI) if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - rapids-logger "Generate Python testing dependencies" + rapids-logger "(cugraph-dgl) Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cugraph_dgl \ @@ -61,12 +67,6 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then conda activate test_cugraph_dgl set -u - if [[ "${RAPIDS_CUDA_VERSION%%.*}" == "11" ]]; then - DGL_CHANNEL="dglteam/label/th23_cu118" - else - DGL_CHANNEL="dglteam/label/th23_cu121" - fi - rapids-print-env @@ -90,7 +90,7 @@ else fi if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - rapids-logger "Generate Python testing dependencies" + rapids-logger "(cugraph-pyg) Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cugraph_pyg \ @@ -129,7 +129,7 @@ else fi if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - rapids-logger "Generate Python testing dependencies" + rapids-logger "(pylibwholegraph) Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_pylibwholegraph \