From 5d2b4e6f2c9733fbb249a818e90f466856dec77d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Mar 2026 12:58:38 -0600 Subject: [PATCH 1/7] ensure 'torch' CUDA wheels are installed in CI --- ci/download-torch-wheels.sh | 40 ++++++++++++++++++++ ci/test_python_integrations.sh | 2 +- ci/test_wheel.sh | 8 ++-- ci/test_wheel_integrations.sh | 35 ++++++++++-------- dependencies.yaml | 67 +++++++++++++++++++++++++++------- 5 files changed, 117 insertions(+), 35 deletions(-) create mode 100755 ci/download-torch-wheels.sh diff --git a/ci/download-torch-wheels.sh b/ci/download-torch-wheels.sh new file mode 100755 index 000000000..7313a006e --- /dev/null +++ b/ci/download-torch-wheels.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# [description] +# +# Downloads a CUDA variant of 'torch' from the correct index, based on CUDA major version. +# +# This exists to avoid using 'pip --extra-index-url', which has these undesirable properties: +# +# - allows for CPU-only 'torch' to be downloaded from pypi.org +# - allows for other non-torch packages like 'numpy' to be downloaded from the PyTorch indices +# - increases solve complexity for 'pip' +# + +set -e -u -o pipefail + +TORCH_WHEEL_DIR="${1}" + +# Ensure CUDA-enabled 'torch' packages are always used. +# +# Downloading + passing the downloaded file as a requirement forces the use of this +# package and ensures 'pip' considers all of its requirements. +# +# Not appending this to PIP_CONSTRAINT, because we don't want the torch '--extra-index-url' +# to leak outside of this script into other 'pip {download,install}'' calls. +rapids-dependency-file-generator \ + --output requirements \ + --file-key "torch_only" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES};require_gpu_pytorch=true" \ +| tee ./torch-constraints.txt + +rapids-pip-retry download \ + --isolated \ + --prefer-binary \ + --no-deps \ + -d "${TORCH_WHEEL_DIR}" \ + --constraint "${PIP_CONSTRAINT}" \ + --constraint ./torch-constraints.txt \ + 'torch' diff --git a/ci/test_python_integrations.sh b/ci/test_python_integrations.sh index 075e2314f..1cecda75d 100755 --- a/ci/test_python_integrations.sh +++ b/ci/test_python_integrations.sh @@ -40,7 +40,7 @@ if [ "${CUDA_MAJOR}" -gt 12 ] || { [ "${CUDA_MAJOR}" -eq 12 ] && [ "${CUDA_MINOR rapids-dependency-file-generator \ --output conda \ --file-key test_pytorch \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES};require_gpu_pytorch=true" \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ | tee env.yaml diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 347340e89..081ddffe6 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -12,17 +12,17 @@ LIBRMM_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid RMM_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" rmm --stable --cuda "$RAPIDS_CUDA_VERSION")") # generate constraints (possibly pinning to oldest support versions of dependencies) -rapids-generate-pip-constraints test_python ./constraints.txt +rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" # notes: # # * echo to expand wildcard before adding `[test]` requires for pip -# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is -# ignored if any other --constraint are passed via the CLI +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI # rapids-pip-retry install \ -v \ - --constraint ./constraints.txt \ + --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${LIBRMM_WHEELHOUSE}"/librmm_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${RMM_WHEELHOUSE}"/rmm_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" diff --git a/ci/test_wheel_integrations.sh b/ci/test_wheel_integrations.sh index c0368f1a3..116624076 100755 --- a/ci/test_wheel_integrations.sh +++ b/ci/test_wheel_integrations.sh @@ -4,8 +4,6 @@ set -eou pipefail -RAPIDS_INIT_PIP_REMOVE_NVIDIA_INDEX="true" -export RAPIDS_INIT_PIP_REMOVE_NVIDIA_INDEX source rapids-init-pip RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" @@ -13,16 +11,16 @@ LIBRMM_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapid RMM_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" rmm --stable --cuda "$RAPIDS_CUDA_VERSION")") # generate constraints (possibly pinning to oldest support versions of dependencies) -rapids-generate-pip-constraints test_python ./constraints.txt +rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" # notes: # # * echo to expand wildcard before adding `[test]` requires for pip -# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is -# ignored if any other --constraint are passed via the CLI +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI # PIP_INSTALL_SHARED_ARGS=( - --constraint=./constraints.txt + --prefer-binary --constraint="${PIP_CONSTRAINT}" "$(echo "${LIBRMM_WHEELHOUSE}"/librmm_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" "$(echo "${RMM_WHEELHOUSE}"/rmm_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" @@ -39,19 +37,24 @@ CUDA_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d'.' -f2) echo "::group::PyTorch Tests" -if [ "${CUDA_MAJOR}" -gt 12 ] || { [ "${CUDA_MAJOR}" -eq 12 ] && [ "${CUDA_MINOR}" -ge 8 ]; }; then - rapids-logger "Generating PyTorch test requirements" - rapids-dependency-file-generator \ - --output requirements \ - --file-key test_wheels_pytorch \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ - | tee test-pytorch-requirements.txt +# Update this when 'torch' publishes CUDA wheels supporting newer CTKs. +# +# See notes in 'dependencies.yaml' for details on supported versions. +if \ + { [ "${CUDA_MAJOR}" -eq 12 ] && [ "${CUDA_MINOR}" -ge 9 ]; } \ + || { [ "${CUDA_MAJOR}" -eq 13 ] && [ "${CUDA_MINOR}" -le 0 ]; }; \ +then + + # ensure a CUDA variant of 'torch' is used + rapids-logger "Downloading PyTorch CUDA wheels" + TORCH_WHEEL_DIR="$(mktemp -d)" + ./ci/download-torch-wheels.sh "${TORCH_WHEEL_DIR}" rapids-logger "Installing PyTorch test requirements" rapids-pip-retry install \ -v \ "${PIP_INSTALL_SHARED_ARGS[@]}" \ - -r test-pytorch-requirements.txt + "${TORCH_WHEEL_DIR}"/torch-*.whl timeout 15m python -m pytest -k "torch" ./python/rmm/rmm/tests \ && EXITCODE_PYTORCH=$? || EXITCODE_PYTORCH=$? @@ -60,7 +63,7 @@ if [ "${CUDA_MAJOR}" -gt 12 ] || { [ "${CUDA_MAJOR}" -eq 12 ] && [ "${CUDA_MINOR EXITCODE="${EXITCODE_PYTORCH}" fi else - rapids-logger "Skipping PyTorch tests (requires CUDA 12.8+, found ${RAPIDS_CUDA_VERSION})" + rapids-logger "Skipping PyTorch tests (requires CUDA 12.6-12.9 or 13.0, found ${RAPIDS_CUDA_VERSION})" fi echo "::endgroup::" @@ -71,7 +74,7 @@ rapids-logger "Generating CuPy test requirements" rapids-dependency-file-generator \ --output requirements \ --file-key test_wheels_cupy \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};use_cuda_wheels=true;require_gpu_pytorch=true" \ | tee test-cupy-requirements.txt rapids-logger "Installing CuPy test requirements" diff --git a/dependencies.yaml b/dependencies.yaml index 19bc1b8fa..319cc900a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -46,10 +46,6 @@ files: - depends_on_cupy - depends_on_librmm - depends_on_rmm - test_wheels_pytorch: - output: none - includes: - - depends_on_pytorch test_wheels_cupy: output: none includes: @@ -131,6 +127,10 @@ files: key: test includes: - test_python + torch_only: + output: none + includes: + - depends_on_pytorch channels: - rapidsai-nightly - rapidsai @@ -397,25 +397,64 @@ dependencies: # pip recognizes the index as a global option for the requirements.txt file - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple depends_on_pytorch: - common: - - output_types: conda - packages: - - pytorch-gpu>=2.10.0 specific: - - output_types: [requirements, pyproject] + - output_types: conda matrices: - matrix: - cuda: "12.*" + require_gpu_pytorch: "true" packages: - - --extra-index-url=https://download.pytorch.org/whl/cu128 + - pytorch-gpu>=2.9 - matrix: packages: - - --extra-index-url=https://download.pytorch.org/whl/cu130 - - output_types: [requirements, pyproject] + - pytorch>=2.9 + # The 'pytorch.org' indices referenced in --extra-index-url below host CPU-only variants too, + # so requirements like '>=' are not safe. + # + # Using '==' and a version with the CUDA specifier like '+cu130' is the most reliable way to ensure + # the packages we want are pulled (at the expense of needing to maintain this list). + # + # 'torch' tightly pins wheels to a single {major}.{minor} CTK version. + # + # This list only contains entries exactly matching CUDA {major}.{minor} that we test in RAPIDS CI, + # to ensure a loud error alerts us to the need to update this list (or CI scripts) when new + # CTKs are added to the support matrix. + - output_types: requirements matrices: + # avoid pulling in 'torch' in places like DLFW builds that prefer to install it other ways + - matrix: + no_pytorch: "true" + packages: + # matrices below ensure CUDA 'torch' packages are used + - matrix: + cuda: "12.9" + dependencies: "oldest" + require_gpu_pytorch: "true" + packages: + - &torch_cu129_index --extra-index-url=https://download.pytorch.org/whl/cu129 + - torch==2.9.0+cu129 + - matrix: + cuda: "12.9" + require_gpu_pytorch: "true" + packages: + - *torch_cu129_index + - torch==2.10.0+cu129 + - matrix: + cuda: "13.0" + dependencies: "oldest" + require_gpu_pytorch: "true" + packages: + - &torch_index_cu13 --extra-index-url=https://download.pytorch.org/whl/cu130 + - torch==2.9.0+cu130 + - matrix: + cuda: "13.0" + require_gpu_pytorch: "true" + packages: + - *torch_index_cu13 + - torch==2.10.0+cu130 - matrix: + require_gpu_pytorch: "false" packages: - - torch>=2.10.0 + - torch>=2.9 depends_on_cupy: common: - output_types: conda From 5d328ff40bbdea3739f5b58f6b8f1e2add922760 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Mar 2026 13:05:16 -0600 Subject: [PATCH 2/7] limit to constraints --- ci/test_wheel.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 081ddffe6..07f940386 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -22,7 +22,6 @@ rapids-generate-pip-constraints test_python "${PIP_CONSTRAINT}" # rapids-pip-retry install \ -v \ - --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${LIBRMM_WHEELHOUSE}"/librmm_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${RMM_WHEELHOUSE}"/rmm_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" From 96a8bcb7cdac37821d2ba45e91b8c3303a7077cb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Mar 2026 13:15:59 -0600 Subject: [PATCH 3/7] revert unnecessary stuff --- ci/test_wheel_integrations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_integrations.sh b/ci/test_wheel_integrations.sh index 116624076..f90742170 100755 --- a/ci/test_wheel_integrations.sh +++ b/ci/test_wheel_integrations.sh @@ -74,7 +74,7 @@ rapids-logger "Generating CuPy test requirements" rapids-dependency-file-generator \ --output requirements \ --file-key test_wheels_cupy \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};use_cuda_wheels=true;require_gpu_pytorch=true" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ | tee test-cupy-requirements.txt rapids-logger "Installing CuPy test requirements" From 9eefdea4d7eaeddcab3b883e6330ae0a7c35d350 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Mar 2026 13:16:43 -0600 Subject: [PATCH 4/7] log message --- ci/test_wheel_integrations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_integrations.sh b/ci/test_wheel_integrations.sh index f90742170..52607d6b9 100755 --- a/ci/test_wheel_integrations.sh +++ b/ci/test_wheel_integrations.sh @@ -63,7 +63,7 @@ then EXITCODE="${EXITCODE_PYTORCH}" fi else - rapids-logger "Skipping PyTorch tests (requires CUDA 12.6-12.9 or 13.0, found ${RAPIDS_CUDA_VERSION})" + rapids-logger "Skipping PyTorch tests (requires CUDA 12.9+ or 13.0, found ${RAPIDS_CUDA_VERSION})" fi echo "::endgroup::" From 9efce2624b03e59cc3533fca0eaa376699a363ab Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 6 Mar 2026 13:57:53 -0600 Subject: [PATCH 5/7] remove outdated comment --- ci/test_wheel_integrations.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/test_wheel_integrations.sh b/ci/test_wheel_integrations.sh index 52607d6b9..30e757341 100755 --- a/ci/test_wheel_integrations.sh +++ b/ci/test_wheel_integrations.sh @@ -31,12 +31,11 @@ EXITCODE=0 rapids-logger "Check GPU usage" nvidia-smi -# Check CUDA version for PyTorch compatibility (requires CUDA 12.8+) +echo "::group::PyTorch Tests" + CUDA_MAJOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d'.' -f1) CUDA_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d'.' -f2) -echo "::group::PyTorch Tests" - # Update this when 'torch' publishes CUDA wheels supporting newer CTKs. # # See notes in 'dependencies.yaml' for details on supported versions. From bd67d6b2d0d6abd0fa6aff5bbd9801ca69d362e4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 13 Mar 2026 10:02:34 -0500 Subject: [PATCH 6/7] make the label spelling match cugraph and cugraph-gnn PRs --- ci/download-torch-wheels.sh | 2 +- ci/test_python_integrations.sh | 2 +- dependencies.yaml | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/download-torch-wheels.sh b/ci/download-torch-wheels.sh index 7313a006e..5a277377d 100755 --- a/ci/download-torch-wheels.sh +++ b/ci/download-torch-wheels.sh @@ -27,7 +27,7 @@ TORCH_WHEEL_DIR="${1}" rapids-dependency-file-generator \ --output requirements \ --file-key "torch_only" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES};require_gpu_pytorch=true" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES};require_gpu=true" \ | tee ./torch-constraints.txt rapids-pip-retry download \ diff --git a/ci/test_python_integrations.sh b/ci/test_python_integrations.sh index 1cecda75d..a82a644df 100755 --- a/ci/test_python_integrations.sh +++ b/ci/test_python_integrations.sh @@ -40,7 +40,7 @@ if [ "${CUDA_MAJOR}" -gt 12 ] || { [ "${CUDA_MAJOR}" -eq 12 ] && [ "${CUDA_MINOR rapids-dependency-file-generator \ --output conda \ --file-key test_pytorch \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES};require_gpu_pytorch=true" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES};require_gpu=true" \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ | tee env.yaml diff --git a/dependencies.yaml b/dependencies.yaml index 319cc900a..5cf0353d6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -401,7 +401,7 @@ dependencies: - output_types: conda matrices: - matrix: - require_gpu_pytorch: "true" + require_gpu: "true" packages: - pytorch-gpu>=2.9 - matrix: @@ -428,31 +428,31 @@ dependencies: - matrix: cuda: "12.9" dependencies: "oldest" - require_gpu_pytorch: "true" + require_gpu: "true" packages: - &torch_cu129_index --extra-index-url=https://download.pytorch.org/whl/cu129 - torch==2.9.0+cu129 - matrix: cuda: "12.9" - require_gpu_pytorch: "true" + require_gpu: "true" packages: - *torch_cu129_index - torch==2.10.0+cu129 - matrix: cuda: "13.0" dependencies: "oldest" - require_gpu_pytorch: "true" + require_gpu: "true" packages: - &torch_index_cu13 --extra-index-url=https://download.pytorch.org/whl/cu130 - torch==2.9.0+cu130 - matrix: cuda: "13.0" - require_gpu_pytorch: "true" + require_gpu: "true" packages: - *torch_index_cu13 - torch==2.10.0+cu130 - matrix: - require_gpu_pytorch: "false" + require_gpu: "false" packages: - torch>=2.9 depends_on_cupy: From a3ca93d95fd9e1478360631ea834787b74b96e68 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 13 Mar 2026 13:09:23 -0500 Subject: [PATCH 7/7] clarify bounds --- ci/test_wheel_integrations.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/test_wheel_integrations.sh b/ci/test_wheel_integrations.sh index 30e757341..6286e68aa 100755 --- a/ci/test_wheel_integrations.sh +++ b/ci/test_wheel_integrations.sh @@ -40,8 +40,8 @@ CUDA_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d'.' -f2) # # See notes in 'dependencies.yaml' for details on supported versions. if \ - { [ "${CUDA_MAJOR}" -eq 12 ] && [ "${CUDA_MINOR}" -ge 9 ]; } \ - || { [ "${CUDA_MAJOR}" -eq 13 ] && [ "${CUDA_MINOR}" -le 0 ]; }; \ + { [ "${CUDA_MAJOR}" -eq 12 ] && [ "${CUDA_MINOR}" -eq 9 ]; } \ + || { [ "${CUDA_MAJOR}" -eq 13 ] && [ "${CUDA_MINOR}" -eq 0 ]; }; \ then # ensure a CUDA variant of 'torch' is used @@ -62,7 +62,7 @@ then EXITCODE="${EXITCODE_PYTORCH}" fi else - rapids-logger "Skipping PyTorch tests (requires CUDA 12.9+ or 13.0, found ${RAPIDS_CUDA_VERSION})" + rapids-logger "Skipping PyTorch tests (requires CUDA 12.9 or 13.0, found ${RAPIDS_CUDA_VERSION})" fi echo "::endgroup::"