From 569295adc7616dd9721340f3fbd04b64de693b59 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 13:29:20 -0400 Subject: [PATCH 01/16] [DO NOT MERGE]: test upstream libkvikio wheel with vendored nvcomp --- ci/build_wheel_libcudf.sh | 8 ++++++++ dependencies.yaml | 2 +- python/libcudf/pyproject.toml | 1 - 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 9337e01a18b2..3802b7e30673 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -17,6 +17,14 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ | tee /tmp/requirements-build.txt + +set -x +LIBKVIKIO_WHEEL=$(rapids-get-artifact "libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a28-py3-none-${AUDITWHEEL_PLAT}.whl") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_WHEEL}" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" +set +x + + rapids-logger "Installing build requirements" rapids-pip-retry install \ -v \ diff --git a/dependencies.yaml b/dependencies.yaml index 32b114057ce7..808277a3f526 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -216,7 +216,7 @@ files: includes: - depends_on_libkvikio - depends_on_librmm - - depends_on_nvcomp + # - depends_on_nvcomp - depends_on_rapids_logger py_build_pylibcudf: output: pyproject diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 8ef792a35558..fd1936ecf0dd 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -40,7 +40,6 @@ classifiers = [ dependencies = [ "libkvikio==25.6.*,>=0.0.0a0", "librmm==25.6.*,>=0.0.0a0", - "nvidia-nvcomp==4.2.0.11", "rapids-logger==0.1.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 9b40d9f8214bbcbc336d9082b57e8ac6f477c8ce Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 13:56:27 -0400 Subject: [PATCH 02/16] bucket path --- ci/build_wheel_libcudf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 3802b7e30673..6352209a9487 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -19,7 +19,7 @@ rapids-dependency-file-generator \ set -x -LIBKVIKIO_WHEEL=$(rapids-get-artifact "libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a28-py3-none-${AUDITWHEEL_PLAT}.whl") +LIBKVIKIO_WHEEL=$(rapids-get-artifact "ci/kvikio/pull-request/702/5437fca/libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a28-py3-none-${AUDITWHEEL_PLAT}.whl") echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_WHEEL}" > /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" set +x From 30073fef641ee8cb8760708c2351eccdb5412f44 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 14:13:26 -0400 Subject: [PATCH 03/16] tarball and wheel have different suffixes --- ci/build_wheel_libcudf.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 6352209a9487..917e1a8bbdd3 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -19,12 +19,14 @@ rapids-dependency-file-generator \ set -x -LIBKVIKIO_WHEEL=$(rapids-get-artifact "ci/kvikio/pull-request/702/5437fca/libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a28-py3-none-${AUDITWHEEL_PLAT}.whl") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_WHEEL}" > /tmp/constraints.txt +#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a28-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/5437fca/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" > /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" set +x - rapids-logger "Installing build requirements" rapids-pip-retry install \ -v \ From 0626b609859d6c9c4fd7caa0f5b6cb381479e83b Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 14:48:25 -0400 Subject: [PATCH 04/16] update sha and version suffix for updated kvikio --- ci/build_wheel_libcudf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 917e1a8bbdd3..36ba16b6beda 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -20,9 +20,9 @@ rapids-dependency-file-generator \ set -x #kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a28-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/5437fca/${LIBKVIKIO_TARBALL}") +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" > /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" set +x From c9f740217722d2c10000e2fdecf2f7db179f9ead Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 15:46:02 -0400 Subject: [PATCH 05/16] make sure we use the updated kvikio wheel everywhere --- ci/build_wheel_cudf.sh | 8 ++++++++ ci/build_wheel_pylibcudf.sh | 9 +++++++++ ci/test_wheel_cudf.sh | 7 +++++++ ci/test_wheel_cudf_polars.sh | 7 +++++++ ci/test_wheel_dask_cudf.sh | 7 +++++++ 5 files changed, 38 insertions(+) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index b6bf12e16264..40e6025017a7 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -18,6 +18,14 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE}/pylibcudf_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" +set -x +#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt +set +x + ./ci/build_wheel.sh cudf ${package_dir} # repair wheels and write to the location that artifact-uploading code expects to find them diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index bfac8c6f7911..20b68018612b 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -16,6 +16,15 @@ LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rap echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/libcudf_*.whl)" > /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" + +set -x +#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt +set +x + ./ci/build_wheel.sh pylibcudf ${package_dir} # repair wheels and write to the location that artifact-uploading code expects to find them diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 9efd8b584fc1..add9c3a989e8 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -15,6 +15,13 @@ rapids-logger "Install cudf, pylibcudf, and test requirements" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf ./constraints.txt +set -x +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt +set +x + # echo to expand wildcard before adding `[extra]` requires for pip rapids-pip-retry install \ -v \ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 90154499c1df..12164f09aeac 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -17,6 +17,13 @@ rapids-logger "Installing cudf_polars and its dependencies" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt +set -x +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt +set +x + # echo to expand wildcard before adding `[test,experimental]` requires for pip rapids-pip-retry install \ -v \ diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 95af2c8ddea6..f84df3034aa8 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -16,6 +16,13 @@ rapids-logger "Install dask_cudf, cudf, pylibcudf, and test requirements" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_dask_cudf ./constraints.txt +set -x +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt +set +x + # echo to expand wildcard before adding `[extra]` requires for pip rapids-pip-retry install \ -v \ From 66b507eb5c2605f28562ed05f58d1cc39288a0e7 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 15:56:14 -0400 Subject: [PATCH 06/16] cleaner downloading of artifacts from kvikio PR --- ci/build_wheel_cudf.sh | 23 ++++++++++++++++------- ci/build_wheel_libcudf.sh | 25 +++++++++++++++++-------- ci/build_wheel_pylibcudf.sh | 24 +++++++++++++++++------- ci/test_wheel_cudf.sh | 22 ++++++++++++++++------ ci/test_wheel_cudf_polars.sh | 22 ++++++++++++++++------ ci/test_wheel_dask_cudf.sh | 22 ++++++++++++++++------ 6 files changed, 98 insertions(+), 40 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 40e6025017a7..8ae2aa6b5e0a 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -18,13 +18,22 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE}/pylibcudf_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" -set -x -#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt -set +x +# ci/use_wheels_from_prs.sh +RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") + +# download wheels, store the directories holding them in variables +LIBKVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp +) +KVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python +) + +# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" +cat >> /tmp/constraints.txt < /tmp/constraints.txt < /tmp/constraints.txt -export PIP_CONSTRAINT="/tmp/constraints.txt" -set +x rapids-logger "Installing build requirements" rapids-pip-retry install \ diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 20b68018612b..a862982b8edb 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -17,13 +17,23 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib export PIP_CONSTRAINT="/tmp/constraints.txt" -set -x -#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt -set +x +# ci/use_wheels_from_prs.sh +RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") + +# download wheels, store the directories holding them in variables +LIBKVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp +) +KVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python +) + +# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" +cat >> /tmp/constraints.txt <> ./constraints.txt -set +x +# ci/use_wheels_from_prs.sh +RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") + +# download wheels, store the directories holding them in variables +LIBKVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp +) +KVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python +) + +# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" +cat >> ./constraints.txt <> ./constraints.txt -set +x +# ci/use_wheels_from_prs.sh +RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") + +# download wheels, store the directories holding them in variables +LIBKVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp +) +KVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python +) + +# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" +cat >> ./constraints.txt <> ./constraints.txt -set +x +# ci/use_wheels_from_prs.sh +RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") + +# download wheels, store the directories holding them in variables +LIBKVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp +) +KVIKIO_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python +) + +# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" +cat >> ./constraints.txt < Date: Tue, 29 Apr 2025 16:05:10 -0400 Subject: [PATCH 07/16] disable using nvcomp runtime wheel --- ci/build_wheel_libcudf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 5f8846e4e6ae..280a587f53c6 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -46,7 +46,7 @@ rapids-pip-retry install \ # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 -export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" +export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=OFF" ./ci/build_wheel.sh "${package_name}" "${package_dir}" # repair wheels and write to the location that artifact-uploading code expects to find them From c3510299b0c699929b4ed199db0ca263b9c3c393 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 16:15:33 -0400 Subject: [PATCH 08/16] Revert "cleaner downloading of artifacts from kvikio PR" This reverts commit 66b507eb5c2605f28562ed05f58d1cc39288a0e7. --- ci/build_wheel_cudf.sh | 23 +++++++---------------- ci/build_wheel_libcudf.sh | 25 ++++++++----------------- ci/build_wheel_pylibcudf.sh | 24 +++++++----------------- ci/test_wheel_cudf.sh | 22 ++++++---------------- ci/test_wheel_cudf_polars.sh | 22 ++++++---------------- ci/test_wheel_dask_cudf.sh | 22 ++++++---------------- 6 files changed, 40 insertions(+), 98 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 8ae2aa6b5e0a..40e6025017a7 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -18,22 +18,13 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE}/pylibcudf_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" -# ci/use_wheels_from_prs.sh -RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") - -# download wheels, store the directories holding them in variables -LIBKVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp -) -KVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python -) - -# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" -cat >> /tmp/constraints.txt <> /tmp/constraints.txt +set +x ./ci/build_wheel.sh cudf ${package_dir} diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 280a587f53c6..255c694796cf 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -17,24 +17,15 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ | tee /tmp/requirements-build.txt -# ci/use_wheels_from_prs.sh -RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") - -# download wheels, store the directories holding them in variables -LIBKVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp -) -KVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python -) - -# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" -cat > /tmp/constraints.txt < /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" +set +x rapids-logger "Installing build requirements" rapids-pip-retry install \ diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index a862982b8edb..20b68018612b 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -17,23 +17,13 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib export PIP_CONSTRAINT="/tmp/constraints.txt" -# ci/use_wheels_from_prs.sh -RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") - -# download wheels, store the directories holding them in variables -LIBKVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp -) -KVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python -) - -# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" -cat >> /tmp/constraints.txt <> /tmp/constraints.txt +set +x ./ci/build_wheel.sh pylibcudf ${package_dir} diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 06140b40c5b3..add9c3a989e8 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -15,22 +15,12 @@ rapids-logger "Install cudf, pylibcudf, and test requirements" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf ./constraints.txt -# ci/use_wheels_from_prs.sh -RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") - -# download wheels, store the directories holding them in variables -LIBKVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp -) -KVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python -) - -# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" -cat >> ./constraints.txt <> ./constraints.txt +set +x # echo to expand wildcard before adding `[extra]` requires for pip rapids-pip-retry install \ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 8f30963bf27d..12164f09aeac 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -17,22 +17,12 @@ rapids-logger "Installing cudf_polars and its dependencies" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt -# ci/use_wheels_from_prs.sh -RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") - -# download wheels, store the directories holding them in variables -LIBKVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp -) -KVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python -) - -# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" -cat >> ./constraints.txt <> ./constraints.txt +set +x # echo to expand wildcard before adding `[test,experimental]` requires for pip rapids-pip-retry install \ diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 400350df5d87..f84df3034aa8 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -16,22 +16,12 @@ rapids-logger "Install dask_cudf, cudf, pylibcudf, and test requirements" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_dask_cudf ./constraints.txt -# ci/use_wheels_from_prs.sh -RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") - -# download wheels, store the directories holding them in variables -LIBKVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 cpp -) -KVIKIO_WHEELHOUSE=$( - RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python -) - -# write a pip constraints file saying e.g. "whenever you encounter a requirement for 'librmm-cu12', use this wheel" -cat >> ./constraints.txt <> ./constraints.txt +set +x # echo to expand wildcard before adding `[extra]` requires for pip rapids-pip-retry install \ From 8e5378ad85a6282a75b1b232e6cdc52515e101fd Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Tue, 29 Apr 2025 16:39:28 -0400 Subject: [PATCH 09/16] Revert "disable using nvcomp runtime wheel" This reverts commit c0af5c799b11024e5c2ac9d450bd2551654b23cf. --- ci/build_wheel_libcudf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 255c694796cf..36ba16b6beda 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -37,7 +37,7 @@ rapids-pip-retry install \ # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 -export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=OFF" +export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" ./ci/build_wheel.sh "${package_name}" "${package_dir}" # repair wheels and write to the location that artifact-uploading code expects to find them From 97faca6adb6222f6351b8d6c531ea577c6d48a69 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 30 Apr 2025 11:13:00 -0400 Subject: [PATCH 10/16] try to fix install paths --- ci/build_wheel_cudf.sh | 1 - ci/build_wheel_pylibcudf.sh | 1 - ci/test_wheel_cudf.sh | 4 ++++ ci/test_wheel_cudf_polars.sh | 4 ++++ ci/test_wheel_dask_cudf.sh | 4 ++++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 40e6025017a7..6f9a899d6a80 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -19,7 +19,6 @@ echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE} export PIP_CONSTRAINT="/tmp/constraints.txt" set -x -#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 20b68018612b..e6cdd531a78d 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -18,7 +18,6 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" set -x -#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index add9c3a989e8..f291392622a3 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -16,6 +16,10 @@ rapids-logger "Install cudf, pylibcudf, and test requirements" rapids-generate-pip-constraints py_test_cudf ./constraints.txt set -x +AUDITWHEEL_ARCH="$(arch)" +export AUDITWHEEL_ARCH +AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" +export AUDITWHEEL_PLAT LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 12164f09aeac..a5d615efec22 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -18,6 +18,10 @@ rapids-logger "Installing cudf_polars and its dependencies" rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt set -x +AUDITWHEEL_ARCH="$(arch)" +export AUDITWHEEL_ARCH +AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" +export AUDITWHEEL_PLAT LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index f84df3034aa8..9b181c64ef6b 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -17,6 +17,10 @@ rapids-logger "Install dask_cudf, cudf, pylibcudf, and test requirements" rapids-generate-pip-constraints py_test_dask_cudf ./constraints.txt set -x +AUDITWHEEL_ARCH="$(arch)" +export AUDITWHEEL_ARCH +AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" +export AUDITWHEEL_PLAT LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") From 6e75249e9255f3fd96bd86087d54de94eaf91804 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 30 Apr 2025 11:13:10 -0400 Subject: [PATCH 11/16] update libcudf RPATH for libkvikio-bundled nvcomp --- ci/build_wheel_libcudf.sh | 2 +- python/libcudf/CMakeLists.txt | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 36ba16b6beda..3a97574fa291 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -37,7 +37,7 @@ rapids-pip-retry install \ # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 -export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" +export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_FROM_LIBKVIKIO_WHEEL=ON" ./ci/build_wheel.sh "${package_name}" "${package_dir}" # repair wheels and write to the location that artifact-uploading code expects to find them diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index b88694ec274f..c3e16c4b4808 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -23,6 +23,7 @@ project( ) option(USE_NVCOMP_RUNTIME_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF) +option(USE_NVCOMP_FROM_LIBKVIKIO_WHEEL "Use nvcomp bundled with libkvikio" OFF) # Check if cudf is already available. If so, it is the user's responsibility to ensure that the # CMake package is also available at build time of the Python cudf package. @@ -55,3 +56,12 @@ if(USE_NVCOMP_RUNTIME_WHEEL) APPEND ) endif() + +if(USE_NVCOMP_FROM_LIBKVIKIO_WHEEL) + set(rpaths "$ORIGIN/../../libkvikio/lib64") + set_property( + TARGET cudf + PROPERTY INSTALL_RPATH ${rpaths} + APPEND + ) +endif() From 3121d4eb25581177d5e56a0cfd9b7e2a1efc5bbd Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 30 Apr 2025 11:40:02 -0400 Subject: [PATCH 12/16] don't export nvcomp --- python/libcudf/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index c3e16c4b4808..2bbdb7012299 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -39,7 +39,7 @@ set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) set(CUDF_BUILD_TESTUTIL OFF) set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) -if(USE_NVCOMP_RUNTIME_WHEEL) +if(USE_NVCOMP_RUNTIME_WHEEL OR USE_NVCOMP_FROM_LIBKVIKIO_WHEEL) set(CUDF_EXPORT_NVCOMP OFF) endif() set(CUDA_STATIC_RUNTIME ON) From a3d9625b958728855e2ed30b3b8626cea79ff6b5 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 30 Apr 2025 13:34:42 -0400 Subject: [PATCH 13/16] make sure upstream libkvikio is installed for pandas and polars tests --- ci/cudf_pandas_scripts/run_tests.sh | 3 +++ ci/test_cudf_polars_polars_tests.sh | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index cefa5d72e457..010942c75ef9 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -56,6 +56,9 @@ else # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python_cudf_pandas ./constraints.txt + LIBKVIKIO_WHEELHOUSE=$(GITHUB_RUN_ID=14738962324 RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python) + echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_WHEELHOUSE}/libkvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl" >> ./constraints.txt + python -m pip install \ -v \ --constraint ./constraints.txt \ diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index 8e4ade406374..456f1a1613b3 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -12,12 +12,15 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) +LIBKVIKIO_WHEELHOUSE=$(GITHUB_RUN_ID=14738962324 RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python) + rapids-logger "Install libcudf, pylibcudf and cudf_polars" rapids-pip-retry install \ -v \ "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ - "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" + "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo "${LIBKVIKIO_WHEELHOUSE}"/libkvikio_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" TAG=$(python -c 'import polars; print(f"py-{polars.__version__}")') From d39e41ad017613cada902cc5060935dd12ab4f2e Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 30 Apr 2025 14:42:52 -0400 Subject: [PATCH 14/16] and again --- ci/cudf_pandas_scripts/run_tests.sh | 12 ++++++++++-- ci/test_cudf_polars_polars_tests.sh | 16 +++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 010942c75ef9..7af3b71ff9c8 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -56,8 +56,16 @@ else # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python_cudf_pandas ./constraints.txt - LIBKVIKIO_WHEELHOUSE=$(GITHUB_RUN_ID=14738962324 RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python) - echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_WHEELHOUSE}/libkvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl" >> ./constraints.txt + set -x + AUDITWHEEL_ARCH="$(arch)" + export AUDITWHEEL_ARCH + AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" + export AUDITWHEEL_PLAT + LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" + LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" + LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") + echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt + set +x python -m pip install \ -v \ diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index 456f1a1613b3..205b45d3dff2 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -12,15 +12,25 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) -LIBKVIKIO_WHEELHOUSE=$(GITHUB_RUN_ID=14738962324 RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact-github kvikio 702 python) +set -x +AUDITWHEEL_ARCH="$(arch)" +export AUDITWHEEL_ARCH +AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" +export AUDITWHEEL_PLAT +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" +set +x + rapids-logger "Install libcudf, pylibcudf and cudf_polars" rapids-pip-retry install \ -v \ "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ - "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ - "$(echo "${LIBKVIKIO_WHEELHOUSE}"/libkvikio_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" + "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" TAG=$(python -c 'import polars; print(f"py-{polars.__version__}")') From 897033393cf073c9f89b397e325faafd9a67fc88 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 1 May 2025 06:58:04 -0400 Subject: [PATCH 15/16] yet another spot to constrain install --- ci/cudf_pandas_scripts/pandas-tests/run.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 6feea4c153c2..c638b529c684 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -17,8 +17,21 @@ CUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-do LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) + +set -x +AUDITWHEEL_ARCH="$(arch)" +export AUDITWHEEL_ARCH +AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" +export AUDITWHEEL_PLAT +LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" +LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" +LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") +echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt +set +x + # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ + --constraint ./constraints.txt \ "$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,pandas-tests]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" From b16cbe61d639455a823d1fc22b5ffe014d26778e Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 2 May 2025 09:37:21 -0400 Subject: [PATCH 16/16] refactor: use upstream nightly wheel for libkvikio+nvcomp --- ci/build_wheel_cudf.sh | 7 ------- ci/build_wheel_libcudf.sh | 14 ++++---------- ci/build_wheel_pylibcudf.sh | 8 -------- ci/cudf_pandas_scripts/pandas-tests/run.sh | 13 ------------- ci/cudf_pandas_scripts/run_tests.sh | 11 ----------- ci/test_cudf_polars_polars_tests.sh | 13 ------------- ci/test_wheel_cudf.sh | 11 ----------- ci/test_wheel_cudf_polars.sh | 11 ----------- ci/test_wheel_dask_cudf.sh | 11 ----------- dependencies.yaml | 3 +++ python/libcudf/CMakeLists.txt | 8 ++++++++ 11 files changed, 15 insertions(+), 95 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 6f9a899d6a80..b6bf12e16264 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -18,13 +18,6 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE}/pylibcudf_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" -set -x -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt -set +x - ./ci/build_wheel.sh cudf ${package_dir} # repair wheels and write to the location that artifact-uploading code expects to find them diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 3a97574fa291..6f12417a131e 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -17,16 +17,6 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ | tee /tmp/requirements-build.txt - -set -x -#kvikio_wheel_cpp_libkvikio_cu11_aarch64.tar.gz -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" > /tmp/constraints.txt -export PIP_CONSTRAINT="/tmp/constraints.txt" -set +x - rapids-logger "Installing build requirements" rapids-pip-retry install \ -v \ @@ -37,6 +27,10 @@ rapids-pip-retry install \ # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 +# TODO(nvcomp): when `nvcomp` supports Python 3.13 and we de-vendor `nvcomp` from `kvikio` +# this should be switched back to using the nvcomp runtime wheel +# https://github.com/rapidsai/build-planning/issues/171 +# export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_FROM_LIBKVIKIO_WHEEL=ON" ./ci/build_wheel.sh "${package_name}" "${package_dir}" diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index e6cdd531a78d..bfac8c6f7911 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -16,14 +16,6 @@ LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rap echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/libcudf_*.whl)" > /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" - -set -x -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt -set +x - ./ci/build_wheel.sh pylibcudf ${package_dir} # repair wheels and write to the location that artifact-uploading code expects to find them diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index c638b529c684..6feea4c153c2 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -17,21 +17,8 @@ CUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-do LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) - -set -x -AUDITWHEEL_ARCH="$(arch)" -export AUDITWHEEL_ARCH -AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" -export AUDITWHEEL_PLAT -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt -set +x - # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - --constraint ./constraints.txt \ "$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,pandas-tests]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 7af3b71ff9c8..cefa5d72e457 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -56,17 +56,6 @@ else # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints test_python_cudf_pandas ./constraints.txt - set -x - AUDITWHEEL_ARCH="$(arch)" - export AUDITWHEEL_ARCH - AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" - export AUDITWHEEL_PLAT - LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" - LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" - LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") - echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt - set +x - python -m pip install \ -v \ --constraint ./constraints.txt \ diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index 205b45d3dff2..8e4ade406374 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -12,19 +12,6 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) -set -x -AUDITWHEEL_ARCH="$(arch)" -export AUDITWHEEL_ARCH -AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" -export AUDITWHEEL_PLAT -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> /tmp/constraints.txt -export PIP_CONSTRAINT="/tmp/constraints.txt" -set +x - - rapids-logger "Install libcudf, pylibcudf and cudf_polars" rapids-pip-retry install \ -v \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index f291392622a3..9efd8b584fc1 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -15,17 +15,6 @@ rapids-logger "Install cudf, pylibcudf, and test requirements" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf ./constraints.txt -set -x -AUDITWHEEL_ARCH="$(arch)" -export AUDITWHEEL_ARCH -AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" -export AUDITWHEEL_PLAT -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt -set +x - # echo to expand wildcard before adding `[extra]` requires for pip rapids-pip-retry install \ -v \ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index a5d615efec22..90154499c1df 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -17,17 +17,6 @@ rapids-logger "Installing cudf_polars and its dependencies" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt -set -x -AUDITWHEEL_ARCH="$(arch)" -export AUDITWHEEL_ARCH -AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" -export AUDITWHEEL_PLAT -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt -set +x - # echo to expand wildcard before adding `[test,experimental]` requires for pip rapids-pip-retry install \ -v \ diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 9b181c64ef6b..95af2c8ddea6 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -16,17 +16,6 @@ rapids-logger "Install dask_cudf, cudf, pylibcudf, and test requirements" # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_dask_cudf ./constraints.txt -set -x -AUDITWHEEL_ARCH="$(arch)" -export AUDITWHEEL_ARCH -AUDITWHEEL_PLAT="manylinux_2_28_$(arch)" -export AUDITWHEEL_PLAT -LIBKVIKIO_WHL="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}-25.6.0a32-py3-none-${AUDITWHEEL_PLAT}.whl" -LIBKVIKIO_TARBALL="kvikio_wheel_cpp_libkvikio_${RAPIDS_PY_CUDA_SUFFIX}_${AUDITWHEEL_ARCH}.tar.gz" -LIBKVIKIO_DIR=$(rapids-get-artifact "ci/kvikio/pull-request/702/7f957eb/${LIBKVIKIO_TARBALL}") -echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://${LIBKVIKIO_DIR}/${LIBKVIKIO_WHL}" >> ./constraints.txt -set +x - # echo to expand wildcard before adding `[extra]` requires for pip rapids-pip-retry install \ -v \ diff --git a/dependencies.yaml b/dependencies.yaml index 808277a3f526..a4e8759516e6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -216,6 +216,9 @@ files: includes: - depends_on_libkvikio - depends_on_librmm + # TODO(nvcomp): when `nvcomp` supports Python 3.13 and we de-vendor `nvcomp` from `kvikio` + # this dependency should be restored + # https://github.com/rapidsai/build-planning/issues/171 # - depends_on_nvcomp - depends_on_rapids_logger py_build_pylibcudf: diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 2bbdb7012299..ca737af08f31 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -23,6 +23,9 @@ project( ) option(USE_NVCOMP_RUNTIME_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF) + +# TODO(nvcomp): when `nvcomp` supports Python 3.13 and we de-vendor `nvcomp` from `kvikio` this +# option should be removed https://github.com/rapidsai/build-planning/issues/171 option(USE_NVCOMP_FROM_LIBKVIKIO_WHEEL "Use nvcomp bundled with libkvikio" OFF) # Check if cudf is already available. If so, it is the user's responsibility to ensure that the @@ -39,6 +42,9 @@ set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) set(CUDF_BUILD_TESTUTIL OFF) set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) + +# TODO(nvcomp): when `nvcomp` supports Python 3.13 and we de-vendor `nvcomp` from `kvikio` the +# libkvikio branch should be removed if(USE_NVCOMP_RUNTIME_WHEEL OR USE_NVCOMP_FROM_LIBKVIKIO_WHEEL) set(CUDF_EXPORT_NVCOMP OFF) endif() @@ -57,6 +63,8 @@ if(USE_NVCOMP_RUNTIME_WHEEL) ) endif() +# TODO(nvcomp): when `nvcomp` supports Python 3.13 and we de-vendor `nvcomp` from `kvikio` this +# block should be removed if(USE_NVCOMP_FROM_LIBKVIKIO_WHEEL) set(rpaths "$ORIGIN/../../libkvikio/lib64") set_property(