From a3e7ad59cdf2162e574995cfedab0ca34fa5de79 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 28 Aug 2025 19:56:38 -0500 Subject: [PATCH 1/2] restore cudf dependency --- ci/run_benchmarks.sh | 7 ----- .../all_cuda-130_arch-aarch64.yaml | 2 ++ .../all_cuda-130_arch-x86_64.yaml | 2 ++ dask_cuda/tests/test_cudf_builtin_spilling.py | 2 +- dask_cuda/tests/test_spill.py | 6 ++--- dependencies.yaml | 26 +++---------------- pyproject.toml | 2 ++ 7 files changed, 14 insertions(+), 33 deletions(-) diff --git a/ci/run_benchmarks.sh b/ci/run_benchmarks.sh index 37ac6bdf..b2c8dd36 100755 --- a/ci/run_benchmarks.sh +++ b/ci/run_benchmarks.sh @@ -6,13 +6,6 @@ set -euo pipefail # Support invoking run_benchmarks.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../dask_cuda -# TODO: run cudf benchmarks unconditionally once there are CUDA 13 cudf packages -# ref: https://github.com/rapidsai/dask-cuda/pull/1536#issuecomment-321247489 -if [[ ! $(python -c "import cudf") ]]; then - echo "skipping benchmarks: cudf packages not available" - exit 0 -fi - python benchmarks/local_cudf_shuffle.py \ --partition-size="1 KiB" \ -d 0 \ diff --git a/conda/environments/all_cuda-130_arch-aarch64.yaml b/conda/environments/all_cuda-130_arch-aarch64.yaml index e6b70cff..0eb9b209 100644 --- a/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -10,6 +10,8 @@ dependencies: - cuda-nvcc-impl - cuda-nvrtc - cuda-version=13.0 +- cudf==25.10.*,>=0.0.0a0 +- dask-cudf==25.10.*,>=0.0.0a0 - distributed-ucxx==0.46.*,>=0.0.0a0 - kvikio==25.10.*,>=0.0.0a0 - numba-cuda>=0.19.1,<0.20.0a0 diff --git a/conda/environments/all_cuda-130_arch-x86_64.yaml b/conda/environments/all_cuda-130_arch-x86_64.yaml index 5cb15cc6..a9dfa77a 100644 --- a/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -10,6 +10,8 @@ dependencies: - cuda-nvcc-impl - cuda-nvrtc - cuda-version=13.0 +- cudf==25.10.*,>=0.0.0a0 +- dask-cudf==25.10.*,>=0.0.0a0 - distributed-ucxx==0.46.*,>=0.0.0a0 - kvikio==25.10.*,>=0.0.0a0 - numba-cuda>=0.19.1,<0.20.0a0 diff --git a/dask_cuda/tests/test_cudf_builtin_spilling.py b/dask_cuda/tests/test_cudf_builtin_spilling.py index e7a4e474..7d21fc85 100644 --- a/dask_cuda/tests/test_cudf_builtin_spilling.py +++ b/dask_cuda/tests/test_cudf_builtin_spilling.py @@ -8,7 +8,6 @@ from dask_cuda.is_spillable_object import is_spillable_object from dask_cuda.proxify_host_file import ProxifyHostFile -cudf = pytest.importorskip("cudf") cupy = pytest.importorskip("cupy") pandas = pytest.importorskip("pandas") @@ -17,6 +16,7 @@ reason="Current version of cudf doesn't support built-in spilling", ) +import cudf # noqa: E402 from cudf.core.buffer.spill_manager import ( # noqa: E402 SpillManager, get_global_manager, diff --git a/dask_cuda/tests/test_spill.py b/dask_cuda/tests/test_spill.py index 8169d852..daa4c5bc 100644 --- a/dask_cuda/tests/test_spill.py +++ b/dask_cuda/tests/test_spill.py @@ -15,10 +15,10 @@ from distributed.utils import Deadline from distributed.utils_test import gen_cluster, gen_test, loop # noqa: F401 -dask_cudf = pytest.importorskip("dask_cudf") +import dask_cudf -from dask_cuda import LocalCUDACluster, utils # noqa: E402 -from dask_cuda.utils_test import IncreasedCloseTimeoutNanny # noqa: E402 +from dask_cuda import LocalCUDACluster, utils +from dask_cuda.utils_test import IncreasedCloseTimeoutNanny if not utils.has_device_memory_resource(): pytest.skip( diff --git a/dependencies.yaml b/dependencies.yaml index 886f54d3..ca8fc71e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -179,10 +179,8 @@ dependencies: common: - output_types: [pyproject] packages: - # TODO: add 'cudf' and 'dask-cudf' back to this dependency list once there are CUDA 13 packages for those - # ref: https://github.com/rapidsai/dask-cuda/pull/1536#issuecomment-3212474898 - # - cudf-cu13==25.10.*,>=0.0.0a0 - # - dask-cudf-cu13==25.10.*,>=0.0.0a0 + - cudf-cu13==25.10.*,>=0.0.0a0 + - dask-cudf-cu13==25.10.*,>=0.0.0a0 - distributed-ucxx-cu13==0.46.*,>=0.0.0a0 - kvikio-cu13==25.10.*,>=0.0.0a0 - ucx-py-cu13==0.46.*,>=0.0.0a0 @@ -194,30 +192,14 @@ dependencies: - pytest - pytest-cov - pytest-timeout - - output_types: [requirements, pyproject] - packages: - # ensure dependencies of 'dask.dataframe' are available - # ref: https://github.com/rapidsai/dask-cuda/pull/1536#discussion_r2298528510 - # TODO: remove this once there are CUDA 13 'dask-cudf' packages - - 'dask[dataframe]' - output_types: [conda] packages: + - cudf==25.10.*,>=0.0.0a0 + - dask-cudf==25.10.*,>=0.0.0a0 - distributed-ucxx==0.46.*,>=0.0.0a0 - kvikio==25.10.*,>=0.0.0a0 - ucx-py==0.46.*,>=0.0.0a0 - numba-cuda>=0.19.1,<0.20.0a0 - specific: - - output_types: [conda] - matrices: - - matrix: - cuda: "12.*" - packages: - # TODO: add 'cudf' and 'dask-cudf' back to the 'common' list above once there are CUDA 13 packages for those - # ref: https://github.com/rapidsai/dask-cuda/pull/1536#issuecomment-3212474898 - - cudf==25.10.*,>=0.0.0a0 - - dask-cudf==25.10.*,>=0.0.0a0 - - matrix: - packages: depends_on_dask_cuda: common: - output_types: conda diff --git a/pyproject.toml b/pyproject.toml index 48dfa08d..6ebd852c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,8 @@ cu12 = [ "ucx-py-cu12==0.46.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. cu13 = [ + "cudf-cu13==25.10.*,>=0.0.0a0", + "dask-cudf-cu13==25.10.*,>=0.0.0a0", "distributed-ucxx-cu13==0.46.*,>=0.0.0a0", "kvikio-cu13==25.10.*,>=0.0.0a0", "numba-cuda[cu13]>=0.19.1,<0.20.0a0", From 644214e2e16874ab70bc0d4a3f450a405254059c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 29 Aug 2025 08:41:08 -0500 Subject: [PATCH 2/2] revert importorskip --- dask_cuda/tests/test_cudf_builtin_spilling.py | 2 +- dask_cuda/tests/test_spill.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dask_cuda/tests/test_cudf_builtin_spilling.py b/dask_cuda/tests/test_cudf_builtin_spilling.py index 7d21fc85..e7a4e474 100644 --- a/dask_cuda/tests/test_cudf_builtin_spilling.py +++ b/dask_cuda/tests/test_cudf_builtin_spilling.py @@ -8,6 +8,7 @@ from dask_cuda.is_spillable_object import is_spillable_object from dask_cuda.proxify_host_file import ProxifyHostFile +cudf = pytest.importorskip("cudf") cupy = pytest.importorskip("cupy") pandas = pytest.importorskip("pandas") @@ -16,7 +17,6 @@ reason="Current version of cudf doesn't support built-in spilling", ) -import cudf # noqa: E402 from cudf.core.buffer.spill_manager import ( # noqa: E402 SpillManager, get_global_manager, diff --git a/dask_cuda/tests/test_spill.py b/dask_cuda/tests/test_spill.py index daa4c5bc..8169d852 100644 --- a/dask_cuda/tests/test_spill.py +++ b/dask_cuda/tests/test_spill.py @@ -15,10 +15,10 @@ from distributed.utils import Deadline from distributed.utils_test import gen_cluster, gen_test, loop # noqa: F401 -import dask_cudf +dask_cudf = pytest.importorskip("dask_cudf") -from dask_cuda import LocalCUDACluster, utils -from dask_cuda.utils_test import IncreasedCloseTimeoutNanny +from dask_cuda import LocalCUDACluster, utils # noqa: E402 +from dask_cuda.utils_test import IncreasedCloseTimeoutNanny # noqa: E402 if not utils.has_device_memory_resource(): pytest.skip(