diff --git a/.devcontainer/README.md b/.devcontainer/README.md
index cc5fac22fde2..91ee7ef85f78 100644
--- a/.devcontainer/README.md
+++ b/.devcontainer/README.md
@@ -20,7 +20,6 @@ This container is a turnkey development environment for building and testing the
By default, the following directories are bind-mounted into the devcontainer:
* `${repo}:/home/coder/cudf`
-* `${repo}/../rapidsmpf:/home/coder/rapidsmpf`
* `${repo}/../.aws:/home/coder/.aws`
* `${repo}/../.local:/home/coder/.local`
* `${repo}/../.cache:/home/coder/.cache`
@@ -29,13 +28,6 @@ By default, the following directories are bind-mounted into the devcontainer:
This ensures caches, configurations, dependencies, and your commits are persisted on the host across container runs.
-The [rapidsmpf](https://github.com/rapidsai/rapidsmpf) repository is a required dependency of `cudf_polars` (that also requires `libcudf`) and must be cloned as a sibling directory to the cudf repo before launching the devcontainer:
-
-```
-# from the parent directory of your cudf clone
-git clone https://github.com/rapidsai/rapidsmpf.git
-```
-
## Launch a Dev Container
To launch a devcontainer from VSCode, open the cuDF repo and select the "Reopen in Container" button in the bottom right:
diff --git a/.devcontainer/cuda12.9-conda/devcontainer.json b/.devcontainer/cuda12.9-conda/devcontainer.json
index f8581936a262..5326e64d8c6c 100644
--- a/.devcontainer/cuda12.9-conda/devcontainer.json
+++ b/.devcontainer/cuda12.9-conda/devcontainer.json
@@ -52,7 +52,7 @@
"initializeCommand": [
"/bin/bash",
"-c",
- "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.9-envs} ${localWorkspaceFolder}/../rapidsmpf"
+ "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.9-envs}"
],
"postAttachCommand": [
"/bin/bash",
@@ -66,8 +66,7 @@
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.9-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
+ "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.9-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
],
"customizations": {
"vscode": {
diff --git a/.devcontainer/cuda12.9-pip/devcontainer.json b/.devcontainer/cuda12.9-pip/devcontainer.json
index 03851a926c8c..914237875c0f 100644
--- a/.devcontainer/cuda12.9-pip/devcontainer.json
+++ b/.devcontainer/cuda12.9-pip/devcontainer.json
@@ -30,7 +30,7 @@
"initializeCommand": [
"/bin/bash",
"-c",
- "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs} ${localWorkspaceFolder}/../rapidsmpf"
+ "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs}"
],
"postAttachCommand": [
"/bin/bash",
@@ -43,8 +43,7 @@
"source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
+ "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent"
],
"customizations": {
"vscode": {
diff --git a/.devcontainer/cuda13.2-conda/devcontainer.json b/.devcontainer/cuda13.2-conda/devcontainer.json
index 78a2f7f64963..a7987bd05dff 100644
--- a/.devcontainer/cuda13.2-conda/devcontainer.json
+++ b/.devcontainer/cuda13.2-conda/devcontainer.json
@@ -52,7 +52,7 @@
"initializeCommand": [
"/bin/bash",
"-c",
- "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda13.2-envs} ${localWorkspaceFolder}/../rapidsmpf"
+ "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda13.2-envs}"
],
"postAttachCommand": [
"/bin/bash",
@@ -66,8 +66,7 @@
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda13.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
+ "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda13.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
],
"customizations": {
"vscode": {
diff --git a/.devcontainer/cuda13.2-pip/devcontainer.json b/.devcontainer/cuda13.2-pip/devcontainer.json
index 7a283715b2e8..181ba40473cc 100644
--- a/.devcontainer/cuda13.2-pip/devcontainer.json
+++ b/.devcontainer/cuda13.2-pip/devcontainer.json
@@ -30,7 +30,7 @@
"initializeCommand": [
"/bin/bash",
"-c",
- "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs} ${localWorkspaceFolder}/../rapidsmpf"
+ "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs}"
],
"postAttachCommand": [
"/bin/bash",
@@ -43,8 +43,7 @@
"source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent",
- "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
+ "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent"
],
"customizations": {
"vscode": {
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
index 031e26e4a048..1a345e4a498f 100644
--- a/.github/workflows/pr.yaml
+++ b/.github/workflows/pr.yaml
@@ -646,7 +646,7 @@ jobs:
package-type: python
pure-wheel: true
wheel-tests-cudf-polars:
- needs: [wheel-build-cudf-polars, changed-files]
+ needs: [wheel-build-cudf-polars, wheel-build-cudf-streaming, changed-files]
permissions:
actions: read
contents: read
@@ -663,7 +663,7 @@ jobs:
container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"
script: "ci/test_wheel_cudf_polars.sh"
cudf-polars-polars-tests:
- needs: [wheel-build-cudf-polars, changed-files]
+ needs: [wheel-build-cudf-polars, wheel-build-cudf-streaming, changed-files]
permissions:
actions: read
contents: read
@@ -732,14 +732,9 @@ jobs:
SCCACHE_DIST_MAX_RETRIES=inf
SCCACHE_SERVER_LOG=sccache=debug
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false
- # Call clone-rapidsmpf before creating the Python env to exclude cudf_polars' dependency on rapidsmpf
- # librapidsmpf-cu13 wheels brings in a hardcoded libnuma-dev cmake target: https://github.com/NVIDIA/cuCascade/issues/118
- # -DBUILD_TESTS=OFF to match rapidsmpf https://github.com/rapidsai/rapidsmpf/blob/main/.github/workflows/pr.yaml#L351 (leads to compilation errors)
- # -DCUDF_BUILD_TESTUTIL=OFF to avoid IMPORTED_GLOBAL promotion errors when cuCascade's find_package(cudf) loads cudf-config.cmake from a CPM subdirectory
build_command: |
sccache --zero-stats;
- clone-rapidsmpf -j$(nproc) -v -q --branch "$(cat ~/cudf/RAPIDS_BRANCH)" --clone-upstream --depth 1 --single-branch --shallow-submodules;
- build-all -j0 -DBUILD_BENCHMARKS=OFF -DBUILD_NUMA_SUPPORT=OFF -DBUILD_TESTS=OFF -DCUDF_BUILD_TESTUTIL=OFF --verbose 2>&1 | tee telemetry-artifacts/build.log;
+ build-all -j0 -DBUILD_BENCHMARKS=ON --verbose 2>&1 | tee telemetry-artifacts/build.log;
sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt;
unit-tests-cudf-pandas:
needs: [wheel-build-cudf, changed-files]
diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh
index 1fb9a4e395f6..50d1ee861f16 100755
--- a/ci/test_cudf_polars_polars_tests.sh
+++ b/ci/test_cudf_polars_polars_tests.sh
@@ -12,6 +12,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")"
CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-github python)
LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")")
+LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
+CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" cudf_streaming --stable --cuda "$RAPIDS_CUDA_VERSION")")
# generate constraints (possibly pinning to oldest support versions of dependencies)
rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}"
@@ -29,7 +31,9 @@ rapids-pip-retry install \
--constraint "${PIP_CONSTRAINT}" \
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
- "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)"
+ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+ "$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+ "$(echo "${CUDF_STREAMING_WHEELHOUSE}"/cudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)"
TAG=$(python -c 'import polars; print(f"py-{polars.__version__}")')
rapids-logger "Clone polars to ${TAG}"
diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh
index 0b73e904dc47..61185c21568a 100755
--- a/ci/test_wheel_cudf_polars.sh
+++ b/ci/test_wheel_cudf_polars.sh
@@ -15,6 +15,10 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF
LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")")
+# Download libcudf_streaming and cudf_streaming built in the previous step
+LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
+CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" cudf_streaming --stable --cuda "$RAPIDS_CUDA_VERSION")")
+
# generate constraints (possibly pinning to oldest support versions of dependencies)
rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}"
@@ -65,6 +69,8 @@ for version in "${VERSIONS[@]}"; do
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,dask,ray]" \
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
"$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+ "$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+ "$(echo "${CUDF_STREAMING_WHEELHOUSE}"/cudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
-r "polars-compat-${version}-requirements.txt"
rapids-logger "Running tests for polars ${version}.*"
diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml
index 838a2c557b47..7e0ad88ddb96 100644
--- a/conda/environments/all_cuda-129_arch-aarch64.yaml
+++ b/conda/environments/all_cuda-129_arch-aarch64.yaml
@@ -44,6 +44,7 @@ dependencies:
- libkvikio==26.8.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvjitlink-dev
+- librapidsmpf==26.8.*,>=0.0.0a0
- librdkafka>=2.11.1,<2.12.0
- librmm==26.8.*,>=0.0.0a0
- make
@@ -87,6 +88,7 @@ dependencies:
- rapids-build-backend>=0.4.0,<0.5.0
- rapids-dask-dependency==26.8.*,>=0.0.0a0
- rapids-logger==0.2.*,>=0.0.0a0
+- rapidsmpf==26.8.*,>=0.0.0a0
- ray-default>=2.55.1
- rich
- rmm==26.8.*,>=0.0.0a0
diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml
index 0a435504e32f..272914ad7635 100644
--- a/conda/environments/all_cuda-129_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-129_arch-x86_64.yaml
@@ -44,6 +44,7 @@ dependencies:
- libkvikio==26.8.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvjitlink-dev
+- librapidsmpf==26.8.*,>=0.0.0a0
- librdkafka>=2.11.1,<2.12.0
- librmm==26.8.*,>=0.0.0a0
- make
@@ -87,6 +88,7 @@ dependencies:
- rapids-build-backend>=0.4.0,<0.5.0
- rapids-dask-dependency==26.8.*,>=0.0.0a0
- rapids-logger==0.2.*,>=0.0.0a0
+- rapidsmpf==26.8.*,>=0.0.0a0
- ray-default>=2.55.1
- rich
- rmm==26.8.*,>=0.0.0a0
diff --git a/conda/environments/all_cuda-132_arch-aarch64.yaml b/conda/environments/all_cuda-132_arch-aarch64.yaml
index 2093888e01fa..9ecba408142c 100644
--- a/conda/environments/all_cuda-132_arch-aarch64.yaml
+++ b/conda/environments/all_cuda-132_arch-aarch64.yaml
@@ -44,6 +44,7 @@ dependencies:
- libkvikio==26.8.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvjitlink-dev
+- librapidsmpf==26.8.*,>=0.0.0a0
- librdkafka>=2.11.1,<2.12.0
- librmm==26.8.*,>=0.0.0a0
- make
@@ -87,6 +88,7 @@ dependencies:
- rapids-build-backend>=0.4.0,<0.5.0
- rapids-dask-dependency==26.8.*,>=0.0.0a0
- rapids-logger==0.2.*,>=0.0.0a0
+- rapidsmpf==26.8.*,>=0.0.0a0
- ray-default>=2.55.1
- rich
- rmm==26.8.*,>=0.0.0a0
diff --git a/conda/environments/all_cuda-132_arch-x86_64.yaml b/conda/environments/all_cuda-132_arch-x86_64.yaml
index 360550447330..ced21db00730 100644
--- a/conda/environments/all_cuda-132_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-132_arch-x86_64.yaml
@@ -44,6 +44,7 @@ dependencies:
- libkvikio==26.8.*,>=0.0.0a0
- libnvcomp-dev==5.2.0.10
- libnvjitlink-dev
+- librapidsmpf==26.8.*,>=0.0.0a0
- librdkafka>=2.11.1,<2.12.0
- librmm==26.8.*,>=0.0.0a0
- make
@@ -87,6 +88,7 @@ dependencies:
- rapids-build-backend>=0.4.0,<0.5.0
- rapids-dask-dependency==26.8.*,>=0.0.0a0
- rapids-logger==0.2.*,>=0.0.0a0
+- rapidsmpf==26.8.*,>=0.0.0a0
- ray-default>=2.55.1
- rich
- rmm==26.8.*,>=0.0.0a0
diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml
index 79ebec2963cb..12aa1e19f162 100644
--- a/conda/recipes/cudf-polars/recipe.yaml
+++ b/conda/recipes/cudf-polars/recipe.yaml
@@ -37,6 +37,7 @@ requirements:
- python
- pylibcudf =${{ version }}
- rapidsmpf =${{ minor_version }}
+ - cudf-streaming =${{ version }}
- polars>=1.35,<1.42
- packaging
- ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }}
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 064549c401dd..960c1c61eb43 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -290,6 +290,16 @@ endfunction()
# * conda environment -----------------------------------------------------------------------------
rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH)
+# ##################################################################################################
+# * find C++ libraries from RAPIDS wheels in in site-packages --------------------------------------
+find_package(Python COMPONENTS Interpreter)
+if(Python_FOUND)
+ include(${rapids-cmake-dir}/cython-core/find_prefix_paths.cmake)
+ rapids_cython_find_prefix_paths("${Python_EXECUTABLE}" wheel_prefix_paths)
+ set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS TRUE)
+ list(APPEND CMAKE_PREFIX_PATH ${wheel_prefix_paths})
+endif()
+
# ##################################################################################################
# * compiler options ------------------------------------------------------------------------------
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules ${CMAKE_MODULE_PATH})
diff --git a/dependencies.yaml b/dependencies.yaml
index 25ed1071f03d..0409d0c4cfe2 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -23,8 +23,10 @@ files:
- depends_on_libkvikio
- depends_on_librmm
- depends_on_libnvcomp
+ - depends_on_librapidsmpf
- depends_on_numba_cuda
- depends_on_rapids_logger
+ - depends_on_rapidsmpf
- depends_on_ray
- depends_on_rmm
- develop
@@ -335,6 +337,7 @@ files:
includes:
- run_cudf_polars
- depends_on_rapidsmpf
+ - depends_on_cudf_streaming
- depends_on_pylibcudf
- depends_on_cuda_python
# TODO(26.10): Remove this alias in favor of dask
@@ -1414,7 +1417,27 @@ dependencies:
common:
- output_types: conda
packages:
- - cudf-streaming==26.8.*,>=0.0.0a0
+ - &cudf_streaming_unsuffixed cudf-streaming==26.8.*,>=0.0.0a0
+ - output_types: requirements
+ packages:
+ # pip recognizes the index as a global option for the requirements.txt file
+ - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+ specific:
+ - output_types: [requirements, pyproject]
+ matrices:
+ - matrix:
+ cuda: "12.*"
+ cuda_suffixed: "true"
+ packages:
+ - cudf-streaming-cu12==26.8.*,>=0.0.0a0
+ - matrix:
+ cuda: "13.*"
+ cuda_suffixed: "true"
+ packages:
+ - cudf-streaming-cu13==26.8.*,>=0.0.0a0
+ - matrix:
+ packages:
+ - *cudf_streaming_unsuffixed
depends_on_ray:
common:
- output_types: conda
diff --git a/python/cudf_polars/cudf_polars/engine/core.py b/python/cudf_polars/cudf_polars/engine/core.py
index 79914d12fe8d..bfe97050ccf3 100644
--- a/python/cudf_polars/cudf_polars/engine/core.py
+++ b/python/cudf_polars/cudf_polars/engine/core.py
@@ -14,14 +14,15 @@
from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeVar
import cuda.core
+
+import polars as pl
+
+from cudf_streaming.streaming.table_chunk import TableChunk
from rapidsmpf.coll import AllGather
from rapidsmpf.config import Options, get_environment_variables
from rapidsmpf.memory.packed_data import PackedData
from rapidsmpf.statistics import Statistics
from rapidsmpf.streaming.core.actor import run_actor_network
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
-import polars as pl
from cudf_polars.containers import DataFrame
from cudf_polars.dsl.ir import IRExecutionContext
@@ -42,10 +43,10 @@
from concurrent.futures import Executor, ThreadPoolExecutor
import rapidsmpf.config
+ from cudf_streaming.streaming.channel_metadata import ChannelMetadata
from rapidsmpf.communicator.communicator import Communicator
from rapidsmpf.memory.buffer_resource import BufferResource
from rapidsmpf.streaming.core.context import Context
- from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
from cudf_polars.dsl.ir import IR
from cudf_polars.streaming.base import PartitionInfo
diff --git a/python/cudf_polars/cudf_polars/engine/dask.py b/python/cudf_polars/cudf_polars/engine/dask.py
index 014627b82c48..46372d0dad04 100644
--- a/python/cudf_polars/cudf_polars/engine/dask.py
+++ b/python/cudf_polars/cudf_polars/engine/dask.py
@@ -16,6 +16,10 @@
import distributed.system
import pynvml
import ucxx._lib.libucxx as ucx_api
+
+import polars as pl
+
+import rmm.mr
from rapidsmpf import bootstrap
from rapidsmpf.communicator.ucxx import barrier, get_root_ucxx_address, new_communicator
from rapidsmpf.config import Options
@@ -24,10 +28,6 @@
from rapidsmpf.statistics import Statistics
from rapidsmpf.streaming.core.context import Context
-import polars as pl
-
-import rmm.mr
-
from cudf_polars.engine.core import (
ClusterInfo,
StreamingEngine,
@@ -44,8 +44,8 @@
if TYPE_CHECKING:
from collections.abc import Callable
+ from cudf_streaming.streaming.channel_metadata import ChannelMetadata
from rapidsmpf.communicator.communicator import Communicator
- from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
from cudf_polars.dsl.ir import IR
from cudf_polars.engine.core import T
diff --git a/python/cudf_polars/cudf_polars/engine/ray.py b/python/cudf_polars/cudf_polars/engine/ray.py
index edf1905e916d..b8824ee55414 100644
--- a/python/cudf_polars/cudf_polars/engine/ray.py
+++ b/python/cudf_polars/cudf_polars/engine/ray.py
@@ -12,6 +12,10 @@
import ray
import ucxx._lib.libucxx as ucx_api
+
+import polars as pl
+
+import rmm.mr
from rapidsmpf import bootstrap
from rapidsmpf.communicator.ucxx import barrier, get_root_ucxx_address, new_communicator
from rapidsmpf.config import Options
@@ -20,10 +24,6 @@
from rapidsmpf.statistics import Statistics
from rapidsmpf.streaming.core.context import Context
-import polars as pl
-
-import rmm.mr
-
from cudf_polars.engine.core import (
ClusterInfo,
StreamingEngine,
@@ -41,10 +41,11 @@
import uuid
from collections.abc import Callable
- from rapidsmpf.communicator.communicator import Communicator
- from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
from ray.actor import ActorHandle
+ from cudf_streaming.streaming.channel_metadata import ChannelMetadata
+ from rapidsmpf.communicator.communicator import Communicator
+
from cudf_polars.dsl.ir import IR
from cudf_polars.engine.core import T
from cudf_polars.engine.options import StreamingOptions
diff --git a/python/cudf_polars/cudf_polars/engine/spmd.py b/python/cudf_polars/cudf_polars/engine/spmd.py
index d334e9f1c95c..96fe41de72b1 100644
--- a/python/cudf_polars/cudf_polars/engine/spmd.py
+++ b/python/cudf_polars/cudf_polars/engine/spmd.py
@@ -10,22 +10,23 @@
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, cast
+import pylibcudf as plc
+from cudf_streaming.integrations.partition import (
+ packed_data_from_cudf_packed_columns,
+ unpack_and_concat,
+)
+from pylibcudf.contiguous_split import pack
from rapidsmpf import bootstrap
from rapidsmpf.coll import AllGather
from rapidsmpf.communicator.single import (
new_communicator as single_communicator,
)
from rapidsmpf.communicator.ucxx import barrier
-from rapidsmpf.integrations.cudf.partition import unpack_and_concat
-from rapidsmpf.memory.packed_data import PackedData
from rapidsmpf.progress_thread import ProgressThread
from rapidsmpf.rmm_resource_adaptor import RmmResourceAdaptor
from rapidsmpf.statistics import Statistics
from rapidsmpf.streaming.core.context import Context
-import pylibcudf as plc
-from pylibcudf.contiguous_split import pack
-
from cudf_polars.containers import DataFrame, DataType
from cudf_polars.engine.core import (
ClusterInfo,
@@ -51,11 +52,11 @@
import uuid
from collections.abc import Callable
+ import polars as pl
+
+ from cudf_streaming.streaming.channel_metadata import ChannelMetadata
from rapidsmpf.communicator.communicator import Communicator
from rapidsmpf.config import Options
- from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
-
- import polars as pl
from cudf_polars.dsl.ir import IR
from cudf_polars.engine.core import T
@@ -162,12 +163,11 @@ def allgather_polars_dataframe(
plc_table = plc.Table.from_arrow(local_df, stream=stream)
- packed_data = PackedData.from_cudf_packed_columns(
+ packed_data = packed_data_from_cudf_packed_columns(
pack(plc_table, stream),
stream,
ctx.br(),
)
-
# Bulk AllGather: each rank contributes once (sequence_number=0)
allgather = AllGather(comm, op_id, ctx.br())
try:
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py
index 911e3f270920..8939e50d8121 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py
@@ -6,18 +6,18 @@
from typing import TYPE_CHECKING
-from rapidsmpf.integrations.cudf.partition import unpack_and_concat
-from rapidsmpf.memory.packed_data import PackedData
-from rapidsmpf.streaming.coll.allgather import AllGather
-
+from cudf_streaming.integrations.partition import (
+ packed_data_from_cudf_packed_columns,
+ unpack_and_concat,
+)
from pylibcudf.contiguous_split import pack
+from rapidsmpf.streaming.coll.allgather import AllGather
if TYPE_CHECKING:
+ import pylibcudf as plc
+ from cudf_streaming.streaming.table_chunk import TableChunk
from rapidsmpf.communicator.communicator import Communicator
from rapidsmpf.streaming.core.context import Context
- from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
- import pylibcudf as plc
from rmm.pylibrmm.stream import Stream
from cudf_polars.dsl.ir import IRExecutionContext
@@ -72,7 +72,7 @@ def insert(self, sequence_number: int, chunk: TableChunk) -> None:
sequence_number,
# TODO: Avoid unnecessary copies.
# See https://github.com/rapidsai/rapidsmpf/issues/933
- PackedData.from_cudf_packed_columns(
+ packed_data_from_cudf_packed_columns(
pack(
chunk.table_view(),
chunk.stream,
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py
index 7da5259afd13..dd78104d2e96 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py
@@ -6,27 +6,26 @@
from typing import TYPE_CHECKING, Any
-from rapidsmpf.communicator.single import new_communicator as single_comm
-from rapidsmpf.config import Options, get_environment_variables
-from rapidsmpf.integrations.cudf.partition import (
+import pylibcudf as plc
+import pylibcudf.partitioning
+from cudf_streaming.integrations.partition import (
partition_and_pack as py_partition_and_pack,
split_and_pack as py_split_and_pack,
unpack_and_concat as py_unpack_and_concat,
)
+from cudf_streaming.streaming.channel_metadata import (
+ ChannelMetadata,
+ HashScheme,
+ Partitioning,
+)
+from cudf_streaming.streaming.table_chunk import TableChunk
+from rapidsmpf.communicator.single import new_communicator as single_comm
+from rapidsmpf.config import Options, get_environment_variables
from rapidsmpf.shuffler import PartitionAssignment
from rapidsmpf.streaming.coll.shuffler import ShufflerAsync
from rapidsmpf.streaming.core.actor import define_actor
from rapidsmpf.streaming.core.context import Context
from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import (
- ChannelMetadata,
- HashScheme,
- Partitioning,
-)
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
-import pylibcudf as plc
-import pylibcudf.partitioning
from cudf_polars.dsl.expr import Col
from cudf_polars.streaming.actor_graph.dispatch import (
@@ -48,7 +47,6 @@
from rapidsmpf.communicator.communicator import Communicator
from rapidsmpf.memory.packed_data import PackedData
from rapidsmpf.streaming.core.channel import Channel
-
from rmm.pylibrmm.stream import Stream
from cudf_polars.dsl.ir import IR, IRExecutionContext
@@ -445,7 +443,7 @@ async def shuffle_actor(
This node combines partition_and_pack, shuffler, and unpack_and_concat
into a single Python node using rapidsmpf.shuffler.Shuffler and utilities
- from rapidsmpf.integrations.cudf.partition.
+ from cudf_streaming.integrations.partition.
Parameters
----------
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py
index 75087141c9b2..f62f2fa05f76 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py
@@ -6,20 +6,19 @@
from typing import TYPE_CHECKING
-from rapidsmpf.shuffler import PartitionAssignment
-from rapidsmpf.streaming.core.actor import define_actor
-from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import (
+import polars as pl
+
+import pylibcudf as plc
+from cudf_streaming.streaming.channel_metadata import (
ChannelMetadata,
OrderKey,
OrderScheme,
Partitioning,
)
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
-import polars as pl
-
-import pylibcudf as plc
+from cudf_streaming.streaming.table_chunk import TableChunk
+from rapidsmpf.shuffler import PartitionAssignment
+from rapidsmpf.streaming.core.actor import define_actor
+from rapidsmpf.streaming.core.message import Message
from cudf_polars.containers import DataFrame, DataType
from cudf_polars.dsl.expr import Col, NamedExpr
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py
index fb7cc4e1439a..1bd440a239f5 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py
@@ -30,14 +30,14 @@
if TYPE_CHECKING:
from collections.abc import MutableMapping
+ import polars as pl
+
+ from cudf_streaming.streaming.channel_metadata import ChannelMetadata
+ from cudf_streaming.streaming.table_chunk import TableChunk
from rapidsmpf.communicator.communicator import Communicator
from rapidsmpf.streaming.core.channel import Channel
from rapidsmpf.streaming.core.context import Context
from rapidsmpf.streaming.core.leaf_actor import DeferredMessages
- from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
- from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
- import polars as pl
from cudf_polars.dsl.ir import IR, IRExecutionContext
from cudf_polars.streaming.actor_graph.dispatch import (
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py
index c9f65da127c1..df2c8d04f4c7 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py
@@ -7,18 +7,17 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
+import pylibcudf as plc
+from cudf_streaming.streaming.channel_metadata import (
+ ChannelMetadata,
+ HashScheme,
+)
+from cudf_streaming.streaming.table_chunk import TableChunk
from rapidsmpf.communicator.single import new_communicator as single_comm
from rapidsmpf.config import Options, get_environment_variables
from rapidsmpf.streaming.core.actor import define_actor
from rapidsmpf.streaming.core.context import Context
from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import (
- ChannelMetadata,
- HashScheme,
-)
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
-import pylibcudf as plc
from cudf_polars.containers import DataType
from cudf_polars.dsl.expr import Col, NamedExpr
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py
index 7f4db5e38bb3..f30b55dbdd3d 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py
@@ -8,15 +8,14 @@
import math
from typing import TYPE_CHECKING, Any
+import pylibcudf as plc
+from cudf_streaming.streaming.channel_metadata import ChannelMetadata
+from cudf_streaming.streaming.table_chunk import TableChunk
from rapidsmpf.memory.memory_reservation import opaque_memory_usage
from rapidsmpf.streaming.core.memory_reserve_or_wait import (
reserve_memory,
)
from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
-import pylibcudf as plc
from cudf_polars.dsl.ir import (
IR,
@@ -481,7 +480,7 @@ def make_rapidsmpf_read_parquet_node(
The RapidsMPF read parquet node, or None if the predicate cannot be
converted to a parquet filter (caller should fall back to scan_node).
"""
- from rapidsmpf.streaming.cudf.parquet import Filter, read_parquet
+ from cudf_streaming.streaming.parquet import Filter, read_parquet
# Build ParquetReaderOptions
try:
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py
index 0130146576a6..fa99192aaf12 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py
@@ -7,25 +7,24 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal
-from rapidsmpf.memory.memory_reservation import opaque_memory_usage
-from rapidsmpf.streaming.core.actor import define_actor
-from rapidsmpf.streaming.core.memory_reserve_or_wait import (
- reserve_memory,
-)
-from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.bloom_filter import BloomFilter
-from rapidsmpf.streaming.cudf.channel_metadata import (
+import pylibcudf as plc
+from cudf_streaming.streaming.bloom_filter import BloomFilter
+from cudf_streaming.streaming.channel_metadata import (
ChannelMetadata,
HashScheme,
Partitioning,
)
-from rapidsmpf.streaming.cudf.table_chunk import (
+from cudf_streaming.streaming.table_chunk import (
TableChunk,
make_table_chunks_available_or_wait,
)
-
-import pylibcudf as plc
from pylibcudf.hashing import LIBCUDF_DEFAULT_HASH_SEED
+from rapidsmpf.memory.memory_reservation import opaque_memory_usage
+from rapidsmpf.streaming.core.actor import define_actor
+from rapidsmpf.streaming.core.memory_reserve_or_wait import (
+ reserve_memory,
+)
+from rapidsmpf.streaming.core.message import Message
from cudf_polars.containers import DataFrame
from cudf_polars.dsl.ir import IR, Join
@@ -60,10 +59,10 @@
from collections.abc import Iterable, MutableMapping
from types import CoroutineType
+ from cudf_streaming.streaming.bloom_filter import BloomFilterChunk
from rapidsmpf.communicator.communicator import Communicator
from rapidsmpf.streaming.core.channel import Channel
from rapidsmpf.streaming.core.context import Context
- from rapidsmpf.streaming.cudf.bloom_filter import BloomFilterChunk
from cudf_polars.dsl.ir import IR, IRExecutionContext
from cudf_polars.streaming.actor_graph.dispatch import SubNetGenerator
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py
index 9432fd7295d8..777fb02c7a64 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py
@@ -7,16 +7,16 @@
import asyncio
from typing import TYPE_CHECKING, Any, cast
+from cudf_streaming.streaming.channel_metadata import ChannelMetadata
+from cudf_streaming.streaming.table_chunk import (
+ TableChunk,
+ make_table_chunks_available_or_wait,
+)
from rapidsmpf.memory.buffer import MemoryType
from rapidsmpf.memory.memory_reservation import opaque_memory_usage
from rapidsmpf.streaming.core.actor import define_actor
from rapidsmpf.streaming.core.message import Message
from rapidsmpf.streaming.core.spillable_messages import SpillableMessages
-from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
-from rapidsmpf.streaming.cudf.table_chunk import (
- TableChunk,
- make_table_chunks_available_or_wait,
-)
from cudf_polars.containers import DataFrame
from cudf_polars.dsl.ir import IR, Empty
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py
index d1569d561b21..4b0f20db5f7b 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py
@@ -40,19 +40,18 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, ClassVar, cast
+import polars as pl
+
+import pylibcudf as plc
+from cudf_streaming.streaming.channel_metadata import ChannelMetadata
+from cudf_streaming.streaming.table_chunk import (
+ TableChunk,
+ make_table_chunks_available_or_wait,
+)
from rapidsmpf.memory.memory_reservation import opaque_memory_usage
from rapidsmpf.shuffler import PartitionAssignment
from rapidsmpf.streaming.core.actor import define_actor
from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
-from rapidsmpf.streaming.cudf.table_chunk import (
- TableChunk,
- make_table_chunks_available_or_wait,
-)
-
-import polars as pl
-
-import pylibcudf as plc
from cudf_polars.containers import Column, DataFrame, DataType
from cudf_polars.dsl.expr import GroupedWindow
@@ -92,7 +91,6 @@
from rapidsmpf.memory.buffer_resource import BufferResource
from rapidsmpf.streaming.core.channel import Channel
from rapidsmpf.streaming.core.context import Context
-
from rmm.pylibrmm.stream import Stream
from cudf_polars.dsl.expr import Col
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py
index fcfc897e1728..ccd62b8c1c15 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py
@@ -7,14 +7,14 @@
import math
from typing import TYPE_CHECKING, Any
-from rapidsmpf.memory.memory_reservation import opaque_memory_usage
-from rapidsmpf.streaming.core.actor import define_actor
-from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
-from rapidsmpf.streaming.cudf.table_chunk import (
+from cudf_streaming.streaming.channel_metadata import ChannelMetadata
+from cudf_streaming.streaming.table_chunk import (
TableChunk,
make_table_chunks_available_or_wait,
)
+from rapidsmpf.memory.memory_reservation import opaque_memory_usage
+from rapidsmpf.streaming.core.actor import define_actor
+from rapidsmpf.streaming.core.message import Message
from cudf_polars.containers import DataFrame
from cudf_polars.streaming.actor_graph.collectives.allgather import AllGatherManager
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py
index a2664fa3a798..bf14a23f1c15 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py
@@ -6,9 +6,9 @@
from typing import TYPE_CHECKING, Any
+from cudf_streaming.streaming.channel_metadata import ChannelMetadata
+from cudf_streaming.streaming.table_chunk import TableChunk
from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
from cudf_polars.dsl.ir import Union
from cudf_polars.streaming.actor_graph.dispatch import (
diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py
index ac87b16d5872..68c5e8a94c0d 100644
--- a/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py
+++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py
@@ -16,24 +16,23 @@
from functools import reduce
from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast
-from rapidsmpf.memory.memory_reservation import opaque_memory_usage
-from rapidsmpf.memory.packed_data import PackedData
-from rapidsmpf.streaming.coll.allgather import AllGather
-from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.cudf.channel_metadata import (
+import pylibcudf as plc
+import rmm.mr
+from cudf_streaming.streaming.channel_metadata import (
ChannelMetadata,
HashScheme,
OrderKey,
OrderScheme,
Partitioning,
)
-from rapidsmpf.streaming.cudf.table_chunk import (
+from cudf_streaming.streaming.table_chunk import (
TableChunk,
make_table_chunks_available_or_wait,
)
-
-import pylibcudf as plc
-import rmm.mr
+from rapidsmpf.memory.memory_reservation import opaque_memory_usage
+from rapidsmpf.memory.packed_data import PackedData
+from rapidsmpf.streaming.coll.allgather import AllGather
+from rapidsmpf.streaming.core.message import Message
import cudf_polars.dsl.tracing
from cudf_polars.containers import DataFrame
@@ -61,7 +60,6 @@
from rapidsmpf.streaming.core.channel import Channel
from rapidsmpf.streaming.core.context import Context
from rapidsmpf.streaming.core.spillable_messages import SpillableMessages
-
from rmm.pylibrmm.stream import Stream
from cudf_polars.dsl.ir import IR, IRExecutionContext
diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py
index 4582c0040019..5786a5351cc4 100644
--- a/python/cudf_polars/cudf_polars/utils/config.py
+++ b/python/cudf_polars/cudf_polars/utils/config.py
@@ -36,13 +36,13 @@
from concurrent.futures import ThreadPoolExecutor
import distributed
- from rapidsmpf.communicator.communicator import Communicator
- from rapidsmpf.streaming.core.context import Context
from ray.actor import ActorHandle
import polars.lazyframe.engine_config
import rmm.mr
+ from rapidsmpf.communicator.communicator import Communicator
+ from rapidsmpf.streaming.core.context import Context
from cudf_polars.engine.ray import RankActor
diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml
index 3f395e4d92ee..557520d9a2fe 100644
--- a/python/cudf_polars/pyproject.toml
+++ b/python/cudf_polars/pyproject.toml
@@ -22,6 +22,7 @@ license-files = ["LICENSE"]
requires-python = ">=3.11"
dependencies = [
"cuda-python>=13.0.1,<14.0",
+ "cudf-streaming==26.8.*,>=0.0.0a0",
"nvidia-ml-py>=12",
"packaging",
"polars>=1.35,<1.42",
@@ -232,7 +233,7 @@ required-imports = ["from __future__ import annotations"]
[tool.ruff.lint.isort.sections]
polars = ["polars"]
-rapids = ["rmm", "pylibcudf"]
+rapids = ["rmm", "pylibcudf", "rapidsmpf", "cudf_streaming"]
[tool.ruff.format]
docstring-code-format = true
diff --git a/python/cudf_polars/tests/streaming/test_allgather.py b/python/cudf_polars/tests/streaming/test_allgather.py
index 48d84afdb00f..54cd49b9afff 100644
--- a/python/cudf_polars/tests/streaming/test_allgather.py
+++ b/python/cudf_polars/tests/streaming/test_allgather.py
@@ -8,11 +8,10 @@
import asyncio
from concurrent.futures import ThreadPoolExecutor
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
import polars as pl
import pylibcudf as plc
+from cudf_streaming.streaming.table_chunk import TableChunk
from cudf_polars.dsl.ir import IRExecutionContext
from cudf_polars.streaming.actor_graph.collectives.allgather import AllGatherManager
diff --git a/python/cudf_polars/tests/streaming/test_dask.py b/python/cudf_polars/tests/streaming/test_dask.py
index 3d81ff5cf1c8..c5cabd323aa8 100644
--- a/python/cudf_polars/tests/streaming/test_dask.py
+++ b/python/cudf_polars/tests/streaming/test_dask.py
@@ -9,10 +9,11 @@
from typing import TYPE_CHECKING
import pytest
-from rapidsmpf.bootstrap import is_running_with_rrun
import polars as pl
+from rapidsmpf.bootstrap import is_running_with_rrun
+
from cudf_polars.engine.options import StreamingOptions
from cudf_polars.testing.asserts import assert_gpu_result_equal
from cudf_polars.utils.config import DaskContext
diff --git a/python/cudf_polars/tests/streaming/test_metadata.py b/python/cudf_polars/tests/streaming/test_metadata.py
index 5c555e3593ba..d634c88561a3 100644
--- a/python/cudf_polars/tests/streaming/test_metadata.py
+++ b/python/cudf_polars/tests/streaming/test_metadata.py
@@ -6,18 +6,18 @@
from __future__ import annotations
import pytest
-from rapidsmpf.streaming.cudf.channel_metadata import (
+
+import polars as pl
+
+import pylibcudf as plc
+from cudf_streaming.streaming.channel_metadata import (
ChannelMetadata,
HashScheme,
OrderKey,
OrderScheme,
Partitioning,
)
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
-import polars as pl
-
-import pylibcudf as plc
+from cudf_streaming.streaming.table_chunk import TableChunk
from cudf_polars import Translator
from cudf_polars.containers import DataFrame, DataType
diff --git a/python/cudf_polars/tests/streaming/test_ray.py b/python/cudf_polars/tests/streaming/test_ray.py
index a4e543d5c628..3b226eda8147 100644
--- a/python/cudf_polars/tests/streaming/test_ray.py
+++ b/python/cudf_polars/tests/streaming/test_ray.py
@@ -9,10 +9,11 @@
from unittest.mock import patch
import pytest
-from rapidsmpf.bootstrap import is_running_with_rrun
import polars as pl
+from rapidsmpf.bootstrap import is_running_with_rrun
+
from cudf_polars.engine.hardware_binding import HardwareBindingPolicy
from cudf_polars.utils.config import RayContext
diff --git a/python/cudf_polars/tests/streaming/test_shuffler.py b/python/cudf_polars/tests/streaming/test_shuffler.py
index 7f4fd46d15e1..51ffe28e7f46 100644
--- a/python/cudf_polars/tests/streaming/test_shuffler.py
+++ b/python/cudf_polars/tests/streaming/test_shuffler.py
@@ -6,14 +6,15 @@
import asyncio
import pytest
-from rapidsmpf.streaming.cudf.channel_metadata import (
+
+import polars as pl
+
+from cudf_streaming.streaming.channel_metadata import (
ChannelMetadata,
HashScheme,
Partitioning,
)
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
-
-import polars as pl
+from cudf_streaming.streaming.table_chunk import TableChunk
from cudf_polars.containers import DataFrame, DataType
from cudf_polars.engine.options import StreamingOptions
diff --git a/python/cudf_polars/tests/streaming/test_spilling.py b/python/cudf_polars/tests/streaming/test_spilling.py
index e368bc3ac1f3..c1d7e5b1cdfc 100644
--- a/python/cudf_polars/tests/streaming/test_spilling.py
+++ b/python/cudf_polars/tests/streaming/test_spilling.py
@@ -9,15 +9,15 @@
from typing import TYPE_CHECKING
import pytest
-from rapidsmpf.memory.buffer import MemoryType
-from rapidsmpf.memory.pinned_memory_resource import is_pinned_memory_resources_supported
-from rapidsmpf.streaming.core.message import Message
-from rapidsmpf.streaming.core.spillable_messages import SpillableMessages
-from rapidsmpf.streaming.cudf.table_chunk import TableChunk
import polars as pl
import pylibcudf as plc
+from cudf_streaming.streaming.table_chunk import TableChunk
+from rapidsmpf.memory.buffer import MemoryType
+from rapidsmpf.memory.pinned_memory_resource import is_pinned_memory_resources_supported
+from rapidsmpf.streaming.core.message import Message
+from rapidsmpf.streaming.core.spillable_messages import SpillableMessages
from cudf_polars.engine.options import StreamingOptions
from cudf_polars.streaming.actor_graph.utils import (
diff --git a/python/cudf_polars/tests/streaming/test_spmd.py b/python/cudf_polars/tests/streaming/test_spmd.py
index 17c2f86d961a..ed4a5b069ba9 100644
--- a/python/cudf_polars/tests/streaming/test_spmd.py
+++ b/python/cudf_polars/tests/streaming/test_spmd.py
@@ -9,12 +9,12 @@
from unittest.mock import patch
import pytest
-from rapidsmpf.bootstrap import is_running_with_rrun
-from rapidsmpf.rmm_resource_adaptor import RmmResourceAdaptor
import polars as pl
import rmm.mr
+from rapidsmpf.bootstrap import is_running_with_rrun
+from rapidsmpf.rmm_resource_adaptor import RmmResourceAdaptor
from cudf_polars.engine.core import _find_memory_error
from cudf_polars.engine.hardware_binding import HardwareBindingPolicy
diff --git a/python/cudf_polars/tests/streaming/test_statistics.py b/python/cudf_polars/tests/streaming/test_statistics.py
index 61d5659dc702..f51605d6bfd7 100644
--- a/python/cudf_polars/tests/streaming/test_statistics.py
+++ b/python/cudf_polars/tests/streaming/test_statistics.py
@@ -7,6 +7,7 @@
from typing import TYPE_CHECKING
import pytest
+
from rapidsmpf.statistics import Statistics
from cudf_polars.engine.options import StreamingOptions
diff --git a/python/cudf_streaming/cudf_streaming/integrations/__init__.py b/python/cudf_streaming/cudf_streaming/integrations/__init__.py
index 743c0b3ec454..37f3e8a1575c 100644
--- a/python/cudf_streaming/cudf_streaming/integrations/__init__.py
+++ b/python/cudf_streaming/cudf_streaming/integrations/__init__.py
@@ -2,6 +2,9 @@
# SPDX-License-Identifier: Apache-2.0
"""Collection of cuDF specific functions."""
-from cudf_streaming.integrations.partition import unpack_and_concat
+from cudf_streaming.integrations.partition import (
+ packed_data_from_cudf_packed_columns,
+ unpack_and_concat,
+)
-__all__ = ["unpack_and_concat"]
+__all__ = ["packed_data_from_cudf_packed_columns", "unpack_and_concat"]
diff --git a/python/cudf_streaming/cudf_streaming/integrations/partition.pxd b/python/cudf_streaming/cudf_streaming/integrations/partition.pxd
index 774ccb338c10..9d674b1bd4c0 100644
--- a/python/cudf_streaming/cudf_streaming/integrations/partition.pxd
+++ b/python/cudf_streaming/cudf_streaming/integrations/partition.pxd
@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES.
# SPDX-License-Identifier: Apache-2.0
+from pylibcudf.contiguous_split cimport PackedColumns
from pylibcudf.table cimport Table
from rmm.pylibrmm.stream cimport Stream
@@ -27,3 +28,8 @@ cpdef object unspill_partitions(
BufferResource br,
object allow_overbooking,
)
+cpdef object packed_data_from_cudf_packed_columns(
+ PackedColumns packed_columns,
+ Stream stream,
+ BufferResource br,
+)
diff --git a/python/cudf_streaming/cudf_streaming/integrations/partition.pyi b/python/cudf_streaming/cudf_streaming/integrations/partition.pyi
index 1133c65ab1c9..efb30be09176 100644
--- a/python/cudf_streaming/cudf_streaming/integrations/partition.pyi
+++ b/python/cudf_streaming/cudf_streaming/integrations/partition.pyi
@@ -4,6 +4,7 @@ from __future__ import annotations
from collections.abc import Iterable
+from pylibcudf.contiguous_split import PackedColumns
from pylibcudf.table import Table
from rapidsmpf.memory.buffer_resource import BufferResource
@@ -39,3 +40,8 @@ def unspill_partitions(
br: BufferResource,
allow_overbooking: bool,
) -> list[PackedData]: ...
+def packed_data_from_cudf_packed_columns(
+ packed_columns: PackedColumns,
+ stream: Stream,
+ br: BufferResource,
+) -> PackedData: ...
diff --git a/python/cudf_streaming/cudf_streaming/integrations/partition.pyx b/python/cudf_streaming/cudf_streaming/integrations/partition.pyx
index cc607d841934..f9fdc5b62203 100644
--- a/python/cudf_streaming/cudf_streaming/integrations/partition.pyx
+++ b/python/cudf_streaming/cudf_streaming/integrations/partition.pyx
@@ -4,16 +4,18 @@
from cython.operator cimport dereference as deref
from cython.operator cimport postincrement
-from libc.stdint cimport uint32_t
+from libc.stdint cimport uint8_t, uint32_t
from libcpp.memory cimport make_unique, unique_ptr
from libcpp.unordered_map cimport unordered_map
from libcpp.utility cimport move
from libcpp.vector cimport vector
+from pylibcudf.contiguous_split cimport PackedColumns
from pylibcudf.libcudf.table.table cimport table as cpp_table
from pylibcudf.libcudf.table.table_view cimport table_view
from pylibcudf.libcudf.types cimport size_type
from pylibcudf.table cimport Table
from rmm.librmm.cuda_stream_view cimport cuda_stream_view
+from rmm.librmm.device_buffer cimport device_buffer
from rmm.pylibrmm.stream cimport Stream
from rapidsmpf._detail.exception_handling cimport ex_handler
@@ -316,6 +318,32 @@ cdef extern from "" nogil:
) except +ex_handler
+cdef extern from *:
+ """
+ #include
+ #include
+ #include
+ #include
+
+ std::unique_ptr cpp_packed_data_from_buffers(
+ std::unique_ptr> metadata,
+ std::unique_ptr gpu_data,
+ rmm::cuda_stream_view stream,
+ rapidsmpf::BufferResource* br
+ ) {
+ return std::make_unique(
+ std::move(metadata), br->move(std::move(gpu_data), stream)
+ );
+ }
+ """
+ unique_ptr[cpp_PackedData] cpp_packed_data_from_buffers(
+ unique_ptr[vector[uint8_t]] metadata,
+ unique_ptr[device_buffer] gpu_data,
+ cuda_stream_view stream,
+ cpp_BufferResource* br,
+ ) except +ex_handler nogil
+
+
cpdef object unspill_partitions(
object partitions,
BufferResource br,
@@ -369,3 +397,59 @@ cpdef object unspill_partitions(
ab,
)
return packed_data_vector_to_list(move(_ret), br)
+
+
+cpdef object packed_data_from_cudf_packed_columns(
+ PackedColumns packed_columns,
+ Stream stream,
+ BufferResource br,
+):
+ """
+ Construct a PackedData from a pylibcudf PackedColumns.
+
+ Takes ownership of the metadata and GPU data from the PackedColumns
+ object, leaving it empty.
+
+ Parameters
+ ----------
+ packed_columns
+ Packed columns from ``pylibcudf.contiguous_split.pack()``.
+ Must not already be empty (already released).
+ stream
+ The CUDA stream on which the preceding ``pack()`` call was performed.
+ Must be the same stream to ensure correct memory ordering.
+ br
+ Buffer resource for memory management.
+
+ Returns
+ -------
+ A new PackedData instance owning the packed column data.
+
+ Raises
+ ------
+ ValueError
+ If the PackedColumns object is empty (already released).
+
+ See Also
+ --------
+ pylibcudf.contiguous_split.pack
+ cudf_streaming.integrations.partition.unpack_and_concat
+ """
+ if packed_columns is None or stream is None or br is None:
+ raise TypeError("Arguments must not be None")
+ cdef cuda_stream_view _stream = stream.view()
+ cdef cpp_BufferResource* _br = br.ptr()
+ cdef PackedData ret = PackedData.__new__(PackedData)
+ with nogil:
+ if not (packed_columns.c_obj != NULL and
+ deref(packed_columns.c_obj).metadata and
+ deref(packed_columns.c_obj).gpu_data):
+ raise ValueError("Cannot release empty PackedColumns")
+ ret.c_obj = cpp_packed_data_from_buffers(
+ move(deref(packed_columns.c_obj).metadata),
+ move(deref(packed_columns.c_obj).gpu_data),
+ _stream,
+ _br,
+ )
+ ret._br = br
+ return ret
diff --git a/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py b/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py
index f48a83e84131..6b55abd01f3c 100644
--- a/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py
+++ b/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py
@@ -9,6 +9,9 @@
import pylibcudf as plc
import pytest
+from cudf_streaming.integrations.partition import (
+ packed_data_from_cudf_packed_columns,
+)
from cudf_streaming.streaming.table_chunk import (
TableChunk,
make_table_chunks_available_or_wait,
@@ -406,7 +409,7 @@ def test_shape_accessor(
expected_shape = (expect.num_rows(), expect.num_columns())
if from_pack:
- pd = PackedData.from_cudf_packed_columns(
+ pd = packed_data_from_cudf_packed_columns(
plc.contiguous_split.pack(expect, stream), stream, context.br()
)
device_chunk = TableChunk.from_packed_data(pd, br=context.br())
@@ -444,7 +447,7 @@ def test_into_packed_data(
) -> None:
expect = random_table(1024)
if from_pack:
- pd = PackedData.from_cudf_packed_columns(
+ pd = packed_data_from_cudf_packed_columns(
plc.contiguous_split.pack(expect, stream), stream, context.br()
)
chunk = TableChunk.from_packed_data(pd, br=context.br())