From 89b1fcf50875ae5eda712d01d16e1f7e6176f002 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Jun 2026 01:27:01 +0000 Subject: [PATCH 01/14] refactor(cudf_polars): use cudf_streaming for cudf-specific streaming imports --- python/cudf_polars/cudf_polars/engine/core.py | 4 ++-- python/cudf_polars/cudf_polars/engine/dask.py | 2 +- python/cudf_polars/cudf_polars/engine/ray.py | 2 +- python/cudf_polars/cudf_polars/engine/spmd.py | 4 ++-- .../actor_graph/collectives/allgather.py | 4 ++-- .../actor_graph/collectives/shuffle.py | 20 +++++++++---------- .../streaming/actor_graph/collectives/sort.py | 10 +++++----- .../cudf_polars/streaming/actor_graph/core.py | 4 ++-- .../streaming/actor_graph/groupby.py | 10 +++++----- .../cudf_polars/streaming/actor_graph/io.py | 6 +++--- .../cudf_polars/streaming/actor_graph/join.py | 20 +++++++++---------- .../streaming/actor_graph/nodes.py | 10 +++++----- .../cudf_polars/streaming/actor_graph/over.py | 10 +++++----- .../streaming/actor_graph/repartition.py | 10 +++++----- .../streaming/actor_graph/union.py | 4 ++-- .../streaming/actor_graph/utils.py | 12 +++++------ python/cudf_polars/pyproject.toml | 1 + 17 files changed, 67 insertions(+), 66 deletions(-) diff --git a/python/cudf_polars/cudf_polars/engine/core.py b/python/cudf_polars/cudf_polars/engine/core.py index 79914d12fe8d..f7881b5ea95a 100644 --- a/python/cudf_polars/cudf_polars/engine/core.py +++ b/python/cudf_polars/cudf_polars/engine/core.py @@ -14,12 +14,12 @@ from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeVar import cuda.core +from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.coll import AllGather from rapidsmpf.config import Options, get_environment_variables from rapidsmpf.memory.packed_data import PackedData from rapidsmpf.statistics import Statistics from rapidsmpf.streaming.core.actor import run_actor_network -from rapidsmpf.streaming.cudf.table_chunk import TableChunk import polars as pl @@ -42,10 +42,10 @@ from concurrent.futures import Executor, ThreadPoolExecutor import rapidsmpf.config + from cudf_streaming.streaming.channel_metadata import ChannelMetadata from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.memory.buffer_resource import BufferResource from rapidsmpf.streaming.core.context import Context - from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata from cudf_polars.dsl.ir import IR from cudf_polars.streaming.base import PartitionInfo diff --git a/python/cudf_polars/cudf_polars/engine/dask.py b/python/cudf_polars/cudf_polars/engine/dask.py index 014627b82c48..f37a5db27998 100644 --- a/python/cudf_polars/cudf_polars/engine/dask.py +++ b/python/cudf_polars/cudf_polars/engine/dask.py @@ -44,8 +44,8 @@ if TYPE_CHECKING: from collections.abc import Callable + from cudf_streaming.streaming.channel_metadata import ChannelMetadata from rapidsmpf.communicator.communicator import Communicator - from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata from cudf_polars.dsl.ir import IR from cudf_polars.engine.core import T diff --git a/python/cudf_polars/cudf_polars/engine/ray.py b/python/cudf_polars/cudf_polars/engine/ray.py index edf1905e916d..686767230e17 100644 --- a/python/cudf_polars/cudf_polars/engine/ray.py +++ b/python/cudf_polars/cudf_polars/engine/ray.py @@ -41,8 +41,8 @@ import uuid from collections.abc import Callable + from cudf_streaming.streaming.channel_metadata import ChannelMetadata from rapidsmpf.communicator.communicator import Communicator - from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata from ray.actor import ActorHandle from cudf_polars.dsl.ir import IR diff --git a/python/cudf_polars/cudf_polars/engine/spmd.py b/python/cudf_polars/cudf_polars/engine/spmd.py index d334e9f1c95c..c6f057d48b12 100644 --- a/python/cudf_polars/cudf_polars/engine/spmd.py +++ b/python/cudf_polars/cudf_polars/engine/spmd.py @@ -10,13 +10,13 @@ from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any, cast +from cudf_streaming.integrations.partition import unpack_and_concat from rapidsmpf import bootstrap from rapidsmpf.coll import AllGather from rapidsmpf.communicator.single import ( new_communicator as single_communicator, ) from rapidsmpf.communicator.ucxx import barrier -from rapidsmpf.integrations.cudf.partition import unpack_and_concat from rapidsmpf.memory.packed_data import PackedData from rapidsmpf.progress_thread import ProgressThread from rapidsmpf.rmm_resource_adaptor import RmmResourceAdaptor @@ -51,9 +51,9 @@ import uuid from collections.abc import Callable + from cudf_streaming.streaming.channel_metadata import ChannelMetadata from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.config import Options - from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata import polars as pl diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py index 911e3f270920..5e899958d95f 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py @@ -6,16 +6,16 @@ from typing import TYPE_CHECKING -from rapidsmpf.integrations.cudf.partition import unpack_and_concat +from cudf_streaming.integrations.partition import unpack_and_concat from rapidsmpf.memory.packed_data import PackedData from rapidsmpf.streaming.coll.allgather import AllGather from pylibcudf.contiguous_split import pack if TYPE_CHECKING: + from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.streaming.core.context import Context - from rapidsmpf.streaming.cudf.table_chunk import TableChunk import pylibcudf as plc from rmm.pylibrmm.stream import Stream diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py index 7da5259afd13..99c2f38d5069 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py @@ -6,24 +6,24 @@ from typing import TYPE_CHECKING, Any -from rapidsmpf.communicator.single import new_communicator as single_comm -from rapidsmpf.config import Options, get_environment_variables -from rapidsmpf.integrations.cudf.partition import ( +from cudf_streaming.integrations.partition import ( partition_and_pack as py_partition_and_pack, split_and_pack as py_split_and_pack, unpack_and_concat as py_unpack_and_concat, ) +from cudf_streaming.streaming.channel_metadata import ( + ChannelMetadata, + HashScheme, + Partitioning, +) +from cudf_streaming.streaming.table_chunk import TableChunk +from rapidsmpf.communicator.single import new_communicator as single_comm +from rapidsmpf.config import Options, get_environment_variables from rapidsmpf.shuffler import PartitionAssignment from rapidsmpf.streaming.coll.shuffler import ShufflerAsync from rapidsmpf.streaming.core.actor import define_actor from rapidsmpf.streaming.core.context import Context from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ( - ChannelMetadata, - HashScheme, - Partitioning, -) -from rapidsmpf.streaming.cudf.table_chunk import TableChunk import pylibcudf as plc import pylibcudf.partitioning @@ -445,7 +445,7 @@ async def shuffle_actor( This node combines partition_and_pack, shuffler, and unpack_and_concat into a single Python node using rapidsmpf.shuffler.Shuffler and utilities - from rapidsmpf.integrations.cudf.partition. + from cudf_streaming.integrations.partition. Parameters ---------- diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py index 75087141c9b2..cec4901d5641 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py @@ -6,16 +6,16 @@ from typing import TYPE_CHECKING -from rapidsmpf.shuffler import PartitionAssignment -from rapidsmpf.streaming.core.actor import define_actor -from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ( +from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, OrderKey, OrderScheme, Partitioning, ) -from rapidsmpf.streaming.cudf.table_chunk import TableChunk +from cudf_streaming.streaming.table_chunk import TableChunk +from rapidsmpf.shuffler import PartitionAssignment +from rapidsmpf.streaming.core.actor import define_actor +from rapidsmpf.streaming.core.message import Message import polars as pl diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py index fb7cc4e1439a..0160681262a1 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py @@ -30,12 +30,12 @@ if TYPE_CHECKING: from collections.abc import MutableMapping + from cudf_streaming.streaming.channel_metadata import ChannelMetadata + from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.streaming.core.channel import Channel from rapidsmpf.streaming.core.context import Context from rapidsmpf.streaming.core.leaf_actor import DeferredMessages - from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata - from rapidsmpf.streaming.cudf.table_chunk import TableChunk import polars as pl diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py index c9f65da127c1..932c3662b99b 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py @@ -7,16 +7,16 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from cudf_streaming.streaming.channel_metadata import ( + ChannelMetadata, + HashScheme, +) +from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.communicator.single import new_communicator as single_comm from rapidsmpf.config import Options, get_environment_variables from rapidsmpf.streaming.core.actor import define_actor from rapidsmpf.streaming.core.context import Context from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ( - ChannelMetadata, - HashScheme, -) -from rapidsmpf.streaming.cudf.table_chunk import TableChunk import pylibcudf as plc diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py index 7f4db5e38bb3..8abf0a25c9fa 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py @@ -8,13 +8,13 @@ import math from typing import TYPE_CHECKING, Any +from cudf_streaming.streaming.channel_metadata import ChannelMetadata +from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.memory.memory_reservation import opaque_memory_usage from rapidsmpf.streaming.core.memory_reserve_or_wait import ( reserve_memory, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata -from rapidsmpf.streaming.cudf.table_chunk import TableChunk import pylibcudf as plc @@ -481,7 +481,7 @@ def make_rapidsmpf_read_parquet_node( The RapidsMPF read parquet node, or None if the predicate cannot be converted to a parquet filter (caller should fall back to scan_node). """ - from rapidsmpf.streaming.cudf.parquet import Filter, read_parquet + from cudf_streaming.streaming.parquet import Filter, read_parquet # Build ParquetReaderOptions try: diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py index 0130146576a6..8cac2a693bac 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py @@ -7,22 +7,22 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal -from rapidsmpf.memory.memory_reservation import opaque_memory_usage -from rapidsmpf.streaming.core.actor import define_actor -from rapidsmpf.streaming.core.memory_reserve_or_wait import ( - reserve_memory, -) -from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.bloom_filter import BloomFilter -from rapidsmpf.streaming.cudf.channel_metadata import ( +from cudf_streaming.streaming.bloom_filter import BloomFilter +from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, Partitioning, ) -from rapidsmpf.streaming.cudf.table_chunk import ( +from cudf_streaming.streaming.table_chunk import ( TableChunk, make_table_chunks_available_or_wait, ) +from rapidsmpf.memory.memory_reservation import opaque_memory_usage +from rapidsmpf.streaming.core.actor import define_actor +from rapidsmpf.streaming.core.memory_reserve_or_wait import ( + reserve_memory, +) +from rapidsmpf.streaming.core.message import Message import pylibcudf as plc from pylibcudf.hashing import LIBCUDF_DEFAULT_HASH_SEED @@ -60,10 +60,10 @@ from collections.abc import Iterable, MutableMapping from types import CoroutineType + from cudf_streaming.streaming.bloom_filter import BloomFilterChunk from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.streaming.core.channel import Channel from rapidsmpf.streaming.core.context import Context - from rapidsmpf.streaming.cudf.bloom_filter import BloomFilterChunk from cudf_polars.dsl.ir import IR, IRExecutionContext from cudf_polars.streaming.actor_graph.dispatch import SubNetGenerator diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py index 9432fd7295d8..777fb02c7a64 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/nodes.py @@ -7,16 +7,16 @@ import asyncio from typing import TYPE_CHECKING, Any, cast +from cudf_streaming.streaming.channel_metadata import ChannelMetadata +from cudf_streaming.streaming.table_chunk import ( + TableChunk, + make_table_chunks_available_or_wait, +) from rapidsmpf.memory.buffer import MemoryType from rapidsmpf.memory.memory_reservation import opaque_memory_usage from rapidsmpf.streaming.core.actor import define_actor from rapidsmpf.streaming.core.message import Message from rapidsmpf.streaming.core.spillable_messages import SpillableMessages -from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata -from rapidsmpf.streaming.cudf.table_chunk import ( - TableChunk, - make_table_chunks_available_or_wait, -) from cudf_polars.containers import DataFrame from cudf_polars.dsl.ir import IR, Empty diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py index d1569d561b21..623e1c9c5cc1 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py @@ -40,15 +40,15 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, cast +from cudf_streaming.streaming.channel_metadata import ChannelMetadata +from cudf_streaming.streaming.table_chunk import ( + TableChunk, + make_table_chunks_available_or_wait, +) from rapidsmpf.memory.memory_reservation import opaque_memory_usage from rapidsmpf.shuffler import PartitionAssignment from rapidsmpf.streaming.core.actor import define_actor from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata -from rapidsmpf.streaming.cudf.table_chunk import ( - TableChunk, - make_table_chunks_available_or_wait, -) import polars as pl diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py index fcfc897e1728..ccd62b8c1c15 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/repartition.py @@ -7,14 +7,14 @@ import math from typing import TYPE_CHECKING, Any -from rapidsmpf.memory.memory_reservation import opaque_memory_usage -from rapidsmpf.streaming.core.actor import define_actor -from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata -from rapidsmpf.streaming.cudf.table_chunk import ( +from cudf_streaming.streaming.channel_metadata import ChannelMetadata +from cudf_streaming.streaming.table_chunk import ( TableChunk, make_table_chunks_available_or_wait, ) +from rapidsmpf.memory.memory_reservation import opaque_memory_usage +from rapidsmpf.streaming.core.actor import define_actor +from rapidsmpf.streaming.core.message import Message from cudf_polars.containers import DataFrame from cudf_polars.streaming.actor_graph.collectives.allgather import AllGatherManager diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py index a2664fa3a798..bf14a23f1c15 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/union.py @@ -6,9 +6,9 @@ from typing import TYPE_CHECKING, Any +from cudf_streaming.streaming.channel_metadata import ChannelMetadata +from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ChannelMetadata -from rapidsmpf.streaming.cudf.table_chunk import TableChunk from cudf_polars.dsl.ir import Union from cudf_polars.streaming.actor_graph.dispatch import ( diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py index ac87b16d5872..1679391574ee 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py @@ -16,21 +16,21 @@ from functools import reduce from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast -from rapidsmpf.memory.memory_reservation import opaque_memory_usage -from rapidsmpf.memory.packed_data import PackedData -from rapidsmpf.streaming.coll.allgather import AllGather -from rapidsmpf.streaming.core.message import Message -from rapidsmpf.streaming.cudf.channel_metadata import ( +from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, OrderKey, OrderScheme, Partitioning, ) -from rapidsmpf.streaming.cudf.table_chunk import ( +from cudf_streaming.streaming.table_chunk import ( TableChunk, make_table_chunks_available_or_wait, ) +from rapidsmpf.memory.memory_reservation import opaque_memory_usage +from rapidsmpf.memory.packed_data import PackedData +from rapidsmpf.streaming.coll.allgather import AllGather +from rapidsmpf.streaming.core.message import Message import pylibcudf as plc import rmm.mr diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 3f395e4d92ee..b41121e7cbc6 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "polars>=1.35,<1.42", "pylibcudf==26.8.*,>=0.0.0a0", "rapidsmpf==26.8.*,>=0.0.0a0", + "cudf_streaming==26.8.*,>=0.0.0a0", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 2f447f1850e4e581b1a9279710b300799c705a19 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Jun 2026 01:27:30 +0000 Subject: [PATCH 02/14] refactor(cudf_polars): update tests to use cudf_streaming imports --- python/cudf_polars/tests/streaming/test_allgather.py | 2 +- python/cudf_polars/tests/streaming/test_metadata.py | 4 ++-- python/cudf_polars/tests/streaming/test_shuffler.py | 4 ++-- python/cudf_polars/tests/streaming/test_spilling.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/cudf_polars/tests/streaming/test_allgather.py b/python/cudf_polars/tests/streaming/test_allgather.py index 48d84afdb00f..c32e43c6d9b7 100644 --- a/python/cudf_polars/tests/streaming/test_allgather.py +++ b/python/cudf_polars/tests/streaming/test_allgather.py @@ -8,7 +8,7 @@ import asyncio from concurrent.futures import ThreadPoolExecutor -from rapidsmpf.streaming.cudf.table_chunk import TableChunk +from cudf_streaming.streaming.table_chunk import TableChunk import polars as pl diff --git a/python/cudf_polars/tests/streaming/test_metadata.py b/python/cudf_polars/tests/streaming/test_metadata.py index 5c555e3593ba..40f2fd47f95a 100644 --- a/python/cudf_polars/tests/streaming/test_metadata.py +++ b/python/cudf_polars/tests/streaming/test_metadata.py @@ -6,14 +6,14 @@ from __future__ import annotations import pytest -from rapidsmpf.streaming.cudf.channel_metadata import ( +from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, OrderKey, OrderScheme, Partitioning, ) -from rapidsmpf.streaming.cudf.table_chunk import TableChunk +from cudf_streaming.streaming.table_chunk import TableChunk import polars as pl diff --git a/python/cudf_polars/tests/streaming/test_shuffler.py b/python/cudf_polars/tests/streaming/test_shuffler.py index 7f4fd46d15e1..65db29541f81 100644 --- a/python/cudf_polars/tests/streaming/test_shuffler.py +++ b/python/cudf_polars/tests/streaming/test_shuffler.py @@ -6,12 +6,12 @@ import asyncio import pytest -from rapidsmpf.streaming.cudf.channel_metadata import ( +from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, Partitioning, ) -from rapidsmpf.streaming.cudf.table_chunk import TableChunk +from cudf_streaming.streaming.table_chunk import TableChunk import polars as pl diff --git a/python/cudf_polars/tests/streaming/test_spilling.py b/python/cudf_polars/tests/streaming/test_spilling.py index e368bc3ac1f3..35d74b2f59ee 100644 --- a/python/cudf_polars/tests/streaming/test_spilling.py +++ b/python/cudf_polars/tests/streaming/test_spilling.py @@ -13,7 +13,7 @@ from rapidsmpf.memory.pinned_memory_resource import is_pinned_memory_resources_supported from rapidsmpf.streaming.core.message import Message from rapidsmpf.streaming.core.spillable_messages import SpillableMessages -from rapidsmpf.streaming.cudf.table_chunk import TableChunk +from cudf_streaming.streaming.table_chunk import TableChunk import polars as pl From 2ef7257203a95a54e72aedb92b4727c02f408799 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Jun 2026 21:04:33 +0000 Subject: [PATCH 03/14] fix(cudf_polars): add cudf_streaming to dependencies.yaml, fix import order --- dependencies.yaml | 23 ++++++++++++++++++- python/cudf_polars/pyproject.toml | 2 +- .../tests/streaming/test_spilling.py | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 25ed1071f03d..69b0a0480f6b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -335,6 +335,7 @@ files: includes: - run_cudf_polars - depends_on_rapidsmpf + - depends_on_cudf_streaming - depends_on_pylibcudf - depends_on_cuda_python # TODO(26.10): Remove this alias in favor of dask @@ -1414,7 +1415,27 @@ dependencies: common: - output_types: conda packages: - - cudf-streaming==26.8.*,>=0.0.0a0 + - &cudf_streaming_unsuffixed cudf-streaming==26.8.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - cudf-streaming-cu12==26.8.*,>=0.0.0a0 + - matrix: + cuda: "13.*" + cuda_suffixed: "true" + packages: + - cudf-streaming-cu13==26.8.*,>=0.0.0a0 + - matrix: + packages: + - *cudf_streaming_unsuffixed depends_on_ray: common: - output_types: conda diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index b41121e7cbc6..9b1ff8d7fd2f 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -22,12 +22,12 @@ license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ "cuda-python>=13.0.1,<14.0", + "cudf-streaming==26.8.*,>=0.0.0a0", "nvidia-ml-py>=12", "packaging", "polars>=1.35,<1.42", "pylibcudf==26.8.*,>=0.0.0a0", "rapidsmpf==26.8.*,>=0.0.0a0", - "cudf_streaming==26.8.*,>=0.0.0a0", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ diff --git a/python/cudf_polars/tests/streaming/test_spilling.py b/python/cudf_polars/tests/streaming/test_spilling.py index 35d74b2f59ee..9db08020aed3 100644 --- a/python/cudf_polars/tests/streaming/test_spilling.py +++ b/python/cudf_polars/tests/streaming/test_spilling.py @@ -9,11 +9,11 @@ from typing import TYPE_CHECKING import pytest +from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.memory.buffer import MemoryType from rapidsmpf.memory.pinned_memory_resource import is_pinned_memory_resources_supported from rapidsmpf.streaming.core.message import Message from rapidsmpf.streaming.core.spillable_messages import SpillableMessages -from cudf_streaming.streaming.table_chunk import TableChunk import polars as pl From 3021f597ea9db49718dec27aa2cf487089f810c2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Jun 2026 22:36:51 +0000 Subject: [PATCH 04/14] Download cudf-streaming wheels in cudf-polars CI tests Add cudf-streaming and libcudf-streaming wheel downloads to the cudf-polars wheel test scripts so they are installed from CI artifacts rather than resolved from a remote index. - Add wheel-build-cudf-streaming to needs for wheel-tests-cudf-polars and cudf-polars-polars-tests in pr.yaml - Download libcudf_streaming and cudf_streaming wheels in test_wheel_cudf_polars.sh and test_cudf_polars_polars_tests.sh - Add cudf-streaming as a runtime dependency in the cudf-polars conda recipe for conda-based test jobs (docs, narwhals) --- .github/workflows/pr.yaml | 4 ++-- ci/test_cudf_polars_polars_tests.sh | 8 +++++++- ci/test_wheel_cudf_polars.sh | 6 ++++++ conda/recipes/cudf-polars/recipe.yaml | 1 + 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 031e26e4a048..ef239ac33acc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -646,7 +646,7 @@ jobs: package-type: python pure-wheel: true wheel-tests-cudf-polars: - needs: [wheel-build-cudf-polars, changed-files] + needs: [wheel-build-cudf-polars, wheel-build-cudf-streaming, changed-files] permissions: actions: read contents: read @@ -663,7 +663,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" script: "ci/test_wheel_cudf_polars.sh" cudf-polars-polars-tests: - needs: [wheel-build-cudf-polars, changed-files] + needs: [wheel-build-cudf-polars, wheel-build-cudf-streaming, changed-files] permissions: actions: read contents: read diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index 1fb9a4e395f6..331b85f2cc90 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -13,6 +13,10 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") +# Download libcudf_streaming and cudf_streaming built in the previous step +LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) +CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" cudf_streaming --stable --cuda "$RAPIDS_CUDA_VERSION")") + # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" @@ -29,7 +33,9 @@ rapids-pip-retry install \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ - "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" + "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo "${CUDF_STREAMING_WHEELHOUSE}"/cudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" TAG=$(python -c 'import polars; print(f"py-{polars.__version__}")') rapids-logger "Clone polars to ${TAG}" diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 0b73e904dc47..61185c21568a 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -15,6 +15,10 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") +# Download libcudf_streaming and cudf_streaming built in the previous step +LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) +CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" cudf_streaming --stable --cuda "$RAPIDS_CUDA_VERSION")") + # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" @@ -65,6 +69,8 @@ for version in "${VERSIONS[@]}"; do "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,dask,ray]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo "${CUDF_STREAMING_WHEELHOUSE}"/cudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ -r "polars-compat-${version}-requirements.txt" rapids-logger "Running tests for polars ${version}.*" diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 79ebec2963cb..12aa1e19f162 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -37,6 +37,7 @@ requirements: - python - pylibcudf =${{ version }} - rapidsmpf =${{ minor_version }} + - cudf-streaming =${{ version }} - polars>=1.35,<1.42 - packaging - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} From 75f17afced56859595901778997c8ba8564de6a5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Jun 2026 23:37:19 +0000 Subject: [PATCH 05/14] Minor cleanup --- ci/test_cudf_polars_polars_tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index 331b85f2cc90..50d1ee861f16 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -12,8 +12,6 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-github python) LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") - -# Download libcudf_streaming and cudf_streaming built in the previous step LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" cudf_streaming --stable --cuda "$RAPIDS_CUDA_VERSION")") From 7dea206a7307333503ef2be57c4d946ea7bc627d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 4 Jun 2026 23:55:12 +0000 Subject: [PATCH 06/14] Remove workaround for building rapidsmpf in devcontainer builds --- .github/workflows/pr.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef239ac33acc..1a345e4a498f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -732,14 +732,9 @@ jobs: SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false - # Call clone-rapidsmpf before creating the Python env to exclude cudf_polars' dependency on rapidsmpf - # librapidsmpf-cu13 wheels brings in a hardcoded libnuma-dev cmake target: https://github.com/NVIDIA/cuCascade/issues/118 - # -DBUILD_TESTS=OFF to match rapidsmpf https://github.com/rapidsai/rapidsmpf/blob/main/.github/workflows/pr.yaml#L351 (leads to compilation errors) - # -DCUDF_BUILD_TESTUTIL=OFF to avoid IMPORTED_GLOBAL promotion errors when cuCascade's find_package(cudf) loads cudf-config.cmake from a CPM subdirectory build_command: | sccache --zero-stats; - clone-rapidsmpf -j$(nproc) -v -q --branch "$(cat ~/cudf/RAPIDS_BRANCH)" --clone-upstream --depth 1 --single-branch --shallow-submodules; - build-all -j0 -DBUILD_BENCHMARKS=OFF -DBUILD_NUMA_SUPPORT=OFF -DBUILD_TESTS=OFF -DCUDF_BUILD_TESTUTIL=OFF --verbose 2>&1 | tee telemetry-artifacts/build.log; + build-all -j0 -DBUILD_BENCHMARKS=ON --verbose 2>&1 | tee telemetry-artifacts/build.log; sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt; unit-tests-cudf-pandas: needs: [wheel-build-cudf, changed-files] From 3efe63b2f37f3f41d2bdb205922c3c4ca9a28a7c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 5 Jun 2026 00:06:09 +0000 Subject: [PATCH 07/14] Update isort sections --- python/cudf_polars/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 9b1ff8d7fd2f..557520d9a2fe 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -233,7 +233,7 @@ required-imports = ["from __future__ import annotations"] [tool.ruff.lint.isort.sections] polars = ["polars"] -rapids = ["rmm", "pylibcudf"] +rapids = ["rmm", "pylibcudf", "rapidsmpf", "cudf_streaming"] [tool.ruff.format] docstring-code-format = true From 5b9fac413ee00084b710e52b093d264ddb9fdb7f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 5 Jun 2026 00:12:11 +0000 Subject: [PATCH 08/14] Apply isort fix --- python/cudf_polars/cudf_polars/engine/core.py | 5 +++-- python/cudf_polars/cudf_polars/engine/dask.py | 8 ++++---- python/cudf_polars/cudf_polars/engine/ray.py | 11 ++++++----- python/cudf_polars/cudf_polars/engine/spmd.py | 9 ++++----- .../streaming/actor_graph/collectives/allgather.py | 6 ++---- .../streaming/actor_graph/collectives/shuffle.py | 6 ++---- .../streaming/actor_graph/collectives/sort.py | 7 +++---- .../cudf_polars/streaming/actor_graph/core.py | 4 ++-- .../cudf_polars/streaming/actor_graph/groupby.py | 3 +-- .../cudf_polars/streaming/actor_graph/io.py | 3 +-- .../cudf_polars/streaming/actor_graph/join.py | 5 ++--- .../cudf_polars/streaming/actor_graph/over.py | 8 +++----- .../cudf_polars/streaming/actor_graph/utils.py | 6 ++---- python/cudf_polars/cudf_polars/utils/config.py | 4 ++-- python/cudf_polars/tests/streaming/test_allgather.py | 3 +-- python/cudf_polars/tests/streaming/test_dask.py | 3 ++- python/cudf_polars/tests/streaming/test_metadata.py | 8 ++++---- python/cudf_polars/tests/streaming/test_ray.py | 3 ++- python/cudf_polars/tests/streaming/test_shuffler.py | 5 +++-- python/cudf_polars/tests/streaming/test_spilling.py | 8 ++++---- python/cudf_polars/tests/streaming/test_spmd.py | 4 ++-- python/cudf_polars/tests/streaming/test_statistics.py | 1 + 22 files changed, 56 insertions(+), 64 deletions(-) diff --git a/python/cudf_polars/cudf_polars/engine/core.py b/python/cudf_polars/cudf_polars/engine/core.py index f7881b5ea95a..bfe97050ccf3 100644 --- a/python/cudf_polars/cudf_polars/engine/core.py +++ b/python/cudf_polars/cudf_polars/engine/core.py @@ -14,6 +14,9 @@ from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeVar import cuda.core + +import polars as pl + from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.coll import AllGather from rapidsmpf.config import Options, get_environment_variables @@ -21,8 +24,6 @@ from rapidsmpf.statistics import Statistics from rapidsmpf.streaming.core.actor import run_actor_network -import polars as pl - from cudf_polars.containers import DataFrame from cudf_polars.dsl.ir import IRExecutionContext from cudf_polars.streaming.actor_graph.collectives import ReserveOpIDs diff --git a/python/cudf_polars/cudf_polars/engine/dask.py b/python/cudf_polars/cudf_polars/engine/dask.py index f37a5db27998..46372d0dad04 100644 --- a/python/cudf_polars/cudf_polars/engine/dask.py +++ b/python/cudf_polars/cudf_polars/engine/dask.py @@ -16,6 +16,10 @@ import distributed.system import pynvml import ucxx._lib.libucxx as ucx_api + +import polars as pl + +import rmm.mr from rapidsmpf import bootstrap from rapidsmpf.communicator.ucxx import barrier, get_root_ucxx_address, new_communicator from rapidsmpf.config import Options @@ -24,10 +28,6 @@ from rapidsmpf.statistics import Statistics from rapidsmpf.streaming.core.context import Context -import polars as pl - -import rmm.mr - from cudf_polars.engine.core import ( ClusterInfo, StreamingEngine, diff --git a/python/cudf_polars/cudf_polars/engine/ray.py b/python/cudf_polars/cudf_polars/engine/ray.py index 686767230e17..b8824ee55414 100644 --- a/python/cudf_polars/cudf_polars/engine/ray.py +++ b/python/cudf_polars/cudf_polars/engine/ray.py @@ -12,6 +12,10 @@ import ray import ucxx._lib.libucxx as ucx_api + +import polars as pl + +import rmm.mr from rapidsmpf import bootstrap from rapidsmpf.communicator.ucxx import barrier, get_root_ucxx_address, new_communicator from rapidsmpf.config import Options @@ -20,10 +24,6 @@ from rapidsmpf.statistics import Statistics from rapidsmpf.streaming.core.context import Context -import polars as pl - -import rmm.mr - from cudf_polars.engine.core import ( ClusterInfo, StreamingEngine, @@ -41,9 +41,10 @@ import uuid from collections.abc import Callable + from ray.actor import ActorHandle + from cudf_streaming.streaming.channel_metadata import ChannelMetadata from rapidsmpf.communicator.communicator import Communicator - from ray.actor import ActorHandle from cudf_polars.dsl.ir import IR from cudf_polars.engine.core import T diff --git a/python/cudf_polars/cudf_polars/engine/spmd.py b/python/cudf_polars/cudf_polars/engine/spmd.py index c6f057d48b12..e7905c845c56 100644 --- a/python/cudf_polars/cudf_polars/engine/spmd.py +++ b/python/cudf_polars/cudf_polars/engine/spmd.py @@ -10,7 +10,9 @@ from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any, cast +import pylibcudf as plc from cudf_streaming.integrations.partition import unpack_and_concat +from pylibcudf.contiguous_split import pack from rapidsmpf import bootstrap from rapidsmpf.coll import AllGather from rapidsmpf.communicator.single import ( @@ -23,9 +25,6 @@ from rapidsmpf.statistics import Statistics from rapidsmpf.streaming.core.context import Context -import pylibcudf as plc -from pylibcudf.contiguous_split import pack - from cudf_polars.containers import DataFrame, DataType from cudf_polars.engine.core import ( ClusterInfo, @@ -51,12 +50,12 @@ import uuid from collections.abc import Callable + import polars as pl + from cudf_streaming.streaming.channel_metadata import ChannelMetadata from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.config import Options - import polars as pl - from cudf_polars.dsl.ir import IR from cudf_polars.engine.core import T from cudf_polars.engine.options import StreamingOptions diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py index 5e899958d95f..1772e9bdb41e 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py @@ -7,17 +7,15 @@ from typing import TYPE_CHECKING from cudf_streaming.integrations.partition import unpack_and_concat +from pylibcudf.contiguous_split import pack from rapidsmpf.memory.packed_data import PackedData from rapidsmpf.streaming.coll.allgather import AllGather -from pylibcudf.contiguous_split import pack - if TYPE_CHECKING: + import pylibcudf as plc from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.streaming.core.context import Context - - import pylibcudf as plc from rmm.pylibrmm.stream import Stream from cudf_polars.dsl.ir import IRExecutionContext diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py index 99c2f38d5069..dd78104d2e96 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/shuffle.py @@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Any +import pylibcudf as plc +import pylibcudf.partitioning from cudf_streaming.integrations.partition import ( partition_and_pack as py_partition_and_pack, split_and_pack as py_split_and_pack, @@ -25,9 +27,6 @@ from rapidsmpf.streaming.core.context import Context from rapidsmpf.streaming.core.message import Message -import pylibcudf as plc -import pylibcudf.partitioning - from cudf_polars.dsl.expr import Col from cudf_polars.streaming.actor_graph.dispatch import ( generate_ir_sub_network, @@ -48,7 +47,6 @@ from rapidsmpf.communicator.communicator import Communicator from rapidsmpf.memory.packed_data import PackedData from rapidsmpf.streaming.core.channel import Channel - from rmm.pylibrmm.stream import Stream from cudf_polars.dsl.ir import IR, IRExecutionContext diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py index cec4901d5641..f62f2fa05f76 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/sort.py @@ -6,6 +6,9 @@ from typing import TYPE_CHECKING +import polars as pl + +import pylibcudf as plc from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, OrderKey, @@ -17,10 +20,6 @@ from rapidsmpf.streaming.core.actor import define_actor from rapidsmpf.streaming.core.message import Message -import polars as pl - -import pylibcudf as plc - from cudf_polars.containers import DataFrame, DataType from cudf_polars.dsl.expr import Col, NamedExpr from cudf_polars.dsl.ir import Empty, Sort diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py index 0160681262a1..1bd440a239f5 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/core.py @@ -30,6 +30,8 @@ if TYPE_CHECKING: from collections.abc import MutableMapping + import polars as pl + from cudf_streaming.streaming.channel_metadata import ChannelMetadata from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.communicator.communicator import Communicator @@ -37,8 +39,6 @@ from rapidsmpf.streaming.core.context import Context from rapidsmpf.streaming.core.leaf_actor import DeferredMessages - import polars as pl - from cudf_polars.dsl.ir import IR, IRExecutionContext from cudf_polars.streaming.actor_graph.dispatch import ( GenState, diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py index 932c3662b99b..df2c8d04f4c7 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/groupby.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +import pylibcudf as plc from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, @@ -18,8 +19,6 @@ from rapidsmpf.streaming.core.context import Context from rapidsmpf.streaming.core.message import Message -import pylibcudf as plc - from cudf_polars.containers import DataType from cudf_polars.dsl.expr import Col, NamedExpr from cudf_polars.dsl.ir import IR, Distinct, GroupBy, Select diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py index 8abf0a25c9fa..f30b55dbdd3d 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py @@ -8,6 +8,7 @@ import math from typing import TYPE_CHECKING, Any +import pylibcudf as plc from cudf_streaming.streaming.channel_metadata import ChannelMetadata from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.memory.memory_reservation import opaque_memory_usage @@ -16,8 +17,6 @@ ) from rapidsmpf.streaming.core.message import Message -import pylibcudf as plc - from cudf_polars.dsl.ir import ( IR, DataFrameScan, diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py index 8cac2a693bac..fa99192aaf12 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/join.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal +import pylibcudf as plc from cudf_streaming.streaming.bloom_filter import BloomFilter from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, @@ -17,6 +18,7 @@ TableChunk, make_table_chunks_available_or_wait, ) +from pylibcudf.hashing import LIBCUDF_DEFAULT_HASH_SEED from rapidsmpf.memory.memory_reservation import opaque_memory_usage from rapidsmpf.streaming.core.actor import define_actor from rapidsmpf.streaming.core.memory_reserve_or_wait import ( @@ -24,9 +26,6 @@ ) from rapidsmpf.streaming.core.message import Message -import pylibcudf as plc -from pylibcudf.hashing import LIBCUDF_DEFAULT_HASH_SEED - from cudf_polars.containers import DataFrame from cudf_polars.dsl.ir import IR, Join from cudf_polars.dsl.utils.naming import names_to_indices diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py index 623e1c9c5cc1..4b0f20db5f7b 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/over.py @@ -40,6 +40,9 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, cast +import polars as pl + +import pylibcudf as plc from cudf_streaming.streaming.channel_metadata import ChannelMetadata from cudf_streaming.streaming.table_chunk import ( TableChunk, @@ -50,10 +53,6 @@ from rapidsmpf.streaming.core.actor import define_actor from rapidsmpf.streaming.core.message import Message -import polars as pl - -import pylibcudf as plc - from cudf_polars.containers import Column, DataFrame, DataType from cudf_polars.dsl.expr import GroupedWindow from cudf_polars.dsl.expressions.base import ExecutionContext @@ -92,7 +91,6 @@ from rapidsmpf.memory.buffer_resource import BufferResource from rapidsmpf.streaming.core.channel import Channel from rapidsmpf.streaming.core.context import Context - from rmm.pylibrmm.stream import Stream from cudf_polars.dsl.expr import Col diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py index 1679391574ee..68c5e8a94c0d 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/utils.py @@ -16,6 +16,8 @@ from functools import reduce from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast +import pylibcudf as plc +import rmm.mr from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, @@ -32,9 +34,6 @@ from rapidsmpf.streaming.coll.allgather import AllGather from rapidsmpf.streaming.core.message import Message -import pylibcudf as plc -import rmm.mr - import cudf_polars.dsl.tracing from cudf_polars.containers import DataFrame from cudf_polars.dsl.expr import Col, NamedExpr @@ -61,7 +60,6 @@ from rapidsmpf.streaming.core.channel import Channel from rapidsmpf.streaming.core.context import Context from rapidsmpf.streaming.core.spillable_messages import SpillableMessages - from rmm.pylibrmm.stream import Stream from cudf_polars.dsl.ir import IR, IRExecutionContext diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index 4582c0040019..5786a5351cc4 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -36,13 +36,13 @@ from concurrent.futures import ThreadPoolExecutor import distributed - from rapidsmpf.communicator.communicator import Communicator - from rapidsmpf.streaming.core.context import Context from ray.actor import ActorHandle import polars.lazyframe.engine_config import rmm.mr + from rapidsmpf.communicator.communicator import Communicator + from rapidsmpf.streaming.core.context import Context from cudf_polars.engine.ray import RankActor diff --git a/python/cudf_polars/tests/streaming/test_allgather.py b/python/cudf_polars/tests/streaming/test_allgather.py index c32e43c6d9b7..54cd49b9afff 100644 --- a/python/cudf_polars/tests/streaming/test_allgather.py +++ b/python/cudf_polars/tests/streaming/test_allgather.py @@ -8,11 +8,10 @@ import asyncio from concurrent.futures import ThreadPoolExecutor -from cudf_streaming.streaming.table_chunk import TableChunk - import polars as pl import pylibcudf as plc +from cudf_streaming.streaming.table_chunk import TableChunk from cudf_polars.dsl.ir import IRExecutionContext from cudf_polars.streaming.actor_graph.collectives.allgather import AllGatherManager diff --git a/python/cudf_polars/tests/streaming/test_dask.py b/python/cudf_polars/tests/streaming/test_dask.py index 3d81ff5cf1c8..c5cabd323aa8 100644 --- a/python/cudf_polars/tests/streaming/test_dask.py +++ b/python/cudf_polars/tests/streaming/test_dask.py @@ -9,10 +9,11 @@ from typing import TYPE_CHECKING import pytest -from rapidsmpf.bootstrap import is_running_with_rrun import polars as pl +from rapidsmpf.bootstrap import is_running_with_rrun + from cudf_polars.engine.options import StreamingOptions from cudf_polars.testing.asserts import assert_gpu_result_equal from cudf_polars.utils.config import DaskContext diff --git a/python/cudf_polars/tests/streaming/test_metadata.py b/python/cudf_polars/tests/streaming/test_metadata.py index 40f2fd47f95a..d634c88561a3 100644 --- a/python/cudf_polars/tests/streaming/test_metadata.py +++ b/python/cudf_polars/tests/streaming/test_metadata.py @@ -6,6 +6,10 @@ from __future__ import annotations import pytest + +import polars as pl + +import pylibcudf as plc from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, @@ -15,10 +19,6 @@ ) from cudf_streaming.streaming.table_chunk import TableChunk -import polars as pl - -import pylibcudf as plc - from cudf_polars import Translator from cudf_polars.containers import DataFrame, DataType from cudf_polars.dsl import expr diff --git a/python/cudf_polars/tests/streaming/test_ray.py b/python/cudf_polars/tests/streaming/test_ray.py index a4e543d5c628..3b226eda8147 100644 --- a/python/cudf_polars/tests/streaming/test_ray.py +++ b/python/cudf_polars/tests/streaming/test_ray.py @@ -9,10 +9,11 @@ from unittest.mock import patch import pytest -from rapidsmpf.bootstrap import is_running_with_rrun import polars as pl +from rapidsmpf.bootstrap import is_running_with_rrun + from cudf_polars.engine.hardware_binding import HardwareBindingPolicy from cudf_polars.utils.config import RayContext diff --git a/python/cudf_polars/tests/streaming/test_shuffler.py b/python/cudf_polars/tests/streaming/test_shuffler.py index 65db29541f81..51ffe28e7f46 100644 --- a/python/cudf_polars/tests/streaming/test_shuffler.py +++ b/python/cudf_polars/tests/streaming/test_shuffler.py @@ -6,6 +6,9 @@ import asyncio import pytest + +import polars as pl + from cudf_streaming.streaming.channel_metadata import ( ChannelMetadata, HashScheme, @@ -13,8 +16,6 @@ ) from cudf_streaming.streaming.table_chunk import TableChunk -import polars as pl - from cudf_polars.containers import DataFrame, DataType from cudf_polars.engine.options import StreamingOptions from cudf_polars.engine.spmd import allgather_polars_dataframe diff --git a/python/cudf_polars/tests/streaming/test_spilling.py b/python/cudf_polars/tests/streaming/test_spilling.py index 9db08020aed3..c1d7e5b1cdfc 100644 --- a/python/cudf_polars/tests/streaming/test_spilling.py +++ b/python/cudf_polars/tests/streaming/test_spilling.py @@ -9,16 +9,16 @@ from typing import TYPE_CHECKING import pytest + +import polars as pl + +import pylibcudf as plc from cudf_streaming.streaming.table_chunk import TableChunk from rapidsmpf.memory.buffer import MemoryType from rapidsmpf.memory.pinned_memory_resource import is_pinned_memory_resources_supported from rapidsmpf.streaming.core.message import Message from rapidsmpf.streaming.core.spillable_messages import SpillableMessages -import polars as pl - -import pylibcudf as plc - from cudf_polars.engine.options import StreamingOptions from cudf_polars.streaming.actor_graph.utils import ( make_spill_function, diff --git a/python/cudf_polars/tests/streaming/test_spmd.py b/python/cudf_polars/tests/streaming/test_spmd.py index 17c2f86d961a..ed4a5b069ba9 100644 --- a/python/cudf_polars/tests/streaming/test_spmd.py +++ b/python/cudf_polars/tests/streaming/test_spmd.py @@ -9,12 +9,12 @@ from unittest.mock import patch import pytest -from rapidsmpf.bootstrap import is_running_with_rrun -from rapidsmpf.rmm_resource_adaptor import RmmResourceAdaptor import polars as pl import rmm.mr +from rapidsmpf.bootstrap import is_running_with_rrun +from rapidsmpf.rmm_resource_adaptor import RmmResourceAdaptor from cudf_polars.engine.core import _find_memory_error from cudf_polars.engine.hardware_binding import HardwareBindingPolicy diff --git a/python/cudf_polars/tests/streaming/test_statistics.py b/python/cudf_polars/tests/streaming/test_statistics.py index 61d5659dc702..f51605d6bfd7 100644 --- a/python/cudf_polars/tests/streaming/test_statistics.py +++ b/python/cudf_polars/tests/streaming/test_statistics.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING import pytest + from rapidsmpf.statistics import Statistics from cudf_polars.engine.options import StreamingOptions From f85db908dff7b935f285636bffc0a2325d4f6a99 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 8 Jun 2026 15:16:49 +0000 Subject: [PATCH 09/14] Revert "Remove workaround for building rapidsmpf in devcontainer builds" This reverts commit 7dea206a7307333503ef2be57c4d946ea7bc627d. --- .github/workflows/pr.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1a345e4a498f..ef239ac33acc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -732,9 +732,14 @@ jobs: SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false + # Call clone-rapidsmpf before creating the Python env to exclude cudf_polars' dependency on rapidsmpf + # librapidsmpf-cu13 wheels brings in a hardcoded libnuma-dev cmake target: https://github.com/NVIDIA/cuCascade/issues/118 + # -DBUILD_TESTS=OFF to match rapidsmpf https://github.com/rapidsai/rapidsmpf/blob/main/.github/workflows/pr.yaml#L351 (leads to compilation errors) + # -DCUDF_BUILD_TESTUTIL=OFF to avoid IMPORTED_GLOBAL promotion errors when cuCascade's find_package(cudf) loads cudf-config.cmake from a CPM subdirectory build_command: | sccache --zero-stats; - build-all -j0 -DBUILD_BENCHMARKS=ON --verbose 2>&1 | tee telemetry-artifacts/build.log; + clone-rapidsmpf -j$(nproc) -v -q --branch "$(cat ~/cudf/RAPIDS_BRANCH)" --clone-upstream --depth 1 --single-branch --shallow-submodules; + build-all -j0 -DBUILD_BENCHMARKS=OFF -DBUILD_NUMA_SUPPORT=OFF -DBUILD_TESTS=OFF -DCUDF_BUILD_TESTUTIL=OFF --verbose 2>&1 | tee telemetry-artifacts/build.log; sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt; unit-tests-cudf-pandas: needs: [wheel-build-cudf, changed-files] From 3952f856bb244297ad118b589f4fb2b37f2606b6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 8 Jun 2026 15:28:03 +0000 Subject: [PATCH 10/14] Reapply "Remove workaround for building rapidsmpf in devcontainer builds" This reverts commit f85db908dff7b935f285636bffc0a2325d4f6a99. --- .github/workflows/pr.yaml | 7 +------ conda/environments/all_cuda-129_arch-aarch64.yaml | 2 ++ conda/environments/all_cuda-129_arch-x86_64.yaml | 2 ++ conda/environments/all_cuda-132_arch-aarch64.yaml | 2 ++ conda/environments/all_cuda-132_arch-x86_64.yaml | 2 ++ dependencies.yaml | 2 ++ 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef239ac33acc..1a345e4a498f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -732,14 +732,9 @@ jobs: SCCACHE_DIST_MAX_RETRIES=inf SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false - # Call clone-rapidsmpf before creating the Python env to exclude cudf_polars' dependency on rapidsmpf - # librapidsmpf-cu13 wheels brings in a hardcoded libnuma-dev cmake target: https://github.com/NVIDIA/cuCascade/issues/118 - # -DBUILD_TESTS=OFF to match rapidsmpf https://github.com/rapidsai/rapidsmpf/blob/main/.github/workflows/pr.yaml#L351 (leads to compilation errors) - # -DCUDF_BUILD_TESTUTIL=OFF to avoid IMPORTED_GLOBAL promotion errors when cuCascade's find_package(cudf) loads cudf-config.cmake from a CPM subdirectory build_command: | sccache --zero-stats; - clone-rapidsmpf -j$(nproc) -v -q --branch "$(cat ~/cudf/RAPIDS_BRANCH)" --clone-upstream --depth 1 --single-branch --shallow-submodules; - build-all -j0 -DBUILD_BENCHMARKS=OFF -DBUILD_NUMA_SUPPORT=OFF -DBUILD_TESTS=OFF -DCUDF_BUILD_TESTUTIL=OFF --verbose 2>&1 | tee telemetry-artifacts/build.log; + build-all -j0 -DBUILD_BENCHMARKS=ON --verbose 2>&1 | tee telemetry-artifacts/build.log; sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt; unit-tests-cudf-pandas: needs: [wheel-build-cudf, changed-files] diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 838a2c557b47..7e0ad88ddb96 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -44,6 +44,7 @@ dependencies: - libkvikio==26.8.*,>=0.0.0a0 - libnvcomp-dev==5.2.0.10 - libnvjitlink-dev +- librapidsmpf==26.8.*,>=0.0.0a0 - librdkafka>=2.11.1,<2.12.0 - librmm==26.8.*,>=0.0.0a0 - make @@ -87,6 +88,7 @@ dependencies: - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.8.*,>=0.0.0a0 - rapids-logger==0.2.*,>=0.0.0a0 +- rapidsmpf==26.8.*,>=0.0.0a0 - ray-default>=2.55.1 - rich - rmm==26.8.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 0a435504e32f..272914ad7635 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -44,6 +44,7 @@ dependencies: - libkvikio==26.8.*,>=0.0.0a0 - libnvcomp-dev==5.2.0.10 - libnvjitlink-dev +- librapidsmpf==26.8.*,>=0.0.0a0 - librdkafka>=2.11.1,<2.12.0 - librmm==26.8.*,>=0.0.0a0 - make @@ -87,6 +88,7 @@ dependencies: - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.8.*,>=0.0.0a0 - rapids-logger==0.2.*,>=0.0.0a0 +- rapidsmpf==26.8.*,>=0.0.0a0 - ray-default>=2.55.1 - rich - rmm==26.8.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-132_arch-aarch64.yaml b/conda/environments/all_cuda-132_arch-aarch64.yaml index 2093888e01fa..9ecba408142c 100644 --- a/conda/environments/all_cuda-132_arch-aarch64.yaml +++ b/conda/environments/all_cuda-132_arch-aarch64.yaml @@ -44,6 +44,7 @@ dependencies: - libkvikio==26.8.*,>=0.0.0a0 - libnvcomp-dev==5.2.0.10 - libnvjitlink-dev +- librapidsmpf==26.8.*,>=0.0.0a0 - librdkafka>=2.11.1,<2.12.0 - librmm==26.8.*,>=0.0.0a0 - make @@ -87,6 +88,7 @@ dependencies: - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.8.*,>=0.0.0a0 - rapids-logger==0.2.*,>=0.0.0a0 +- rapidsmpf==26.8.*,>=0.0.0a0 - ray-default>=2.55.1 - rich - rmm==26.8.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-132_arch-x86_64.yaml b/conda/environments/all_cuda-132_arch-x86_64.yaml index 360550447330..ced21db00730 100644 --- a/conda/environments/all_cuda-132_arch-x86_64.yaml +++ b/conda/environments/all_cuda-132_arch-x86_64.yaml @@ -44,6 +44,7 @@ dependencies: - libkvikio==26.8.*,>=0.0.0a0 - libnvcomp-dev==5.2.0.10 - libnvjitlink-dev +- librapidsmpf==26.8.*,>=0.0.0a0 - librdkafka>=2.11.1,<2.12.0 - librmm==26.8.*,>=0.0.0a0 - make @@ -87,6 +88,7 @@ dependencies: - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.8.*,>=0.0.0a0 - rapids-logger==0.2.*,>=0.0.0a0 +- rapidsmpf==26.8.*,>=0.0.0a0 - ray-default>=2.55.1 - rich - rmm==26.8.*,>=0.0.0a0 diff --git a/dependencies.yaml b/dependencies.yaml index 69b0a0480f6b..0409d0c4cfe2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -23,8 +23,10 @@ files: - depends_on_libkvikio - depends_on_librmm - depends_on_libnvcomp + - depends_on_librapidsmpf - depends_on_numba_cuda - depends_on_rapids_logger + - depends_on_rapidsmpf - depends_on_ray - depends_on_rmm - develop From 9fb2ab80cddaf11f315f5a2f871a9a184bd97584 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 8 Jun 2026 11:07:41 -0500 Subject: [PATCH 11/14] Remove rapidsmpf sibling mount from devcontainers --- .devcontainer/README.md | 8 -------- .devcontainer/cuda12.9-conda/devcontainer.json | 5 ++--- .devcontainer/cuda12.9-pip/devcontainer.json | 5 ++--- .devcontainer/cuda13.2-conda/devcontainer.json | 5 ++--- .devcontainer/cuda13.2-pip/devcontainer.json | 5 ++--- 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index cc5fac22fde2..91ee7ef85f78 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -20,7 +20,6 @@ This container is a turnkey development environment for building and testing the By default, the following directories are bind-mounted into the devcontainer: * `${repo}:/home/coder/cudf` -* `${repo}/../rapidsmpf:/home/coder/rapidsmpf` * `${repo}/../.aws:/home/coder/.aws` * `${repo}/../.local:/home/coder/.local` * `${repo}/../.cache:/home/coder/.cache` @@ -29,13 +28,6 @@ By default, the following directories are bind-mounted into the devcontainer: This ensures caches, configurations, dependencies, and your commits are persisted on the host across container runs. -The [rapidsmpf](https://github.com/rapidsai/rapidsmpf) repository is a required dependency of `cudf_polars` (that also requires `libcudf`) and must be cloned as a sibling directory to the cudf repo before launching the devcontainer: - -``` -# from the parent directory of your cudf clone -git clone https://github.com/rapidsai/rapidsmpf.git -``` - ## Launch a Dev Container To launch a devcontainer from VSCode, open the cuDF repo and select the "Reopen in Container" button in the bottom right:
diff --git a/.devcontainer/cuda12.9-conda/devcontainer.json b/.devcontainer/cuda12.9-conda/devcontainer.json index f8581936a262..5326e64d8c6c 100644 --- a/.devcontainer/cuda12.9-conda/devcontainer.json +++ b/.devcontainer/cuda12.9-conda/devcontainer.json @@ -52,7 +52,7 @@ "initializeCommand": [ "/bin/bash", "-c", - "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.9-envs} ${localWorkspaceFolder}/../rapidsmpf" + "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.9-envs}" ], "postAttachCommand": [ "/bin/bash", @@ -66,8 +66,7 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.9-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.9-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.9-pip/devcontainer.json b/.devcontainer/cuda12.9-pip/devcontainer.json index 03851a926c8c..914237875c0f 100644 --- a/.devcontainer/cuda12.9-pip/devcontainer.json +++ b/.devcontainer/cuda12.9-pip/devcontainer.json @@ -30,7 +30,7 @@ "initializeCommand": [ "/bin/bash", "-c", - "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs} ${localWorkspaceFolder}/../rapidsmpf" + "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs}" ], "postAttachCommand": [ "/bin/bash", @@ -43,8 +43,7 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda13.2-conda/devcontainer.json b/.devcontainer/cuda13.2-conda/devcontainer.json index 78a2f7f64963..a7987bd05dff 100644 --- a/.devcontainer/cuda13.2-conda/devcontainer.json +++ b/.devcontainer/cuda13.2-conda/devcontainer.json @@ -52,7 +52,7 @@ "initializeCommand": [ "/bin/bash", "-c", - "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda13.2-envs} ${localWorkspaceFolder}/../rapidsmpf" + "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda13.2-envs}" ], "postAttachCommand": [ "/bin/bash", @@ -66,8 +66,7 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda13.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda13.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda13.2-pip/devcontainer.json b/.devcontainer/cuda13.2-pip/devcontainer.json index 7a283715b2e8..181ba40473cc 100644 --- a/.devcontainer/cuda13.2-pip/devcontainer.json +++ b/.devcontainer/cuda13.2-pip/devcontainer.json @@ -30,7 +30,7 @@ "initializeCommand": [ "/bin/bash", "-c", - "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs} ${localWorkspaceFolder}/../rapidsmpf" + "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs}" ], "postAttachCommand": [ "/bin/bash", @@ -43,8 +43,7 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda13.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" ], "customizations": { "vscode": { From 70a8ad5269e31d44dc64aeb1748e3216d3ea96b6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 8 Jun 2026 16:42:26 +0000 Subject: [PATCH 12/14] Support finding RAPIDS wheels from site-packages in pure CMake builds --- cpp/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 064549c401dd..a927dde70d33 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -290,6 +290,15 @@ endfunction() # * conda environment ----------------------------------------------------------------------------- rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) +# ################################################################################################## +# * find C++ libraries from RAPIDS wheels in in site-packages -------------------------------------- +find_package(Python COMPONENTS Interpreter) +if(Python_FOUND) + rapids_cython_find_prefix_paths("${Python_EXECUTABLE}" wheel_prefix_paths) + set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS TRUE) + list(APPEND CMAKE_PREFIX_PATH ${wheel_prefix_paths}) +endif() + # ################################################################################################## # * compiler options ------------------------------------------------------------------------------ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules ${CMAKE_MODULE_PATH}) From 3a9b5016c94ae2257a66418f1b48575652385168 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 8 Jun 2026 17:07:38 +0000 Subject: [PATCH 13/14] Add include --- cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a927dde70d33..960c1c61eb43 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -294,6 +294,7 @@ rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) # * find C++ libraries from RAPIDS wheels in in site-packages -------------------------------------- find_package(Python COMPONENTS Interpreter) if(Python_FOUND) + include(${rapids-cmake-dir}/cython-core/find_prefix_paths.cmake) rapids_cython_find_prefix_paths("${Python_EXECUTABLE}" wheel_prefix_paths) set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS TRUE) list(APPEND CMAKE_PREFIX_PATH ${wheel_prefix_paths}) From ca414c10b2dcdc99f2b3f139cbe10237a91bbd77 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 8 Jun 2026 18:02:59 -0700 Subject: [PATCH 14/14] Migrate from_cudf_packed_columns --- python/cudf_polars/cudf_polars/engine/spmd.py | 9 +- .../actor_graph/collectives/allgather.py | 8 +- .../cudf_streaming/integrations/__init__.py | 7 +- .../cudf_streaming/integrations/partition.pxd | 6 ++ .../cudf_streaming/integrations/partition.pyi | 6 ++ .../cudf_streaming/integrations/partition.pyx | 86 ++++++++++++++++++- .../cudf_streaming/tests/test_table_chunk.py | 7 +- 7 files changed, 117 insertions(+), 12 deletions(-) diff --git a/python/cudf_polars/cudf_polars/engine/spmd.py b/python/cudf_polars/cudf_polars/engine/spmd.py index e7905c845c56..96fe41de72b1 100644 --- a/python/cudf_polars/cudf_polars/engine/spmd.py +++ b/python/cudf_polars/cudf_polars/engine/spmd.py @@ -11,7 +11,10 @@ from typing import TYPE_CHECKING, Any, cast import pylibcudf as plc -from cudf_streaming.integrations.partition import unpack_and_concat +from cudf_streaming.integrations.partition import ( + packed_data_from_cudf_packed_columns, + unpack_and_concat, +) from pylibcudf.contiguous_split import pack from rapidsmpf import bootstrap from rapidsmpf.coll import AllGather @@ -19,7 +22,6 @@ new_communicator as single_communicator, ) from rapidsmpf.communicator.ucxx import barrier -from rapidsmpf.memory.packed_data import PackedData from rapidsmpf.progress_thread import ProgressThread from rapidsmpf.rmm_resource_adaptor import RmmResourceAdaptor from rapidsmpf.statistics import Statistics @@ -161,12 +163,11 @@ def allgather_polars_dataframe( plc_table = plc.Table.from_arrow(local_df, stream=stream) - packed_data = PackedData.from_cudf_packed_columns( + packed_data = packed_data_from_cudf_packed_columns( pack(plc_table, stream), stream, ctx.br(), ) - # Bulk AllGather: each rank contributes once (sequence_number=0) allgather = AllGather(comm, op_id, ctx.br()) try: diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py index 1772e9bdb41e..8939e50d8121 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/collectives/allgather.py @@ -6,9 +6,11 @@ from typing import TYPE_CHECKING -from cudf_streaming.integrations.partition import unpack_and_concat +from cudf_streaming.integrations.partition import ( + packed_data_from_cudf_packed_columns, + unpack_and_concat, +) from pylibcudf.contiguous_split import pack -from rapidsmpf.memory.packed_data import PackedData from rapidsmpf.streaming.coll.allgather import AllGather if TYPE_CHECKING: @@ -70,7 +72,7 @@ def insert(self, sequence_number: int, chunk: TableChunk) -> None: sequence_number, # TODO: Avoid unnecessary copies. # See https://github.com/rapidsai/rapidsmpf/issues/933 - PackedData.from_cudf_packed_columns( + packed_data_from_cudf_packed_columns( pack( chunk.table_view(), chunk.stream, diff --git a/python/cudf_streaming/cudf_streaming/integrations/__init__.py b/python/cudf_streaming/cudf_streaming/integrations/__init__.py index 743c0b3ec454..37f3e8a1575c 100644 --- a/python/cudf_streaming/cudf_streaming/integrations/__init__.py +++ b/python/cudf_streaming/cudf_streaming/integrations/__init__.py @@ -2,6 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 """Collection of cuDF specific functions.""" -from cudf_streaming.integrations.partition import unpack_and_concat +from cudf_streaming.integrations.partition import ( + packed_data_from_cudf_packed_columns, + unpack_and_concat, +) -__all__ = ["unpack_and_concat"] +__all__ = ["packed_data_from_cudf_packed_columns", "unpack_and_concat"] diff --git a/python/cudf_streaming/cudf_streaming/integrations/partition.pxd b/python/cudf_streaming/cudf_streaming/integrations/partition.pxd index 774ccb338c10..9d674b1bd4c0 100644 --- a/python/cudf_streaming/cudf_streaming/integrations/partition.pxd +++ b/python/cudf_streaming/cudf_streaming/integrations/partition.pxd @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 +from pylibcudf.contiguous_split cimport PackedColumns from pylibcudf.table cimport Table from rmm.pylibrmm.stream cimport Stream @@ -27,3 +28,8 @@ cpdef object unspill_partitions( BufferResource br, object allow_overbooking, ) +cpdef object packed_data_from_cudf_packed_columns( + PackedColumns packed_columns, + Stream stream, + BufferResource br, +) diff --git a/python/cudf_streaming/cudf_streaming/integrations/partition.pyi b/python/cudf_streaming/cudf_streaming/integrations/partition.pyi index 1133c65ab1c9..efb30be09176 100644 --- a/python/cudf_streaming/cudf_streaming/integrations/partition.pyi +++ b/python/cudf_streaming/cudf_streaming/integrations/partition.pyi @@ -4,6 +4,7 @@ from __future__ import annotations from collections.abc import Iterable +from pylibcudf.contiguous_split import PackedColumns from pylibcudf.table import Table from rapidsmpf.memory.buffer_resource import BufferResource @@ -39,3 +40,8 @@ def unspill_partitions( br: BufferResource, allow_overbooking: bool, ) -> list[PackedData]: ... +def packed_data_from_cudf_packed_columns( + packed_columns: PackedColumns, + stream: Stream, + br: BufferResource, +) -> PackedData: ... diff --git a/python/cudf_streaming/cudf_streaming/integrations/partition.pyx b/python/cudf_streaming/cudf_streaming/integrations/partition.pyx index cc607d841934..f9fdc5b62203 100644 --- a/python/cudf_streaming/cudf_streaming/integrations/partition.pyx +++ b/python/cudf_streaming/cudf_streaming/integrations/partition.pyx @@ -4,16 +4,18 @@ from cython.operator cimport dereference as deref from cython.operator cimport postincrement -from libc.stdint cimport uint32_t +from libc.stdint cimport uint8_t, uint32_t from libcpp.memory cimport make_unique, unique_ptr from libcpp.unordered_map cimport unordered_map from libcpp.utility cimport move from libcpp.vector cimport vector +from pylibcudf.contiguous_split cimport PackedColumns from pylibcudf.libcudf.table.table cimport table as cpp_table from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport size_type from pylibcudf.table cimport Table from rmm.librmm.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.device_buffer cimport device_buffer from rmm.pylibrmm.stream cimport Stream from rapidsmpf._detail.exception_handling cimport ex_handler @@ -316,6 +318,32 @@ cdef extern from "" nogil: ) except +ex_handler +cdef extern from *: + """ + #include + #include + #include + #include + + std::unique_ptr cpp_packed_data_from_buffers( + std::unique_ptr> metadata, + std::unique_ptr gpu_data, + rmm::cuda_stream_view stream, + rapidsmpf::BufferResource* br + ) { + return std::make_unique( + std::move(metadata), br->move(std::move(gpu_data), stream) + ); + } + """ + unique_ptr[cpp_PackedData] cpp_packed_data_from_buffers( + unique_ptr[vector[uint8_t]] metadata, + unique_ptr[device_buffer] gpu_data, + cuda_stream_view stream, + cpp_BufferResource* br, + ) except +ex_handler nogil + + cpdef object unspill_partitions( object partitions, BufferResource br, @@ -369,3 +397,59 @@ cpdef object unspill_partitions( ab, ) return packed_data_vector_to_list(move(_ret), br) + + +cpdef object packed_data_from_cudf_packed_columns( + PackedColumns packed_columns, + Stream stream, + BufferResource br, +): + """ + Construct a PackedData from a pylibcudf PackedColumns. + + Takes ownership of the metadata and GPU data from the PackedColumns + object, leaving it empty. + + Parameters + ---------- + packed_columns + Packed columns from ``pylibcudf.contiguous_split.pack()``. + Must not already be empty (already released). + stream + The CUDA stream on which the preceding ``pack()`` call was performed. + Must be the same stream to ensure correct memory ordering. + br + Buffer resource for memory management. + + Returns + ------- + A new PackedData instance owning the packed column data. + + Raises + ------ + ValueError + If the PackedColumns object is empty (already released). + + See Also + -------- + pylibcudf.contiguous_split.pack + cudf_streaming.integrations.partition.unpack_and_concat + """ + if packed_columns is None or stream is None or br is None: + raise TypeError("Arguments must not be None") + cdef cuda_stream_view _stream = stream.view() + cdef cpp_BufferResource* _br = br.ptr() + cdef PackedData ret = PackedData.__new__(PackedData) + with nogil: + if not (packed_columns.c_obj != NULL and + deref(packed_columns.c_obj).metadata and + deref(packed_columns.c_obj).gpu_data): + raise ValueError("Cannot release empty PackedColumns") + ret.c_obj = cpp_packed_data_from_buffers( + move(deref(packed_columns.c_obj).metadata), + move(deref(packed_columns.c_obj).gpu_data), + _stream, + _br, + ) + ret._br = br + return ret diff --git a/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py b/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py index f48a83e84131..6b55abd01f3c 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py @@ -9,6 +9,9 @@ import pylibcudf as plc import pytest +from cudf_streaming.integrations.partition import ( + packed_data_from_cudf_packed_columns, +) from cudf_streaming.streaming.table_chunk import ( TableChunk, make_table_chunks_available_or_wait, @@ -406,7 +409,7 @@ def test_shape_accessor( expected_shape = (expect.num_rows(), expect.num_columns()) if from_pack: - pd = PackedData.from_cudf_packed_columns( + pd = packed_data_from_cudf_packed_columns( plc.contiguous_split.pack(expect, stream), stream, context.br() ) device_chunk = TableChunk.from_packed_data(pd, br=context.br()) @@ -444,7 +447,7 @@ def test_into_packed_data( ) -> None: expect = random_table(1024) if from_pack: - pd = PackedData.from_cudf_packed_columns( + pd = packed_data_from_cudf_packed_columns( plc.contiguous_split.pack(expect, stream), stream, context.br() ) chunk = TableChunk.from_packed_data(pd, br=context.br())