Add Python/Cython bindings for libcudf_streaming

.github/CODEOWNERS

@@ -10,6 +10,7 @@ python/            @rapidsai/cudf-python-codeowners
 notebooks/         @rapidsai/cudf-python-codeowners
 python/dask_cudf/  @rapidsai/cudf-dask-codeowners
 python/cudf_polars/ @rapidsai/cudf-polars-codeowners
+python/cudf_streaming/ @rapidsai/rapidsmpf-python-codeowners
 
 #cmake code owners
 CMakeLists.txt @rapidsai/cudf-cmake-codeowners

.github/workflows/build.yaml

@@ -217,6 +217,46 @@ jobs:
       date: ${{ inputs.date }}
       package-name: libcudf_streaming
       package-type: cpp
+  wheel-build-cudf-streaming:
+    needs: [telemetry-setup, wheel-build-libcudf-streaming, wheel-build-pylibcudf]
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    secrets: inherit # zizmor: ignore[secrets-inherit]
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
+    with:
+      # Build a wheel for each CUDA x ARCH x minimum supported Python version
+      matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      node_type: cpu16
+      script: ci/build_wheel_cudf_streaming.sh
+      package-name: cudf_streaming
+      package-type: python
+  wheel-publish-cudf-streaming:
+    needs: wheel-build-cudf-streaming
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@main
+    secrets:
+      CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN: ${{ secrets.CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN }}
+      RAPIDSAI_PYPI_TOKEN: ${{ secrets.RAPIDSAI_PYPI_TOKEN }}
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cudf_streaming
+      package-type: python
   wheel-build-pylibcudf:
     needs: [telemetry-setup, wheel-build-libcudf]
     permissions:

.github/workflows/pr.yaml

@@ -27,6 +27,8 @@ jobs:
       - docs-build
       - wheel-build-libcudf
       - wheel-build-libcudf-streaming
+      - wheel-build-cudf-streaming
+      - wheel-tests-cudf-streaming
       - wheel-build-pylibcudf
       - wheel-build-cudf
       - wheel-tests-cudf
@@ -542,6 +544,38 @@ jobs:
       script: "ci/build_wheel_libcudf_streaming.sh"
       package-name: libcudf_streaming
       package-type: cpp
+  wheel-build-cudf-streaming:
+    needs: [checks, wheel-build-libcudf-streaming, wheel-build-pylibcudf]
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    secrets: inherit # zizmor: ignore[secrets-inherit]
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
+    with:
+      # Build a wheel for each CUDA x ARCH x minimum supported Python version
+      matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber)))
+      build_type: pull-request
+      node_type: cpu16
+      script: "ci/build_wheel_cudf_streaming.sh"
+      package-name: cudf_streaming
+      package-type: python
+  wheel-tests-cudf-streaming:
+    needs: [wheel-build-cudf-streaming, changed-files]
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    secrets: inherit # zizmor: ignore[secrets-inherit]
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels
+    with:
+      build_type: pull-request
+      script: ci/test_wheel_cudf_streaming.sh
   wheel-build-pylibcudf:
     needs: [checks, wheel-build-libcudf]
     permissions:

build.sh

@@ -18,8 +18,8 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd "$(dirname "$0")"; pwd)
 
-VALIDARGS="clean libcudf pylibcudf cudf cudf_polars dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz libcudf_streaming -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings"
-HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudf_polars] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [libcudf_streaming] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
+VALIDARGS="clean libcudf pylibcudf cudf cudf_polars dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz libcudf_streaming cudf_streaming -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings"
+HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudf_polars] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [libcudf_streaming] [cudf_streaming] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
    clean                         - remove all existing build artifacts and configuration (start
                                    over)
    libcudf                       - build the cudf C++ code only
@@ -33,6 +33,7 @@ HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudf_polars] [dask_cudf] [benchma
    cudf_kafka                    - build the cudf_kafka Python package
    custreamz                     - build the custreamz Python package
    libcudf_streaming              - build the libcudf_streaming C++ code only
+   cudf_streaming                - build the cudf_streaming Python package
    -v                            - verbose build mode
    -g                            - build for debug
    -n                            - no install step (does not affect Python)
@@ -367,3 +368,13 @@ if hasArg libcudf_streaming; then
         cmake --build . -j"${PARALLEL_LEVEL}" --target install ${VERBOSE_FLAG}
     fi
 fi
+
+# build cudf_streaming Python package
+if hasArg cudf_streaming; then
+    cd "${REPODIR}/python/cudf_streaming"
+    SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${STREAMING_LIB_BUILD_DIR};${EXTRA_CMAKE_ARGS[*]}" \
+        python -m pip install \
+            "${PYTHON_ARGS_FOR_INSTALL[@]}" \
+            "${PY_API_ARGS[@]}" \
+            .
+fi

ci/build_python.sh

@@ -63,6 +63,16 @@ rapids-telemetry-record build-cudf_kafka.log \
 rapids-telemetry-record sccache-stats-cudf_kafka.txt sccache --show-adv-stats
 sccache --stop-server >/dev/null 2>&1 || true
 
+rapids-logger "Building cudf_streaming"
+
+rapids-telemetry-record build-cudf_streaming.log \
+    rattler-build build --recipe conda/recipes/cudf_streaming \
+                    "${RATTLER_ARGS[@]}" \
+                    "${RATTLER_CHANNELS[@]}"
+
+rapids-telemetry-record sccache-stats-cudf_streaming.txt sccache --show-adv-stats
+sccache --stop-server >/dev/null 2>&1 || true
+
 # remove build_cache directory
 rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache
 

ci/build_wheel_cudf_streaming.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+source rapids-init-pip
+
+package_name="cudf-streaming"
+package_dir="python/cudf_streaming"
+dependency_file_key_suffix="cudf_streaming"
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")"
+
+# Downloads libcudf_streaming wheel from this current build,
+# then ensures 'cudf_streaming' wheel builds always use the 'libcudf_streaming' just built in the same CI run.
+LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
+echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_STREAMING_WHEELHOUSE}/libcudf_streaming_*.whl)" >> "${PIP_CONSTRAINT}"
+
+rapids-logger "Generating build requirements"
+
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key "py_build_${dependency_file_key_suffix}" \
+  --file-key "py_rapids_build_${dependency_file_key_suffix}" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \
+  | tee /tmp/requirements-build.txt
+
+rapids-logger "Installing build requirements"
+rapids-pip-retry install \
+    -v \
+    --prefer-binary \
+    -r /tmp/requirements-build.txt
+
+# build with '--no-build-isolation', for better sccache hit rate
+# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735)
+export PIP_NO_BUILD_ISOLATION=0
+
+# TODO: move this variable into `ci-wheel`
+# Format Python limited API version string
+RAPIDS_PY_API="cp${RAPIDS_PY_VERSION//./}"
+export RAPIDS_PY_API
+
+./ci/build_wheel.sh "${package_name}" "${package_dir}" --stable
+
+# repair wheels and write to the location that artifact-uploading code expects to find them
+python -m auditwheel repair \
+    --exclude libcudf.so \
+    --exclude libcudf_streaming.so \
+    --exclude librapidsmpf.so \
+    --exclude librapids_logger.so \
+    --exclude librmm.so \
+    --exclude libucxx.so \
+    --exclude libucp.so.0 \
+    -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" \
+    ${package_dir}/dist/*
+
+./ci/validate_wheel.sh "${package_dir}" "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}"
+
+RAPIDS_PACKAGE_NAME="$(rapids-package-name wheel_python cudf_streaming --stable --cuda)"
+export RAPIDS_PACKAGE_NAME

ci/run_cudf_streaming_pytests.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+# Support invoking run_cudf_streaming_pytests.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_streaming/cudf_streaming/tests
+
+# OpenMPI specific options (CI runs as root)
+export OMPI_ALLOW_RUN_AS_ROOT=1
+export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export OMPI_MCA_opal_cuda_support=1
+
+# cudf_streaming tests require MPI for the communicator fixtures.
+# Run with mpirun; currently single-rank only tests exist.
+mpirun --map-by node --bind-to none -np 1 \
+  python -m pytest --cache-clear "$@" .

ci/test_python_other.sh

@@ -54,5 +54,9 @@ rapids-logger "pytest cudf-polars"
   --durations=10 --durations-min=10 \
   -ra
 
+rapids-logger "pytest cudf_streaming"
+timeout 30m ./ci/run_cudf_streaming_pytests.sh \
+  --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-streaming.xml"
+
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}

ci/test_wheel_cudf_streaming.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+source rapids-init-pip
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")"
+
+# Download cudf_streaming, libcudf_streaming, and pylibcudf built in previous steps
+CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" cudf_streaming --stable --cuda "$RAPIDS_CUDA_VERSION")")
+LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
+LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
+PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")")
+
+# generate constraints (possibly pinning to oldest support versions of dependencies)
+rapids-generate-pip-constraints py_test_cudf_streaming "${PIP_CONSTRAINT}"
+
+rapids-logger "Install cudf_streaming and its dependencies"
+
+rapids-pip-retry install \
+    -v \
+    --prefer-binary \
+    --constraint "${PIP_CONSTRAINT}" \
+    "$(echo "${CUDF_STREAMING_WHEELHOUSE}"/cudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \
+    "$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+    "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+    "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)"
+
+rapids-logger "pytest cudf_streaming"
+pushd python/cudf_streaming/cudf_streaming/tests
+EXITCODE=0
+timeout 30m python -m pytest \
+  --cache-clear \
+  --numprocesses=8 \
+  --dist=worksteal \
+  . || EXITCODE=$?
+
+# Exit code 5 means no tests were collected (all skipped); acceptable when
+# communicator support (MPI/UCXX) is unavailable in the wheel test environment.
+if [ ${EXITCODE} -ne 0 ] && [ ${EXITCODE} -ne 5 ]; then
+  exit ${EXITCODE}
+fi
+popd

ci/validate_wheel.sh

@@ -32,6 +32,10 @@ elif [[ "${package_dir}" == "python/libcudf_streaming" ]]; then
     PYDISTCHECK_ARGS+=(
         --max-allowed-size-compressed '100M'
     )
+elif [[ "${package_dir}" == "python/cudf_streaming" ]]; then
+    PYDISTCHECK_ARGS+=(
+        --max-allowed-size-compressed '75M'
+    )
 elif [[ "${package_dir}" != "python/cudf" ]] && \
      [[ "${package_dir}" != "python/cudf_polars" ]] && \
      [[ "${package_dir}" != "python/dask_cudf" ]] && \

conda/recipes/cudf_streaming/conda_build_config.yaml

@@ -0,0 +1,17 @@
+c_compiler_version:
+  - 14
+
+cxx_compiler_version:
+  - 14
+
+c_stdlib:
+  - sysroot
+
+c_stdlib_version:
+  - "2.28"
+
+cmake_version:
+  - ">=4.0"
+
+cuda_compiler:
+  - cuda-nvcc