Create new libcudf_streaming C++ library

.github/CODEOWNERS

@@ -2,6 +2,9 @@
 cpp/                 @rapidsai/cudf-cpp-codeowners
 python/cudf/udf_cpp/ @rapidsai/cudf-cpp-codeowners
 
+#libcudf_streaming code owners (order matters, this supersedes cpp/ for files in this directory)
+cpp/libcudf_streaming/  @rapidsai/rapidsmpf-cpp-codeowners
+
 #python code owners
 python/            @rapidsai/cudf-python-codeowners
 notebooks/         @rapidsai/cudf-python-codeowners
@@ -12,6 +15,9 @@ python/cudf_polars/ @rapidsai/cudf-polars-codeowners
 CMakeLists.txt @rapidsai/cudf-cmake-codeowners
 **/cmake/      @rapidsai/cudf-cmake-codeowners
 *.cmake        @rapidsai/cudf-cmake-codeowners
+cpp/libcudf_streaming/CMakeLists.txt @rapidsai/rapidsmpf-cmake-codeowners
+cpp/libcudf_streaming/**/cmake/      @rapidsai/rapidsmpf-cmake-codeowners
+cpp/libcudf_streaming/*.cmake        @rapidsai/rapidsmpf-cmake-codeowners
 
 #java code owners
 java/              @rapidsai/cudf-java-codeowners

.github/workflows/build.yaml

@@ -177,6 +177,46 @@ jobs:
       date: ${{ inputs.date }}
       package-name: libcudf
       package-type: cpp
+  wheel-build-libcudf-streaming:
+    needs: [telemetry-setup, wheel-build-libcudf]
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    secrets: inherit # zizmor: ignore[secrets-inherit]
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
+    with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      node_type: cpu16
+      script: ci/build_wheel_libcudf_streaming.sh
+      package-name: libcudf_streaming
+      package-type: cpp
+  wheel-publish-libcudf-streaming:
+    needs: wheel-build-libcudf-streaming
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@main
+    secrets:
+      CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN: ${{ secrets.CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN }}
+      RAPIDSAI_PYPI_TOKEN: ${{ secrets.RAPIDSAI_PYPI_TOKEN }}
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libcudf_streaming
+      package-type: cpp
   wheel-build-pylibcudf:
     needs: [telemetry-setup, wheel-build-libcudf]
     permissions:

.github/workflows/pr.yaml

@@ -26,6 +26,7 @@ jobs:
       - conda-notebook-tests
       - docs-build
       - wheel-build-libcudf
+      - wheel-build-libcudf-streaming
       - wheel-build-pylibcudf
       - wheel-build-cudf
       - wheel-tests-cudf
@@ -523,6 +524,24 @@ jobs:
       script: "ci/build_wheel_libcudf.sh"
       package-name: libcudf
       package-type: cpp
+  wheel-build-libcudf-streaming:
+    needs: [checks, wheel-build-libcudf]
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    secrets: inherit # zizmor: ignore[secrets-inherit]
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
+    with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: pull-request
+      node_type: cpu16
+      script: "ci/build_wheel_libcudf_streaming.sh"
+      package-name: libcudf_streaming
+      package-type: cpp
   wheel-build-pylibcudf:
     needs: [checks, wheel-build-libcudf]
     permissions:

build.sh

@@ -18,8 +18,8 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd "$(dirname "$0")"; pwd)
 
-VALIDARGS="clean libcudf pylibcudf cudf cudf_polars dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings"
-HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudf_polars] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
+VALIDARGS="clean libcudf pylibcudf cudf cudf_polars dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz libcudf_streaming -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings"
+HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudf_polars] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [libcudf_streaming] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
    clean                         - remove all existing build artifacts and configuration (start
                                    over)
    libcudf                       - build the cudf C++ code only
@@ -32,6 +32,7 @@ HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudf_polars] [dask_cudf] [benchma
    libcudf_kafka                 - build the libcudf_kafka C++ code only
    cudf_kafka                    - build the cudf_kafka Python package
    custreamz                     - build the custreamz Python package
+   libcudf_streaming              - build the libcudf_streaming C++ code only
    -v                            - verbose build mode
    -g                            - build for debug
    -n                            - no install step (does not affect Python)
@@ -51,14 +52,15 @@ HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudf_polars] [dask_cudf] [benchma
 "
 LIB_BUILD_DIR=${LIB_BUILD_DIR:=${REPODIR}/cpp/build}
 KAFKA_LIB_BUILD_DIR=${KAFKA_LIB_BUILD_DIR:=${REPODIR}/cpp/libcudf_kafka/build}
+STREAMING_LIB_BUILD_DIR=${STREAMING_LIB_BUILD_DIR:=${REPODIR}/cpp/libcudf_streaming/build}
 CUDF_KAFKA_BUILD_DIR=${REPODIR}/python/cudf_kafka/build
 CUDF_BUILD_DIR=${REPODIR}/python/cudf/build
 DASK_CUDF_BUILD_DIR=${REPODIR}/python/dask_cudf/build
 PYLIBCUDF_BUILD_DIR=${REPODIR}/python/pylibcudf/build
 CUSTREAMZ_BUILD_DIR=${REPODIR}/python/custreamz/build
 CUDF_JAR_JAVA_BUILD_DIR="$REPODIR/java/target"
 
-BUILD_DIRS="${LIB_BUILD_DIR} ${CUDF_BUILD_DIR} ${DASK_CUDF_BUILD_DIR} ${KAFKA_LIB_BUILD_DIR} ${CUDF_KAFKA_BUILD_DIR} ${CUSTREAMZ_BUILD_DIR} ${CUDF_JAR_JAVA_BUILD_DIR} ${PYLIBCUDF_BUILD_DIR}"
+BUILD_DIRS="${LIB_BUILD_DIR} ${CUDF_BUILD_DIR} ${DASK_CUDF_BUILD_DIR} ${KAFKA_LIB_BUILD_DIR} ${CUDF_KAFKA_BUILD_DIR} ${CUSTREAMZ_BUILD_DIR} ${CUDF_JAR_JAVA_BUILD_DIR} ${PYLIBCUDF_BUILD_DIR} ${STREAMING_LIB_BUILD_DIR}"
 
 # Set defaults for vars modified by flags to this script
 VERBOSE_FLAG=""
@@ -199,7 +201,7 @@ fi
 ################################################################################
 # Configure, build, and install libcudf
 
-if buildAll || hasArg libcudf || hasArg pylibcudf || hasArg cudf ; then
+if buildAll || hasArg libcudf || hasArg libcudf_streaming || hasArg pylibcudf || hasArg cudf ; then
     if (( BUILD_ALL_GPU_ARCH == 0 )); then
         CUDF_CMAKE_CUDA_ARCHITECTURES="${CUDF_CMAKE_CUDA_ARCHITECTURES:-NATIVE}"
         if [[ "$CUDF_CMAKE_CUDA_ARCHITECTURES" == "NATIVE" ]]; then
@@ -347,3 +349,21 @@ if hasArg custreamz; then
     cd "${REPODIR}/python/custreamz"
     python -m pip install "${PYTHON_ARGS_FOR_INSTALL[@]}" .
 fi
+
+# Build libcudf_streaming library
+if hasArg libcudf_streaming; then
+    cmake -S "$REPODIR/cpp/libcudf_streaming" -B "${STREAMING_LIB_BUILD_DIR}" \
+          -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
+          -DCMAKE_CUDA_ARCHITECTURES="${CUDF_CMAKE_CUDA_ARCHITECTURES}" \
+          -DBUILD_TESTS=${BUILD_TESTS} \
+          -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+          "${EXTRA_CMAKE_ARGS[@]}"
+
+
+    cd "${STREAMING_LIB_BUILD_DIR}"
+    cmake --build . -j"${PARALLEL_LEVEL}" ${VERBOSE_FLAG}
+
+    if [[ ${INSTALL_TARGET} != "" ]]; then
+        cmake --build . -j"${PARALLEL_LEVEL}" --target install ${VERBOSE_FLAG}
+    fi
+fi

ci/build_wheel_libcudf_streaming.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+source rapids-init-pip
+
+package_name="libcudf_streaming"
+package_dir="python/libcudf_streaming"
+
+rapids-logger "Generating build requirements"
+
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key "py_build_${package_name}" \
+  --file-key "py_rapids_build_${package_name}" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \
+| tee /tmp/requirements-build.txt
+
+rapids-logger "Installing build requirements"
+rapids-pip-retry install \
+    -v \
+    --prefer-binary \
+    -r /tmp/requirements-build.txt
+
+# build with '--no-build-isolation', for better sccache hit rate
+# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735)
+export PIP_NO_BUILD_ISOLATION=0
+
+./ci/build_wheel.sh "${package_name}" "${package_dir}"
+
+# repair wheels and write to the location that artifact-uploading code expects to find them
+python -m auditwheel repair \
+    --exclude libcudf.so \
+    --exclude librapidsmpf.so \
+    --exclude librapids_logger.so \
+    --exclude librmm.so \
+    --exclude libucxx.so \
+    --exclude libucp.so.0 \
+    -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" \
+    ${package_dir}/dist/*
+
+./ci/validate_wheel.sh "${package_dir}" "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}"

ci/release/update-version.sh

@@ -137,13 +137,14 @@ fi
 
 # Centralized version file update
 echo "${NEXT_FULL_TAG}" > VERSION
+# The cudf version file must be a copy, see https://github.com/rapidsai/cudf/pull/18198
 echo "${NEXT_FULL_TAG}" > python/cudf/cudf/VERSION
 echo "${RAPIDS_BRANCH_NAME}" > RAPIDS_BRANCH
 
 DEPENDENCIES=(
   cudf
-  cudf_kafka
   cudf-polars
+  cudf_kafka
   cugraph
   cuml
   custreamz
@@ -152,8 +153,8 @@ DEPENDENCIES=(
   kvikio
   libcudf
   libcudf-example
-  libcudf_kafka
   libcudf-tests
+  libcudf_kafka
   libkvikio
   librmm
   pylibcudf

ci/run_cudf_streaming_ctests.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+# Support customizing the ctests' install location
+# First, try the installed location (CI/conda environments)
+installed_test_location="${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcudf_streaming/"
+# Fall back to the build directory (devcontainer environments)
+script_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+devcontainers_test_location="${script_dir}/../cpp/libcudf_streaming/build/latest"
+buildsh_test_location="${STREAMING_LIB_BUILD_DIR:-${script_dir}/../cpp/libcudf_streaming/build}"
+
+if [[ -d "${installed_test_location}" ]]; then
+    cd "${installed_test_location}"
+elif [[ -d "${devcontainers_test_location}" ]]; then
+    cd "${devcontainers_test_location}"
+elif [[ -d "${buildsh_test_location}" ]]; then
+    cd "${buildsh_test_location}"
+else
+    echo "Error: Test location not found. Searched:" >&2
+    echo "  - ${installed_test_location}" >&2
+    echo "  - ${devcontainers_test_location}" >&2
+    echo "  - ${buildsh_test_location}" >&2
+    exit 1
+fi
+
+ctest --output-on-failure --no-tests=error "$@"

ci/test_cpp.sh

@@ -32,5 +32,11 @@ if (( SUITEERROR == 0 )); then
     SUITEERROR=$?
 fi
 
+if (( SUITEERROR == 0 )); then
+    rapids-logger "Run libcudf_streaming gtests"
+    timeout 30m ./ci/run_cudf_streaming_ctests.sh -j20
+    SUITEERROR=$?
+fi
+
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}

ci/validate_wheel.sh

@@ -28,6 +28,10 @@ if [[ "${package_dir}" == "python/libcudf" ]]; then
             --max-allowed-size-compressed '350M'
         )
     fi
+elif [[ "${package_dir}" == "python/libcudf_streaming" ]]; then
+    PYDISTCHECK_ARGS+=(
+        --max-allowed-size-compressed '100M'
+    )
 elif [[ "${package_dir}" != "python/cudf" ]] && \
      [[ "${package_dir}" != "python/cudf_polars" ]] && \
      [[ "${package_dir}" != "python/dask_cudf" ]] && \

conda/recipes/libcudf/recipe.yaml

@@ -38,7 +38,7 @@ cache:
         done
 
         ./build.sh -n -v \
-            libcudf libcudf_kafka benchmarks tests \
+            libcudf libcudf_kafka libcudf_streaming benchmarks tests \
             --build_metrics --incl_cache_stats --allgpuarch \
             --cmake-args="\"-DCUDF_ENABLE_ARROW_S3=ON\" $cmake_args"
       secrets:
@@ -83,6 +83,7 @@ cache:
       - ${{ stdlib("c") }}
     host:
       - librmm =${{ minor_version }}
+      - librapidsmpf =${{ minor_version }}
       - libkvikio =${{ minor_version }}
       - cuda-cudart-dev
       - cuda-nvrtc-dev
@@ -216,6 +217,53 @@ outputs:
       license: Apache-2.0
       summary: libcudf_kafka library
 
+  - package:
+      name: libcudf-streaming
+      version: ${{ version }}
+    build:
+      script:
+        - cmake --install cpp/libcudf_streaming/build
+      string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }}
+      dynamic_linking:
+        overlinking_behavior: "error"
+    requirements:
+      build:
+        - cmake ${{ cmake_version }}
+        - ${{ stdlib("c") }}
+      host:
+        - ${{ pin_subpackage("libcudf", exact=True) }}
+        - librmm =${{ minor_version }}
+        - librapidsmpf =${{ minor_version }}
+        - cuda-version =${{ cuda_version }}
+        - cuda-cudart-dev
+      run:
+        - ${{ pin_subpackage("libcudf", exact=True) }}
+        - librmm =${{ minor_version }}
+        - librapidsmpf =${{ minor_version }}
+        - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }}
+      ignore_run_exports:
+        by_name:
+          - cuda-cudart
+          - cuda-nvrtc
+          - cuda-nvtx
+          - cuda-version
+          - flatbuffers
+          - libcudf
+          - libcufile
+          - libcurand
+          - libkvikio
+          - librdkafka
+          - librmm
+          - libzlib
+          - libnvcomp
+    tests:
+      - script:
+        - test -f $PREFIX/lib/libcudf_streaming.so
+    about:
+      homepage: https://rapids.ai/
+      license: Apache-2.0
+      summary: libcudf-streaming library
+
   - package:
       name: libcudf-example
       version: ${{ version }}
@@ -301,6 +349,7 @@ outputs:
       script:
         - cmake --install cpp/build --component testing
         - cmake --install cpp/libcudf_kafka/build --component testing
+        - cmake --install cpp/libcudf_streaming/build --component testing
       string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }}
       dynamic_linking:
         overlinking_behavior: "error"
@@ -313,6 +362,7 @@ outputs:
       host:
         - ${{ pin_subpackage("libcudf", exact=True) }}
         - ${{ pin_subpackage("libcudf_kafka", exact=True) }}
+        - ${{ pin_subpackage("libcudf-streaming", exact=True) }}
         - cuda-version =${{ cuda_version }}
         - libcurand-dev
         - cuda-cudart-dev
@@ -321,6 +371,7 @@ outputs:
         - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }}
         - ${{ pin_subpackage("libcudf", exact=True) }}
         - ${{ pin_subpackage("libcudf_kafka", exact=True) }}
+        - ${{ pin_subpackage("libcudf-streaming", exact=True) }}
         - libcurand
         - cuda-sanitizer-api
       ignore_run_exports: