From 3003a8c6579cb05c34c1c74c88531152337836d2 Mon Sep 17 00:00:00 2001 From: paul-aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:46:13 -0700 Subject: [PATCH 1/6] Add cuDF JAR build support for all Maven classifiers (#23261) * Split the JAR build into three composable stages (static libcudf build, per-classifier JAR packaging, and Maven-repo gather) so each stage is independently runnable in CI and locally. * Link against a static libcudf built from source per CUDA version rather than a conda shared libcudf. * Emit the Maven classifier based on the host architecture the build runs on, introducing a new `-arm64` suffix to distinguish aarch64 JARs from their x86_64 counterparts. * Add `test_java_build_local.sh` as a one-command local reproducer of the full CI matrix for the host arch, with per-step timings and GPU compute-capability auto-detection. Contributes to https://github.com/rapidsai/cudf/issues/22204 Authors: - https://github.com/paul-aiyedun Approvers: - Mike Sarahan (https://github.com/msarahan) - Tim Liu (https://github.com/NvTimLiu) URL: https://github.com/rapidsai/cudf/pull/23261 --- .github/workflows/build.yaml | 70 +++++ dependencies.yaml | 24 ++ java/ci/README.md | 107 ++++++- java/ci/argparse.sh | 36 +++ java/ci/assemble_maven_repo.sh | 180 +++++++++++ java/ci/build_cudf_java_jar.sh | 246 +++++++++++++++ java/ci/build_cudf_java_jar_in_container.sh | 121 +++++++ java/ci/build_static_libcudf.sh | 138 ++++++++ java/ci/build_static_libcudf_in_container.sh | 95 ++++++ java/ci/test_java_build_local.sh | 315 +++++++++++++++++++ java/pom.xml | 7 + 11 files changed, 1326 insertions(+), 13 deletions(-) create mode 100644 java/ci/argparse.sh create mode 100755 java/ci/assemble_maven_repo.sh create mode 100755 java/ci/build_cudf_java_jar.sh create mode 100755 java/ci/build_cudf_java_jar_in_container.sh create mode 100755 java/ci/build_static_libcudf.sh create mode 100755 java/ci/build_static_libcudf_in_container.sh create mode 100755 java/ci/test_java_build_local.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f6bddc6cc93..c1a386577155 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -471,3 +471,73 @@ jobs: with: push: true cuda: '["12.9", "13.3"]' + # Build the cuDF Java JAR for every Maven classifier. + java-build: + needs: [telemetry-setup] + strategy: + fail-fast: false + matrix: + include: + - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } + runs-on: ${{ matrix.runner }} + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Build static libcudf + run: | + ./java/ci/build_static_libcudf.sh \ + --output-dir "${RUNNER_TEMP}/libcudf" \ + --cuda-version "${{ matrix.cuda }}" + - name: Build cuDF Java JAR + run: | + ./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir "${RUNNER_TEMP}/libcudf" \ + --output-dir "${RUNNER_TEMP}/jars" \ + --cuda-version "${{ matrix.cuda }}" + - name: Upload per-entry JAR artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} + # Ship only the JARs and POMs. Exclude the per-classifier Maven build scratch dir. + path: | + ${{ runner.temp }}/jars + !${{ runner.temp }}/jars/.mvn-temp-target + if-no-files-found: error + # Assemble the per-classifier JARs into one Maven-repository layout. + java-gather: + needs: [java-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Download per-entry JAR artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cudf_java_cuda* + path: ${{ runner.temp }}/jars + merge-multiple: true + - name: Assemble Maven repository layout + run: | + ./java/ci/assemble_maven_repo.sh \ + --jars-dir "${RUNNER_TEMP}/jars" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload combined Maven repository artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error diff --git a/dependencies.yaml b/dependencies.yaml index f054189a8d37..6c6c9ac67996 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -129,6 +129,19 @@ files: - depends_on_cudf_polars - depends_on_ray - depends_on_cudf_streaming + build_java: + # Toolchain for building static libcudf from source and packaging the cuDF + # Java JAR. `depends_on_libcudf` is excluded because the Java build links + # against a static libcudf built from source, not a conda shared libcudf. + output: none + includes: + - build_base + - build_all + - build_cpp + - cuda + - cuda_static + - cuda_version + - build_java test_java: output: none includes: @@ -632,6 +645,17 @@ dependencies: - croaring==4.4.2 - flatbuffers==24.3.25 - librdkafka<2.15.0a0 + build_java: + common: + - output_types: conda + packages: + - boost + # cuda_profiler_api.h is used by the JNI layer (CudaJni.cpp) but is + # not pulled in by the base `cuda` dev packages. + - cuda-profiler-api + - make + - maven + - openjdk=8.* depends_on_libnvcomp: common: - output_types: conda diff --git a/java/ci/README.md b/java/ci/README.md index 5ef4850a2bb1..219e08e5eaff 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -1,11 +1,100 @@ # Build Jar artifact of cuDF -## Build the docker image +## Recommended: self-contained release build scripts -### Prerequisite +The scripts under `java/ci/` build the cuDF Java JAR for every Maven classifier the +same way locally and in CI (GitHub Actions is only a thin wrapper that adds +artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image, +runs the build in a throwaway container, and writes its output to a host +directory. No local `docker build` is required, and no GPU is required to build. -1. Docker should be installed. -2. [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) should be installed. +### Prerequisites + +1. Docker is installed and the current user can run `docker`. +2. Network access to pull `rapidsai/ci-conda:-latest`. + +### Local one-command shortcut + +For local testing only, `java/ci/test_java_build_local.sh` runs Steps 1-3 end-to-end for both CUDA 12 and CUDA 13 on the host architecture. + +```bash +./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test +``` + +### Step 1 - Build the static libcudf install tree + +```bash +./java/ci/build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version 12.9 +``` + +This produces a static libcudf install tree (`lib/libcudf.a` plus its static +dependencies) under the given output directory. Build outputs are host-user-owned +so plain `rm -rf` works. + +### Step 2 - Package the cuDF Java JAR for one classifier + +```bash +./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir /tmp/libcudf-cuda12 \ + --output-dir /tmp/jars \ + --cuda-version 12.9 +``` + +This compiles the JNI layer against the static libcudf from Step 1 and emits a +single classifier JAR (e.g. `cudf-26.08.0-SNAPSHOT-cuda12.jar`) plus its POM +into a classifier-named subdirectory under `--output-dir`: + +``` +/tmp/jars/cuda12/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The classifier is derived from `--cuda-version` (major) + host arch (`uname +-m`): `cuda12` / `cuda13` on `x86_64`, `cuda12-arm64` / `cuda13-arm64` on +`aarch64`. Producing the ARM classifiers requires a real `aarch64` host. +Repeat Step 2 for each classifier, pointing `--libcudf-dir` at the matching +static libcudf tree and using the same `--output-dir` (each classifier lands +in its own subdirectory). Concurrent invocations for different classifiers +are safe because each nests its own bind-mount over `/repo/java/target` +inside the container. + +### Step 3 - Assemble the Maven repository layout + +```bash +./java/ci/assemble_maven_repo.sh \ + --jars-dir /tmp/jars \ + --output-dir /tmp/maven-repo +``` + +This walks every subdirectory of `--jars-dir` (each subdir name IS the +classifier), gathers the per-classifier JAR and shared POM, derives the +artifact version from the JAR filenames (requiring a single unique version +across subdirs), and lays them out as: + +``` +/tmp/maven-repo/ai/rapids/cudf/26.08.0-SNAPSHOT/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT-cuda13.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The set of classifiers is whatever subdirectories are present under +`--jars-dir`. For a local `x86_64`-only run, populate `/tmp/jars/cuda12/` +and `/tmp/jars/cuda13/`. For the full four-way release build, add +`/tmp/jars/cuda12-arm64/` and `/tmp/jars/cuda13-arm64/`. + +In GitHub Actions (`.github/workflows/build.yaml`), the `java-build` matrix job +runs Steps 1-2 per (CUDA x arch) entry and uploads each classifier subdir as a +per-entry artifact. The separate `java-gather` job downloads them (with +`merge-multiple: true`, so all subdirs land in a single parent dir), runs +Step 3, and uploads the combined `cudf_java_maven_repo` artifact. + +## Legacy: manual Dockerfile.rocky build (obsolete) + +> The `java/ci/Dockerfile.rocky` + `java/ci/build-in-docker.sh` flow below is the +> old build path. It is retained for reference but superseded by the +> self-contained scripts above. ### Build the docker image @@ -20,25 +109,19 @@ The following CUDA versions are supported w/ CUDA Enhanced Compatibility: Change the --build-arg CUDA_VERSION to what you need. You can replace the tag "cudf-build:12.9.1-devel-rocky8" with another name you like. -## Start the docker then build - -### Start the docker +### Start the docker then build Run below command to start a docker container with GPU. ```bash nvidia-docker run -it cudf-build:12.9.1-devel-rocky8 bash ``` -### Download the cuDF source code - You can download the cuDF repo in the docker container or you can mount it into the container. Here I choose to download again in the container. ```bash git clone --recursive https://github.com/rapidsai/cudf.git -b main ``` -### Build cuDF jar with devtoolset - ```bash cd cudf export WORKSPACE=`pwd` @@ -46,6 +129,4 @@ source java/ci/env.sh ${sclCMD} "java/ci/build-in-docker.sh" ``` -### The output - You can find the cuDF jar in java/target/ like cudf-26.08.0-SNAPSHOT-cuda12.jar. diff --git a/java/ci/argparse.sh b/java/ci/argparse.sh new file mode 100644 index 000000000000..9a06d35c9df8 --- /dev/null +++ b/java/ci/argparse.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared argparse helpers for the java/ci/ host orchestrator scripts. +# Meant to be sourced, not executed: +# . "${SCRIPT_DIR}/argparse.sh" + +# require_value +# Check: exit 1 when is empty (i.e. the flag was passed +# without its argument, or was the last token on the command line). +require_value() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} requires a value" >&2 + exit 1 + fi +} + +# require_arg +# Check: assert that a required flag was actually supplied by the +# caller. Prints the script's print_help (if defined) then exits 1 on failure. +# Preserves the existing behavior of showing help after a "required flag missing" +# error. +require_arg() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} is required." >&2 + if declare -F print_help > /dev/null; then + print_help + fi + exit 1 + fi +} diff --git a/java/ci/assemble_maven_repo.sh b/java/ci/assemble_maven_repo.sh new file mode 100755 index 000000000000..b1d3e65a457c --- /dev/null +++ b/java/ci/assemble_maven_repo.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Decoupled gather step: assemble per-classifier cuDF Java JARs into a single +# Maven-repository-layout directory. +# +# Input: --jars-dir contains one subdirectory per classifier, each holding +# exactly one cudf--.jar and a cudf-.pom. Subdir +# names ARE the classifier names, and the artifact version is derived from +# the JAR filenames (all subdirs must agree). +# +# Output layout: +# /ai/rapids/cudf//cudf--.jar +# /ai/rapids/cudf//cudf-.pom + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +GROUP_PATH="ai/rapids" +ARTIFACT_ID="cudf" + +JARS_DIR="" +OUTPUT_DIR="" + +print_help() { + cat << EOF + +Usage: assemble_maven_repo.sh --jars-dir --output-dir + +Gathers per-classifier cuDF Java JARs into a single Maven-repository-layout tree. + +REQUIRED: + -j, --jars-dir Parent directory containing one subdirectory per + classifier (each holding cudf--.jar + and cudf-.pom). Subdir name is the classifier. + -o, --output-dir Directory to receive the combined Maven-repository layout. + +OPTIONS: + -h, --help Show this help message. + +EXAMPLE: + assemble_maven_repo.sh --jars-dir /tmp/jars --output-dir /tmp/maven-repo + # given /tmp/jars/{cuda12,cuda13}/ inputs, produces: + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda12.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda13.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -j|--jars-dir) + require_value "$1" "$2" + JARS_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --jars-dir "${JARS_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" + +if [[ ! -d ${JARS_DIR} ]]; then + echo "Error: --jars-dir '${JARS_DIR}' does not exist." + exit 1 +fi + +if [[ -e ${OUTPUT_DIR} && -n "$(ls -A "${OUTPUT_DIR}" 2>/dev/null)" ]]; then + echo "Error: --output-dir '${OUTPUT_DIR}' must be empty or nonexistent" >&2 + exit 1 +fi + +ASSEMBLE_FINISHED=0 +cleanup_partial_output() { + if [[ ${ASSEMBLE_FINISHED} -eq 0 && -e ${OUTPUT_DIR} ]]; then + echo "Assembly did not complete; removing partial output at ${OUTPUT_DIR}" >&2 + rm -rf "${OUTPUT_DIR}" + fi +} +trap cleanup_partial_output EXIT + +echo "Assembling Maven repository layout" +echo " jars dir: ${JARS_DIR}" +echo " output dir: ${OUTPUT_DIR}" + +# Walk every classifier subdirectory. Each subdir must contain exactly one +# cudf-*-.jar. The version is derived from the filename and must +# match across all subdirs. +FIRST_VERSION="" +CLASSIFIERS_SEEN="" + +for subdir in "${JARS_DIR}"/*/; do + classifier=$(basename "${subdir}") + + jar="" + for candidate in "${subdir}"cudf-*-"${classifier}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${jar}" ]]; then + echo "Error: multiple JARs in ${subdir} match cudf-*-${classifier}.jar" >&2 + exit 1 + fi + jar=${candidate} + fi + done + + if [[ -z "${jar}" ]]; then + echo "Error: no cudf-*-${classifier}.jar found in ${subdir}" >&2 + exit 1 + fi + + # Filename is cudf--.jar. Peel prefix and suffix. + base=$(basename "${jar}" .jar) + version=$(echo "${base}" | sed -e 's/^cudf-//' -e "s/-${classifier}$//") + + if [[ -z "${FIRST_VERSION}" ]]; then + FIRST_VERSION=${version} + elif [[ "${version}" != "${FIRST_VERSION}" ]]; then + echo "Error: inconsistent versions across subdirs: ${FIRST_VERSION} vs ${version} (${subdir})" >&2 + exit 1 + fi + + DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${version}" + mkdir -p "${DEST_DIR}" + cp -f "${jar}" "${DEST_DIR}/" + echo " + $(basename "${jar}")" + CLASSIFIERS_SEEN="${CLASSIFIERS_SEEN} ${classifier}" +done + +if [[ -z "${FIRST_VERSION}" ]]; then + echo "Error: no classifier subdirs found under ${JARS_DIR}" >&2 + exit 1 +fi + +# POM is identical across subdirs; copy the first one found. +POM_SRC="" +for subdir in "${JARS_DIR}"/*/; do + candidate=${subdir}cudf-${FIRST_VERSION}.pom + if [[ -f "${candidate}" ]]; then + POM_SRC=${candidate} + break + fi +done + +if [[ -z "${POM_SRC}" ]]; then + echo "Error: no cudf-${FIRST_VERSION}.pom found under ${JARS_DIR}" >&2 + exit 1 +fi + +DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${FIRST_VERSION}" +cp -f "${POM_SRC}" "${DEST_DIR}/cudf-${FIRST_VERSION}.pom" +echo " + cudf-${FIRST_VERSION}.pom" + +echo "Maven repository assembled successfully at ${OUTPUT_DIR}" +echo "Classifiers present:${CLASSIFIERS_SEEN}" + +ASSEMBLE_FINISHED=1 diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh new file mode 100755 index 000000000000..084b00c68478 --- /dev/null +++ b/java/ci/build_cudf_java_jar.sh @@ -0,0 +1,246 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained packaging of the cuDF Java JAR for a single classifier. +# +# Consumes a prebuilt static libcudf install tree (from build_static_libcudf.sh), +# compiles the JNI layer against it inside a throwaway RAPIDS ci-conda container, +# and emits the single classifier JAR (plus its POM) to a per-classifier +# subdirectory under --output-dir. This script is layout-agnostic: it produces +# one classifier's artifacts and knows nothing about the combined +# Maven-repository layout (see java/ci/assemble_maven_repo.sh). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +LIBCUDF_DIR="" +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_cudf_java_jar.sh --libcudf-dir --output-dir \\ + --cuda-version [OPTIONS] + +Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-conda +container, linking against a prebuilt static libcudf. Always builds for the +host architecture (uname -m). The build image is fixed to +rapidsai/ci-conda:-latest (version derived from the VERSION +file). + +The classifier is derived from --cuda-version (major) + host arch (uname -m), +mirroring the pom.xml Groovy logic: "cuda" for x86_64, +"cuda-arm64" for aarch64. The classifier JAR + POM are written to +//. Concurrent invocations targeting different +classifiers are safe. + +REQUIRED: + -l, --libcudf-dir Static libcudf install tree produced by + build_static_libcudf.sh. + -o, --output-dir Host parent directory. The script creates and writes + to //, which must not already + exist. + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + Must match --cuda-version of the static libcudf tree; + determines the cuda12/cuda13 classifier. + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. Must match the value passed to + build_static_libcudf.sh when producing the static + libcudf tree in --libcudf-dir, or device linking of + libcudfjni.so against libcudf.a will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_cudf_java_jar.sh -l /tmp/libcudf-cuda12 -o /tmp/jars -c 12.9 + build_cudf_java_jar.sh -l /tmp/libcudf-cuda13 -o /tmp/jars -c 13.3 -A 80 + # writes: + # /tmp/jars/cuda12/cudf--cuda12.jar + # /tmp/jars/cuda12/cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -l|--libcudf-dir) + require_value "$1" "$2" + LIBCUDF_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --libcudf-dir "${LIBCUDF_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +if [[ ! -d ${LIBCUDF_DIR} ]]; then + echo "Error: --libcudf-dir '${LIBCUDF_DIR}' does not exist." + exit 1 +fi + +# Derive the Maven classifier from --cuda-version major + host arch, mirroring +# the pom.xml Groovy logic: "cuda" for x86_64, "cuda-arm64" for +# aarch64. +CUDA_MAJOR="$(echo "${CUDA_VERSION}" | cut -d. -f1)" +HOST_ARCH="$(uname -m)" +case "${HOST_ARCH}" in + x86_64) + CLASSIFIER="cuda${CUDA_MAJOR}" + ;; + aarch64|arm64) + CLASSIFIER="cuda${CUDA_MAJOR}-arm64" + ;; + *) + echo "Error: Unsupported host arch '${HOST_ARCH}' (expected x86_64 or aarch64)" >&2 + exit 1 + ;; +esac + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" +LIBCUDF_DIR="$(cd "${LIBCUDF_DIR}" && pwd)" + +CLASSIFIER_OUT="${OUTPUT_DIR}/${CLASSIFIER}" +if [[ -e ${CLASSIFIER_OUT} ]]; then + echo "Error: classifier output '${CLASSIFIER_OUT}' already exists. Remove it before re-running." >&2 + exit 1 +fi +mkdir -p "${CLASSIFIER_OUT}" + +# Per-classifier scratch dir for Maven's java/target/. Nested bind-mount over +# /repo/java/target inside the container isolates concurrent invocations +# (each classifier gets its own target/). The `.mvn-temp-target/` prefix +# keeps this dir invisible to the default `*/` globbing in +# assemble_maven_repo.sh's classifier discovery loop. +# +# Recreate the scratch on every launch: the in-container mvn cannot clean a +# bind-mount point (rmdir on /repo/java/target fails with EBUSY), so the +# host wrapper is responsible for guaranteeing a clean starting target/. +TARGET_SCRATCH="${OUTPUT_DIR}/.mvn-temp-target/${CLASSIFIER}" +rm -rf "${TARGET_SCRATCH}" +mkdir -p "${TARGET_SCRATCH}" + +echo "Packaging cuDF Java JAR" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " classifier: ${CLASSIFIER}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " libcudf dir: ${LIBCUDF_DIR}" +echo " output dir: ${CLASSIFIER_OUT}" +echo " target dir: ${TARGET_SCRATCH}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${LIBCUDF_DIR}:/libcudf:ro" + --volume "${CLASSIFIER_OUT}:/output" + --volume "${TARGET_SCRATCH}:/repo/java/target" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_cudf_java_jar_in_container.sh + +# Post-run: assert exactly one main classifier JAR + one POM, and that the +# JAR's classifier suffix matches the subdir name we chose (catches pom drift). +PRODUCED_JAR="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*-"${CLASSIFIER}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_JAR}" ]]; then + echo "Error: multiple JARs matching cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_JAR=${candidate} + fi +done + +if [[ -z "${PRODUCED_JAR}" ]]; then + echo "Error: no cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +PRODUCED_POM="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*.pom; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_POM}" ]]; then + echo "Error: multiple POMs found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_POM=${candidate} + fi +done + +if [[ -z "${PRODUCED_POM}" ]]; then + echo "Error: no cudf-*.pom found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +echo "cuDF Java JAR build succeeded:" +echo " $(basename "${PRODUCED_JAR}")" +echo " $(basename "${PRODUCED_POM}")" diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh new file mode 100755 index 000000000000..98fce46c4f4a --- /dev/null +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container packaging of the cuDF Java JAR for a single classifier. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_cudf_java_jar.sh. It generates the build_java conda toolchain +# environment, compiles the JNI layer against a prebuilt static libcudf +# (mounted at /libcudf), and packages the cuDF Java JAR. The resulting +# classifier JAR and its POM are copied to /output. /output and +# /repo/java/target are chowned to HOST_UID:HOST_GID on exit so the host user +# owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output and /repo/java/target +# (both required). + +set -e + +OUTPUT_DIR=/output +REPO_ROOT=/repo +CUDF_INSTALL_DIR=/libcudf + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + +_chown_outputs_on_exit() { + chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true +} +trap _chown_outputs_on_exit EXIT + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +BUILD_ARG=( + -B + "-Dmaven.repo.local=/tmp/.m2" + "-Dparallel.level=${PARALLEL_LEVEL}" + "-DskipTests=true" + "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON" + "-DCUDF_JNI_LIBCUDF_STATIC=ON" + "-DUSE_GDS=OFF" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + BUILD_ARG+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +cd "${REPO_ROOT}/java" + +CUDF_VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" +rapids-logger "Packaging cuDF Java JAR version ${CUDF_VERSION} (libcudf: ${CUDF_INSTALL_DIR})" + +# The `clean` goal is intentionally omitted: /repo/java/target is a +# bind-mount point, so when `mvn clean` attempts to remove the directory, +# it fails with EBUSY. The host wrapper (build_cudf_java_jar.sh) recreates +# the scratch dir before each container launch to guarantee target/ starts empty. +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" mvn package "${BUILD_ARG[@]}" + +MAIN_JAR="" +for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do + case "${candidate}" in + *-tests.jar|*-sources.jar|*-javadoc.jar) + continue + ;; + esac + if [[ -f ${candidate} ]]; then + MAIN_JAR=${candidate} + break + fi +done + +if [[ -z ${MAIN_JAR} ]]; then + echo "Error: no cuDF classifier JAR produced under target/" + ls -l target/ || true + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" +cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" +cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" + +rapids-logger "Emitted $(basename "${MAIN_JAR}") + cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" diff --git a/java/ci/build_static_libcudf.sh b/java/ci/build_static_libcudf.sh new file mode 100755 index 000000000000..2fa2e435471c --- /dev/null +++ b/java/ci/build_static_libcudf.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained build of a static libcudf install tree. +# +# Pulls the RAPIDS ci-conda image, builds libcudf with BUILD_SHARED_LIBS=OFF +# inside a throwaway container, and installs the static libcudf tree (libcudf.a +# plus its static dependencies) into a directory on the host. No GPU is required +# to build. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_static_libcudf.sh --output-dir --cuda-version [OPTIONS] + +Builds a static libcudf install tree inside a RAPIDS ci-conda container and +writes it to a directory on the host. Always builds for the host architecture +(uname -m). The build image is fixed to rapidsai/ci-conda:-latest +(version derived from the VERSION file). + +REQUIRED: + -o, --output-dir Host directory to receive the static install tree + (libcudf.a and its static dependencies). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. When packaging the cuDF Java JAR + against this static libcudf tree, pass the same value + to build_cudf_java_jar.sh --cmake-cuda-architectures + or device linking of libcudfjni.so against libcudf.a + will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version "12.9" + build_static_libcudf.sh -o /tmp/libcudf-cuda13 -c 13.3 -A "80" + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" + +echo "Building static libcudf" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " output dir: ${OUTPUT_DIR}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${OUTPUT_DIR}:/output" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_static_libcudf_in_container.sh + +if [[ -f "${OUTPUT_DIR}/lib/libcudf.a" || -f "${OUTPUT_DIR}/lib64/libcudf.a" ]]; then + echo "Static libcudf build succeeded: ${OUTPUT_DIR}" +else + echo "Error: expected libcudf.a not found under ${OUTPUT_DIR}/lib or ${OUTPUT_DIR}/lib64" + exit 1 +fi diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh new file mode 100755 index 000000000000..e77637e747b2 --- /dev/null +++ b/java/ci/build_static_libcudf_in_container.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container build of a static libcudf install tree. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_static_libcudf.sh. It generates the build_java conda toolchain +# environment, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs the +# resulting static libcudf (plus its static dependencies) into /output. Then +# chowns /output to HOST_UID:HOST_GID so the host user owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output (both required). + +set -e + +INSTALL_PREFIX=/output +REPO_ROOT=/repo +BUILD_DIR=/tmp/libcudf-build + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +CMAKE_ARGS=( + -S "${REPO_ROOT}/cpp" + -B "${BUILD_DIR}" + -GNinja + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTS=OFF + -DUSE_NVTX=ON + -DCUDF_LARGE_STRINGS_DISABLED=ON + -DCUDF_USE_ARROW_STATIC=ON + -DCUDF_ENABLE_ARROW_S3=OFF + -DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON + -DRMM_LOGGING_LEVEL=OFF + -DCUDF_KVIKIO_REMOTE_IO=OFF +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + CMAKE_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +rapids-logger "Configuring static libcudf" +cmake "${CMAKE_ARGS[@]}" + +rapids-logger "Building static libcudf with ${PARALLEL_LEVEL} jobs" +cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" + +rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" +cmake --install "${BUILD_DIR}" + +rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" +chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" diff --git a/java/ci/test_java_build_local.sh b/java/ci/test_java_build_local.sh new file mode 100755 index 000000000000..1d67ad62771d --- /dev/null +++ b/java/ci/test_java_build_local.sh @@ -0,0 +1,315 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Local end-to-end verification of the build workflow: builds the static +# libcudf install tree and the classifier JAR for both CUDA 12 and CUDA 13, +# then runs the decoupled gather step to assemble the combined +# Maven-repository layout. Mirrors what the java-build matrix + java-gather +# jobs in .github/workflows/build.yaml do in CI. +# +# Runs on either x86_64 or aarch64 hosts. Each invocation covers only the +# host architecture: on x86_64 it produces the "cuda12" and "cuda13" +# classifier JARs; on aarch64 it produces "cuda12-arm64" and "cuda13-arm64". +# The arm64 suffix is added automatically by the child build scripts (via +# pom.xml's classifier logic keyed off `uname -m`). To cover all four +# release classifiers, run this script once on each architecture. +# +# Both static libcudf builds run in parallel, and both JAR builds run in +# parallel (each uses a nested bind-mount over /repo/java/target inside the +# container to isolate Maven output). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +WORK_DIR="" +PARALLEL_LEVEL="$(nproc)" +CMAKE_CUDA_ARCHITECTURES="" + +# CUDA versions built for both classifiers. Keep in sync with the java-build +# matrix in .github/workflows/build.yaml. +CUDA12_VERSION="12.9" +CUDA13_VERSION="13.3" + +# Wall-clock timing variables. +STEP_NAMES=() +STEP_ELAPSED=() + +print_help() { + cat << EOF + +Usage: test_java_build_local.sh --work-dir [OPTIONS] + +Runs the full build+gather pipeline for both CUDA 12 and CUDA 13 on the host +architecture (x86_64 or aarch64) and assembles the combined Maven-repository +layout. + +REQUIRED: + -w, --work-dir Scratch directory for build outputs. Subtrees created: + /libcudf-cuda12 static libcudf (CUDA 12) + /libcudf-cuda13 static libcudf (CUDA 13) + /jars/ per-classifier JAR + POM + /maven-repo combined Maven layout + where is "cuda12" / "cuda13" on x86_64 + and "cuda12-arm64" / "cuda13-arm64" on aarch64. + +OPTIONS: + -j, --parallel Total build parallelism (default: nproc = ${PARALLEL_LEVEL}). + Each concurrent JAR build gets --parallel/2 to avoid + RAM pressure from two parallel nvcc runs. + -A, --cmake-cuda-architectures + CUDA architecture list (e.g. "80" or "80;90") passed to + both build scripts. The literal value "all" is a + sentinel meaning "do not pass --cmake-cuda-architectures + to child scripts" — child scripts then fall back to + cuDF's default RAPIDS full architecture list (slow). + Default: auto-detect the local GPU's compute + capability via nvidia-smi (e.g. Ampere -> "80"). If + nvidia-smi is missing or returns nothing, falls back + to "all". + -h, --help Show this help message. + +EXAMPLES: + # Fast run (auto-detect local GPU arch): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test + + # Explicit override: + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures 80 + + # Full RAPIDS arch list (slow, e.g. GPU-less host): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures all + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -w|--work-dir) + require_value "$1" "$2" + WORK_DIR=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +log_step() { + echo + echo "============================================================" + echo "== $1" + echo "============================================================" +} + +format_elapsed() { + local s=$1 + printf '%dm %02ds' $((s/60)) $((s%60)) +} + +record_step_end() { + local name=$1 + local start=$2 + local elapsed=$((SECONDS - start)) + STEP_NAMES+=("${name}") + STEP_ELAPSED+=("${elapsed}") + echo + echo "== ${name} completed in $(format_elapsed "${elapsed}")" +} + +parse_args "$@" + +require_arg --work-dir "${WORK_DIR}" + +mkdir -p "${WORK_DIR}" +WORK_DIR="$(cd "${WORK_DIR}" && pwd)" +LOG_DIR="${WORK_DIR}/logs" +mkdir -p "${LOG_DIR}" + +# Remove outputs from any prior run so cmake/mvn does not see stale artifacts. +rm -rf "${WORK_DIR}/libcudf-cuda12" \ + "${WORK_DIR}/libcudf-cuda13" \ + "${WORK_DIR}/jars" \ + "${WORK_DIR}/maven-repo" + +# Auto-detect the local GPU's compute capability when the flag was not passed. +# "all" is the sentinel value that means "do not forward this flag to child +# scripts". Child scripts fall back to cuDF's default RAPIDS architecture +# list (slow but correct on GPU-less hosts). +if [[ -z ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DETECTED="" + if command -v nvidia-smi > /dev/null 2>&1; then + DETECTED=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' | tr -d ' ') + fi + if [[ -n ${DETECTED} ]]; then + CMAKE_CUDA_ARCHITECTURES=${DETECTED} + echo "Auto-detected local GPU compute capability: ${CMAKE_CUDA_ARCHITECTURES}" + else + CMAKE_CUDA_ARCHITECTURES="all" + echo "No local GPU detected; falling back to cuDF's default RAPIDS architecture list" + fi +fi + +echo "cuDF Java local build verification" +echo " host arch: $(uname -m)" +echo " work dir: ${WORK_DIR}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " cuda12 version: ${CUDA12_VERSION}" +echo " cuda13 version: ${CUDA13_VERSION}" +echo " cmake cuda architectures: ${CMAKE_CUDA_ARCHITECTURES}" +echo " logs: ${LOG_DIR}/{static,jar}_cuda{12,13}.log" + +# When forwarding to child scripts, "all" means "don't pass the flag". +CHILD_CMAKE_ARGS=() +if [[ ${CMAKE_CUDA_ARCHITECTURES} != "all" ]]; then + CHILD_CMAKE_ARGS=(--cmake-cuda-architectures "${CMAKE_CUDA_ARCHITECTURES}") +fi + +# Both Step 1 and Step 2 launch two concurrent builds. Each build gets half +# of PARALLEL_LEVEL so together they stay within PARALLEL_LEVEL. +STEP_PARALLEL=$((PARALLEL_LEVEL / 2)) +if [[ ${STEP_PARALLEL} -lt 1 ]]; then + STEP_PARALLEL=1 +fi + +# Step 1: static libcudf builds in parallel. +log_step "Step 1: building static libcudf for CUDA 12 and CUDA 13 in parallel" +STEP1_START=${SECONDS} + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda12" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda12.log" 2>&1 & +STATIC_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda13" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda13.log" 2>&1 & +STATIC_CUDA13_PID=$! + +echo " cuda12 static pid: ${STATIC_CUDA12_PID} (tail -f ${LOG_DIR}/static_cuda12.log)" +echo " cuda13 static pid: ${STATIC_CUDA13_PID} (tail -f ${LOG_DIR}/static_cuda13.log)" + +STATIC_CUDA12_RC=0 +STATIC_CUDA13_RC=0 +if ! wait "${STATIC_CUDA12_PID}"; then + STATIC_CUDA12_RC=1 +fi +if ! wait "${STATIC_CUDA13_PID}"; then + STATIC_CUDA13_RC=1 +fi + +if [[ ${STATIC_CUDA12_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 12 build failed." + echo "See ${LOG_DIR}/static_cuda12.log" +fi +if [[ ${STATIC_CUDA13_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 13 build failed." + echo "See ${LOG_DIR}/static_cuda13.log" +fi +if [[ ${STATIC_CUDA12_RC} -ne 0 || ${STATIC_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 1: static libcudf (parallel run)" "${STEP1_START}" + +# Step 2: JAR builds in parallel. Each build's container nests a bind-mount +# over /repo/java/target so concurrent Maven runs don't clobber each other. +log_step "Step 2: packaging cuDF Java JARs for cuda12 and cuda13 in parallel" +STEP2_START=${SECONDS} + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda12" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda12.log" 2>&1 & +JAR_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda13" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda13.log" 2>&1 & +JAR_CUDA13_PID=$! + +echo " cuda12 jar pid: ${JAR_CUDA12_PID} (tail -f ${LOG_DIR}/jar_cuda12.log)" +echo " cuda13 jar pid: ${JAR_CUDA13_PID} (tail -f ${LOG_DIR}/jar_cuda13.log)" + +JAR_CUDA12_RC=0 +JAR_CUDA13_RC=0 +if ! wait "${JAR_CUDA12_PID}"; then + JAR_CUDA12_RC=1 +fi +if ! wait "${JAR_CUDA13_PID}"; then + JAR_CUDA13_RC=1 +fi + +if [[ ${JAR_CUDA12_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 12 build failed." + echo "See ${LOG_DIR}/jar_cuda12.log" +fi +if [[ ${JAR_CUDA13_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 13 build failed." + echo "See ${LOG_DIR}/jar_cuda13.log" +fi +if [[ ${JAR_CUDA12_RC} -ne 0 || ${JAR_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 2: JAR builds (parallel run)" "${STEP2_START}" + +# Step 3: gather into a combined Maven-repository layout. +log_step "Step 3: assembling combined Maven-repository layout" +STEP3_START=${SECONDS} + +"${SCRIPT_DIR}/assemble_maven_repo.sh" \ + --jars-dir "${WORK_DIR}/jars" \ + --output-dir "${WORK_DIR}/maven-repo" + +record_step_end "Step 3: assemble Maven repo" "${STEP3_START}" + +# Derive the assembled version from the output tree for display purposes. +CUDF_VERSION=$(basename "$(ls -d "${WORK_DIR}/maven-repo/ai/rapids/cudf"/*/ | head -1)") + +log_step "Success" +echo "Combined Maven repository:" +echo " ${WORK_DIR}/maven-repo/ai/rapids/cudf/${CUDF_VERSION}/" +echo +echo "Timings:" +for i in "${!STEP_NAMES[@]}"; do + printf ' %-45s %s\n' "${STEP_NAMES[$i]}" "$(format_elapsed "${STEP_ELAPSED[$i]}")" +done +printf ' %-45s %s\n' "Total wall time" "$(format_elapsed "${SECONDS}")" diff --git a/java/pom.xml b/java/pom.xml index ee2a8cf7a603..44d957cab553 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -649,6 +649,13 @@ def cm = cudaPattern.matcher(nvccout) if (cm.find()) { def classifier = 'cuda' + cm.group(1) + // Emit "cuda" on x86_64 and "cuda-arm64" on + // aarch64 so a single pom produces a distinct Maven + // classifier per architecture. + def osArch = System.getProperty('os.arch') + if (osArch == 'aarch64' || osArch == 'arm64') { + classifier = classifier + '-arm64' + } project.properties['cuda.classifier'] = classifier } else { throw new RuntimeException('could not find CUDA version') From 496874ecb8a1ddadcf610b850a59d88ce5488dbf Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 21 Jul 2026 17:35:04 +0100 Subject: [PATCH 2/6] Add timeout stacktrace utility for cudf-polars tests, remove pytest-timeout (#23332) ## Description Timing out an individual test is actually not useful since it is not safe to cancel a test at an arbitrary point in execution: that might leave a collective dangling. Moreover, many of the Polars tests run quite close to the, arbitrarily chosen, timeout limit on CI runners if they are heavily loaded. Since the signal we actually want is the state of a hanging process, instead just add a test-suite level timeout with a utility to print the state of the hanging processes. closes https://github.com/rapidsai/cudf/issues/22948 ## Checklist - [x] I am familiar with the [Contributing Guidelines](https://github.com/rapidsai/cudf/blob/HEAD/CONTRIBUTING.md). - [x] New or existing tests cover these changes. - [x] The documentation is up to date with these changes. --- ci/run_cudf_polars_polars_tests.sh | 13 +- ci/run_cudf_polars_pytests.sh | 7 +- ci/test_wheel_cudf_polars.sh | 3 + ci/timeout_with_stack.py | 353 ++++++++++++++++++ .../all_cuda-129_arch-aarch64.yaml | 3 +- .../all_cuda-129_arch-x86_64.yaml | 3 +- .../all_cuda-133_arch-aarch64.yaml | 3 +- .../all_cuda-133_arch-x86_64.yaml | 3 +- dependencies.yaml | 7 +- python/cudf_polars/pyproject.toml | 3 +- python/cudf_polars/tests/conftest.py | 3 - .../tests/expressions/test_rolling.py | 1 - .../cudf_polars/tests/streaming/test_scan.py | 1 - .../cudf_polars/tests/streaming/test_sort.py | 1 - 14 files changed, 385 insertions(+), 19 deletions(-) create mode 100644 ci/timeout_with_stack.py diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index 5302942db173..3778e1a9035a 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -4,6 +4,8 @@ set -euo pipefail +TIMEOUT_TOOL_PATH="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/timeout_with_stack.py + # Support invoking run_cudf_polars_pytests.sh outside the script directory # Assumption, polars has been cloned in the root of the repo. cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../polars/ @@ -60,16 +62,18 @@ DESELECTED_TESTS_STR=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") # Don't quote the `DESELECTED_...` variable because `pytest` can't handle # multiple quoted arguments inline # shellcheck disable=SC2086 +# Fail fast (-x) because failed tests pollute the state echo "Run polars tests with injected in-memory GPU engine" -python -m pytest \ +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest \ --import-mode=importlib \ --cache-clear \ + -x \ -m "" \ -p cudf_polars.testing.inject_gpu_engine \ -n 4 \ --dist=worksteal \ --tb=native \ - --timeout=240 \ --durations 10 --durations-min 10 \ -ra \ $DESELECTED_TESTS_STR \ @@ -81,9 +85,11 @@ python -m pytest \ echo "Run polars tests with injected SPMD GPU engine, small blocksize" CUDF_POLARS__EXECUTOR__TARGET_PARTITION_SIZE=805306368 \ CUDF_POLARS__EXECUTOR__FALLBACK_MODE=silent \ - python -m pytest \ +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest \ --import-mode=importlib \ --cache-clear \ + -x \ -v \ -m "" \ -p cudf_polars.testing.inject_gpu_engine \ @@ -91,7 +97,6 @@ CUDF_POLARS__EXECUTOR__FALLBACK_MODE=silent \ -n 4 \ --dist=worksteal \ --tb=native \ - --timeout=240 \ --durations 10 --durations-min 10 \ -ra \ $DESELECTED_TESTS_STR \ diff --git a/ci/run_cudf_polars_pytests.sh b/ci/run_cudf_polars_pytests.sh index 82d1ccd4879f..3fac6910c5a8 100755 --- a/ci/run_cudf_polars_pytests.sh +++ b/ci/run_cudf_polars_pytests.sh @@ -1,11 +1,14 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail # It is essential to cd into python/cudf_polars as `pytest-xdist` + `coverage` seem to work only at this directory level. # Support invoking run_cudf_polars_pytests.sh outside the script directory +TIMEOUT_TOOL_PATH="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/timeout_with_stack.py + cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_polars/ -python -m pytest --cache-clear "$@" tests +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest --cache-clear "$@" tests diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 09315bbe981c..d9df9c05d4eb 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -89,12 +89,14 @@ for version in "${VERSIONS[@]}"; do COVERAGE_ARGS=(--no-cov) fi + # Fail fast (-x) rather than trying to continue because failed tests pollute the state ./ci/run_cudf_polars_pytests.sh \ -vv \ "${COVERAGE_ARGS[@]}" \ --numprocesses=4 \ --dist=worksteal \ --durations 10 --durations-min 10 \ + -x \ -ra \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-${version}.xml" @@ -106,6 +108,7 @@ for version in "${VERSIONS[@]}"; do EXITCODE=1 FAILED+=("${version}") rapids-logger "Tests failed for polars ${version}.*" + break else PASSED+=("${version}") rapids-logger "Tests passed for polars ${version}.*" diff --git a/ci/timeout_with_stack.py b/ci/timeout_with_stack.py new file mode 100644 index 000000000000..02f62408fe9e --- /dev/null +++ b/ci/timeout_with_stack.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Module for running commands with timeout and capturing stack traces. + +This module provides functionality to run commands with a timeout, capture stack traces +of processes that exceed the timeout, and properly terminate process trees. + +See Also +-------- +subprocess.Popen : For running subprocesses without timeout. +psutil.Process : For process management and information. + +Examples +-------- +>>> from timeout_with_stack import run_with_timeout +>>> exit_code = run_with_timeout(["sleep", "10"], timeout=5) +>>> print(f"Process exited with code: {exit_code}") +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import signal +import subprocess +import sys +import time +from contextlib import suppress +from enum import IntEnum +from typing import TYPE_CHECKING + +import psutil + +if TYPE_CHECKING: + from collections.abc import Sequence + from types import FrameType + + +class StackType(IntEnum): + """Enum representing the type of stack trace to capture.""" + + C = 0 + Python = 1 + + +def get_child_pids(pid: int) -> list[int]: + """ + Get all child PIDs of a given process. + + This function retrieves all child process IDs (PIDs) of a given process, + including recursively nested child processes. + + Parameters + ---------- + pid + The process ID of the parent process. + + Returns + ------- + A list of child process IDs. Returns an empty list if the parent process + does not exist. + + See Also + -------- + psutil.Process.children : For getting child processes. + + Examples + -------- + >>> from timeout_with_stack import get_child_pids + >>> child_pids = get_child_pids(1234) + >>> print(f"Child PIDs: {child_pids}") + """ + try: + parent = psutil.Process(pid) + children = parent.children(recursive=True) + return [p.pid for p in children] + except psutil.NoSuchProcess: + return [] + + +def capture_stack_trace(pid: int, stack_type=StackType.C) -> None: + """ + Capture stack trace for a given process. + + This function captures the stack trace of a process using GDB. It prints the + stack trace to stdout, which can be useful for debugging hanging or long-running + processes. + + Parameters + ---------- + pid + The process ID of the process to capture stack trace for. + stack_type + The stack type to extract, either C or Python. + + See Also + -------- + capture_all_stacks : For capturing stack traces of a process and its children. + + Examples + -------- + >>> from timeout_with_stack import capture_stack_trace + >>> capture_stack_trace(1234) + """ + if stack_type is StackType.C: + bt_command = "thread apply all bt" + print(f"\nCapturing C stack trace for process {pid}:") + else: + bt_command = "thread apply all py-bt" + print(f"\nCapturing Python stack trace for process {pid}:") + gdb = shutil.which("gdb") + if gdb is None: + print(f"Skipping stack trace for process {pid}: gdb not found") + return + + try: + proc = subprocess.run( + [ + gdb, + "--quiet", + "--pid", + str(pid), + "-ex", + "set pagination off", + "-ex", + "set confirm off", + "-ex", + bt_command, + "-ex", + "quit", + ], + capture_output=True, + text=True, + check=False, + timeout=120, + ) + except subprocess.TimeoutExpired: + print(f"Timed out capturing stack trace for process {pid}") + return + + print(proc.stdout) + if proc.stderr: + print(proc.stderr, file=sys.stderr) + + +def capture_all_stacks(pid: int, *, enable_python: bool = False) -> None: + """ + Capture stack traces for parent and all child processes. + + This function captures stack traces for both the parent process and all its + child processes. It first captures the parent's stack trace, then recursively + captures stack traces for all child processes. + + Parameters + ---------- + pid + The process ID of the parent process. + enable_python + Whether to capture Python stack traces. + + See Also + -------- + capture_stack_trace : For capturing stack trace of a single process. + get_child_pids : For getting child process IDs. + + Examples + -------- + >>> from timeout_with_stack import capture_all_stacks + >>> capture_all_stacks(1234, enable_python=True) + """ + # Capture parent process stack + if enable_python: + capture_stack_trace(pid, stack_type=StackType.Python) + capture_stack_trace(pid, stack_type=StackType.C) + + # Get and capture all child processes + child_pids = get_child_pids(pid) + for child_pid in child_pids: + if enable_python: + capture_stack_trace(child_pid, stack_type=StackType.Python) + capture_stack_trace(child_pid, stack_type=StackType.C) + + +def terminate_process_tree(pid: int) -> None: + """ + Terminate a process and all its children. + + This function terminates a process and all its child processes. It first + attempts to gracefully terminate all processes, then forcefully kills any + remaining processes after a timeout. + + Parameters + ---------- + pid + The process ID of the parent process to terminate. + + See Also + -------- + psutil.Process.terminate : For gracefully terminating a process. + psutil.Process.kill : For forcefully killing a process. + + Examples + -------- + >>> from timeout_with_stack import terminate_process_tree + >>> terminate_process_tree(1234) + """ + try: + parent = psutil.Process(pid) + children = parent.children(recursive=True) + + # Terminate children first + for child in children: + with suppress(psutil.NoSuchProcess): + child.terminate() + + # Create a copy of children list + terminated_children = list(children) + + # Wait for all children to terminate + for child in terminated_children: + with suppress(psutil.TimeoutExpired): + child.wait(timeout=3) + + # Kill any remaining children + for child in terminated_children: + with suppress(psutil.NoSuchProcess): + child.kill() + + # Terminate parent + parent.terminate() + try: + parent.wait(timeout=3) + except psutil.TimeoutExpired: + parent.kill() + + except psutil.NoSuchProcess: + pass + + +def install_signal_handler(pid: int, *signals: signal.Signals) -> None: + """ + Install signal handler that terminates the given pid on receiving any of signals + """ + + def handler(signum: int, frame: FrameType | None) -> None: + print(f"Received {signum=}, terminating process tree") + terminate_process_tree(pid) + sys.exit(signum) + + for sig in signals: + signal.signal(sig, handler) + + +def run_with_timeout( + cmd: Sequence[str], timeout: float, *, enable_python: bool = False +) -> int: + """ + Run a command with a timeout and capture stack traces if it exceeds the timeout. + + This function runs a command with a specified timeout. If the command exceeds + the timeout, it captures stack traces of the process and its children before + terminating them. It handles keyboard interrupts gracefully. + + Parameters + ---------- + cmd + The command and its arguments to run. + timeout + Maximum time in seconds to allow the command to run. + enable_python + Whether to capture Python stack traces. + + Returns + ------- + Return code of the command, or 124 if timeout occurred, or signal.SIGINT + if interrupted by keyboard. + + See Also + -------- + subprocess.Popen : For running subprocesses without timeout. + capture_all_stacks : For capturing stack traces of processes. + + Examples + -------- + >>> from timeout_with_stack import run_with_timeout + >>> exit_code = run_with_timeout(["sleep", "10"], timeout=5, enable_python=True) + >>> print(f"Process exited with code: {exit_code}") + """ + # Start the process with a new process group + # Note: preexec_fn is used here as we need to create a new process group + # for proper termination of child processes + process = subprocess.Popen( + cmd, + preexec_fn=os.setsid, + ) + install_signal_handler( + process.pid, + signal.SIGTERM, + signal.SIGABRT, + signal.SIGHUP, + signal.SIGQUIT, + ) + start_time = time.time() + + try: + while time.time() - start_time < timeout: + if process.poll() is not None: + return process.returncode + time.sleep(0.1) + + print(f"\nProcess timed out after {timeout} seconds") + print("Capturing stack traces for all processes...") + + # Capture stacks for parent and all children + capture_all_stacks(process.pid, enable_python=enable_python) + + # Terminate the entire process tree + print("\nTerminating process tree...") + terminate_process_tree(process.pid) + except KeyboardInterrupt: + print("\nReceived keyboard interrupt") + terminate_process_tree(process.pid) + return signal.SIGINT + else: + return 124 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run a command with timeout and capture stack traces" + ) + parser.add_argument("timeout", type=float, help="Timeout in seconds") + parser.add_argument( + "--enable-python", + action="store_true", + help="Enable Python stack trace capture", + ) + parser.add_argument( + "command", nargs=argparse.REMAINDER, help="Command to run" + ) + + args = parser.parse_args() + + if not args.command: + parser.error("No command specified") + + exit_code = run_with_timeout( + args.command, args.timeout, enable_python=args.enable_python + ) + sys.exit(exit_code) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 79a81086c3b0..3cb635245e2d 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-aarch64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index a4a887efbbb4..c75d11503f60 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-133_arch-aarch64.yaml b/conda/environments/all_cuda-133_arch-aarch64.yaml index 4aa4ae68a4dc..2206e925f896 100644 --- a/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-aarch64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-133_arch-x86_64.yaml b/conda/environments/all_cuda-133_arch-x86_64.yaml index 5648f35aa5ff..f36f02a5c21d 100644 --- a/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/dependencies.yaml b/dependencies.yaml index 6c6c9ac67996..c2041b67228e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -1196,13 +1196,18 @@ dependencies: packages: - rich - pytest-httpserver - - pytest-timeout - zstandard + # Used by ci/timeout_with_stack.py utility + - psutil # The polars test suite constructs pandas objects for interop and # dask-cuda pulls pandas in transitively (unbounded), so constrain it # here to exclude the 3.0.4 release that segfaults on pd.Timedelta # (https://github.com/pandas-dev/pandas/issues/66086). - *pandas + - output_types: conda + packages: + # Used by timeout_with_stack.py utility + - gdb test_python_narwhals: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 37307aa739f8..6007e165747f 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -45,9 +45,9 @@ classifiers = [ test = [ "dask-cuda==26.8.*,>=0.0.0a0", "pandas>=3.0.0,<3.0.4a0", + "psutil", "pytest-cov", "pytest-httpserver", - "pytest-timeout", "pytest-xdist", "pytest<9.1.0", "rich", @@ -91,7 +91,6 @@ filterwarnings = [ "error", "ignore:Port .* is already in use.:UserWarning", ] -timeout = 45 xfail_strict = true [tool.coverage.report] diff --git a/python/cudf_polars/tests/conftest.py b/python/cudf_polars/tests/conftest.py index 218592d52a39..22e034b37d78 100644 --- a/python/cudf_polars/tests/conftest.py +++ b/python/cudf_polars/tests/conftest.py @@ -339,9 +339,6 @@ def engine_raise_on_fail() -> pl.GPUEngine: def timeout_seconds() -> int: """ Conservative timeout for APIs that accept a timeout parameter. - - Since pytest-timeout is installed, ensure this value is less than timeout - in python/cudf_polars/pyproject.toml. """ return 30 diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index e80bb25679c1..570497147d99 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -358,7 +358,6 @@ def test_rank_over_with_null_values( @pytest.mark.parametrize("method", ["ordinal", "dense", "min", "max", "average"]) @pytest.mark.parametrize("descending", [False, True]) @pytest.mark.parametrize("order_by", [None, ["g2", pl.col("x2") * 2]]) -@pytest.mark.timeout(120) def test_rank_over_with_null_group_keys( engine: pl.GPUEngine, df: pl.LazyFrame, diff --git a/python/cudf_polars/tests/streaming/test_scan.py b/python/cudf_polars/tests/streaming/test_scan.py index 13e88ead7731..15ba081e69fb 100644 --- a/python/cudf_polars/tests/streaming/test_scan.py +++ b/python/cudf_polars/tests/streaming/test_scan.py @@ -73,7 +73,6 @@ def df(): ("parquet", pl.scan_parquet), ], ) -@pytest.mark.timeout(90) def test_parallel_scan( tmp_path: Path, df: pl.DataFrame, diff --git a/python/cudf_polars/tests/streaming/test_sort.py b/python/cudf_polars/tests/streaming/test_sort.py index 71e1535988e3..9707ba9886b6 100644 --- a/python/cudf_polars/tests/streaming/test_sort.py +++ b/python/cudf_polars/tests/streaming/test_sort.py @@ -87,7 +87,6 @@ def large_frames(): ) -@pytest.mark.timeout(120) def test_sort(df, engine): q = df.sort(by=["y", "z"]) assert_gpu_result_equal(q, engine=engine) From edbf8cf0c8f73c2b80e758b59cf0ef2ab4441ef5 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 21 Jul 2026 11:42:59 -0500 Subject: [PATCH 3/6] Stop mutating the source column dtype in equivalent-type numerical casts (#23364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split out of #23255 (1/6). `NumericalColumn.as_numerical_column` short-circuits casts between equivalent dtypes (same pylibcudf type, e.g. `float64` → `Float64`), but implemented the shortcut by assigning the target dtype onto `self._dtype` in place. The column object is shared with the caller's Series/DataFrame, so the *source* object silently changed dtype as a side effect of the cast. This returns a fresh column over the same pylibcudf data instead (`nans_to_nulls` first for float → masked casts), and adds a classic regression test. Fixes 5 pandas-tests (`test_stack_nullable_dtype[*]`, `test_loc_set_nan_in_categorical_series[Float64]`, `test_assert_series_equal_extension_dtype_mismatch`, `test_assert_frame_equal_extension_dtype_mismatch`); their xfail entries are removed. Attribution verified by running the node ids against an isolated build containing only this fix (they pass) and a clean build (they fail). Independent of the other #23255 split PRs; can merge in any order. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23364 --- python/cudf/cudf/core/column/numerical.py | 19 ++++++++------- .../pandas/scripts/pandas-testing-plugin.py | 5 ---- .../cudf/tests/series/methods/test_astype.py | 24 +++++++++++++++++++ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index a1d14959e7ca..251514aa66ef 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -911,20 +911,23 @@ def as_numerical_column(self, dtype: DtypeObj) -> NumericalColumn: self.dtype ): # Short-circuit the cast if the dtypes are equivalent - # but not the same type object. + # but not the same type object. Do NOT mutate self._dtype: + # the column object may be shared with the caller's frame. if ( is_pandas_nullable_extension_dtype(dtype) and isinstance(self.dtype, np.dtype) and self.dtype.kind == "f" ): - # If the dtype is a pandas nullable extension type, we need to - # float column doesn't have any NaNs. + # NaNs must become nulls before viewing as a masked dtype. res = self.nans_to_nulls() - res._dtype = dtype - return res - else: - self._dtype = dtype - return self + return cast( + "NumericalColumn", + ColumnBase.create(res.plc_column, dtype), + ) + return cast( + "NumericalColumn", + ColumnBase.create(self.plc_column, dtype), + ) if self.dtype.kind == "f" and dtype.kind in "iu": if not is_pandas_nullable_extension_dtype(dtype) and ( self.nan_count > 0 diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index f09d75e33b39..5f3155adb71b 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1721,8 +1721,6 @@ def pytest_unconfigure(config): "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[True]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_level[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[True]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_order_with_unsorted_levels_multi_row_2[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[True]": "TODO: Add a reason for failure", @@ -2511,7 +2509,6 @@ def pytest_unconfigure(config): "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-pyarrow-True]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-False]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-True]": 'AssertionError: Column name="0" are different', - "tests/indexing/test_loc.py::TestLocWithMultiIndex::test_loc_set_nan_in_categorical_series[Float64]": "TODO: Add a reason for failure", "tests/indexing/test_loc.py::test_loc_getitem_multiindex_tuple_level": "AssertionError: DataFrame Expected type , found instead", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask0-values0-object]": "TODO: Add a reason for failure", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask1-values0-object]": "TODO: Add a reason for failure", @@ -3841,13 +3838,11 @@ def pytest_unconfigure(config): "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2NS]": "TODO: Add a reason for failure", "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2Us]": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[None]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[nan]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_frame_equal_index_dtype_mismatch[True-df11-df21-DataFrame\\\\.index level \\\\[0\\\\] are different]": "Failed: DID NOT RAISE ", "tests/util/test_assert_index_equal.py::test_index_equal_range_categories[True-True]": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_series_equal.py::test_assert_series_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_assert_series_equal_int_tol": "AssertionError: left is not an ExtensionArray", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[Int64]": "Failed: DID NOT RAISE ", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[int64]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/series/methods/test_astype.py b/python/cudf/cudf/tests/series/methods/test_astype.py index a3eaa0020cfa..3071da934e13 100644 --- a/python/cudf/cudf/tests/series/methods/test_astype.py +++ b/python/cudf/cudf/tests/series/methods/test_astype.py @@ -1626,3 +1626,27 @@ def test_astype_aware_to_naive_raises(): cudf_ser.astype("datetime64[ns]") with pytest.raises(TypeError): pd_ser.astype("datetime64[ns]") + + +@pytest.mark.parametrize( + "data, src_dtype, masked_dtype", + [ + ([1.0, 2.0, float("nan")], "float64", pd.Float64Dtype()), + ([1, 2, 3], "int64", pd.Int64Dtype()), + ], +) +def test_astype_masked_equivalent_dtype_no_source_mutation( + data, src_dtype, masked_dtype +): + # casting to the equivalent masked dtype takes a short-circuit path; + # it must not mutate the source column's dtype in place (the column + # is shared with the source Series/frame) + ser = cudf.Series(data, dtype=src_dtype) + result = ser.astype(masked_dtype) + + assert ser.dtype == np.dtype(src_dtype) + assert result.dtype == masked_dtype + assert_eq( + result.to_pandas(), + pd.Series(data, dtype=src_dtype).astype(masked_dtype), + ) From a537b771b8e20055a047bc30828d3b959ece4a80 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 21 Jul 2026 13:02:19 -0500 Subject: [PATCH 4/6] Mirror cudf.pandas class-level monkeypatches onto the real type (#23001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split out of #22927 per review. ## Problem A class-level attribute write on a cudf.pandas proxy type — e.g. `monkeypatch.setattr(pd.ExcelFile, "parse", fn)` — was only applied to the proxy class. Code that runs under `disable_module_accelerator()` (such as the pandas fallback path of `pd.read_excel`) resolves attributes from the *real* class, so a patch applied only to the proxy was invisible to it. ## Fix Add `_FastSlowProxyMeta.__setattr__`/`__delattr__` to mirror runtime class-level patches onto the underlying "slow" (real) type: - `__setattr__` mirrors the assignment after translating the assigned value into "slow" space. A plain value is forwarded as-is. Re-assigning the proxy's *pristine* attribute for a name (which is exactly what `monkeypatch.setattr` / `mock.patch.object` save and re-assign on undo) translates to the slow type's pristine attribute for that name — restored if the slow type had one of its own, or removed if it didn't (leaving any inherited implementation visible). Proxy machinery (e.g. a saved `pd.ExcelFile.parse`, a `_MethodProxy`) unwraps to the slow object it delegates to. - `__delattr__` mirrors a deletion as a deletion. Nothing is restored on delete. The pristine state is a per-type map `name -> (pristine proxy attribute, pristine slow class-dict entry)` snapshotted once, when `make_*_proxy_type` finishes building the type (the same point mirroring is enabled via `_fsproxy_mirror_slow_overrides`). It is a fixed translation table, not runtime patch tracking: there is no stash of "what to put back", and undo works for any code that follows the standard save/patch/re-assign pattern (pytest `monkeypatch`, `unittest.mock.patch.object`, manual saves) because the saved value itself identifies the pristine state. The translation is what makes mirroring safe at all — the values readable off a proxy type live in proxy space, and forwarding e.g. the saved `columns` property or `eval`/`query` functions verbatim onto `pandas.DataFrame` would install cudf machinery on the real class (for `columns` this infinitely recurses on the fallback path). cudf.pandas's own custom methods (`DataFrame.eval`/`query`) are installed via the new `_setattr_fsproxy_no_mirror` helper, which registers them as part of the proxy's pristine state without forwarding them to pandas. ## Tests Adds unit tests in `cudf_pandas_tests/test_fast_slow_proxy.py` covering: set/delete mirroring, monkeypatch round-trips (new attr, existing attr, nested, `delattr`), `mock.patch.object` (which saves the raw descriptor without resolving it), properties, plain data attributes, `staticmethod`/`classmethod` descriptor preservation, methods the slow type only inherits, and the no-mirror helper; plus an end-to-end test in `test_cudf_pandas.py` that patches/unpatches `DataFrame.columns`/`eval` and `Series.str` and checks real pandas is restored and functional. Removes 14 now-passing xfails from the pandas-tests plugin (13× `read_excel` engine-selection tests that monkeypatch the engine, plus a monkeypatch-registered custom accessor). The attribution of these 14 to the proxy fix (vs the Excel-reader fixes remaining in #22927) was verified locally by running each removed xfail with the proxy fix in isolation. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23001 --- python/cudf/cudf/pandas/_wrappers/pandas.py | 7 +- python/cudf/cudf/pandas/fast_slow_proxy.py | 189 ++++++++++ .../pandas/scripts/pandas-testing-plugin.py | 14 - .../cudf_pandas_tests/test_cudf_pandas.py | 32 ++ .../cudf_pandas_tests/test_fast_slow_proxy.py | 339 ++++++++++++++++++ 5 files changed, 565 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index 621163c4be2b..6cba72851da4 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -47,6 +47,7 @@ _FastSlowAttribute, _FunctionProxy, _maybe_wrap_result, + _setattr_fsproxy_no_mirror, _State, _Unusable, is_proxy_object, @@ -1691,8 +1692,10 @@ def _df_query_method(self, *args, local_dict=None, global_dict=None, **kwargs): ) -DataFrame.eval = _df_eval_method -DataFrame.query = _df_query_method +# These custom implementations are installed by cudf.pandas itself and must +# not be mirrored onto (and clobber) the real ``pandas.DataFrame``. +_setattr_fsproxy_no_mirror(DataFrame, "eval", _df_eval_method) +_setattr_fsproxy_no_mirror(DataFrame, "query", _df_query_method) _JsonReader = make_intermediate_proxy_type( "_JsonReader", diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index 20600c7fe9db..60fa07dc37b4 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -362,6 +362,11 @@ def _fsproxy_state(self) -> _State: final_type_map[fast_type] = cls final_type_map[slow_type] = cls + # Proxy type fully constructed: snapshot its pristine state and, from + # here on, mirror class-level attribute writes (genuine runtime + # monkeypatches) onto the underlying "slow" (real) type. + _enable_fsproxy_mirroring(cls) + return cls @@ -511,6 +516,11 @@ def _fsproxy_fast_to_slow(self): intermediate_type_map[fast_type] = cls intermediate_type_map[slow_type] = cls + # Proxy type fully constructed: snapshot its pristine state and, from + # here on, mirror class-level attribute writes (genuine runtime + # monkeypatches) onto the underlying "slow" (real) type. + _enable_fsproxy_mirroring(cls) + return cls @@ -560,6 +570,55 @@ def get_registered_functions(): return dict() +_SLOW_ABSENT = object() + + +def _enable_fsproxy_mirroring(cls: type) -> None: + """Finalize a proxy type for class-level patch mirroring. + + Snapshots the proxy type's pristine public class attributes together + with the slow type's pristine class-dict entries for the same names, + then enables mirroring of class-level attribute writes/deletions onto + the slow type (see ``_FastSlowProxyMeta.__setattr__``/``__delattr__``). + + The snapshot is a fixed translation table, not runtime patch tracking: + re-assigning the proxy's pristine attribute for ``name`` (which is what + ``monkeypatch``/``mock.patch`` save and re-assign on undo) translates + to restoring the slow type's pristine attribute for ``name``. + """ + slow = cls._fsproxy_slow_type # type: ignore[attr-defined] + pristine = { + name: (value, slow.__dict__.get(name, _SLOW_ABSENT)) + for name, value in cls.__dict__.items() + if not name.startswith("_") + } + type.__setattr__(cls, "_fsproxy_pristine_attrs", pristine) + type.__setattr__(cls, "_fsproxy_mirror_slow_overrides", True) + + +def _setattr_fsproxy_no_mirror(cls: type, name: str, value: Any) -> None: + """Install a cudf.pandas-internal attribute on a proxy type. + + ``_FastSlowProxyMeta.__setattr__`` mirrors class-level attribute writes + onto the underlying "slow" (real) type so that runtime monkeypatches stay + visible to the pandas fallback path. cudf.pandas itself installs a handful + of custom methods (e.g. ``DataFrame.query``/``DataFrame.eval``) onto the + proxy classes that must *not* clobber pandas' genuine implementations; use + this helper for those (rare) assignments, after the proxy type has been + fully constructed by ``make_*_proxy_type``. The attribute is registered as + part of the proxy's pristine state so that a later save/patch/re-assign + cycle restores the slow type's own attribute rather than forwarding the + cudf-internal object to it. + """ + type.__setattr__(cls, name, value) + if not name.startswith("_"): + slow = cls._fsproxy_slow_type # type: ignore[attr-defined] + cls._fsproxy_pristine_attrs[name] = ( # type: ignore[attr-defined] + value, + slow.__dict__.get(name, _SLOW_ABSENT), + ) + + class _FastSlowProxyMeta(type): """ Metaclass used to dynamically find class attributes and @@ -578,6 +637,136 @@ def _fsproxy_slow(self) -> type: def _fsproxy_fast(self) -> type: return self._fsproxy_fast_type + def __new__(mcls, *args, **kwargs): + cls = super().__new__(mcls, *args, **kwargs) + # Per-proxy-type switch controlling whether class-level attribute + # writes are mirrored onto the underlying "slow" (real) type (see + # ``__setattr__``/``__delattr__``). It starts disabled so that the + # attributes installed while the proxy type is being built are not + # forwarded to the real type; ``make_*_proxy_type`` enables it once + # construction is complete. Initialized in ``__new__`` rather than + # ``__init__`` because cooperating metaclasses may perform + # class-level attribute writes from their own ``__new__`` — e.g. + # ``ABCMeta.__new__`` assigns ``__abstractmethods__``, dispatching + # to ``__setattr__`` below before ``__init__`` ever runs. + type.__setattr__(cls, "_fsproxy_mirror_slow_overrides", False) + return cls + + def __setattr__(cls, name, value): + # Class-level attribute assignments on a proxy type (e.g. + # ``monkeypatch.setattr(pd.ExcelFile, "parse", fn)``) must also be + # mirrored onto the underlying "slow" (real) type. Code that runs + # under ``disable_module_accelerator()`` (e.g. the pandas fallback + # path of ``pd.read_excel``) resolves attributes from the real + # class, not the proxy, so a patch applied only to the proxy would + # otherwise be invisible to that code. The assigned value is first + # translated into its slow-space equivalent: re-assigning the + # proxy's pristine attribute translates to the slow type's pristine + # attribute, and proxy machinery is unwrapped to the slow object it + # delegates to, so save/patch/re-assign cycles round-trip on the + # real type as well. + type.__setattr__(cls, name, value) + if not cls._fsproxy_mirror_slow_overrides: + # The proxy type is still being constructed (or this is a + # non-pandas proxy): only mirror user/runtime monkeypatches, + # never the custom methods cudf.pandas installs on the proxy + # classes itself. + return + if name.startswith("_"): + return + slow = cls._fsproxy_slow_type + try: + # Mirroring is best-effort: translating a wrapped proxy instance + # can require a fast-to-slow conversion, which may itself fail; + # never let that escape an otherwise-successful assignment. + pristine = cls._fsproxy_pristine_attrs + entry = pristine.get(name) + if entry is not None and value is entry[0]: + # The proxy's pristine attribute for ``name`` is being + # re-assigned (e.g. ``monkeypatch``/``mock.patch`` undo + # re-setting the saved class-dict entry). Its slow-space + # equivalent is the slow type's pristine attribute. + if entry[1] is _SLOW_ABSENT: + # The slow type never defined ``name`` itself: the proxy + # mirrors the slow type's *dir*, so it has pristine + # attributes for methods the slow type only inherits + # (e.g. ``DataFrame.head`` lives on ``NDFrame``). + # Mirroring a patch for such a name added a shadowing + # entry to the slow type's dict; undoing the patch must + # remove that entry again so the inherited + # implementation becomes visible. It may legitimately be + # missing (the mirror is best-effort), hence the guard. + if name in slow.__dict__: + delattr(slow, name) + else: + setattr(slow, name, entry[1]) + return + # Otherwise translate the assigned value into "slow" space + # before mirroring it: a value read off a proxy type (e.g. the + # original that a caller saves before patching and re-assigns + # to undo) is proxy machinery wrapping a slow-side object, and + # mirroring it verbatim would install that machinery on the + # real type. Unwrap it to the slow object it delegates to, so + # save/patch/re-assign cycles round-trip on the real type; + # values with no determinable slow-side equivalent are not + # mirrored at all. + if isinstance(value, _FastSlowAttribute): + # The proxy's own delegating descriptor (a *pristine* one is + # already handled by identity above; this covers a + # descriptor obtained some other way): its slow equivalent + # is the method it wraps, if it ever resolved one. + attr = value._attr + if not isinstance(attr, _MethodProxy): + return + value = attr + if isinstance(value, _FunctionProxy): + unwrapped = value._fsproxy_slow + if entry is not None and entry[1] is not _SLOW_ABSENT: + # Re-assigning a saved ``cls.method`` (a ``_MethodProxy`` + # over the *resolved* slow attribute): if it resolves + # back to the slow type's pristine attribute, restore + # the pristine class-dict entry itself so + # ``classmethod``/``staticmethod`` descriptors are not + # degraded to their bound/plain-function forms. + descriptor = entry[1] + try: + resolved = ( + descriptor.__get__(None, slow) + if hasattr(type(descriptor), "__get__") + else descriptor + ) + if unwrapped is resolved or unwrapped == resolved: + setattr(slow, name, descriptor) + return + except Exception: + pass + setattr(slow, name, unwrapped) + elif isinstance(value, _FastSlowProxy): + setattr(slow, name, value._fsproxy_slow) + elif isinstance(value, _FastSlowProxyMeta): + slow_type = getattr(value, "_fsproxy_slow_type", None) + if slow_type is not None: + setattr(slow, name, slow_type) + else: + setattr(slow, name, value) + except Exception: + pass + + def __delattr__(cls, name): + # Mirror class-level attribute *deletions* onto the underlying "slow" + # (real) type as well, for the same reason as ``__setattr__``: after + # ``del cls.name`` the attribute is gone from the proxy, so it must + # also be gone from the real type seen by fallback code. + type.__delattr__(cls, name) + if not cls._fsproxy_mirror_slow_overrides: + return + if name.startswith("_"): + return + try: + delattr(cls._fsproxy_slow_type, name) + except (AttributeError, TypeError): + pass + def __dir__(self): # Try to return the cached dir of the slow object, but if it # doesn't exist, fall back to the default implementation. diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 5f3155adb71b..4f749565dfc8 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -2539,19 +2539,6 @@ def pytest_unconfigure(config): "tests/io/excel/test_odswriter.py::test_cell_value_type[test string-string-string-value-test string]": "TODO: Add a reason for failure", "tests/io/excel/test_odswriter.py::test_cell_value_type[value4-date-date-value-2010-10-10T10:10:10]": "TODO: Add a reason for failure", "tests/io/excel/test_odswriter.py::test_cell_value_type[value5-date-date-value-2010-10-10]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.ods')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsb')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsx')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('odf', '.ods')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('openpyxl', '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('openpyxl', '.xlsx')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('pyxlsb', '.xlsb')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('xlrd', '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xlsx')]": "TODO: Add a reason for failure", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[False]": "AttributeError: _compute. Did you mean: 'compare'?", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[True]": "AttributeError: _compute. Did you mean: 'compare'?", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[columns]": "AttributeError: _compute. Did you mean: 'compare'?", @@ -3685,7 +3672,6 @@ def pytest_unconfigure(config): "tests/test_algos.py::TestValueCounts::test_value_counts_dropna": "pandas keeps bool-with-None data as object dtype; cudf stores it as a masked bool column", "tests/test_algos.py::TestValueCounts::test_value_counts_stability": "asserts that kind='quicksort' produces an unstable order; cudf sorts are always stable", "tests/test_col.py::test_cached_property": "AssertionError: assert False", - "tests/test_col.py::test_custom_accessor": "AttributeError: 'Series' object has no attribute 'xyz'", "tests/test_common.py::test_serializable[obj0]": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[False]": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[True]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index cbf3e17bfdc4..792e4f2cf314 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -2186,6 +2186,38 @@ def test_module_proxy_write_through_config(monkeypatch): cf.register_option("foo", 1) +def test_class_monkeypatch_roundtrip_restores_real_pandas(monkeypatch): + # Class-level patches on proxy types are mirrored onto the real pandas + # type (so they stay visible to fallback code running under + # ``disable_module_accelerator``); undoing them must restore the real + # type's own attributes — including for attributes that cudf.pandas + # replaces on the proxy, like ``columns``/``eval``/``str``. + real_df = xpd.DataFrame._fsproxy_slow + real_series = xpd.Series._fsproxy_slow + orig_columns = real_df.__dict__["columns"] + orig_eval = real_df.__dict__["eval"] + orig_str = real_series.__dict__["str"] + + def fake_eval(self, *args, **kwargs): + return "patched" + + monkeypatch.setattr(xpd.DataFrame, "eval", fake_eval) + assert real_df.__dict__["eval"] is fake_eval + monkeypatch.setattr( + xpd.DataFrame, "columns", property(lambda self: "patched") + ) + monkeypatch.setattr(xpd.Series, "str", property(lambda self: "patched")) + monkeypatch.undo() + + assert real_df.__dict__["columns"] is orig_columns + assert real_df.__dict__["eval"] is orig_eval + assert real_series.__dict__["str"] is orig_str + # The real type must remain fully functional on the fallback path. + df = real_df({"a": [1, 2]}) + assert list(df.columns) == ["a"] + assert list(df.eval("b = a + 1").columns) == ["a", "b"] + + @pytest.mark.parametrize("box", ["Series", "array"]) @pytest.mark.parametrize("na_value", [pd.NA, np.nan], ids=["NA", "NaN"]) @pytest.mark.parametrize("storage", ["python", "pyarrow"]) diff --git a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py index 30d2124edcbf..5e22badc4c0e 100644 --- a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py +++ b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py @@ -12,7 +12,9 @@ import cudf.pandas.fast_slow_proxy from cudf.pandas.fast_slow_proxy import ( _fast_arg, + _FastSlowAttribute, _FunctionProxy, + _setattr_fsproxy_no_mirror, _slow_arg, _transform_arg, _Unusable, @@ -747,3 +749,340 @@ def test_tuple_with_attrs_transform(): assert b is bprime assert c == cprime and c is not cprime assert d == dprime and d is not dprime + + +def _make_mirror_proxy(): + class Fast: + pass + + class SlowBase: + def inherited(self): + return "base" + + class Slow(SlowBase): + const = 42 + + def existing(self): + return "slow original" + + @property + def prop(self): + return "slow prop" + + @staticmethod + def smethod(x): + return x + 1 + + @classmethod + def cmethod(cls): + return cls.__name__ + + Pxy = make_final_proxy_type( + "Pxy", + Fast, + Slow, + fast_to_slow=lambda fast: Slow(), + slow_to_fast=lambda slow: Fast(), + ) + return Fast, Slow, Pxy + + +def test_class_attr_mirroring_enabled_after_construction(): + # ``make_*_proxy_type`` enables per-type mirroring once the proxy type is + # fully built (it starts disabled so the methods installed during + # construction are not forwarded to the real type). + _, _, Pxy = _make_mirror_proxy() + assert Pxy.__dict__["_fsproxy_mirror_slow_overrides"] is True + + +def test_class_attr_setattr_mirrored_to_slow(): + # A class-level attribute write on the proxy is mirrored onto the + # underlying "slow" (real) type so it is visible to fallback code. + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.new_method = patched + assert Slow.__dict__.get("new_method") is patched + + +def test_class_attr_delattr_mirrored_to_slow(): + # Deleting a class-level attribute on the proxy mirrors the deletion + # onto the slow type (the ``__delattr__`` path). + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.new_method = patched + assert "new_method" in Slow.__dict__ + + del Pxy.new_method + assert "new_method" not in Pxy.__dict__ + assert "new_method" not in Slow.__dict__ + + +def test_class_attr_delattr_existing_removes_from_slow(): + # Deleting is deleting, not restoring: patching an existing attribute + # and then deleting it removes it from both the proxy and the slow type, + # exactly as the same sequence would on a plain Python class. + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + del Pxy.existing + assert "existing" not in Pxy.__dict__ + assert "existing" not in Slow.__dict__ + + +def test_class_attr_restore_existing_slow_attr(): + # Re-assigning the proxy's pristine class-dict entry (as ``monkeypatch`` + # and ``mock.patch`` teardown do) restores the slow type's pristine + # attribute. Note: no prior class-level getattr — the saved descriptor + # is unresolved, as in the ``mock.patch.object`` flow. + _, Slow, Pxy = _make_mirror_proxy() + saved = Pxy.__dict__["existing"] + assert isinstance(saved, _FastSlowAttribute) + original_slow = Slow.__dict__["existing"] + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + Pxy.existing = saved + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_restore_via_saved_method_proxy(): + # Re-assigning a saved ``Pxy.method`` (a ``_MethodProxy`` over the + # resolved slow attribute, not the class-dict descriptor) also restores + # the slow type's pristine attribute. + _, Slow, Pxy = _make_mirror_proxy() + saved = getattr(Pxy, "existing") + original_slow = Slow.__dict__["existing"] + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + Pxy.existing = saved + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_monkeypatch_roundtrip(monkeypatch): + # End-to-end: ``monkeypatch`` of a brand-new attribute mirrors onto the + # slow type, and teardown (which deletes it) mirrors the deletion. + _, Slow, Pxy = _make_mirror_proxy() + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "brand_new", fake, raising=False) + assert Slow.__dict__.get("brand_new") is fake + + monkeypatch.undo() + assert "brand_new" not in Pxy.__dict__ + assert "brand_new" not in Slow.__dict__ + + +def test_class_attr_monkeypatch_existing_roundtrip(monkeypatch): + # End-to-end: ``monkeypatch`` of a pre-existing method mirrors the patch + # onto the slow type, and teardown (which re-assigns the saved proxy + # descriptor) restores the slow type's original implementation. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "existing", fake) + assert Slow.__dict__["existing"] is fake + + monkeypatch.undo() + assert isinstance(Pxy.__dict__["existing"], _FastSlowAttribute) + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_nested_monkeypatch_existing_roundtrip(): + # Nested patches of the same pre-existing method unwind in order, + # each level restoring the slow type to the previous state. + from _pytest.monkeypatch import MonkeyPatch + + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake1(self): + return "fake1" + + def fake2(self): + return "fake2" + + mp1, mp2 = MonkeyPatch(), MonkeyPatch() + mp1.setattr(Pxy, "existing", fake1) + assert Slow.__dict__["existing"] is fake1 + mp2.setattr(Pxy, "existing", fake2) + assert Slow.__dict__["existing"] is fake2 + + mp2.undo() + assert Slow.__dict__["existing"] is fake1 + mp1.undo() + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_mock_patch_object_roundtrip(): + # ``unittest.mock.patch.object`` saves the raw class-dict entry without + # a prior getattr (so the saved descriptor is never resolved); undo must + # still restore the slow type's pristine attribute. + from unittest import mock + + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake(self): + return "fake" + + with mock.patch.object(Pxy, "existing", fake): + assert Slow.__dict__["existing"] is fake + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_property_monkeypatch_roundtrip(monkeypatch): + # Patching a property mirrors it onto the slow type; undo restores the + # slow type's pristine property object. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["prop"] + + fake = property(lambda self: "fake") + monkeypatch.setattr(Pxy, "prop", fake) + assert Slow.__dict__["prop"] is fake + assert Slow().prop == "fake" + + monkeypatch.undo() + assert Slow.__dict__["prop"] is original_slow + assert Slow().prop == "slow prop" + + +def test_class_attr_data_attr_monkeypatch_roundtrip(monkeypatch): + # Patching a plain class data attribute round-trips on the slow type. + _, Slow, Pxy = _make_mirror_proxy() + + monkeypatch.setattr(Pxy, "const", 99) + assert Slow.const == 99 + + monkeypatch.undo() + assert Slow.const == 42 + + +def test_class_attr_staticmethod_monkeypatch_roundtrip(monkeypatch): + # Undo restores the slow type's pristine ``staticmethod`` descriptor, + # not the plain function it resolves to (which would break instance + # calls by receiving ``self``). + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["smethod"] + assert isinstance(original_slow, staticmethod) + + monkeypatch.setattr(Pxy, "smethod", staticmethod(lambda x: x - 1)) + assert Slow.smethod(1) == 0 + + monkeypatch.undo() + assert Slow.__dict__["smethod"] is original_slow + assert Slow().smethod(1) == 2 + + +def test_class_attr_classmethod_monkeypatch_roundtrip(monkeypatch): + # Undo restores the slow type's pristine ``classmethod`` descriptor, + # not the class-bound method it resolves to (which would pin ``cls`` + # for subclasses). + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["cmethod"] + assert isinstance(original_slow, classmethod) + + monkeypatch.setattr(Pxy, "cmethod", classmethod(lambda cls: "fake")) + assert Slow.cmethod() == "fake" + + monkeypatch.undo() + assert Slow.__dict__["cmethod"] is original_slow + assert Slow.cmethod() == "Slow" + + +def test_class_attr_inherited_method_monkeypatch_roundtrip(monkeypatch): + # Patching a method the slow type only inherits mirrors it into the slow + # type's own dict; undo removes that entry again (rather than copying + # the base-class implementation into the subclass), leaving the + # inherited implementation visible. + _, Slow, Pxy = _make_mirror_proxy() + assert "inherited" not in Slow.__dict__ + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "inherited", fake) + assert Slow.__dict__["inherited"] is fake + + monkeypatch.undo() + assert "inherited" not in Slow.__dict__ + assert Slow().inherited() == "base" + + +def test_class_attr_monkeypatch_delattr_roundtrip(monkeypatch): + # ``monkeypatch.delattr`` mirrors the deletion onto the slow type, and + # undo (which re-assigns the saved pristine descriptor) restores the + # slow type's pristine attribute. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + monkeypatch.delattr(Pxy, "existing") + assert "existing" not in Pxy.__dict__ + assert "existing" not in Slow.__dict__ + + monkeypatch.undo() + assert Slow.__dict__["existing"] is original_slow + + +def test_setattr_fsproxy_no_mirror_skips_slow(): + # ``_setattr_fsproxy_no_mirror`` sets a class attribute on the proxy + # without forwarding it to the slow type (used for cudf.pandas' own custom + # methods such as ``DataFrame.query``/``eval``). + _, Slow, Pxy = _make_mirror_proxy() + + def custom(self): + return "custom" + + _setattr_fsproxy_no_mirror(Pxy, "custom_method", custom) + assert Pxy.__dict__["custom_method"] is custom + assert "custom_method" not in Slow.__dict__ + + +def test_setattr_fsproxy_no_mirror_monkeypatch_roundtrip(monkeypatch): + # A cudf-installed custom attribute (e.g. ``DataFrame.eval``/``query``) + # participates in the pristine state: monkeypatching it and undoing + # restores the proxy's custom object AND the slow type's own genuine + # implementation — the cudf-internal object is never forwarded to the + # slow type. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def custom(self): + return "cudf custom" + + _setattr_fsproxy_no_mirror(Pxy, "existing", custom) + assert Slow.__dict__["existing"] is original_slow + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "existing", fake) + assert Slow.__dict__["existing"] is fake + + monkeypatch.undo() + assert Pxy.__dict__["existing"] is custom + assert Slow.__dict__["existing"] is original_slow From 56f55035daa3025a4636091e866081c88a472822 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 21 Jul 2026 14:26:09 -0500 Subject: [PATCH 5/6] Empty commit to trigger a build (#23376) This is an empty commit to trigger a build. This is needed after the RMM ABI break in https://github.com/rapidsai/rmm/pull/2462. From 2e3d7545a3679bc890a2c539a2a45ec6abdb5c5f Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:31:39 -0400 Subject: [PATCH 6/6] Remove rmm::device_buffer forward declaration from types.hpp (#23373) Fixes build error introduced by an RMM change. The forward declaration in types.hpp is not actually needed and removing it fixes the build errors. Authors: - David Wendt (https://github.com/davidwendt) - James Lamb (https://github.com/jameslamb) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23373 --- cpp/include/cudf/types.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index c69fc8111750..9c631dd31654 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -38,14 +38,6 @@ * @brief Type declarations for libcudf. */ -// Forward declarations -/// @cond -namespace rmm { -class device_buffer; -/// @endcond - -} // namespace rmm - namespace CUDF_EXPORT cudf { // Forward declaration class column;