diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 97c3add68349..1e166c0a97ac 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -471,3 +471,73 @@ jobs: with: push: true cuda: '["12.9", "13.3"]' + # Build the cuDF Java JAR for every Maven classifier. + java-build: + needs: [telemetry-setup] + strategy: + fail-fast: false + matrix: + include: + - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } + runs-on: ${{ matrix.runner }} + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Build static libcudf + run: | + ./java/ci/build_static_libcudf.sh \ + --output-dir "${RUNNER_TEMP}/libcudf" \ + --cuda-version "${{ matrix.cuda }}" + - name: Build cuDF Java JAR + run: | + ./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir "${RUNNER_TEMP}/libcudf" \ + --output-dir "${RUNNER_TEMP}/jars" \ + --cuda-version "${{ matrix.cuda }}" + - name: Upload per-entry JAR artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} + # Ship only the JARs and POMs. Exclude the per-classifier Maven build scratch dir. + path: | + ${{ runner.temp }}/jars + !${{ runner.temp }}/jars/.mvn-temp-target + if-no-files-found: error + # Assemble the per-classifier JARs into one Maven-repository layout. + java-gather: + needs: [java-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Download per-entry JAR artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cudf_java_cuda* + path: ${{ runner.temp }}/jars + merge-multiple: true + - name: Assemble Maven repository layout + run: | + ./java/ci/assemble_maven_repo.sh \ + --jars-dir "${RUNNER_TEMP}/jars" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload combined Maven repository artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index 5302942db173..3778e1a9035a 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -4,6 +4,8 @@ set -euo pipefail +TIMEOUT_TOOL_PATH="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/timeout_with_stack.py + # Support invoking run_cudf_polars_pytests.sh outside the script directory # Assumption, polars has been cloned in the root of the repo. cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../polars/ @@ -60,16 +62,18 @@ DESELECTED_TESTS_STR=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") # Don't quote the `DESELECTED_...` variable because `pytest` can't handle # multiple quoted arguments inline # shellcheck disable=SC2086 +# Fail fast (-x) because failed tests pollute the state echo "Run polars tests with injected in-memory GPU engine" -python -m pytest \ +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest \ --import-mode=importlib \ --cache-clear \ + -x \ -m "" \ -p cudf_polars.testing.inject_gpu_engine \ -n 4 \ --dist=worksteal \ --tb=native \ - --timeout=240 \ --durations 10 --durations-min 10 \ -ra \ $DESELECTED_TESTS_STR \ @@ -81,9 +85,11 @@ python -m pytest \ echo "Run polars tests with injected SPMD GPU engine, small blocksize" CUDF_POLARS__EXECUTOR__TARGET_PARTITION_SIZE=805306368 \ CUDF_POLARS__EXECUTOR__FALLBACK_MODE=silent \ - python -m pytest \ +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest \ --import-mode=importlib \ --cache-clear \ + -x \ -v \ -m "" \ -p cudf_polars.testing.inject_gpu_engine \ @@ -91,7 +97,6 @@ CUDF_POLARS__EXECUTOR__FALLBACK_MODE=silent \ -n 4 \ --dist=worksteal \ --tb=native \ - --timeout=240 \ --durations 10 --durations-min 10 \ -ra \ $DESELECTED_TESTS_STR \ diff --git a/ci/run_cudf_polars_pytests.sh b/ci/run_cudf_polars_pytests.sh index 82d1ccd4879f..3fac6910c5a8 100755 --- a/ci/run_cudf_polars_pytests.sh +++ b/ci/run_cudf_polars_pytests.sh @@ -1,11 +1,14 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail # It is essential to cd into python/cudf_polars as `pytest-xdist` + `coverage` seem to work only at this directory level. # Support invoking run_cudf_polars_pytests.sh outside the script directory +TIMEOUT_TOOL_PATH="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/timeout_with_stack.py + cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_polars/ -python -m pytest --cache-clear "$@" tests +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest --cache-clear "$@" tests diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 09315bbe981c..d9df9c05d4eb 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -89,12 +89,14 @@ for version in "${VERSIONS[@]}"; do COVERAGE_ARGS=(--no-cov) fi + # Fail fast (-x) rather than trying to continue because failed tests pollute the state ./ci/run_cudf_polars_pytests.sh \ -vv \ "${COVERAGE_ARGS[@]}" \ --numprocesses=4 \ --dist=worksteal \ --durations 10 --durations-min 10 \ + -x \ -ra \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-${version}.xml" @@ -106,6 +108,7 @@ for version in "${VERSIONS[@]}"; do EXITCODE=1 FAILED+=("${version}") rapids-logger "Tests failed for polars ${version}.*" + break else PASSED+=("${version}") rapids-logger "Tests passed for polars ${version}.*" diff --git a/ci/timeout_with_stack.py b/ci/timeout_with_stack.py new file mode 100644 index 000000000000..02f62408fe9e --- /dev/null +++ b/ci/timeout_with_stack.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Module for running commands with timeout and capturing stack traces. + +This module provides functionality to run commands with a timeout, capture stack traces +of processes that exceed the timeout, and properly terminate process trees. + +See Also +-------- +subprocess.Popen : For running subprocesses without timeout. +psutil.Process : For process management and information. + +Examples +-------- +>>> from timeout_with_stack import run_with_timeout +>>> exit_code = run_with_timeout(["sleep", "10"], timeout=5) +>>> print(f"Process exited with code: {exit_code}") +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import signal +import subprocess +import sys +import time +from contextlib import suppress +from enum import IntEnum +from typing import TYPE_CHECKING + +import psutil + +if TYPE_CHECKING: + from collections.abc import Sequence + from types import FrameType + + +class StackType(IntEnum): + """Enum representing the type of stack trace to capture.""" + + C = 0 + Python = 1 + + +def get_child_pids(pid: int) -> list[int]: + """ + Get all child PIDs of a given process. + + This function retrieves all child process IDs (PIDs) of a given process, + including recursively nested child processes. + + Parameters + ---------- + pid + The process ID of the parent process. + + Returns + ------- + A list of child process IDs. Returns an empty list if the parent process + does not exist. + + See Also + -------- + psutil.Process.children : For getting child processes. + + Examples + -------- + >>> from timeout_with_stack import get_child_pids + >>> child_pids = get_child_pids(1234) + >>> print(f"Child PIDs: {child_pids}") + """ + try: + parent = psutil.Process(pid) + children = parent.children(recursive=True) + return [p.pid for p in children] + except psutil.NoSuchProcess: + return [] + + +def capture_stack_trace(pid: int, stack_type=StackType.C) -> None: + """ + Capture stack trace for a given process. + + This function captures the stack trace of a process using GDB. It prints the + stack trace to stdout, which can be useful for debugging hanging or long-running + processes. + + Parameters + ---------- + pid + The process ID of the process to capture stack trace for. + stack_type + The stack type to extract, either C or Python. + + See Also + -------- + capture_all_stacks : For capturing stack traces of a process and its children. + + Examples + -------- + >>> from timeout_with_stack import capture_stack_trace + >>> capture_stack_trace(1234) + """ + if stack_type is StackType.C: + bt_command = "thread apply all bt" + print(f"\nCapturing C stack trace for process {pid}:") + else: + bt_command = "thread apply all py-bt" + print(f"\nCapturing Python stack trace for process {pid}:") + gdb = shutil.which("gdb") + if gdb is None: + print(f"Skipping stack trace for process {pid}: gdb not found") + return + + try: + proc = subprocess.run( + [ + gdb, + "--quiet", + "--pid", + str(pid), + "-ex", + "set pagination off", + "-ex", + "set confirm off", + "-ex", + bt_command, + "-ex", + "quit", + ], + capture_output=True, + text=True, + check=False, + timeout=120, + ) + except subprocess.TimeoutExpired: + print(f"Timed out capturing stack trace for process {pid}") + return + + print(proc.stdout) + if proc.stderr: + print(proc.stderr, file=sys.stderr) + + +def capture_all_stacks(pid: int, *, enable_python: bool = False) -> None: + """ + Capture stack traces for parent and all child processes. + + This function captures stack traces for both the parent process and all its + child processes. It first captures the parent's stack trace, then recursively + captures stack traces for all child processes. + + Parameters + ---------- + pid + The process ID of the parent process. + enable_python + Whether to capture Python stack traces. + + See Also + -------- + capture_stack_trace : For capturing stack trace of a single process. + get_child_pids : For getting child process IDs. + + Examples + -------- + >>> from timeout_with_stack import capture_all_stacks + >>> capture_all_stacks(1234, enable_python=True) + """ + # Capture parent process stack + if enable_python: + capture_stack_trace(pid, stack_type=StackType.Python) + capture_stack_trace(pid, stack_type=StackType.C) + + # Get and capture all child processes + child_pids = get_child_pids(pid) + for child_pid in child_pids: + if enable_python: + capture_stack_trace(child_pid, stack_type=StackType.Python) + capture_stack_trace(child_pid, stack_type=StackType.C) + + +def terminate_process_tree(pid: int) -> None: + """ + Terminate a process and all its children. + + This function terminates a process and all its child processes. It first + attempts to gracefully terminate all processes, then forcefully kills any + remaining processes after a timeout. + + Parameters + ---------- + pid + The process ID of the parent process to terminate. + + See Also + -------- + psutil.Process.terminate : For gracefully terminating a process. + psutil.Process.kill : For forcefully killing a process. + + Examples + -------- + >>> from timeout_with_stack import terminate_process_tree + >>> terminate_process_tree(1234) + """ + try: + parent = psutil.Process(pid) + children = parent.children(recursive=True) + + # Terminate children first + for child in children: + with suppress(psutil.NoSuchProcess): + child.terminate() + + # Create a copy of children list + terminated_children = list(children) + + # Wait for all children to terminate + for child in terminated_children: + with suppress(psutil.TimeoutExpired): + child.wait(timeout=3) + + # Kill any remaining children + for child in terminated_children: + with suppress(psutil.NoSuchProcess): + child.kill() + + # Terminate parent + parent.terminate() + try: + parent.wait(timeout=3) + except psutil.TimeoutExpired: + parent.kill() + + except psutil.NoSuchProcess: + pass + + +def install_signal_handler(pid: int, *signals: signal.Signals) -> None: + """ + Install signal handler that terminates the given pid on receiving any of signals + """ + + def handler(signum: int, frame: FrameType | None) -> None: + print(f"Received {signum=}, terminating process tree") + terminate_process_tree(pid) + sys.exit(signum) + + for sig in signals: + signal.signal(sig, handler) + + +def run_with_timeout( + cmd: Sequence[str], timeout: float, *, enable_python: bool = False +) -> int: + """ + Run a command with a timeout and capture stack traces if it exceeds the timeout. + + This function runs a command with a specified timeout. If the command exceeds + the timeout, it captures stack traces of the process and its children before + terminating them. It handles keyboard interrupts gracefully. + + Parameters + ---------- + cmd + The command and its arguments to run. + timeout + Maximum time in seconds to allow the command to run. + enable_python + Whether to capture Python stack traces. + + Returns + ------- + Return code of the command, or 124 if timeout occurred, or signal.SIGINT + if interrupted by keyboard. + + See Also + -------- + subprocess.Popen : For running subprocesses without timeout. + capture_all_stacks : For capturing stack traces of processes. + + Examples + -------- + >>> from timeout_with_stack import run_with_timeout + >>> exit_code = run_with_timeout(["sleep", "10"], timeout=5, enable_python=True) + >>> print(f"Process exited with code: {exit_code}") + """ + # Start the process with a new process group + # Note: preexec_fn is used here as we need to create a new process group + # for proper termination of child processes + process = subprocess.Popen( + cmd, + preexec_fn=os.setsid, + ) + install_signal_handler( + process.pid, + signal.SIGTERM, + signal.SIGABRT, + signal.SIGHUP, + signal.SIGQUIT, + ) + start_time = time.time() + + try: + while time.time() - start_time < timeout: + if process.poll() is not None: + return process.returncode + time.sleep(0.1) + + print(f"\nProcess timed out after {timeout} seconds") + print("Capturing stack traces for all processes...") + + # Capture stacks for parent and all children + capture_all_stacks(process.pid, enable_python=enable_python) + + # Terminate the entire process tree + print("\nTerminating process tree...") + terminate_process_tree(process.pid) + except KeyboardInterrupt: + print("\nReceived keyboard interrupt") + terminate_process_tree(process.pid) + return signal.SIGINT + else: + return 124 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run a command with timeout and capture stack traces" + ) + parser.add_argument("timeout", type=float, help="Timeout in seconds") + parser.add_argument( + "--enable-python", + action="store_true", + help="Enable Python stack trace capture", + ) + parser.add_argument( + "command", nargs=argparse.REMAINDER, help="Command to run" + ) + + args = parser.parse_args() + + if not args.command: + parser.error("No command specified") + + exit_code = run_with_timeout( + args.command, args.timeout, enable_python=args.enable_python + ) + sys.exit(exit_code) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index ff472eb88459..39689c166210 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-aarch64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index faf8536eb029..fed3fe1734ec 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-133_arch-aarch64.yaml b/conda/environments/all_cuda-133_arch-aarch64.yaml index 808dffee68be..01f99d1cc46a 100644 --- a/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-aarch64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-133_arch-x86_64.yaml b/conda/environments/all_cuda-133_arch-x86_64.yaml index 66c228ac07bb..8d132dcba708 100644 --- a/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index c69fc8111750..9c631dd31654 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -38,14 +38,6 @@ * @brief Type declarations for libcudf. */ -// Forward declarations -/// @cond -namespace rmm { -class device_buffer; -/// @endcond - -} // namespace rmm - namespace CUDF_EXPORT cudf { // Forward declaration class column; diff --git a/dependencies.yaml b/dependencies.yaml index 573da38533a9..5eb907f5649a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -129,6 +129,19 @@ files: - depends_on_cudf_polars - depends_on_ray - depends_on_cudf_streaming + build_java: + # Toolchain for building static libcudf from source and packaging the cuDF + # Java JAR. `depends_on_libcudf` is excluded because the Java build links + # against a static libcudf built from source, not a conda shared libcudf. + output: none + includes: + - build_base + - build_all + - build_cpp + - cuda + - cuda_static + - cuda_version + - build_java test_java: output: none includes: @@ -632,6 +645,17 @@ dependencies: - croaring==4.4.2 - flatbuffers==24.3.25 - librdkafka<2.15.0a0 + build_java: + common: + - output_types: conda + packages: + - boost + # cuda_profiler_api.h is used by the JNI layer (CudaJni.cpp) but is + # not pulled in by the base `cuda` dev packages. + - cuda-profiler-api + - make + - maven + - openjdk=8.* depends_on_libnvcomp: common: - output_types: conda @@ -1172,13 +1196,18 @@ dependencies: packages: - rich - pytest-httpserver - - pytest-timeout - zstandard + # Used by ci/timeout_with_stack.py utility + - psutil # The polars test suite constructs pandas objects for interop and # dask-cuda pulls pandas in transitively (unbounded), so constrain it # here to exclude the 3.0.4 release that segfaults on pd.Timedelta # (https://github.com/pandas-dev/pandas/issues/66086). - *pandas + - output_types: conda + packages: + # Used by timeout_with_stack.py utility + - gdb test_python_narwhals: common: - output_types: [conda, requirements, pyproject] diff --git a/java/ci/README.md b/java/ci/README.md index 3f3060ef5b45..af8ed4b3be4b 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -1,11 +1,100 @@ # Build Jar artifact of cuDF -## Build the docker image +## Recommended: self-contained release build scripts -### Prerequisite +The scripts under `java/ci/` build the cuDF Java JAR for every Maven classifier the +same way locally and in CI (GitHub Actions is only a thin wrapper that adds +artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image, +runs the build in a throwaway container, and writes its output to a host +directory. No local `docker build` is required, and no GPU is required to build. -1. Docker should be installed. -2. [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) should be installed. +### Prerequisites + +1. Docker is installed and the current user can run `docker`. +2. Network access to pull `rapidsai/ci-conda:-latest`. + +### Local one-command shortcut + +For local testing only, `java/ci/test_java_build_local.sh` runs Steps 1-3 end-to-end for both CUDA 12 and CUDA 13 on the host architecture. + +```bash +./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test +``` + +### Step 1 - Build the static libcudf install tree + +```bash +./java/ci/build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version 12.9 +``` + +This produces a static libcudf install tree (`lib/libcudf.a` plus its static +dependencies) under the given output directory. Build outputs are host-user-owned +so plain `rm -rf` works. + +### Step 2 - Package the cuDF Java JAR for one classifier + +```bash +./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir /tmp/libcudf-cuda12 \ + --output-dir /tmp/jars \ + --cuda-version 12.9 +``` + +This compiles the JNI layer against the static libcudf from Step 1 and emits a +single classifier JAR (e.g. `cudf-26.08.0-SNAPSHOT-cuda12.jar`) plus its POM +into a classifier-named subdirectory under `--output-dir`: + +``` +/tmp/jars/cuda12/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The classifier is derived from `--cuda-version` (major) + host arch (`uname +-m`): `cuda12` / `cuda13` on `x86_64`, `cuda12-arm64` / `cuda13-arm64` on +`aarch64`. Producing the ARM classifiers requires a real `aarch64` host. +Repeat Step 2 for each classifier, pointing `--libcudf-dir` at the matching +static libcudf tree and using the same `--output-dir` (each classifier lands +in its own subdirectory). Concurrent invocations for different classifiers +are safe because each nests its own bind-mount over `/repo/java/target` +inside the container. + +### Step 3 - Assemble the Maven repository layout + +```bash +./java/ci/assemble_maven_repo.sh \ + --jars-dir /tmp/jars \ + --output-dir /tmp/maven-repo +``` + +This walks every subdirectory of `--jars-dir` (each subdir name IS the +classifier), gathers the per-classifier JAR and shared POM, derives the +artifact version from the JAR filenames (requiring a single unique version +across subdirs), and lays them out as: + +``` +/tmp/maven-repo/ai/rapids/cudf/26.08.0-SNAPSHOT/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT-cuda13.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The set of classifiers is whatever subdirectories are present under +`--jars-dir`. For a local `x86_64`-only run, populate `/tmp/jars/cuda12/` +and `/tmp/jars/cuda13/`. For the full four-way release build, add +`/tmp/jars/cuda12-arm64/` and `/tmp/jars/cuda13-arm64/`. + +In GitHub Actions (`.github/workflows/build.yaml`), the `java-build` matrix job +runs Steps 1-2 per (CUDA x arch) entry and uploads each classifier subdir as a +per-entry artifact. The separate `java-gather` job downloads them (with +`merge-multiple: true`, so all subdirs land in a single parent dir), runs +Step 3, and uploads the combined `cudf_java_maven_repo` artifact. + +## Legacy: manual Dockerfile.rocky build (obsolete) + +> The `java/ci/Dockerfile.rocky` + `java/ci/build-in-docker.sh` flow below is the +> old build path. It is retained for reference but superseded by the +> self-contained scripts above. ### Build the docker image @@ -20,25 +109,19 @@ The following CUDA versions are supported w/ CUDA Enhanced Compatibility: Change the --build-arg CUDA_VERSION to what you need. You can replace the tag "cudf-build:12.9.1-devel-rocky8" with another name you like. -## Start the docker then build - -### Start the docker +### Start the docker then build Run below command to start a docker container with GPU. ```bash nvidia-docker run -it cudf-build:12.9.1-devel-rocky8 bash ``` -### Download the cuDF source code - You can download the cuDF repo in the docker container or you can mount it into the container. Here I choose to download again in the container. ```bash git clone --recursive https://github.com/rapidsai/cudf.git -b main ``` -### Build cuDF jar with devtoolset - ```bash cd cudf export WORKSPACE=`pwd` diff --git a/java/ci/argparse.sh b/java/ci/argparse.sh new file mode 100644 index 000000000000..9a06d35c9df8 --- /dev/null +++ b/java/ci/argparse.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared argparse helpers for the java/ci/ host orchestrator scripts. +# Meant to be sourced, not executed: +# . "${SCRIPT_DIR}/argparse.sh" + +# require_value +# Check: exit 1 when is empty (i.e. the flag was passed +# without its argument, or was the last token on the command line). +require_value() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} requires a value" >&2 + exit 1 + fi +} + +# require_arg +# Check: assert that a required flag was actually supplied by the +# caller. Prints the script's print_help (if defined) then exits 1 on failure. +# Preserves the existing behavior of showing help after a "required flag missing" +# error. +require_arg() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} is required." >&2 + if declare -F print_help > /dev/null; then + print_help + fi + exit 1 + fi +} diff --git a/java/ci/assemble_maven_repo.sh b/java/ci/assemble_maven_repo.sh new file mode 100755 index 000000000000..b1d3e65a457c --- /dev/null +++ b/java/ci/assemble_maven_repo.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Decoupled gather step: assemble per-classifier cuDF Java JARs into a single +# Maven-repository-layout directory. +# +# Input: --jars-dir contains one subdirectory per classifier, each holding +# exactly one cudf--.jar and a cudf-.pom. Subdir +# names ARE the classifier names, and the artifact version is derived from +# the JAR filenames (all subdirs must agree). +# +# Output layout: +# /ai/rapids/cudf//cudf--.jar +# /ai/rapids/cudf//cudf-.pom + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +GROUP_PATH="ai/rapids" +ARTIFACT_ID="cudf" + +JARS_DIR="" +OUTPUT_DIR="" + +print_help() { + cat << EOF + +Usage: assemble_maven_repo.sh --jars-dir --output-dir + +Gathers per-classifier cuDF Java JARs into a single Maven-repository-layout tree. + +REQUIRED: + -j, --jars-dir Parent directory containing one subdirectory per + classifier (each holding cudf--.jar + and cudf-.pom). Subdir name is the classifier. + -o, --output-dir Directory to receive the combined Maven-repository layout. + +OPTIONS: + -h, --help Show this help message. + +EXAMPLE: + assemble_maven_repo.sh --jars-dir /tmp/jars --output-dir /tmp/maven-repo + # given /tmp/jars/{cuda12,cuda13}/ inputs, produces: + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda12.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda13.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -j|--jars-dir) + require_value "$1" "$2" + JARS_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --jars-dir "${JARS_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" + +if [[ ! -d ${JARS_DIR} ]]; then + echo "Error: --jars-dir '${JARS_DIR}' does not exist." + exit 1 +fi + +if [[ -e ${OUTPUT_DIR} && -n "$(ls -A "${OUTPUT_DIR}" 2>/dev/null)" ]]; then + echo "Error: --output-dir '${OUTPUT_DIR}' must be empty or nonexistent" >&2 + exit 1 +fi + +ASSEMBLE_FINISHED=0 +cleanup_partial_output() { + if [[ ${ASSEMBLE_FINISHED} -eq 0 && -e ${OUTPUT_DIR} ]]; then + echo "Assembly did not complete; removing partial output at ${OUTPUT_DIR}" >&2 + rm -rf "${OUTPUT_DIR}" + fi +} +trap cleanup_partial_output EXIT + +echo "Assembling Maven repository layout" +echo " jars dir: ${JARS_DIR}" +echo " output dir: ${OUTPUT_DIR}" + +# Walk every classifier subdirectory. Each subdir must contain exactly one +# cudf-*-.jar. The version is derived from the filename and must +# match across all subdirs. +FIRST_VERSION="" +CLASSIFIERS_SEEN="" + +for subdir in "${JARS_DIR}"/*/; do + classifier=$(basename "${subdir}") + + jar="" + for candidate in "${subdir}"cudf-*-"${classifier}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${jar}" ]]; then + echo "Error: multiple JARs in ${subdir} match cudf-*-${classifier}.jar" >&2 + exit 1 + fi + jar=${candidate} + fi + done + + if [[ -z "${jar}" ]]; then + echo "Error: no cudf-*-${classifier}.jar found in ${subdir}" >&2 + exit 1 + fi + + # Filename is cudf--.jar. Peel prefix and suffix. + base=$(basename "${jar}" .jar) + version=$(echo "${base}" | sed -e 's/^cudf-//' -e "s/-${classifier}$//") + + if [[ -z "${FIRST_VERSION}" ]]; then + FIRST_VERSION=${version} + elif [[ "${version}" != "${FIRST_VERSION}" ]]; then + echo "Error: inconsistent versions across subdirs: ${FIRST_VERSION} vs ${version} (${subdir})" >&2 + exit 1 + fi + + DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${version}" + mkdir -p "${DEST_DIR}" + cp -f "${jar}" "${DEST_DIR}/" + echo " + $(basename "${jar}")" + CLASSIFIERS_SEEN="${CLASSIFIERS_SEEN} ${classifier}" +done + +if [[ -z "${FIRST_VERSION}" ]]; then + echo "Error: no classifier subdirs found under ${JARS_DIR}" >&2 + exit 1 +fi + +# POM is identical across subdirs; copy the first one found. +POM_SRC="" +for subdir in "${JARS_DIR}"/*/; do + candidate=${subdir}cudf-${FIRST_VERSION}.pom + if [[ -f "${candidate}" ]]; then + POM_SRC=${candidate} + break + fi +done + +if [[ -z "${POM_SRC}" ]]; then + echo "Error: no cudf-${FIRST_VERSION}.pom found under ${JARS_DIR}" >&2 + exit 1 +fi + +DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${FIRST_VERSION}" +cp -f "${POM_SRC}" "${DEST_DIR}/cudf-${FIRST_VERSION}.pom" +echo " + cudf-${FIRST_VERSION}.pom" + +echo "Maven repository assembled successfully at ${OUTPUT_DIR}" +echo "Classifiers present:${CLASSIFIERS_SEEN}" + +ASSEMBLE_FINISHED=1 diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh new file mode 100755 index 000000000000..084b00c68478 --- /dev/null +++ b/java/ci/build_cudf_java_jar.sh @@ -0,0 +1,246 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained packaging of the cuDF Java JAR for a single classifier. +# +# Consumes a prebuilt static libcudf install tree (from build_static_libcudf.sh), +# compiles the JNI layer against it inside a throwaway RAPIDS ci-conda container, +# and emits the single classifier JAR (plus its POM) to a per-classifier +# subdirectory under --output-dir. This script is layout-agnostic: it produces +# one classifier's artifacts and knows nothing about the combined +# Maven-repository layout (see java/ci/assemble_maven_repo.sh). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +LIBCUDF_DIR="" +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_cudf_java_jar.sh --libcudf-dir --output-dir \\ + --cuda-version [OPTIONS] + +Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-conda +container, linking against a prebuilt static libcudf. Always builds for the +host architecture (uname -m). The build image is fixed to +rapidsai/ci-conda:-latest (version derived from the VERSION +file). + +The classifier is derived from --cuda-version (major) + host arch (uname -m), +mirroring the pom.xml Groovy logic: "cuda" for x86_64, +"cuda-arm64" for aarch64. The classifier JAR + POM are written to +//. Concurrent invocations targeting different +classifiers are safe. + +REQUIRED: + -l, --libcudf-dir Static libcudf install tree produced by + build_static_libcudf.sh. + -o, --output-dir Host parent directory. The script creates and writes + to //, which must not already + exist. + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + Must match --cuda-version of the static libcudf tree; + determines the cuda12/cuda13 classifier. + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. Must match the value passed to + build_static_libcudf.sh when producing the static + libcudf tree in --libcudf-dir, or device linking of + libcudfjni.so against libcudf.a will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_cudf_java_jar.sh -l /tmp/libcudf-cuda12 -o /tmp/jars -c 12.9 + build_cudf_java_jar.sh -l /tmp/libcudf-cuda13 -o /tmp/jars -c 13.3 -A 80 + # writes: + # /tmp/jars/cuda12/cudf--cuda12.jar + # /tmp/jars/cuda12/cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -l|--libcudf-dir) + require_value "$1" "$2" + LIBCUDF_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --libcudf-dir "${LIBCUDF_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +if [[ ! -d ${LIBCUDF_DIR} ]]; then + echo "Error: --libcudf-dir '${LIBCUDF_DIR}' does not exist." + exit 1 +fi + +# Derive the Maven classifier from --cuda-version major + host arch, mirroring +# the pom.xml Groovy logic: "cuda" for x86_64, "cuda-arm64" for +# aarch64. +CUDA_MAJOR="$(echo "${CUDA_VERSION}" | cut -d. -f1)" +HOST_ARCH="$(uname -m)" +case "${HOST_ARCH}" in + x86_64) + CLASSIFIER="cuda${CUDA_MAJOR}" + ;; + aarch64|arm64) + CLASSIFIER="cuda${CUDA_MAJOR}-arm64" + ;; + *) + echo "Error: Unsupported host arch '${HOST_ARCH}' (expected x86_64 or aarch64)" >&2 + exit 1 + ;; +esac + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" +LIBCUDF_DIR="$(cd "${LIBCUDF_DIR}" && pwd)" + +CLASSIFIER_OUT="${OUTPUT_DIR}/${CLASSIFIER}" +if [[ -e ${CLASSIFIER_OUT} ]]; then + echo "Error: classifier output '${CLASSIFIER_OUT}' already exists. Remove it before re-running." >&2 + exit 1 +fi +mkdir -p "${CLASSIFIER_OUT}" + +# Per-classifier scratch dir for Maven's java/target/. Nested bind-mount over +# /repo/java/target inside the container isolates concurrent invocations +# (each classifier gets its own target/). The `.mvn-temp-target/` prefix +# keeps this dir invisible to the default `*/` globbing in +# assemble_maven_repo.sh's classifier discovery loop. +# +# Recreate the scratch on every launch: the in-container mvn cannot clean a +# bind-mount point (rmdir on /repo/java/target fails with EBUSY), so the +# host wrapper is responsible for guaranteeing a clean starting target/. +TARGET_SCRATCH="${OUTPUT_DIR}/.mvn-temp-target/${CLASSIFIER}" +rm -rf "${TARGET_SCRATCH}" +mkdir -p "${TARGET_SCRATCH}" + +echo "Packaging cuDF Java JAR" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " classifier: ${CLASSIFIER}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " libcudf dir: ${LIBCUDF_DIR}" +echo " output dir: ${CLASSIFIER_OUT}" +echo " target dir: ${TARGET_SCRATCH}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${LIBCUDF_DIR}:/libcudf:ro" + --volume "${CLASSIFIER_OUT}:/output" + --volume "${TARGET_SCRATCH}:/repo/java/target" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_cudf_java_jar_in_container.sh + +# Post-run: assert exactly one main classifier JAR + one POM, and that the +# JAR's classifier suffix matches the subdir name we chose (catches pom drift). +PRODUCED_JAR="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*-"${CLASSIFIER}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_JAR}" ]]; then + echo "Error: multiple JARs matching cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_JAR=${candidate} + fi +done + +if [[ -z "${PRODUCED_JAR}" ]]; then + echo "Error: no cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +PRODUCED_POM="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*.pom; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_POM}" ]]; then + echo "Error: multiple POMs found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_POM=${candidate} + fi +done + +if [[ -z "${PRODUCED_POM}" ]]; then + echo "Error: no cudf-*.pom found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +echo "cuDF Java JAR build succeeded:" +echo " $(basename "${PRODUCED_JAR}")" +echo " $(basename "${PRODUCED_POM}")" diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh new file mode 100755 index 000000000000..98fce46c4f4a --- /dev/null +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container packaging of the cuDF Java JAR for a single classifier. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_cudf_java_jar.sh. It generates the build_java conda toolchain +# environment, compiles the JNI layer against a prebuilt static libcudf +# (mounted at /libcudf), and packages the cuDF Java JAR. The resulting +# classifier JAR and its POM are copied to /output. /output and +# /repo/java/target are chowned to HOST_UID:HOST_GID on exit so the host user +# owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output and /repo/java/target +# (both required). + +set -e + +OUTPUT_DIR=/output +REPO_ROOT=/repo +CUDF_INSTALL_DIR=/libcudf + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + +_chown_outputs_on_exit() { + chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true +} +trap _chown_outputs_on_exit EXIT + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +BUILD_ARG=( + -B + "-Dmaven.repo.local=/tmp/.m2" + "-Dparallel.level=${PARALLEL_LEVEL}" + "-DskipTests=true" + "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON" + "-DCUDF_JNI_LIBCUDF_STATIC=ON" + "-DUSE_GDS=OFF" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + BUILD_ARG+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +cd "${REPO_ROOT}/java" + +CUDF_VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" +rapids-logger "Packaging cuDF Java JAR version ${CUDF_VERSION} (libcudf: ${CUDF_INSTALL_DIR})" + +# The `clean` goal is intentionally omitted: /repo/java/target is a +# bind-mount point, so when `mvn clean` attempts to remove the directory, +# it fails with EBUSY. The host wrapper (build_cudf_java_jar.sh) recreates +# the scratch dir before each container launch to guarantee target/ starts empty. +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" mvn package "${BUILD_ARG[@]}" + +MAIN_JAR="" +for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do + case "${candidate}" in + *-tests.jar|*-sources.jar|*-javadoc.jar) + continue + ;; + esac + if [[ -f ${candidate} ]]; then + MAIN_JAR=${candidate} + break + fi +done + +if [[ -z ${MAIN_JAR} ]]; then + echo "Error: no cuDF classifier JAR produced under target/" + ls -l target/ || true + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" +cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" +cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" + +rapids-logger "Emitted $(basename "${MAIN_JAR}") + cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" diff --git a/java/ci/build_static_libcudf.sh b/java/ci/build_static_libcudf.sh new file mode 100755 index 000000000000..2fa2e435471c --- /dev/null +++ b/java/ci/build_static_libcudf.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained build of a static libcudf install tree. +# +# Pulls the RAPIDS ci-conda image, builds libcudf with BUILD_SHARED_LIBS=OFF +# inside a throwaway container, and installs the static libcudf tree (libcudf.a +# plus its static dependencies) into a directory on the host. No GPU is required +# to build. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_static_libcudf.sh --output-dir --cuda-version [OPTIONS] + +Builds a static libcudf install tree inside a RAPIDS ci-conda container and +writes it to a directory on the host. Always builds for the host architecture +(uname -m). The build image is fixed to rapidsai/ci-conda:-latest +(version derived from the VERSION file). + +REQUIRED: + -o, --output-dir Host directory to receive the static install tree + (libcudf.a and its static dependencies). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. When packaging the cuDF Java JAR + against this static libcudf tree, pass the same value + to build_cudf_java_jar.sh --cmake-cuda-architectures + or device linking of libcudfjni.so against libcudf.a + will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version "12.9" + build_static_libcudf.sh -o /tmp/libcudf-cuda13 -c 13.3 -A "80" + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" + +echo "Building static libcudf" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " output dir: ${OUTPUT_DIR}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${OUTPUT_DIR}:/output" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_static_libcudf_in_container.sh + +if [[ -f "${OUTPUT_DIR}/lib/libcudf.a" || -f "${OUTPUT_DIR}/lib64/libcudf.a" ]]; then + echo "Static libcudf build succeeded: ${OUTPUT_DIR}" +else + echo "Error: expected libcudf.a not found under ${OUTPUT_DIR}/lib or ${OUTPUT_DIR}/lib64" + exit 1 +fi diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh new file mode 100755 index 000000000000..e77637e747b2 --- /dev/null +++ b/java/ci/build_static_libcudf_in_container.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container build of a static libcudf install tree. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_static_libcudf.sh. It generates the build_java conda toolchain +# environment, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs the +# resulting static libcudf (plus its static dependencies) into /output. Then +# chowns /output to HOST_UID:HOST_GID so the host user owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output (both required). + +set -e + +INSTALL_PREFIX=/output +REPO_ROOT=/repo +BUILD_DIR=/tmp/libcudf-build + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +CMAKE_ARGS=( + -S "${REPO_ROOT}/cpp" + -B "${BUILD_DIR}" + -GNinja + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTS=OFF + -DUSE_NVTX=ON + -DCUDF_LARGE_STRINGS_DISABLED=ON + -DCUDF_USE_ARROW_STATIC=ON + -DCUDF_ENABLE_ARROW_S3=OFF + -DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON + -DRMM_LOGGING_LEVEL=OFF + -DCUDF_KVIKIO_REMOTE_IO=OFF +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + CMAKE_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +rapids-logger "Configuring static libcudf" +cmake "${CMAKE_ARGS[@]}" + +rapids-logger "Building static libcudf with ${PARALLEL_LEVEL} jobs" +cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" + +rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" +cmake --install "${BUILD_DIR}" + +rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" +chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" diff --git a/java/ci/test_java_build_local.sh b/java/ci/test_java_build_local.sh new file mode 100755 index 000000000000..1d67ad62771d --- /dev/null +++ b/java/ci/test_java_build_local.sh @@ -0,0 +1,315 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Local end-to-end verification of the build workflow: builds the static +# libcudf install tree and the classifier JAR for both CUDA 12 and CUDA 13, +# then runs the decoupled gather step to assemble the combined +# Maven-repository layout. Mirrors what the java-build matrix + java-gather +# jobs in .github/workflows/build.yaml do in CI. +# +# Runs on either x86_64 or aarch64 hosts. Each invocation covers only the +# host architecture: on x86_64 it produces the "cuda12" and "cuda13" +# classifier JARs; on aarch64 it produces "cuda12-arm64" and "cuda13-arm64". +# The arm64 suffix is added automatically by the child build scripts (via +# pom.xml's classifier logic keyed off `uname -m`). To cover all four +# release classifiers, run this script once on each architecture. +# +# Both static libcudf builds run in parallel, and both JAR builds run in +# parallel (each uses a nested bind-mount over /repo/java/target inside the +# container to isolate Maven output). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +WORK_DIR="" +PARALLEL_LEVEL="$(nproc)" +CMAKE_CUDA_ARCHITECTURES="" + +# CUDA versions built for both classifiers. Keep in sync with the java-build +# matrix in .github/workflows/build.yaml. +CUDA12_VERSION="12.9" +CUDA13_VERSION="13.3" + +# Wall-clock timing variables. +STEP_NAMES=() +STEP_ELAPSED=() + +print_help() { + cat << EOF + +Usage: test_java_build_local.sh --work-dir [OPTIONS] + +Runs the full build+gather pipeline for both CUDA 12 and CUDA 13 on the host +architecture (x86_64 or aarch64) and assembles the combined Maven-repository +layout. + +REQUIRED: + -w, --work-dir Scratch directory for build outputs. Subtrees created: + /libcudf-cuda12 static libcudf (CUDA 12) + /libcudf-cuda13 static libcudf (CUDA 13) + /jars/ per-classifier JAR + POM + /maven-repo combined Maven layout + where is "cuda12" / "cuda13" on x86_64 + and "cuda12-arm64" / "cuda13-arm64" on aarch64. + +OPTIONS: + -j, --parallel Total build parallelism (default: nproc = ${PARALLEL_LEVEL}). + Each concurrent JAR build gets --parallel/2 to avoid + RAM pressure from two parallel nvcc runs. + -A, --cmake-cuda-architectures + CUDA architecture list (e.g. "80" or "80;90") passed to + both build scripts. The literal value "all" is a + sentinel meaning "do not pass --cmake-cuda-architectures + to child scripts" — child scripts then fall back to + cuDF's default RAPIDS full architecture list (slow). + Default: auto-detect the local GPU's compute + capability via nvidia-smi (e.g. Ampere -> "80"). If + nvidia-smi is missing or returns nothing, falls back + to "all". + -h, --help Show this help message. + +EXAMPLES: + # Fast run (auto-detect local GPU arch): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test + + # Explicit override: + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures 80 + + # Full RAPIDS arch list (slow, e.g. GPU-less host): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures all + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -w|--work-dir) + require_value "$1" "$2" + WORK_DIR=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +log_step() { + echo + echo "============================================================" + echo "== $1" + echo "============================================================" +} + +format_elapsed() { + local s=$1 + printf '%dm %02ds' $((s/60)) $((s%60)) +} + +record_step_end() { + local name=$1 + local start=$2 + local elapsed=$((SECONDS - start)) + STEP_NAMES+=("${name}") + STEP_ELAPSED+=("${elapsed}") + echo + echo "== ${name} completed in $(format_elapsed "${elapsed}")" +} + +parse_args "$@" + +require_arg --work-dir "${WORK_DIR}" + +mkdir -p "${WORK_DIR}" +WORK_DIR="$(cd "${WORK_DIR}" && pwd)" +LOG_DIR="${WORK_DIR}/logs" +mkdir -p "${LOG_DIR}" + +# Remove outputs from any prior run so cmake/mvn does not see stale artifacts. +rm -rf "${WORK_DIR}/libcudf-cuda12" \ + "${WORK_DIR}/libcudf-cuda13" \ + "${WORK_DIR}/jars" \ + "${WORK_DIR}/maven-repo" + +# Auto-detect the local GPU's compute capability when the flag was not passed. +# "all" is the sentinel value that means "do not forward this flag to child +# scripts". Child scripts fall back to cuDF's default RAPIDS architecture +# list (slow but correct on GPU-less hosts). +if [[ -z ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DETECTED="" + if command -v nvidia-smi > /dev/null 2>&1; then + DETECTED=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' | tr -d ' ') + fi + if [[ -n ${DETECTED} ]]; then + CMAKE_CUDA_ARCHITECTURES=${DETECTED} + echo "Auto-detected local GPU compute capability: ${CMAKE_CUDA_ARCHITECTURES}" + else + CMAKE_CUDA_ARCHITECTURES="all" + echo "No local GPU detected; falling back to cuDF's default RAPIDS architecture list" + fi +fi + +echo "cuDF Java local build verification" +echo " host arch: $(uname -m)" +echo " work dir: ${WORK_DIR}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " cuda12 version: ${CUDA12_VERSION}" +echo " cuda13 version: ${CUDA13_VERSION}" +echo " cmake cuda architectures: ${CMAKE_CUDA_ARCHITECTURES}" +echo " logs: ${LOG_DIR}/{static,jar}_cuda{12,13}.log" + +# When forwarding to child scripts, "all" means "don't pass the flag". +CHILD_CMAKE_ARGS=() +if [[ ${CMAKE_CUDA_ARCHITECTURES} != "all" ]]; then + CHILD_CMAKE_ARGS=(--cmake-cuda-architectures "${CMAKE_CUDA_ARCHITECTURES}") +fi + +# Both Step 1 and Step 2 launch two concurrent builds. Each build gets half +# of PARALLEL_LEVEL so together they stay within PARALLEL_LEVEL. +STEP_PARALLEL=$((PARALLEL_LEVEL / 2)) +if [[ ${STEP_PARALLEL} -lt 1 ]]; then + STEP_PARALLEL=1 +fi + +# Step 1: static libcudf builds in parallel. +log_step "Step 1: building static libcudf for CUDA 12 and CUDA 13 in parallel" +STEP1_START=${SECONDS} + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda12" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda12.log" 2>&1 & +STATIC_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda13" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda13.log" 2>&1 & +STATIC_CUDA13_PID=$! + +echo " cuda12 static pid: ${STATIC_CUDA12_PID} (tail -f ${LOG_DIR}/static_cuda12.log)" +echo " cuda13 static pid: ${STATIC_CUDA13_PID} (tail -f ${LOG_DIR}/static_cuda13.log)" + +STATIC_CUDA12_RC=0 +STATIC_CUDA13_RC=0 +if ! wait "${STATIC_CUDA12_PID}"; then + STATIC_CUDA12_RC=1 +fi +if ! wait "${STATIC_CUDA13_PID}"; then + STATIC_CUDA13_RC=1 +fi + +if [[ ${STATIC_CUDA12_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 12 build failed." + echo "See ${LOG_DIR}/static_cuda12.log" +fi +if [[ ${STATIC_CUDA13_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 13 build failed." + echo "See ${LOG_DIR}/static_cuda13.log" +fi +if [[ ${STATIC_CUDA12_RC} -ne 0 || ${STATIC_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 1: static libcudf (parallel run)" "${STEP1_START}" + +# Step 2: JAR builds in parallel. Each build's container nests a bind-mount +# over /repo/java/target so concurrent Maven runs don't clobber each other. +log_step "Step 2: packaging cuDF Java JARs for cuda12 and cuda13 in parallel" +STEP2_START=${SECONDS} + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda12" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda12.log" 2>&1 & +JAR_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda13" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda13.log" 2>&1 & +JAR_CUDA13_PID=$! + +echo " cuda12 jar pid: ${JAR_CUDA12_PID} (tail -f ${LOG_DIR}/jar_cuda12.log)" +echo " cuda13 jar pid: ${JAR_CUDA13_PID} (tail -f ${LOG_DIR}/jar_cuda13.log)" + +JAR_CUDA12_RC=0 +JAR_CUDA13_RC=0 +if ! wait "${JAR_CUDA12_PID}"; then + JAR_CUDA12_RC=1 +fi +if ! wait "${JAR_CUDA13_PID}"; then + JAR_CUDA13_RC=1 +fi + +if [[ ${JAR_CUDA12_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 12 build failed." + echo "See ${LOG_DIR}/jar_cuda12.log" +fi +if [[ ${JAR_CUDA13_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 13 build failed." + echo "See ${LOG_DIR}/jar_cuda13.log" +fi +if [[ ${JAR_CUDA12_RC} -ne 0 || ${JAR_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 2: JAR builds (parallel run)" "${STEP2_START}" + +# Step 3: gather into a combined Maven-repository layout. +log_step "Step 3: assembling combined Maven-repository layout" +STEP3_START=${SECONDS} + +"${SCRIPT_DIR}/assemble_maven_repo.sh" \ + --jars-dir "${WORK_DIR}/jars" \ + --output-dir "${WORK_DIR}/maven-repo" + +record_step_end "Step 3: assemble Maven repo" "${STEP3_START}" + +# Derive the assembled version from the output tree for display purposes. +CUDF_VERSION=$(basename "$(ls -d "${WORK_DIR}/maven-repo/ai/rapids/cudf"/*/ | head -1)") + +log_step "Success" +echo "Combined Maven repository:" +echo " ${WORK_DIR}/maven-repo/ai/rapids/cudf/${CUDF_VERSION}/" +echo +echo "Timings:" +for i in "${!STEP_NAMES[@]}"; do + printf ' %-45s %s\n' "${STEP_NAMES[$i]}" "$(format_elapsed "${STEP_ELAPSED[$i]}")" +done +printf ' %-45s %s\n' "Total wall time" "$(format_elapsed "${SECONDS}")" diff --git a/java/pom.xml b/java/pom.xml index 723dd39c2cf6..8521de5567e4 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -657,6 +657,13 @@ def cm = cudaPattern.matcher(nvccout) if (cm.find()) { def classifier = 'cuda' + cm.group(1) + // Emit "cuda" on x86_64 and "cuda-arm64" on + // aarch64 so a single pom produces a distinct Maven + // classifier per architecture. + def osArch = System.getProperty('os.arch') + if (osArch == 'aarch64' || osArch == 'arm64') { + classifier = classifier + '-arm64' + } project.properties['cuda.classifier'] = classifier } else { throw new RuntimeException('could not find CUDA version') diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index a1d14959e7ca..251514aa66ef 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -911,20 +911,23 @@ def as_numerical_column(self, dtype: DtypeObj) -> NumericalColumn: self.dtype ): # Short-circuit the cast if the dtypes are equivalent - # but not the same type object. + # but not the same type object. Do NOT mutate self._dtype: + # the column object may be shared with the caller's frame. if ( is_pandas_nullable_extension_dtype(dtype) and isinstance(self.dtype, np.dtype) and self.dtype.kind == "f" ): - # If the dtype is a pandas nullable extension type, we need to - # float column doesn't have any NaNs. + # NaNs must become nulls before viewing as a masked dtype. res = self.nans_to_nulls() - res._dtype = dtype - return res - else: - self._dtype = dtype - return self + return cast( + "NumericalColumn", + ColumnBase.create(res.plc_column, dtype), + ) + return cast( + "NumericalColumn", + ColumnBase.create(self.plc_column, dtype), + ) if self.dtype.kind == "f" and dtype.kind in "iu": if not is_pandas_nullable_extension_dtype(dtype) and ( self.nan_count > 0 diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index 621163c4be2b..6cba72851da4 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -47,6 +47,7 @@ _FastSlowAttribute, _FunctionProxy, _maybe_wrap_result, + _setattr_fsproxy_no_mirror, _State, _Unusable, is_proxy_object, @@ -1691,8 +1692,10 @@ def _df_query_method(self, *args, local_dict=None, global_dict=None, **kwargs): ) -DataFrame.eval = _df_eval_method -DataFrame.query = _df_query_method +# These custom implementations are installed by cudf.pandas itself and must +# not be mirrored onto (and clobber) the real ``pandas.DataFrame``. +_setattr_fsproxy_no_mirror(DataFrame, "eval", _df_eval_method) +_setattr_fsproxy_no_mirror(DataFrame, "query", _df_query_method) _JsonReader = make_intermediate_proxy_type( "_JsonReader", diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index 20600c7fe9db..60fa07dc37b4 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -362,6 +362,11 @@ def _fsproxy_state(self) -> _State: final_type_map[fast_type] = cls final_type_map[slow_type] = cls + # Proxy type fully constructed: snapshot its pristine state and, from + # here on, mirror class-level attribute writes (genuine runtime + # monkeypatches) onto the underlying "slow" (real) type. + _enable_fsproxy_mirroring(cls) + return cls @@ -511,6 +516,11 @@ def _fsproxy_fast_to_slow(self): intermediate_type_map[fast_type] = cls intermediate_type_map[slow_type] = cls + # Proxy type fully constructed: snapshot its pristine state and, from + # here on, mirror class-level attribute writes (genuine runtime + # monkeypatches) onto the underlying "slow" (real) type. + _enable_fsproxy_mirroring(cls) + return cls @@ -560,6 +570,55 @@ def get_registered_functions(): return dict() +_SLOW_ABSENT = object() + + +def _enable_fsproxy_mirroring(cls: type) -> None: + """Finalize a proxy type for class-level patch mirroring. + + Snapshots the proxy type's pristine public class attributes together + with the slow type's pristine class-dict entries for the same names, + then enables mirroring of class-level attribute writes/deletions onto + the slow type (see ``_FastSlowProxyMeta.__setattr__``/``__delattr__``). + + The snapshot is a fixed translation table, not runtime patch tracking: + re-assigning the proxy's pristine attribute for ``name`` (which is what + ``monkeypatch``/``mock.patch`` save and re-assign on undo) translates + to restoring the slow type's pristine attribute for ``name``. + """ + slow = cls._fsproxy_slow_type # type: ignore[attr-defined] + pristine = { + name: (value, slow.__dict__.get(name, _SLOW_ABSENT)) + for name, value in cls.__dict__.items() + if not name.startswith("_") + } + type.__setattr__(cls, "_fsproxy_pristine_attrs", pristine) + type.__setattr__(cls, "_fsproxy_mirror_slow_overrides", True) + + +def _setattr_fsproxy_no_mirror(cls: type, name: str, value: Any) -> None: + """Install a cudf.pandas-internal attribute on a proxy type. + + ``_FastSlowProxyMeta.__setattr__`` mirrors class-level attribute writes + onto the underlying "slow" (real) type so that runtime monkeypatches stay + visible to the pandas fallback path. cudf.pandas itself installs a handful + of custom methods (e.g. ``DataFrame.query``/``DataFrame.eval``) onto the + proxy classes that must *not* clobber pandas' genuine implementations; use + this helper for those (rare) assignments, after the proxy type has been + fully constructed by ``make_*_proxy_type``. The attribute is registered as + part of the proxy's pristine state so that a later save/patch/re-assign + cycle restores the slow type's own attribute rather than forwarding the + cudf-internal object to it. + """ + type.__setattr__(cls, name, value) + if not name.startswith("_"): + slow = cls._fsproxy_slow_type # type: ignore[attr-defined] + cls._fsproxy_pristine_attrs[name] = ( # type: ignore[attr-defined] + value, + slow.__dict__.get(name, _SLOW_ABSENT), + ) + + class _FastSlowProxyMeta(type): """ Metaclass used to dynamically find class attributes and @@ -578,6 +637,136 @@ def _fsproxy_slow(self) -> type: def _fsproxy_fast(self) -> type: return self._fsproxy_fast_type + def __new__(mcls, *args, **kwargs): + cls = super().__new__(mcls, *args, **kwargs) + # Per-proxy-type switch controlling whether class-level attribute + # writes are mirrored onto the underlying "slow" (real) type (see + # ``__setattr__``/``__delattr__``). It starts disabled so that the + # attributes installed while the proxy type is being built are not + # forwarded to the real type; ``make_*_proxy_type`` enables it once + # construction is complete. Initialized in ``__new__`` rather than + # ``__init__`` because cooperating metaclasses may perform + # class-level attribute writes from their own ``__new__`` — e.g. + # ``ABCMeta.__new__`` assigns ``__abstractmethods__``, dispatching + # to ``__setattr__`` below before ``__init__`` ever runs. + type.__setattr__(cls, "_fsproxy_mirror_slow_overrides", False) + return cls + + def __setattr__(cls, name, value): + # Class-level attribute assignments on a proxy type (e.g. + # ``monkeypatch.setattr(pd.ExcelFile, "parse", fn)``) must also be + # mirrored onto the underlying "slow" (real) type. Code that runs + # under ``disable_module_accelerator()`` (e.g. the pandas fallback + # path of ``pd.read_excel``) resolves attributes from the real + # class, not the proxy, so a patch applied only to the proxy would + # otherwise be invisible to that code. The assigned value is first + # translated into its slow-space equivalent: re-assigning the + # proxy's pristine attribute translates to the slow type's pristine + # attribute, and proxy machinery is unwrapped to the slow object it + # delegates to, so save/patch/re-assign cycles round-trip on the + # real type as well. + type.__setattr__(cls, name, value) + if not cls._fsproxy_mirror_slow_overrides: + # The proxy type is still being constructed (or this is a + # non-pandas proxy): only mirror user/runtime monkeypatches, + # never the custom methods cudf.pandas installs on the proxy + # classes itself. + return + if name.startswith("_"): + return + slow = cls._fsproxy_slow_type + try: + # Mirroring is best-effort: translating a wrapped proxy instance + # can require a fast-to-slow conversion, which may itself fail; + # never let that escape an otherwise-successful assignment. + pristine = cls._fsproxy_pristine_attrs + entry = pristine.get(name) + if entry is not None and value is entry[0]: + # The proxy's pristine attribute for ``name`` is being + # re-assigned (e.g. ``monkeypatch``/``mock.patch`` undo + # re-setting the saved class-dict entry). Its slow-space + # equivalent is the slow type's pristine attribute. + if entry[1] is _SLOW_ABSENT: + # The slow type never defined ``name`` itself: the proxy + # mirrors the slow type's *dir*, so it has pristine + # attributes for methods the slow type only inherits + # (e.g. ``DataFrame.head`` lives on ``NDFrame``). + # Mirroring a patch for such a name added a shadowing + # entry to the slow type's dict; undoing the patch must + # remove that entry again so the inherited + # implementation becomes visible. It may legitimately be + # missing (the mirror is best-effort), hence the guard. + if name in slow.__dict__: + delattr(slow, name) + else: + setattr(slow, name, entry[1]) + return + # Otherwise translate the assigned value into "slow" space + # before mirroring it: a value read off a proxy type (e.g. the + # original that a caller saves before patching and re-assigns + # to undo) is proxy machinery wrapping a slow-side object, and + # mirroring it verbatim would install that machinery on the + # real type. Unwrap it to the slow object it delegates to, so + # save/patch/re-assign cycles round-trip on the real type; + # values with no determinable slow-side equivalent are not + # mirrored at all. + if isinstance(value, _FastSlowAttribute): + # The proxy's own delegating descriptor (a *pristine* one is + # already handled by identity above; this covers a + # descriptor obtained some other way): its slow equivalent + # is the method it wraps, if it ever resolved one. + attr = value._attr + if not isinstance(attr, _MethodProxy): + return + value = attr + if isinstance(value, _FunctionProxy): + unwrapped = value._fsproxy_slow + if entry is not None and entry[1] is not _SLOW_ABSENT: + # Re-assigning a saved ``cls.method`` (a ``_MethodProxy`` + # over the *resolved* slow attribute): if it resolves + # back to the slow type's pristine attribute, restore + # the pristine class-dict entry itself so + # ``classmethod``/``staticmethod`` descriptors are not + # degraded to their bound/plain-function forms. + descriptor = entry[1] + try: + resolved = ( + descriptor.__get__(None, slow) + if hasattr(type(descriptor), "__get__") + else descriptor + ) + if unwrapped is resolved or unwrapped == resolved: + setattr(slow, name, descriptor) + return + except Exception: + pass + setattr(slow, name, unwrapped) + elif isinstance(value, _FastSlowProxy): + setattr(slow, name, value._fsproxy_slow) + elif isinstance(value, _FastSlowProxyMeta): + slow_type = getattr(value, "_fsproxy_slow_type", None) + if slow_type is not None: + setattr(slow, name, slow_type) + else: + setattr(slow, name, value) + except Exception: + pass + + def __delattr__(cls, name): + # Mirror class-level attribute *deletions* onto the underlying "slow" + # (real) type as well, for the same reason as ``__setattr__``: after + # ``del cls.name`` the attribute is gone from the proxy, so it must + # also be gone from the real type seen by fallback code. + type.__delattr__(cls, name) + if not cls._fsproxy_mirror_slow_overrides: + return + if name.startswith("_"): + return + try: + delattr(cls._fsproxy_slow_type, name) + except (AttributeError, TypeError): + pass + def __dir__(self): # Try to return the cached dir of the slow object, but if it # doesn't exist, fall back to the default implementation. diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index f09d75e33b39..4f749565dfc8 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1721,8 +1721,6 @@ def pytest_unconfigure(config): "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[True]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_level[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[True]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_order_with_unsorted_levels_multi_row_2[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[True]": "TODO: Add a reason for failure", @@ -2511,7 +2509,6 @@ def pytest_unconfigure(config): "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-pyarrow-True]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-False]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-True]": 'AssertionError: Column name="0" are different', - "tests/indexing/test_loc.py::TestLocWithMultiIndex::test_loc_set_nan_in_categorical_series[Float64]": "TODO: Add a reason for failure", "tests/indexing/test_loc.py::test_loc_getitem_multiindex_tuple_level": "AssertionError: DataFrame Expected type , found instead", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask0-values0-object]": "TODO: Add a reason for failure", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask1-values0-object]": "TODO: Add a reason for failure", @@ -2542,19 +2539,6 @@ def pytest_unconfigure(config): "tests/io/excel/test_odswriter.py::test_cell_value_type[test string-string-string-value-test string]": "TODO: Add a reason for failure", "tests/io/excel/test_odswriter.py::test_cell_value_type[value4-date-date-value-2010-10-10T10:10:10]": "TODO: Add a reason for failure", "tests/io/excel/test_odswriter.py::test_cell_value_type[value5-date-date-value-2010-10-10]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.ods')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsb')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsx')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('odf', '.ods')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('openpyxl', '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('openpyxl', '.xlsx')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('pyxlsb', '.xlsb')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('xlrd', '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xlsx')]": "TODO: Add a reason for failure", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[False]": "AttributeError: _compute. Did you mean: 'compare'?", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[True]": "AttributeError: _compute. Did you mean: 'compare'?", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[columns]": "AttributeError: _compute. Did you mean: 'compare'?", @@ -3688,7 +3672,6 @@ def pytest_unconfigure(config): "tests/test_algos.py::TestValueCounts::test_value_counts_dropna": "pandas keeps bool-with-None data as object dtype; cudf stores it as a masked bool column", "tests/test_algos.py::TestValueCounts::test_value_counts_stability": "asserts that kind='quicksort' produces an unstable order; cudf sorts are always stable", "tests/test_col.py::test_cached_property": "AssertionError: assert False", - "tests/test_col.py::test_custom_accessor": "AttributeError: 'Series' object has no attribute 'xyz'", "tests/test_common.py::test_serializable[obj0]": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[False]": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[True]": "TODO: Add a reason for failure", @@ -3841,13 +3824,11 @@ def pytest_unconfigure(config): "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2NS]": "TODO: Add a reason for failure", "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2Us]": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[None]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[nan]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_frame_equal_index_dtype_mismatch[True-df11-df21-DataFrame\\\\.index level \\\\[0\\\\] are different]": "Failed: DID NOT RAISE ", "tests/util/test_assert_index_equal.py::test_index_equal_range_categories[True-True]": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_series_equal.py::test_assert_series_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_assert_series_equal_int_tol": "AssertionError: left is not an ExtensionArray", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[Int64]": "Failed: DID NOT RAISE ", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[int64]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/series/methods/test_astype.py b/python/cudf/cudf/tests/series/methods/test_astype.py index 6a0fa7b3a05f..ef37cf00923b 100644 --- a/python/cudf/cudf/tests/series/methods/test_astype.py +++ b/python/cudf/cudf/tests/series/methods/test_astype.py @@ -1667,3 +1667,27 @@ def test_string_astype_int_invalid_underscores_raises(data, dtype): lfunc_args_and_kwargs=((), {"dtype": dtype}), rfunc_args_and_kwargs=((), {"dtype": dtype}), ) + + +@pytest.mark.parametrize( + "data, src_dtype, masked_dtype", + [ + ([1.0, 2.0, float("nan")], "float64", pd.Float64Dtype()), + ([1, 2, 3], "int64", pd.Int64Dtype()), + ], +) +def test_astype_masked_equivalent_dtype_no_source_mutation( + data, src_dtype, masked_dtype +): + # casting to the equivalent masked dtype takes a short-circuit path; + # it must not mutate the source column's dtype in place (the column + # is shared with the source Series/frame) + ser = cudf.Series(data, dtype=src_dtype) + result = ser.astype(masked_dtype) + + assert ser.dtype == np.dtype(src_dtype) + assert result.dtype == masked_dtype + assert_eq( + result.to_pandas(), + pd.Series(data, dtype=src_dtype).astype(masked_dtype), + ) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index cbf3e17bfdc4..792e4f2cf314 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -2186,6 +2186,38 @@ def test_module_proxy_write_through_config(monkeypatch): cf.register_option("foo", 1) +def test_class_monkeypatch_roundtrip_restores_real_pandas(monkeypatch): + # Class-level patches on proxy types are mirrored onto the real pandas + # type (so they stay visible to fallback code running under + # ``disable_module_accelerator``); undoing them must restore the real + # type's own attributes — including for attributes that cudf.pandas + # replaces on the proxy, like ``columns``/``eval``/``str``. + real_df = xpd.DataFrame._fsproxy_slow + real_series = xpd.Series._fsproxy_slow + orig_columns = real_df.__dict__["columns"] + orig_eval = real_df.__dict__["eval"] + orig_str = real_series.__dict__["str"] + + def fake_eval(self, *args, **kwargs): + return "patched" + + monkeypatch.setattr(xpd.DataFrame, "eval", fake_eval) + assert real_df.__dict__["eval"] is fake_eval + monkeypatch.setattr( + xpd.DataFrame, "columns", property(lambda self: "patched") + ) + monkeypatch.setattr(xpd.Series, "str", property(lambda self: "patched")) + monkeypatch.undo() + + assert real_df.__dict__["columns"] is orig_columns + assert real_df.__dict__["eval"] is orig_eval + assert real_series.__dict__["str"] is orig_str + # The real type must remain fully functional on the fallback path. + df = real_df({"a": [1, 2]}) + assert list(df.columns) == ["a"] + assert list(df.eval("b = a + 1").columns) == ["a", "b"] + + @pytest.mark.parametrize("box", ["Series", "array"]) @pytest.mark.parametrize("na_value", [pd.NA, np.nan], ids=["NA", "NaN"]) @pytest.mark.parametrize("storage", ["python", "pyarrow"]) diff --git a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py index 30d2124edcbf..5e22badc4c0e 100644 --- a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py +++ b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py @@ -12,7 +12,9 @@ import cudf.pandas.fast_slow_proxy from cudf.pandas.fast_slow_proxy import ( _fast_arg, + _FastSlowAttribute, _FunctionProxy, + _setattr_fsproxy_no_mirror, _slow_arg, _transform_arg, _Unusable, @@ -747,3 +749,340 @@ def test_tuple_with_attrs_transform(): assert b is bprime assert c == cprime and c is not cprime assert d == dprime and d is not dprime + + +def _make_mirror_proxy(): + class Fast: + pass + + class SlowBase: + def inherited(self): + return "base" + + class Slow(SlowBase): + const = 42 + + def existing(self): + return "slow original" + + @property + def prop(self): + return "slow prop" + + @staticmethod + def smethod(x): + return x + 1 + + @classmethod + def cmethod(cls): + return cls.__name__ + + Pxy = make_final_proxy_type( + "Pxy", + Fast, + Slow, + fast_to_slow=lambda fast: Slow(), + slow_to_fast=lambda slow: Fast(), + ) + return Fast, Slow, Pxy + + +def test_class_attr_mirroring_enabled_after_construction(): + # ``make_*_proxy_type`` enables per-type mirroring once the proxy type is + # fully built (it starts disabled so the methods installed during + # construction are not forwarded to the real type). + _, _, Pxy = _make_mirror_proxy() + assert Pxy.__dict__["_fsproxy_mirror_slow_overrides"] is True + + +def test_class_attr_setattr_mirrored_to_slow(): + # A class-level attribute write on the proxy is mirrored onto the + # underlying "slow" (real) type so it is visible to fallback code. + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.new_method = patched + assert Slow.__dict__.get("new_method") is patched + + +def test_class_attr_delattr_mirrored_to_slow(): + # Deleting a class-level attribute on the proxy mirrors the deletion + # onto the slow type (the ``__delattr__`` path). + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.new_method = patched + assert "new_method" in Slow.__dict__ + + del Pxy.new_method + assert "new_method" not in Pxy.__dict__ + assert "new_method" not in Slow.__dict__ + + +def test_class_attr_delattr_existing_removes_from_slow(): + # Deleting is deleting, not restoring: patching an existing attribute + # and then deleting it removes it from both the proxy and the slow type, + # exactly as the same sequence would on a plain Python class. + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + del Pxy.existing + assert "existing" not in Pxy.__dict__ + assert "existing" not in Slow.__dict__ + + +def test_class_attr_restore_existing_slow_attr(): + # Re-assigning the proxy's pristine class-dict entry (as ``monkeypatch`` + # and ``mock.patch`` teardown do) restores the slow type's pristine + # attribute. Note: no prior class-level getattr — the saved descriptor + # is unresolved, as in the ``mock.patch.object`` flow. + _, Slow, Pxy = _make_mirror_proxy() + saved = Pxy.__dict__["existing"] + assert isinstance(saved, _FastSlowAttribute) + original_slow = Slow.__dict__["existing"] + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + Pxy.existing = saved + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_restore_via_saved_method_proxy(): + # Re-assigning a saved ``Pxy.method`` (a ``_MethodProxy`` over the + # resolved slow attribute, not the class-dict descriptor) also restores + # the slow type's pristine attribute. + _, Slow, Pxy = _make_mirror_proxy() + saved = getattr(Pxy, "existing") + original_slow = Slow.__dict__["existing"] + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + Pxy.existing = saved + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_monkeypatch_roundtrip(monkeypatch): + # End-to-end: ``monkeypatch`` of a brand-new attribute mirrors onto the + # slow type, and teardown (which deletes it) mirrors the deletion. + _, Slow, Pxy = _make_mirror_proxy() + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "brand_new", fake, raising=False) + assert Slow.__dict__.get("brand_new") is fake + + monkeypatch.undo() + assert "brand_new" not in Pxy.__dict__ + assert "brand_new" not in Slow.__dict__ + + +def test_class_attr_monkeypatch_existing_roundtrip(monkeypatch): + # End-to-end: ``monkeypatch`` of a pre-existing method mirrors the patch + # onto the slow type, and teardown (which re-assigns the saved proxy + # descriptor) restores the slow type's original implementation. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "existing", fake) + assert Slow.__dict__["existing"] is fake + + monkeypatch.undo() + assert isinstance(Pxy.__dict__["existing"], _FastSlowAttribute) + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_nested_monkeypatch_existing_roundtrip(): + # Nested patches of the same pre-existing method unwind in order, + # each level restoring the slow type to the previous state. + from _pytest.monkeypatch import MonkeyPatch + + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake1(self): + return "fake1" + + def fake2(self): + return "fake2" + + mp1, mp2 = MonkeyPatch(), MonkeyPatch() + mp1.setattr(Pxy, "existing", fake1) + assert Slow.__dict__["existing"] is fake1 + mp2.setattr(Pxy, "existing", fake2) + assert Slow.__dict__["existing"] is fake2 + + mp2.undo() + assert Slow.__dict__["existing"] is fake1 + mp1.undo() + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_mock_patch_object_roundtrip(): + # ``unittest.mock.patch.object`` saves the raw class-dict entry without + # a prior getattr (so the saved descriptor is never resolved); undo must + # still restore the slow type's pristine attribute. + from unittest import mock + + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake(self): + return "fake" + + with mock.patch.object(Pxy, "existing", fake): + assert Slow.__dict__["existing"] is fake + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_property_monkeypatch_roundtrip(monkeypatch): + # Patching a property mirrors it onto the slow type; undo restores the + # slow type's pristine property object. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["prop"] + + fake = property(lambda self: "fake") + monkeypatch.setattr(Pxy, "prop", fake) + assert Slow.__dict__["prop"] is fake + assert Slow().prop == "fake" + + monkeypatch.undo() + assert Slow.__dict__["prop"] is original_slow + assert Slow().prop == "slow prop" + + +def test_class_attr_data_attr_monkeypatch_roundtrip(monkeypatch): + # Patching a plain class data attribute round-trips on the slow type. + _, Slow, Pxy = _make_mirror_proxy() + + monkeypatch.setattr(Pxy, "const", 99) + assert Slow.const == 99 + + monkeypatch.undo() + assert Slow.const == 42 + + +def test_class_attr_staticmethod_monkeypatch_roundtrip(monkeypatch): + # Undo restores the slow type's pristine ``staticmethod`` descriptor, + # not the plain function it resolves to (which would break instance + # calls by receiving ``self``). + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["smethod"] + assert isinstance(original_slow, staticmethod) + + monkeypatch.setattr(Pxy, "smethod", staticmethod(lambda x: x - 1)) + assert Slow.smethod(1) == 0 + + monkeypatch.undo() + assert Slow.__dict__["smethod"] is original_slow + assert Slow().smethod(1) == 2 + + +def test_class_attr_classmethod_monkeypatch_roundtrip(monkeypatch): + # Undo restores the slow type's pristine ``classmethod`` descriptor, + # not the class-bound method it resolves to (which would pin ``cls`` + # for subclasses). + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["cmethod"] + assert isinstance(original_slow, classmethod) + + monkeypatch.setattr(Pxy, "cmethod", classmethod(lambda cls: "fake")) + assert Slow.cmethod() == "fake" + + monkeypatch.undo() + assert Slow.__dict__["cmethod"] is original_slow + assert Slow.cmethod() == "Slow" + + +def test_class_attr_inherited_method_monkeypatch_roundtrip(monkeypatch): + # Patching a method the slow type only inherits mirrors it into the slow + # type's own dict; undo removes that entry again (rather than copying + # the base-class implementation into the subclass), leaving the + # inherited implementation visible. + _, Slow, Pxy = _make_mirror_proxy() + assert "inherited" not in Slow.__dict__ + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "inherited", fake) + assert Slow.__dict__["inherited"] is fake + + monkeypatch.undo() + assert "inherited" not in Slow.__dict__ + assert Slow().inherited() == "base" + + +def test_class_attr_monkeypatch_delattr_roundtrip(monkeypatch): + # ``monkeypatch.delattr`` mirrors the deletion onto the slow type, and + # undo (which re-assigns the saved pristine descriptor) restores the + # slow type's pristine attribute. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + monkeypatch.delattr(Pxy, "existing") + assert "existing" not in Pxy.__dict__ + assert "existing" not in Slow.__dict__ + + monkeypatch.undo() + assert Slow.__dict__["existing"] is original_slow + + +def test_setattr_fsproxy_no_mirror_skips_slow(): + # ``_setattr_fsproxy_no_mirror`` sets a class attribute on the proxy + # without forwarding it to the slow type (used for cudf.pandas' own custom + # methods such as ``DataFrame.query``/``eval``). + _, Slow, Pxy = _make_mirror_proxy() + + def custom(self): + return "custom" + + _setattr_fsproxy_no_mirror(Pxy, "custom_method", custom) + assert Pxy.__dict__["custom_method"] is custom + assert "custom_method" not in Slow.__dict__ + + +def test_setattr_fsproxy_no_mirror_monkeypatch_roundtrip(monkeypatch): + # A cudf-installed custom attribute (e.g. ``DataFrame.eval``/``query``) + # participates in the pristine state: monkeypatching it and undoing + # restores the proxy's custom object AND the slow type's own genuine + # implementation — the cudf-internal object is never forwarded to the + # slow type. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def custom(self): + return "cudf custom" + + _setattr_fsproxy_no_mirror(Pxy, "existing", custom) + assert Slow.__dict__["existing"] is original_slow + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "existing", fake) + assert Slow.__dict__["existing"] is fake + + monkeypatch.undo() + assert Pxy.__dict__["existing"] is custom + assert Slow.__dict__["existing"] is original_slow diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 1185af4f6dc9..44825a801a42 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -45,9 +45,9 @@ classifiers = [ test = [ "dask-cuda==26.10.*,>=0.0.0a0", "pandas>=3.0.0,<3.0.4a0", + "psutil", "pytest-cov", "pytest-httpserver", - "pytest-timeout", "pytest-xdist", "pytest<9.1.0", "rich", @@ -91,7 +91,6 @@ filterwarnings = [ "error", "ignore:Port .* is already in use.:UserWarning", ] -timeout = 45 xfail_strict = true [tool.coverage.report] diff --git a/python/cudf_polars/tests/conftest.py b/python/cudf_polars/tests/conftest.py index 218592d52a39..22e034b37d78 100644 --- a/python/cudf_polars/tests/conftest.py +++ b/python/cudf_polars/tests/conftest.py @@ -339,9 +339,6 @@ def engine_raise_on_fail() -> pl.GPUEngine: def timeout_seconds() -> int: """ Conservative timeout for APIs that accept a timeout parameter. - - Since pytest-timeout is installed, ensure this value is less than timeout - in python/cudf_polars/pyproject.toml. """ return 30 diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index e80bb25679c1..570497147d99 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -358,7 +358,6 @@ def test_rank_over_with_null_values( @pytest.mark.parametrize("method", ["ordinal", "dense", "min", "max", "average"]) @pytest.mark.parametrize("descending", [False, True]) @pytest.mark.parametrize("order_by", [None, ["g2", pl.col("x2") * 2]]) -@pytest.mark.timeout(120) def test_rank_over_with_null_group_keys( engine: pl.GPUEngine, df: pl.LazyFrame, diff --git a/python/cudf_polars/tests/streaming/test_scan.py b/python/cudf_polars/tests/streaming/test_scan.py index 13e88ead7731..15ba081e69fb 100644 --- a/python/cudf_polars/tests/streaming/test_scan.py +++ b/python/cudf_polars/tests/streaming/test_scan.py @@ -73,7 +73,6 @@ def df(): ("parquet", pl.scan_parquet), ], ) -@pytest.mark.timeout(90) def test_parallel_scan( tmp_path: Path, df: pl.DataFrame, diff --git a/python/cudf_polars/tests/streaming/test_sort.py b/python/cudf_polars/tests/streaming/test_sort.py index 71e1535988e3..9707ba9886b6 100644 --- a/python/cudf_polars/tests/streaming/test_sort.py +++ b/python/cudf_polars/tests/streaming/test_sort.py @@ -87,7 +87,6 @@ def large_frames(): ) -@pytest.mark.timeout(120) def test_sort(df, engine): q = df.sort(by=["y", "z"]) assert_gpu_result_equal(q, engine=engine)