From 66e67c6382de8aa225589a3434c0ff9977146cad Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:41:38 +0000 Subject: [PATCH 1/4] Add cuDF JAR build support for all Maven classifiers * Split the JAR build into three composable stages (static libcudf build, per-classifier JAR packaging, and Maven-repo gather) so each stage is independently runnable in CI and locally. * Link against a static libcudf built from source per CUDA version rather than a conda shared libcudf. * Emit the Maven classifier based on the host architecture the build runs on, introducing a new `-arm64` suffix to distinguish aarch64 JARs from their x86_64 counterparts. * Add `test_java_build_local.sh` as a one-command local reproducer of the full CI matrix for the host arch, with per-step timings and GPU compute-capability auto-detection. --- .github/workflows/build.yaml | 70 +++++ dependencies.yaml | 24 ++ java/ci/README.md | 99 +++++- java/ci/argparse.sh | 36 +++ java/ci/assemble_maven_repo.sh | 164 ++++++++++ java/ci/build_cudf_java_jar.sh | 241 ++++++++++++++ java/ci/build_cudf_java_jar_in_container.sh | 116 +++++++ java/ci/build_static_libcudf.sh | 138 ++++++++ java/ci/build_static_libcudf_in_container.sh | 92 ++++++ java/ci/test_java_build_local.sh | 314 +++++++++++++++++++ java/pom.xml | 7 + 11 files changed, 1288 insertions(+), 13 deletions(-) create mode 100644 java/ci/argparse.sh create mode 100755 java/ci/assemble_maven_repo.sh create mode 100755 java/ci/build_cudf_java_jar.sh create mode 100755 java/ci/build_cudf_java_jar_in_container.sh create mode 100755 java/ci/build_static_libcudf.sh create mode 100755 java/ci/build_static_libcudf_in_container.sh create mode 100755 java/ci/test_java_build_local.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f6bddc6cc93..c1a386577155 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -471,3 +471,73 @@ jobs: with: push: true cuda: '["12.9", "13.3"]' + # Build the cuDF Java JAR for every Maven classifier. + java-build: + needs: [telemetry-setup] + strategy: + fail-fast: false + matrix: + include: + - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } + runs-on: ${{ matrix.runner }} + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Build static libcudf + run: | + ./java/ci/build_static_libcudf.sh \ + --output-dir "${RUNNER_TEMP}/libcudf" \ + --cuda-version "${{ matrix.cuda }}" + - name: Build cuDF Java JAR + run: | + ./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir "${RUNNER_TEMP}/libcudf" \ + --output-dir "${RUNNER_TEMP}/jars" \ + --cuda-version "${{ matrix.cuda }}" + - name: Upload per-entry JAR artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} + # Ship only the JARs and POMs. Exclude the per-classifier Maven build scratch dir. + path: | + ${{ runner.temp }}/jars + !${{ runner.temp }}/jars/.mvn-temp-target + if-no-files-found: error + # Assemble the per-classifier JARs into one Maven-repository layout. + java-gather: + needs: [java-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Download per-entry JAR artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cudf_java_cuda* + path: ${{ runner.temp }}/jars + merge-multiple: true + - name: Assemble Maven repository layout + run: | + ./java/ci/assemble_maven_repo.sh \ + --jars-dir "${RUNNER_TEMP}/jars" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload combined Maven repository artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error diff --git a/dependencies.yaml b/dependencies.yaml index f054189a8d37..6c6c9ac67996 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -129,6 +129,19 @@ files: - depends_on_cudf_polars - depends_on_ray - depends_on_cudf_streaming + build_java: + # Toolchain for building static libcudf from source and packaging the cuDF + # Java JAR. `depends_on_libcudf` is excluded because the Java build links + # against a static libcudf built from source, not a conda shared libcudf. + output: none + includes: + - build_base + - build_all + - build_cpp + - cuda + - cuda_static + - cuda_version + - build_java test_java: output: none includes: @@ -632,6 +645,17 @@ dependencies: - croaring==4.4.2 - flatbuffers==24.3.25 - librdkafka<2.15.0a0 + build_java: + common: + - output_types: conda + packages: + - boost + # cuda_profiler_api.h is used by the JNI layer (CudaJni.cpp) but is + # not pulled in by the base `cuda` dev packages. + - cuda-profiler-api + - make + - maven + - openjdk=8.* depends_on_libnvcomp: common: - output_types: conda diff --git a/java/ci/README.md b/java/ci/README.md index 5ef4850a2bb1..07415daa0453 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -1,11 +1,92 @@ # Build Jar artifact of cuDF -## Build the docker image +## Recommended: self-contained release build scripts -### Prerequisite +The scripts under `ci/` build the cuDF Java JAR for every Maven classifier the +same way locally and in CI (GitHub Actions is only a thin wrapper that adds +artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image, +runs the build in a throwaway container, and writes its output to a host +directory. No local `docker build` is required, and no GPU is required to build. -1. Docker should be installed. -2. [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) should be installed. +### Prerequisites + +1. Docker is installed and the current user can run `docker`. +2. Network access to pull `rapidsai/ci-conda:-latest`. + +### Step 1 - Build the static libcudf install tree + +```bash +./java/ci/build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version 12.9 +``` + +This produces a static libcudf install tree (`lib/libcudf.a` plus its static +dependencies) under the given output directory. Build outputs are host-user-owned +so plain `rm -rf` works. + +### Step 2 - Package the cuDF Java JAR for one classifier + +```bash +./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir /tmp/libcudf-cuda12 \ + --output-dir /tmp/jars \ + --cuda-version 12.9 +``` + +This compiles the JNI layer against the static libcudf from Step 1 and emits a +single classifier JAR (e.g. `cudf-26.08.0-SNAPSHOT-cuda12.jar`) plus its POM +into a classifier-named subdirectory under `--output-dir`: + +``` +/tmp/jars/cuda12/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The classifier is derived from `--cuda-version` (major) + host arch (`uname +-m`): `cuda12` / `cuda13` on `x86_64`, `cuda12-arm64` / `cuda13-arm64` on +`aarch64`. Producing the ARM classifiers requires a real `aarch64` host. +Repeat Step 2 for each classifier, pointing `--libcudf-dir` at the matching +static libcudf tree and using the same `--output-dir` (each classifier lands +in its own subdirectory). Concurrent invocations for different classifiers +are safe because each nests its own bind-mount over `/repo/java/target` +inside the container. + +### Step 3 - Assemble the Maven repository layout + +```bash +./java/ci/assemble_maven_repo.sh \ + --jars-dir /tmp/jars \ + --output-dir /tmp/maven-repo +``` + +This walks every subdirectory of `--jars-dir` (each subdir name IS the +classifier), gathers the per-classifier JAR and shared POM, derives the +artifact version from the JAR filenames (requiring a single unique version +across subdirs), and lays them out as: + +``` +/tmp/maven-repo/ai/rapids/cudf/26.08.0-SNAPSHOT/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT-cuda13.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The set of classifiers is whatever subdirectories are present under +`--jars-dir`. For a local `x86_64`-only run, populate `/tmp/jars/cuda12/` +and `/tmp/jars/cuda13/`. For the full four-way release build, add +`/tmp/jars/cuda12-arm64/` and `/tmp/jars/cuda13-arm64/`. + +In GitHub Actions (`.github/workflows/build.yaml`), the `java-build` matrix job +runs Steps 1-2 per (CUDA x arch) entry and uploads each classifier subdir as a +per-entry artifact. The separate `java-gather` job downloads them (with +`merge-multiple: true`, so all subdirs land in a single parent dir), runs +Step 3, and uploads the combined `cudf_java_maven_repo` artifact. + +## Legacy: manual Dockerfile.rocky build (obsolete) + +> The `java/ci/Dockerfile.rocky` + `java/ci/build-in-docker.sh` flow below is the +> old build path. It is retained for reference but superseded by the +> self-contained scripts above. ### Build the docker image @@ -20,25 +101,19 @@ The following CUDA versions are supported w/ CUDA Enhanced Compatibility: Change the --build-arg CUDA_VERSION to what you need. You can replace the tag "cudf-build:12.9.1-devel-rocky8" with another name you like. -## Start the docker then build - -### Start the docker +### Start the docker then build Run below command to start a docker container with GPU. ```bash nvidia-docker run -it cudf-build:12.9.1-devel-rocky8 bash ``` -### Download the cuDF source code - You can download the cuDF repo in the docker container or you can mount it into the container. Here I choose to download again in the container. ```bash git clone --recursive https://github.com/rapidsai/cudf.git -b main ``` -### Build cuDF jar with devtoolset - ```bash cd cudf export WORKSPACE=`pwd` @@ -46,6 +121,4 @@ source java/ci/env.sh ${sclCMD} "java/ci/build-in-docker.sh" ``` -### The output - You can find the cuDF jar in java/target/ like cudf-26.08.0-SNAPSHOT-cuda12.jar. diff --git a/java/ci/argparse.sh b/java/ci/argparse.sh new file mode 100644 index 000000000000..9a06d35c9df8 --- /dev/null +++ b/java/ci/argparse.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared argparse helpers for the java/ci/ host orchestrator scripts. +# Meant to be sourced, not executed: +# . "${SCRIPT_DIR}/argparse.sh" + +# require_value +# Check: exit 1 when is empty (i.e. the flag was passed +# without its argument, or was the last token on the command line). +require_value() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} requires a value" >&2 + exit 1 + fi +} + +# require_arg +# Check: assert that a required flag was actually supplied by the +# caller. Prints the script's print_help (if defined) then exits 1 on failure. +# Preserves the existing behavior of showing help after a "required flag missing" +# error. +require_arg() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} is required." >&2 + if declare -F print_help > /dev/null; then + print_help + fi + exit 1 + fi +} diff --git a/java/ci/assemble_maven_repo.sh b/java/ci/assemble_maven_repo.sh new file mode 100755 index 000000000000..ab9ebf3ab89c --- /dev/null +++ b/java/ci/assemble_maven_repo.sh @@ -0,0 +1,164 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Decoupled gather step: assemble per-classifier cuDF Java JARs into a single +# Maven-repository-layout directory. +# +# Input: --jars-dir contains one subdirectory per classifier, each holding +# exactly one cudf--.jar and a cudf-.pom. Subdir +# names ARE the classifier names, and the artifact version is derived from +# the JAR filenames (all subdirs must agree). +# +# Output layout: +# /ai/rapids/cudf//cudf--.jar +# /ai/rapids/cudf//cudf-.pom + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +GROUP_PATH="ai/rapids" +ARTIFACT_ID="cudf" + +JARS_DIR="" +OUTPUT_DIR="" + +print_help() { + cat << EOF + +Usage: assemble_maven_repo.sh --jars-dir --output-dir + +Gathers per-classifier cuDF Java JARs into a single Maven-repository-layout tree. + +REQUIRED: + -j, --jars-dir Parent directory containing one subdirectory per + classifier (each holding cudf--.jar + and cudf-.pom). Subdir name is the classifier. + -o, --output-dir Directory to receive the combined Maven-repository layout. + +OPTIONS: + -h, --help Show this help message. + +EXAMPLE: + assemble_maven_repo.sh --jars-dir /tmp/jars --output-dir /tmp/maven-repo + # given /tmp/jars/{cuda12,cuda13}/ inputs, produces: + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda12.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda13.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -j|--jars-dir) + require_value "$1" "$2" + JARS_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --jars-dir "${JARS_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" + +if [[ ! -d ${JARS_DIR} ]]; then + echo "Error: --jars-dir '${JARS_DIR}' does not exist." + exit 1 +fi + +echo "Assembling Maven repository layout" +echo " jars dir: ${JARS_DIR}" +echo " output dir: ${OUTPUT_DIR}" + +# Walk every classifier subdirectory. Each subdir must contain exactly one +# cudf-*-.jar. The version is derived from the filename and must +# match across all subdirs. +FIRST_VERSION="" +CLASSIFIERS_SEEN="" + +for subdir in "${JARS_DIR}"/*/; do + classifier=$(basename "${subdir}") + + jar="" + for candidate in "${subdir}"cudf-*-"${classifier}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${jar}" ]]; then + echo "Error: multiple JARs in ${subdir} match cudf-*-${classifier}.jar" >&2 + exit 1 + fi + jar=${candidate} + fi + done + + if [[ -z "${jar}" ]]; then + echo "Error: no cudf-*-${classifier}.jar found in ${subdir}" >&2 + exit 1 + fi + + # Filename is cudf--.jar. Peel prefix and suffix. + base=$(basename "${jar}" .jar) + version=$(echo "${base}" | sed -e 's/^cudf-//' -e "s/-${classifier}$//") + + if [[ -z "${FIRST_VERSION}" ]]; then + FIRST_VERSION=${version} + elif [[ "${version}" != "${FIRST_VERSION}" ]]; then + echo "Error: inconsistent versions across subdirs: ${FIRST_VERSION} vs ${version} (${subdir})" >&2 + exit 1 + fi + + DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${version}" + mkdir -p "${DEST_DIR}" + cp -f "${jar}" "${DEST_DIR}/" + echo " + $(basename "${jar}")" + CLASSIFIERS_SEEN="${CLASSIFIERS_SEEN} ${classifier}" +done + +if [[ -z "${FIRST_VERSION}" ]]; then + echo "Error: no classifier subdirs found under ${JARS_DIR}" >&2 + exit 1 +fi + +# POM is identical across subdirs; copy the first one found. +POM_SRC="" +for subdir in "${JARS_DIR}"/*/; do + candidate=${subdir}cudf-${FIRST_VERSION}.pom + if [[ -f "${candidate}" ]]; then + POM_SRC=${candidate} + break + fi +done + +if [[ -z "${POM_SRC}" ]]; then + echo "Error: no cudf-${FIRST_VERSION}.pom found under ${JARS_DIR}" >&2 + exit 1 +fi + +DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${FIRST_VERSION}" +cp -f "${POM_SRC}" "${DEST_DIR}/cudf-${FIRST_VERSION}.pom" +echo " + cudf-${FIRST_VERSION}.pom" + +echo "Maven repository assembled successfully at ${OUTPUT_DIR}" +echo "Classifiers present:${CLASSIFIERS_SEEN}" diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh new file mode 100755 index 000000000000..1f5fc87bca40 --- /dev/null +++ b/java/ci/build_cudf_java_jar.sh @@ -0,0 +1,241 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained packaging of the cuDF Java JAR for a single classifier. +# +# Consumes a prebuilt static libcudf install tree (from build_static_libcudf.sh), +# compiles the JNI layer against it inside a throwaway RAPIDS ci-conda container, +# and emits the single classifier JAR (plus its POM) to a per-classifier +# subdirectory under --output-dir. This script is layout-agnostic: it produces +# one classifier's artifacts and knows nothing about the combined +# Maven-repository layout (see java/ci/assemble_maven_repo.sh). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +LIBCUDF_DIR="" +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_cudf_java_jar.sh --libcudf-dir --output-dir \\ + --cuda-version [OPTIONS] + +Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-conda +container, linking against a prebuilt static libcudf. Always builds for the +host architecture (uname -m). The build image is fixed to +rapidsai/ci-conda:-latest (version derived from the VERSION +file). + +The classifier is derived from --cuda-version (major) + host arch (uname -m), +mirroring the pom.xml Groovy logic: "cuda" for x86_64, +"cuda-arm64" for aarch64. The classifier JAR + POM are written to +//. Concurrent invocations targeting different +classifiers are safe. + +REQUIRED: + -l, --libcudf-dir Static libcudf install tree produced by + build_static_libcudf.sh. + -o, --output-dir Host parent directory. The script creates and writes + to //. + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + Must match --cuda-version of the static libcudf tree; + determines the cuda12/cuda13 classifier. + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. Must match the value passed to + build_static_libcudf.sh when producing the static + libcudf tree in --libcudf-dir, or device linking of + libcudfjni.so against libcudf.a will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_cudf_java_jar.sh -l /tmp/libcudf-cuda12 -o /tmp/jars -c 12.9 + build_cudf_java_jar.sh -l /tmp/libcudf-cuda13 -o /tmp/jars -c 13.3 -A 80 + # writes: + # /tmp/jars/cuda12/cudf--cuda12.jar + # /tmp/jars/cuda12/cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -l|--libcudf-dir) + require_value "$1" "$2" + LIBCUDF_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --libcudf-dir "${LIBCUDF_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +if [[ ! -d ${LIBCUDF_DIR} ]]; then + echo "Error: --libcudf-dir '${LIBCUDF_DIR}' does not exist." + exit 1 +fi + +# Derive the Maven classifier from --cuda-version major + host arch, mirroring +# the pom.xml Groovy logic: "cuda" for x86_64, "cuda-arm64" for +# aarch64. +CUDA_MAJOR="$(echo "${CUDA_VERSION}" | cut -d. -f1)" +HOST_ARCH="$(uname -m)" +case "${HOST_ARCH}" in + x86_64) + CLASSIFIER="cuda${CUDA_MAJOR}" + ;; + aarch64|arm64) + CLASSIFIER="cuda${CUDA_MAJOR}-arm64" + ;; + *) + echo "Error: Unsupported host arch '${HOST_ARCH}' (expected x86_64 or aarch64)" >&2 + exit 1 + ;; +esac + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" +LIBCUDF_DIR="$(cd "${LIBCUDF_DIR}" && pwd)" + +CLASSIFIER_OUT="${OUTPUT_DIR}/${CLASSIFIER}" +mkdir -p "${CLASSIFIER_OUT}" + +# Per-classifier scratch dir for Maven's java/target/. Nested bind-mount over +# /repo/java/target inside the container isolates concurrent invocations +# (each classifier gets its own target/). The `.mvn-temp-target/` prefix +# keeps this dir invisible to the default `*/` globbing in +# assemble_maven_repo.sh's classifier discovery loop. +# +# Recreate the scratch on every launch: the in-container mvn cannot clean a +# bind-mount point (rmdir on /repo/java/target fails with EBUSY), so the +# host wrapper is responsible for guaranteeing a clean starting target/. +TARGET_SCRATCH="${OUTPUT_DIR}/.mvn-temp-target/${CLASSIFIER}" +rm -rf "${TARGET_SCRATCH}" +mkdir -p "${TARGET_SCRATCH}" + +echo "Packaging cuDF Java JAR" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " classifier: ${CLASSIFIER}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " libcudf dir: ${LIBCUDF_DIR}" +echo " output dir: ${CLASSIFIER_OUT}" +echo " target dir: ${TARGET_SCRATCH}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${LIBCUDF_DIR}:/libcudf:ro" + --volume "${CLASSIFIER_OUT}:/output" + --volume "${TARGET_SCRATCH}:/repo/java/target" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_cudf_java_jar_in_container.sh + +# Post-run: assert exactly one main classifier JAR + one POM, and that the +# JAR's classifier suffix matches the subdir name we chose (catches pom drift). +PRODUCED_JAR="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*-"${CLASSIFIER}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_JAR}" ]]; then + echo "Error: multiple JARs matching cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_JAR=${candidate} + fi +done + +if [[ -z "${PRODUCED_JAR}" ]]; then + echo "Error: no cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +PRODUCED_POM="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*.pom; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_POM}" ]]; then + echo "Error: multiple POMs found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_POM=${candidate} + fi +done + +if [[ -z "${PRODUCED_POM}" ]]; then + echo "Error: no cudf-*.pom found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +echo "cuDF Java JAR build succeeded:" +echo " $(basename "${PRODUCED_JAR}")" +echo " $(basename "${PRODUCED_POM}")" diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh new file mode 100755 index 000000000000..231d5ac2d498 --- /dev/null +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container packaging of the cuDF Java JAR for a single classifier. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_cudf_java_jar.sh. It generates the build_java conda toolchain +# environment, compiles the JNI layer against a prebuilt static libcudf +# (mounted at /libcudf), and packages the cuDF Java JAR. The resulting +# classifier JAR and its POM are copied to /output. Then chowns /output and +# /repo/java/target to HOST_UID:HOST_GID so the host user owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output and /repo/java/target +# (both required). + +set -e + +OUTPUT_DIR=/output +REPO_ROOT=/repo +CUDF_INSTALL_DIR=/libcudf + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +BUILD_ARG=( + -B + "-Dmaven.repo.local=/tmp/.m2" + "-Dparallel.level=${PARALLEL_LEVEL}" + "-DskipTests=true" + "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON" + "-DCUDF_JNI_LIBCUDF_STATIC=ON" + "-DUSE_GDS=OFF" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + BUILD_ARG+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +cd "${REPO_ROOT}/java" + +CUDF_VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" +rapids-logger "Packaging cuDF Java JAR version ${CUDF_VERSION} (libcudf: ${CUDF_INSTALL_DIR})" + +# The `clean` goal is intentionally omitted: /repo/java/target is a +# bind-mount point, so when `mvn clean` attempts to remove the directory, +# it fails with EBUSY. The host wrapper (build_cudf_java_jar.sh) recreates +# the scratch dir before each container launch to guarantee target/ starts empty. +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" mvn package "${BUILD_ARG[@]}" + +MAIN_JAR="" +for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do + case "${candidate}" in + *-tests.jar|*-sources.jar|*-javadoc.jar) + continue + ;; + esac + if [[ -f ${candidate} ]]; then + MAIN_JAR=${candidate} + break + fi +done + +if [[ -z ${MAIN_JAR} ]]; then + echo "Error: no cuDF classifier JAR produced under target/" + ls -l target/ || true + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" +cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" +cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" + +rapids-logger "Emitted $(basename "${MAIN_JAR}") + cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" + +if [[ -n ${HOST_UID} && -n ${HOST_GID} ]]; then + rapids-logger "Chowning ${OUTPUT_DIR} and ${REPO_ROOT}/java/target to ${HOST_UID}:${HOST_GID}" + chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" + chown -R "${HOST_UID}:${HOST_GID}" "${REPO_ROOT}/java/target" +fi diff --git a/java/ci/build_static_libcudf.sh b/java/ci/build_static_libcudf.sh new file mode 100755 index 000000000000..2fa2e435471c --- /dev/null +++ b/java/ci/build_static_libcudf.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained build of a static libcudf install tree. +# +# Pulls the RAPIDS ci-conda image, builds libcudf with BUILD_SHARED_LIBS=OFF +# inside a throwaway container, and installs the static libcudf tree (libcudf.a +# plus its static dependencies) into a directory on the host. No GPU is required +# to build. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_static_libcudf.sh --output-dir --cuda-version [OPTIONS] + +Builds a static libcudf install tree inside a RAPIDS ci-conda container and +writes it to a directory on the host. Always builds for the host architecture +(uname -m). The build image is fixed to rapidsai/ci-conda:-latest +(version derived from the VERSION file). + +REQUIRED: + -o, --output-dir Host directory to receive the static install tree + (libcudf.a and its static dependencies). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. When packaging the cuDF Java JAR + against this static libcudf tree, pass the same value + to build_cudf_java_jar.sh --cmake-cuda-architectures + or device linking of libcudfjni.so against libcudf.a + will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version "12.9" + build_static_libcudf.sh -o /tmp/libcudf-cuda13 -c 13.3 -A "80" + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" + +echo "Building static libcudf" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " output dir: ${OUTPUT_DIR}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${OUTPUT_DIR}:/output" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_static_libcudf_in_container.sh + +if [[ -f "${OUTPUT_DIR}/lib/libcudf.a" || -f "${OUTPUT_DIR}/lib64/libcudf.a" ]]; then + echo "Static libcudf build succeeded: ${OUTPUT_DIR}" +else + echo "Error: expected libcudf.a not found under ${OUTPUT_DIR}/lib or ${OUTPUT_DIR}/lib64" + exit 1 +fi diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh new file mode 100755 index 000000000000..579973399b36 --- /dev/null +++ b/java/ci/build_static_libcudf_in_container.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container build of a static libcudf install tree. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_static_libcudf.sh. It generates the build_java conda toolchain +# environment, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs the +# resulting static libcudf (plus its static dependencies) into /output. Then +# chowns /output to HOST_UID:HOST_GID so the host user owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output (both required). + +set -e + +INSTALL_PREFIX=/output +REPO_ROOT=/repo +BUILD_DIR=/tmp/libcudf-build + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +CMAKE_ARGS=( + -S "${REPO_ROOT}/cpp" + -B "${BUILD_DIR}" + -GNinja + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTS=OFF + -DUSE_NVTX=ON + -DCUDF_LARGE_STRINGS_DISABLED=ON + -DCUDF_USE_ARROW_STATIC=ON + -DCUDF_ENABLE_ARROW_S3=OFF + -DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON + -DRMM_LOGGING_LEVEL=OFF + -DCUDF_KVIKIO_REMOTE_IO=OFF +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + CMAKE_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +rapids-logger "Configuring static libcudf" +cmake "${CMAKE_ARGS[@]}" + +rapids-logger "Building static libcudf with ${PARALLEL_LEVEL} jobs" +cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" + +rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" +cmake --install "${BUILD_DIR}" + +if [[ -n ${HOST_UID} && -n ${HOST_GID} ]]; then + rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" + chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" +fi diff --git a/java/ci/test_java_build_local.sh b/java/ci/test_java_build_local.sh new file mode 100755 index 000000000000..b4467da1109f --- /dev/null +++ b/java/ci/test_java_build_local.sh @@ -0,0 +1,314 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Local end-to-end verification of the build workflow: builds the static +# libcudf install tree and the classifier JAR for both CUDA 12 and CUDA 13, +# then runs the decoupled gather step to assemble the combined +# Maven-repository layout. Mirrors what the java-build matrix + java-gather +# jobs in .github/workflows/build.yaml do in CI. +# +# Runs on either x86_64 or aarch64 hosts. Each invocation covers only the +# host architecture: on x86_64 it produces the "cuda12" and "cuda13" +# classifier JARs; on aarch64 it produces "cuda12-arm64" and "cuda13-arm64". +# The arm64 suffix is added automatically by the child build scripts (via +# pom.xml's classifier logic keyed off `uname -m`). To cover all four +# release classifiers, run this script once on each architecture. +# +# Both static libcudf builds run in parallel, and both JAR builds run in +# parallel (each uses a nested bind-mount over /repo/java/target inside the +# container to isolate Maven output). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +WORK_DIR="" +PARALLEL_LEVEL="$(nproc)" +CMAKE_CUDA_ARCHITECTURES="" + +# CUDA versions built for both classifiers. Keep in sync with the java-build +# matrix in .github/workflows/build.yaml. +CUDA12_VERSION="12.9" +CUDA13_VERSION="13.3" + +# Wall-clock timing variables. +STEP_NAMES=() +STEP_ELAPSED=() + +print_help() { + cat << EOF + +Usage: test_java_build_local.sh --work-dir [OPTIONS] + +Runs the full build+gather pipeline for both CUDA 12 and CUDA 13 on the host +architecture (x86_64 or aarch64) and assembles the combined Maven-repository +layout. + +REQUIRED: + -w, --work-dir Scratch directory for build outputs. Subtrees created: + /libcudf-cuda12 static libcudf (CUDA 12) + /libcudf-cuda13 static libcudf (CUDA 13) + /jars/ per-classifier JAR + POM + /maven-repo combined Maven layout + where is "cuda12" / "cuda13" on x86_64 + and "cuda12-arm64" / "cuda13-arm64" on aarch64. + +OPTIONS: + -j, --parallel Total build parallelism (default: nproc = ${PARALLEL_LEVEL}). + Each concurrent JAR build gets --parallel/2 to avoid + RAM pressure from two parallel nvcc runs. + -A, --cmake-cuda-architectures + CUDA architecture list (e.g. "80" or "80;90") passed to + both build scripts. The literal value "all" is a + sentinel meaning "do not pass --cmake-cuda-architectures + to child scripts" — child scripts then fall back to + cuDF's default RAPIDS full architecture list (slow). + Default: auto-detect the local GPU's compute + capability via nvidia-smi (e.g. Ampere -> "80"). If + nvidia-smi is missing or returns nothing, falls back + to "all". + -h, --help Show this help message. + +EXAMPLES: + # Fast run (auto-detect local GPU arch): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test + + # Explicit override: + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures 80 + + # Full RAPIDS arch list (slow, e.g. GPU-less host): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures all + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -w|--work-dir) + require_value "$1" "$2" + WORK_DIR=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +log_step() { + echo + echo "============================================================" + echo "== $1" + echo "============================================================" +} + +format_elapsed() { + local s=$1 + printf '%dm %02ds' $((s/60)) $((s%60)) +} + +record_step_end() { + local name=$1 + local start=$2 + local elapsed=$((SECONDS - start)) + STEP_NAMES+=("${name}") + STEP_ELAPSED+=("${elapsed}") + echo + echo "== ${name} completed in $(format_elapsed "${elapsed}")" +} + +parse_args "$@" + +require_arg --work-dir "${WORK_DIR}" + +mkdir -p "${WORK_DIR}" +WORK_DIR="$(cd "${WORK_DIR}" && pwd)" +LOG_DIR="${WORK_DIR}/logs" +mkdir -p "${LOG_DIR}" + +# Remove outputs from any prior run so cmake/mvn does not see stale artifacts. +rm -rf "${WORK_DIR}/libcudf-cuda12" \ + "${WORK_DIR}/libcudf-cuda13" \ + "${WORK_DIR}/jars" \ + "${WORK_DIR}/maven-repo" + +# Auto-detect the local GPU's compute capability when the flag was not passed. +# "all" is the sentinel value that means "do not forward this flag to child +# scripts". Child scripts fall back to cuDF's default RAPIDS architecture +# list (slow but correct on GPU-less hosts). +if [[ -z ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DETECTED="" + if command -v nvidia-smi > /dev/null 2>&1; then + DETECTED=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' | tr -d ' ') + fi + if [[ -n ${DETECTED} ]]; then + CMAKE_CUDA_ARCHITECTURES=${DETECTED} + echo "Auto-detected local GPU compute capability: ${CMAKE_CUDA_ARCHITECTURES}" + else + CMAKE_CUDA_ARCHITECTURES="all" + echo "No local GPU detected; falling back to cuDF's default RAPIDS architecture list" + fi +fi + +echo "cuDF Java local build verification" +echo " host arch: $(uname -m)" +echo " work dir: ${WORK_DIR}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " cuda12 version: ${CUDA12_VERSION}" +echo " cuda13 version: ${CUDA13_VERSION}" +echo " cmake cuda architectures: ${CMAKE_CUDA_ARCHITECTURES}" +echo " logs: ${LOG_DIR}/{static,jar}_cuda{12,13}.log" + +# When forwarding to child scripts, "all" means "don't pass the flag". +CHILD_CMAKE_ARGS=() +if [[ ${CMAKE_CUDA_ARCHITECTURES} != "all" ]]; then + CHILD_CMAKE_ARGS=(--cmake-cuda-architectures "${CMAKE_CUDA_ARCHITECTURES}") +fi + +# JAR builds run in parallel; give each half of the CPU budget. +JAR_PARALLEL=$((PARALLEL_LEVEL / 2)) +if [[ ${JAR_PARALLEL} -lt 1 ]]; then + JAR_PARALLEL=1 +fi + +# Step 1: static libcudf builds in parallel. +log_step "Step 1: building static libcudf for CUDA 12 and CUDA 13 in parallel" +STEP1_START=${SECONDS} + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda12" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${PARALLEL_LEVEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda12.log" 2>&1 & +STATIC_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda13" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${PARALLEL_LEVEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda13.log" 2>&1 & +STATIC_CUDA13_PID=$! + +echo " cuda12 static pid: ${STATIC_CUDA12_PID} (tail -f ${LOG_DIR}/static_cuda12.log)" +echo " cuda13 static pid: ${STATIC_CUDA13_PID} (tail -f ${LOG_DIR}/static_cuda13.log)" + +STATIC_CUDA12_RC=0 +STATIC_CUDA13_RC=0 +if ! wait "${STATIC_CUDA12_PID}"; then + STATIC_CUDA12_RC=1 +fi +if ! wait "${STATIC_CUDA13_PID}"; then + STATIC_CUDA13_RC=1 +fi + +if [[ ${STATIC_CUDA12_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 12 build failed." + echo "See ${LOG_DIR}/static_cuda12.log" +fi +if [[ ${STATIC_CUDA13_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 13 build failed." + echo "See ${LOG_DIR}/static_cuda13.log" +fi +if [[ ${STATIC_CUDA12_RC} -ne 0 || ${STATIC_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 1: static libcudf (parallel run)" "${STEP1_START}" + +# Step 2: JAR builds in parallel. Each build's container nests a bind-mount +# over /repo/java/target so concurrent Maven runs don't clobber each other. +log_step "Step 2: packaging cuDF Java JARs for cuda12 and cuda13 in parallel" +STEP2_START=${SECONDS} + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda12" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${JAR_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda12.log" 2>&1 & +JAR_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda13" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${JAR_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda13.log" 2>&1 & +JAR_CUDA13_PID=$! + +echo " cuda12 jar pid: ${JAR_CUDA12_PID} (tail -f ${LOG_DIR}/jar_cuda12.log)" +echo " cuda13 jar pid: ${JAR_CUDA13_PID} (tail -f ${LOG_DIR}/jar_cuda13.log)" + +JAR_CUDA12_RC=0 +JAR_CUDA13_RC=0 +if ! wait "${JAR_CUDA12_PID}"; then + JAR_CUDA12_RC=1 +fi +if ! wait "${JAR_CUDA13_PID}"; then + JAR_CUDA13_RC=1 +fi + +if [[ ${JAR_CUDA12_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 12 build failed." + echo "See ${LOG_DIR}/jar_cuda12.log" +fi +if [[ ${JAR_CUDA13_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 13 build failed." + echo "See ${LOG_DIR}/jar_cuda13.log" +fi +if [[ ${JAR_CUDA12_RC} -ne 0 || ${JAR_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 2: JAR builds (parallel run)" "${STEP2_START}" + +# Step 3: gather into a combined Maven-repository layout. +log_step "Step 3: assembling combined Maven-repository layout" +STEP3_START=${SECONDS} + +"${SCRIPT_DIR}/assemble_maven_repo.sh" \ + --jars-dir "${WORK_DIR}/jars" \ + --output-dir "${WORK_DIR}/maven-repo" + +record_step_end "Step 3: assemble Maven repo" "${STEP3_START}" + +# Derive the assembled version from the output tree for display purposes. +CUDF_VERSION=$(basename "$(ls -d "${WORK_DIR}/maven-repo/ai/rapids/cudf"/*/ | head -1)") + +log_step "Success" +echo "Combined Maven repository:" +echo " ${WORK_DIR}/maven-repo/ai/rapids/cudf/${CUDF_VERSION}/" +echo +echo "Timings:" +for i in "${!STEP_NAMES[@]}"; do + printf ' %-45s %s\n' "${STEP_NAMES[$i]}" "$(format_elapsed "${STEP_ELAPSED[$i]}")" +done +printf ' %-45s %s\n' "Total wall time" "$(format_elapsed "${SECONDS}")" diff --git a/java/pom.xml b/java/pom.xml index ee2a8cf7a603..44d957cab553 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -649,6 +649,13 @@ def cm = cudaPattern.matcher(nvccout) if (cm.find()) { def classifier = 'cuda' + cm.group(1) + // Emit "cuda" on x86_64 and "cuda-arm64" on + // aarch64 so a single pom produces a distinct Maven + // classifier per architecture. + def osArch = System.getProperty('os.arch') + if (osArch == 'aarch64' || osArch == 'arm64') { + classifier = classifier + '-arm64' + } project.properties['cuda.classifier'] = classifier } else { throw new RuntimeException('could not find CUDA version') From 4e8d9db847dfba4fa559ae031df68b99c4a32856 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:19:26 +0000 Subject: [PATCH 2/4] Address coderabbit comments --- java/ci/README.md | 10 +++++++++- java/ci/assemble_maven_repo.sh | 16 +++++++++++++++ java/ci/build_cudf_java_jar.sh | 7 ++++++- java/ci/build_cudf_java_jar_in_container.sh | 21 ++++++++++++-------- java/ci/build_static_libcudf_in_container.sh | 11 ++++++---- java/ci/test_java_build_local.sh | 17 ++++++++-------- 6 files changed, 60 insertions(+), 22 deletions(-) diff --git a/java/ci/README.md b/java/ci/README.md index 07415daa0453..219e08e5eaff 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -2,7 +2,7 @@ ## Recommended: self-contained release build scripts -The scripts under `ci/` build the cuDF Java JAR for every Maven classifier the +The scripts under `java/ci/` build the cuDF Java JAR for every Maven classifier the same way locally and in CI (GitHub Actions is only a thin wrapper that adds artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image, runs the build in a throwaway container, and writes its output to a host @@ -13,6 +13,14 @@ directory. No local `docker build` is required, and no GPU is required to build. 1. Docker is installed and the current user can run `docker`. 2. Network access to pull `rapidsai/ci-conda:-latest`. +### Local one-command shortcut + +For local testing only, `java/ci/test_java_build_local.sh` runs Steps 1-3 end-to-end for both CUDA 12 and CUDA 13 on the host architecture. + +```bash +./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test +``` + ### Step 1 - Build the static libcudf install tree ```bash diff --git a/java/ci/assemble_maven_repo.sh b/java/ci/assemble_maven_repo.sh index ab9ebf3ab89c..b1d3e65a457c 100755 --- a/java/ci/assemble_maven_repo.sh +++ b/java/ci/assemble_maven_repo.sh @@ -89,6 +89,20 @@ if [[ ! -d ${JARS_DIR} ]]; then exit 1 fi +if [[ -e ${OUTPUT_DIR} && -n "$(ls -A "${OUTPUT_DIR}" 2>/dev/null)" ]]; then + echo "Error: --output-dir '${OUTPUT_DIR}' must be empty or nonexistent" >&2 + exit 1 +fi + +ASSEMBLE_FINISHED=0 +cleanup_partial_output() { + if [[ ${ASSEMBLE_FINISHED} -eq 0 && -e ${OUTPUT_DIR} ]]; then + echo "Assembly did not complete; removing partial output at ${OUTPUT_DIR}" >&2 + rm -rf "${OUTPUT_DIR}" + fi +} +trap cleanup_partial_output EXIT + echo "Assembling Maven repository layout" echo " jars dir: ${JARS_DIR}" echo " output dir: ${OUTPUT_DIR}" @@ -162,3 +176,5 @@ echo " + cudf-${FIRST_VERSION}.pom" echo "Maven repository assembled successfully at ${OUTPUT_DIR}" echo "Classifiers present:${CLASSIFIERS_SEEN}" + +ASSEMBLE_FINISHED=1 diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh index 1f5fc87bca40..084b00c68478 100755 --- a/java/ci/build_cudf_java_jar.sh +++ b/java/ci/build_cudf_java_jar.sh @@ -47,7 +47,8 @@ REQUIRED: -l, --libcudf-dir Static libcudf install tree produced by build_static_libcudf.sh. -o, --output-dir Host parent directory. The script creates and writes - to //. + to //, which must not already + exist. -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). Must match --cuda-version of the static libcudf tree; determines the cuda12/cuda13 classifier. @@ -151,6 +152,10 @@ OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" LIBCUDF_DIR="$(cd "${LIBCUDF_DIR}" && pwd)" CLASSIFIER_OUT="${OUTPUT_DIR}/${CLASSIFIER}" +if [[ -e ${CLASSIFIER_OUT} ]]; then + echo "Error: classifier output '${CLASSIFIER_OUT}' already exists. Remove it before re-running." >&2 + exit 1 +fi mkdir -p "${CLASSIFIER_OUT}" # Per-classifier scratch dir for Maven's java/target/. Nested bind-mount over diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh index 231d5ac2d498..98fce46c4f4a 100755 --- a/java/ci/build_cudf_java_jar_in_container.sh +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -8,8 +8,9 @@ # java/ci/build_cudf_java_jar.sh. It generates the build_java conda toolchain # environment, compiles the JNI layer against a prebuilt static libcudf # (mounted at /libcudf), and packages the cuDF Java JAR. The resulting -# classifier JAR and its POM are copied to /output. Then chowns /output and -# /repo/java/target to HOST_UID:HOST_GID so the host user owns the outputs. +# classifier JAR and its POM are copied to /output. /output and +# /repo/java/target are chowned to HOST_UID:HOST_GID on exit so the host user +# owns the outputs. # # Inputs (environment variables): # RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). @@ -31,6 +32,16 @@ if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then exit 1 fi +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + +_chown_outputs_on_exit() { + chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true +} +trap _chown_outputs_on_exit EXIT + if [[ -z ${PARALLEL_LEVEL} ]]; then PARALLEL_LEVEL=$(nproc) fi @@ -108,9 +119,3 @@ cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" rapids-logger "Emitted $(basename "${MAIN_JAR}") + cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" - -if [[ -n ${HOST_UID} && -n ${HOST_GID} ]]; then - rapids-logger "Chowning ${OUTPUT_DIR} and ${REPO_ROOT}/java/target to ${HOST_UID}:${HOST_GID}" - chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" - chown -R "${HOST_UID}:${HOST_GID}" "${REPO_ROOT}/java/target" -fi diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh index 579973399b36..e77637e747b2 100755 --- a/java/ci/build_static_libcudf_in_container.sh +++ b/java/ci/build_static_libcudf_in_container.sh @@ -29,6 +29,11 @@ if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then exit 1 fi +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + if [[ -z ${PARALLEL_LEVEL} ]]; then PARALLEL_LEVEL=$(nproc) fi @@ -86,7 +91,5 @@ cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" cmake --install "${BUILD_DIR}" -if [[ -n ${HOST_UID} && -n ${HOST_GID} ]]; then - rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" - chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" -fi +rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" +chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" diff --git a/java/ci/test_java_build_local.sh b/java/ci/test_java_build_local.sh index b4467da1109f..1d67ad62771d 100755 --- a/java/ci/test_java_build_local.sh +++ b/java/ci/test_java_build_local.sh @@ -189,10 +189,11 @@ if [[ ${CMAKE_CUDA_ARCHITECTURES} != "all" ]]; then CHILD_CMAKE_ARGS=(--cmake-cuda-architectures "${CMAKE_CUDA_ARCHITECTURES}") fi -# JAR builds run in parallel; give each half of the CPU budget. -JAR_PARALLEL=$((PARALLEL_LEVEL / 2)) -if [[ ${JAR_PARALLEL} -lt 1 ]]; then - JAR_PARALLEL=1 +# Both Step 1 and Step 2 launch two concurrent builds. Each build gets half +# of PARALLEL_LEVEL so together they stay within PARALLEL_LEVEL. +STEP_PARALLEL=$((PARALLEL_LEVEL / 2)) +if [[ ${STEP_PARALLEL} -lt 1 ]]; then + STEP_PARALLEL=1 fi # Step 1: static libcudf builds in parallel. @@ -202,7 +203,7 @@ STEP1_START=${SECONDS} "${SCRIPT_DIR}/build_static_libcudf.sh" \ --output-dir "${WORK_DIR}/libcudf-cuda12" \ --cuda-version "${CUDA12_VERSION}" \ - --parallel "${PARALLEL_LEVEL}" \ + --parallel "${STEP_PARALLEL}" \ "${CHILD_CMAKE_ARGS[@]}" \ > "${LOG_DIR}/static_cuda12.log" 2>&1 & STATIC_CUDA12_PID=$! @@ -210,7 +211,7 @@ STATIC_CUDA12_PID=$! "${SCRIPT_DIR}/build_static_libcudf.sh" \ --output-dir "${WORK_DIR}/libcudf-cuda13" \ --cuda-version "${CUDA13_VERSION}" \ - --parallel "${PARALLEL_LEVEL}" \ + --parallel "${STEP_PARALLEL}" \ "${CHILD_CMAKE_ARGS[@]}" \ > "${LOG_DIR}/static_cuda13.log" 2>&1 & STATIC_CUDA13_PID=$! @@ -250,7 +251,7 @@ STEP2_START=${SECONDS} --libcudf-dir "${WORK_DIR}/libcudf-cuda12" \ --output-dir "${WORK_DIR}/jars" \ --cuda-version "${CUDA12_VERSION}" \ - --parallel "${JAR_PARALLEL}" \ + --parallel "${STEP_PARALLEL}" \ "${CHILD_CMAKE_ARGS[@]}" \ > "${LOG_DIR}/jar_cuda12.log" 2>&1 & JAR_CUDA12_PID=$! @@ -259,7 +260,7 @@ JAR_CUDA12_PID=$! --libcudf-dir "${WORK_DIR}/libcudf-cuda13" \ --output-dir "${WORK_DIR}/jars" \ --cuda-version "${CUDA13_VERSION}" \ - --parallel "${JAR_PARALLEL}" \ + --parallel "${STEP_PARALLEL}" \ "${CHILD_CMAKE_ARGS[@]}" \ > "${LOG_DIR}/jar_cuda13.log" 2>&1 & JAR_CUDA13_PID=$! From f6b02ea3e3e3c37fd37e846c3d9043f5a25ad2fa Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Mon, 20 Jul 2026 05:41:52 +0000 Subject: [PATCH 3/4] TEMPORARY: Add java-build and java-gather jobs to pr.yaml for testing --- .github/workflows/pr.yaml | 70 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d2a097dbdf43..192bc56c36ba 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,6 +44,9 @@ jobs: - narwhals-tests - telemetry-setup - third-party-integration-tests-cudf-pandas + # TEMPORARY: revert java-build and java-gather additions before merging + - java-build + - java-gather uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 permissions: contents: read @@ -861,6 +864,73 @@ jobs: pull-requests: read uses: ./.github/workflows/spark-rapids-jni.yaml if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java + # TEMPORARY: revert addition of the java-build and java-gather jobs before merging. + # Keep this in sync with the same jobs in build.yaml. + java-build: + needs: [telemetry-setup] + strategy: + fail-fast: false + matrix: + include: + - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } + runs-on: ${{ matrix.runner }} + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Build static libcudf + run: | + ./java/ci/build_static_libcudf.sh \ + --output-dir "${RUNNER_TEMP}/libcudf" \ + --cuda-version "${{ matrix.cuda }}" + - name: Build cuDF Java JAR + run: | + ./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir "${RUNNER_TEMP}/libcudf" \ + --output-dir "${RUNNER_TEMP}/jars" \ + --cuda-version "${{ matrix.cuda }}" + - name: Upload per-entry JAR artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} + path: | + ${{ runner.temp }}/jars + !${{ runner.temp }}/jars/.mvn-temp-target + if-no-files-found: error + java-gather: + needs: [java-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Download per-entry JAR artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cudf_java_cuda* + path: ${{ runner.temp }}/jars + merge-multiple: true + - name: Assemble Maven repository layout + run: | + ./java/ci/assemble_maven_repo.sh \ + --jars-dir "${RUNNER_TEMP}/jars" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload combined Maven repository artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error telemetry-summarize: # This job must use a self-hosted runner to record telemetry traces. runs-on: linux-amd64-cpu4 From 0014c452961b3ec51358f7ac6ac7ce2eb160bf89 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:08:43 +0000 Subject: [PATCH 4/4] Revert "TEMPORARY: Add java-build and java-gather jobs to pr.yaml for testing" This reverts commit f6b02ea3e3e3c37fd37e846c3d9043f5a25ad2fa. --- .github/workflows/pr.yaml | 70 --------------------------------------- 1 file changed, 70 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 192bc56c36ba..d2a097dbdf43 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,9 +44,6 @@ jobs: - narwhals-tests - telemetry-setup - third-party-integration-tests-cudf-pandas - # TEMPORARY: revert java-build and java-gather additions before merging - - java-build - - java-gather uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 permissions: contents: read @@ -864,73 +861,6 @@ jobs: pull-requests: read uses: ./.github/workflows/spark-rapids-jni.yaml if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java - # TEMPORARY: revert addition of the java-build and java-gather jobs before merging. - # Keep this in sync with the same jobs in build.yaml. - java-build: - needs: [telemetry-setup] - strategy: - fail-fast: false - matrix: - include: - - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } - - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } - - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } - - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } - runs-on: ${{ matrix.runner }} - permissions: - contents: read - steps: - - name: Checkout code repo - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - fetch-depth: 0 - persist-credentials: false - - name: Build static libcudf - run: | - ./java/ci/build_static_libcudf.sh \ - --output-dir "${RUNNER_TEMP}/libcudf" \ - --cuda-version "${{ matrix.cuda }}" - - name: Build cuDF Java JAR - run: | - ./java/ci/build_cudf_java_jar.sh \ - --libcudf-dir "${RUNNER_TEMP}/libcudf" \ - --output-dir "${RUNNER_TEMP}/jars" \ - --cuda-version "${{ matrix.cuda }}" - - name: Upload per-entry JAR artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} - path: | - ${{ runner.temp }}/jars - !${{ runner.temp }}/jars/.mvn-temp-target - if-no-files-found: error - java-gather: - needs: [java-build] - runs-on: linux-amd64-cpu4 - permissions: - contents: read - steps: - - name: Checkout code repo - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - name: Download per-entry JAR artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: cudf_java_cuda* - path: ${{ runner.temp }}/jars - merge-multiple: true - - name: Assemble Maven repository layout - run: | - ./java/ci/assemble_maven_repo.sh \ - --jars-dir "${RUNNER_TEMP}/jars" \ - --output-dir "${RUNNER_TEMP}/maven-repo" - - name: Upload combined Maven repository artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: cudf_java_maven_repo - path: ${{ runner.temp }}/maven-repo - if-no-files-found: error telemetry-summarize: # This job must use a self-hosted runner to record telemetry traces. runs-on: linux-amd64-cpu4