diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c1a386577155..e647f09885b8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -541,3 +541,23 @@ jobs: name: cudf_java_maven_repo path: ${{ runner.temp }}/maven-repo if-no-files-found: error + + # Publish tagged release candidates to Maven Central via the Sonatype + # Central Publisher Portal. + java-publish: + needs: [java-gather] + if: ${{ github.ref_type == 'tag' && (inputs.build_type || 'branch') == 'branch' }} + permissions: + actions: read + contents: read + uses: rapidsai/shared-workflows/.github/workflows/maven-publish.yaml@release/26.08 + secrets: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + MAVEN_DEPLOY_TOKEN: ${{ secrets.MAVEN_DEPLOY_TOKEN }} + with: + publication-type: 'rc' + artifact-name: cudf_java_maven_repo + source-git-sha: ${{ inputs.sha || github.sha }} + # false = validate + drop (safe). true = stage PENDING for manual publish. + stage-for-maven-central-publish: true diff --git a/java/ci/README.md b/java/ci/README.md index e537f2fb890b..51244e6348f0 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -34,12 +34,15 @@ so plain `rm -rf` works. ### Step 2 - Package the cuDF Java JAR for one classifier ```bash -./java/ci/build_cudf_java_jar.sh \ +GITHUB_REF=refs/heads/my-branch ./java/ci/build_cudf_java_jar.sh \ --libcudf-dir /tmp/libcudf-cuda12 \ --output-dir /tmp/jars \ --cuda-version 12.9 ``` +`GITHUB_REF` is required and selects release-tag vs SNAPSHOT versioning. See +the versioning section below. + This compiles the JNI layer against the static libcudf from Step 1 and emits the classifier JAR (e.g. `cudf-26.08.0-SNAPSHOT-cuda12.jar`), a classifier-independent sources jar and javadoc jar, and the POM into a @@ -72,12 +75,14 @@ inside the container. This walks every subdirectory of `--jars-dir` (each subdir name IS the classifier), gathers the per-classifier JAR, one shared sources jar, one -shared javadoc jar, and the shared POM, derives the artifact version from -the JAR filenames (requiring a single unique version across subdirs), and +shared javadoc jar, the shared POM, and seeds an unclassified primary JAR +as a copy of the `cuda12` classifier. Derives the artifact version from +the JAR filenames (requiring a single unique version across subdirs) and lays them out as: ``` /tmp/maven-repo/ai/rapids/cudf/26.08.0-SNAPSHOT/ + cudf-26.08.0-SNAPSHOT.jar cudf-26.08.0-SNAPSHOT-cuda12.jar cudf-26.08.0-SNAPSHOT-cuda13.jar cudf-26.08.0-SNAPSHOT-sources.jar @@ -88,7 +93,27 @@ lays them out as: The set of classifiers is whatever subdirectories are present under `--jars-dir`. For a local `x86_64`-only run, populate `/tmp/jars/cuda12/` and `/tmp/jars/cuda13/`. For the full four-way release build, add -`/tmp/jars/cuda12-arm64/` and `/tmp/jars/cuda13-arm64/`. +`/tmp/jars/cuda12-arm64/` and `/tmp/jars/cuda13-arm64/`. The `cuda12` +subdirectory is required because the unclassified primary JAR is copied from +it, so an `aarch64`-only set of subdirectories is not a valid gather input. + +### Release Tag vs SNAPSHOT versioning + +Release tag CI runs (`GITHUB_REF=refs/tags/vYY.MM.PP`) produce release-versioned +JARs (`cudf-26.08.0-*.jar`). All other runs produce `-SNAPSHOT`. Gated by +[`rapids-is-release-build`](https://github.com/rapidsai/gha-tools/blob/main/tools/rapids-is-release-build). +`GITHUB_REF` is required. `test_java_build_local.sh` defaults it to the current +branch. + +To rehearse the release path locally: + +```bash +GITHUB_REF=refs/tags/vYY.MM.PP ./java/ci/test_java_build_local.sh +``` + +Rewrites `java/pom.xml` in place. Restore with `git checkout java/pom.xml`. + +### GitHub Actions In GitHub Actions (`.github/workflows/build.yaml`), the `java-build` matrix job runs Steps 1-2 per (CUDA x arch) entry and uploads each classifier subdir as a diff --git a/java/ci/assemble_maven_repo.sh b/java/ci/assemble_maven_repo.sh index 96bf8c8f33f3..2e384fd2c1d0 100755 --- a/java/ci/assemble_maven_repo.sh +++ b/java/ci/assemble_maven_repo.sh @@ -166,12 +166,26 @@ if [[ -z "${FIRST_VERSION}" ]]; then exit 1 fi +DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${FIRST_VERSION}" + +# Seed the unclassified primary from cuda12. Maven Central serves this to +# consumers depending on ai.rapids:cudf without a . +PRIMARY_SOURCE="${DEST_DIR}/cudf-${FIRST_VERSION}-cuda12.jar" +if [[ ! -f "${PRIMARY_SOURCE}" ]]; then + echo "Error: ${PRIMARY_SOURCE} missing." >&2 + exit 1 +fi +UNCLASSIFIED="${DEST_DIR}/cudf-${FIRST_VERSION}.jar" +if [[ ! -f "${UNCLASSIFIED}" ]]; then + cp -f "${PRIMARY_SOURCE}" "${UNCLASSIFIED}" + echo " + cudf-${FIRST_VERSION}.jar (unclassified primary, copy of cuda12)" +fi + # Sources and javadoc jars are classifier-independent (pure Java, no arch or # cuda variation). Every classifier subdir produces byte-equivalent copies; # pick the lexicographically first subdir's copy as canonical. Fail fast if # any subdir is missing either file - that indicates -Prelease or # -Pjavadoc-jdk17 did not activate for that classifier's build. -DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${FIRST_VERSION}" FIRST_CLASSIFIER_SUBDIR="" for subdir in "${JARS_DIR}"/*/; do if [[ -z ${FIRST_CLASSIFIER_SUBDIR} ]]; then diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh index 084b00c68478..acdf38feb735 100755 --- a/java/ci/build_cudf_java_jar.sh +++ b/java/ci/build_cudf_java_jar.sh @@ -121,6 +121,8 @@ require_arg --libcudf-dir "${LIBCUDF_DIR}" require_arg --output-dir "${OUTPUT_DIR}" require_arg --cuda-version "${CUDA_VERSION}" +: "${GITHUB_REF:?GITHUB_REF must be set.}" + if [[ ! -d ${LIBCUDF_DIR} ]]; then echo "Error: --libcudf-dir '${LIBCUDF_DIR}' does not exist." exit 1 @@ -194,6 +196,7 @@ DOCKER_ARGS=( --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" --env HOST_UID="$(id -u)" --env HOST_GID="$(id -g)" + --env GITHUB_REF="${GITHUB_REF}" ) if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh index e02ea3171508..9f93cf5edb98 100755 --- a/java/ci/build_cudf_java_jar_in_container.sh +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -38,7 +38,16 @@ if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then exit 1 fi +# TEMPORARY: same patch as spark-rapids-jni +# (NVIDIA/cudf-spark-jni patches/0001-use-static-nvcomp-from-libcudf.patch). +# Static CUDF_JNI_LIBCUDF_STATIC=ON JARs embed nvcomp in libcudf and do not +# package libnvcomp.so. NativeDepsLoader still requires it unless this patch +# drops "nvcomp" from loadOrder for packaging only. +NVCOMP_LOADER_PATCH="${REPO_ROOT}/java/ci/patches/0001-use-static-nvcomp-from-libcudf.patch" + _chown_outputs_on_exit() { + # Best-effort: leave /repo clean even if packaging failed mid-build. + git -C "${REPO_ROOT}" apply --reverse "${NVCOMP_LOADER_PATCH}" >/dev/null 2>&1 || true chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true } trap _chown_outputs_on_exit EXIT @@ -107,8 +116,29 @@ fi cd "${REPO_ROOT}/java" CUDF_VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" + +# Release tag builds strip -SNAPSHOT and rewrite the POM so packaged artifacts +# carry the release version. Non-release builds keep -SNAPSHOT. +if rapids-is-release-build; then + CUDF_VERSION="${CUDF_VERSION%-SNAPSHOT}" + mvn versions:set -DnewVersion="${CUDF_VERSION}" -DgenerateBackupPoms=false "${BUILD_ARG[@]}" +fi + rapids-logger "Packaging cuDF Java JAR version ${CUDF_VERSION} (libcudf: ${CUDF_INSTALL_DIR})" +# PIC libspdlog.a / libfmt.a are installed into the static libcudf prefix +# (same tree as rmm) and linked via $ from the packaging +# cmake patch. Fail early if this mount is missing them. +for _logging_lib in libfmt.a libspdlog.a; do + if [[ ! -f ${CUDF_INSTALL_DIR}/lib/${_logging_lib} ]]; then + echo "Error: ${CUDF_INSTALL_DIR}/lib/${_logging_lib} missing; rebuild static libcudf with install_static_logging_libs" >&2 + exit 1 + fi +done + +rapids-logger "Applying temporary nvcomp NativeDepsLoader patch (same as spark-rapids-jni)" +git -C "${REPO_ROOT}" apply "${NVCOMP_LOADER_PATCH}" + # The `clean` goal is intentionally omitted: /repo/java/target is a # bind-mount point, so when `mvn clean` attempts to remove the directory, # it fails with EBUSY. The host wrapper (build_cudf_java_jar.sh) recreates @@ -162,4 +192,6 @@ done cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" +bash "${REPO_ROOT}/java/ci/check_jar_libcudf_needed.sh" "${OUTPUT_DIR}/$(basename "${MAIN_JAR}")" + rapids-logger "Emitted $(basename "${MAIN_JAR}"), cudf-${CUDF_VERSION}-sources.jar, cudf-${CUDF_VERSION}-javadoc.jar, and cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh index e77637e747b2..10c5c94c4a92 100755 --- a/java/ci/build_static_libcudf_in_container.sh +++ b/java/ci/build_static_libcudf_in_container.sh @@ -34,6 +34,14 @@ if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then exit 1 fi +SPDLOG_FMT_PATCH="${REPO_ROOT}/java/ci/patches/0002-static-spdlog-fmt-for-java-libcudf.patch" + +_cleanup_on_exit() { + git -C "${REPO_ROOT}" apply --reverse "${SPDLOG_FMT_PATCH}" >/dev/null 2>&1 || true + chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" 2>/dev/null || true +} +trap _cleanup_on_exit EXIT + if [[ -z ${PARALLEL_LEVEL} ]]; then PARALLEL_LEVEL=$(nproc) fi @@ -62,6 +70,13 @@ if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache fi +# PIC libfmt.a / libspdlog.a into the same install prefix as rmm / rapids_logger. +# shellcheck disable=SC1091 +. "${REPO_ROOT}/java/ci/install_static_logging_libs.sh" +install_static_logging_libs "${INSTALL_PREFIX}" + +git -C "${REPO_ROOT}" apply "${SPDLOG_FMT_PATCH}" + CMAKE_ARGS=( -S "${REPO_ROOT}/cpp" -B "${BUILD_DIR}" @@ -91,5 +106,4 @@ cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" cmake --install "${BUILD_DIR}" -rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" -chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" +rapids-logger "Emitted static libcudf install tree to ${INSTALL_PREFIX}" diff --git a/java/ci/check_jar_libcudf_needed.sh b/java/ci/check_jar_libcudf_needed.sh new file mode 100755 index 000000000000..b4bc39995128 --- /dev/null +++ b/java/ci/check_jar_libcudf_needed.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Fail if a cudf classifier JAR's libcudf.so DT_NEEDs libspdlog or libfmt. +# +# Usage: check_jar_libcudf_needed.sh + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +JAR="$(realpath "$1")" +if [[ ! -f ${JAR} ]]; then + echo "Error: JAR not found: ${JAR}" >&2 + exit 1 +fi + +if ! command -v readelf >/dev/null 2>&1; then + echo "Error: readelf not found (install binutils)" >&2 + exit 1 +fi + +if ! command -v unzip >/dev/null 2>&1; then + echo "Error: unzip not found" >&2 + exit 1 +fi + +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +MEMBER="$(unzip -Z1 "${JAR}" | grep -E '(^|/)libcudf\.so$' | head -1 || true)" +if [[ -z ${MEMBER} ]]; then + echo "Error: ${JAR} does not contain libcudf.so" >&2 + exit 1 +fi + +# -j flattens the JAR-internal path (e.g. native/.../libcudf.so) into ${TMP}. +unzip -j -o -q "${JAR}" "${MEMBER}" -d "${TMP}" + +SO="${TMP}/libcudf.so" +if [[ ! -f ${SO} ]]; then + echo "Error: failed to extract libcudf.so from ${JAR}" >&2 + exit 1 +fi + +if readelf -d "${SO}" | grep -E 'NEEDED.*\[(libspdlog|libfmt)\.so'; then + echo "Error: ${JAR} libcudf.so still depends on shared spdlog/fmt:" >&2 + readelf -d "${SO}" | grep NEEDED >&2 || true + exit 1 +fi + +echo "OK: no libspdlog/libfmt DT_NEEDED in $(basename "${JAR}")" diff --git a/java/ci/install_static_logging_libs.sh b/java/ci/install_static_logging_libs.sh new file mode 100755 index 000000000000..02a74d3d18b2 --- /dev/null +++ b/java/ci/install_static_logging_libs.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Install PIC static libfmt.a / libspdlog.a into the static libcudf install +# prefix (same tree as rmm / rapids_logger). Versions match the shared libs +# from the ci-conda build_java env (libfmt.so.12.1.0, libspdlog.so.1.17.0). +# +# Usage: +# . install_static_logging_libs.sh +# install_static_logging_libs "${INSTALL_PREFIX}" + +install_static_libfmt() { + local prefix="$1" + local src + src="$(mktemp -d /tmp/cudf-java-fmt.XXXXXX)" + + rapids-logger "Installing static libfmt.a (12.1.0) into ${prefix}" + curl -fsSL "https://github.com/fmtlib/fmt/archive/refs/tags/12.1.0.tar.gz" \ + | tar -xz -C "${src}" + cmake -S "${src}/fmt-12.1.0" -B "${src}/build" -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${prefix}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DFMT_DOC=OFF \ + -DFMT_TEST=OFF + cmake --build "${src}/build" --parallel "${PARALLEL_LEVEL}" + cmake --install "${src}/build" + rm -rf "${src}" + [[ -f ${prefix}/lib/libfmt.a ]] \ + || { echo "Error: failed to install ${prefix}/lib/libfmt.a" >&2; return 1; } +} + +install_static_libspdlog() { + local prefix="$1" + local src + src="$(mktemp -d /tmp/cudf-java-spdlog.XXXXXX)" + + rapids-logger "Installing static libspdlog.a (v1.17.0) into ${prefix}" + curl -fsSL "https://github.com/gabime/spdlog/archive/refs/tags/v1.17.0.tar.gz" \ + | tar -xz -C "${src}" + cmake -S "${src}/spdlog-1.17.0" -B "${src}/build" -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${prefix}" \ + -DCMAKE_PREFIX_PATH="${prefix}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DSPDLOG_BUILD_SHARED=OFF \ + -DSPDLOG_BUILD_PIC=ON \ + -DSPDLOG_FMT_EXTERNAL=ON \ + -DSPDLOG_BUILD_EXAMPLE=OFF \ + -DSPDLOG_BUILD_TESTS=OFF \ + -DSPDLOG_BUILD_BENCH=OFF + cmake --build "${src}/build" --parallel "${PARALLEL_LEVEL}" + cmake --install "${src}/build" + rm -rf "${src}" + [[ -f ${prefix}/lib/libspdlog.a ]] \ + || { echo "Error: failed to install ${prefix}/lib/libspdlog.a" >&2; return 1; } +} + +install_static_logging_libs() { + local prefix="$1" + if [[ -z ${prefix} ]]; then + echo "Error: install_static_logging_libs requires an install prefix" >&2 + return 1 + fi + mkdir -p "${prefix}" + + [[ -f ${prefix}/lib/libfmt.a ]] || install_static_libfmt "${prefix}" + [[ -f ${prefix}/lib/libspdlog.a ]] || install_static_libspdlog "${prefix}" +} diff --git a/java/ci/patches/0001-use-static-nvcomp-from-libcudf.patch b/java/ci/patches/0001-use-static-nvcomp-from-libcudf.patch new file mode 100644 index 000000000000..6beeec85a59e --- /dev/null +++ b/java/ci/patches/0001-use-static-nvcomp-from-libcudf.patch @@ -0,0 +1,14 @@ +diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +index 0000000000..0000000000 100644 +--- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java ++++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +@@ -69,9 +69,6 @@ public class NativeDepsLoader { + * subsequent stages are loaded. + */ + private static final String[][] loadOrder = new String[][]{ +- new String[]{ +- "nvcomp" +- }, + new String[]{ + "cudf" + }, diff --git a/java/ci/patches/0002-static-spdlog-fmt-for-java-libcudf.patch b/java/ci/patches/0002-static-spdlog-fmt-for-java-libcudf.patch new file mode 100644 index 000000000000..8a850f933a5b --- /dev/null +++ b/java/ci/patches/0002-static-spdlog-fmt-for-java-libcudf.patch @@ -0,0 +1,29 @@ +diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt +index 0000000000..1111111111 100644 +--- a/cpp/CMakeLists.txt ++++ b/cpp/CMakeLists.txt +@@ -1359,4 +1359,23 @@ if(TARGET spdlog::spdlog) + else() +- target_link_libraries(cudf PRIVATE ${_spdlog_link}) ++ # TEMPORARY (java packaging): absorb PIC static spdlog/fmt from the ++ # libcudf install prefix (same tree as rmm). BUILD_INTERFACE uses the ++ # libs found at configure time; INSTALL_INTERFACE uses $ ++ # so consumers (JAR container at /libcudf) resolve them like other deps. ++ find_library( ++ CUDF_JAVA_SPDLOG_STATIC NAMES libspdlog.a ++ PATHS "${CMAKE_INSTALL_PREFIX}/lib" NO_DEFAULT_PATH REQUIRED ++ ) ++ find_library( ++ CUDF_JAVA_FMT_STATIC NAMES libfmt.a ++ PATHS "${CMAKE_INSTALL_PREFIX}/lib" NO_DEFAULT_PATH REQUIRED ++ ) ++ target_link_libraries( ++ cudf ++ PRIVATE ++ "$>" ++ "$>" ++ "$/lib/libspdlog.a>>" ++ "$/lib/libfmt.a>>" ++ ) + endif() + endif() diff --git a/java/ci/test_java_build_local.sh b/java/ci/test_java_build_local.sh index 1d67ad62771d..56ff7cc136fe 100755 --- a/java/ci/test_java_build_local.sh +++ b/java/ci/test_java_build_local.sh @@ -143,6 +143,16 @@ record_step_end() { parse_args "$@" +# Mirror GHA's GITHUB_REF locally so the container distinguishes SNAPSHOT from release-tag builds. +if [[ -z "${GITHUB_REF:-}" ]]; then + GITHUB_REF="$(git -C "${SCRIPT_DIR}" symbolic-ref HEAD 2>/dev/null || true)" +fi +if [[ -z "${GITHUB_REF:-}" ]]; then + echo "Error: could not derive GITHUB_REF from HEAD (detached?). Set it explicitly, e.g. GITHUB_REF=refs/heads/my-branch." >&2 + exit 1 +fi +export GITHUB_REF + require_arg --work-dir "${WORK_DIR}" mkdir -p "${WORK_DIR}"