From 8a210cfd2a2d8356da68004df0d22412d8b6945c Mon Sep 17 00:00:00 2001 From: Firestarman Date: Tue, 8 Jul 2025 19:53:43 +0800 Subject: [PATCH 1/9] fix build errors in JNI for nvcomp5 Signed-off-by: Firestarman --- .../rapids/cudf/nvcomp/BatchedCompressor.java | 7 ++- .../cudf/nvcomp/BatchedDecompressor.java | 6 ++- .../cudf/nvcomp/BatchedLZ4Compressor.java | 4 +- .../cudf/nvcomp/BatchedLZ4Decompressor.java | 6 ++- .../cudf/nvcomp/BatchedZstdCompressor.java | 4 +- .../cudf/nvcomp/BatchedZstdDecompressor.java | 6 ++- .../java/ai/rapids/cudf/nvcomp/NvcompJni.java | 14 ++++-- java/src/main/native/src/NvcompJni.cpp | 48 ++++++++++++------- 8 files changed, 63 insertions(+), 32 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedCompressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedCompressor.java index 72dfcdb3cb59..5a0ff5740298 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedCompressor.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedCompressor.java @@ -91,7 +91,8 @@ public DeviceMemoryBuffer[] compress(BaseDeviceMemoryBuffer[] origInputs, Cuda.S buildAddrsAndSizes(inputs, inputChunkAddrs, inputChunkSizes, compressedBuffers, outputChunkAddrs); - final long tempBufferSize = batchedCompressGetTempSize(numChunks, chunkSize); + final long tempBufferSize = batchedCompressGetTempSize(numChunks, chunkSize, + numChunks * chunkSize); try (DeviceMemoryBuffer addrsAndSizes = putAddrsAndSizesOnDevice(inputChunkAddrs, inputChunkSizes, outputChunkAddrs, stream); DeviceMemoryBuffer tempBuffer = @@ -308,9 +309,11 @@ private long[] calcOutputBufferSizes(int[] chunksPerInput, long[] outputChunkSiz * Get the temporary workspace size required to perform compression of an entire batch. * @param batchSize number of chunks in the batch * @param maxChunkSize maximum size of an uncompressed chunk in bytes + * @param totalSize Upper bound on the total uncompressed size of all chunks * @return The size of required temporary workspace in bytes to compress the batch. */ - protected abstract long batchedCompressGetTempSize(long batchSize, long maxChunkSize); + protected abstract long batchedCompressGetTempSize(long batchSize, long maxChunkSize, + long totalSize); /** * Asynchronously compress a batch of buffers. Note that compressedSizesOutPtr must diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedDecompressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedDecompressor.java index 5543d2dcb646..af195c84cc7a 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedDecompressor.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedDecompressor.java @@ -71,7 +71,8 @@ public void decompressAsync(BaseDeviceMemoryBuffer[] origInputs, totalChunks += numBufferChunks; } - final long tempBufferSize = batchedDecompressGetTempSize(totalChunks, chunkSize); + final long tempBufferSize = batchedDecompressGetTempSize(totalChunks, chunkSize, + totalChunks * chunkSize); try (DeviceMemoryBuffer devAddrsSizes = buildAddrsSizesBuffer(chunkSize, totalChunks, inputs.getArray(), chunksPerInput, outputs, stream); DeviceMemoryBuffer devTemp = DeviceMemoryBuffer.allocate(tempBufferSize)) { @@ -198,10 +199,11 @@ private static HostMemoryBuffer fetchMetadata(long totalChunks, BaseDeviceMemory * Computes the temporary storage size in bytes needed to decompress a compressed batch. * @param numChunks number of chunks in the batch * @param maxUncompressedChunkBytes maximum uncompressed size of any chunk in bytes + * @param maxTotalSize Upper bound on the total uncompressed size of all chunks * @return number of temporary storage bytes needed to decompress the batch */ protected abstract long batchedDecompressGetTempSize(long numChunks, - long maxUncompressedChunkBytes); + long maxUncompressedChunkBytes, long maxTotalSize); /** * Asynchronously decompress a batch of compressed data buffers. diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Compressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Compressor.java index 58c0e7ee1691..d8b8d4c616e0 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Compressor.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Compressor.java @@ -32,8 +32,8 @@ public BatchedLZ4Compressor(long chunkSize, long maxIntermediateBufferSize) { } @Override - protected long batchedCompressGetTempSize(long batchSize, long maxChunkSize) { - return NvcompJni.batchedLZ4CompressGetTempSize(batchSize, maxChunkSize); + protected long batchedCompressGetTempSize(long batchSize, long maxChunkSize, long totalSize) { + return NvcompJni.batchedLZ4CompressGetTempSize(batchSize, maxChunkSize, totalSize); } @Override diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Decompressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Decompressor.java index d78d537ea13b..82e425ae234d 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Decompressor.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedLZ4Decompressor.java @@ -41,8 +41,10 @@ public static void decompressAsync(long chunkSize, BaseDeviceMemoryBuffer[] orig } @Override - protected long batchedDecompressGetTempSize(long numChunks, long maxUncompressedChunkBytes) { - return NvcompJni.batchedLZ4DecompressGetTempSize(numChunks, maxUncompressedChunkBytes); + protected long batchedDecompressGetTempSize(long numChunks, long maxUncompressedChunkBytes, + long maxTotalSize) { + return NvcompJni.batchedLZ4DecompressGetTempSize(numChunks, maxUncompressedChunkBytes, + maxTotalSize); } @Override diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdCompressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdCompressor.java index 0532b4aa86d8..9c4a8aca52e1 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdCompressor.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdCompressor.java @@ -31,8 +31,8 @@ public BatchedZstdCompressor(long chunkSize, long maxIntermediateBufferSize) { } @Override - protected long batchedCompressGetTempSize(long batchSize, long maxChunkSize) { - return NvcompJni.batchedZstdCompressGetTempSize(batchSize, maxChunkSize); + protected long batchedCompressGetTempSize(long batchSize, long maxChunkSize, long totalSize) { + return NvcompJni.batchedZstdCompressGetTempSize(batchSize, maxChunkSize, totalSize); } @Override diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdDecompressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdDecompressor.java index ba11a236834d..3ca21ac0b7e3 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdDecompressor.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/BatchedZstdDecompressor.java @@ -23,8 +23,10 @@ public BatchedZstdDecompressor(long chunkSize) { } @Override - protected long batchedDecompressGetTempSize(long numChunks, long maxUncompressedChunkBytes) { - return NvcompJni.batchedZstdDecompressGetTempSize(numChunks, maxUncompressedChunkBytes); + protected long batchedDecompressGetTempSize(long numChunks, long maxUncompressedChunkBytes, + long maxTotalSize) { + return NvcompJni.batchedZstdDecompressGetTempSize(numChunks, maxUncompressedChunkBytes, + maxTotalSize); } @Override diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java b/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java index 1a21629a2082..f940f781ffbc 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java @@ -29,9 +29,10 @@ class NvcompJni { * Get the temporary workspace size required to perform compression of entire LZ4 batch. * @param batchSize number of chunks in the batch * @param maxChunkSize maximum size of an uncompressed chunk in bytes + * @param maxTotalSize Upper bound on the total uncompressed size of all chunks * @return The size of required temporary workspace in bytes to compress the batch. */ - static native long batchedLZ4CompressGetTempSize(long batchSize, long maxChunkSize); + static native long batchedLZ4CompressGetTempSize(long batchSize, long maxChunkSize, long maxTotalSize); /** * Get the maximum size any chunk could compress to in a LZ4 batch. This is the minimum amount of @@ -74,11 +75,13 @@ static native void batchedLZ4CompressAsync( * Computes the temporary storage size in bytes needed to decompress a LZ4-compressed batch. * @param numChunks number of chunks in the batch * @param maxUncompressedChunkBytes maximum uncompressed size of any chunk in bytes + * @param maxTotalSize Upper bound on the total uncompressed size of all chunks * @return number of temporary storage bytes needed to decompress the batch */ static native long batchedLZ4DecompressGetTempSize( long numChunks, - long maxUncompressedChunkBytes); + long maxUncompressedChunkBytes, + long maxTotalSize); /** * Asynchronously decompress a batch of LZ4-compressed data buffers. @@ -121,9 +124,10 @@ static native void batchedLZ4GetDecompressSizeAsync( * Get the temporary workspace size required to perform compression of entire zstd batch. * @param batchSize number of chunks in the batch * @param maxChunkSize maximum size of an uncompressed chunk in bytes + * @param maxTotalSize Upper bound on the total uncompressed size of all chunks * @return The size of required temporary workspace in bytes to compress the batch. */ - static native long batchedZstdCompressGetTempSize(long batchSize, long maxChunkSize); + static native long batchedZstdCompressGetTempSize(long batchSize, long maxChunkSize, long maxTotalSize); /** * Get the maximum size any chunk could compress to in a ZSTD batch. This is the minimum @@ -167,11 +171,13 @@ static native void batchedZstdCompressAsync( * ZSTD-compressed batch. * @param numChunks number of chunks in the batch * @param maxUncompressedChunkBytes maximum uncompressed size of any chunk in bytes + * @param maxTotalSize Upper bound on the total uncompressed size of all chunks * @return number of temporary storage bytes needed to decompress the batch */ static native long batchedZstdDecompressGetTempSize( long numChunks, - long maxUncompressedChunkBytes); + long maxUncompressedChunkBytes, + long maxTotalSize); /** * Asynchronously decompress a batch of ZSTD-compressed data buffers. diff --git a/java/src/main/native/src/NvcompJni.cpp b/java/src/main/native/src/NvcompJni.cpp index 0b3bf1916b92..4fa4fc372f56 100644 --- a/java/src/main/native/src/NvcompJni.cpp +++ b/java/src/main/native/src/NvcompJni.cpp @@ -63,15 +63,16 @@ extern "C" { // methods for lz4 JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4CompressGetTempSize( - JNIEnv* env, jclass, jlong j_batch_size, jlong j_max_chunk_size) + JNIEnv* env, jclass, jlong j_batch_size, jlong j_max_chunk_size, jlong j_max_total_size) { try { cudf::jni::auto_set_device(env); auto batch_size = static_cast(j_batch_size); auto max_chunk_size = static_cast(j_max_chunk_size); + auto total_size = static_cast(j_max_total_size); std::size_t temp_size = 0; - auto status = nvcompBatchedLZ4CompressGetTempSize( - batch_size, max_chunk_size, nvcompBatchedLZ4DefaultOpts, &temp_size); + auto status = nvcompBatchedLZ4CompressGetTempSizeAsync( + batch_size, max_chunk_size, nvcompBatchedLZ4CompressDefaultOpts, &temp_size, total_size); check_nvcomp_status(env, status); return static_cast(temp_size); } @@ -88,7 +89,7 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4CompressGetMaxOutputChunkSize(JNI auto max_chunk_size = static_cast(j_max_chunk_size); std::size_t max_output_size = 0; auto status = nvcompBatchedLZ4CompressGetMaxOutputChunkSize( - max_chunk_size, nvcompBatchedLZ4DefaultOpts, &max_output_size); + max_chunk_size, nvcompBatchedLZ4CompressDefaultOpts, &max_output_size); check_nvcomp_status(env, status); return static_cast(max_output_size); } @@ -119,7 +120,10 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4CompressAsync(JNIEnv* env, auto out_ptrs = reinterpret_cast(j_out_ptrs); auto compressed_out_sizes = reinterpret_cast(j_compressed_sizes_out_ptr); auto stream = reinterpret_cast(j_stream); - auto status = nvcompBatchedLZ4CompressAsync(in_ptrs, + // FIXME how to use these statuses ? They are not used either in the corresponding + // decompressor. + auto comp_statuses = rmm::device_uvector(batch_size, stream); + auto status = nvcompBatchedLZ4CompressAsync(in_ptrs, in_sizes, chunk_size, batch_size, @@ -127,7 +131,8 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4CompressAsync(JNIEnv* env, temp_size, out_ptrs, compressed_out_sizes, - nvcompBatchedLZ4DefaultOpts, + nvcompBatchedLZ4CompressDefaultOpts, + comp_statuses.data(), stream); check_nvcomp_status(env, status); } @@ -135,14 +140,16 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4CompressAsync(JNIEnv* env, } JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4DecompressGetTempSize( - JNIEnv* env, jclass, jlong j_batch_size, jlong j_chunk_size) + JNIEnv* env, jclass, jlong j_batch_size, jlong j_chunk_size, jlong j_max_total_size) { try { cudf::jni::auto_set_device(env); auto batch_size = static_cast(j_batch_size); auto chunk_size = static_cast(j_chunk_size); + auto total_size = static_cast(j_max_total_size); std::size_t temp_size = 0; - auto status = nvcompBatchedLZ4DecompressGetTempSize(batch_size, chunk_size, &temp_size); + auto status = + nvcompBatchedLZ4DecompressGetTempSizeAsync(batch_size, chunk_size, &temp_size, total_size); check_nvcomp_status(env, status); return static_cast(temp_size); } @@ -181,6 +188,7 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4DecompressAsync(JNIEnv* env, temp_ptr, temp_size, uncompressed_ptrs, + nvcompBatchedLZ4DecompressDefaultOpts, uncompressed_statuses.data(), stream); check_nvcomp_status(env, status); @@ -218,15 +226,16 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4GetDecompressSizeAsync(JNIEnv* en // methods for zstd JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdCompressGetTempSize( - JNIEnv* env, jclass, jlong j_batch_size, jlong j_max_chunk_size) + JNIEnv* env, jclass, jlong j_batch_size, jlong j_max_chunk_size, jlong j_max_total_size) { try { cudf::jni::auto_set_device(env); auto batch_size = static_cast(j_batch_size); auto max_chunk_size = static_cast(j_max_chunk_size); + auto total_size = static_cast(j_max_total_size); std::size_t temp_size = 0; - auto status = nvcompBatchedZstdCompressGetTempSize( - batch_size, max_chunk_size, nvcompBatchedZstdDefaultOpts, &temp_size); + auto status = nvcompBatchedZstdCompressGetTempSizeAsync( + batch_size, max_chunk_size, nvcompBatchedZstdCompressDefaultOpts, &temp_size, total_size); check_nvcomp_status(env, status); return static_cast(temp_size); } @@ -242,7 +251,7 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdCompressGetMaxOutputChunkSize( auto max_chunk_size = static_cast(j_max_chunk_size); std::size_t max_output_size = 0; auto status = nvcompBatchedZstdCompressGetMaxOutputChunkSize( - max_chunk_size, nvcompBatchedZstdDefaultOpts, &max_output_size); + max_chunk_size, nvcompBatchedZstdCompressDefaultOpts, &max_output_size); check_nvcomp_status(env, status); return static_cast(max_output_size); } @@ -273,7 +282,10 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdCompressAsync(JNIEnv* env, auto out_ptrs = reinterpret_cast(j_out_ptrs); auto compressed_out_sizes = reinterpret_cast(j_compressed_sizes_out_ptr); auto stream = reinterpret_cast(j_stream); - auto status = nvcompBatchedZstdCompressAsync(in_ptrs, + // FIXME how to use these statuses ? They are not used either in the corresponding + // decompression. + auto comp_statuses = rmm::device_uvector(batch_size, stream); + auto status = nvcompBatchedZstdCompressAsync(in_ptrs, in_sizes, chunk_size, batch_size, @@ -281,7 +293,8 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdCompressAsync(JNIEnv* env, temp_size, out_ptrs, compressed_out_sizes, - nvcompBatchedZstdDefaultOpts, + nvcompBatchedZstdCompressDefaultOpts, + comp_statuses.data(), stream); check_nvcomp_status(env, status); } @@ -289,14 +302,16 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdCompressAsync(JNIEnv* env, } JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdDecompressGetTempSize( - JNIEnv* env, jclass, jlong j_batch_size, jlong j_chunk_size) + JNIEnv* env, jclass, jlong j_batch_size, jlong j_chunk_size, jlong j_max_total_size) { try { cudf::jni::auto_set_device(env); auto batch_size = static_cast(j_batch_size); auto chunk_size = static_cast(j_chunk_size); + auto total_size = static_cast(j_max_total_size); std::size_t temp_size = 0; - auto status = nvcompBatchedZstdDecompressGetTempSize(batch_size, chunk_size, &temp_size); + auto status = + nvcompBatchedZstdDecompressGetTempSizeAsync(batch_size, chunk_size, &temp_size, total_size); check_nvcomp_status(env, status); return static_cast(temp_size); } @@ -335,6 +350,7 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdDecompressAsync(JNIEnv* env, temp_ptr, temp_size, uncompressed_ptrs, + nvcompBatchedZstdDecompressDefaultOpts, uncompressed_statuses.data(), stream); check_nvcomp_status(env, status); From aa2c40afe4e8ed317ce6ac2d673474157bec14ea Mon Sep 17 00:00:00 2001 From: Firestarman Date: Tue, 8 Jul 2025 21:40:57 +0800 Subject: [PATCH 2/9] update comment Signed-off-by: Firestarman --- java/src/main/native/src/NvcompJni.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/native/src/NvcompJni.cpp b/java/src/main/native/src/NvcompJni.cpp index 4fa4fc372f56..9e127dd7119d 100644 --- a/java/src/main/native/src/NvcompJni.cpp +++ b/java/src/main/native/src/NvcompJni.cpp @@ -283,7 +283,7 @@ Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdCompressAsync(JNIEnv* env, auto compressed_out_sizes = reinterpret_cast(j_compressed_sizes_out_ptr); auto stream = reinterpret_cast(j_stream); // FIXME how to use these statuses ? They are not used either in the corresponding - // decompression. + // decompressor. auto comp_statuses = rmm::device_uvector(batch_size, stream); auto status = nvcompBatchedZstdCompressAsync(in_ptrs, in_sizes, From 951010e1b0adf6846405f507ef4f015b8cb56d4a Mon Sep 17 00:00:00 2001 From: Firestarman Date: Mon, 14 Jul 2025 14:52:02 +0800 Subject: [PATCH 3/9] API change 2 Signed-off-by: Firestarman --- java/src/main/native/src/NvcompJni.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/src/main/native/src/NvcompJni.cpp b/java/src/main/native/src/NvcompJni.cpp index 9e127dd7119d..4e35f2f5688e 100644 --- a/java/src/main/native/src/NvcompJni.cpp +++ b/java/src/main/native/src/NvcompJni.cpp @@ -148,8 +148,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4Decompres auto chunk_size = static_cast(j_chunk_size); auto total_size = static_cast(j_max_total_size); std::size_t temp_size = 0; - auto status = - nvcompBatchedLZ4DecompressGetTempSizeAsync(batch_size, chunk_size, &temp_size, total_size); + auto status = nvcompBatchedLZ4DecompressGetTempSizeAsync( + batch_size, chunk_size, nvcompBatchedLZ4DecompressDefaultOpts, &temp_size, total_size); check_nvcomp_status(env, status); return static_cast(temp_size); } @@ -310,8 +310,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedZstdDecompre auto chunk_size = static_cast(j_chunk_size); auto total_size = static_cast(j_max_total_size); std::size_t temp_size = 0; - auto status = - nvcompBatchedZstdDecompressGetTempSizeAsync(batch_size, chunk_size, &temp_size, total_size); + auto status = nvcompBatchedZstdDecompressGetTempSizeAsync( + batch_size, chunk_size, nvcompBatchedZstdDecompressDefaultOpts, &temp_size, total_size); check_nvcomp_status(env, status); return static_cast(temp_size); } From 8892a6fdab2fc4f2abdfc0443a8808570d07e2df Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Fri, 8 Aug 2025 10:53:55 -0700 Subject: [PATCH 4/9] use special rapids-cmake branch --- cmake/rapids_config.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/rapids_config.cmake b/cmake/rapids_config.cmake index b2c54a3f27dc..ef7d0418f5b4 100644 --- a/cmake/rapids_config.cmake +++ b/cmake/rapids_config.cmake @@ -35,6 +35,9 @@ if(NOT _rapids_branch) ) endif() +set(rapids-cmake-repo vuule/rapids-cmake) +set(rapids-cmake-branch fea-nvcomp-5) + if(NOT rapids-cmake-version) set(rapids-cmake-version "${RAPIDS_VERSION_MAJOR_MINOR}") endif() From 396fb17ef255d9d81e069ff36848197b0ce7c2c6 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Fri, 8 Aug 2025 13:03:05 -0700 Subject: [PATCH 5/9] update dependencies --- conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- dependencies.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 0e95832dddd9..a785af1b10ce 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -40,7 +40,7 @@ dependencies: - jupyter_client - libcurand-dev - libkvikio==25.10.*,>=0.0.0a0 -- libnvcomp-dev==4.2.0.11 +- libnvcomp-dev==5.0.0.6 - librdkafka>=2.8.0,<2.9.0a0 - librmm==25.10.*,>=0.0.0a0 - make diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index e96b8d819536..e00668e70ff4 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==25.10.*,>=0.0.0a0 -- libnvcomp-dev==4.2.0.11 +- libnvcomp-dev==5.0.0.6 - librdkafka>=2.8.0,<2.9.0a0 - librmm==25.10.*,>=0.0.0a0 - make diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 7ca201655855..7acdfdd9698b 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -26,7 +26,7 @@ flatbuffers_version: - "=24.3.25" nvcomp_version: - - "=4.2.0.11" + - "=5.0.0.6" zlib_version: - ">=1.2.13" diff --git a/dependencies.yaml b/dependencies.yaml index f214bde574e6..205e39a8155a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -460,7 +460,7 @@ dependencies: - output_types: conda packages: # Align nvcomp version with rapids-cmake - - libnvcomp-dev==4.2.0.11 + - libnvcomp-dev==5.0.0.6 rapids_build_skbuild: common: - output_types: [conda, requirements, pyproject] From 3ac71f3c501280e100e7a2aaaea07051745d8796 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Wed, 13 Aug 2025 11:30:00 -0700 Subject: [PATCH 6/9] wheel update --- ci/build_wheel_libcudf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 768ee5c8c0bf..b0fc281a73c6 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -36,7 +36,7 @@ export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_FROM_LIBKVIKIO_WHEEL=ON" # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ - --exclude libnvcomp.so.4 \ + --exclude libnvcomp.so.* \ --exclude libkvikio.so \ --exclude librapids_logger.so \ --exclude librmm.so \ From 6f053c9f12f852db32eee8bac8ff3a31b4db1a28 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Thu, 14 Aug 2025 10:48:39 -0700 Subject: [PATCH 7/9] remove excludes for testing --- ci/build_wheel_cudf.sh | 1 - ci/build_wheel_libcudf.sh | 1 - ci/build_wheel_pylibcudf.sh | 1 - 3 files changed, 3 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index af289fe7229e..7ada7cffba7b 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -24,7 +24,6 @@ echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE} # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ --exclude libcudf.so \ - --exclude libnvcomp.so \ --exclude libkvikio.so \ --exclude librapids_logger.so \ --exclude librmm.so \ diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index b0fc281a73c6..ecff9942607c 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -36,7 +36,6 @@ export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_FROM_LIBKVIKIO_WHEEL=ON" # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ - --exclude libnvcomp.so.* \ --exclude libkvikio.so \ --exclude librapids_logger.so \ --exclude librmm.so \ diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 3f99f75ceb47..97312841b2a8 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -22,7 +22,6 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ --exclude libcudf.so \ - --exclude libnvcomp.so \ --exclude libkvikio.so \ --exclude librapids_logger.so \ --exclude librmm.so \ From d7c989ff17e26a52fd3fe2fc990aacb65a20b812 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Mon, 18 Aug 2025 12:01:23 -0700 Subject: [PATCH 8/9] revert wheel changes --- ci/build_wheel_cudf.sh | 1 + ci/build_wheel_libcudf.sh | 1 + ci/build_wheel_pylibcudf.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 7ada7cffba7b..af289fe7229e 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -24,6 +24,7 @@ echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE} # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ --exclude libcudf.so \ + --exclude libnvcomp.so \ --exclude libkvikio.so \ --exclude librapids_logger.so \ --exclude librmm.so \ diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index ecff9942607c..ae0ab29c7f82 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -37,6 +37,7 @@ export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_FROM_LIBKVIKIO_WHEEL=ON" # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ --exclude libkvikio.so \ + --exclude libnvcomp.so.5 \ --exclude librapids_logger.so \ --exclude librmm.so \ -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" \ diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 97312841b2a8..3f99f75ceb47 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -22,6 +22,7 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_WHEELHOUSE}/lib # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ --exclude libcudf.so \ + --exclude libnvcomp.so \ --exclude libkvikio.so \ --exclude librapids_logger.so \ --exclude librmm.so \ From 050de17d318bc00ee0616d2fd7463d20f1417f42 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 19 Aug 2025 16:49:04 -0700 Subject: [PATCH 9/9] remove cmake hack --- cmake/rapids_config.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmake/rapids_config.cmake b/cmake/rapids_config.cmake index ef7d0418f5b4..b2c54a3f27dc 100644 --- a/cmake/rapids_config.cmake +++ b/cmake/rapids_config.cmake @@ -35,9 +35,6 @@ if(NOT _rapids_branch) ) endif() -set(rapids-cmake-repo vuule/rapids-cmake) -set(rapids-cmake-branch fea-nvcomp-5) - if(NOT rapids-cmake-version) set(rapids-cmake-version "${RAPIDS_VERSION_MAJOR_MINOR}") endif()