From 95365a0a136a2487dc0ad505d33510215c09d8b2 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Fri, 16 May 2025 15:51:16 -0700 Subject: [PATCH 1/6] best guess --- cpp/src/io/comp/comp.cpp | 2 +- cpp/src/io/comp/nvcomp_adapter.cpp | 205 ++++++++++++++++++++++++++++- cpp/src/io/comp/nvcomp_adapter.hpp | 12 +- cpp/src/io/comp/uncomp.cpp | 1 - 4 files changed, 214 insertions(+), 6 deletions(-) diff --git a/cpp/src/io/comp/comp.cpp b/cpp/src/io/comp/comp.cpp index 48d5e2bc3ef..8c0ac881fb5 100644 --- a/cpp/src/io/comp/comp.cpp +++ b/cpp/src/io/comp/comp.cpp @@ -438,7 +438,7 @@ std::optional compress_max_allowed_chunk_size(compression_type compressi return 1ul; } - return nvcomp::required_alignment(*nvcomp_type); + return nvcomp::compress_required_alignment(*nvcomp_type); } [[nodiscard]] size_t max_compressed_size(compression_type compression, uint32_t uncompressed_size) diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index f0647e1c080..2564c27b0b2 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -16,6 +16,7 @@ #include "nvcomp_adapter.hpp" +#include "io/utilities/getenv_or.hpp" #include "nvcomp_adapter.cuh" #include @@ -65,6 +66,13 @@ namespace { return "compression_type(" + std::to_string(static_cast(compression)) + ")"; } +#if NVCOMP_VER_MAJOR >= 5 +[[nodiscard]] bool use_hw_decompression() +{ + return getenv_or("LIBCUDF_HW_DECOMPRESSION", std::string{"ON"}) == "ON"; +} +#endif + #define CHECK_NVCOMP_STATUS(status) \ do { \ CUDF_EXPECTS(status == nvcompStatus_t::nvcompSuccess, \ @@ -76,6 +84,26 @@ namespace { CUDF_FAIL("Unsupported compression type: " + compression_type_name(compression)); \ } while (0) +#if NVCOMP_VER_MAJOR >= 5 +// Dispatcher for nvcompBatchedDecompressGetTempSize +template +auto batched_decompress_get_temp_size(compression_type compression, Args&&... args) +{ + switch (compression) { + case compression_type::SNAPPY: + return nvcompBatchedSnappyDecompressGetTempSize(std::forward(args)...); + case compression_type::ZSTD: + return nvcompBatchedZstdDecompressGetTempSize(std::forward(args)...); + case compression_type::LZ4: + return nvcompBatchedLZ4DecompressGetTempSize(std::forward(args)...); + case compression_type::DEFLATE: + return nvcompBatchedDeflateDecompressGetTempSize(std::forward(args)...); + case compression_type::GZIP: + return nvcompBatchedGzipDecompressGetTempSize(std::forward(args)...); + default: UNSUPPORTED_COMPRESSION(compression); + } +} +#else // Dispatcher for nvcompBatchedDecompressGetTempSizeEx template auto batched_decompress_get_temp_size_ex(compression_type compression, Args&&... args) @@ -94,7 +122,92 @@ auto batched_decompress_get_temp_size_ex(compression_type compression, Args&&... default: UNSUPPORTED_COMPRESSION(compression); } } +#endif +#if NVCOMP_VER_MAJOR >= 5 +// Dispatcher for nvcompBatchedDecompressAsync +template +auto batched_decompress_async(compression_type compression, + bool use_hw_decompression, + const void* const* device_compressed_chunk_ptrs, + const size_t* device_compressed_chunk_bytes, + const size_t* device_uncompressed_buffer_bytes, + size_t* device_uncompressed_chunk_bytes, + size_t num_chunks, + void* const device_temp_ptr, + size_t temp_bytes, + void* const* device_uncompressed_chunk_ptrs, + nvcompStatus_t* device_statuses, + rmm::cuda_stream_view stream) +{ + switch (compression) { + case compression_type::SNAPPY: { + auto opts = nvcompBatchedSnappyDecompressDefaultOpts; + // TODO: adjust opts.algorithm based on use_hw_decompression - no details on this yet + return nvcompBatchedSnappyDecompressAsync(device_compressed_chunk_ptrs, + device_compressed_chunk_bytes, + device_uncompressed_buffer_bytes, + device_uncompressed_chunk_bytes, + num_chunks, + device_temp_ptr, + temp_bytes, + device_uncompressed_chunk_ptrs, + opts, + device_statuses, + stream.value()); + } + case compression_type::ZSTD: + return nvcompBatchedZstdDecompressAsync(device_compressed_chunk_ptrs, + device_compressed_chunk_bytes, + device_uncompressed_buffer_bytes, + device_uncompressed_chunk_bytes, + num_chunks, + device_temp_ptr, + temp_bytes, + device_uncompressed_chunk_ptrs, + nvcompBatchedZstdDecompressDefaultOpts, + device_statuses, + stream.value()); + case compression_type::DEFLATE: + return nvcompBatchedDeflateDecompressAsync(device_compressed_chunk_ptrs, + device_compressed_chunk_bytes, + device_uncompressed_buffer_bytes, + device_uncompressed_chunk_bytes, + num_chunks, + device_temp_ptr, + temp_bytes, + device_uncompressed_chunk_ptrs, + nvcompBatchedDeflateDecompressDefaultOpts, + device_statuses, + stream.value()); + case compression_type::LZ4: + return nvcompBatchedLZ4DecompressAsync(device_compressed_chunk_ptrs, + device_compressed_chunk_bytes, + device_uncompressed_buffer_bytes, + device_uncompressed_chunk_bytes, + num_chunks, + device_temp_ptr, + temp_bytes, + device_uncompressed_chunk_ptrs, + nvcompBatchedLZ4DecompressDefaultOpts, + device_statuses, + stream.value()); + case compression_type::GZIP: + return nvcompBatchedGzipDecompressAsync(device_compressed_chunk_ptrs, + device_compressed_chunk_bytes, + device_uncompressed_buffer_bytes, + device_uncompressed_chunk_bytes, + num_chunks, + device_temp_ptr, + temp_bytes, + device_uncompressed_chunk_ptrs, + nvcompBatchedGzipDecompressDefaultOpts, + device_statuses, + stream.value()); + default: UNSUPPORTED_COMPRESSION(compression); + } +} +#else // Dispatcher for nvcompBatchedDecompressAsync template auto batched_decompress_async(compression_type compression, Args&&... args) @@ -112,6 +225,7 @@ auto batched_decompress_async(compression_type compression, Args&&... args) default: UNSUPPORTED_COMPRESSION(compression); } } +#endif size_t batched_compress_temp_size(compression_type compression, size_t batch_size, @@ -165,6 +279,9 @@ void batched_compress_async(compression_type compression, size_t temp_bytes, void* const* device_compressed_ptrs, size_t* device_compressed_bytes, +#if NVCOMP_VER_MAJOR >= 5 + nvcompStatus_t* device_nvcomp_statuses, +#endif rmm::cuda_stream_view stream) { nvcompStatus_t nvcomp_status = nvcompStatus_t::nvcompSuccess; @@ -179,6 +296,9 @@ void batched_compress_async(compression_type compression, device_compressed_ptrs, device_compressed_bytes, nvcompBatchedSnappyDefaultOpts, +#if NVCOMP_VER_MAJOR >= 5 + device_nvcomp_statuses, +#endif stream.value()); break; case compression_type::DEFLATE: @@ -191,6 +311,9 @@ void batched_compress_async(compression_type compression, device_compressed_ptrs, device_compressed_bytes, nvcompBatchedDeflateDefaultOpts, +#if NVCOMP_VER_MAJOR >= 5 + device_nvcomp_statuses, +#endif stream.value()); break; case compression_type::ZSTD: @@ -203,6 +326,9 @@ void batched_compress_async(compression_type compression, device_compressed_ptrs, device_compressed_bytes, nvcompBatchedZstdDefaultOpts, +#if NVCOMP_VER_MAJOR >= 5 + device_nvcomp_statuses, +#endif stream.value()); break; case compression_type::LZ4: @@ -215,6 +341,9 @@ void batched_compress_async(compression_type compression, device_compressed_ptrs, device_compressed_bytes, nvcompBatchedLZ4DefaultOpts, +#if NVCOMP_VER_MAJOR >= 5 + device_nvcomp_statuses, +#endif stream.value()); break; default: UNSUPPORTED_COMPRESSION(compression); @@ -280,9 +409,14 @@ size_t batched_decompress_temp_size(compression_type compression, size_t max_uncomp_chunk_size, size_t max_total_uncomp_size) { - size_t temp_size = 0; + size_t temp_size = 0; +#if NVCOMP_VER_MAJOR >= 5 + nvcompStatus_t const nvcomp_status = batched_decompress_get_temp_size( + compression, num_chunks, max_uncomp_chunk_size, max_total_uncomp_size, &temp_size); +#else nvcompStatus_t const nvcomp_status = batched_decompress_get_temp_size_ex( compression, num_chunks, max_uncomp_chunk_size, &temp_size, max_total_uncomp_size); +#endif CHECK_NVCOMP_STATUS(nvcomp_status); return temp_size; } @@ -306,6 +440,9 @@ void batched_decompress(compression_type compression, compression, num_chunks, max_uncomp_chunk_size, max_total_uncomp_size); rmm::device_buffer scratch(temp_size, stream); auto const nvcomp_status = batched_decompress_async(compression, +#if NVCOMP_VER_MAJOR >= 5 + use_hw_decompression(), +#endif nvcomp_args.input_data_ptrs.data(), nvcomp_args.input_data_sizes.data(), nvcomp_args.output_data_sizes.data(), @@ -378,6 +515,9 @@ void batched_compress(compression_type compression, CUDF_EXPECTS(is_aligned(scratch.data(), 8), "Compression failed, misaligned scratch buffer"); rmm::device_uvector actual_compressed_data_sizes(num_chunks, stream); +#if NVCOMP_VER_MAJOR >= 5 + rmm::device_uvector nvcomp_statuses(num_chunks, stream); +#endif batched_compress_async(compression, nvcomp_args.input_data_ptrs.data(), @@ -388,9 +528,16 @@ void batched_compress(compression_type compression, scratch.size(), nvcomp_args.output_data_ptrs.data(), actual_compressed_data_sizes.data(), +#if NVCOMP_VER_MAJOR >= 5 + nvcomp_statuses.data(), +#endif stream.value()); +#if NVCOMP_VER_MAJOR >= 5 + update_compression_results(nvcomp_statuses, actual_compressed_data_sizes, results, stream); +#else update_compression_results(actual_compressed_data_sizes, results, stream); +#endif } feature_status_parameters::feature_status_parameters() @@ -480,8 +627,54 @@ std::optional is_decompression_disabled(compression_type compressio return reason; } +#if NVCOMP_VER_MAJOR >= 5 +size_t compress_required_alignment(compression_type compression) +{ + nvcompAlignmentRequirements_t alignments{}; + nvcompStatus_t status; + switch (compression) { + case compression_type::GZIP: + case compression_type::DEFLATE: + status = nvcompBatchedDeflateCompressGetRequiredAlignments(nvcompBatchedDeflateDefaultOpts, + &alignments); + case compression_type::SNAPPY: + status = nvcompBatchedSnappyCompressGetRequiredAlignments(nvcompBatchedSnappyDefaultOpts, + &alignments); + case compression_type::ZSTD: + status = + nvcompBatchedZstdCompressGetRequiredAlignments(nvcompBatchedZstdDefaultOpts, &alignments); + case compression_type::LZ4: + status = + nvcompBatchedLZ4CompressGetRequiredAlignments(nvcompBatchedLZ4DefaultOpts, &alignments); + default: UNSUPPORTED_COMPRESSION(compression); + } + CHECK_NVCOMP_STATUS(status); + return std::max( + {alignments.input_alignment, alignments.output_alignment, alignments.temp_alignment}); +} -size_t required_alignment(compression_type compression) +size_t decompress_required_alignment(compression_type compression) +{ + nvcompAlignmentRequirements_t alignments{}; + nvcompStatus_t status; + switch (compression) { + case compression_type::GZIP: + case compression_type::DEFLATE: + status = nvcompBatchedDeflateDecompressGetRequiredAlignments(&alignments); + case compression_type::SNAPPY: + status = nvcompBatchedSnappyDecompressGetRequiredAlignments(&alignments); + case compression_type::ZSTD: + status = nvcompBatchedZstdDecompressGetRequiredAlignments(&alignments); + case compression_type::LZ4: + status = nvcompBatchedLZ4DecompressGetRequiredAlignments(&alignments); + default: UNSUPPORTED_COMPRESSION(compression); + } + CHECK_NVCOMP_STATUS(status); + return std::max( + {alignments.input_alignment, alignments.output_alignment, alignments.temp_alignment}); +} +#else +size_t compress_required_alignment(compression_type compression) { switch (compression) { case compression_type::GZIP: @@ -493,6 +686,14 @@ size_t required_alignment(compression_type compression) } } +// TODO: check alignment in readers; we can't align input, but should make sure output is aligned +size_t decompress_required_alignment(compression_type compression) +{ + // nvcompBatchedDecompressGetRequiredAlignments is not available in nvcomp < 5.0 + return compress_required_alignment(compression); +} +#endif + std::optional compress_max_allowed_chunk_size(compression_type compression) { switch (compression) { diff --git a/cpp/src/io/comp/nvcomp_adapter.hpp b/cpp/src/io/comp/nvcomp_adapter.hpp index 5c402523168..f7e02b0fb4f 100644 --- a/cpp/src/io/comp/nvcomp_adapter.hpp +++ b/cpp/src/io/comp/nvcomp_adapter.hpp @@ -73,12 +73,20 @@ size_t batched_decompress_temp_size(compression_type compression, uint32_t max_uncomp_chunk_size); /** - * @brief Gets input and output alignment requirements for the given compression type. + * @brief Gets input and output alignment requirements for compression. * * @param compression Compression type * @returns required alignment */ -[[nodiscard]] size_t required_alignment(compression_type compression); +[[nodiscard]] size_t compress_required_alignment(compression_type compression); + +/** + * @brief Gets input and output alignment requirements for decompression. + * + * @param compression Compression type + * @returns required alignment + */ +[[nodiscard]] size_t decompress_required_alignment(compression_type compression); /** * @brief Maximum size of uncompressed chunks that can be compressed with nvCOMP. diff --git a/cpp/src/io/comp/uncomp.cpp b/cpp/src/io/comp/uncomp.cpp index 7f2f9ead78e..92778ec88f8 100644 --- a/cpp/src/io/comp/uncomp.cpp +++ b/cpp/src/io/comp/uncomp.cpp @@ -17,7 +17,6 @@ #include "common_internal.hpp" #include "gpuinflate.hpp" #include "io/utilities/getenv_or.hpp" -#include "io/utilities/hostdevice_vector.hpp" #include "io_uncomp.hpp" #include "nvcomp_adapter.hpp" #include "unbz2.hpp" // bz2 uncompress From 421b9ac92365b56b7d68f39a5f259b564ea694d5 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Wed, 21 May 2025 15:43:03 -0700 Subject: [PATCH 2/6] opts rename --- cpp/src/io/comp/nvcomp_adapter.cpp | 87 +++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 14 deletions(-) diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index 2564c27b0b2..51a45b9bd4d 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -269,6 +269,7 @@ size_t batched_compress_temp_size(compression_type compression, return temp_size; } +#if NVCOMP_VER_MAJOR >= 5 // Dispatcher for nvcompBatchedCompressAsync void batched_compress_async(compression_type compression, void const* const* device_uncompressed_ptrs, @@ -279,9 +280,7 @@ void batched_compress_async(compression_type compression, size_t temp_bytes, void* const* device_compressed_ptrs, size_t* device_compressed_bytes, -#if NVCOMP_VER_MAJOR >= 5 nvcompStatus_t* device_nvcomp_statuses, -#endif rmm::cuda_stream_view stream) { nvcompStatus_t nvcomp_status = nvcompStatus_t::nvcompSuccess; @@ -295,10 +294,8 @@ void batched_compress_async(compression_type compression, temp_bytes, device_compressed_ptrs, device_compressed_bytes, - nvcompBatchedSnappyDefaultOpts, -#if NVCOMP_VER_MAJOR >= 5 + nvcompBatchedSnappyCompressionDefaultOpts, device_nvcomp_statuses, -#endif stream.value()); break; case compression_type::DEFLATE: @@ -310,10 +307,8 @@ void batched_compress_async(compression_type compression, temp_bytes, device_compressed_ptrs, device_compressed_bytes, - nvcompBatchedDeflateDefaultOpts, -#if NVCOMP_VER_MAJOR >= 5 + nvcompBatchedDeflateCompressionDefaultOpts, device_nvcomp_statuses, -#endif stream.value()); break; case compression_type::ZSTD: @@ -325,10 +320,8 @@ void batched_compress_async(compression_type compression, temp_bytes, device_compressed_ptrs, device_compressed_bytes, - nvcompBatchedZstdDefaultOpts, -#if NVCOMP_VER_MAJOR >= 5 + nvcompBatchedZstdCompressionDefaultOpts, device_nvcomp_statuses, -#endif stream.value()); break; case compression_type::LZ4: @@ -340,16 +333,82 @@ void batched_compress_async(compression_type compression, temp_bytes, device_compressed_ptrs, device_compressed_bytes, - nvcompBatchedLZ4DefaultOpts, -#if NVCOMP_VER_MAJOR >= 5 + nvcompBatchedLZ4CompressionDefaultOpts, device_nvcomp_statuses, -#endif stream.value()); break; default: UNSUPPORTED_COMPRESSION(compression); } CHECK_NVCOMP_STATUS(nvcomp_status); } +#else +// Dispatcher for nvcompBatchedCompressAsync +void batched_compress_async(compression_type compression, + void const* const* device_uncompressed_ptrs, + size_t const* device_uncompressed_bytes, + size_t max_uncompressed_chunk_bytes, + size_t batch_size, + void* device_temp_ptr, + size_t temp_bytes, + void* const* device_compressed_ptrs, + size_t* device_compressed_bytes, + rmm::cuda_stream_view stream) +{ + nvcompStatus_t nvcomp_status = nvcompStatus_t::nvcompSuccess; + switch (compression) { + case compression_type::SNAPPY: + nvcomp_status = nvcompBatchedSnappyCompressAsync(device_uncompressed_ptrs, + device_uncompressed_bytes, + max_uncompressed_chunk_bytes, + batch_size, + device_temp_ptr, + temp_bytes, + device_compressed_ptrs, + device_compressed_bytes, + nvcompBatchedSnappyDefaultOpts, + stream.value()); + break; + case compression_type::DEFLATE: + nvcomp_status = nvcompBatchedDeflateCompressAsync(device_uncompressed_ptrs, + device_uncompressed_bytes, + max_uncompressed_chunk_bytes, + batch_size, + device_temp_ptr, + temp_bytes, + device_compressed_ptrs, + device_compressed_bytes, + nvcompBatchedDeflateDefaultOpts, + stream.value()); + break; + case compression_type::ZSTD: + nvcomp_status = nvcompBatchedZstdCompressAsync(device_uncompressed_ptrs, + device_uncompressed_bytes, + max_uncompressed_chunk_bytes, + batch_size, + device_temp_ptr, + temp_bytes, + device_compressed_ptrs, + device_compressed_bytes, + nvcompBatchedZstdDefaultOpts, + stream.value()); + break; + case compression_type::LZ4: + nvcomp_status = nvcompBatchedLZ4CompressAsync(device_uncompressed_ptrs, + device_uncompressed_bytes, + max_uncompressed_chunk_bytes, + batch_size, + device_temp_ptr, + temp_bytes, + device_compressed_ptrs, + device_compressed_bytes, + nvcompBatchedLZ4DefaultOpts, + stream.value()); + break; + default: UNSUPPORTED_COMPRESSION(compression); + } + CHECK_NVCOMP_STATUS(nvcomp_status); +} +#endif bool is_aligned(void const* ptr, std::uintptr_t alignment) noexcept { From 33ee1a7eca4a956b28b39171008b90699f100af9 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Wed, 21 May 2025 16:23:44 -0700 Subject: [PATCH 3/6] Update nvcomp_adapter.hpp --- cpp/src/io/comp/nvcomp_adapter.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/io/comp/nvcomp_adapter.hpp b/cpp/src/io/comp/nvcomp_adapter.hpp index f7e02b0fb4f..b3a7d3ed7c0 100644 --- a/cpp/src/io/comp/nvcomp_adapter.hpp +++ b/cpp/src/io/comp/nvcomp_adapter.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * Copyright (c) 2022-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From f2bd0fbf30be11ae3791e075249513735aeaf66a Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Fri, 23 May 2025 14:46:03 -0700 Subject: [PATCH 4/6] updates --- cpp/src/io/comp/nvcomp_adapter.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index 51a45b9bd4d..f3d236b9a81 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -67,9 +67,11 @@ namespace { } #if NVCOMP_VER_MAJOR >= 5 -[[nodiscard]] bool use_hw_decompression() +[[nodiscard]] std::optional use_hw_decompression() { - return getenv_or("LIBCUDF_HW_DECOMPRESSION", std::string{"ON"}) == "ON"; + auto const env = getenv("LIBCUDF_HW_DECOMPRESSION"); + if (env == nullptr) { return std::nullopt; } + return std::string{env} == "ON"; } #endif @@ -128,7 +130,7 @@ auto batched_decompress_get_temp_size_ex(compression_type compression, Args&&... // Dispatcher for nvcompBatchedDecompressAsync template auto batched_decompress_async(compression_type compression, - bool use_hw_decompression, + std::optional use_hw_decompression, const void* const* device_compressed_chunk_ptrs, const size_t* device_compressed_chunk_bytes, const size_t* device_uncompressed_buffer_bytes, @@ -143,6 +145,10 @@ auto batched_decompress_async(compression_type compression, switch (compression) { case compression_type::SNAPPY: { auto opts = nvcompBatchedSnappyDecompressDefaultOpts; + if (use_hw_decompression.has_value()) { + opts.backend = *use_hw_decompression ? NVCOMP_DECOMPRESS_BACKEND_HARDWARE + : NVCOMP_DECOMPRESS_BACKEND_CUDA; + } // TODO: adjust opts.algorithm based on use_hw_decompression - no details on this yet return nvcompBatchedSnappyDecompressAsync(device_compressed_chunk_ptrs, device_compressed_chunk_bytes, @@ -470,6 +476,7 @@ size_t batched_decompress_temp_size(compression_type compression, { size_t temp_size = 0; #if NVCOMP_VER_MAJOR >= 5 + // TODO: decompression options are expected to be added as parameters in the future nvcompStatus_t const nvcomp_status = batched_decompress_get_temp_size( compression, num_chunks, max_uncomp_chunk_size, max_total_uncomp_size, &temp_size); #else From eb0b651c01d5ff56909d5049cee69b94ff3b8b4a Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 27 May 2025 11:28:02 -0700 Subject: [PATCH 5/6] remove old TODO --- cpp/src/io/comp/nvcomp_adapter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index f3d236b9a81..db412842c9c 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -149,7 +149,6 @@ auto batched_decompress_async(compression_type compression, opts.backend = *use_hw_decompression ? NVCOMP_DECOMPRESS_BACKEND_HARDWARE : NVCOMP_DECOMPRESS_BACKEND_CUDA; } - // TODO: adjust opts.algorithm based on use_hw_decompression - no details on this yet return nvcompBatchedSnappyDecompressAsync(device_compressed_chunk_ptrs, device_compressed_chunk_bytes, device_uncompressed_buffer_bytes, From b85ebd397d2493d6ff1cf463467abff66753322d Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 27 May 2025 16:19:30 -0700 Subject: [PATCH 6/6] code reviews --- cpp/src/io/comp/nvcomp_adapter.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index f3d236b9a81..3045a3b5edd 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -71,7 +71,10 @@ namespace { { auto const env = getenv("LIBCUDF_HW_DECOMPRESSION"); if (env == nullptr) { return std::nullopt; } - return std::string{env} == "ON"; + std::string val{env}; + std::transform( + val.begin(), val.end(), val.begin(), [](unsigned char c) { return std::toupper(c); }); + return val == "ON"; } #endif @@ -131,12 +134,12 @@ auto batched_decompress_get_temp_size_ex(compression_type compression, Args&&... template auto batched_decompress_async(compression_type compression, std::optional use_hw_decompression, - const void* const* device_compressed_chunk_ptrs, - const size_t* device_compressed_chunk_bytes, - const size_t* device_uncompressed_buffer_bytes, + void const* const* device_compressed_chunk_ptrs, + size_t const* device_compressed_chunk_bytes, + size_t const* device_uncompressed_buffer_bytes, size_t* device_uncompressed_chunk_bytes, size_t num_chunks, - void* const device_temp_ptr, + void* device_temp_ptr, size_t temp_bytes, void* const* device_uncompressed_chunk_ptrs, nvcompStatus_t* device_statuses,