From 97c46d85613a4cd1b8c9e5aecefb6534dcdbab54 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:30:34 +0000 Subject: [PATCH 01/42] Add basics --- cpp/include/cudf/io/parquet.hpp | 33 ++++++++++++ cpp/src/io/parquet/decode_fixed.cu | 87 +++++++++++++++++++++++++++--- cpp/src/io/parquet/parquet_gpu.hpp | 5 +- cpp/src/io/parquet/reader_impl.cpp | 46 +++++++++++++++- cpp/src/io/parquet/reader_impl.hpp | 2 + 5 files changed, 164 insertions(+), 9 deletions(-) diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 148dc15b5c69..b58c4a11da4e 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -110,6 +110,8 @@ class parquet_reader_options { type_id _decimal_width{type_id::EMPTY}; // Whether to use JIT compilation for filtering bool _use_jit_filter = false; + // Best-effort: try to output DICTIONARY32 columns for fully dict-encoded string columns + bool _try_output_dict_columns = false; // Whether column name matching is case sensitive. In case of multiple // case-insensitive matches, the first matched column is selected bool _case_sensitive_names = true; @@ -340,6 +342,18 @@ class parquet_reader_options { return _prepend_row_index_column; } + /** + * @brief Returns whether the reader should try to output DICTIONARY32 columns. + * + * When true, the reader may output DICTIONARY32 columns for fully dict-encoded + * string columns instead of fully decoded STRING columns. A DICTIONARY32 column + * consists of an INT32 indices child and a STRING keys child. + * Best-effort: falls back to STRING if the column has mixed encoding. + * + * @return `true` if the reader should try to output DICTIONARY32 columns + */ + [[nodiscard]] bool is_enabled_try_output_dict_columns() const { return _try_output_dict_columns; } + /** * @brief Set a new source location * @@ -633,6 +647,13 @@ class parquet_reader_options { * @param val Boolean indicating whether to prepend the row index column. */ void enable_prepend_row_index_column(bool val) { _prepend_row_index_column = val; } + + /** + * @brief Sets to enable/disable trying to output DICTIONARY32 columns. + * + * @param val Boolean indicating whether to try to output DICTIONARY32 columns + */ + void enable_try_output_dict_columns(bool val) { _try_output_dict_columns = val; } }; /** @@ -931,6 +952,18 @@ class parquet_reader_options_builder { return *this; } + /** + * @brief Sets to enable/disable trying to output DICTIONARY32 columns. + * + * @param val Boolean value whether to try to output DICTIONARY32 columns + * @return this for chaining + */ + parquet_reader_options_builder& try_output_dict_columns(bool val) + { + options._try_output_dict_columns = val; + return *this; + } + /** * @brief move parquet_reader_options member once it's built. */ diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 604c5d4c5bbb..095e6f2c16f5 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -81,6 +81,49 @@ __device__ static void scan_block_exclusive_sum( } } +template +__device__ void decode_dict_indices_as_int32( + page_state_s* s, state_buf* const sb, int start, int end, int t) +{ + constexpr int num_warps = block_size / cudf::detail::warp_size; + constexpr int max_batch_size = num_warps * cudf::detail::warp_size; + + int const leaf_level_index = s->col.max_nesting_depth - 1; + auto const data_out = s->nesting_info[leaf_level_index].data_out; + + int const skipped_leaf_values = s->page.skipped_leaf_values; + + int pos = start; + while (pos < end) { + int const batch_size = min(max_batch_size, end - pos); + int const target_pos = pos + batch_size; + int const thread_pos = pos + t; + + int const dst_pos = [&]() { + if constexpr (copy_mode_t == copy_mode::DIRECT) { + return thread_pos - s->first_row; + } else { + int dst_pos = sb->nz_idx[rolling_index(thread_pos)]; + if constexpr (!has_lists_t) { dst_pos -= s->first_row; } + return dst_pos; + } + }(); + + if (thread_pos < target_pos && dst_pos >= 0) { + int const src_pos = [&]() { + if constexpr (has_lists_t) { return thread_pos + skipped_leaf_values; } + return thread_pos; + }(); + + auto* dst = reinterpret_cast(data_out) + dst_pos; + *dst = sb->dict_idx[rolling_index(src_pos)]; + } + + pos += batch_size; + __syncthreads(); + } +} + template __device__ void decode_fixed_width_values( page_state_s* s, state_buf* const sb, int start, int end, int t) @@ -914,7 +957,18 @@ CUDF_HOST_DEVICE constexpr bool has_dict() (kernel_mask_t == decode_kernel_mask::FIXED_WIDTH_DICT_LIST) || (kernel_mask_t == decode_kernel_mask::STRING_DICT) || (kernel_mask_t == decode_kernel_mask::STRING_DICT_NESTED) || - (kernel_mask_t == decode_kernel_mask::STRING_DICT_LIST); + (kernel_mask_t == decode_kernel_mask::STRING_DICT_LIST) || + (kernel_mask_t == decode_kernel_mask::DICT_INT32) || + (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED) || + (kernel_mask_t == decode_kernel_mask::DICT_INT32_LIST); +} + +template +constexpr bool is_dict_int32_output() +{ + return (kernel_mask_t == decode_kernel_mask::DICT_INT32) || + (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED) || + (kernel_mask_t == decode_kernel_mask::DICT_INT32_LIST); } template @@ -934,7 +988,8 @@ CUDF_HOST_DEVICE constexpr bool has_nesting() (kernel_mask_t == decode_kernel_mask::BYTE_STREAM_SPLIT_FIXED_WIDTH_NESTED) || (kernel_mask_t == decode_kernel_mask::STRING_NESTED) || (kernel_mask_t == decode_kernel_mask::STRING_DICT_NESTED) || - (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_NESTED); + (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_NESTED) || + (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED); } template @@ -946,7 +1001,8 @@ CUDF_HOST_DEVICE constexpr bool has_lists() (kernel_mask_t == decode_kernel_mask::BYTE_STREAM_SPLIT_FIXED_WIDTH_LIST) || (kernel_mask_t == decode_kernel_mask::STRING_LIST) || (kernel_mask_t == decode_kernel_mask::STRING_DICT_LIST) || - (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_LIST); + (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_LIST) || + (kernel_mask_t == decode_kernel_mask::DICT_INT32_LIST); } template @@ -996,6 +1052,7 @@ CUDF_KERNEL void __launch_bounds__(decode_block_size_t, 8) constexpr bool split_decode_t = is_split_decode(); constexpr bool has_strings_t = (static_cast(kernel_mask_t) & STRINGS_MASK_NON_DELTA) != 0; + constexpr bool is_dict_int32_t = is_dict_int32_output(); constexpr int rolling_buf_size = decode_block_size_t * 2; constexpr int rle_run_buffer_size = rle_stream_required_run_buffer_size(); @@ -1170,7 +1227,10 @@ CUDF_KERNEL void __launch_bounds__(decode_block_size_t, 8) } auto decode_values = [&]() { - if constexpr (has_strings_t) { + if constexpr (is_dict_int32_t) { + decode_dict_indices_as_int32( + s, sb, valid_count, next_valid_count, t); + } else if constexpr (has_strings_t) { uint32_t* const str_offsets = s->setup.col.column_string_offset_base + page_string_offset_indices[page_idx]; string_output_offset = @@ -1199,10 +1259,14 @@ CUDF_KERNEL void __launch_bounds__(decode_block_size_t, 8) } // Zero-fill null positions after decoding valid values - if constexpr (has_strings_t || has_lists_t) { + if constexpr (has_strings_t || has_lists_t || is_dict_int32_t) { if (process_nulls) { - uint32_t const dtype_len = has_strings_t ? sizeof(cudf::size_type) : s->dtype_len; - int const num_values = [&]() { + uint32_t const dtype_len = [&]() -> uint32_t { + if constexpr (is_dict_int32_t) { return sizeof(int32_t); } + if constexpr (has_strings_t) { return sizeof(cudf::size_type); } + return s->dtype_len; + }(); + int const num_values = [&]() { if constexpr (has_lists_t) { auto const& ni = s->nesting_info[s->setup.col.max_nesting_depth - 1]; return ni.valid_map_offset - init_valid_map_offset; @@ -1366,6 +1430,15 @@ void decode_page_data(cudf::detail::hostdevice_span pages, case decode_kernel_mask::STRING_STREAM_SPLIT_LIST: launch_kernel(int_tag_t<128>{}, kernel_tag_t{}); break; + case decode_kernel_mask::DICT_INT32: + launch_kernel(int_tag_t<128>{}, kernel_tag_t{}); + break; + case decode_kernel_mask::DICT_INT32_NESTED: + launch_kernel(int_tag_t<128>{}, kernel_tag_t{}); + break; + case decode_kernel_mask::DICT_INT32_LIST: + launch_kernel(int_tag_t<128>{}, kernel_tag_t{}); + break; default: CUDF_EXPECTS(false, "Kernel type not handled by this function"); break; } } diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 6adc4e1b270e..37b9ec12d7f5 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -223,7 +223,10 @@ enum class decode_kernel_mask { STRING_STREAM_SPLIT = (1 << 23), // Run decode kernel for BYTE_STREAM_SPLIT string data STRING_STREAM_SPLIT_NESTED = (1 << 24), // Run decode kernel for nested BYTE_STREAM_SPLIT string data - STRING_STREAM_SPLIT_LIST = (1 << 25) // Run decode kernel for list BYTE_STREAM_SPLIT string data + STRING_STREAM_SPLIT_LIST = (1 << 25), // Run decode kernel for list BYTE_STREAM_SPLIT string data + DICT_INT32 = (1 << 26), // Run decode kernel for dict string → INT32 indices + DICT_INT32_NESTED = (1 << 27), // Run decode kernel for nested dict string → INT32 indices + DICT_INT32_LIST = (1 << 28), // Run decode kernel for list dict string → INT32 indices }; constexpr uint32_t STRINGS_MASK_NON_DELTA = BitOr(decode_kernel_mask::STRING, diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 307015ec2c3f..4c5630585331 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -14,7 +14,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -262,6 +264,21 @@ void reader_impl::decode_page_data(read_mode mode, size_t skip_rows, size_t num_ decode_data(decode_kernel_mask::STRING_STREAM_SPLIT_LIST); } + // launch dict-index-as-int32 decoder for flat columns + if (BitAnd(kernel_mask, decode_kernel_mask::DICT_INT32) != 0) { + decode_data(decode_kernel_mask::DICT_INT32); + } + + // launch dict-index-as-int32 decoder for nested columns + if (BitAnd(kernel_mask, decode_kernel_mask::DICT_INT32_NESTED) != 0) { + decode_data(decode_kernel_mask::DICT_INT32_NESTED); + } + + // launch dict-index-as-int32 decoder for list columns + if (BitAnd(kernel_mask, decode_kernel_mask::DICT_INT32_LIST) != 0) { + decode_data(decode_kernel_mask::DICT_INT32_LIST); + } + // launch delta byte array decoder if (BitAnd(kernel_mask, decode_kernel_mask::DELTA_BYTE_ARRAY) != 0) { decode_delta_byte_array(subpass.pages, @@ -524,7 +541,8 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, options.is_enabled_use_jit_filter(), options.is_enabled_case_sensitive_names(), options.is_enabled_prepend_source_index_column(), - options.is_enabled_prepend_row_index_column()}, + options.is_enabled_prepend_row_index_column(), + options.is_enabled_try_output_dict_columns()}, _sources{std::move(sources)}, _output_chunk_read_limit{chunk_read_limit}, _input_pass_read_limit{pass_read_limit} @@ -917,6 +935,32 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, apply_decimal_width_cast(out_columns); + // Best-effort: when the user requested DICTIONARY32 output for dict-encoded string columns, + // and the direct parquet-dict transcode fast path has not been wired up yet, fall back to + // dictionary-encoding each decoded STRING column post-hoc. + // + // TODO: replace this fallback with a direct transcode path that: + // 1) detects at preprocess time whether all chunks of a string column are fully dict-encoded, + // 2) flips the page kernel_mask from STRING_DICT* to DICT_INT32*, + // 3) allocates the output buffer as INT32 (indices) instead of STRING, + // 4) builds the keys child directly from the page dictionary's string bytes, + // 5) assembles a DICTIONARY32 column without a second pass through cudf::dictionary::encode. + if (_options.try_output_dict_columns) { + static bool logged_fallback = false; + if (!logged_fallback) { + CUDF_LOG_WARN( + "Parquet reader: try_output_dict_columns is enabled, but the direct parquet-dict " + "transcode fast path is not yet available; falling back to dictionary::encode() on " + "decoded STRING columns."); + logged_fallback = true; + } + for (auto& col : out_columns) { + if (col and col->type().id() == type_id::STRING) { + col = cudf::dictionary::encode(col->view(), data_type{type_id::INT32}, _stream, _mr); + } + } + } + if (!_output_metadata) { populate_metadata(out_metadata); // Finally, save the output table metadata into `_output_metadata` for reuse next time. diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 76d88f52e310..4bd3fff50eb8 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -542,6 +542,8 @@ class reader_impl { bool prepend_source_index_column = false; // Whether to prepend the file-local row index column to the output bool prepend_row_index_column = false; + // Whether to try outputting DICTIONARY32 columns for fully dict-encoded string columns + bool try_output_dict_columns = false; } _options; // name to reference converter to extract AST output filter From a9bb2210e4af03f53450c5560e13ae21ce041d8d Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:35:09 +0000 Subject: [PATCH 02/42] Add basic test --- cpp/tests/CMakeLists.txt | 1 + cpp/tests/io/parquet_reader_dict_test.cpp | 126 ++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 cpp/tests/io/parquet_reader_dict_test.cpp diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 25938c6b5e05..06c0462ebed9 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -340,6 +340,7 @@ ConfigureTest( io/parquet_chunked_writer_test.cpp io/parquet_common.cpp io/parquet_misc_test.cpp + io/parquet_reader_dict_test.cpp io/parquet_reader_test.cpp io/parquet_test.cpp io/parquet_v2_test.cpp diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp new file mode 100644 index 000000000000..7837b39331aa --- /dev/null +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -0,0 +1,126 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "parquet_common.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +constexpr cudf::size_type num_rows = 5000; +constexpr cudf::size_type cardinality = num_rows / 10; +constexpr cudf::size_type row_group_size = 1000; +constexpr unsigned int seed = 0xcece; +constexpr double null_probability = 0.1; + +cudf::test::strings_column_wrapper make_low_cardinality_strings() +{ + std::mt19937 engine(seed); + std::uniform_int_distribution value_dist(0, cardinality - 1); + std::bernoulli_distribution null_dist(null_probability); + + std::vector strings(num_rows); + std::vector valids(num_rows); + for (cudf::size_type i = 0; i < num_rows; ++i) { + strings[i] = "str_" + std::to_string(value_dist(engine)); + valids[i] = not null_dist(engine); + } + + return cudf::test::strings_column_wrapper(strings.begin(), strings.end(), valids.begin()); +} + +void write_parquet(cudf::table_view const& input, std::string const& filepath) +{ + auto const options = + cudf::io::chunked_parquet_writer_options::builder(cudf::io::sink_info{filepath}) + .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) + .compression(cudf::io::compression_type::NONE) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .build(); + + cudf::io::chunked_parquet_writer writer(options); + for (auto offset = 0; offset < input.num_rows(); offset += row_group_size) { + auto const length = std::min(row_group_size, input.num_rows() - offset); + auto const chunk = cudf::slice(input, {offset, offset + length}); + writer.write(chunk.front()); + } + writer.close(); +} + +cudf::io::table_with_metadata read_parquet_as_dict(std::string const& filepath) +{ + auto const read_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .try_output_dict_columns(true) + .build(); + return cudf::io::read_parquet(read_opts); +} + +} // namespace + +struct ParquetReaderDictTest : public cudf::test::BaseFixture {}; + +// A flat string column that is fully dictionary-encoded in every row group should be returned +// as a DICTIONARY32 column when `try_output_dict_columns` is enabled, and the decoded keys +// should match the original input. +TEST_F(ParquetReaderDictTest, FlatStringDictTranscode) +{ + auto input_col = make_low_cardinality_strings(); + + auto const input_tbl = cudf::table_view{{input_col}}; + auto const filepath = temp_env->get_temp_filepath("FlatStringDictTranscode.parquet"); + write_parquet(input_tbl, filepath); + + auto const dict_input = cudf::dictionary::encode(input_col); + auto const dict_input_view = cudf::dictionary_column_view(dict_input->view()); + auto const decoded_input = cudf::dictionary::decode(dict_input_view); + + auto const read_table = read_parquet_as_dict(filepath).tbl; + ASSERT_EQ(read_table->num_rows(), num_rows); + ASSERT_EQ(read_table->num_columns(), 1); + + auto const read_col = read_table->view().column(0); + ASSERT_EQ(read_col.type().id(), cudf::type_id::DICTIONARY32) + << "Expected the reader to produce a DICTIONARY32 column when try_output_dict_columns is on"; + + cudf::dictionary_column_view dict_read_view(read_col); + auto const decoded_read = cudf::dictionary::decode(dict_read_view); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, decoded_read->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(decoded_input->view(), decoded_read->view()); +} + +// When the option is not set, the reader should still produce a plain STRING column, regardless +// of whether the source file is fully dictionary-encoded. +TEST_F(ParquetReaderDictTest, FlatStringNoTranscodeByDefault) +{ + auto input_col = make_low_cardinality_strings(); + + auto const input_tbl = cudf::table_view{{input_col}}; + auto const filepath = temp_env->get_temp_filepath("FlatStringNoTranscodeByDefault.parquet"); + write_parquet(input_tbl, filepath); + + auto const read_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}).build(); + auto const read_table = cudf::io::read_parquet(read_opts).tbl; + + ASSERT_EQ(read_table->num_columns(), 1); + auto const read_col = read_table->view().column(0); + ASSERT_EQ(read_col.type().id(), cudf::type_id::STRING); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, read_col); +} From b3a108e37cdbb0c56ce2757b51ee3650cbb9fe7e Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 21 Apr 2026 23:11:59 +0000 Subject: [PATCH 03/42] Remaining infrastructure --- cpp/CMakeLists.txt | 1 + cpp/src/io/parquet/decode_fixed.cu | 2 +- cpp/src/io/parquet/reader_impl.cpp | 50 ++- cpp/src/io/parquet/reader_impl.hpp | 40 +++ .../io/parquet/reader_impl_dict_transcode.cu | 326 ++++++++++++++++++ 5 files changed, 400 insertions(+), 19 deletions(-) create mode 100644 cpp/src/io/parquet/reader_impl_dict_transcode.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 81e447291a5b..605e78eb0c14 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -811,6 +811,7 @@ add_library( src/io/parquet/reader_impl.cpp src/io/parquet/reader_impl_chunking.cu src/io/parquet/reader_impl_chunking_utils.cu + src/io/parquet/reader_impl_dict_transcode.cu src/io/parquet/reader_impl_helpers.cpp src/io/parquet/reader_impl_preprocess.cu src/io/parquet/reader_impl_preprocess_utils.cu diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 095e6f2c16f5..77415039dfde 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -964,7 +964,7 @@ CUDF_HOST_DEVICE constexpr bool has_dict() } template -constexpr bool is_dict_int32_output() +CUDF_HOST_DEVICE constexpr bool is_dict_int32_output() { return (kernel_mask_t == decode_kernel_mask::DICT_INT32) || (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED) || diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 4c5630585331..8a22c2886272 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -547,6 +547,14 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, _output_chunk_read_limit{chunk_read_limit}, _input_pass_read_limit{pass_read_limit} { + // Direct parquet-dict → DICTIONARY32 transcode currently only supports single-pass, non-chunked + // reads. Splitting rowgroups across passes/subpasses would require aligning dictionary keys + // across passes, which we don't support yet. + CUDF_EXPECTS( + not _options.try_output_dict_columns or (chunk_read_limit == 0 and pass_read_limit == 0), + "try_output_dict_columns is only supported for single-pass reads; it cannot be combined " + "with a non-zero chunk_read_limit or pass_read_limit."); + // Open and parse the source dataset metadata CUDF_EXPECTS(file_metadatas.empty() or file_metadatas.size() == _sources.size(), "Encountered a mismatch in the number of provided data sources and metadatas"); @@ -713,6 +721,12 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) auto& subpass = *pass.subpass; auto const& read_info = subpass.output_chunk_read_info[subpass.current_output_chunk]; + // If the caller asked for direct parquet-dict → DICTIONARY32 transcode, detect per-column + // eligibility and mutate `_output_buffers` / `subpass.pages` before we allocate column buffers + // or dispatch decode kernels. This has to happen before `preprocess_chunk_strings` / + // `allocate_columns` because those branch on `subpass.kernel_mask` and on `out_buf.type`. + prepare_dict_transcode(); + // computes: // PageNestingInfo::batch_size for each level of nesting, for each page, taking row bounds into // account. PageInfo::skipped_values, which tells us where to start decoding in the input to @@ -735,6 +749,11 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // Allocate memory buffers for the output columns. allocate_columns(mode, read_info.skip_rows, read_info.num_rows); + // Zero-init the INT32 index buffers of dict-transcoded columns before launching decode, so + // that null positions (which the DICT_INT32 kernel does not write to) carry well-defined + // indices in the produced DICTIONARY32 output. + zero_init_dict_transcoded_index_buffers(); + // Parse data into the output buffers. decode_page_data(mode, read_info.skip_rows, read_info.num_rows); @@ -762,6 +781,12 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) } } + // For any columns that were selected for direct parquet-dict → DICTIONARY32 transcode in + // `prepare_dict_transcode`, the entries in `out_columns` are currently INT32 indices columns. + // Assemble them into DICTIONARY32 columns here by attaching per-chunk keys and shifting + // per-chunk indices so the concatenation refers to the unified keys child. + assemble_dict_transcoded_columns(out_columns); + out_columns = cudf::structs::detail::enforce_null_consistency(std::move(out_columns), _stream, _mr); @@ -935,25 +960,14 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, apply_decimal_width_cast(out_columns); - // Best-effort: when the user requested DICTIONARY32 output for dict-encoded string columns, - // and the direct parquet-dict transcode fast path has not been wired up yet, fall back to - // dictionary-encoding each decoded STRING column post-hoc. - // - // TODO: replace this fallback with a direct transcode path that: - // 1) detects at preprocess time whether all chunks of a string column are fully dict-encoded, - // 2) flips the page kernel_mask from STRING_DICT* to DICT_INT32*, - // 3) allocates the output buffer as INT32 (indices) instead of STRING, - // 4) builds the keys child directly from the page dictionary's string bytes, - // 5) assembles a DICTIONARY32 column without a second pass through cudf::dictionary::encode. + // When the user requested DICTIONARY32 output for flat string columns, the direct transcode + // fast path in `prepare_dict_transcode`/`assemble_dict_transcoded_columns` has already + // assembled DICTIONARY32 columns for all *eligible* flat STRING columns (i.e. those whose + // chunks were fully dictionary-encoded). For columns that were *not* eligible (e.g. chunks + // with mixed or non-dictionary encodings, nested schemas, or columns added as empty columns + // above), fall back to a post-hoc `cudf::dictionary::encode` so the user still gets a + // DICTIONARY32 column from every flat string column in the output table. if (_options.try_output_dict_columns) { - static bool logged_fallback = false; - if (!logged_fallback) { - CUDF_LOG_WARN( - "Parquet reader: try_output_dict_columns is enabled, but the direct parquet-dict " - "transcode fast path is not yet available; falling back to dictionary::encode() on " - "decoded STRING columns."); - logged_fallback = true; - } for (auto& col : out_columns) { if (col and col->type().id() == type_id::STRING) { col = cudf::dictionary::encode(col->view(), data_type{type_id::INT32}, _stream, _mr); diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 4bd3fff50eb8..13190e3436c6 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -188,6 +188,41 @@ class reader_impl { */ void preprocess_chunk_strings(read_mode mode, row_range const& read_info); + /** + * @brief Detect per-column eligibility for direct Parquet-dict → DICTIONARY32 transcode, and + * apply the required host-side mutations to `_output_buffers` and `subpass.pages` so that the + * subsequent allocate/decode path produces INT32 indices for eligible columns. + * + * Must be called after `prepare_data()` (so that `pass.chunks`, `pass.pages` and + * `subpass.pages` are populated on the host) and before `preprocess_chunk_strings()` / + * `allocate_columns()` / `decode_page_data()`. + * + * Populates `_dict_transcode_eligible` with a bool per input column indicating whether the + * column will be assembled as a DICTIONARY32 output later in `assemble_dict_transcoded_columns`. + */ + void prepare_dict_transcode(); + + /** + * @brief Zero-initialize the INT32 output buffers of dict-transcoded columns so that null rows + * carry a well-defined dictionary index (the `DICT_INT32` kernel skips null positions). + * + * Must be called after `allocate_columns` and before `decode_page_data`. + */ + void zero_init_dict_transcoded_index_buffers(); + + /** + * @brief Assemble DICTIONARY32 output columns for input columns that were marked eligible by + * `prepare_dict_transcode`. Each chunk's INT32 indices produced by the `DICT_INT32` kernel are + * shifted by the cumulative number of keys from prior chunks, and the per-chunk keys (built + * from `pass.str_dict_index`) are concatenated into a single keys child. + * + * Non-eligible flat STRING columns are left untouched here and are expected to go through the + * post-hoc `cudf::dictionary::encode` fallback in `finalize_output`. + * + * @param out_columns The output columns vector to mutate in place. + */ + void assemble_dict_transcoded_columns(std::vector>& out_columns); + /** * @brief Copies over the relevant page mask information for the subpass */ @@ -602,6 +637,11 @@ class reader_impl { std::size_t _output_chunk_read_limit{0}; // output chunk size limit in bytes std::size_t _input_pass_read_limit{0}; // input pass memory usage limit in bytes + + // Per-input-column flag indicating whether that column was selected for direct + // Parquet-dict → DICTIONARY32 transcode in `prepare_dict_transcode()`. Populated before decode + // and consumed in `assemble_dict_transcoded_columns()`. + std::vector _dict_transcode_eligible; }; } // namespace cudf::io::parquet::detail diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu new file mode 100644 index 000000000000..354144b3157e --- /dev/null +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -0,0 +1,326 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "reader_impl.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace cudf::io::parquet::detail { + +namespace { + +// Host-side counterpart of `is_string_col` in parquet_gpu.hpp. Kept narrow: for direct +// Parquet-dict → DICTIONARY32 transcode we only accept pure BYTE_ARRAY columns without a +// DECIMAL logical type and without the strings-to-categorical flag. FIXED_LEN_BYTE_ARRAY is +// deliberately excluded because it is typically a binary payload. +[[nodiscard]] bool is_host_byte_array_string_chunk(ColumnChunkDesc const& chunk) +{ + if (chunk.physical_type != Type::BYTE_ARRAY) { return false; } + if (chunk.is_strings_to_cat) { return false; } + if (chunk.logical_type.has_value() and chunk.logical_type->type == LogicalType::DECIMAL) { + return false; + } + return true; +} + +// Is the given page encoding a dictionary-indices encoding? Both PLAIN_DICTIONARY (legacy) and +// RLE_DICTIONARY are valid encodings for data pages that reference a parquet dictionary page. +[[nodiscard]] bool is_dict_data_page_encoding(Encoding enc) +{ + return enc == Encoding::PLAIN_DICTIONARY or enc == Encoding::RLE_DICTIONARY; +} + +} // namespace + +void reader_impl::prepare_dict_transcode() +{ + CUDF_FUNC_RANGE(); + + _dict_transcode_eligible.assign(_input_columns.size(), false); + + if (not _options.try_output_dict_columns) { return; } + if (_pass_itm_data == nullptr or _pass_itm_data->subpass == nullptr) { return; } + + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; + + if (pass.chunks.empty() or subpass.pages.size() == 0) { return; } + + // Step 1: determine per-input-column eligibility. A column is eligible iff + // - the corresponding output buffer is currently typed as STRING (i.e. a flat string column), + // - every chunk of that column is a BYTE_ARRAY string chunk with a dictionary page, + // - every data page of every chunk of that column uses (PLAIN|RLE)_DICTIONARY encoding, + // - the chunk has a flat (non-list, non-nested) schema. + // + // We scan host-side pass.chunks and pass.pages here rather than subpass.pages because + // subpass.pages may be a subset. For single-pass single-subpass reads (the only configuration + // in which try_output_dict_columns is supported), subpass.pages == pass.pages. + auto const num_input_cols = _input_columns.size(); + + std::vector col_has_string_buffer(num_input_cols, false); + std::vector col_all_chunks_string = std::vector(num_input_cols, true); + std::vector col_has_any_chunk = std::vector(num_input_cols, false); + std::vector col_all_pages_dict = std::vector(num_input_cols, true); + + for (size_t i = 0; i < num_input_cols; ++i) { + auto const& input_col = _input_columns[i]; + // Flat columns have nesting_depth == 1, and the root output buffer is the leaf. + if (input_col.nesting_depth() != 1) { continue; } + auto const& out_buf = _output_buffers[input_col.nesting[0]]; + if (out_buf.type.id() == type_id::STRING) { col_has_string_buffer[i] = true; } + } + + for (size_t c = 0; c < pass.chunks.size(); ++c) { + auto const& chunk = pass.chunks[c]; + auto const col_idx = chunk.src_col_index; + if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } + col_has_any_chunk[col_idx] = true; + if (chunk.max_nesting_depth != 1 or chunk.max_level[level_type::REPETITION] != 0 or + not is_host_byte_array_string_chunk(chunk) or chunk.num_dict_pages < 1) { + col_all_chunks_string[col_idx] = false; + } + } + + for (auto const& page : pass.pages) { + if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } + auto const chunk_idx = page.chunk_idx; + if (chunk_idx < 0 or static_cast(chunk_idx) >= pass.chunks.size()) { continue; } + auto const col_idx = pass.chunks[chunk_idx].src_col_index; + if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } + if (not is_dict_data_page_encoding(page.encoding)) { + col_all_pages_dict[col_idx] = false; + } + } + + for (size_t i = 0; i < num_input_cols; ++i) { + _dict_transcode_eligible[i] = col_has_string_buffer[i] and col_has_any_chunk[i] and + col_all_chunks_string[i] and col_all_pages_dict[i]; + } + + auto const num_eligible = + std::count(_dict_transcode_eligible.begin(), _dict_transcode_eligible.end(), true); + if (num_eligible == 0) { return; } + + // Step 2: flip the output buffer type for eligible columns from STRING → INT32. The subsequent + // `allocate_columns` call will then allocate an INT32 buffer that the DICT_INT32 kernel can + // write directly into. + for (size_t i = 0; i < num_input_cols; ++i) { + if (not _dict_transcode_eligible[i]) { continue; } + auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; + out_buf.type = data_type{type_id::INT32}; + // Leaf flat columns have no children; nothing more to adjust. + } + + // Step 3: rewrite per-page kernel_mask for eligible columns on the host subpass pages from + // STRING_DICT → DICT_INT32, then H2D so the device pages agree. Only the flat variant is + // considered here (eligibility requires `max_nesting_depth == 1`). + bool any_rewritten = false; + for (size_t p = 0; p < subpass.pages.size(); ++p) { + auto& page = subpass.pages[p]; + if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } + auto const chunk_idx = page.chunk_idx; + if (chunk_idx < 0 or static_cast(chunk_idx) >= pass.chunks.size()) { continue; } + auto const col_idx = pass.chunks[chunk_idx].src_col_index; + if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } + if (not _dict_transcode_eligible[col_idx]) { continue; } + if (page.kernel_mask == decode_kernel_mask::STRING_DICT) { + page.kernel_mask = decode_kernel_mask::DICT_INT32; + any_rewritten = true; + } + } + + if (any_rewritten) { + // Push the rewritten kernel_masks back to device so subsequent decode kernels dispatch + // correctly. Then refresh the aggregated subpass.kernel_mask on the host by re-OR'ing all + // page kernel_masks. + subpass.pages.host_to_device_async(_stream); + uint32_t refreshed = 0; + for (size_t p = 0; p < subpass.pages.size(); ++p) { + refreshed |= static_cast(subpass.pages[p].kernel_mask); + } + subpass.kernel_mask = refreshed; + _stream.synchronize(); + } +} + +void reader_impl::zero_init_dict_transcoded_index_buffers() +{ + if (not _options.try_output_dict_columns) { return; } + if (_dict_transcode_eligible.empty()) { return; } + + // The `DICT_INT32` kernel only writes to positions with valid definition levels, leaving null + // slots untouched. Since `allocate_columns` uses `memset_data=false` by default, the INT32 + // output buffer for a transcoded column may contain uninitialized bytes at null positions. + // Zero them here so null rows carry a well-defined (valid) index into the dictionary keys. + for (size_t i = 0; i < _input_columns.size(); ++i) { + if (not _dict_transcode_eligible[i]) { continue; } + auto const& input_col = _input_columns[i]; + auto& out_buf = _output_buffers[input_col.nesting[0]]; + if (out_buf.type.id() != type_id::INT32) { continue; } + if (out_buf.data() == nullptr or out_buf.size == 0) { continue; } + CUDF_CUDA_TRY(cudaMemsetAsync( + out_buf.data(), 0, static_cast(out_buf.size) * sizeof(int32_t), _stream.value())); + } +} + +namespace { + +// Build a STRING keys column covering the dictionary entries of a contiguous range of chunks of a +// single input column. `str_dict_index` is the device-resident pointer to the pass-wide +// `string_index_pair` buffer. `entry_count` is the total number of entries contributed by the +// range. Because the pass's str_dict_index buffer already stores pairs in chunk-index order with +// packed offsets, and our caller passes a range covering a contiguous sub-slice for one column, +// we can simply wrap the pointer in a span and hand it to the strings factory. +[[nodiscard]] std::unique_ptr make_keys_column_from_index_pairs( + string_index_pair const* begin, + size_type entry_count, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + if (entry_count <= 0) { return cudf::make_empty_column(data_type{type_id::STRING}); } + return cudf::strings::detail::make_strings_column(begin, begin + entry_count, stream, mr); +} + +} // namespace + +void reader_impl::assemble_dict_transcoded_columns( + std::vector>& out_columns) +{ + CUDF_FUNC_RANGE(); + + if (not _options.try_output_dict_columns) { return; } + if (_dict_transcode_eligible.empty()) { return; } + if (_pass_itm_data == nullptr) { return; } + + auto& pass = *_pass_itm_data; + + // For each eligible input column, collect its chunks in row-group order, build a per-chunk + // DICTIONARY32 segment (local 0-based indices + per-chunk keys column), and concatenate. + // + // IMPORTANT: Each segment carries row-group-local indices into its own keys column. We do NOT + // pre-shift indices into a global keyspace, because `cudf::concatenate` on dictionary columns + // (via `cudf::dictionary::detail::concatenate`) already re-maps the indices using + // `compute_children_offsets_fn`. Pre-shifting would cause double-offsetting and out-of-bounds + // reads in the `dispatch_compute_indices` kernel. + for (size_t i = 0; i < _input_columns.size(); ++i) { + if (not _dict_transcode_eligible[i]) { continue; } + + // Gather the ordered list of chunk indices belonging to this input column. + std::vector chunk_indices; + chunk_indices.reserve(pass.chunks.size() / std::max(_input_columns.size(), 1)); + for (size_t c = 0; c < pass.chunks.size(); ++c) { + if (pass.chunks[c].src_col_index == static_cast(i)) { chunk_indices.push_back(c); } + } + if (chunk_indices.empty()) { continue; } + + // Per-chunk key counts derived from the dictionary page's num_input_values, mirrored back to + // the host when `pass.pages` was copied by `decode_page_headers`. + std::vector chunk_key_counts(chunk_indices.size(), 0); + for (size_t k = 0; k < chunk_indices.size(); ++k) { + auto const& chunk = pass.chunks[chunk_indices[k]]; + if (chunk.dict_page != nullptr) { + for (auto const& page : pass.pages) { + if (page.chunk_idx == static_cast(chunk_indices[k]) and + (page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { + chunk_key_counts[k] = static_cast(page.num_input_values); + break; + } + } + } + } + + // Grab ownership of the decoded INT32 indices column and its raw device pointer so we can + // carve per-chunk slices out of it without additional synchronization. + auto& indices_col = out_columns[i]; + CUDF_EXPECTS(indices_col != nullptr and indices_col->type().id() == type_id::INT32, + "Expected INT32 indices column for dict-transcoded flat string column"); + + std::vector chunk_row_offsets(chunk_indices.size() + 1, 0); + for (size_t k = 0; k < chunk_indices.size(); ++k) { + chunk_row_offsets[k + 1] = + chunk_row_offsets[k] + static_cast(pass.chunks[chunk_indices[k]].num_rows); + } + + auto indices_contents = indices_col->release(); + auto* indices_data = static_cast(indices_contents.data->data()); + auto const indices_size = + static_cast(indices_contents.data->size() / sizeof(int32_t)); + CUDF_EXPECTS(indices_size == chunk_row_offsets.back(), + "Row counts on pass chunks must sum to the indices column size"); + + std::vector> dict_segments; + dict_segments.reserve(chunk_indices.size()); + + for (size_t k = 0; k < chunk_indices.size(); ++k) { + auto const& chunk = pass.chunks[chunk_indices[k]]; + auto const row_begin = chunk_row_offsets[k]; + auto const row_end = chunk_row_offsets[k + 1]; + auto const key_count = chunk_key_counts[k]; + auto const chunk_nrows = row_end - row_begin; + + // Copy this chunk's slice of (unshifted, local-to-chunk) INT32 indices into its own buffer. + rmm::device_buffer seg_data(chunk_nrows * sizeof(int32_t), _stream, _mr); + if (chunk_nrows > 0) { + CUDF_CUDA_TRY(cudaMemcpyAsync(seg_data.data(), + indices_data + row_begin, + chunk_nrows * sizeof(int32_t), + cudaMemcpyDeviceToDevice, + _stream.value())); + } + + // Slice the null mask into this chunk's range. + rmm::device_buffer seg_null_mask{}; + size_type seg_null_count = 0; + if (indices_contents.null_mask != nullptr and indices_contents.null_mask->size() > 0) { + auto const* src_mask_ptr = + static_cast(indices_contents.null_mask->data()); + seg_null_mask = cudf::detail::copy_bitmask(src_mask_ptr, row_begin, row_end, _stream, _mr); + seg_null_count = cudf::null_count(src_mask_ptr, row_begin, row_end, _stream); + } + + auto seg_indices = std::make_unique(data_type{type_id::INT32}, + chunk_nrows, + std::move(seg_data), + std::move(seg_null_mask), + seg_null_count); + + auto seg_keys = + make_keys_column_from_index_pairs(chunk.str_dict_index, key_count, _stream, _mr); + + // Assemble a DICTIONARY32 column for this chunk segment. Indices are local 0-based; the + // subsequent `cudf::concatenate` will rewrite them against the unified, deduplicated keys. + auto seg_dict = + cudf::make_dictionary_column(std::move(seg_keys), std::move(seg_indices), _stream, _mr); + dict_segments.emplace_back(std::move(seg_dict)); + } + + // Concatenate all segments into the final DICTIONARY32 column for this input column. + // `cudf::dictionary::detail::concatenate` deduplicates + sorts keys and recomputes indices. + if (dict_segments.size() == 1) { + out_columns[i] = std::move(dict_segments.front()); + } else { + std::vector segment_views; + segment_views.reserve(dict_segments.size()); + for (auto const& seg : dict_segments) { + segment_views.emplace_back(seg->view()); + } + out_columns[i] = cudf::concatenate(segment_views, _stream, _mr); + } + } +} + +} // namespace cudf::io::parquet::detail From 040cb9f6368f4d60d377d11e0321e9e284cb07f9 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 21 Apr 2026 23:42:20 +0000 Subject: [PATCH 04/42] Code cleanup --- cpp/src/io/parquet/reader_impl.cpp | 7 +- .../io/parquet/reader_impl_dict_transcode.cu | 424 +++++++++--------- 2 files changed, 218 insertions(+), 213 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 8a22c2886272..9e559dc624d2 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -965,12 +965,13 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, // assembled DICTIONARY32 columns for all *eligible* flat STRING columns (i.e. those whose // chunks were fully dictionary-encoded). For columns that were *not* eligible (e.g. chunks // with mixed or non-dictionary encodings, nested schemas, or columns added as empty columns - // above), fall back to a post-hoc `cudf::dictionary::encode` so the user still gets a + // above), fall back to a post-hoc `dictionary::detail::encode` so the user still gets a // DICTIONARY32 column from every flat string column in the output table. if (_options.try_output_dict_columns) { for (auto& col : out_columns) { if (col and col->type().id() == type_id::STRING) { - col = cudf::dictionary::encode(col->view(), data_type{type_id::INT32}, _stream, _mr); + col = cudf::dictionary::detail::encode( + col->view(), data_type{type_id::INT32}, _stream, _mr); } } } diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 354144b3157e..be2d3a9f1767 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -7,23 +7,26 @@ #include #include -#include -#include +#include +#include #include #include -#include #include #include -#include + +#include #include +#include +#include +#include #include namespace cudf::io::parquet::detail { namespace { -// Host-side counterpart of `is_string_col` in parquet_gpu.hpp. Kept narrow: for direct +// Host-side counterpart of `is_string_col` in `parquet_gpu.hpp`. Kept narrow: for direct // Parquet-dict → DICTIONARY32 transcode we only accept pure BYTE_ARRAY columns without a // DECIMAL logical type and without the strings-to-categorical flag. FIXED_LEN_BYTE_ARRAY is // deliberately excluded because it is typically a binary payload. @@ -37,13 +40,49 @@ namespace { return true; } -// Is the given page encoding a dictionary-indices encoding? Both PLAIN_DICTIONARY (legacy) and -// RLE_DICTIONARY are valid encodings for data pages that reference a parquet dictionary page. +// Both PLAIN_DICTIONARY (legacy) and RLE_DICTIONARY are valid encodings for data pages that +// reference a parquet dictionary page. [[nodiscard]] bool is_dict_data_page_encoding(Encoding enc) { return enc == Encoding::PLAIN_DICTIONARY or enc == Encoding::RLE_DICTIONARY; } +// Per-input-column eligibility flags. Each column must satisfy all of these conditions to be +// eligible for direct Parquet-dict → DICTIONARY32 transcode. +struct column_eligibility { + bool has_string_buffer = false; + bool has_any_chunk = false; + bool all_chunks_string = true; + bool all_pages_dict = true; + + [[nodiscard]] bool is_eligible() const + { + return has_string_buffer and has_any_chunk and all_chunks_string and all_pages_dict; + } +}; + +// Classify a chunk against its column's eligibility state. +void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) +{ + e.has_any_chunk = true; + if (chunk.max_nesting_depth != 1 or chunk.max_level[level_type::REPETITION] != 0 or + not is_host_byte_array_string_chunk(chunk) or chunk.num_dict_pages < 1) { + e.all_chunks_string = false; + } +} + +// Build a STRING keys column covering the dictionary entries of a single chunk of a single input +// column. `begin` points into the pass-wide `string_index_pair` buffer. +[[nodiscard]] std::unique_ptr make_keys_column_from_index_pairs( + string_index_pair const* begin, + size_type entry_count, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + if (entry_count <= 0) { return cudf::make_empty_column(data_type{type_id::STRING}); } + return cudf::strings::detail::make_strings_column(begin, begin + entry_count, stream, mr); +} + } // namespace void reader_impl::prepare_dict_transcode() @@ -60,143 +99,123 @@ void reader_impl::prepare_dict_transcode() if (pass.chunks.empty() or subpass.pages.size() == 0) { return; } - // Step 1: determine per-input-column eligibility. A column is eligible iff + // Determine per-input-column eligibility. A column is eligible iff // - the corresponding output buffer is currently typed as STRING (i.e. a flat string column), // - every chunk of that column is a BYTE_ARRAY string chunk with a dictionary page, // - every data page of every chunk of that column uses (PLAIN|RLE)_DICTIONARY encoding, // - the chunk has a flat (non-list, non-nested) schema. // - // We scan host-side pass.chunks and pass.pages here rather than subpass.pages because - // subpass.pages may be a subset. For single-pass single-subpass reads (the only configuration - // in which try_output_dict_columns is supported), subpass.pages == pass.pages. + // We scan host-side `pass.chunks` and `pass.pages` here rather than `subpass.pages` because + // `subpass.pages` may be a subset. For single-pass single-subpass reads (the only configuration + // in which `try_output_dict_columns` is supported), `subpass.pages == pass.pages`. auto const num_input_cols = _input_columns.size(); - - std::vector col_has_string_buffer(num_input_cols, false); - std::vector col_all_chunks_string = std::vector(num_input_cols, true); - std::vector col_has_any_chunk = std::vector(num_input_cols, false); - std::vector col_all_pages_dict = std::vector(num_input_cols, true); - - for (size_t i = 0; i < num_input_cols; ++i) { - auto const& input_col = _input_columns[i]; - // Flat columns have nesting_depth == 1, and the root output buffer is the leaf. - if (input_col.nesting_depth() != 1) { continue; } - auto const& out_buf = _output_buffers[input_col.nesting[0]]; - if (out_buf.type.id() == type_id::STRING) { col_has_string_buffer[i] = true; } - } - - for (size_t c = 0; c < pass.chunks.size(); ++c) { - auto const& chunk = pass.chunks[c]; - auto const col_idx = chunk.src_col_index; + std::vector elig(num_input_cols); + + // Seed from the output buffer type: only flat STRING columns have a single leaf buffer whose + // type is STRING and can be flipped in place. + std::for_each( + cuda::counting_iterator{0}, cuda::counting_iterator{num_input_cols}, [&](size_t i) { + auto const& input_col = _input_columns[i]; + if (input_col.nesting_depth() != 1) { return; } + auto const& out_buf = _output_buffers[input_col.nesting[0]]; + if (out_buf.type.id() == type_id::STRING) { elig[i].has_string_buffer = true; } + }); + + // Fold per-chunk info into the per-column eligibility flags. + for (auto const& chunk : pass.chunks) { + auto const col_idx = chunk.src_col_index; if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } - col_has_any_chunk[col_idx] = true; - if (chunk.max_nesting_depth != 1 or chunk.max_level[level_type::REPETITION] != 0 or - not is_host_byte_array_string_chunk(chunk) or chunk.num_dict_pages < 1) { - col_all_chunks_string[col_idx] = false; - } + update_from_chunk(elig[col_idx], chunk); } + // Any non-dictionary data-page encoding disqualifies the whole column. Dictionary pages + // themselves (PAGEINFO_FLAGS_DICTIONARY) are skipped since they are not data pages. for (auto const& page : pass.pages) { if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } auto const chunk_idx = page.chunk_idx; if (chunk_idx < 0 or static_cast(chunk_idx) >= pass.chunks.size()) { continue; } auto const col_idx = pass.chunks[chunk_idx].src_col_index; if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } - if (not is_dict_data_page_encoding(page.encoding)) { - col_all_pages_dict[col_idx] = false; - } + if (not is_dict_data_page_encoding(page.encoding)) { elig[col_idx].all_pages_dict = false; } } - for (size_t i = 0; i < num_input_cols; ++i) { - _dict_transcode_eligible[i] = col_has_string_buffer[i] and col_has_any_chunk[i] and - col_all_chunks_string[i] and col_all_pages_dict[i]; - } + std::transform( + elig.begin(), elig.end(), _dict_transcode_eligible.begin(), [](column_eligibility const& e) { + return e.is_eligible(); + }); auto const num_eligible = std::count(_dict_transcode_eligible.begin(), _dict_transcode_eligible.end(), true); if (num_eligible == 0) { return; } - // Step 2: flip the output buffer type for eligible columns from STRING → INT32. The subsequent + // Flip the output buffer type for eligible columns from STRING → INT32. The subsequent // `allocate_columns` call will then allocate an INT32 buffer that the DICT_INT32 kernel can // write directly into. - for (size_t i = 0; i < num_input_cols; ++i) { - if (not _dict_transcode_eligible[i]) { continue; } - auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; - out_buf.type = data_type{type_id::INT32}; - // Leaf flat columns have no children; nothing more to adjust. - } - - // Step 3: rewrite per-page kernel_mask for eligible columns on the host subpass pages from + std::for_each( + cuda::counting_iterator{0}, cuda::counting_iterator{num_input_cols}, [&](size_t i) { + if (not _dict_transcode_eligible[i]) { return; } + auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; + out_buf.type = data_type{type_id::INT32}; + }); + + // Rewrite per-page `kernel_mask` for eligible columns on the host subpass pages from // STRING_DICT → DICT_INT32, then H2D so the device pages agree. Only the flat variant is // considered here (eligibility requires `max_nesting_depth == 1`). bool any_rewritten = false; - for (size_t p = 0; p < subpass.pages.size(); ++p) { - auto& page = subpass.pages[p]; - if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } + std::for_each(subpass.pages.host_begin(), subpass.pages.host_end(), [&](PageInfo& page) { + if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { return; } auto const chunk_idx = page.chunk_idx; - if (chunk_idx < 0 or static_cast(chunk_idx) >= pass.chunks.size()) { continue; } + if (chunk_idx < 0 or static_cast(chunk_idx) >= pass.chunks.size()) { return; } auto const col_idx = pass.chunks[chunk_idx].src_col_index; - if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } - if (not _dict_transcode_eligible[col_idx]) { continue; } + if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { return; } + if (not _dict_transcode_eligible[col_idx]) { return; } if (page.kernel_mask == decode_kernel_mask::STRING_DICT) { page.kernel_mask = decode_kernel_mask::DICT_INT32; any_rewritten = true; } - } - - if (any_rewritten) { - // Push the rewritten kernel_masks back to device so subsequent decode kernels dispatch - // correctly. Then refresh the aggregated subpass.kernel_mask on the host by re-OR'ing all - // page kernel_masks. - subpass.pages.host_to_device_async(_stream); - uint32_t refreshed = 0; - for (size_t p = 0; p < subpass.pages.size(); ++p) { - refreshed |= static_cast(subpass.pages[p].kernel_mask); - } - subpass.kernel_mask = refreshed; - _stream.synchronize(); - } + }); + + if (not any_rewritten) { return; } + + // Push the rewritten `kernel_mask`s back to device so subsequent decode kernels dispatch + // correctly. Then refresh the aggregated `subpass.kernel_mask` on the host by re-OR'ing all + // page kernel masks. + subpass.pages.host_to_device_async(_stream); + subpass.kernel_mask = std::transform_reduce( + subpass.pages.host_begin(), + subpass.pages.host_end(), + uint32_t{0}, + std::bit_or<>{}, + [](PageInfo const& page) { return static_cast(page.kernel_mask); }); + _stream.synchronize(); } void reader_impl::zero_init_dict_transcoded_index_buffers() { + CUDF_FUNC_RANGE(); + if (not _options.try_output_dict_columns) { return; } if (_dict_transcode_eligible.empty()) { return; } // The `DICT_INT32` kernel only writes to positions with valid definition levels, leaving null - // slots untouched. Since `allocate_columns` uses `memset_data=false` by default, the INT32 - // output buffer for a transcoded column may contain uninitialized bytes at null positions. - // Zero them here so null rows carry a well-defined (valid) index into the dictionary keys. - for (size_t i = 0; i < _input_columns.size(); ++i) { - if (not _dict_transcode_eligible[i]) { continue; } - auto const& input_col = _input_columns[i]; - auto& out_buf = _output_buffers[input_col.nesting[0]]; - if (out_buf.type.id() != type_id::INT32) { continue; } - if (out_buf.data() == nullptr or out_buf.size == 0) { continue; } - CUDF_CUDA_TRY(cudaMemsetAsync( - out_buf.data(), 0, static_cast(out_buf.size) * sizeof(int32_t), _stream.value())); - } + // slots untouched. Since `allocate_columns` does not zero-initialize fixed-width buffers by + // default, the INT32 output buffer for a transcoded column may contain uninitialized bytes at + // null positions. Zero them here so null rows carry a well-defined (valid) index into the + // dictionary keys -- a requirement for `cudf::dictionary::detail::concatenate` to correctly + // remap indices below. + std::for_each( + cuda::counting_iterator{0}, + cuda::counting_iterator{_input_columns.size()}, + [&](size_t i) { + if (not _dict_transcode_eligible[i]) { return; } + auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; + if (out_buf.type.id() != type_id::INT32) { return; } + if (out_buf.data() == nullptr or out_buf.size == 0) { return; } + CUDF_CUDA_TRY(cudaMemsetAsync( + out_buf.data(), 0, static_cast(out_buf.size) * sizeof(int32_t), _stream.value())); + }); } -namespace { - -// Build a STRING keys column covering the dictionary entries of a contiguous range of chunks of a -// single input column. `str_dict_index` is the device-resident pointer to the pass-wide -// `string_index_pair` buffer. `entry_count` is the total number of entries contributed by the -// range. Because the pass's str_dict_index buffer already stores pairs in chunk-index order with -// packed offsets, and our caller passes a range covering a contiguous sub-slice for one column, -// we can simply wrap the pointer in a span and hand it to the strings factory. -[[nodiscard]] std::unique_ptr make_keys_column_from_index_pairs( - string_index_pair const* begin, - size_type entry_count, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - if (entry_count <= 0) { return cudf::make_empty_column(data_type{type_id::STRING}); } - return cudf::strings::detail::make_strings_column(begin, begin + entry_count, stream, mr); -} - -} // namespace - void reader_impl::assemble_dict_transcoded_columns( std::vector>& out_columns) { @@ -206,121 +225,106 @@ void reader_impl::assemble_dict_transcoded_columns( if (_dict_transcode_eligible.empty()) { return; } if (_pass_itm_data == nullptr) { return; } - auto& pass = *_pass_itm_data; + auto const& pass = *_pass_itm_data; // For each eligible input column, collect its chunks in row-group order, build a per-chunk // DICTIONARY32 segment (local 0-based indices + per-chunk keys column), and concatenate. // // IMPORTANT: Each segment carries row-group-local indices into its own keys column. We do NOT - // pre-shift indices into a global keyspace, because `cudf::concatenate` on dictionary columns - // (via `cudf::dictionary::detail::concatenate`) already re-maps the indices using - // `compute_children_offsets_fn`. Pre-shifting would cause double-offsetting and out-of-bounds - // reads in the `dispatch_compute_indices` kernel. - for (size_t i = 0; i < _input_columns.size(); ++i) { - if (not _dict_transcode_eligible[i]) { continue; } - - // Gather the ordered list of chunk indices belonging to this input column. - std::vector chunk_indices; - chunk_indices.reserve(pass.chunks.size() / std::max(_input_columns.size(), 1)); - for (size_t c = 0; c < pass.chunks.size(); ++c) { - if (pass.chunks[c].src_col_index == static_cast(i)) { chunk_indices.push_back(c); } - } - if (chunk_indices.empty()) { continue; } - - // Per-chunk key counts derived from the dictionary page's num_input_values, mirrored back to - // the host when `pass.pages` was copied by `decode_page_headers`. - std::vector chunk_key_counts(chunk_indices.size(), 0); - for (size_t k = 0; k < chunk_indices.size(); ++k) { - auto const& chunk = pass.chunks[chunk_indices[k]]; - if (chunk.dict_page != nullptr) { - for (auto const& page : pass.pages) { - if (page.chunk_idx == static_cast(chunk_indices[k]) and - (page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { - chunk_key_counts[k] = static_cast(page.num_input_values); - break; - } - } + // pre-shift indices into a global keyspace, because `cudf::dictionary::detail::concatenate` + // already re-maps the indices using `compute_children_offsets_fn`. Pre-shifting would cause + // double-offsetting and out-of-bounds reads in the `dispatch_compute_indices` kernel. + std::for_each( + cuda::counting_iterator{0}, + cuda::counting_iterator{_input_columns.size()}, + [&](size_t i) { + if (not _dict_transcode_eligible[i]) { return; } + + // Gather chunk indices for this input column in row-group order. + std::vector chunk_indices; + chunk_indices.reserve(pass.chunks.size() / std::max(_input_columns.size(), 1)); + std::copy_if(cuda::counting_iterator{0}, + cuda::counting_iterator{pass.chunks.size()}, + std::back_inserter(chunk_indices), + [&](size_t c) { return pass.chunks[c].src_col_index == static_cast(i); }); + if (chunk_indices.empty()) { return; } + + // Per-chunk key counts from the dictionary page's `num_input_values`, mirrored back to + // host when `pass.pages` was copied by `decode_page_headers`. + std::vector chunk_key_counts(chunk_indices.size(), 0); + std::transform(chunk_indices.begin(), + chunk_indices.end(), + chunk_key_counts.begin(), + [&](size_t chunk_idx) -> size_type { + if (pass.chunks[chunk_idx].dict_page == nullptr) { return 0; } + for (auto const& page : pass.pages) { + if (page.chunk_idx == static_cast(chunk_idx) and + (page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { + return static_cast(page.num_input_values); + } + } + return size_type{0}; + }); + + // Grab ownership of the decoded INT32 indices column; we slice it into zero-copy + // per-chunk views to feed into the per-segment dictionary column builders. + auto& indices_col = out_columns[i]; + CUDF_EXPECTS(indices_col != nullptr and indices_col->type().id() == type_id::INT32, + "Expected INT32 indices column for dict-transcoded flat string column"); + auto indices_owner = std::move(indices_col); + column_view const indices_view{indices_owner->view()}; + + // Per-chunk boundaries along the row axis: chunk k occupies rows + // [chunk_row_offsets[k], chunk_row_offsets[k+1]). + std::vector chunk_row_offsets(chunk_indices.size() + 1, 0); + std::transform( + chunk_indices.begin(), + chunk_indices.end(), + chunk_row_offsets.begin() + 1, + [&](size_t chunk_idx) { return static_cast(pass.chunks[chunk_idx].num_rows); }); + std::inclusive_scan( + chunk_row_offsets.begin() + 1, chunk_row_offsets.end(), chunk_row_offsets.begin() + 1); + CUDF_EXPECTS(chunk_row_offsets.back() == indices_view.size(), + "Row counts on pass chunks must sum to the indices column size"); + + // Build a DICTIONARY32 segment for every chunk. Each segment carries row-group-local + // indices into its own keys child; `cudf::detail::concatenate` rewrites the indices + // against the unified, deduplicated keys. We deep-copy each sliced view into a fresh + // offset-zero indices column before handing it to `make_dictionary_column`: the + // `make_dictionary_column(column_view, column_view, ...)` path would otherwise + // double-apply the slice's offset when constructing the internal indices child. + std::vector> dict_segments(chunk_indices.size()); + std::transform(cuda::counting_iterator{0}, + cuda::counting_iterator{chunk_indices.size()}, + dict_segments.begin(), + [&](size_t k) { + auto const chunk_idx = chunk_indices[k]; + auto const& chunk = pass.chunks[chunk_idx]; + + auto const seg_view = cudf::detail::slice( + indices_view, chunk_row_offsets[k], chunk_row_offsets[k + 1], _stream); + auto seg_indices = std::make_unique(seg_view, _stream, _mr); + + auto seg_keys = make_keys_column_from_index_pairs( + chunk.str_dict_index, chunk_key_counts[k], _stream, _mr); + + return cudf::make_dictionary_column( + std::move(seg_keys), std::move(seg_indices), _stream, _mr); + }); + + // Concatenate all segments into the final DICTIONARY32 column for this input column. + // `cudf::dictionary::detail::concatenate` deduplicates + sorts keys and recomputes indices. + if (dict_segments.size() == 1) { + out_columns[i] = std::move(dict_segments.front()); + } else { + std::vector segment_views(dict_segments.size()); + std::transform( + dict_segments.begin(), dict_segments.end(), segment_views.begin(), [](auto const& seg) { + return seg->view(); + }); + out_columns[i] = cudf::detail::concatenate(segment_views, _stream, _mr); } - } - - // Grab ownership of the decoded INT32 indices column and its raw device pointer so we can - // carve per-chunk slices out of it without additional synchronization. - auto& indices_col = out_columns[i]; - CUDF_EXPECTS(indices_col != nullptr and indices_col->type().id() == type_id::INT32, - "Expected INT32 indices column for dict-transcoded flat string column"); - - std::vector chunk_row_offsets(chunk_indices.size() + 1, 0); - for (size_t k = 0; k < chunk_indices.size(); ++k) { - chunk_row_offsets[k + 1] = - chunk_row_offsets[k] + static_cast(pass.chunks[chunk_indices[k]].num_rows); - } - - auto indices_contents = indices_col->release(); - auto* indices_data = static_cast(indices_contents.data->data()); - auto const indices_size = - static_cast(indices_contents.data->size() / sizeof(int32_t)); - CUDF_EXPECTS(indices_size == chunk_row_offsets.back(), - "Row counts on pass chunks must sum to the indices column size"); - - std::vector> dict_segments; - dict_segments.reserve(chunk_indices.size()); - - for (size_t k = 0; k < chunk_indices.size(); ++k) { - auto const& chunk = pass.chunks[chunk_indices[k]]; - auto const row_begin = chunk_row_offsets[k]; - auto const row_end = chunk_row_offsets[k + 1]; - auto const key_count = chunk_key_counts[k]; - auto const chunk_nrows = row_end - row_begin; - - // Copy this chunk's slice of (unshifted, local-to-chunk) INT32 indices into its own buffer. - rmm::device_buffer seg_data(chunk_nrows * sizeof(int32_t), _stream, _mr); - if (chunk_nrows > 0) { - CUDF_CUDA_TRY(cudaMemcpyAsync(seg_data.data(), - indices_data + row_begin, - chunk_nrows * sizeof(int32_t), - cudaMemcpyDeviceToDevice, - _stream.value())); - } - - // Slice the null mask into this chunk's range. - rmm::device_buffer seg_null_mask{}; - size_type seg_null_count = 0; - if (indices_contents.null_mask != nullptr and indices_contents.null_mask->size() > 0) { - auto const* src_mask_ptr = - static_cast(indices_contents.null_mask->data()); - seg_null_mask = cudf::detail::copy_bitmask(src_mask_ptr, row_begin, row_end, _stream, _mr); - seg_null_count = cudf::null_count(src_mask_ptr, row_begin, row_end, _stream); - } - - auto seg_indices = std::make_unique(data_type{type_id::INT32}, - chunk_nrows, - std::move(seg_data), - std::move(seg_null_mask), - seg_null_count); - - auto seg_keys = - make_keys_column_from_index_pairs(chunk.str_dict_index, key_count, _stream, _mr); - - // Assemble a DICTIONARY32 column for this chunk segment. Indices are local 0-based; the - // subsequent `cudf::concatenate` will rewrite them against the unified, deduplicated keys. - auto seg_dict = - cudf::make_dictionary_column(std::move(seg_keys), std::move(seg_indices), _stream, _mr); - dict_segments.emplace_back(std::move(seg_dict)); - } - - // Concatenate all segments into the final DICTIONARY32 column for this input column. - // `cudf::dictionary::detail::concatenate` deduplicates + sorts keys and recomputes indices. - if (dict_segments.size() == 1) { - out_columns[i] = std::move(dict_segments.front()); - } else { - std::vector segment_views; - segment_views.reserve(dict_segments.size()); - for (auto const& seg : dict_segments) { - segment_views.emplace_back(seg->view()); - } - out_columns[i] = cudf::concatenate(segment_views, _stream, _mr); - } - } + }); } } // namespace cudf::io::parquet::detail From 98ebabc058e325ba71b3f50a33857ace4d2bce1b Mon Sep 17 00:00:00 2001 From: ykiran Date: Tue, 12 May 2026 12:36:34 -0700 Subject: [PATCH 05/42] Minor changes --- cpp/src/io/parquet/decode_fixed.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 77415039dfde..9be0e972d22a 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -1017,7 +1017,7 @@ CUDF_HOST_DEVICE constexpr bool is_split_decode() } /** - * @brief Kernel for computing fixed width non dictionary column data stored in the pages + * @brief Kernel for computing fixed width column data stored in the pages * * This function will write the page data and the page data's validity to the * output specified in the page's column chunk. If necessary, additional From 1b5ea78de359607d1b99ebfdaf13bb7f30a1caa1 Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 15 May 2026 13:30:47 -0700 Subject: [PATCH 06/42] Added list test --- cpp/tests/io/parquet_reader_dict_test.cpp | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 7837b39331aa..d63941a494e1 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include +#include #include #include #include @@ -26,6 +28,8 @@ constexpr cudf::size_type num_rows = 5000; constexpr cudf::size_type cardinality = num_rows / 10; constexpr cudf::size_type row_group_size = 1000; constexpr unsigned int seed = 0xcece; +constexpr unsigned int list_strings_seed = seed ^ 0xA5701DUL; +constexpr cudf::size_type max_elements_per_list = 8; constexpr double null_probability = 0.1; cudf::test::strings_column_wrapper make_low_cardinality_strings() @@ -44,6 +48,33 @@ cudf::test::strings_column_wrapper make_low_cardinality_strings() return cudf::test::strings_column_wrapper(strings.begin(), strings.end(), valids.begin()); } +std::unique_ptr make_low_cardinality_lists_of_strings() +{ + std::mt19937 engine(list_strings_seed); + std::uniform_int_distribution value_dist(0, cardinality - 1); + std::uniform_int_distribution len_dist(0, max_elements_per_list); + + std::vector offsets; + offsets.reserve(num_rows + 1); + offsets.push_back(0); + std::vector child_strings; + for (cudf::size_type row = 0; row < num_rows; ++row) { + auto const len = len_dist(engine); + for (int e = 0; e < len; ++e) { + child_strings.push_back("str_" + std::to_string(value_dist(engine))); + } + offsets.push_back(offsets.back() + static_cast(len)); + } + + auto child = cudf::test::strings_column_wrapper(child_strings.begin(), child_strings.end()); + auto offsets_col = + cudf::test::fixed_width_column_wrapper(offsets.begin(), offsets.end()) + .release(); + + return cudf::make_lists_column( + num_rows, std::move(offsets_col), child.release(), 0, rmm::device_buffer{}); +} + void write_parquet(cudf::table_view const& input, std::string const& filepath) { auto const options = @@ -124,3 +155,24 @@ TEST_F(ParquetReaderDictTest, FlatStringNoTranscodeByDefault) ASSERT_EQ(read_col.type().id(), cudf::type_id::STRING); CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, read_col); } + +// List is not eligible for Parquet-dictionary → DICTIONARY32 transcode (flat string columns +// only). With `try_output_dict_columns` enabled, the reader still round-trips as LIST. +TEST_F(ParquetReaderDictTest, ListOfStringsDictEncodedWithTryOutputDictOption) +{ + auto list_col = make_low_cardinality_lists_of_strings(); + + auto const input_tbl = cudf::table_view{{list_col->view()}}; + auto const filepath = + temp_env->get_temp_filepath("ListOfStringsDictEncodedWithTryOutputDictOption.parquet"); + write_parquet(input_tbl, filepath); + + auto const read_table = read_parquet_as_dict(filepath).tbl; + ASSERT_EQ(read_table->num_rows(), input_tbl.num_rows()); + ASSERT_EQ(read_table->num_columns(), 1); + + auto const read_col = read_table->view().column(0); + ASSERT_EQ(read_col.type().id(), cudf::type_id::LIST) + << "List must remain LIST when try_output_dict_columns is on (transcode is flat-only)"; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(list_col->view(), read_col); +} \ No newline at end of file From 428c1112cd9c09b40dac53df632579b6d6f4eeae Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 15 May 2026 16:22:40 -0700 Subject: [PATCH 07/42] Cleanup prepare --- .../io/parquet/reader_impl_dict_transcode.cu | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index be2d3a9f1767..04c2cef2c113 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -71,6 +71,52 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) } } +// Per-input-column eligibility for Parquet-dict → DICTIONARY32. A column is eligible iff +// - the corresponding output buffer is currently typed as STRING (i.e. a flat string column), +// - every chunk of that column is a BYTE_ARRAY string chunk with a dictionary page, +// - every data page of every chunk of that column uses (PLAIN|RLE)_DICTIONARY encoding, +// - the chunk has a flat (non-list, non-nested) schema. +// +// We scan host-side `pass.chunks` and `pass.pages` here rather than `subpass.pages` because +// `subpass.pages` may be a subset. For single-pass single-subpass reads (the only configuration +// in which `try_output_dict_columns` is supported), `subpass.pages == pass.pages`. +[[nodiscard]] std::vector compute_dict_transcode_eligibility( + pass_intermediate_data const& pass, + std::vector const& input_columns, + std::vector const& output_buffers) +{ + auto const num_input_cols = input_columns.size(); + std::vector elig(num_input_cols); + + //Check if the output buffer is a flat string column + std::for_each(cuda::counting_iterator{0}, + cuda::counting_iterator{num_input_cols}, + [&](size_t i) { + auto const& input_col = input_columns[i]; + if (input_col.nesting_depth() != 1) { return; } + if (output_buffers[input_col.nesting[0]].type.id() == type_id::STRING) { + elig[i].has_string_buffer = true; + } + }); + + // Fold per-chunk info into the per-column eligibility flags. + for (auto const& chunk : pass.chunks) { + auto const col_idx = chunk.src_col_index; + update_from_chunk(elig[col_idx], chunk); + } + + // Any non-dictionary data-page encoding disqualifies the whole column. Dictionary pages + // themselves (PAGEINFO_FLAGS_DICTIONARY) are skipped since they are not data pages. + for (auto const& page : pass.pages) { + if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } + auto const chunk_idx = page.chunk_idx; + auto const col_idx = pass.chunks[chunk_idx].src_col_index; + if (not is_dict_data_page_encoding(page.encoding)) { elig[col_idx].all_pages_dict = false; } + } + + return elig; +} + // Build a STRING keys column covering the dictionary entries of a single chunk of a single input // column. `begin` points into the pass-wide `string_index_pair` buffer. [[nodiscard]] std::unique_ptr make_keys_column_from_index_pairs( @@ -99,46 +145,7 @@ void reader_impl::prepare_dict_transcode() if (pass.chunks.empty() or subpass.pages.size() == 0) { return; } - // Determine per-input-column eligibility. A column is eligible iff - // - the corresponding output buffer is currently typed as STRING (i.e. a flat string column), - // - every chunk of that column is a BYTE_ARRAY string chunk with a dictionary page, - // - every data page of every chunk of that column uses (PLAIN|RLE)_DICTIONARY encoding, - // - the chunk has a flat (non-list, non-nested) schema. - // - // We scan host-side `pass.chunks` and `pass.pages` here rather than `subpass.pages` because - // `subpass.pages` may be a subset. For single-pass single-subpass reads (the only configuration - // in which `try_output_dict_columns` is supported), `subpass.pages == pass.pages`. - auto const num_input_cols = _input_columns.size(); - std::vector elig(num_input_cols); - - // Seed from the output buffer type: only flat STRING columns have a single leaf buffer whose - // type is STRING and can be flipped in place. - std::for_each( - cuda::counting_iterator{0}, cuda::counting_iterator{num_input_cols}, [&](size_t i) { - auto const& input_col = _input_columns[i]; - if (input_col.nesting_depth() != 1) { return; } - auto const& out_buf = _output_buffers[input_col.nesting[0]]; - if (out_buf.type.id() == type_id::STRING) { elig[i].has_string_buffer = true; } - }); - - // Fold per-chunk info into the per-column eligibility flags. - for (auto const& chunk : pass.chunks) { - auto const col_idx = chunk.src_col_index; - if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } - update_from_chunk(elig[col_idx], chunk); - } - - // Any non-dictionary data-page encoding disqualifies the whole column. Dictionary pages - // themselves (PAGEINFO_FLAGS_DICTIONARY) are skipped since they are not data pages. - for (auto const& page : pass.pages) { - if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } - auto const chunk_idx = page.chunk_idx; - if (chunk_idx < 0 or static_cast(chunk_idx) >= pass.chunks.size()) { continue; } - auto const col_idx = pass.chunks[chunk_idx].src_col_index; - if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { continue; } - if (not is_dict_data_page_encoding(page.encoding)) { elig[col_idx].all_pages_dict = false; } - } - + auto const elig = compute_dict_transcode_eligibility(pass, _input_columns, _output_buffers); std::transform( elig.begin(), elig.end(), _dict_transcode_eligible.begin(), [](column_eligibility const& e) { return e.is_eligible(); @@ -148,7 +155,7 @@ void reader_impl::prepare_dict_transcode() std::count(_dict_transcode_eligible.begin(), _dict_transcode_eligible.end(), true); if (num_eligible == 0) { return; } - // Flip the output buffer type for eligible columns from STRING → INT32. The subsequent + // Change the output buffer type for eligible columns from STRING → INT32. The subsequent // `allocate_columns` call will then allocate an INT32 buffer that the DICT_INT32 kernel can // write directly into. std::for_each( @@ -165,9 +172,7 @@ void reader_impl::prepare_dict_transcode() std::for_each(subpass.pages.host_begin(), subpass.pages.host_end(), [&](PageInfo& page) { if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { return; } auto const chunk_idx = page.chunk_idx; - if (chunk_idx < 0 or static_cast(chunk_idx) >= pass.chunks.size()) { return; } auto const col_idx = pass.chunks[chunk_idx].src_col_index; - if (col_idx < 0 or static_cast(col_idx) >= num_input_cols) { return; } if (not _dict_transcode_eligible[col_idx]) { return; } if (page.kernel_mask == decode_kernel_mask::STRING_DICT) { page.kernel_mask = decode_kernel_mask::DICT_INT32; From 139d9898b68c79c409c3575169d8d6b76be9e1ba Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 15 May 2026 17:45:42 -0700 Subject: [PATCH 08/42] Added return value for prepare --- cpp/src/io/parquet/reader_impl.cpp | 10 +++++----- cpp/src/io/parquet/reader_impl.hpp | 11 ++++++---- .../io/parquet/reader_impl_dict_transcode.cu | 20 +++++++++---------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 9e559dc624d2..044f7e8a9795 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -725,7 +725,7 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // eligibility and mutate `_output_buffers` / `subpass.pages` before we allocate column buffers // or dispatch decode kernels. This has to happen before `preprocess_chunk_strings` / // `allocate_columns` because those branch on `subpass.kernel_mask` and on `out_buf.type`. - prepare_dict_transcode(); + bool const dict_transcode_active = prepare_dict_transcode(); // computes: // PageNestingInfo::batch_size for each level of nesting, for each page, taking row bounds into @@ -752,7 +752,7 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // Zero-init the INT32 index buffers of dict-transcoded columns before launching decode, so // that null positions (which the DICT_INT32 kernel does not write to) carry well-defined // indices in the produced DICTIONARY32 output. - zero_init_dict_transcoded_index_buffers(); + if (dict_transcode_active) { zero_init_dict_transcoded_index_buffers(); } // Parse data into the output buffers. decode_page_data(mode, read_info.skip_rows, read_info.num_rows); @@ -783,9 +783,9 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // For any columns that were selected for direct parquet-dict → DICTIONARY32 transcode in // `prepare_dict_transcode`, the entries in `out_columns` are currently INT32 indices columns. - // Assemble them into DICTIONARY32 columns here by attaching per-chunk keys and shifting - // per-chunk indices so the concatenation refers to the unified keys child. - assemble_dict_transcoded_columns(out_columns); + // Assemble them into DICTIONARY32 columns here by attaching per-chunk keys; concatenate + // remaps indices to the unified keys child. + if (dict_transcode_active) { assemble_dict_transcoded_columns(out_columns); } out_columns = cudf::structs::detail::enforce_null_consistency(std::move(out_columns), _stream, _mr); diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 13190e3436c6..279e7ed4ab99 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -199,8 +199,11 @@ class reader_impl { * * Populates `_dict_transcode_eligible` with a bool per input column indicating whether the * column will be assembled as a DICTIONARY32 output later in `assemble_dict_transcoded_columns`. + * + * @return True if dict transcode is active for this read (eligible columns had output types and + * decode masks updated and pushed to the device). False otherwise. */ - void prepare_dict_transcode(); + [[nodiscard]] bool prepare_dict_transcode(); /** * @brief Zero-initialize the INT32 output buffers of dict-transcoded columns so that null rows @@ -212,9 +215,9 @@ class reader_impl { /** * @brief Assemble DICTIONARY32 output columns for input columns that were marked eligible by - * `prepare_dict_transcode`. Each chunk's INT32 indices produced by the `DICT_INT32` kernel are - * shifted by the cumulative number of keys from prior chunks, and the per-chunk keys (built - * from `pass.str_dict_index`) are concatenated into a single keys child. + * `prepare_dict_transcode`. Per-chunk keys (from `pass.str_dict_index`) and INT32 indices are + * concatenated; `cudf::dictionary::detail::concatenate` remaps indices to deduplicated keys + * (indices are not pre-shifted by the reader). * * Non-eligible flat STRING columns are left untouched here and are expected to go through the * post-hoc `cudf::dictionary::encode` fallback in `finalize_output`. diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 04c2cef2c113..0423bc4c9c33 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -131,19 +131,19 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) } // namespace -void reader_impl::prepare_dict_transcode() +bool reader_impl::prepare_dict_transcode() { CUDF_FUNC_RANGE(); _dict_transcode_eligible.assign(_input_columns.size(), false); - if (not _options.try_output_dict_columns) { return; } - if (_pass_itm_data == nullptr or _pass_itm_data->subpass == nullptr) { return; } + if (not _options.try_output_dict_columns) { return false; } + if (_pass_itm_data == nullptr or _pass_itm_data->subpass == nullptr) { return false; } auto& pass = *_pass_itm_data; auto& subpass = *pass.subpass; - if (pass.chunks.empty() or subpass.pages.size() == 0) { return; } + if (pass.chunks.empty() or subpass.pages.size() == 0) { return false; } auto const elig = compute_dict_transcode_eligibility(pass, _input_columns, _output_buffers); std::transform( @@ -153,7 +153,9 @@ void reader_impl::prepare_dict_transcode() auto const num_eligible = std::count(_dict_transcode_eligible.begin(), _dict_transcode_eligible.end(), true); - if (num_eligible == 0) { return; } + if (num_eligible == 0) { return false; } + + auto const num_input_cols = _input_columns.size(); // Change the output buffer type for eligible columns from STRING → INT32. The subsequent // `allocate_columns` call will then allocate an INT32 buffer that the DICT_INT32 kernel can @@ -180,7 +182,7 @@ void reader_impl::prepare_dict_transcode() } }); - if (not any_rewritten) { return; } + if (not any_rewritten) { return false; } // Push the rewritten `kernel_mask`s back to device so subsequent decode kernels dispatch // correctly. Then refresh the aggregated `subpass.kernel_mask` on the host by re-OR'ing all @@ -193,15 +195,13 @@ void reader_impl::prepare_dict_transcode() std::bit_or<>{}, [](PageInfo const& page) { return static_cast(page.kernel_mask); }); _stream.synchronize(); + return true; } void reader_impl::zero_init_dict_transcoded_index_buffers() { CUDF_FUNC_RANGE(); - if (not _options.try_output_dict_columns) { return; } - if (_dict_transcode_eligible.empty()) { return; } - // The `DICT_INT32` kernel only writes to positions with valid definition levels, leaving null // slots untouched. Since `allocate_columns` does not zero-initialize fixed-width buffers by // default, the INT32 output buffer for a transcoded column may contain uninitialized bytes at @@ -226,8 +226,6 @@ void reader_impl::assemble_dict_transcoded_columns( { CUDF_FUNC_RANGE(); - if (not _options.try_output_dict_columns) { return; } - if (_dict_transcode_eligible.empty()) { return; } if (_pass_itm_data == nullptr) { return; } auto const& pass = *_pass_itm_data; From 2c7f06931c163ebe859d84688d2c9a98724b03c0 Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 15 Jun 2026 13:18:27 -0700 Subject: [PATCH 09/42] Feedback --- cpp/src/dictionary/detail/concatenate.cu | 1 + cpp/src/io/parquet/reader_impl.cpp | 1 + cpp/src/io/parquet/reader_impl_dict_transcode.cu | 2 ++ 3 files changed, 4 insertions(+) diff --git a/cpp/src/dictionary/detail/concatenate.cu b/cpp/src/dictionary/detail/concatenate.cu index 04bfe521eb47..26a91952cfe9 100644 --- a/cpp/src/dictionary/detail/concatenate.cu +++ b/cpp/src/dictionary/detail/concatenate.cu @@ -153,6 +153,7 @@ struct map_indices_fn { } // namespace +//TODO: Overload function to accept multiple vectors to do the concatenate at once, with a 2D kernel. std::unique_ptr concatenate(host_span columns, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 044f7e8a9795..628adf8956cf 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -686,6 +686,7 @@ void reader_impl::preprocess_chunk_strings(read_mode mode, row_range const& read table_with_metadata reader_impl::read_chunk_internal(read_mode mode) { + //TODO: Having local views instead of offsets for the segments/ CUDF_FUNC_RANGE(); // If `_output_metadata` has been constructed, just copy it over. diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 0423bc4c9c33..c7f8bff48b4a 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -296,6 +296,7 @@ void reader_impl::assemble_dict_transcoded_columns( // offset-zero indices column before handing it to `make_dictionary_column`: the // `make_dictionary_column(column_view, column_view, ...)` path would otherwise // double-apply the slice's offset when constructing the internal indices child. + // TODO: Make it a viewable instead of reconstructing. std::vector> dict_segments(chunk_indices.size()); std::transform(cuda::counting_iterator{0}, cuda::counting_iterator{chunk_indices.size()}, @@ -315,6 +316,7 @@ void reader_impl::assemble_dict_transcoded_columns( std::move(seg_keys), std::move(seg_indices), _stream, _mr); }); + // TODO: For one row group, we can handle all of them at once. // Concatenate all segments into the final DICTIONARY32 column for this input column. // `cudf::dictionary::detail::concatenate` deduplicates + sorts keys and recomputes indices. if (dict_segments.size() == 1) { From 58cd231fc040bbf5a46edfc29f9506fdefde64dd Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 15 Jun 2026 13:30:53 -0700 Subject: [PATCH 10/42] Change assemble_dict_transcoded_columns to build per-chunk column_view --- .../io/parquet/reader_impl_dict_transcode.cu | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index c7f8bff48b4a..de19f0dbdba6 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -269,8 +269,9 @@ void reader_impl::assemble_dict_transcoded_columns( return size_type{0}; }); - // Grab ownership of the decoded INT32 indices column; we slice it into zero-copy - // per-chunk views to feed into the per-segment dictionary column builders. + // Grab ownership of the decoded INT32 indices column. Its buffer is shared (aliased) by + // every per-chunk DICTIONARY32 view below via the parent view's offset/size, so it must + // stay alive until the per-column concatenate/assembly completes. auto& indices_col = out_columns[i]; CUDF_EXPECTS(indices_col != nullptr and indices_col->type().id() == type_id::INT32, "Expected INT32 indices column for dict-transcoded flat string column"); @@ -290,44 +291,45 @@ void reader_impl::assemble_dict_transcoded_columns( CUDF_EXPECTS(chunk_row_offsets.back() == indices_view.size(), "Row counts on pass chunks must sum to the indices column size"); - // Build a DICTIONARY32 segment for every chunk. Each segment carries row-group-local - // indices into its own keys child; `cudf::detail::concatenate` rewrites the indices - // against the unified, deduplicated keys. We deep-copy each sliced view into a fresh - // offset-zero indices column before handing it to `make_dictionary_column`: the - // `make_dictionary_column(column_view, column_view, ...)` path would otherwise - // double-apply the slice's offset when constructing the internal indices child. - // TODO: Make it a viewable instead of reconstructing. - std::vector> dict_segments(chunk_indices.size()); + // Build a DICTIONARY32 *view* for every chunk without copying the decoded indices. Each + // view's keys child is this chunk's own STRING keys column (which must be materialized from + // the parquet dictionary page), while its indices child aliases the shared, already-decoded + // INT32 buffer. We select each chunk's row range via the parent dictionary view's + // `offset`/`size` rather than slicing the indices child: `get_indices_annotated()` rebuilds + // the indices view from the child's `head()` plus the parent's offset/size, so a sliced + // child (carrying its own offset) would be ignored. `cudf::detail::concatenate` then + // rewrites the row-group-local indices against the unified, deduplicated keys. + std::vector> seg_keys_owners(chunk_indices.size()); + std::vector dict_segment_views(chunk_indices.size()); std::transform(cuda::counting_iterator{0}, cuda::counting_iterator{chunk_indices.size()}, - dict_segments.begin(), + dict_segment_views.begin(), [&](size_t k) { auto const chunk_idx = chunk_indices[k]; auto const& chunk = pass.chunks[chunk_idx]; - auto const seg_view = cudf::detail::slice( - indices_view, chunk_row_offsets[k], chunk_row_offsets[k + 1], _stream); - auto seg_indices = std::make_unique(seg_view, _stream, _mr); - - auto seg_keys = make_keys_column_from_index_pairs( + seg_keys_owners[k] = make_keys_column_from_index_pairs( chunk.str_dict_index, chunk_key_counts[k], _stream, _mr); - return cudf::make_dictionary_column( - std::move(seg_keys), std::move(seg_indices), _stream, _mr); + auto const seg_rows = chunk_row_offsets[k + 1] - chunk_row_offsets[k]; + return column_view{data_type{type_id::DICTIONARY32}, + seg_rows, + nullptr, // dictionary parent holds no data + nullptr, // non-nullable transcode path + 0, // null count + chunk_row_offsets[k], // reslices shared indices child + {indices_view, seg_keys_owners[k]->view()}}; }); - // TODO: For one row group, we can handle all of them at once. - // Concatenate all segments into the final DICTIONARY32 column for this input column. - // `cudf::dictionary::detail::concatenate` deduplicates + sorts keys and recomputes indices. - if (dict_segments.size() == 1) { - out_columns[i] = std::move(dict_segments.front()); + // Materialize the final DICTIONARY32 column for this input column. + if (dict_segment_views.size() == 1) { + // Single row group: the parquet dictionary page keys are already unique, so no dedup is + // needed. Take ownership of the decoded INT32 indices buffer directly (zero copy). + out_columns[i] = cudf::make_dictionary_column( + std::move(seg_keys_owners.front()), std::move(indices_owner), _stream, _mr); } else { - std::vector segment_views(dict_segments.size()); - std::transform( - dict_segments.begin(), dict_segments.end(), segment_views.begin(), [](auto const& seg) { - return seg->view(); - }); - out_columns[i] = cudf::detail::concatenate(segment_views, _stream, _mr); + // `cudf::detail::concatenate` deduplicates + sorts keys and recomputes indices. + out_columns[i] = cudf::detail::concatenate(dict_segment_views, _stream, _mr); } }); } From d57d70f5ba705d6f864b6925cc8759386b9e77b4 Mon Sep 17 00:00:00 2001 From: ykiran Date: Wed, 24 Jun 2026 09:58:40 -0700 Subject: [PATCH 11/42] Parquet dictionary benchmark --- cpp/benchmarks/CMakeLists.txt | 2 +- .../io/parquet/parquet_reader_dict.cpp | 119 ++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 cpp/benchmarks/io/parquet/parquet_reader_dict.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 1b57e3b23666..7e062b512d75 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -297,7 +297,7 @@ ConfigureNVBench( # * parquet reader benchmark ---------------------------------------------------------------------- ConfigureNVBench( PARQUET_READER_NVBENCH io/parquet/parquet_reader_input.cpp io/parquet/parquet_reader_options.cpp - io/parquet/reader_common.cpp + io/parquet/parquet_reader_dict.cpp io/parquet/reader_common.cpp ) # ################################################################################################## diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp new file mode 100644 index 000000000000..05b8083ca231 --- /dev/null +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +#include +#include + +#include + +// Benchmark for the parquet-dictionary -> cudf DICTIONARY32 transcode fast path enabled by +// `parquet_reader_options::try_output_dict_columns`. It reads a fully dictionary-encoded set of +// low-cardinality string columns both with the option off (the column materializes as STRING) and +// with the option on (the reader keeps the dictionary representation and emits DICTIONARY32). The +// `try_output_dict_columns` axis lets the two paths be compared directly. + +namespace { + +// The transcode fast path requires every data page of an eligible column to be dictionary-encoded. +// Forcing `dictionary_policy::ALWAYS` together with low cardinality guarantees this for the +// generated data. +void write_dict_encoded_parquet(cudf::table_view const& view, + cuio_source_sink_pair& source_sink, + int64_t row_group_size_rows) +{ + cudf::io::parquet_writer_options write_opts = + cudf::io::parquet_writer_options::builder(source_sink.make_sink_info(), view) + .compression(cudf::io::compression_type::NONE) + .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN); + // Sentinel 0 == use cuDF default row-group sizing. + if (row_group_size_rows > 0) { write_opts.set_row_group_size_rows(row_group_size_rows); } + cudf::io::write_parquet(write_opts); +} + +} // namespace + +void BM_parquet_read_dict_transcode(nvbench::state& state) +{ + auto const cardinality = static_cast(state.get_int64("cardinality")); + auto const data_size = static_cast(state.get_int64("data_size")); + auto const num_cols = static_cast(state.get_int64("num_cols")); + auto const rg_size_rows = state.get_int64("row_group_size_rows"); + auto const try_dict = static_cast(state.get_int64("try_output_dict_columns")); + auto const avg_string_length = + static_cast(state.get_int64("avg_string_length")); + auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); + + // corresponds to 3 sigma (full width 6 sigma: 99.7% of range) + auto const half_width = avg_string_length >> 3; + auto const length_min = avg_string_length - half_width; + auto const length_max = avg_string_length + half_width; + + data_profile const profile = + data_profile_builder() + .cardinality(cardinality) + .avg_run_length(1) + .distribution(data_type::STRING, distribution_id::NORMAL, length_min, length_max); + + auto const d_type = get_type_or_group(static_cast(data_type::STRING)); + auto const tbl = + create_random_table(cycle_dtypes(d_type, num_cols), table_size_bytes{data_size}, profile); + auto const view = tbl->view(); + + cuio_source_sink_pair source_sink(source_type); + write_dict_encoded_parquet(view, source_sink, rg_size_rows); + + cudf::io::parquet_reader_options read_opts = + cudf::io::parquet_reader_options::builder(source_sink.make_source_info()) + .try_output_dict_columns(try_dict); + + // Sanity check (outside the timed region): when the option is on the eligible string columns must + // come back as DICTIONARY32, otherwise the benchmark would silently measure the plain path. + if (try_dict) { + auto const probe = cudf::io::read_parquet(read_opts); + CUDF_EXPECTS(probe.tbl->num_columns() == num_cols, "Unexpected number of columns"); + CUDF_EXPECTS(probe.tbl->view().column(0).type().id() == cudf::type_id::DICTIONARY32, + "try_output_dict_columns did not produce a DICTIONARY32 column; check that the " + "generated data is fully dictionary-encoded"); + } + + auto mem_stats_logger = cudf::memory_stats_logger(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); + state.exec( + nvbench::exec_tag::sync | nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) { + drop_page_cache_if_enabled(read_opts.get_source().filepaths()); + + timer.start(); + auto const result = cudf::io::read_parquet(read_opts); + timer.stop(); + + CUDF_EXPECTS(result.tbl->num_columns() == num_cols, "Unexpected number of columns"); + }); + + auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); + state.add_element_count(static_cast(data_size) / time, "bytes_per_second"); + state.add_element_count(static_cast(view.num_rows()) / time, "rows_per_sec"); + state.add_buffer_size( + mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); + state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); +} + +NVBENCH_BENCH(BM_parquet_read_dict_transcode) + .set_name("parquet_read_dict_transcode") + .add_string_axis("io_type", {"DEVICE_BUFFER"}) + .set_min_samples(4) + .add_int64_axis("try_output_dict_columns", {0, 1}) + .add_int64_axis("cardinality", {100, 1'000, 10'000}) + .add_int64_axis("num_cols", {1, 8}) + .add_int64_axis("data_size", {512 << 20}) + .add_int64_axis("avg_string_length", {16}) + // Sentinel 0 == default row groups; small values force multiple row groups, exercising the + // per-row-group key concatenation / index remapping path. + .add_int64_axis("row_group_size_rows", {0, 100'000}); From 3065d703b0ca4cd822bfe3f7d211637e9f2786d2 Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 29 Jun 2026 12:49:24 -0700 Subject: [PATCH 12/42] Added more documentation --- cpp/src/io/parquet/decode_fixed.cu | 44 +++++++++ .../io/parquet/reader_impl_dict_transcode.cu | 96 ++++++++++++++----- 2 files changed, 115 insertions(+), 25 deletions(-) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 9be0e972d22a..5bc1b0a24062 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -81,6 +81,23 @@ __device__ static void scan_block_exclusive_sum( } } +/** + * @brief Write a batch of decoded dictionary indices directly as INT32 output values. + * + * Used by the Parquet-dict → DICTIONARY32 transcode path: instead of materializing the dictionary + * keys, the per-row dictionary indices are emitted verbatim as the INT32 indices child of the + * output DICTIONARY32 column. + * + * @tparam block_size Number of threads per block + * @tparam has_lists_t Whether the column has a list (repetition) level + * @tparam copy_mode_t Whether destination positions are direct or indirect (nz_idx) mapped + * @tparam state_buf Page state buffer type providing the decoded dictionary indices + * @param s Page decode state for the current page + * @param sb Page state buffers holding the decoded dictionary indices + * @param start First value position (within the page) to write in this batch + * @param end One-past-the-last value position to write in this batch + * @param t Thread index within the block + */ template __device__ void decode_dict_indices_as_int32( page_state_s* s, state_buf* const sb, int start, int end, int t) @@ -949,6 +966,12 @@ __device__ void skip_ahead_in_decoding(page_state_s* s, block.sync(); } +/** + * @brief Check if the kernel mask decodes dictionary-encoded data (has a dictionary stream). + * + * @tparam kernel_mask_t The decode kernel mask to test + * @return True for fixed-width, string and INT32-index dictionary masks + */ template CUDF_HOST_DEVICE constexpr bool has_dict() { @@ -963,6 +986,15 @@ CUDF_HOST_DEVICE constexpr bool has_dict() (kernel_mask_t == decode_kernel_mask::DICT_INT32_LIST); } +/** + * @brief Check whether the kernel mask decodes parquet dictionary indices directly to an INT32 column. + * + * These masks back the Parquet-dict → DICTIONARY32 transcode path, where the decoded output is the + * INT32 indices child of a DICTIONARY32 column rather than fully materialized values. + * + * @tparam kernel_mask_t The decode kernel mask to test + * @return True for the DICT_INT32, DICT_INT32_NESTED and DICT_INT32_LIST masks + */ template CUDF_HOST_DEVICE constexpr bool is_dict_int32_output() { @@ -979,6 +1011,12 @@ CUDF_HOST_DEVICE constexpr bool has_bools() (kernel_mask_t == decode_kernel_mask::BOOLEAN_LIST); } +/** + * @brief Check if the kernel mask decodes a (non-list) nested column. + * + * @tparam kernel_mask_t The decode kernel mask to test + * @return True for the `*_NESTED` masks + */ template CUDF_HOST_DEVICE constexpr bool has_nesting() { @@ -992,6 +1030,12 @@ CUDF_HOST_DEVICE constexpr bool has_nesting() (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED); } +/** + * @brief Check if the kernel mask decodes a list column (has a repetition level). + * + * @tparam kernel_mask_t The decode kernel mask to test + * @return True for the `*_LIST` masks + */ template CUDF_HOST_DEVICE constexpr bool has_lists() { diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index de19f0dbdba6..714b0049344c 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -26,10 +26,17 @@ namespace cudf::io::parquet::detail { namespace { -// Host-side counterpart of `is_string_col` in `parquet_gpu.hpp`. Kept narrow: for direct -// Parquet-dict → DICTIONARY32 transcode we only accept pure BYTE_ARRAY columns without a -// DECIMAL logical type and without the strings-to-categorical flag. FIXED_LEN_BYTE_ARRAY is -// deliberately excluded because it is typically a binary payload. +/** + * @brief Host-side check for whether a column chunk decodes to a plain string column. + * + * Host-side counterpart of `is_string_col` in `parquet_gpu.hpp`. Kept narrow: for direct + * Parquet-dict → DICTIONARY32 transcode we only accept pure BYTE_ARRAY columns without a + * DECIMAL logical type and without the strings-to-categorical flag. FIXED_LEN_BYTE_ARRAY is + * deliberately excluded because it is typically a binary payload. + * + * @param chunk The column chunk descriptor to classify + * @return True if the chunk is a plain BYTE_ARRAY string chunk eligible for transcode + */ [[nodiscard]] bool is_host_byte_array_string_chunk(ColumnChunkDesc const& chunk) { if (chunk.physical_type != Type::BYTE_ARRAY) { return false; } @@ -40,28 +47,48 @@ namespace { return true; } -// Both PLAIN_DICTIONARY (legacy) and RLE_DICTIONARY are valid encodings for data pages that -// reference a parquet dictionary page. +/** + * @brief Whether a data-page encoding references a parquet dictionary page. + * + * Both PLAIN_DICTIONARY (legacy) and RLE_DICTIONARY are valid encodings for data pages that + * reference a parquet dictionary page. + * + * @param enc The data-page encoding to test + * @return True if the encoding is a dictionary data-page encoding + */ [[nodiscard]] bool is_dict_data_page_encoding(Encoding enc) { return enc == Encoding::PLAIN_DICTIONARY or enc == Encoding::RLE_DICTIONARY; } -// Per-input-column eligibility flags. Each column must satisfy all of these conditions to be -// eligible for direct Parquet-dict → DICTIONARY32 transcode. +/** + * @brief Per-input-column eligibility flags for Parquet-dict → DICTIONARY32 transcode. + * + * Each column must satisfy all of these conditions to be eligible for direct transcode. + */ struct column_eligibility { - bool has_string_buffer = false; - bool has_any_chunk = false; - bool all_chunks_string = true; - bool all_pages_dict = true; - + bool has_string_buffer = false; ///< Output buffer is currently typed as STRING + bool has_any_chunk = false; ///< At least one chunk was seen for this column + bool all_chunks_string = true; ///< Every chunk is a flat BYTE_ARRAY string chunk with a dict + bool all_pages_dict = true; ///< Every data page uses a dictionary encoding + + /** + * @brief Whether the column satisfies every transcode-eligibility condition. + * + * @return True if the column is eligible for direct DICTIONARY32 transcode + */ [[nodiscard]] bool is_eligible() const { return has_string_buffer and has_any_chunk and all_chunks_string and all_pages_dict; } }; -// Classify a chunk against its column's eligibility state. +/** + * @brief Fold a single chunk's properties into its column's eligibility state. + * + * @param e The per-column eligibility state to update in place + * @param chunk The column chunk descriptor to classify + */ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) { e.has_any_chunk = true; @@ -71,15 +98,24 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) } } -// Per-input-column eligibility for Parquet-dict → DICTIONARY32. A column is eligible iff -// - the corresponding output buffer is currently typed as STRING (i.e. a flat string column), -// - every chunk of that column is a BYTE_ARRAY string chunk with a dictionary page, -// - every data page of every chunk of that column uses (PLAIN|RLE)_DICTIONARY encoding, -// - the chunk has a flat (non-list, non-nested) schema. -// -// We scan host-side `pass.chunks` and `pass.pages` here rather than `subpass.pages` because -// `subpass.pages` may be a subset. For single-pass single-subpass reads (the only configuration -// in which `try_output_dict_columns` is supported), `subpass.pages == pass.pages`. +/** + * @brief Compute per-input-column eligibility for Parquet-dict → DICTIONARY32 transcode. + * + * A column is eligible iff + * - the corresponding output buffer is currently typed as STRING (i.e. a flat string column), + * - every chunk of that column is a BYTE_ARRAY string chunk with a dictionary page, + * - every data page of every chunk of that column uses (PLAIN|RLE)_DICTIONARY encoding, + * - the chunk has a flat (non-list, non-nested) schema. + * + * We scan host-side `pass.chunks` and `pass.pages` here rather than `subpass.pages` because + * `subpass.pages` may be a subset. For single-pass single-subpass reads (the only configuration + * in which `try_output_dict_columns` is supported), `subpass.pages == pass.pages`. + * + * @param pass The pass intermediate data holding host-side chunks and pages + * @param input_columns The reader's input column descriptors + * @param output_buffers The output column buffers (used to detect flat STRING columns) + * @return A vector of per-input-column eligibility records, indexed by input column + */ [[nodiscard]] std::vector compute_dict_transcode_eligibility( pass_intermediate_data const& pass, std::vector const& input_columns, @@ -117,8 +153,18 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) return elig; } -// Build a STRING keys column covering the dictionary entries of a single chunk of a single input -// column. `begin` points into the pass-wide `string_index_pair` buffer. +/** + * @brief Build a STRING keys column from a chunk's dictionary entries. + * + * Builds a STRING keys column covering the dictionary entries of a single chunk of a single input + * column. `begin` points into the pass-wide `string_index_pair` buffer. + * + * @param begin Pointer to the first `string_index_pair` entry for this chunk's dictionary + * @param entry_count Number of dictionary entries (keys) for this chunk + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's memory + * @return A STRING column holding this chunk's dictionary keys (empty if `entry_count <= 0`) + */ [[nodiscard]] std::unique_ptr make_keys_column_from_index_pairs( string_index_pair const* begin, size_type entry_count, From 629b36d9df620c433a36c08728ce376c2bfffc06 Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 29 Jun 2026 12:54:21 -0700 Subject: [PATCH 13/42] Removed uinnecessary synchronization --- cpp/src/io/parquet/reader_impl_dict_transcode.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 714b0049344c..fcee97db88e1 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -231,8 +231,9 @@ bool reader_impl::prepare_dict_transcode() if (not any_rewritten) { return false; } // Push the rewritten `kernel_mask`s back to device so subsequent decode kernels dispatch - // correctly. Then refresh the aggregated `subpass.kernel_mask` on the host by re-OR'ing all - // page kernel masks. + // correctly. The copy is enqueued on `_stream`, so no explicit synchronization is required. The host + // source buffer (`subpass.pages`) is owned by the subpass and is neither freed nor re-mutated + // before the copy completes. subpass.pages.host_to_device_async(_stream); subpass.kernel_mask = std::transform_reduce( subpass.pages.host_begin(), @@ -240,7 +241,6 @@ bool reader_impl::prepare_dict_transcode() uint32_t{0}, std::bit_or<>{}, [](PageInfo const& page) { return static_cast(page.kernel_mask); }); - _stream.synchronize(); return true; } From 83ed466f2d57663bd13e352852ed903e63266e5b Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 29 Jun 2026 13:04:23 -0700 Subject: [PATCH 14/42] Added fallback to STRING for AST filters --- cpp/src/io/parquet/reader_impl.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 628adf8956cf..6c5a443182e3 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -555,6 +555,15 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, "try_output_dict_columns is only supported for single-pass reads; it cannot be combined " "with a non-zero chunk_read_limit or pass_read_limit."); + // AST filters do not support dictionary columns yet (see the column selection below). The + // transcode fast path and the `finalize_output` fallback both convert flat STRING columns to + // DICTIONARY32 *before* the filter is evaluated, which would feed dictionary columns to the AST. + // Since `try_output_dict_columns` is best-effort, we silently disable it for filtered reads so the + // filter still operates on STRING columns (the columns are simply returned as STRING). + if (_options.try_output_dict_columns and options.get_filter().has_value()) { + _options.try_output_dict_columns = false; + } + // Open and parse the source dataset metadata CUDF_EXPECTS(file_metadatas.empty() or file_metadatas.size() == _sources.size(), "Encountered a mismatch in the number of provided data sources and metadatas"); From 681a226b1b6e9f2afb906f3d5030c650bbca5618 Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 29 Jun 2026 13:07:13 -0700 Subject: [PATCH 15/42] Added new tests --- cpp/tests/io/parquet_reader_dict_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index d63941a494e1..9f33a7c6431f 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -21,6 +22,7 @@ #include #include #include +#include namespace { From 48458b1774252b0906435d6278e39b4eaeac0f8c Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 29 Jun 2026 13:27:40 -0700 Subject: [PATCH 16/42] Added UTF-8 characters to test --- cpp/tests/io/parquet_reader_dict_test.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 9f33a7c6431f..640829e96a10 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,20 @@ constexpr unsigned int list_strings_seed = seed ^ 0xA5701DUL; constexpr cudf::size_type max_elements_per_list = 8; constexpr double null_probability = 0.1; +// Per-distinct-value prefixes deliberately mixing ASCII with multi-byte UTF-8 (accented Latin, +// Greek, CJK, and an emoji) so the transcode/fallback paths are exercised on non-ASCII keys. The +// numeric suffix in `make_value_string` keeps every distinct value a distinct string, preserving +// the intended cardinality. +std::array const utf8_prefixes{ + "str", "café", "naïve", "Ωμέγα", "日本語", "🚀rocket"}; + +// Map a dictionary value to a UTF-8 string. Distinct values map to distinct +// strings via the numeric suffix. +std::string make_value_string(int value) +{ + return std::string{utf8_prefixes[value % utf8_prefixes.size()]} + "_" + std::to_string(value); +} + cudf::test::strings_column_wrapper make_low_cardinality_strings() { std::mt19937 engine(seed); @@ -43,7 +58,7 @@ cudf::test::strings_column_wrapper make_low_cardinality_strings() std::vector strings(num_rows); std::vector valids(num_rows); for (cudf::size_type i = 0; i < num_rows; ++i) { - strings[i] = "str_" + std::to_string(value_dist(engine)); + strings[i] = make_value_string(value_dist(engine)); valids[i] = not null_dist(engine); } @@ -63,7 +78,7 @@ std::unique_ptr make_low_cardinality_lists_of_strings() for (cudf::size_type row = 0; row < num_rows; ++row) { auto const len = len_dist(engine); for (int e = 0; e < len; ++e) { - child_strings.push_back("str_" + std::to_string(value_dist(engine))); + child_strings.push_back(make_value_string(value_dist(engine))); } offsets.push_back(offsets.back() + static_cast(len)); } From 1eb6a259e9fbb918a01329d34d71d168ed90db2e Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 29 Jun 2026 13:30:29 -0700 Subject: [PATCH 17/42] Added more edge case tests --- cpp/tests/io/parquet_reader_dict_test.cpp | 65 +++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 640829e96a10..5a512fac1929 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -192,4 +192,69 @@ TEST_F(ParquetReaderDictTest, ListOfStringsDictEncodedWithTryOutputDictOption) ASSERT_EQ(read_col.type().id(), cudf::type_id::LIST) << "List must remain LIST when try_output_dict_columns is on (transcode is flat-only)"; CUDF_TEST_EXPECT_COLUMNS_EQUAL(list_col->view(), read_col); +} + +// Edge case: empty input. A zero-row flat STRING column must round-trip through the transcode +// path without error and reproduce the empty input (whether it comes back as STRING or as an +// empty DICTIONARY32 via the best-effort fallback encode). +TEST_F(ParquetReaderDictTest, EmptyFlatStringDictTranscode) +{ + std::vector const empty; + auto const input_col = cudf::test::strings_column_wrapper(empty.begin(), empty.end()); + + auto const input_tbl = cudf::table_view{{input_col}}; + auto const filepath = temp_env->get_temp_filepath("EmptyFlatStringDictTranscode.parquet"); + + // Write directly: the chunked row-group loop in `write_parquet` would skip a zero-row table. + auto const write_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, input_tbl) + .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) + .compression(cudf::io::compression_type::NONE) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .build(); + cudf::io::write_parquet(write_opts); + + auto const read_table = read_parquet_as_dict(filepath).tbl; + ASSERT_EQ(read_table->num_rows(), 0); + ASSERT_EQ(read_table->num_columns(), 1); + + auto const read_col = read_table->view().column(0); + if (read_col.type().id() == cudf::type_id::DICTIONARY32) { + auto const decoded = cudf::dictionary::decode(cudf::dictionary_column_view(read_col)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, decoded->view()); + } else { + ASSERT_EQ(read_col.type().id(), cudf::type_id::STRING); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, read_col); + } +} + +// Edge case: sliced input. Writing a sliced (non-zero offset, reduced size) flat STRING column +// must transcode correctly -- the reader's DICTIONARY32 output must decode back to exactly the +// sliced rows (including nulls), not the underlying full column. +TEST_F(ParquetReaderDictTest, SlicedFlatStringDictTranscode) +{ + auto const full_col = make_low_cardinality_strings(); + + // Interior slice so the view carries a non-zero offset and a reduced size, spanning multiple + // row groups to also exercise the per-row-group key concatenation / index remapping path. + auto const slice_start = row_group_size + 7; + auto const slice_end = num_rows - 13; + auto const sliced = cudf::slice(static_cast(full_col), + {slice_start, slice_end}) + .front(); + + auto const input_tbl = cudf::table_view{{sliced}}; + auto const filepath = temp_env->get_temp_filepath("SlicedFlatStringDictTranscode.parquet"); + write_parquet(input_tbl, filepath); + + auto const read_table = read_parquet_as_dict(filepath).tbl; + ASSERT_EQ(read_table->num_rows(), slice_end - slice_start); + ASSERT_EQ(read_table->num_columns(), 1); + + auto const read_col = read_table->view().column(0); + ASSERT_EQ(read_col.type().id(), cudf::type_id::DICTIONARY32) + << "Expected a DICTIONARY32 column for a fully dict-encoded sliced string input"; + + auto const decoded_read = cudf::dictionary::decode(cudf::dictionary_column_view(read_col)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(sliced, decoded_read->view()); } \ No newline at end of file From 744e7e5314e82acf3ca72f6caea6610ccae1193a Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 29 Jun 2026 13:40:07 -0700 Subject: [PATCH 18/42] Formatting and licenses --- .../io/parquet/parquet_reader_dict.cpp | 33 +++++++++---------- cpp/src/dictionary/detail/concatenate.cu | 3 +- cpp/src/io/parquet/decode_fixed.cu | 3 +- cpp/src/io/parquet/reader_impl.cpp | 10 +++--- .../io/parquet/reader_impl_dict_transcode.cu | 31 +++++++++-------- cpp/tests/io/parquet_reader_dict_test.cpp | 30 ++++++++--------- 6 files changed, 54 insertions(+), 56 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 05b8083ca231..368df02e0049 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -42,14 +42,13 @@ void write_dict_encoded_parquet(cudf::table_view const& view, void BM_parquet_read_dict_transcode(nvbench::state& state) { - auto const cardinality = static_cast(state.get_int64("cardinality")); - auto const data_size = static_cast(state.get_int64("data_size")); - auto const num_cols = static_cast(state.get_int64("num_cols")); - auto const rg_size_rows = state.get_int64("row_group_size_rows"); - auto const try_dict = static_cast(state.get_int64("try_output_dict_columns")); - auto const avg_string_length = - static_cast(state.get_int64("avg_string_length")); - auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); + auto const cardinality = static_cast(state.get_int64("cardinality")); + auto const data_size = static_cast(state.get_int64("data_size")); + auto const num_cols = static_cast(state.get_int64("num_cols")); + auto const rg_size_rows = state.get_int64("row_group_size_rows"); + auto const try_dict = static_cast(state.get_int64("try_output_dict_columns")); + auto const avg_string_length = static_cast(state.get_int64("avg_string_length")); + auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); // corresponds to 3 sigma (full width 6 sigma: 99.7% of range) auto const half_width = avg_string_length >> 3; @@ -86,16 +85,16 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) auto mem_stats_logger = cudf::memory_stats_logger(); state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); - state.exec( - nvbench::exec_tag::sync | nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) { - drop_page_cache_if_enabled(read_opts.get_source().filepaths()); + state.exec(nvbench::exec_tag::sync | nvbench::exec_tag::timer, + [&](nvbench::launch& launch, auto& timer) { + drop_page_cache_if_enabled(read_opts.get_source().filepaths()); - timer.start(); - auto const result = cudf::io::read_parquet(read_opts); - timer.stop(); + timer.start(); + auto const result = cudf::io::read_parquet(read_opts); + timer.stop(); - CUDF_EXPECTS(result.tbl->num_columns() == num_cols, "Unexpected number of columns"); - }); + CUDF_EXPECTS(result.tbl->num_columns() == num_cols, "Unexpected number of columns"); + }); auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); state.add_element_count(static_cast(data_size) / time, "bytes_per_second"); diff --git a/cpp/src/dictionary/detail/concatenate.cu b/cpp/src/dictionary/detail/concatenate.cu index 26a91952cfe9..f8904504c5ae 100644 --- a/cpp/src/dictionary/detail/concatenate.cu +++ b/cpp/src/dictionary/detail/concatenate.cu @@ -153,7 +153,8 @@ struct map_indices_fn { } // namespace -//TODO: Overload function to accept multiple vectors to do the concatenate at once, with a 2D kernel. +// TODO: Overload function to accept multiple vectors to do the concatenate at once, with a 2D +// kernel. std::unique_ptr concatenate(host_span columns, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 5bc1b0a24062..93d6a77c96ef 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -987,7 +987,8 @@ CUDF_HOST_DEVICE constexpr bool has_dict() } /** - * @brief Check whether the kernel mask decodes parquet dictionary indices directly to an INT32 column. + * @brief Check whether the kernel mask decodes parquet dictionary indices directly to an INT32 + * column. * * These masks back the Parquet-dict → DICTIONARY32 transcode path, where the decoded output is the * INT32 indices child of a DICTIONARY32 column rather than fully materialized values. diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 6c5a443182e3..c76d136bd152 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -558,8 +558,8 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, // AST filters do not support dictionary columns yet (see the column selection below). The // transcode fast path and the `finalize_output` fallback both convert flat STRING columns to // DICTIONARY32 *before* the filter is evaluated, which would feed dictionary columns to the AST. - // Since `try_output_dict_columns` is best-effort, we silently disable it for filtered reads so the - // filter still operates on STRING columns (the columns are simply returned as STRING). + // Since `try_output_dict_columns` is best-effort, we silently disable it for filtered reads so + // the filter still operates on STRING columns (the columns are simply returned as STRING). if (_options.try_output_dict_columns and options.get_filter().has_value()) { _options.try_output_dict_columns = false; } @@ -695,7 +695,7 @@ void reader_impl::preprocess_chunk_strings(read_mode mode, row_range const& read table_with_metadata reader_impl::read_chunk_internal(read_mode mode) { - //TODO: Having local views instead of offsets for the segments/ + // TODO: Having local views instead of offsets for the segments/ CUDF_FUNC_RANGE(); // If `_output_metadata` has been constructed, just copy it over. @@ -980,8 +980,8 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, if (_options.try_output_dict_columns) { for (auto& col : out_columns) { if (col and col->type().id() == type_id::STRING) { - col = cudf::dictionary::detail::encode( - col->view(), data_type{type_id::INT32}, _stream, _mr); + col = + cudf::dictionary::detail::encode(col->view(), data_type{type_id::INT32}, _stream, _mr); } } } diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index fcee97db88e1..d0ccb4ee9d58 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -124,16 +124,15 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) auto const num_input_cols = input_columns.size(); std::vector elig(num_input_cols); - //Check if the output buffer is a flat string column - std::for_each(cuda::counting_iterator{0}, - cuda::counting_iterator{num_input_cols}, - [&](size_t i) { - auto const& input_col = input_columns[i]; - if (input_col.nesting_depth() != 1) { return; } - if (output_buffers[input_col.nesting[0]].type.id() == type_id::STRING) { - elig[i].has_string_buffer = true; - } - }); + // Check if the output buffer is a flat string column + std::for_each( + cuda::counting_iterator{0}, cuda::counting_iterator{num_input_cols}, [&](size_t i) { + auto const& input_col = input_columns[i]; + if (input_col.nesting_depth() != 1) { return; } + if (output_buffers[input_col.nesting[0]].type.id() == type_id::STRING) { + elig[i].has_string_buffer = true; + } + }); // Fold per-chunk info into the per-column eligibility flags. for (auto const& chunk : pass.chunks) { @@ -146,7 +145,7 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) for (auto const& page : pass.pages) { if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } auto const chunk_idx = page.chunk_idx; - auto const col_idx = pass.chunks[chunk_idx].src_col_index; + auto const col_idx = pass.chunks[chunk_idx].src_col_index; if (not is_dict_data_page_encoding(page.encoding)) { elig[col_idx].all_pages_dict = false; } } @@ -220,7 +219,7 @@ bool reader_impl::prepare_dict_transcode() std::for_each(subpass.pages.host_begin(), subpass.pages.host_end(), [&](PageInfo& page) { if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { return; } auto const chunk_idx = page.chunk_idx; - auto const col_idx = pass.chunks[chunk_idx].src_col_index; + auto const col_idx = pass.chunks[chunk_idx].src_col_index; if (not _dict_transcode_eligible[col_idx]) { return; } if (page.kernel_mask == decode_kernel_mask::STRING_DICT) { page.kernel_mask = decode_kernel_mask::DICT_INT32; @@ -231,9 +230,9 @@ bool reader_impl::prepare_dict_transcode() if (not any_rewritten) { return false; } // Push the rewritten `kernel_mask`s back to device so subsequent decode kernels dispatch - // correctly. The copy is enqueued on `_stream`, so no explicit synchronization is required. The host - // source buffer (`subpass.pages`) is owned by the subpass and is neither freed nor re-mutated - // before the copy completes. + // correctly. The copy is enqueued on `_stream`, so no explicit synchronization is required. The + // host source buffer (`subpass.pages`) is owned by the subpass and is neither freed nor + // re-mutated before the copy completes. subpass.pages.host_to_device_async(_stream); subpass.kernel_mask = std::transform_reduce( subpass.pages.host_begin(), diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 5a512fac1929..69c1e0d1e92d 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -18,22 +18,22 @@ #include #include +#include #include #include #include #include #include -#include namespace { -constexpr cudf::size_type num_rows = 5000; -constexpr cudf::size_type cardinality = num_rows / 10; -constexpr cudf::size_type row_group_size = 1000; -constexpr unsigned int seed = 0xcece; -constexpr unsigned int list_strings_seed = seed ^ 0xA5701DUL; +constexpr cudf::size_type num_rows = 5000; +constexpr cudf::size_type cardinality = num_rows / 10; +constexpr cudf::size_type row_group_size = 1000; +constexpr unsigned int seed = 0xcece; +constexpr unsigned int list_strings_seed = seed ^ 0xA5701DUL; constexpr cudf::size_type max_elements_per_list = 8; -constexpr double null_probability = 0.1; +constexpr double null_probability = 0.1; // Per-distinct-value prefixes deliberately mixing ASCII with multi-byte UTF-8 (accented Latin, // Greek, CJK, and an emoji) so the transcode/fallback paths are exercised on non-ASCII keys. The @@ -112,10 +112,9 @@ void write_parquet(cudf::table_view const& input, std::string const& filepath) cudf::io::table_with_metadata read_parquet_as_dict(std::string const& filepath) { - auto const read_opts = - cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) - .try_output_dict_columns(true) - .build(); + auto const read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .try_output_dict_columns(true) + .build(); return cudf::io::read_parquet(read_opts); } @@ -239,9 +238,8 @@ TEST_F(ParquetReaderDictTest, SlicedFlatStringDictTranscode) // row groups to also exercise the per-row-group key concatenation / index remapping path. auto const slice_start = row_group_size + 7; auto const slice_end = num_rows - 13; - auto const sliced = cudf::slice(static_cast(full_col), - {slice_start, slice_end}) - .front(); + auto const sliced = + cudf::slice(static_cast(full_col), {slice_start, slice_end}).front(); auto const input_tbl = cudf::table_view{{sliced}}; auto const filepath = temp_env->get_temp_filepath("SlicedFlatStringDictTranscode.parquet"); @@ -257,4 +255,4 @@ TEST_F(ParquetReaderDictTest, SlicedFlatStringDictTranscode) auto const decoded_read = cudf::dictionary::decode(cudf::dictionary_column_view(read_col)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(sliced, decoded_read->view()); -} \ No newline at end of file +} From 44225bdb08e9ec8bc8bf45344fc35485843a2fed Mon Sep 17 00:00:00 2001 From: ykiran Date: Tue, 30 Jun 2026 17:47:23 -0700 Subject: [PATCH 19/42] MR feedback --- .../io/parquet/parquet_reader_dict.cpp | 14 ++-- cpp/include/cudf/io/parquet.hpp | 41 ++++------ cpp/src/dictionary/detail/concatenate.cu | 2 - cpp/src/io/parquet/reader_impl.cpp | 29 +++---- cpp/src/io/parquet/reader_impl.hpp | 2 +- .../io/parquet/reader_impl_dict_transcode.cu | 78 +++++++++++++------ cpp/tests/io/parquet_reader_dict_test.cpp | 10 +-- 7 files changed, 95 insertions(+), 81 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 368df02e0049..0a69827c094c 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -14,10 +14,10 @@ #include // Benchmark for the parquet-dictionary -> cudf DICTIONARY32 transcode fast path enabled by -// `parquet_reader_options::try_output_dict_columns`. It reads a fully dictionary-encoded set of +// `parquet_reader_options::output_dict_columns`. It reads a fully dictionary-encoded set of // low-cardinality string columns both with the option off (the column materializes as STRING) and // with the option on (the reader keeps the dictionary representation and emits DICTIONARY32). The -// `try_output_dict_columns` axis lets the two paths be compared directly. +// `output_dict_columns` axis lets the two paths be compared directly. namespace { @@ -46,7 +46,7 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) auto const data_size = static_cast(state.get_int64("data_size")); auto const num_cols = static_cast(state.get_int64("num_cols")); auto const rg_size_rows = state.get_int64("row_group_size_rows"); - auto const try_dict = static_cast(state.get_int64("try_output_dict_columns")); + auto const output_dict = static_cast(state.get_int64("output_dict_columns")); auto const avg_string_length = static_cast(state.get_int64("avg_string_length")); auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); @@ -71,15 +71,15 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) cudf::io::parquet_reader_options read_opts = cudf::io::parquet_reader_options::builder(source_sink.make_source_info()) - .try_output_dict_columns(try_dict); + .output_dict_columns(output_dict); // Sanity check (outside the timed region): when the option is on the eligible string columns must // come back as DICTIONARY32, otherwise the benchmark would silently measure the plain path. - if (try_dict) { + if (output_dict) { auto const probe = cudf::io::read_parquet(read_opts); CUDF_EXPECTS(probe.tbl->num_columns() == num_cols, "Unexpected number of columns"); CUDF_EXPECTS(probe.tbl->view().column(0).type().id() == cudf::type_id::DICTIONARY32, - "try_output_dict_columns did not produce a DICTIONARY32 column; check that the " + "output_dict_columns did not produce a DICTIONARY32 column; check that the " "generated data is fully dictionary-encoded"); } @@ -108,7 +108,7 @@ NVBENCH_BENCH(BM_parquet_read_dict_transcode) .set_name("parquet_read_dict_transcode") .add_string_axis("io_type", {"DEVICE_BUFFER"}) .set_min_samples(4) - .add_int64_axis("try_output_dict_columns", {0, 1}) + .add_int64_axis("output_dict_columns", {0, 1}) .add_int64_axis("cardinality", {100, 1'000, 10'000}) .add_int64_axis("num_cols", {1, 8}) .add_int64_axis("data_size", {512 << 20}) diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index b58c4a11da4e..e4d9d099a57c 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -110,8 +110,8 @@ class parquet_reader_options { type_id _decimal_width{type_id::EMPTY}; // Whether to use JIT compilation for filtering bool _use_jit_filter = false; - // Best-effort: try to output DICTIONARY32 columns for fully dict-encoded string columns - bool _try_output_dict_columns = false; + // For flat string columns, output DICT32 encoded string columns + bool _output_dict_columns = false; // Whether column name matching is case sensitive. In case of multiple // case-insensitive matches, the first matched column is selected bool _case_sensitive_names = true; @@ -345,14 +345,19 @@ class parquet_reader_options { /** * @brief Returns whether the reader should try to output DICTIONARY32 columns. * - * When true, the reader may output DICTIONARY32 columns for fully dict-encoded + * When true, the reader outputs DICTIONARY32 columns for fully dict-encoded * string columns instead of fully decoded STRING columns. A DICTIONARY32 column * consists of an INT32 indices child and a STRING keys child. - * Best-effort: falls back to STRING if the column has mixed encoding. * - * @return `true` if the reader should try to output DICTIONARY32 columns + * AST filters do not support dictionary columns yet, so when a filter is set this option is + * silently disabled and the columns are returned as STRING for the filter to operate on. + * + * @return `true` if the reader should output DICTIONARY32 columns for flat string columns */ - [[nodiscard]] bool is_enabled_try_output_dict_columns() const { return _try_output_dict_columns; } + [[nodiscard]] bool is_enabled_output_dict_columns() const + { + return _output_dict_columns and not _filter.has_value(); + } /** * @brief Set a new source location @@ -651,9 +656,9 @@ class parquet_reader_options { /** * @brief Sets to enable/disable trying to output DICTIONARY32 columns. * - * @param val Boolean indicating whether to try to output DICTIONARY32 columns + * @param val Boolean indicating whether to output DICTIONARY32 columns for flat string columns */ - void enable_try_output_dict_columns(bool val) { _try_output_dict_columns = val; } + void enable_output_dict_columns(bool val) { _output_dict_columns = val; } }; /** @@ -929,26 +934,14 @@ class parquet_reader_options_builder { } /** - * @brief Sets whether to prepend a source file index column to the output. + * @brief Sets to enable/disable trying to output DICTIONARY32 columns. * - * @param val Boolean indicating whether to prepend a source file index column + * @param val Boolean value whether to try to output DICTIONARY32 columns * @return this for chaining */ parquet_reader_options_builder& prepend_source_index_column(bool val) { - options.enable_prepend_source_index_column(val); - return *this; - } - - /** - * @brief Sets whether to prepend a file-local row index column to the output. - * - * @param val Boolean indicating whether to prepend a row index column - * @return this for chaining - */ - parquet_reader_options_builder& prepend_row_index_column(bool val) - { - options.enable_prepend_row_index_column(val); + options._prepend_source_index_column = val; return *this; } @@ -960,7 +953,7 @@ class parquet_reader_options_builder { */ parquet_reader_options_builder& try_output_dict_columns(bool val) { - options._try_output_dict_columns = val; + options._output_dict_columns = val; return *this; } diff --git a/cpp/src/dictionary/detail/concatenate.cu b/cpp/src/dictionary/detail/concatenate.cu index f8904504c5ae..04bfe521eb47 100644 --- a/cpp/src/dictionary/detail/concatenate.cu +++ b/cpp/src/dictionary/detail/concatenate.cu @@ -153,8 +153,6 @@ struct map_indices_fn { } // namespace -// TODO: Overload function to accept multiple vectors to do the concatenate at once, with a 2D -// kernel. std::unique_ptr concatenate(host_span columns, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index c76d136bd152..59d1002a5583 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -542,26 +542,21 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, options.is_enabled_case_sensitive_names(), options.is_enabled_prepend_source_index_column(), options.is_enabled_prepend_row_index_column(), - options.is_enabled_try_output_dict_columns()}, + options.is_enabled_output_dict_columns()}, _sources{std::move(sources)}, _output_chunk_read_limit{chunk_read_limit}, _input_pass_read_limit{pass_read_limit} { - // Direct parquet-dict → DICTIONARY32 transcode currently only supports single-pass, non-chunked - // reads. Splitting rowgroups across passes/subpasses would require aligning dictionary keys - // across passes, which we don't support yet. - CUDF_EXPECTS( - not _options.try_output_dict_columns or (chunk_read_limit == 0 and pass_read_limit == 0), - "try_output_dict_columns is only supported for single-pass reads; it cannot be combined " - "with a non-zero chunk_read_limit or pass_read_limit."); - - // AST filters do not support dictionary columns yet (see the column selection below). The - // transcode fast path and the `finalize_output` fallback both convert flat STRING columns to - // DICTIONARY32 *before* the filter is evaluated, which would feed dictionary columns to the AST. - // Since `try_output_dict_columns` is best-effort, we silently disable it for filtered reads so - // the filter still operates on STRING columns (the columns are simply returned as STRING). - if (_options.try_output_dict_columns and options.get_filter().has_value()) { - _options.try_output_dict_columns = false; + // The direct parquet-dict → DICTIONARY32 transcode fast path only supports single-pass, + // non-chunked reads. Splitting rowgroups across passes/subpasses would require aligning + // dictionary keys across passes, which are not supported yet. In that scenario, we silently + // skip the fast path in `prepare_dict_transcode` and still produce DICTIONARY32 output + // via the post-hoc `dictionary::detail::encode` fallback in `finalize_output`. + if (_options.output_dict_columns and (chunk_read_limit != 0 or pass_read_limit != 0)) { + CUDF_LOG_WARN( + "output_dict_columns: the direct parquet-dict transcode fast path is disabled for chunked / " + "multi-pass reads (non-zero chunk_read_limit or pass_read_limit); falling back to encoding " + "DICTIONARY32 columns at the output."); } // Open and parse the source dataset metadata @@ -977,7 +972,7 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, // with mixed or non-dictionary encodings, nested schemas, or columns added as empty columns // above), fall back to a post-hoc `dictionary::detail::encode` so the user still gets a // DICTIONARY32 column from every flat string column in the output table. - if (_options.try_output_dict_columns) { + if (_options.output_dict_columns) { for (auto& col : out_columns) { if (col and col->type().id() == type_id::STRING) { col = diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 279e7ed4ab99..624e4c7a2e6c 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -581,7 +581,7 @@ class reader_impl { // Whether to prepend the file-local row index column to the output bool prepend_row_index_column = false; // Whether to try outputting DICTIONARY32 columns for fully dict-encoded string columns - bool try_output_dict_columns = false; + bool output_dict_columns = false; } _options; // name to reference converter to extract AST output filter diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index d0ccb4ee9d58..508691aed685 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -10,9 +10,12 @@ #include #include #include +#include +#include #include #include #include +#include #include @@ -109,7 +112,7 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) * * We scan host-side `pass.chunks` and `pass.pages` here rather than `subpass.pages` because * `subpass.pages` may be a subset. For single-pass single-subpass reads (the only configuration - * in which `try_output_dict_columns` is supported), `subpass.pages == pass.pages`. + * in which `output_dict_columns` is supported), `subpass.pages == pass.pages`. * * @param pass The pass intermediate data holding host-side chunks and pages * @param input_columns The reader's input column descriptors @@ -124,14 +127,13 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) auto const num_input_cols = input_columns.size(); std::vector elig(num_input_cols); - // Check if the output buffer is a flat string column - std::for_each( - cuda::counting_iterator{0}, cuda::counting_iterator{num_input_cols}, [&](size_t i) { - auto const& input_col = input_columns[i]; - if (input_col.nesting_depth() != 1) { return; } - if (output_buffers[input_col.nesting[0]].type.id() == type_id::STRING) { - elig[i].has_string_buffer = true; - } + // Mark columns whose output buffer is a flat string column. + std::transform( + input_columns.begin(), input_columns.end(), elig.begin(), [&](input_column_info const& col) { + column_eligibility e{}; + e.has_string_buffer = + col.nesting_depth() == 1 and output_buffers[col.nesting[0]].type.id() == type_id::STRING; + return e; }); // Fold per-chunk info into the per-column eligibility flags. @@ -182,7 +184,13 @@ bool reader_impl::prepare_dict_transcode() _dict_transcode_eligible.assign(_input_columns.size(), false); - if (not _options.try_output_dict_columns) { return false; } + if (not _options.output_dict_columns) { return false; } + + // The fast path requires the whole column to live in a single subpass. For chunked / multi-pass + // reads (non-zero chunk or pass read limit) we skip it and let `finalize_output` produce the + // DICTIONARY32 columns via a post-hoc `dictionary::detail::encode` instead. + if (_output_chunk_read_limit != 0 or _input_pass_read_limit != 0) { return false; } + if (_pass_itm_data == nullptr or _pass_itm_data->subpass == nullptr) { return false; } auto& pass = *_pass_itm_data; @@ -253,17 +261,25 @@ void reader_impl::zero_init_dict_transcoded_index_buffers() // null positions. Zero them here so null rows carry a well-defined (valid) index into the // dictionary keys -- a requirement for `cudf::dictionary::detail::concatenate` to correctly // remap indices below. - std::for_each( - cuda::counting_iterator{0}, - cuda::counting_iterator{_input_columns.size()}, - [&](size_t i) { - if (not _dict_transcode_eligible[i]) { return; } - auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; - if (out_buf.type.id() != type_id::INT32) { return; } - if (out_buf.data() == nullptr or out_buf.size == 0) { return; } - CUDF_CUDA_TRY(cudaMemsetAsync( - out_buf.data(), 0, static_cast(out_buf.size) * sizeof(int32_t), _stream.value())); - }); + std::vector> index_bufs; + index_bufs.reserve(_input_columns.size()); + std::for_each(cuda::counting_iterator{0}, + cuda::counting_iterator{_input_columns.size()}, + [&](size_t i) { + if (not _dict_transcode_eligible[i]) { return; } + auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; + if (out_buf.type.id() != type_id::INT32) { return; } + if (out_buf.data() == nullptr or out_buf.size == 0) { return; } + index_bufs.emplace_back(static_cast(out_buf.data()), + static_cast(out_buf.size)); + }); + + if (index_bufs.empty()) { return; } + + // Zero all eligible index buffers in a single batched operation instead of one memset per column. + auto const pinned_index_bufs = cudf::detail::make_pinned_vector( + cudf::host_span const>{index_bufs}, _stream); + cudf::detail::batched_memset(pinned_index_bufs, 0, _stream); } void reader_impl::assemble_dict_transcoded_columns( @@ -297,6 +313,13 @@ void reader_impl::assemble_dict_transcoded_columns( [&](size_t c) { return pass.chunks[c].src_col_index == static_cast(i); }); if (chunk_indices.empty()) { return; } + // `out_columns` is indexed by output-buffer (root column) ordinal, not input-column + // ordinal: a nested struct/list column contributes one entry to `_output_buffers` but one + // entry per leaf to `_input_columns`, so `i` and the corresponding root index can diverge + // as soon as any nested column precedes this one. Eligibility requires a flat (depth-1) + // column, so `nesting[0]` is the correct, and only, output-buffer index to use here. + auto const out_idx = static_cast(_input_columns[i].nesting[0]); + // Per-chunk key counts from the dictionary page's `num_input_values`, mirrored back to // host when `pass.pages` was copied by `decode_page_headers`. std::vector chunk_key_counts(chunk_indices.size(), 0); @@ -317,7 +340,7 @@ void reader_impl::assemble_dict_transcoded_columns( // Grab ownership of the decoded INT32 indices column. Its buffer is shared (aliased) by // every per-chunk DICTIONARY32 view below via the parent view's offset/size, so it must // stay alive until the per-column concatenate/assembly completes. - auto& indices_col = out_columns[i]; + auto& indices_col = out_columns[out_idx]; CUDF_EXPECTS(indices_col != nullptr and indices_col->type().id() == type_id::INT32, "Expected INT32 indices column for dict-transcoded flat string column"); auto indices_owner = std::move(indices_col); @@ -370,11 +393,16 @@ void reader_impl::assemble_dict_transcoded_columns( if (dict_segment_views.size() == 1) { // Single row group: the parquet dictionary page keys are already unique, so no dedup is // needed. Take ownership of the decoded INT32 indices buffer directly (zero copy). - out_columns[i] = cudf::make_dictionary_column( + out_columns[out_idx] = cudf::make_dictionary_column( std::move(seg_keys_owners.front()), std::move(indices_owner), _stream, _mr); } else { - // `cudf::detail::concatenate` deduplicates + sorts keys and recomputes indices. - out_columns[i] = cudf::detail::concatenate(dict_segment_views, _stream, _mr); + // `cudf::detail::concatenate` deduplicates + sorts keys and recomputes indices. This is + // required today because DICTIONARY32 keys are assumed unique and sorted. When + // https://github.com/rapidsai/cudf/pull/22839 lands and relaxes that constraint, this + // could be replaced with a cheaper path: plain-concatenate the per-chunk keys columns + // (keeping cross-chunk duplicates) and offset-shift each chunk's row-group-local indices + // by the running total of prior chunks' key counts, avoiding the dedup/sort entirely. + out_columns[out_idx] = cudf::detail::concatenate(dict_segment_views, _stream, _mr); } }); } diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 69c1e0d1e92d..789ceb30f11e 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -113,7 +113,7 @@ void write_parquet(cudf::table_view const& input, std::string const& filepath) cudf::io::table_with_metadata read_parquet_as_dict(std::string const& filepath) { auto const read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) - .try_output_dict_columns(true) + .output_dict_columns(true) .build(); return cudf::io::read_parquet(read_opts); } @@ -123,7 +123,7 @@ cudf::io::table_with_metadata read_parquet_as_dict(std::string const& filepath) struct ParquetReaderDictTest : public cudf::test::BaseFixture {}; // A flat string column that is fully dictionary-encoded in every row group should be returned -// as a DICTIONARY32 column when `try_output_dict_columns` is enabled, and the decoded keys +// as a DICTIONARY32 column when `output_dict_columns` is enabled, and the decoded keys // should match the original input. TEST_F(ParquetReaderDictTest, FlatStringDictTranscode) { @@ -143,7 +143,7 @@ TEST_F(ParquetReaderDictTest, FlatStringDictTranscode) auto const read_col = read_table->view().column(0); ASSERT_EQ(read_col.type().id(), cudf::type_id::DICTIONARY32) - << "Expected the reader to produce a DICTIONARY32 column when try_output_dict_columns is on"; + << "Expected the reader to produce a DICTIONARY32 column when output_dict_columns is on"; cudf::dictionary_column_view dict_read_view(read_col); auto const decoded_read = cudf::dictionary::decode(dict_read_view); @@ -173,7 +173,7 @@ TEST_F(ParquetReaderDictTest, FlatStringNoTranscodeByDefault) } // List is not eligible for Parquet-dictionary → DICTIONARY32 transcode (flat string columns -// only). With `try_output_dict_columns` enabled, the reader still round-trips as LIST. +// only). With `output_dict_columns` enabled, the reader still round-trips as LIST. TEST_F(ParquetReaderDictTest, ListOfStringsDictEncodedWithTryOutputDictOption) { auto list_col = make_low_cardinality_lists_of_strings(); @@ -189,7 +189,7 @@ TEST_F(ParquetReaderDictTest, ListOfStringsDictEncodedWithTryOutputDictOption) auto const read_col = read_table->view().column(0); ASSERT_EQ(read_col.type().id(), cudf::type_id::LIST) - << "List must remain LIST when try_output_dict_columns is on (transcode is flat-only)"; + << "List must remain LIST when output_dict_columns is on (transcode is flat-only)"; CUDF_TEST_EXPECT_COLUMNS_EQUAL(list_col->view(), read_col); } From 2a1ea2d1852499d9195eb31d5c36bc220ad14ae5 Mon Sep 17 00:00:00 2001 From: ykiran Date: Wed, 1 Jul 2026 19:15:15 -0700 Subject: [PATCH 20/42] More bug fixes --- cpp/src/io/parquet/reader_impl.cpp | 2 +- cpp/src/io/parquet/reader_impl.hpp | 10 ++++++- .../io/parquet/reader_impl_dict_transcode.cu | 28 +++++++++++++------ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 59d1002a5583..5b8c691ec19e 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -730,7 +730,7 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // eligibility and mutate `_output_buffers` / `subpass.pages` before we allocate column buffers // or dispatch decode kernels. This has to happen before `preprocess_chunk_strings` / // `allocate_columns` because those branch on `subpass.kernel_mask` and on `out_buf.type`. - bool const dict_transcode_active = prepare_dict_transcode(); + bool const dict_transcode_active = prepare_dict_transcode(mode); // computes: // PageNestingInfo::batch_size for each level of nesting, for each page, taking row bounds into diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 624e4c7a2e6c..524d0ed38164 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -200,10 +200,18 @@ class reader_impl { * Populates `_dict_transcode_eligible` with a bool per input column indicating whether the * column will be assembled as a DICTIONARY32 output later in `assemble_dict_transcoded_columns`. * + * The fast path is also skipped when custom row bounds are in effect (see + * `uses_custom_row_bounds`): `assemble_dict_transcoded_columns` derives per-chunk row segments + * from the full, unadjusted `ColumnChunkDesc::num_rows`, which would not match the decoded + * indices column's size once a `skip_rows` / `num_rows` slice is applied. Skipped columns still + * get DICTIONARY32 output via the post-hoc `dictionary::detail::encode` fallback in + * `finalize_output`. + * + * @param mode Value indicating if the data sources are read all at once or chunk by chunk * @return True if dict transcode is active for this read (eligible columns had output types and * decode masks updated and pushed to the device). False otherwise. */ - [[nodiscard]] bool prepare_dict_transcode(); + [[nodiscard]] bool prepare_dict_transcode(read_mode mode); /** * @brief Zero-initialize the INT32 output buffers of dict-transcoded columns so that null rows diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 508691aed685..455be6a88bcb 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -178,7 +178,7 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) } // namespace -bool reader_impl::prepare_dict_transcode() +bool reader_impl::prepare_dict_transcode(read_mode mode) { CUDF_FUNC_RANGE(); @@ -191,6 +191,11 @@ bool reader_impl::prepare_dict_transcode() // DICTIONARY32 columns via a post-hoc `dictionary::detail::encode` instead. if (_output_chunk_read_limit != 0 or _input_pass_read_limit != 0) { return false; } + // Custom row bounds (`skip_rows` / `num_rows`) slice the decoded output to fewer rows than the + // full, unadjusted chunks that `assemble_dict_transcoded_columns` segments by. Skip the fast + // path here too and fall back to the post-hoc encode. + if (uses_custom_row_bounds(mode)) { return false; } + if (_pass_itm_data == nullptr or _pass_itm_data->subpass == nullptr) { return false; } auto& pass = *_pass_itm_data; @@ -365,8 +370,11 @@ void reader_impl::assemble_dict_transcoded_columns( // INT32 buffer. We select each chunk's row range via the parent dictionary view's // `offset`/`size` rather than slicing the indices child: `get_indices_annotated()` rebuilds // the indices view from the child's `head()` plus the parent's offset/size, so a sliced - // child (carrying its own offset) would be ignored. `cudf::detail::concatenate` then - // rewrites the row-group-local indices against the unified, deduplicated keys. + // child (carrying its own offset) would be ignored. For the same reason the null mask must + // also live on the parent view (sourced from the shared `indices_view`'s mask): the indices + // child's own null mask, if it had one, would be ignored by `get_indices_annotated()`, and a + // hardcoded null_count of 0 would silently turn nulls into a valid index (0) once + // `cudf::detail::concatenate` rewrites indices against the unified, deduplicated keys. std::vector> seg_keys_owners(chunk_indices.size()); std::vector dict_segment_views(chunk_indices.size()); std::transform(cuda::counting_iterator{0}, @@ -379,13 +387,17 @@ void reader_impl::assemble_dict_transcoded_columns( seg_keys_owners[k] = make_keys_column_from_index_pairs( chunk.str_dict_index, chunk_key_counts[k], _stream, _mr); - auto const seg_rows = chunk_row_offsets[k + 1] - chunk_row_offsets[k]; + auto const seg_begin = chunk_row_offsets[k]; + auto const seg_end = chunk_row_offsets[k + 1]; + auto const seg_rows = seg_end - seg_begin; + auto const seg_null_count = + indices_view.null_count(seg_begin, seg_end, _stream); return column_view{data_type{type_id::DICTIONARY32}, seg_rows, - nullptr, // dictionary parent holds no data - nullptr, // non-nullable transcode path - 0, // null count - chunk_row_offsets[k], // reslices shared indices child + nullptr, // dictionary parent holds no data + indices_view.null_mask(), // shared with indices_view + seg_null_count, + seg_begin, // reslices shared indices child + null mask {indices_view, seg_keys_owners[k]->view()}}; }); From d063325c60975bfb669f19823d0233e9040ca206 Mon Sep 17 00:00:00 2001 From: ykiran Date: Wed, 1 Jul 2026 19:28:10 -0700 Subject: [PATCH 21/42] Formatting fixes --- cpp/src/io/parquet/reader_impl_dict_transcode.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 455be6a88bcb..d834a67c0ede 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -394,7 +394,7 @@ void reader_impl::assemble_dict_transcoded_columns( indices_view.null_count(seg_begin, seg_end, _stream); return column_view{data_type{type_id::DICTIONARY32}, seg_rows, - nullptr, // dictionary parent holds no data + nullptr, // dictionary parent holds no data indices_view.null_mask(), // shared with indices_view seg_null_count, seg_begin, // reslices shared indices child + null mask From b444c7be7653874084892878ccfb741069355bac Mon Sep 17 00:00:00 2001 From: ykiran Date: Tue, 7 Jul 2026 16:24:45 -0700 Subject: [PATCH 22/42] Refined parquet benchmark --- .../io/parquet/parquet_reader_dict.cpp | 99 +++++++++++++++---- 1 file changed, 81 insertions(+), 18 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 0a69827c094c..1a118d5f464d 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -8,19 +8,49 @@ #include #include +#include +#include #include +#include #include #include +#include +#include +#include +#include +#include + // Benchmark for the parquet-dictionary -> cudf DICTIONARY32 transcode fast path enabled by // `parquet_reader_options::output_dict_columns`. It reads a fully dictionary-encoded set of -// low-cardinality string columns both with the option off (the column materializes as STRING) and -// with the option on (the reader keeps the dictionary representation and emits DICTIONARY32). The -// `output_dict_columns` axis lets the two paths be compared directly. +// low-cardinality string columns under three modes, selected by the `mode` axis, so the transcode +// path can be judged against both a lower and an upper reference: +// +// - "plain_string": reader default; columns materialize as STRING. This is the cheapest possible +// read (no dictionary built) and serves as the lower-bound reference -- it does +// strictly less work and produces a different (STRING) output. +// - "decode_encode": read as STRING, then `cudf::dictionary::encode` each column to DICTIONARY32. +// This is the pre-existing way to obtain a dictionary column and is the fair +// apples-to-apples baseline the transcode fast path aims to beat. +// - "transcode": `output_dict_columns=true`; the reader keeps the dictionary representation and +// emits DICTIONARY32 directly, skipping string materialization. +// +// Both "decode_encode" and "transcode" produce DICTIONARY32 output, so their times and peak memory +// are directly comparable; "plain_string" shows the floor cost of just decoding. namespace { +enum class bench_mode { plain_string, decode_encode, transcode }; + +[[nodiscard]] bench_mode parse_mode(std::string const& mode) +{ + if (mode == "plain_string") { return bench_mode::plain_string; } + if (mode == "decode_encode") { return bench_mode::decode_encode; } + if (mode == "transcode") { return bench_mode::transcode; } + CUDF_FAIL("Unknown benchmark mode: " + mode); +} + // The transcode fast path requires every data page of an eligible column to be dictionary-encoded. // Forcing `dictionary_policy::ALWAYS` together with low cardinality guarantees this for the // generated data. @@ -46,10 +76,20 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) auto const data_size = static_cast(state.get_int64("data_size")); auto const num_cols = static_cast(state.get_int64("num_cols")); auto const rg_size_rows = state.get_int64("row_group_size_rows"); - auto const output_dict = static_cast(state.get_int64("output_dict_columns")); + auto const mode = parse_mode(state.get_string("mode")); auto const avg_string_length = static_cast(state.get_int64("avg_string_length")); auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); + // nvbench axes form a full Cartesian product, so the (large data_size x single-column) cell is + // generated even though we don't want it: a single ~2 GB STRING column would overflow 32-bit + // string offsets. Skip that combination and keep the large data size to multi-column runs only. + if (num_cols == 1 and data_size > (std::size_t{512} << 20)) { + state.skip( + "Single-column reads above 512 MiB would overflow 32-bit string offsets; the large " + "data_size point is restricted to multi-column configurations"); + return; + } + // corresponds to 3 sigma (full width 6 sigma: 99.7% of range) auto const half_width = avg_string_length >> 3; auto const length_min = avg_string_length - half_width; @@ -71,16 +111,37 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) cudf::io::parquet_reader_options read_opts = cudf::io::parquet_reader_options::builder(source_sink.make_source_info()) - .output_dict_columns(output_dict); - - // Sanity check (outside the timed region): when the option is on the eligible string columns must - // come back as DICTIONARY32, otherwise the benchmark would silently measure the plain path. - if (output_dict) { - auto const probe = cudf::io::read_parquet(read_opts); - CUDF_EXPECTS(probe.tbl->num_columns() == num_cols, "Unexpected number of columns"); - CUDF_EXPECTS(probe.tbl->view().column(0).type().id() == cudf::type_id::DICTIONARY32, - "output_dict_columns did not produce a DICTIONARY32 column; check that the " - "generated data is fully dictionary-encoded"); + .output_dict_columns(mode == bench_mode::transcode); + + // Perform the full work for the selected mode: read, and for `decode_encode` additionally encode + // each STRING column to DICTIONARY32. Returns the resulting table so it can be reused for both the + // outside-the-timed-region verification and the timed measurement. + auto const run_mode = [&]() -> std::unique_ptr { + auto result = cudf::io::read_parquet(read_opts); + if (mode == bench_mode::decode_encode) { + std::vector> encoded; + encoded.reserve(result.tbl->num_columns()); + for (auto const& col : result.tbl->view()) { + encoded.push_back(cudf::dictionary::encode(col)); + } + return std::make_unique(std::move(encoded)); + } + return std::move(result.tbl); + }; + + // Sanity check (outside the timed region, run for every mode so warm-up is symmetric): confirm the + // produced column types match the mode, otherwise the benchmark would silently measure the wrong + // path (e.g. `transcode` falling back to STRING because the data is not fully dictionary-encoded). + { + auto const probe = run_mode(); + CUDF_EXPECTS(probe->num_columns() == num_cols, "Unexpected number of columns"); + auto const expected_id = + (mode == bench_mode::plain_string) ? cudf::type_id::STRING : cudf::type_id::DICTIONARY32; + for (auto const& col : probe->view()) { + CUDF_EXPECTS(col.type().id() == expected_id, + "Produced column type does not match the benchmark mode; check that the " + "generated data is fully dictionary-encoded"); + } } auto mem_stats_logger = cudf::memory_stats_logger(); @@ -90,10 +151,10 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) drop_page_cache_if_enabled(read_opts.get_source().filepaths()); timer.start(); - auto const result = cudf::io::read_parquet(read_opts); + auto const result = run_mode(); timer.stop(); - CUDF_EXPECTS(result.tbl->num_columns() == num_cols, "Unexpected number of columns"); + CUDF_EXPECTS(result->num_columns() == num_cols, "Unexpected number of columns"); }); auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); @@ -108,10 +169,12 @@ NVBENCH_BENCH(BM_parquet_read_dict_transcode) .set_name("parquet_read_dict_transcode") .add_string_axis("io_type", {"DEVICE_BUFFER"}) .set_min_samples(4) - .add_int64_axis("output_dict_columns", {0, 1}) + .add_string_axis("mode", {"plain_string", "decode_encode", "transcode"}) .add_int64_axis("cardinality", {100, 1'000, 10'000}) .add_int64_axis("num_cols", {1, 8}) - .add_int64_axis("data_size", {512 << 20}) + // 2 GiB point is single-column-skipped in the body (see state.skip): a lone ~2 GB string column + // would overflow 32-bit string offsets, so the large size only runs with num_cols > 1. + .add_int64_axis("data_size", {std::int64_t{512} << 20, std::int64_t{2} << 30}) .add_int64_axis("avg_string_length", {16}) // Sentinel 0 == default row groups; small values force multiple row groups, exercising the // per-row-group key concatenation / index remapping path. From cb65d0ec70f851c0a6fe3ff46b1c3906a1acf0fd Mon Sep 17 00:00:00 2001 From: ykiran Date: Tue, 7 Jul 2026 16:59:03 -0700 Subject: [PATCH 23/42] Added table to bench output --- .../io/parquet/parquet_reader_dict.cpp | 102 +++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 1a118d5f464d..b51c9867972c 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -16,10 +16,14 @@ #include +#include #include #include +#include +#include #include #include +#include #include // Benchmark for the parquet-dictionary -> cudf DICTIONARY32 transcode fast path enabled by @@ -51,6 +55,90 @@ enum class bench_mode { plain_string, decode_encode, transcode }; CUDF_FAIL("Unknown benchmark mode: " + mode); } +// nvbench invokes the benchmark once per axis combination and prints its own results table; there is +// no cross-state hook, so a single invocation cannot compare `transcode` against `decode_encode`. +// This collector accumulates each run's CPU/GPU mean time, keyed by every setting except `mode`, and +// prints a relative comparison table from its destructor -- i.e. at program exit, after nvbench's +// own output. `decode_encode` is the 100% reference and `transcode` is shown as a percentage of it. +struct run_settings { + std::int64_t cardinality; + std::int64_t num_cols; + std::int64_t data_size; + std::int64_t avg_string_length; + std::int64_t row_group_size_rows; + + bool operator<(run_settings const& o) const + { + return std::tie(cardinality, num_cols, data_size, avg_string_length, row_group_size_rows) < + std::tie( + o.cardinality, o.num_cols, o.data_size, o.avg_string_length, o.row_group_size_rows); + } +}; + +struct mode_timing { + double cpu_ms = 0.0; + double gpu_ms = 0.0; + bool present = false; +}; + +class comparison_collector { + public: + void record(run_settings const& key, bench_mode mode, double cpu_ms, double gpu_ms) + { + auto& row = _rows[key]; + auto& slot = + (mode == bench_mode::decode_encode) + ? row.decode_encode + : ((mode == bench_mode::transcode) ? row.transcode : row.plain_string); + slot = mode_timing{cpu_ms, gpu_ms, true}; + } + + ~comparison_collector() { print(); } + + private: + struct row { + mode_timing plain_string; + mode_timing decode_encode; + mode_timing transcode; + }; + + void print() const + { + auto const has_pair = [](row const& r) { return r.decode_encode.present and r.transcode.present; }; + if (std::none_of(_rows.begin(), _rows.end(), [&](auto const& kv) { return has_pair(kv.second); })) { + return; + } + + std::printf("\n# transcode vs decode_encode (decode_encode = 100%% reference)\n\n"); + std::printf( + "| cardinality | num_cols | data_size (MiB) | row_group_size_rows | decode_encode CPU (ms) | " + "transcode CPU (ms) | transcode CPU %% | decode_encode GPU (ms) | transcode GPU (ms) | " + "transcode GPU %% |\n"); + std::printf("|---|---|---|---|---|---|---|---|---|---|\n"); + for (auto const& [key, r] : _rows) { + if (not has_pair(r)) { continue; } + auto const cpu_pct = 100.0 * r.transcode.cpu_ms / r.decode_encode.cpu_ms; + auto const gpu_pct = 100.0 * r.transcode.gpu_ms / r.decode_encode.gpu_ms; + std::printf("| %lld | %lld | %lld | %lld | %.3f | %.3f | %.1f%% | %.3f | %.3f | %.1f%% |\n", + static_cast(key.cardinality), + static_cast(key.num_cols), + static_cast(key.data_size >> 20), + static_cast(key.row_group_size_rows), + r.decode_encode.cpu_ms, + r.transcode.cpu_ms, + cpu_pct, + r.decode_encode.gpu_ms, + r.transcode.gpu_ms, + gpu_pct); + } + std::printf("\n"); + } + + std::map _rows; +}; + +comparison_collector g_comparison_collector; + // The transcode fast path requires every data page of an eligible column to be dictionary-encoded. // Forcing `dictionary_policy::ALWAYS` together with low cardinality guarantees this for the // generated data. @@ -157,12 +245,24 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) CUDF_EXPECTS(result->num_columns() == num_cols, "Unexpected number of columns"); }); - auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); + auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); + auto const cpu_time = state.get_summary("nv/cold/time/cpu/mean").get_float64("value"); state.add_element_count(static_cast(data_size) / time, "bytes_per_second"); state.add_element_count(static_cast(view.num_rows()) / time, "rows_per_sec"); state.add_buffer_size( mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); + + // Record this run for the end-of-program transcode-vs-decode_encode comparison table. Times are + // reported by nvbench in seconds; store as milliseconds. + g_comparison_collector.record(run_settings{cardinality, + num_cols, + static_cast(data_size), + avg_string_length, + rg_size_rows}, + mode, + cpu_time * 1e3, + time * 1e3); } NVBENCH_BENCH(BM_parquet_read_dict_transcode) From 1fcf4ab0a20cb5d7b0582243e4d0c6c429bf1ebe Mon Sep 17 00:00:00 2001 From: ykiran Date: Tue, 7 Jul 2026 18:18:39 -0700 Subject: [PATCH 24/42] Modified sweep in benchmark --- .../io/parquet/parquet_reader_dict.cpp | 267 ++++++++++++------ 1 file changed, 183 insertions(+), 84 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index b51c9867972c..332d8df8060b 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -27,24 +28,37 @@ #include // Benchmark for the parquet-dictionary -> cudf DICTIONARY32 transcode fast path enabled by -// `parquet_reader_options::output_dict_columns`. It reads a fully dictionary-encoded set of -// low-cardinality string columns under three modes, selected by the `mode` axis, so the transcode -// path can be judged against both a lower and an upper reference: +// `parquet_reader_options::output_dict_columns`. A single fully dictionary-encoded string column is +// read under three modes, selected by the `mode` axis, so the transcode path can be judged against +// both a lower and an upper reference: // -// - "plain_string": reader default; columns materialize as STRING. This is the cheapest possible -// read (no dictionary built) and serves as the lower-bound reference -- it does -// strictly less work and produces a different (STRING) output. -// - "decode_encode": read as STRING, then `cudf::dictionary::encode` each column to DICTIONARY32. -// This is the pre-existing way to obtain a dictionary column and is the fair -// apples-to-apples baseline the transcode fast path aims to beat. +// - "plain_string": reader default; the column materializes as STRING. This is the cheapest +// possible read (no dictionary built) and serves as the lower-bound reference -- +// it does strictly less work and produces a different (STRING) output. +// - "decode_encode": read as STRING, then `cudf::dictionary::encode` to DICTIONARY32. This is the +// pre-existing way to obtain a dictionary column and is the fair apples-to-apples +// baseline the transcode fast path aims to beat. // - "transcode": `output_dict_columns=true`; the reader keeps the dictionary representation and // emits DICTIONARY32 directly, skipping string materialization. // // Both "decode_encode" and "transcode" produce DICTIONARY32 output, so their times and peak memory -// are directly comparable; "plain_string" shows the floor cost of just decoding. +// are directly comparable; "plain_string" shows the floor cost of just decoding. A relative +// comparison table (decode_encode = 100%%) is printed at program exit (see comparison_collector). +// +// The sweep varies four axes: cardinality, total table size, rows per row group, and rows per data +// page. A single column (num_cols == 1) is used so a row group can hold as many distinct values as +// possible: the writer picks the dictionary index bit width per row group from the distinct values +// it contains, capped at MAX_DICT_BITS (24). Cardinality therefore ranges up to 2^24, the point at +// which 24-bit indices are required. At high distinct-per-row-group counts the writer may abandon +// dictionary encoding (indices exceed 24 bits, or plain encoding is smaller); when that leaves the +// column ineligible for transcode, that state is skipped rather than measured. The largest +// single-column configurations exceed 2^31 characters and rely on cuDF's default (enabled) large +// strings support to switch to 64-bit offsets automatically. namespace { +constexpr cudf::size_type num_cols = 1; + enum class bench_mode { plain_string, decode_encode, transcode }; [[nodiscard]] bench_mode parse_mode(std::string const& mode) @@ -55,23 +69,46 @@ enum class bench_mode { plain_string, decode_encode, transcode }; CUDF_FAIL("Unknown benchmark mode: " + mode); } -// nvbench invokes the benchmark once per axis combination and prints its own results table; there is -// no cross-state hook, so a single invocation cannot compare `transcode` against `decode_encode`. -// This collector accumulates each run's CPU/GPU mean time, keyed by every setting except `mode`, and -// prints a relative comparison table from its destructor -- i.e. at program exit, after nvbench's -// own output. `decode_encode` is the 100% reference and `transcode` is shown as a percentage of it. +// Upper-bound estimate of the dictionary index bit width the writer will use for a row group. The +// width is derived per row group from its distinct value count, which is at most +// min(cardinality, rows in the row group). This over-estimates when the (last) row group is shorter +// than `row_group_size_rows` or when hash collisions reduce distinct counts. +[[nodiscard]] int approx_dict_bits(std::int64_t cardinality, std::int64_t row_group_size_rows) +{ + auto const distinct = std::min(cardinality, row_group_size_rows); + if (distinct <= 1) { return 1; } + int bits = 0; + auto max_index = distinct - 1; + while (max_index > 0) { + ++bits; + max_index >>= 1; + } + return bits; +} + +// nvbench invokes the benchmark once per axis combination, prints its own results table, and omits +// skipped states from it; there is also no cross-state hook, so a single invocation cannot group the +// three modes of a configuration together. This collector accumulates each run's CPU/GPU mean time +// (and any transcode skip reason), keyed by every setting except `mode`, and prints one row per +// configuration from its destructor -- i.e. at program exit, after nvbench's own output -- with all +// three modes in fixed order (plain_string, decode_encode, transcode) so each configuration is +// grouped and ordered regardless of nvbench's state ordering or its omission of skipped states. struct run_settings { std::int64_t cardinality; - std::int64_t num_cols; std::int64_t data_size; - std::int64_t avg_string_length; std::int64_t row_group_size_rows; + std::int64_t max_page_size_rows; + std::int64_t avg_string_length; bool operator<(run_settings const& o) const { - return std::tie(cardinality, num_cols, data_size, avg_string_length, row_group_size_rows) < - std::tie( - o.cardinality, o.num_cols, o.data_size, o.avg_string_length, o.row_group_size_rows); + return std::tie( + cardinality, data_size, row_group_size_rows, max_page_size_rows, avg_string_length) < + std::tie(o.cardinality, + o.data_size, + o.row_group_size_rows, + o.max_page_size_rows, + o.avg_string_length); } }; @@ -85,12 +122,18 @@ class comparison_collector { public: void record(run_settings const& key, bench_mode mode, double cpu_ms, double gpu_ms) { - auto& row = _rows[key]; - auto& slot = - (mode == bench_mode::decode_encode) - ? row.decode_encode - : ((mode == bench_mode::transcode) ? row.transcode : row.plain_string); - slot = mode_timing{cpu_ms, gpu_ms, true}; + auto& r = _rows[key]; + auto& slot = (mode == bench_mode::decode_encode) + ? r.decode_encode + : ((mode == bench_mode::transcode) ? r.transcode : r.plain_string); + slot = mode_timing{cpu_ms, gpu_ms, true}; + } + + // Record that transcode was skipped for a configuration, with a short reason shown in the table. + // `transcode` is the only mode this benchmark ever skips. + void record_skip(run_settings const& key, std::string reason) + { + _rows[key].transcode_note = std::move(reason); } ~comparison_collector() { print(); } @@ -100,36 +143,60 @@ class comparison_collector { mode_timing plain_string; mode_timing decode_encode; mode_timing transcode; + std::string transcode_note; // reason shown when `transcode` was skipped as ineligible }; void print() const { - auto const has_pair = [](row const& r) { return r.decode_encode.present and r.transcode.present; }; - if (std::none_of(_rows.begin(), _rows.end(), [&](auto const& kv) { return has_pair(kv.second); })) { - return; - } + if (_rows.empty()) { return; } - std::printf("\n# transcode vs decode_encode (decode_encode = 100%% reference)\n\n"); + std::printf("\n# Per-configuration mode comparison " + "(order: plain_string, decode_encode, transcode)\n\n"); std::printf( - "| cardinality | num_cols | data_size (MiB) | row_group_size_rows | decode_encode CPU (ms) | " - "transcode CPU (ms) | transcode CPU %% | decode_encode GPU (ms) | transcode GPU (ms) | " - "transcode GPU %% |\n"); - std::printf("|---|---|---|---|---|---|---|---|---|---|\n"); + "| cardinality | ~dict_bits | data_size (MiB) | row_group_size_rows | max_page_size_rows | " + "plain_string CPU (ms) | decode_encode CPU (ms) | transcode CPU (ms) | " + "plain_string GPU (ms) | decode_encode GPU (ms) | transcode GPU (ms) | " + "transcode CPU speedup %% | transcode GPU speedup %% |\n"); + std::printf("|---|---|---|---|---|---|---|---|---|---|---|---|---|\n"); + + auto const num = [](double v) { + std::array buf{}; + std::snprintf(buf.data(), buf.size(), "%.3f", v); + return std::string{buf.data()}; + }; + // Timing cell: the value if the mode ran, otherwise the skip reason (transcode only) or "-". + auto const cell = [&](mode_timing const& t, double mode_timing::*field, std::string const& note) { + if (t.present) { return num(t.*field); } + return note.empty() ? std::string{"-"} : note; + }; + + // Speedup of transcode over the decode_encode baseline, as a signed percentage of baseline time + // saved: 100 * (decode_encode - transcode) / decode_encode. Positive = transcode is faster, + // negative = slower. "-" when either mode is missing. + auto const speedup = + [](mode_timing const& base, mode_timing const& cand, double mode_timing::*field) { + if (not(base.present and cand.present)) { return std::string{"-"}; } + std::array buf{}; + std::snprintf( + buf.data(), buf.size(), "%+.1f%%", 100.0 * (base.*field - cand.*field) / (base.*field)); + return std::string{buf.data()}; + }; + for (auto const& [key, r] : _rows) { - if (not has_pair(r)) { continue; } - auto const cpu_pct = 100.0 * r.transcode.cpu_ms / r.decode_encode.cpu_ms; - auto const gpu_pct = 100.0 * r.transcode.gpu_ms / r.decode_encode.gpu_ms; - std::printf("| %lld | %lld | %lld | %lld | %.3f | %.3f | %.1f%% | %.3f | %.3f | %.1f%% |\n", + std::printf("| %lld | %d | %lld | %lld | %lld | %s | %s | %s | %s | %s | %s | %s | %s |\n", static_cast(key.cardinality), - static_cast(key.num_cols), + approx_dict_bits(key.cardinality, key.row_group_size_rows), static_cast(key.data_size >> 20), static_cast(key.row_group_size_rows), - r.decode_encode.cpu_ms, - r.transcode.cpu_ms, - cpu_pct, - r.decode_encode.gpu_ms, - r.transcode.gpu_ms, - gpu_pct); + static_cast(key.max_page_size_rows), + cell(r.plain_string, &mode_timing::cpu_ms, std::string{}).c_str(), + cell(r.decode_encode, &mode_timing::cpu_ms, std::string{}).c_str(), + cell(r.transcode, &mode_timing::cpu_ms, r.transcode_note).c_str(), + cell(r.plain_string, &mode_timing::gpu_ms, std::string{}).c_str(), + cell(r.decode_encode, &mode_timing::gpu_ms, std::string{}).c_str(), + cell(r.transcode, &mode_timing::gpu_ms, r.transcode_note).c_str(), + speedup(r.decode_encode, r.transcode, &mode_timing::cpu_ms).c_str(), + speedup(r.decode_encode, r.transcode, &mode_timing::gpu_ms).c_str()); } std::printf("\n"); } @@ -140,19 +207,25 @@ class comparison_collector { comparison_collector g_comparison_collector; // The transcode fast path requires every data page of an eligible column to be dictionary-encoded. -// Forcing `dictionary_policy::ALWAYS` together with low cardinality guarantees this for the -// generated data. +// Forcing `dictionary_policy::ALWAYS` maximizes the chance of full dictionary encoding; the writer +// can still fall back to plain when indices exceed MAX_DICT_BITS or plain is smaller, in which case +// the transcode state is skipped by the caller. void write_dict_encoded_parquet(cudf::table_view const& view, cuio_source_sink_pair& source_sink, - int64_t row_group_size_rows) + std::int64_t row_group_size_rows, + std::int64_t max_page_size_rows) { cudf::io::parquet_writer_options write_opts = cudf::io::parquet_writer_options::builder(source_sink.make_sink_info(), view) .compression(cudf::io::compression_type::NONE) .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN); - // Sentinel 0 == use cuDF default row-group sizing. - if (row_group_size_rows > 0) { write_opts.set_row_group_size_rows(row_group_size_rows); } + if (row_group_size_rows > 0) { + write_opts.set_row_group_size_rows(static_cast(row_group_size_rows)); + } + if (max_page_size_rows > 0) { + write_opts.set_max_page_size_rows(static_cast(max_page_size_rows)); + } cudf::io::write_parquet(write_opts); } @@ -162,22 +235,12 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) { auto const cardinality = static_cast(state.get_int64("cardinality")); auto const data_size = static_cast(state.get_int64("data_size")); - auto const num_cols = static_cast(state.get_int64("num_cols")); auto const rg_size_rows = state.get_int64("row_group_size_rows"); + auto const page_size_rows = state.get_int64("max_page_size_rows"); auto const mode = parse_mode(state.get_string("mode")); auto const avg_string_length = static_cast(state.get_int64("avg_string_length")); auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); - // nvbench axes form a full Cartesian product, so the (large data_size x single-column) cell is - // generated even though we don't want it: a single ~2 GB STRING column would overflow 32-bit - // string offsets. Skip that combination and keep the large data size to multi-column runs only. - if (num_cols == 1 and data_size > (std::size_t{512} << 20)) { - state.skip( - "Single-column reads above 512 MiB would overflow 32-bit string offsets; the large " - "data_size point is restricted to multi-column configurations"); - return; - } - // corresponds to 3 sigma (full width 6 sigma: 99.7% of range) auto const half_width = avg_string_length >> 3; auto const length_min = avg_string_length - half_width; @@ -195,7 +258,7 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) auto const view = tbl->view(); cuio_source_sink_pair source_sink(source_type); - write_dict_encoded_parquet(view, source_sink, rg_size_rows); + write_dict_encoded_parquet(view, source_sink, rg_size_rows, page_size_rows); cudf::io::parquet_reader_options read_opts = cudf::io::parquet_reader_options::builder(source_sink.make_source_info()) @@ -217,18 +280,50 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) return std::move(result.tbl); }; - // Sanity check (outside the timed region, run for every mode so warm-up is symmetric): confirm the - // produced column types match the mode, otherwise the benchmark would silently measure the wrong - // path (e.g. `transcode` falling back to STRING because the data is not fully dictionary-encoded). + // Verification (outside the timed region, run for every mode so warm-up is symmetric). For + // `transcode`, the writer may have fallen back to plain encoding at high cardinality / large row + // groups, leaving the column ineligible for the fast path; in that case skip the state rather than + // silently measuring the plain path or aborting the whole sweep. { auto const probe = run_mode(); - CUDF_EXPECTS(probe->num_columns() == num_cols, "Unexpected number of columns"); - auto const expected_id = - (mode == bench_mode::plain_string) ? cudf::type_id::STRING : cudf::type_id::DICTIONARY32; - for (auto const& col : probe->view()) { - CUDF_EXPECTS(col.type().id() == expected_id, - "Produced column type does not match the benchmark mode; check that the " - "generated data is fully dictionary-encoded"); + // Bind the table_view to a local: `probe->view()` returns a temporary, so calling it separately + // for begin() and end() would yield iterators into two different temporaries (mismatched- + // iterator UB). Iterate a single view instead. + auto const probe_view = probe->view(); + CUDF_EXPECTS(probe_view.num_columns() == num_cols, "Unexpected number of columns"); + auto const all_of_type = [&](cudf::type_id id) { + return std::all_of(probe_view.begin(), probe_view.end(), [id](auto const& col) { + return col.type().id() == id; + }); + }; + auto const actual_type_id = static_cast(probe_view.column(0).type().id()); + if (mode == bench_mode::plain_string) { + if (not all_of_type(cudf::type_id::STRING)) { + state.skip("plain_string produced unexpected type_id=" + std::to_string(actual_type_id) + + " (expected STRING=" + + std::to_string(static_cast(cudf::type_id::STRING)) + ")"); + return; + } + } else if (mode == bench_mode::decode_encode) { + if (not all_of_type(cudf::type_id::DICTIONARY32)) { + state.skip("decode_encode produced unexpected type_id=" + std::to_string(actual_type_id) + + " (expected DICTIONARY32=" + + std::to_string(static_cast(cudf::type_id::DICTIONARY32)) + ")"); + return; + } + } else if (not all_of_type(cudf::type_id::DICTIONARY32)) { + // Record the skip so the end-of-program per-configuration table can show why transcode has no + // timing for this configuration (nvbench omits skipped states from its own table). + g_comparison_collector.record_skip(run_settings{cardinality, + static_cast(data_size), + rg_size_rows, + page_size_rows, + avg_string_length}, + "skipped: plain fallback"); + state.skip( + "transcode did not produce DICTIONARY32: at this cardinality / row-group size the writer " + "fell back to plain encoding, making the column ineligible for the fast path"); + return; } } @@ -256,10 +351,10 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) // Record this run for the end-of-program transcode-vs-decode_encode comparison table. Times are // reported by nvbench in seconds; store as milliseconds. g_comparison_collector.record(run_settings{cardinality, - num_cols, static_cast(data_size), - avg_string_length, - rg_size_rows}, + rg_size_rows, + page_size_rows, + avg_string_length}, mode, cpu_time * 1e3, time * 1e3); @@ -270,12 +365,16 @@ NVBENCH_BENCH(BM_parquet_read_dict_transcode) .add_string_axis("io_type", {"DEVICE_BUFFER"}) .set_min_samples(4) .add_string_axis("mode", {"plain_string", "decode_encode", "transcode"}) - .add_int64_axis("cardinality", {100, 1'000, 10'000}) - .add_int64_axis("num_cols", {1, 8}) - // 2 GiB point is single-column-skipped in the body (see state.skip): a lone ~2 GB string column - // would overflow 32-bit string offsets, so the large size only runs with num_cols > 1. + // Cardinality spans up to 2^24, the point at which per-row-group dictionary indices need the + // maximum 24 bits the writer supports (MAX_DICT_BITS); beyond that the writer abandons dictionary + // encoding. Achieved bits = ceil(log2(min(cardinality, rows per row group))). + .add_int64_axis("cardinality", {1 << 10, 1 << 15, 1 << 20, 1 << 24}) + // Total table size (single column). The largest points exceed 2^31 chars and rely on cuDF's + // default large strings support (64-bit offsets). .add_int64_axis("data_size", {std::int64_t{512} << 20, std::int64_t{2} << 30}) - .add_int64_axis("avg_string_length", {16}) - // Sentinel 0 == default row groups; small values force multiple row groups, exercising the - // per-row-group key concatenation / index remapping path. - .add_int64_axis("row_group_size_rows", {0, 100'000}); + // Rows per row group: small (many row groups -> stresses per-row-group key concatenation) to very + // large (>= 2^24 so a single row group can hold enough distinct values to reach 24 dict bits). + .add_int64_axis("row_group_size_rows", {100'000, 1'000'000, 20'000'000}) + // Rows per data page: small to large. + .add_int64_axis("max_page_size_rows", {20'000, 100'000, 1'000'000}) + .add_int64_axis("avg_string_length", {16}); From 2c204d016df113f22140be1f6971469c828e0c34 Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 20 Jul 2026 12:40:56 -0700 Subject: [PATCH 25/42] Fix mangled rebase --- cpp/include/cudf/io/parquet.hpp | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index e4d9d099a57c..2283b7908cb0 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -934,16 +934,28 @@ class parquet_reader_options_builder { } /** - * @brief Sets to enable/disable trying to output DICTIONARY32 columns. + * @brief Sets whether to prepend a source file index column to the output. * - * @param val Boolean value whether to try to output DICTIONARY32 columns + * @param val Boolean indicating whether to prepend a source file index column * @return this for chaining */ - parquet_reader_options_builder& prepend_source_index_column(bool val) - { - options._prepend_source_index_column = val; - return *this; - } + parquet_reader_options_builder& prepend_source_index_column(bool val) + { + options.enable_prepend_source_index_column(val); + return *this; + } + + /** + * @brief Sets whether to prepend a file-local row index column to the output. + * + * @param val Boolean indicating whether to prepend a row index column + * @return this for chaining + */ + parquet_reader_options_builder& prepend_row_index_column(bool val) + { + options.enable_prepend_row_index_column(val); + return *this; + } /** * @brief Sets to enable/disable trying to output DICTIONARY32 columns. @@ -951,7 +963,7 @@ class parquet_reader_options_builder { * @param val Boolean value whether to try to output DICTIONARY32 columns * @return this for chaining */ - parquet_reader_options_builder& try_output_dict_columns(bool val) + parquet_reader_options_builder& output_dict_columns(bool val) { options._output_dict_columns = val; return *this; From be085f771e627d98d054ecfcb71943a676d74d4e Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 20 Jul 2026 16:00:47 -0700 Subject: [PATCH 26/42] Test changes --- .../io/parquet/parquet_reader_dict.cpp | 49 ++++++++++--------- cpp/include/cudf/io/parquet.hpp | 40 +++++++-------- cpp/src/dictionary/detail/concatenate.cu | 3 ++ cpp/tests/io/parquet_reader_dict_test.cpp | 18 +++---- 4 files changed, 58 insertions(+), 52 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 332d8df8060b..3dade95d9a23 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -33,12 +33,13 @@ // both a lower and an upper reference: // // - "plain_string": reader default; the column materializes as STRING. This is the cheapest -// possible read (no dictionary built) and serves as the lower-bound reference -- -// it does strictly less work and produces a different (STRING) output. +// possible read (no dictionary built) and serves as the lower-bound reference +// -- it does strictly less work and produces a different (STRING) output. // - "decode_encode": read as STRING, then `cudf::dictionary::encode` to DICTIONARY32. This is the -// pre-existing way to obtain a dictionary column and is the fair apples-to-apples -// baseline the transcode fast path aims to beat. -// - "transcode": `output_dict_columns=true`; the reader keeps the dictionary representation and +// pre-existing way to obtain a dictionary column and is the fair +// apples-to-apples baseline the transcode fast path aims to beat. +// - "transcode": `output_dict_columns=true`; the reader keeps the dictionary representation +// and // emits DICTIONARY32 directly, skipping string materialization. // // Both "decode_encode" and "transcode" produce DICTIONARY32 output, so their times and peak memory @@ -87,11 +88,11 @@ enum class bench_mode { plain_string, decode_encode, transcode }; } // nvbench invokes the benchmark once per axis combination, prints its own results table, and omits -// skipped states from it; there is also no cross-state hook, so a single invocation cannot group the -// three modes of a configuration together. This collector accumulates each run's CPU/GPU mean time -// (and any transcode skip reason), keyed by every setting except `mode`, and prints one row per -// configuration from its destructor -- i.e. at program exit, after nvbench's own output -- with all -// three modes in fixed order (plain_string, decode_encode, transcode) so each configuration is +// skipped states from it; there is also no cross-state hook, so a single invocation cannot group +// the three modes of a configuration together. This collector accumulates each run's CPU/GPU mean +// time (and any transcode skip reason), keyed by every setting except `mode`, and prints one row +// per configuration from its destructor -- i.e. at program exit, after nvbench's own output -- with +// all three modes in fixed order (plain_string, decode_encode, transcode) so each configuration is // grouped and ordered regardless of nvbench's state ordering or its omission of skipped states. struct run_settings { std::int64_t cardinality; @@ -150,8 +151,9 @@ class comparison_collector { { if (_rows.empty()) { return; } - std::printf("\n# Per-configuration mode comparison " - "(order: plain_string, decode_encode, transcode)\n\n"); + std::printf( + "\n# Per-configuration mode comparison " + "(order: plain_string, decode_encode, transcode)\n\n"); std::printf( "| cardinality | ~dict_bits | data_size (MiB) | row_group_size_rows | max_page_size_rows | " "plain_string CPU (ms) | decode_encode CPU (ms) | transcode CPU (ms) | " @@ -165,16 +167,17 @@ class comparison_collector { return std::string{buf.data()}; }; // Timing cell: the value if the mode ran, otherwise the skip reason (transcode only) or "-". - auto const cell = [&](mode_timing const& t, double mode_timing::*field, std::string const& note) { - if (t.present) { return num(t.*field); } - return note.empty() ? std::string{"-"} : note; - }; + auto const cell = + [&](mode_timing const& t, double mode_timing::* field, std::string const& note) { + if (t.present) { return num(t.*field); } + return note.empty() ? std::string{"-"} : note; + }; // Speedup of transcode over the decode_encode baseline, as a signed percentage of baseline time // saved: 100 * (decode_encode - transcode) / decode_encode. Positive = transcode is faster, // negative = slower. "-" when either mode is missing. auto const speedup = - [](mode_timing const& base, mode_timing const& cand, double mode_timing::*field) { + [](mode_timing const& base, mode_timing const& cand, double mode_timing::* field) { if (not(base.present and cand.present)) { return std::string{"-"}; } std::array buf{}; std::snprintf( @@ -265,8 +268,8 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) .output_dict_columns(mode == bench_mode::transcode); // Perform the full work for the selected mode: read, and for `decode_encode` additionally encode - // each STRING column to DICTIONARY32. Returns the resulting table so it can be reused for both the - // outside-the-timed-region verification and the timed measurement. + // each STRING column to DICTIONARY32. Returns the resulting table so it can be reused for both + // the outside-the-timed-region verification and the timed measurement. auto const run_mode = [&]() -> std::unique_ptr { auto result = cudf::io::read_parquet(read_opts); if (mode == bench_mode::decode_encode) { @@ -282,8 +285,8 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) // Verification (outside the timed region, run for every mode so warm-up is symmetric). For // `transcode`, the writer may have fallen back to plain encoding at high cardinality / large row - // groups, leaving the column ineligible for the fast path; in that case skip the state rather than - // silently measuring the plain path or aborting the whole sweep. + // groups, leaving the column ineligible for the fast path; in that case skip the state rather + // than silently measuring the plain path or aborting the whole sweep. { auto const probe = run_mode(); // Bind the table_view to a local: `probe->view()` returns a temporary, so calling it separately @@ -300,8 +303,8 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) if (mode == bench_mode::plain_string) { if (not all_of_type(cudf::type_id::STRING)) { state.skip("plain_string produced unexpected type_id=" + std::to_string(actual_type_id) + - " (expected STRING=" + - std::to_string(static_cast(cudf::type_id::STRING)) + ")"); + " (expected STRING=" + std::to_string(static_cast(cudf::type_id::STRING)) + + ")"); return; } } else if (mode == bench_mode::decode_encode) { diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 2283b7908cb0..0e9452f3a035 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -345,9 +345,9 @@ class parquet_reader_options { /** * @brief Returns whether the reader should try to output DICTIONARY32 columns. * - * When true, the reader outputs DICTIONARY32 columns for fully dict-encoded - * string columns instead of fully decoded STRING columns. A DICTIONARY32 column - * consists of an INT32 indices child and a STRING keys child. + * When true, the reader outputs DICTIONARY32 columns (instead of fully decoded STRING columns) + * for fully dict-encoded string columns . A DICTIONARY32 column consists of an INT32 indices + * child and a STRING keys child. * * AST filters do not support dictionary columns yet, so when a filter is set this option is * silently disabled and the columns are returned as STRING for the filter to operate on. @@ -939,23 +939,23 @@ class parquet_reader_options_builder { * @param val Boolean indicating whether to prepend a source file index column * @return this for chaining */ - parquet_reader_options_builder& prepend_source_index_column(bool val) - { - options.enable_prepend_source_index_column(val); - return *this; - } - - /** - * @brief Sets whether to prepend a file-local row index column to the output. - * - * @param val Boolean indicating whether to prepend a row index column - * @return this for chaining - */ - parquet_reader_options_builder& prepend_row_index_column(bool val) - { - options.enable_prepend_row_index_column(val); - return *this; - } + parquet_reader_options_builder& prepend_source_index_column(bool val) + { + options.enable_prepend_source_index_column(val); + return *this; + } + + /** + * @brief Sets whether to prepend a file-local row index column to the output. + * + * @param val Boolean indicating whether to prepend a row index column + * @return this for chaining + */ + parquet_reader_options_builder& prepend_row_index_column(bool val) + { + options.enable_prepend_row_index_column(val); + return *this; + } /** * @brief Sets to enable/disable trying to output DICTIONARY32 columns. diff --git a/cpp/src/dictionary/detail/concatenate.cu b/cpp/src/dictionary/detail/concatenate.cu index 04bfe521eb47..9b2b275aa9c9 100644 --- a/cpp/src/dictionary/detail/concatenate.cu +++ b/cpp/src/dictionary/detail/concatenate.cu @@ -175,6 +175,9 @@ std::unique_ptr concatenate(host_span columns, return keys; }); + // TODO: Overload function to accept multiple vectors to do the concatenate at once, with a 2D + // kernel. The keys concatenate below and the indices concatenate further down are two separate + // launches over the same set of input columns and could be fused into a single batched call. // first, concatenate all the keys auto all_keys = cudf::detail::concatenate(keys_views, stream, cudf::get_current_device_resource_ref()); diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 789ceb30f11e..4e06dfa3ea4d 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -94,20 +94,20 @@ std::unique_ptr make_low_cardinality_lists_of_strings() void write_parquet(cudf::table_view const& input, std::string const& filepath) { + // Produce row groups consisting of `row_group_size` rows, with a single (non-chunked) write. Row + // groups are built from whole page fragments, so `max_page_fragment_size` must also be lowered to + // `row_group_size` + // -- otherwise the default 5000-row fragment would force row groups to snap to multiples of 5000 + // instead of the requested size. auto const options = - cudf::io::chunked_parquet_writer_options::builder(cudf::io::sink_info{filepath}) + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, input) .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) .compression(cudf::io::compression_type::NONE) .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .row_group_size_rows(row_group_size) + .max_page_fragment_size(row_group_size) .build(); - - cudf::io::chunked_parquet_writer writer(options); - for (auto offset = 0; offset < input.num_rows(); offset += row_group_size) { - auto const length = std::min(row_group_size, input.num_rows() - offset); - auto const chunk = cudf::slice(input, {offset, offset + length}); - writer.write(chunk.front()); - } - writer.close(); + cudf::io::write_parquet(options); } cudf::io::table_with_metadata read_parquet_as_dict(std::string const& filepath) From c8916e2d99a7aee4c376dd8c6cf189c168866204 Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 20 Jul 2026 16:28:13 -0700 Subject: [PATCH 27/42] test signing From 62251181aee349d4f3c482d40607013d977d4cad Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 20 Jul 2026 16:52:27 -0700 Subject: [PATCH 28/42] Minor fixes --- cpp/benchmarks/io/parquet/parquet_reader_dict.cpp | 1 + cpp/include/cudf/io/parquet.hpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 3dade95d9a23..0a6ef1becbbb 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 0e9452f3a035..6dfb74a68518 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -965,7 +965,7 @@ class parquet_reader_options_builder { */ parquet_reader_options_builder& output_dict_columns(bool val) { - options._output_dict_columns = val; + options.enable_output_dict_columns(val); return *this; } From e9db447a829fb56bcf828f3490b31b0c1337199d Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 20 Jul 2026 17:10:40 -0700 Subject: [PATCH 29/42] Trimmed dead code --- cpp/src/io/parquet/decode_fixed.cu | 22 ++++-------------- cpp/src/io/parquet/parquet_gpu.hpp | 5 ++-- cpp/src/io/parquet/reader_impl.cpp | 23 ++++--------------- .../io/parquet/reader_impl_dict_transcode.cu | 2 -- 4 files changed, 13 insertions(+), 39 deletions(-) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 93d6a77c96ef..ecd23a5a0b46 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -981,9 +981,7 @@ CUDF_HOST_DEVICE constexpr bool has_dict() (kernel_mask_t == decode_kernel_mask::STRING_DICT) || (kernel_mask_t == decode_kernel_mask::STRING_DICT_NESTED) || (kernel_mask_t == decode_kernel_mask::STRING_DICT_LIST) || - (kernel_mask_t == decode_kernel_mask::DICT_INT32) || - (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED) || - (kernel_mask_t == decode_kernel_mask::DICT_INT32_LIST); + (kernel_mask_t == decode_kernel_mask::DICT_INT32); } /** @@ -994,14 +992,12 @@ CUDF_HOST_DEVICE constexpr bool has_dict() * INT32 indices child of a DICTIONARY32 column rather than fully materialized values. * * @tparam kernel_mask_t The decode kernel mask to test - * @return True for the DICT_INT32, DICT_INT32_NESTED and DICT_INT32_LIST masks + * @return True for the DICT_INT32 mask */ template CUDF_HOST_DEVICE constexpr bool is_dict_int32_output() { - return (kernel_mask_t == decode_kernel_mask::DICT_INT32) || - (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED) || - (kernel_mask_t == decode_kernel_mask::DICT_INT32_LIST); + return (kernel_mask_t == decode_kernel_mask::DICT_INT32); } template @@ -1027,8 +1023,7 @@ CUDF_HOST_DEVICE constexpr bool has_nesting() (kernel_mask_t == decode_kernel_mask::BYTE_STREAM_SPLIT_FIXED_WIDTH_NESTED) || (kernel_mask_t == decode_kernel_mask::STRING_NESTED) || (kernel_mask_t == decode_kernel_mask::STRING_DICT_NESTED) || - (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_NESTED) || - (kernel_mask_t == decode_kernel_mask::DICT_INT32_NESTED); + (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_NESTED); } /** @@ -1046,8 +1041,7 @@ CUDF_HOST_DEVICE constexpr bool has_lists() (kernel_mask_t == decode_kernel_mask::BYTE_STREAM_SPLIT_FIXED_WIDTH_LIST) || (kernel_mask_t == decode_kernel_mask::STRING_LIST) || (kernel_mask_t == decode_kernel_mask::STRING_DICT_LIST) || - (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_LIST) || - (kernel_mask_t == decode_kernel_mask::DICT_INT32_LIST); + (kernel_mask_t == decode_kernel_mask::STRING_STREAM_SPLIT_LIST); } template @@ -1478,12 +1472,6 @@ void decode_page_data(cudf::detail::hostdevice_span pages, case decode_kernel_mask::DICT_INT32: launch_kernel(int_tag_t<128>{}, kernel_tag_t{}); break; - case decode_kernel_mask::DICT_INT32_NESTED: - launch_kernel(int_tag_t<128>{}, kernel_tag_t{}); - break; - case decode_kernel_mask::DICT_INT32_LIST: - launch_kernel(int_tag_t<128>{}, kernel_tag_t{}); - break; default: CUDF_EXPECTS(false, "Kernel type not handled by this function"); break; } } diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 37b9ec12d7f5..2b4e3a715b02 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -225,8 +225,9 @@ enum class decode_kernel_mask { (1 << 24), // Run decode kernel for nested BYTE_STREAM_SPLIT string data STRING_STREAM_SPLIT_LIST = (1 << 25), // Run decode kernel for list BYTE_STREAM_SPLIT string data DICT_INT32 = (1 << 26), // Run decode kernel for dict string → INT32 indices - DICT_INT32_NESTED = (1 << 27), // Run decode kernel for nested dict string → INT32 indices - DICT_INT32_LIST = (1 << 28), // Run decode kernel for list dict string → INT32 indices + // TODO: add DICT_INT32_NESTED (1 << 27) and DICT_INT32_LIST (1 << 28) to extend the Parquet-dict + // → DICTIONARY32 transcode fast path to nested and list string columns. Only flat string columns + // are supported today (see compute_dict_transcode_eligibility / prepare_dict_transcode). }; constexpr uint32_t STRINGS_MASK_NON_DELTA = BitOr(decode_kernel_mask::STRING, diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 5b8c691ec19e..5eff64ae90dd 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -265,20 +265,13 @@ void reader_impl::decode_page_data(read_mode mode, size_t skip_rows, size_t num_ } // launch dict-index-as-int32 decoder for flat columns + // TODO: extend the Parquet-dict → DICTIONARY32 transcode to nested and list string columns by + // adding DICT_INT32_NESTED / DICT_INT32_LIST launches here. Only flat string columns are + // transcoded today; nested/list columns fall back to the normal decode path. if (BitAnd(kernel_mask, decode_kernel_mask::DICT_INT32) != 0) { decode_data(decode_kernel_mask::DICT_INT32); } - // launch dict-index-as-int32 decoder for nested columns - if (BitAnd(kernel_mask, decode_kernel_mask::DICT_INT32_NESTED) != 0) { - decode_data(decode_kernel_mask::DICT_INT32_NESTED); - } - - // launch dict-index-as-int32 decoder for list columns - if (BitAnd(kernel_mask, decode_kernel_mask::DICT_INT32_LIST) != 0) { - decode_data(decode_kernel_mask::DICT_INT32_LIST); - } - // launch delta byte array decoder if (BitAnd(kernel_mask, decode_kernel_mask::DELTA_BYTE_ARRAY) != 0) { decode_delta_byte_array(subpass.pages, @@ -690,7 +683,6 @@ void reader_impl::preprocess_chunk_strings(read_mode mode, row_range const& read table_with_metadata reader_impl::read_chunk_internal(read_mode mode) { - // TODO: Having local views instead of offsets for the segments/ CUDF_FUNC_RANGE(); // If `_output_metadata` has been constructed, just copy it over. @@ -965,14 +957,9 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, apply_decimal_width_cast(out_columns); - // When the user requested DICTIONARY32 output for flat string columns, the direct transcode - // fast path in `prepare_dict_transcode`/`assemble_dict_transcoded_columns` has already - // assembled DICTIONARY32 columns for all *eligible* flat STRING columns (i.e. those whose - // chunks were fully dictionary-encoded). For columns that were *not* eligible (e.g. chunks - // with mixed or non-dictionary encodings, nested schemas, or columns added as empty columns - // above), fall back to a post-hoc `dictionary::detail::encode` so the user still gets a - // DICTIONARY32 column from every flat string column in the output table. if (_options.output_dict_columns) { + // For columns that were not eligible for the direct transcode fast path, fall back to a + // post-hoc `dictionary::detail::encode`. for (auto& col : out_columns) { if (col and col->type().id() == type_id::STRING) { col = diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index d834a67c0ede..5705caf699cb 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -196,8 +196,6 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) // path here too and fall back to the post-hoc encode. if (uses_custom_row_bounds(mode)) { return false; } - if (_pass_itm_data == nullptr or _pass_itm_data->subpass == nullptr) { return false; } - auto& pass = *_pass_itm_data; auto& subpass = *pass.subpass; From 43be5c28ace32d1b16442156e479dfa2f45f784e Mon Sep 17 00:00:00 2001 From: ykiran Date: Wed, 22 Jul 2026 15:06:01 -0700 Subject: [PATCH 30/42] Cleanup --- .../io/parquet/parquet_reader_dict.cpp | 184 ++++++++++-------- cpp/include/cudf/io/parquet.hpp | 8 +- cpp/src/io/parquet/decode_fixed.cu | 3 - cpp/src/io/parquet/parquet_gpu.hpp | 4 +- 4 files changed, 107 insertions(+), 92 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 0a6ef1becbbb..9d663cabe064 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -29,45 +30,45 @@ #include // Benchmark for the parquet-dictionary -> cudf DICTIONARY32 transcode fast path enabled by -// `parquet_reader_options::output_dict_columns`. A single fully dictionary-encoded string column is +// `parquet_reader_options::output_dict_columns`. A flat fully dictionary-encoded string column is // read under three modes, selected by the `mode` axis, so the transcode path can be judged against // both a lower and an upper reference: // -// - "plain_string": reader default; the column materializes as STRING. This is the cheapest -// possible read (no dictionary built) and serves as the lower-bound reference -// -- it does strictly less work and produces a different (STRING) output. -// - "decode_encode": read as STRING, then `cudf::dictionary::encode` to DICTIONARY32. This is the -// pre-existing way to obtain a dictionary column and is the fair -// apples-to-apples baseline the transcode fast path aims to beat. -// - "transcode": `output_dict_columns=true`; the reader keeps the dictionary representation -// and -// emits DICTIONARY32 directly, skipping string materialization. +// - "materialize_string": reader default; the column materializes as STRING. The cheapest possible +// read (no dictionary built); serves as the lower-bound reference. +// - "materialize_string_and_encode_dict": materialize as STRING, then `cudf::dictionary::encode` +// to DICTIONARY32. The pre-existing way to obtain a dictionary column and the +// fair apples-to-apples baseline the transcode fast path aims to beat. +// - "direct_dict_transcode": `output_dict_columns=true`; the reader keeps the dictionary +// representation and emits DICTIONARY32 directly, skipping string materialization. // -// Both "decode_encode" and "transcode" produce DICTIONARY32 output, so their times and peak memory -// are directly comparable; "plain_string" shows the floor cost of just decoding. A relative -// comparison table (decode_encode = 100%%) is printed at program exit (see comparison_collector). +// Both "materialize_string_and_encode_dict" and "direct_dict_transcode" produce DICTIONARY32 output, +// so their times and peak memory are directly comparable; "materialize_string" shows the floor cost +// of just decoding. A relative comparison table (materialize_string_and_encode_dict = 100%%) is +// printed at program exit (see comparison_collector). // -// The sweep varies four axes: cardinality, total table size, rows per row group, and rows per data -// page. A single column (num_cols == 1) is used so a row group can hold as many distinct values as -// possible: the writer picks the dictionary index bit width per row group from the distinct values -// it contains, capped at MAX_DICT_BITS (24). Cardinality therefore ranges up to 2^24, the point at -// which 24-bit indices are required. At high distinct-per-row-group counts the writer may abandon -// dictionary encoding (indices exceed 24 bits, or plain encoding is smaller); when that leaves the -// column ineligible for transcode, that state is skipped rather than measured. The largest -// single-column configurations exceed 2^31 characters and rely on cuDF's default (enabled) large -// strings support to switch to 64-bit offsets automatically. +// The sweep varies cardinality, rows per row group, and rows per data page at a fixed table size +// (kept modest so the sweep stays light for local/CI runs). A single column (num_cols == 1) is used +// so a row group can hold as many distinct values as possible: the writer picks the dictionary index +// bit width per row group from the distinct values it contains, capped at MAX_DICT_BITS (24). +// Cardinality therefore ranges up to 2^24, the point at which 24-bit indices are required. At high +// distinct-per-row-group counts the writer may abandon dictionary encoding (indices exceed 24 bits, +// or plain encoding is smaller); when that leaves the column ineligible for transcode, that state is +// skipped rather than measured. namespace { constexpr cudf::size_type num_cols = 1; -enum class bench_mode { plain_string, decode_encode, transcode }; +enum class bench_mode { materialize_string, materialize_string_and_encode_dict, direct_dict_transcode }; [[nodiscard]] bench_mode parse_mode(std::string const& mode) { - if (mode == "plain_string") { return bench_mode::plain_string; } - if (mode == "decode_encode") { return bench_mode::decode_encode; } - if (mode == "transcode") { return bench_mode::transcode; } + if (mode == "materialize_string") { return bench_mode::materialize_string; } + if (mode == "materialize_string_and_encode_dict") { + return bench_mode::materialize_string_and_encode_dict; + } + if (mode == "direct_dict_transcode") { return bench_mode::direct_dict_transcode; } CUDF_FAIL("Unknown benchmark mode: " + mode); } @@ -79,21 +80,16 @@ enum class bench_mode { plain_string, decode_encode, transcode }; { auto const distinct = std::min(cardinality, row_group_size_rows); if (distinct <= 1) { return 1; } - int bits = 0; - auto max_index = distinct - 1; - while (max_index > 0) { - ++bits; - max_index >>= 1; - } - return bits; + return static_cast(std::bit_width(static_cast(distinct - 1))); } // nvbench invokes the benchmark once per axis combination, prints its own results table, and omits // skipped states from it; there is also no cross-state hook, so a single invocation cannot group // the three modes of a configuration together. This collector accumulates each run's CPU/GPU mean -// time (and any transcode skip reason), keyed by every setting except `mode`, and prints one row -// per configuration from its destructor -- i.e. at program exit, after nvbench's own output -- with -// all three modes in fixed order (plain_string, decode_encode, transcode) so each configuration is +// time (and any direct_dict_transcode skip reason), keyed by every setting except `mode`, and prints +// one row per configuration from its destructor -- i.e. at program exit, after nvbench's own output +// -- with all three modes in fixed order (materialize_string, materialize_string_and_encode_dict, +// direct_dict_transcode) so each configuration is // grouped and ordered regardless of nvbench's state ordering or its omission of skipped states. struct run_settings { std::int64_t cardinality; @@ -125,27 +121,29 @@ class comparison_collector { void record(run_settings const& key, bench_mode mode, double cpu_ms, double gpu_ms) { auto& r = _rows[key]; - auto& slot = (mode == bench_mode::decode_encode) - ? r.decode_encode - : ((mode == bench_mode::transcode) ? r.transcode : r.plain_string); - slot = mode_timing{cpu_ms, gpu_ms, true}; + auto& slot = (mode == bench_mode::materialize_string_and_encode_dict) + ? r.materialize_string_and_encode_dict + : ((mode == bench_mode::direct_dict_transcode) ? r.direct_dict_transcode + : r.materialize_string); + slot = mode_timing{cpu_ms, gpu_ms, true}; } - // Record that transcode was skipped for a configuration, with a short reason shown in the table. - // `transcode` is the only mode this benchmark ever skips. + // Record that direct_dict_transcode was skipped for a configuration, with a short reason shown in + // the table. `direct_dict_transcode` is the only mode this benchmark ever skips. void record_skip(run_settings const& key, std::string reason) { - _rows[key].transcode_note = std::move(reason); + _rows[key].direct_dict_transcode_note = std::move(reason); } ~comparison_collector() { print(); } private: struct row { - mode_timing plain_string; - mode_timing decode_encode; - mode_timing transcode; - std::string transcode_note; // reason shown when `transcode` was skipped as ineligible + mode_timing materialize_string; + mode_timing materialize_string_and_encode_dict; + mode_timing direct_dict_transcode; + // reason shown when `direct_dict_transcode` was skipped as ineligible + std::string direct_dict_transcode_note; }; void print() const @@ -154,12 +152,14 @@ class comparison_collector { std::printf( "\n# Per-configuration mode comparison " - "(order: plain_string, decode_encode, transcode)\n\n"); + "(order: materialize_string, materialize_string_and_encode_dict, " + "direct_dict_transcode)\n\n"); std::printf( "| cardinality | ~dict_bits | data_size (MiB) | row_group_size_rows | max_page_size_rows | " - "plain_string CPU (ms) | decode_encode CPU (ms) | transcode CPU (ms) | " - "plain_string GPU (ms) | decode_encode GPU (ms) | transcode GPU (ms) | " - "transcode CPU speedup %% | transcode GPU speedup %% |\n"); + "materialize_string CPU (ms) | materialize_string_and_encode_dict CPU (ms) | " + "direct_dict_transcode CPU (ms) | materialize_string GPU (ms) | " + "materialize_string_and_encode_dict GPU (ms) | direct_dict_transcode GPU (ms) | " + "direct_dict_transcode CPU speedup %% | direct_dict_transcode GPU speedup %% |\n"); std::printf("|---|---|---|---|---|---|---|---|---|---|---|---|---|\n"); auto const num = [](double v) { @@ -167,16 +167,18 @@ class comparison_collector { std::snprintf(buf.data(), buf.size(), "%.3f", v); return std::string{buf.data()}; }; - // Timing cell: the value if the mode ran, otherwise the skip reason (transcode only) or "-". + // Timing cell: the value if the mode ran, else the skip reason (direct_dict_transcode only) or "-". auto const cell = [&](mode_timing const& t, double mode_timing::* field, std::string const& note) { if (t.present) { return num(t.*field); } return note.empty() ? std::string{"-"} : note; }; - // Speedup of transcode over the decode_encode baseline, as a signed percentage of baseline time - // saved: 100 * (decode_encode - transcode) / decode_encode. Positive = transcode is faster, - // negative = slower. "-" when either mode is missing. + // Speedup of direct_dict_transcode over the materialize_string_and_encode_dict baseline, as a + // signed percentage of baseline time saved: + // 100 * (materialize_string_and_encode_dict - direct_dict_transcode) / + // materialize_string_and_encode_dict. + // Positive = direct_dict_transcode faster, negative = slower. "-" when either mode is missing. auto const speedup = [](mode_timing const& base, mode_timing const& cand, double mode_timing::* field) { if (not(base.present and cand.present)) { return std::string{"-"}; } @@ -193,14 +195,21 @@ class comparison_collector { static_cast(key.data_size >> 20), static_cast(key.row_group_size_rows), static_cast(key.max_page_size_rows), - cell(r.plain_string, &mode_timing::cpu_ms, std::string{}).c_str(), - cell(r.decode_encode, &mode_timing::cpu_ms, std::string{}).c_str(), - cell(r.transcode, &mode_timing::cpu_ms, r.transcode_note).c_str(), - cell(r.plain_string, &mode_timing::gpu_ms, std::string{}).c_str(), - cell(r.decode_encode, &mode_timing::gpu_ms, std::string{}).c_str(), - cell(r.transcode, &mode_timing::gpu_ms, r.transcode_note).c_str(), - speedup(r.decode_encode, r.transcode, &mode_timing::cpu_ms).c_str(), - speedup(r.decode_encode, r.transcode, &mode_timing::gpu_ms).c_str()); + cell(r.materialize_string, &mode_timing::cpu_ms, std::string{}).c_str(), + cell(r.materialize_string_and_encode_dict, &mode_timing::cpu_ms, std::string{}).c_str(), + cell(r.direct_dict_transcode, &mode_timing::cpu_ms, r.direct_dict_transcode_note) + .c_str(), + cell(r.materialize_string, &mode_timing::gpu_ms, std::string{}).c_str(), + cell(r.materialize_string_and_encode_dict, &mode_timing::gpu_ms, std::string{}) + .c_str(), + cell(r.direct_dict_transcode, &mode_timing::gpu_ms, r.direct_dict_transcode_note) + .c_str(), + speedup( + r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::cpu_ms) + .c_str(), + speedup( + r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::gpu_ms) + .c_str()); } std::printf("\n"); } @@ -213,7 +222,7 @@ comparison_collector g_comparison_collector; // The transcode fast path requires every data page of an eligible column to be dictionary-encoded. // Forcing `dictionary_policy::ALWAYS` maximizes the chance of full dictionary encoding; the writer // can still fall back to plain when indices exceed MAX_DICT_BITS or plain is smaller, in which case -// the transcode state is skipped by the caller. +// the direct_dict_transcode state is skipped by the caller. void write_dict_encoded_parquet(cudf::table_view const& view, cuio_source_sink_pair& source_sink, std::int64_t row_group_size_rows, @@ -266,14 +275,14 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) cudf::io::parquet_reader_options read_opts = cudf::io::parquet_reader_options::builder(source_sink.make_source_info()) - .output_dict_columns(mode == bench_mode::transcode); + .output_dict_columns(mode == bench_mode::direct_dict_transcode); - // Perform the full work for the selected mode: read, and for `decode_encode` additionally encode + // Perform the full work for the selected mode: read, and for `materialize_string_and_encode_dict` additionally encode // each STRING column to DICTIONARY32. Returns the resulting table so it can be reused for both // the outside-the-timed-region verification and the timed measurement. auto const run_mode = [&]() -> std::unique_ptr { auto result = cudf::io::read_parquet(read_opts); - if (mode == bench_mode::decode_encode) { + if (mode == bench_mode::materialize_string_and_encode_dict) { std::vector> encoded; encoded.reserve(result.tbl->num_columns()); for (auto const& col : result.tbl->view()) { @@ -285,7 +294,8 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) }; // Verification (outside the timed region, run for every mode so warm-up is symmetric). For - // `transcode`, the writer may have fallen back to plain encoding at high cardinality / large row + // `direct_dict_transcode`, the writer may have fallen back to plain encoding at high cardinality / + // large row // groups, leaving the column ineligible for the fast path; in that case skip the state rather // than silently measuring the plain path or aborting the whole sweep. { @@ -301,22 +311,23 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) }); }; auto const actual_type_id = static_cast(probe_view.column(0).type().id()); - if (mode == bench_mode::plain_string) { + if (mode == bench_mode::materialize_string) { if (not all_of_type(cudf::type_id::STRING)) { - state.skip("plain_string produced unexpected type_id=" + std::to_string(actual_type_id) + + state.skip("materialize_string produced unexpected type_id=" + std::to_string(actual_type_id) + " (expected STRING=" + std::to_string(static_cast(cudf::type_id::STRING)) + ")"); return; } - } else if (mode == bench_mode::decode_encode) { + } else if (mode == bench_mode::materialize_string_and_encode_dict) { if (not all_of_type(cudf::type_id::DICTIONARY32)) { - state.skip("decode_encode produced unexpected type_id=" + std::to_string(actual_type_id) + + state.skip("materialize_string_and_encode_dict produced unexpected type_id=" + std::to_string(actual_type_id) + " (expected DICTIONARY32=" + std::to_string(static_cast(cudf::type_id::DICTIONARY32)) + ")"); return; } } else if (not all_of_type(cudf::type_id::DICTIONARY32)) { - // Record the skip so the end-of-program per-configuration table can show why transcode has no + // Record the skip so the end-of-program per-configuration table can show why + // direct_dict_transcode has no // timing for this configuration (nvbench omits skipped states from its own table). g_comparison_collector.record_skip(run_settings{cardinality, static_cast(data_size), @@ -325,7 +336,8 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) avg_string_length}, "skipped: plain fallback"); state.skip( - "transcode did not produce DICTIONARY32: at this cardinality / row-group size the writer " + "direct_dict_transcode did not produce DICTIONARY32: at this cardinality / row-group size " + "the writer " "fell back to plain encoding, making the column ineligible for the fast path"); return; } @@ -352,7 +364,8 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); - // Record this run for the end-of-program transcode-vs-decode_encode comparison table. Times are + // Record this run for the end-of-program direct_dict_transcode-vs-materialize_string_and_encode_dict + // comparison table. Times are // reported by nvbench in seconds; store as milliseconds. g_comparison_collector.record(run_settings{cardinality, static_cast(data_size), @@ -368,17 +381,18 @@ NVBENCH_BENCH(BM_parquet_read_dict_transcode) .set_name("parquet_read_dict_transcode") .add_string_axis("io_type", {"DEVICE_BUFFER"}) .set_min_samples(4) - .add_string_axis("mode", {"plain_string", "decode_encode", "transcode"}) - // Cardinality spans up to 2^24, the point at which per-row-group dictionary indices need the + .add_string_axis( + "mode", {"materialize_string", "materialize_string_and_encode_dict", "direct_dict_transcode"}) + // Cardinality: low, mid, and 2^24 -- the point at which per-row-group dictionary indices need the // maximum 24 bits the writer supports (MAX_DICT_BITS); beyond that the writer abandons dictionary // encoding. Achieved bits = ceil(log2(min(cardinality, rows per row group))). - .add_int64_axis("cardinality", {1 << 10, 1 << 15, 1 << 20, 1 << 24}) - // Total table size (single column). The largest points exceed 2^31 chars and rely on cuDF's - // default large strings support (64-bit offsets). - .add_int64_axis("data_size", {std::int64_t{512} << 20, std::int64_t{2} << 30}) - // Rows per row group: small (many row groups -> stresses per-row-group key concatenation) to very - // large (>= 2^24 so a single row group can hold enough distinct values to reach 24 dict bits). + .add_int64_axis("cardinality", {1 << 10, 1 << 20, 1 << 24}) + // Fixed table size, kept modest so the sweep stays light for local/CI runs (peak memory and + // per-state runtime scale with this). + .add_int64_axis("data_size", {std::int64_t{512} << 20}) + // Rows per row group: small (many row groups -> stresses per-row-group key concatenation), + // default (1M), and very large (>= 2^24 so a single row group can reach the 24-bit dict boundary). .add_int64_axis("row_group_size_rows", {100'000, 1'000'000, 20'000'000}) - // Rows per data page: small to large. - .add_int64_axis("max_page_size_rows", {20'000, 100'000, 1'000'000}) + // Rows per data page: small and large (page size is a second-order factor for this benchmark). + .add_int64_axis("max_page_size_rows", {20'000, 1'000'000}) .add_int64_axis("avg_string_length", {16}); diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 6dfb74a68518..eeba337af443 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -958,9 +958,13 @@ class parquet_reader_options_builder { } /** - * @brief Sets to enable/disable trying to output DICTIONARY32 columns. + * @brief Sets options for enabling/disabling output of DICTIONARY32 columns. + * + * @param val Boolean value whether to output DICTIONARY32 columns + * + * @note When enabled, the output columns will be of type DICTIONARY32. When disabled, the output + * columns will be of type STRING. * - * @param val Boolean value whether to try to output DICTIONARY32 columns * @return this for chaining */ parquet_reader_options_builder& output_dict_columns(bool val) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index ecd23a5a0b46..075325d49827 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -988,9 +988,6 @@ CUDF_HOST_DEVICE constexpr bool has_dict() * @brief Check whether the kernel mask decodes parquet dictionary indices directly to an INT32 * column. * - * These masks back the Parquet-dict → DICTIONARY32 transcode path, where the decoded output is the - * INT32 indices child of a DICTIONARY32 column rather than fully materialized values. - * * @tparam kernel_mask_t The decode kernel mask to test * @return True for the DICT_INT32 mask */ diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 2b4e3a715b02..2870f9c49b0a 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -226,8 +226,8 @@ enum class decode_kernel_mask { STRING_STREAM_SPLIT_LIST = (1 << 25), // Run decode kernel for list BYTE_STREAM_SPLIT string data DICT_INT32 = (1 << 26), // Run decode kernel for dict string → INT32 indices // TODO: add DICT_INT32_NESTED (1 << 27) and DICT_INT32_LIST (1 << 28) to extend the Parquet-dict - // → DICTIONARY32 transcode fast path to nested and list string columns. Only flat string columns - // are supported today (see compute_dict_transcode_eligibility / prepare_dict_transcode). + // → DICTIONARY32 transcode fast path to nested and list string columns. Currently, only flat string columns + // are supported(see compute_dict_transcode_eligibility / prepare_dict_transcode). }; constexpr uint32_t STRINGS_MASK_NON_DELTA = BitOr(decode_kernel_mask::STRING, From c56e6ea94164b9cd9828b27e1ed5e9b6d79c0b2d Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 27 Jul 2026 14:39:46 -0700 Subject: [PATCH 31/42] More MR feedback --- .../io/parquet/parquet_reader_dict.cpp | 106 +++++++++--------- cpp/src/io/parquet/parquet_gpu.hpp | 4 +- cpp/src/io/parquet/reader_impl.cpp | 5 - .../io/parquet/reader_impl_dict_transcode.cu | 106 +++++++++++------- 4 files changed, 121 insertions(+), 100 deletions(-) diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp index 9d663cabe064..c5e32e6dae2d 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp @@ -34,33 +34,39 @@ // read under three modes, selected by the `mode` axis, so the transcode path can be judged against // both a lower and an upper reference: // -// - "materialize_string": reader default; the column materializes as STRING. The cheapest possible +// - "materialize_string": reader default; the column materializes as STRING. The cheapest +// possible // read (no dictionary built); serves as the lower-bound reference. // - "materialize_string_and_encode_dict": materialize as STRING, then `cudf::dictionary::encode` // to DICTIONARY32. The pre-existing way to obtain a dictionary column and the // fair apples-to-apples baseline the transcode fast path aims to beat. // - "direct_dict_transcode": `output_dict_columns=true`; the reader keeps the dictionary -// representation and emits DICTIONARY32 directly, skipping string materialization. +// representation and emits DICTIONARY32 directly, skipping string +// materialization. // -// Both "materialize_string_and_encode_dict" and "direct_dict_transcode" produce DICTIONARY32 output, -// so their times and peak memory are directly comparable; "materialize_string" shows the floor cost -// of just decoding. A relative comparison table (materialize_string_and_encode_dict = 100%%) is -// printed at program exit (see comparison_collector). +// Both "materialize_string_and_encode_dict" and "direct_dict_transcode" produce DICTIONARY32 +// output, so their times and peak memory are directly comparable; "materialize_string" shows the +// floor cost of just decoding. A relative comparison table (materialize_string_and_encode_dict = +// 100%%) is printed at program exit (see comparison_collector). // // The sweep varies cardinality, rows per row group, and rows per data page at a fixed table size // (kept modest so the sweep stays light for local/CI runs). A single column (num_cols == 1) is used -// so a row group can hold as many distinct values as possible: the writer picks the dictionary index -// bit width per row group from the distinct values it contains, capped at MAX_DICT_BITS (24). +// so a row group can hold as many distinct values as possible: the writer picks the dictionary +// index bit width per row group from the distinct values it contains, capped at MAX_DICT_BITS (24). // Cardinality therefore ranges up to 2^24, the point at which 24-bit indices are required. At high // distinct-per-row-group counts the writer may abandon dictionary encoding (indices exceed 24 bits, -// or plain encoding is smaller); when that leaves the column ineligible for transcode, that state is -// skipped rather than measured. +// or plain encoding is smaller); when that leaves the column ineligible for transcode, that state +// is skipped rather than measured. namespace { constexpr cudf::size_type num_cols = 1; -enum class bench_mode { materialize_string, materialize_string_and_encode_dict, direct_dict_transcode }; +enum class bench_mode { + materialize_string, + materialize_string_and_encode_dict, + direct_dict_transcode +}; [[nodiscard]] bench_mode parse_mode(std::string const& mode) { @@ -86,8 +92,9 @@ enum class bench_mode { materialize_string, materialize_string_and_encode_dict, // nvbench invokes the benchmark once per axis combination, prints its own results table, and omits // skipped states from it; there is also no cross-state hook, so a single invocation cannot group // the three modes of a configuration together. This collector accumulates each run's CPU/GPU mean -// time (and any direct_dict_transcode skip reason), keyed by every setting except `mode`, and prints -// one row per configuration from its destructor -- i.e. at program exit, after nvbench's own output +// time (and any direct_dict_transcode skip reason), keyed by every setting except `mode`, and +// prints one row per configuration from its destructor -- i.e. at program exit, after nvbench's own +// output // -- with all three modes in fixed order (materialize_string, materialize_string_and_encode_dict, // direct_dict_transcode) so each configuration is // grouped and ordered regardless of nvbench's state ordering or its omission of skipped states. @@ -125,7 +132,7 @@ class comparison_collector { ? r.materialize_string_and_encode_dict : ((mode == bench_mode::direct_dict_transcode) ? r.direct_dict_transcode : r.materialize_string); - slot = mode_timing{cpu_ms, gpu_ms, true}; + slot = mode_timing{cpu_ms, gpu_ms, true}; } // Record that direct_dict_transcode was skipped for a configuration, with a short reason shown in @@ -167,7 +174,8 @@ class comparison_collector { std::snprintf(buf.data(), buf.size(), "%.3f", v); return std::string{buf.data()}; }; - // Timing cell: the value if the mode ran, else the skip reason (direct_dict_transcode only) or "-". + // Timing cell: the value if the mode ran, else the skip reason (direct_dict_transcode only) or + // "-". auto const cell = [&](mode_timing const& t, double mode_timing::* field, std::string const& note) { if (t.present) { return num(t.*field); } @@ -189,27 +197,23 @@ class comparison_collector { }; for (auto const& [key, r] : _rows) { - std::printf("| %lld | %d | %lld | %lld | %lld | %s | %s | %s | %s | %s | %s | %s | %s |\n", - static_cast(key.cardinality), - approx_dict_bits(key.cardinality, key.row_group_size_rows), - static_cast(key.data_size >> 20), - static_cast(key.row_group_size_rows), - static_cast(key.max_page_size_rows), - cell(r.materialize_string, &mode_timing::cpu_ms, std::string{}).c_str(), - cell(r.materialize_string_and_encode_dict, &mode_timing::cpu_ms, std::string{}).c_str(), - cell(r.direct_dict_transcode, &mode_timing::cpu_ms, r.direct_dict_transcode_note) - .c_str(), - cell(r.materialize_string, &mode_timing::gpu_ms, std::string{}).c_str(), - cell(r.materialize_string_and_encode_dict, &mode_timing::gpu_ms, std::string{}) - .c_str(), - cell(r.direct_dict_transcode, &mode_timing::gpu_ms, r.direct_dict_transcode_note) - .c_str(), - speedup( - r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::cpu_ms) - .c_str(), - speedup( - r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::gpu_ms) - .c_str()); + std::printf( + "| %lld | %d | %lld | %lld | %lld | %s | %s | %s | %s | %s | %s | %s | %s |\n", + static_cast(key.cardinality), + approx_dict_bits(key.cardinality, key.row_group_size_rows), + static_cast(key.data_size >> 20), + static_cast(key.row_group_size_rows), + static_cast(key.max_page_size_rows), + cell(r.materialize_string, &mode_timing::cpu_ms, std::string{}).c_str(), + cell(r.materialize_string_and_encode_dict, &mode_timing::cpu_ms, std::string{}).c_str(), + cell(r.direct_dict_transcode, &mode_timing::cpu_ms, r.direct_dict_transcode_note).c_str(), + cell(r.materialize_string, &mode_timing::gpu_ms, std::string{}).c_str(), + cell(r.materialize_string_and_encode_dict, &mode_timing::gpu_ms, std::string{}).c_str(), + cell(r.direct_dict_transcode, &mode_timing::gpu_ms, r.direct_dict_transcode_note).c_str(), + speedup(r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::cpu_ms) + .c_str(), + speedup(r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::gpu_ms) + .c_str()); } std::printf("\n"); } @@ -277,9 +281,9 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) cudf::io::parquet_reader_options::builder(source_sink.make_source_info()) .output_dict_columns(mode == bench_mode::direct_dict_transcode); - // Perform the full work for the selected mode: read, and for `materialize_string_and_encode_dict` additionally encode - // each STRING column to DICTIONARY32. Returns the resulting table so it can be reused for both - // the outside-the-timed-region verification and the timed measurement. + // Perform the full work for the selected mode: read, and for `materialize_string_and_encode_dict` + // additionally encode each STRING column to DICTIONARY32. Returns the resulting table so it can + // be reused for both the outside-the-timed-region verification and the timed measurement. auto const run_mode = [&]() -> std::unique_ptr { auto result = cudf::io::read_parquet(read_opts); if (mode == bench_mode::materialize_string_and_encode_dict) { @@ -294,10 +298,9 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) }; // Verification (outside the timed region, run for every mode so warm-up is symmetric). For - // `direct_dict_transcode`, the writer may have fallen back to plain encoding at high cardinality / - // large row - // groups, leaving the column ineligible for the fast path; in that case skip the state rather - // than silently measuring the plain path or aborting the whole sweep. + // `direct_dict_transcode`, the writer may have fallen back to plain encoding at high cardinality + // / large row groups, leaving the column ineligible for the fast path; in that case skip the + // state rather than silently measuring the plain path or aborting the whole sweep. { auto const probe = run_mode(); // Bind the table_view to a local: `probe->view()` returns a temporary, so calling it separately @@ -313,15 +316,15 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) auto const actual_type_id = static_cast(probe_view.column(0).type().id()); if (mode == bench_mode::materialize_string) { if (not all_of_type(cudf::type_id::STRING)) { - state.skip("materialize_string produced unexpected type_id=" + std::to_string(actual_type_id) + - " (expected STRING=" + std::to_string(static_cast(cudf::type_id::STRING)) + - ")"); + state.skip( + "materialize_string produced unexpected type_id=" + std::to_string(actual_type_id) + + " (expected STRING=" + std::to_string(static_cast(cudf::type_id::STRING)) + ")"); return; } } else if (mode == bench_mode::materialize_string_and_encode_dict) { if (not all_of_type(cudf::type_id::DICTIONARY32)) { - state.skip("materialize_string_and_encode_dict produced unexpected type_id=" + std::to_string(actual_type_id) + - " (expected DICTIONARY32=" + + state.skip("materialize_string_and_encode_dict produced unexpected type_id=" + + std::to_string(actual_type_id) + " (expected DICTIONARY32=" + std::to_string(static_cast(cudf::type_id::DICTIONARY32)) + ")"); return; } @@ -364,8 +367,8 @@ void BM_parquet_read_dict_transcode(nvbench::state& state) mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); - // Record this run for the end-of-program direct_dict_transcode-vs-materialize_string_and_encode_dict - // comparison table. Times are + // Record this run for the end-of-program + // direct_dict_transcode-vs-materialize_string_and_encode_dict comparison table. Times are // reported by nvbench in seconds; store as milliseconds. g_comparison_collector.record(run_settings{cardinality, static_cast(data_size), @@ -391,7 +394,8 @@ NVBENCH_BENCH(BM_parquet_read_dict_transcode) // per-state runtime scale with this). .add_int64_axis("data_size", {std::int64_t{512} << 20}) // Rows per row group: small (many row groups -> stresses per-row-group key concatenation), - // default (1M), and very large (>= 2^24 so a single row group can reach the 24-bit dict boundary). + // default (1M), and very large (>= 2^24 so a single row group can reach the 24-bit dict + // boundary). .add_int64_axis("row_group_size_rows", {100'000, 1'000'000, 20'000'000}) // Rows per data page: small and large (page size is a second-order factor for this benchmark). .add_int64_axis("max_page_size_rows", {20'000, 1'000'000}) diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 2870f9c49b0a..d5575138afba 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -226,8 +226,8 @@ enum class decode_kernel_mask { STRING_STREAM_SPLIT_LIST = (1 << 25), // Run decode kernel for list BYTE_STREAM_SPLIT string data DICT_INT32 = (1 << 26), // Run decode kernel for dict string → INT32 indices // TODO: add DICT_INT32_NESTED (1 << 27) and DICT_INT32_LIST (1 << 28) to extend the Parquet-dict - // → DICTIONARY32 transcode fast path to nested and list string columns. Currently, only flat string columns - // are supported(see compute_dict_transcode_eligibility / prepare_dict_transcode). + // → DICTIONARY32 transcode fast path to nested and list string columns. Currently, only flat + // string columns are supported(see compute_dict_transcode_eligibility / prepare_dict_transcode). }; constexpr uint32_t STRINGS_MASK_NON_DELTA = BitOr(decode_kernel_mask::STRING, diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 5eff64ae90dd..b28de75a9866 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -746,11 +746,6 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // Allocate memory buffers for the output columns. allocate_columns(mode, read_info.skip_rows, read_info.num_rows); - // Zero-init the INT32 index buffers of dict-transcoded columns before launching decode, so - // that null positions (which the DICT_INT32 kernel does not write to) carry well-defined - // indices in the produced DICTIONARY32 output. - if (dict_transcode_active) { zero_init_dict_transcoded_index_buffers(); } - // Parse data into the output buffers. decode_page_data(mode, read_info.skip_rows, read_info.num_rows); diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 5705caf699cb..9c0f77560948 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -340,13 +341,28 @@ void reader_impl::assemble_dict_transcoded_columns( return size_type{0}; }); - // Grab ownership of the decoded INT32 indices column. Its buffer is shared (aliased) by - // every per-chunk DICTIONARY32 view below via the parent view's offset/size, so it must - // stay alive until the per-column concatenate/assembly completes. auto& indices_col = out_columns[out_idx]; CUDF_EXPECTS(indices_col != nullptr and indices_col->type().id() == type_id::INT32, "Expected INT32 indices column for dict-transcoded flat string column"); auto indices_owner = std::move(indices_col); + + // Single row group fast path: keys are already unique (one dict page), no dedup needed. + // Take ownership of the decoded INT32 indices buffer directly (zero copy), skipping the + // offset/null-count/segment-view work that is only needed for multi-chunk concatenation. + if (chunk_indices.size() == 1) { + auto const& chunk = pass.chunks[chunk_indices[0]]; + out_columns[out_idx] = + cudf::make_dictionary_column(make_keys_column_from_index_pairs( + chunk.str_dict_index, chunk_key_counts[0], _stream, _mr), + std::move(indices_owner), + _stream, + _mr); + return; + } + + // Multi-row-group path: the indices buffer is shared (aliased) by per-chunk DICTIONARY32 + // views below via the parent's offset/size, so it must stay alive until concatenate + // completes. column_view const indices_view{indices_owner->view()}; // Per-chunk boundaries along the row axis: chunk k occupies rows @@ -362,6 +378,21 @@ void reader_impl::assemble_dict_transcoded_columns( CUDF_EXPECTS(chunk_row_offsets.back() == indices_view.size(), "Row counts on pass chunks must sum to the indices column size"); + // Pre-compute null counts for all segments in a single kernel launch. Building the + // column_views below requires a per-segment null count, and calling null_count(begin, end) + // inside the loop would launch one kernel per chunk. Batch them here instead. + std::vector seg_null_counts(chunk_indices.size(), 0); + if (indices_view.nullable()) { + std::vector indices_pairs; + indices_pairs.reserve(chunk_indices.size() * 2); + for (size_t k = 0; k < chunk_indices.size(); ++k) { + indices_pairs.push_back(chunk_row_offsets[k]); + indices_pairs.push_back(chunk_row_offsets[k + 1]); + } + seg_null_counts = + cudf::detail::segmented_null_count(indices_view.null_mask(), indices_pairs, _stream); + } + // Build a DICTIONARY32 *view* for every chunk without copying the decoded indices. Each // view's keys child is this chunk's own STRING keys column (which must be materialized from // the parquet dictionary page), while its indices child aliases the shared, already-decoded @@ -375,45 +406,36 @@ void reader_impl::assemble_dict_transcoded_columns( // `cudf::detail::concatenate` rewrites indices against the unified, deduplicated keys. std::vector> seg_keys_owners(chunk_indices.size()); std::vector dict_segment_views(chunk_indices.size()); - std::transform(cuda::counting_iterator{0}, - cuda::counting_iterator{chunk_indices.size()}, - dict_segment_views.begin(), - [&](size_t k) { - auto const chunk_idx = chunk_indices[k]; - auto const& chunk = pass.chunks[chunk_idx]; - - seg_keys_owners[k] = make_keys_column_from_index_pairs( - chunk.str_dict_index, chunk_key_counts[k], _stream, _mr); - - auto const seg_begin = chunk_row_offsets[k]; - auto const seg_end = chunk_row_offsets[k + 1]; - auto const seg_rows = seg_end - seg_begin; - auto const seg_null_count = - indices_view.null_count(seg_begin, seg_end, _stream); - return column_view{data_type{type_id::DICTIONARY32}, - seg_rows, - nullptr, // dictionary parent holds no data - indices_view.null_mask(), // shared with indices_view - seg_null_count, - seg_begin, // reslices shared indices child + null mask - {indices_view, seg_keys_owners[k]->view()}}; - }); - - // Materialize the final DICTIONARY32 column for this input column. - if (dict_segment_views.size() == 1) { - // Single row group: the parquet dictionary page keys are already unique, so no dedup is - // needed. Take ownership of the decoded INT32 indices buffer directly (zero copy). - out_columns[out_idx] = cudf::make_dictionary_column( - std::move(seg_keys_owners.front()), std::move(indices_owner), _stream, _mr); - } else { - // `cudf::detail::concatenate` deduplicates + sorts keys and recomputes indices. This is - // required today because DICTIONARY32 keys are assumed unique and sorted. When - // https://github.com/rapidsai/cudf/pull/22839 lands and relaxes that constraint, this - // could be replaced with a cheaper path: plain-concatenate the per-chunk keys columns - // (keeping cross-chunk duplicates) and offset-shift each chunk's row-group-local indices - // by the running total of prior chunks' key counts, avoiding the dedup/sort entirely. - out_columns[out_idx] = cudf::detail::concatenate(dict_segment_views, _stream, _mr); - } + std::transform( + cuda::counting_iterator{0}, + cuda::counting_iterator{chunk_indices.size()}, + dict_segment_views.begin(), + [&](size_t k) { + auto const chunk_idx = chunk_indices[k]; + auto const& chunk = pass.chunks[chunk_idx]; + + seg_keys_owners[k] = make_keys_column_from_index_pairs( + chunk.str_dict_index, chunk_key_counts[k], _stream, get_current_device_resource_ref()); + + auto const seg_begin = chunk_row_offsets[k]; + auto const seg_end = chunk_row_offsets[k + 1]; + auto const seg_rows = seg_end - seg_begin; + return column_view{data_type{type_id::DICTIONARY32}, + seg_rows, + nullptr, // dictionary parent holds no data + indices_view.null_mask(), // shared with indices_view + seg_null_counts[k], + seg_begin, // reslices shared indices child + null mask + {indices_view, seg_keys_owners[k]->view()}}; + }); + + // `cudf::detail::concatenate` deduplicates + sorts keys and recomputes indices. This is + // required today because DICTIONARY32 keys are assumed unique and sorted. When + // https://github.com/rapidsai/cudf/pull/22839 lands and relaxes that constraint, this + // could be replaced with a cheaper path: plain-concatenate the per-chunk keys columns + // (keeping cross-chunk duplicates) and offset-shift each chunk's row-group-local indices + // by the running total of prior chunks' key counts, avoiding the dedup/sort entirely. + out_columns[out_idx] = cudf::detail::concatenate(dict_segment_views, _stream, _mr); }); } From 378c97b228d6441880c37c6d86110e0e6d5832cb Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 10:51:45 -0700 Subject: [PATCH 32/42] More MR feedback --- cpp/include/cudf/io/parquet.hpp | 24 ++++++-------- cpp/src/io/parquet/parquet_gpu.hpp | 3 -- cpp/src/io/parquet/reader_impl.cpp | 32 ++++++++++--------- .../io/parquet/reader_impl_dict_transcode.cu | 5 +++ cpp/tests/io/parquet_reader_dict_test.cpp | 2 ++ 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index eeba337af443..6e227d1f62d4 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -110,7 +110,7 @@ class parquet_reader_options { type_id _decimal_width{type_id::EMPTY}; // Whether to use JIT compilation for filtering bool _use_jit_filter = false; - // For flat string columns, output DICT32 encoded string columns + // Whether to output flat string columns as DICT32 encoded columns bool _output_dict_columns = false; // Whether column name matching is case sensitive. In case of multiple // case-insensitive matches, the first matched column is selected @@ -343,21 +343,17 @@ class parquet_reader_options { } /** - * @brief Returns whether the reader should try to output DICTIONARY32 columns. + * @brief Returns whether the reader returns flat string columns as DICTIONARY32 encoded columns * - * When true, the reader outputs DICTIONARY32 columns (instead of fully decoded STRING columns) - * for fully dict-encoded string columns . A DICTIONARY32 column consists of an INT32 indices + * When true, the reader outputs STRING columns as DICTIONARY32 encoded columns. A DICTIONARY32 column consists of an INT32 indices * child and a STRING keys child. * - * AST filters do not support dictionary columns yet, so when a filter is set this option is - * silently disabled and the columns are returned as STRING for the filter to operate on. + * When AST/JIT filters are set, the direct transcode fast path is disabled. + * String columns are materialized, then operated on by the filter. The filtered results are then encoded as DICTIONARY32 columns. * - * @return `true` if the reader should output DICTIONARY32 columns for flat string columns + * @return `true` if the reader returns flat string columns as DICTIONARY32 encoded columns */ - [[nodiscard]] bool is_enabled_output_dict_columns() const - { - return _output_dict_columns and not _filter.has_value(); - } + [[nodiscard]] bool is_enabled_output_dict_columns() const { return _output_dict_columns; } /** * @brief Set a new source location @@ -654,7 +650,7 @@ class parquet_reader_options { void enable_prepend_row_index_column(bool val) { _prepend_row_index_column = val; } /** - * @brief Sets to enable/disable trying to output DICTIONARY32 columns. + * @brief Sets to enable/disable trying to output DICTIONARY32 columns for flat string columns. * * @param val Boolean indicating whether to output DICTIONARY32 columns for flat string columns */ @@ -958,9 +954,9 @@ class parquet_reader_options_builder { } /** - * @brief Sets options for enabling/disabling output of DICTIONARY32 columns. + * @brief Sets options for enabling/disabling output of DICTIONARY32 columns for flat string columns. * - * @param val Boolean value whether to output DICTIONARY32 columns + * @param val Boolean value whether to output flat string columns as DICTIONARY32 encoded columns * * @note When enabled, the output columns will be of type DICTIONARY32. When disabled, the output * columns will be of type STRING. diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index d5575138afba..d299cfe7615b 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -225,9 +225,6 @@ enum class decode_kernel_mask { (1 << 24), // Run decode kernel for nested BYTE_STREAM_SPLIT string data STRING_STREAM_SPLIT_LIST = (1 << 25), // Run decode kernel for list BYTE_STREAM_SPLIT string data DICT_INT32 = (1 << 26), // Run decode kernel for dict string → INT32 indices - // TODO: add DICT_INT32_NESTED (1 << 27) and DICT_INT32_LIST (1 << 28) to extend the Parquet-dict - // → DICTIONARY32 transcode fast path to nested and list string columns. Currently, only flat - // string columns are supported(see compute_dict_transcode_eligibility / prepare_dict_transcode). }; constexpr uint32_t STRINGS_MASK_NON_DELTA = BitOr(decode_kernel_mask::STRING, diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index b28de75a9866..52d4c45228a6 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -265,9 +265,6 @@ void reader_impl::decode_page_data(read_mode mode, size_t skip_rows, size_t num_ } // launch dict-index-as-int32 decoder for flat columns - // TODO: extend the Parquet-dict → DICTIONARY32 transcode to nested and list string columns by - // adding DICT_INT32_NESTED / DICT_INT32_LIST launches here. Only flat string columns are - // transcoded today; nested/list columns fall back to the normal decode path. if (BitAnd(kernel_mask, decode_kernel_mask::DICT_INT32) != 0) { decode_data(decode_kernel_mask::DICT_INT32); } @@ -541,10 +538,7 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, _input_pass_read_limit{pass_read_limit} { // The direct parquet-dict → DICTIONARY32 transcode fast path only supports single-pass, - // non-chunked reads. Splitting rowgroups across passes/subpasses would require aligning - // dictionary keys across passes, which are not supported yet. In that scenario, we silently - // skip the fast path in `prepare_dict_transcode` and still produce DICTIONARY32 output - // via the post-hoc `dictionary::detail::encode` fallback in `finalize_output`. + // non-chunked reads. if (_options.output_dict_columns and (chunk_read_limit != 0 or pass_read_limit != 0)) { CUDF_LOG_WARN( "output_dict_columns: the direct parquet-dict transcode fast path is disabled for chunked / " @@ -952,16 +946,23 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, apply_decimal_width_cast(out_columns); - if (_options.output_dict_columns) { - // For columns that were not eligible for the direct transcode fast path, fall back to a - // post-hoc `dictionary::detail::encode`. - for (auto& col : out_columns) { + // Encode any remaining flat STRING columns to DICTIONARY32 via a post-hoc + // `dictionary::detail::encode`: columns not produced by the direct transcode fast path, or all of + // them when the fast path was disabled (e.g. under a filter). This is applied to the FINAL output + // table below -- after any filter is evaluated -- so the filter still operates on STRING columns + // and the dictionary is built over only the surviving rows. + auto const encode_output_dict_columns = + [&](std::unique_ptr tbl) -> std::unique_ptr
{ + if (not _options.output_dict_columns) { return tbl; } + auto columns = tbl->release(); + for (auto& col : columns) { if (col and col->type().id() == type_id::STRING) { col = cudf::dictionary::detail::encode(col->view(), data_type{type_id::INT32}, _stream, _mr); } } - } + return std::make_unique
(std::move(columns)); + }; if (!_output_metadata) { populate_metadata(out_metadata); @@ -1030,15 +1031,16 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, // Exclude columns present in filter only in output auto output_table = cudf::detail::apply_mask( only_output, *predicate, cudf::detail::mask_type::RETENTION, _stream, _mr); - return {std::move(output_table), std::move(out_metadata)}; + return {encode_output_dict_columns(std::move(output_table)), std::move(out_metadata)}; } else { auto output_table = cudf::filter( read_table->view(), final_filter_expr.value().get(), only_output, _stream, _mr); - return {std::move(output_table), std::move(out_metadata)}; + return {encode_output_dict_columns(std::move(output_table)), std::move(out_metadata)}; } } - return {std::make_unique
(std::move(out_columns)), std::move(out_metadata)}; + return {encode_output_dict_columns(std::make_unique
(std::move(out_columns))), + std::move(out_metadata)}; } table_with_metadata reader_impl::read() diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 9c0f77560948..c1bea863d664 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -197,6 +197,11 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) // path here too and fall back to the post-hoc encode. if (uses_custom_row_bounds(mode)) { return false; } + // AST/JIT filters evaluate predicates on materialized STRING columns, so the direct transcode + // fast path cannot run under a filter. Skip it and let `finalize_output` encode the filtered + // STRING result to DICTIONARY32 via the post-hoc `dictionary::detail::encode` fallback. + if (_expr_conv.get_converted_expr().has_value()) { return false; } + auto& pass = *_pass_itm_data; auto& subpass = *pass.subpass; diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 4e06dfa3ea4d..996a9ffdce06 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -18,6 +18,8 @@ #include #include +#include + #include #include #include From e3fd96bc45574e03d030fb41bf57ae60742e9544 Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 15:28:11 -0700 Subject: [PATCH 33/42] Batched memset for nullables --- cpp/src/io/parquet/reader_impl_dict_transcode.cu | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index c1bea863d664..29be54bc57eb 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -270,12 +270,25 @@ void reader_impl::zero_init_dict_transcoded_index_buffers() // null positions. Zero them here so null rows carry a well-defined (valid) index into the // dictionary keys -- a requirement for `cudf::dictionary::detail::concatenate` to correctly // remap indices below. + // + // Only nullable columns need this: the kernel decodes non-nullable columns (max definition level + // 0) in DIRECT mode, which writes every output position, so their buffers are fully initialized by + // decode. This mirrors `is_nullable()` in the decode kernel (max_level[DEFINITION] > 0). + if (_pass_itm_data == nullptr) { return; } + auto const& pass = *_pass_itm_data; + std::vector col_nullable(_input_columns.size(), false); + for (auto const& chunk : pass.chunks) { + if (chunk.max_level[level_type::DEFINITION] > 0) { col_nullable[chunk.src_col_index] = true; } + } + std::vector> index_bufs; index_bufs.reserve(_input_columns.size()); std::for_each(cuda::counting_iterator{0}, cuda::counting_iterator{_input_columns.size()}, [&](size_t i) { if (not _dict_transcode_eligible[i]) { return; } + // Non-nullable columns decode in DIRECT mode and write every slot -> no zeroing. + if (not col_nullable[i]) { return; } auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; if (out_buf.type.id() != type_id::INT32) { return; } if (out_buf.data() == nullptr or out_buf.size == 0) { return; } From 00320a477a7a3d6be4a3a8f17b2c517dd9fd5994 Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 16:43:50 -0700 Subject: [PATCH 34/42] Cleanup --- cpp/src/io/parquet/reader_impl.hpp | 31 ++------- .../io/parquet/reader_impl_dict_transcode.cu | 65 +++++-------------- 2 files changed, 24 insertions(+), 72 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 524d0ed38164..14a3d5a1ca23 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -190,26 +190,13 @@ class reader_impl { /** * @brief Detect per-column eligibility for direct Parquet-dict → DICTIONARY32 transcode, and - * apply the required host-side mutations to `_output_buffers` and `subpass.pages` so that the - * subsequent allocate/decode path produces INT32 indices for eligible columns. + * apply the required host-side mutations to `_output_buffers` and `subpass.pages`. * - * Must be called after `prepare_data()` (so that `pass.chunks`, `pass.pages` and - * `subpass.pages` are populated on the host) and before `preprocess_chunk_strings()` / - * `allocate_columns()` / `decode_page_data()`. - * - * Populates `_dict_transcode_eligible` with a bool per input column indicating whether the + * Must be called after `prepare_data()`. Populates `_dict_transcode_eligible` with a bool per input column indicating whether the * column will be assembled as a DICTIONARY32 output later in `assemble_dict_transcoded_columns`. * - * The fast path is also skipped when custom row bounds are in effect (see - * `uses_custom_row_bounds`): `assemble_dict_transcoded_columns` derives per-chunk row segments - * from the full, unadjusted `ColumnChunkDesc::num_rows`, which would not match the decoded - * indices column's size once a `skip_rows` / `num_rows` slice is applied. Skipped columns still - * get DICTIONARY32 output via the post-hoc `dictionary::detail::encode` fallback in - * `finalize_output`. - * * @param mode Value indicating if the data sources are read all at once or chunk by chunk - * @return True if dict transcode is active for this read (eligible columns had output types and - * decode masks updated and pushed to the device). False otherwise. + * @return True if dict transcode is active for this read. False otherwise */ [[nodiscard]] bool prepare_dict_transcode(read_mode mode); @@ -223,14 +210,9 @@ class reader_impl { /** * @brief Assemble DICTIONARY32 output columns for input columns that were marked eligible by - * `prepare_dict_transcode`. Per-chunk keys (from `pass.str_dict_index`) and INT32 indices are - * concatenated; `cudf::dictionary::detail::concatenate` remaps indices to deduplicated keys - * (indices are not pre-shifted by the reader). - * - * Non-eligible flat STRING columns are left untouched here and are expected to go through the - * post-hoc `cudf::dictionary::encode` fallback in `finalize_output`. + * `prepare_dict_transcode`. * - * @param out_columns The output columns vector to mutate in place. + * @param out_columns The output columns vector to transcode in place. */ void assemble_dict_transcoded_columns(std::vector>& out_columns); @@ -650,8 +632,7 @@ class reader_impl { std::size_t _input_pass_read_limit{0}; // input pass memory usage limit in bytes // Per-input-column flag indicating whether that column was selected for direct - // Parquet-dict → DICTIONARY32 transcode in `prepare_dict_transcode()`. Populated before decode - // and consumed in `assemble_dict_transcoded_columns()`. + // Parquet-dict → DICTIONARY32 transcode. std::vector _dict_transcode_eligible; }; diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 29be54bc57eb..f555413f298c 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -33,15 +33,12 @@ namespace { /** * @brief Host-side check for whether a column chunk decodes to a plain string column. * - * Host-side counterpart of `is_string_col` in `parquet_gpu.hpp`. Kept narrow: for direct - * Parquet-dict → DICTIONARY32 transcode we only accept pure BYTE_ARRAY columns without a - * DECIMAL logical type and without the strings-to-categorical flag. FIXED_LEN_BYTE_ARRAY is - * deliberately excluded because it is typically a binary payload. + * Host-side counterpart of `is_string_col` in `parquet_gpu.hpp` * * @param chunk The column chunk descriptor to classify * @return True if the chunk is a plain BYTE_ARRAY string chunk eligible for transcode */ -[[nodiscard]] bool is_host_byte_array_string_chunk(ColumnChunkDesc const& chunk) +[[nodiscard]] bool is_byte_array_string_chunk(ColumnChunkDesc const& chunk) { if (chunk.physical_type != Type::BYTE_ARRAY) { return false; } if (chunk.is_strings_to_cat) { return false; } @@ -52,10 +49,7 @@ namespace { } /** - * @brief Whether a data-page encoding references a parquet dictionary page. - * - * Both PLAIN_DICTIONARY (legacy) and RLE_DICTIONARY are valid encodings for data pages that - * reference a parquet dictionary page. + * @brief Whether a data-page encoding that contains a dictionary page. * * @param enc The data-page encoding to test * @return True if the encoding is a dictionary data-page encoding @@ -97,7 +91,7 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) { e.has_any_chunk = true; if (chunk.max_nesting_depth != 1 or chunk.max_level[level_type::REPETITION] != 0 or - not is_host_byte_array_string_chunk(chunk) or chunk.num_dict_pages < 1) { + not is_byte_array_string_chunk(chunk) or chunk.num_dict_pages < 1) { e.all_chunks_string = false; } } @@ -108,13 +102,9 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) * A column is eligible iff * - the corresponding output buffer is currently typed as STRING (i.e. a flat string column), * - every chunk of that column is a BYTE_ARRAY string chunk with a dictionary page, - * - every data page of every chunk of that column uses (PLAIN|RLE)_DICTIONARY encoding, + * - every data page of every chunk of that column uses DICTIONARY encoding, * - the chunk has a flat (non-list, non-nested) schema. * - * We scan host-side `pass.chunks` and `pass.pages` here rather than `subpass.pages` because - * `subpass.pages` may be a subset. For single-pass single-subpass reads (the only configuration - * in which `output_dict_columns` is supported), `subpass.pages == pass.pages`. - * * @param pass The pass intermediate data holding host-side chunks and pages * @param input_columns The reader's input column descriptors * @param output_buffers The output column buffers (used to detect flat STRING columns) @@ -158,9 +148,6 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) /** * @brief Build a STRING keys column from a chunk's dictionary entries. * - * Builds a STRING keys column covering the dictionary entries of a single chunk of a single input - * column. `begin` points into the pass-wide `string_index_pair` buffer. - * * @param begin Pointer to the first `string_index_pair` entry for this chunk's dictionary * @param entry_count Number of dictionary entries (keys) for this chunk * @param stream CUDA stream used for device memory operations and kernel launches @@ -192,9 +179,7 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) // DICTIONARY32 columns via a post-hoc `dictionary::detail::encode` instead. if (_output_chunk_read_limit != 0 or _input_pass_read_limit != 0) { return false; } - // Custom row bounds (`skip_rows` / `num_rows`) slice the decoded output to fewer rows than the - // full, unadjusted chunks that `assemble_dict_transcoded_columns` segments by. Skip the fast - // path here too and fall back to the post-hoc encode. + // Skip the fast path if custom row bounds are in effect. if (uses_custom_row_bounds(mode)) { return false; } // AST/JIT filters evaluate predicates on materialized STRING columns, so the direct transcode @@ -219,9 +204,7 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) auto const num_input_cols = _input_columns.size(); - // Change the output buffer type for eligible columns from STRING → INT32. The subsequent - // `allocate_columns` call will then allocate an INT32 buffer that the DICT_INT32 kernel can - // write directly into. + // Change the output buffer type for eligible columns from STRING → INT32. std::for_each( cuda::counting_iterator{0}, cuda::counting_iterator{num_input_cols}, [&](size_t i) { if (not _dict_transcode_eligible[i]) { return; } @@ -230,8 +213,7 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) }); // Rewrite per-page `kernel_mask` for eligible columns on the host subpass pages from - // STRING_DICT → DICT_INT32, then H2D so the device pages agree. Only the flat variant is - // considered here (eligibility requires `max_nesting_depth == 1`). + // STRING_DICT → DICT_INT32, then H2D so the device pages agree. bool any_rewritten = false; std::for_each(subpass.pages.host_begin(), subpass.pages.host_end(), [&](PageInfo& page) { if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { return; } @@ -247,9 +229,7 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) if (not any_rewritten) { return false; } // Push the rewritten `kernel_mask`s back to device so subsequent decode kernels dispatch - // correctly. The copy is enqueued on `_stream`, so no explicit synchronization is required. The - // host source buffer (`subpass.pages`) is owned by the subpass and is neither freed nor - // re-mutated before the copy completes. + // correctly. subpass.pages.host_to_device_async(_stream); subpass.kernel_mask = std::transform_reduce( subpass.pages.host_begin(), @@ -273,7 +253,7 @@ void reader_impl::zero_init_dict_transcoded_index_buffers() // // Only nullable columns need this: the kernel decodes non-nullable columns (max definition level // 0) in DIRECT mode, which writes every output position, so their buffers are fully initialized by - // decode. This mirrors `is_nullable()` in the decode kernel (max_level[DEFINITION] > 0). + // decode. if (_pass_itm_data == nullptr) { return; } auto const& pass = *_pass_itm_data; std::vector col_nullable(_input_columns.size(), false); @@ -411,17 +391,13 @@ void reader_impl::assemble_dict_transcoded_columns( cudf::detail::segmented_null_count(indices_view.null_mask(), indices_pairs, _stream); } - // Build a DICTIONARY32 *view* for every chunk without copying the decoded indices. Each - // view's keys child is this chunk's own STRING keys column (which must be materialized from - // the parquet dictionary page), while its indices child aliases the shared, already-decoded - // INT32 buffer. We select each chunk's row range via the parent dictionary view's - // `offset`/`size` rather than slicing the indices child: `get_indices_annotated()` rebuilds - // the indices view from the child's `head()` plus the parent's offset/size, so a sliced - // child (carrying its own offset) would be ignored. For the same reason the null mask must - // also live on the parent view (sourced from the shared `indices_view`'s mask): the indices - // child's own null mask, if it had one, would be ignored by `get_indices_annotated()`, and a - // hardcoded null_count of 0 would silently turn nulls into a valid index (0) once - // `cudf::detail::concatenate` rewrites indices against the unified, deduplicated keys. + // Build a per-chunk DICTIONARY32 *view* that aliases the shared decoded INT32 buffer (no + // copy): keys = this chunk's STRING column, indices = `indices_view`. The row range, null + // mask, and null count must all live on the *parent* view (via offset/size), not the indices + // child, because `get_indices_annotated()` rebuilds the indices from the child's `head()` plus + // the parent's offset/size/null_mask -- anything set on the child is ignored. A wrong null + // count (e.g. a hardcoded 0) would silently turn nulls into a valid index once + // `cudf::detail::concatenate` remaps the indices against the unified keys. std::vector> seg_keys_owners(chunk_indices.size()); std::vector dict_segment_views(chunk_indices.size()); std::transform( @@ -447,12 +423,7 @@ void reader_impl::assemble_dict_transcoded_columns( {indices_view, seg_keys_owners[k]->view()}}; }); - // `cudf::detail::concatenate` deduplicates + sorts keys and recomputes indices. This is - // required today because DICTIONARY32 keys are assumed unique and sorted. When - // https://github.com/rapidsai/cudf/pull/22839 lands and relaxes that constraint, this - // could be replaced with a cheaper path: plain-concatenate the per-chunk keys columns - // (keeping cross-chunk duplicates) and offset-shift each chunk's row-group-local indices - // by the running total of prior chunks' key counts, avoiding the dedup/sort entirely. + // `cudf::detail::concatenate` deduplicates + sorts keys and recomputes indices. out_columns[out_idx] = cudf::detail::concatenate(dict_segment_views, _stream, _mr); }); } From 822c500de50d2c6eacade5a1088112788117d432 Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 16:56:44 -0700 Subject: [PATCH 35/42] Formatting fixes --- cpp/include/cudf/io/parquet.hpp | 10 ++++++---- cpp/src/io/parquet/reader_impl.cpp | 2 +- cpp/src/io/parquet/reader_impl.hpp | 7 ++++--- cpp/src/io/parquet/reader_impl_dict_transcode.cu | 10 +++++----- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 6e227d1f62d4..bbb7766dff76 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -345,11 +345,12 @@ class parquet_reader_options { /** * @brief Returns whether the reader returns flat string columns as DICTIONARY32 encoded columns * - * When true, the reader outputs STRING columns as DICTIONARY32 encoded columns. A DICTIONARY32 column consists of an INT32 indices - * child and a STRING keys child. + * When true, the reader outputs STRING columns as DICTIONARY32 encoded columns. A DICTIONARY32 + * column consists of an INT32 indices child and a STRING keys child. * * When AST/JIT filters are set, the direct transcode fast path is disabled. - * String columns are materialized, then operated on by the filter. The filtered results are then encoded as DICTIONARY32 columns. + * String columns are materialized, then operated on by the filter. The filtered results are then + * encoded as DICTIONARY32 columns. * * @return `true` if the reader returns flat string columns as DICTIONARY32 encoded columns */ @@ -954,7 +955,8 @@ class parquet_reader_options_builder { } /** - * @brief Sets options for enabling/disabling output of DICTIONARY32 columns for flat string columns. + * @brief Sets options for enabling/disabling output of DICTIONARY32 columns for flat string + * columns. * * @param val Boolean value whether to output flat string columns as DICTIONARY32 encoded columns * diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 52d4c45228a6..bdb81b831e1c 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -538,7 +538,7 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, _input_pass_read_limit{pass_read_limit} { // The direct parquet-dict → DICTIONARY32 transcode fast path only supports single-pass, - // non-chunked reads. + // non-chunked reads. if (_options.output_dict_columns and (chunk_read_limit != 0 or pass_read_limit != 0)) { CUDF_LOG_WARN( "output_dict_columns: the direct parquet-dict transcode fast path is disabled for chunked / " diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 14a3d5a1ca23..f1c9fc93e6e6 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -192,8 +192,9 @@ class reader_impl { * @brief Detect per-column eligibility for direct Parquet-dict → DICTIONARY32 transcode, and * apply the required host-side mutations to `_output_buffers` and `subpass.pages`. * - * Must be called after `prepare_data()`. Populates `_dict_transcode_eligible` with a bool per input column indicating whether the - * column will be assembled as a DICTIONARY32 output later in `assemble_dict_transcoded_columns`. + * Must be called after `prepare_data()`. Populates `_dict_transcode_eligible` with a bool per + * input column indicating whether the column will be assembled as a DICTIONARY32 output later in + * `assemble_dict_transcoded_columns`. * * @param mode Value indicating if the data sources are read all at once or chunk by chunk * @return True if dict transcode is active for this read. False otherwise @@ -210,7 +211,7 @@ class reader_impl { /** * @brief Assemble DICTIONARY32 output columns for input columns that were marked eligible by - * `prepare_dict_transcode`. + * `prepare_dict_transcode`. * * @param out_columns The output columns vector to transcode in place. */ diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index f555413f298c..237172628d69 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -252,8 +252,8 @@ void reader_impl::zero_init_dict_transcoded_index_buffers() // remap indices below. // // Only nullable columns need this: the kernel decodes non-nullable columns (max definition level - // 0) in DIRECT mode, which writes every output position, so their buffers are fully initialized by - // decode. + // 0) in DIRECT mode, which writes every output position, so their buffers are fully initialized + // by decode. if (_pass_itm_data == nullptr) { return; } auto const& pass = *_pass_itm_data; std::vector col_nullable(_input_columns.size(), false); @@ -394,9 +394,9 @@ void reader_impl::assemble_dict_transcoded_columns( // Build a per-chunk DICTIONARY32 *view* that aliases the shared decoded INT32 buffer (no // copy): keys = this chunk's STRING column, indices = `indices_view`. The row range, null // mask, and null count must all live on the *parent* view (via offset/size), not the indices - // child, because `get_indices_annotated()` rebuilds the indices from the child's `head()` plus - // the parent's offset/size/null_mask -- anything set on the child is ignored. A wrong null - // count (e.g. a hardcoded 0) would silently turn nulls into a valid index once + // child, because `get_indices_annotated()` rebuilds the indices from the child's `head()` + // plus the parent's offset/size/null_mask -- anything set on the child is ignored. A wrong + // null count (e.g. a hardcoded 0) would silently turn nulls into a valid index once // `cudf::detail::concatenate` remaps the indices against the unified keys. std::vector> seg_keys_owners(chunk_indices.size()); std::vector dict_segment_views(chunk_indices.size()); From 158d4a2f117c086c6aeb5ec0d5fd6780eaa894bb Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 17:05:06 -0700 Subject: [PATCH 36/42] Helpder and code reuse --- cpp/src/io/parquet/page_hdr.cu | 6 ++-- cpp/src/io/parquet/parquet_gpu.hpp | 13 ++++++++- .../io/parquet/reader_impl_dict_transcode.cu | 28 +++++-------------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cpp/src/io/parquet/page_hdr.cu b/cpp/src/io/parquet/page_hdr.cu index b69c26a4d5eb..71dc290f5901 100644 --- a/cpp/src/io/parquet/page_hdr.cu +++ b/cpp/src/io/parquet/page_hdr.cu @@ -232,8 +232,7 @@ __device__ decode_kernel_mask kernel_mask_for_page(PageInfo const& page, return is_list(chunk) ? decode_kernel_mask::STRING_LIST : is_nested(chunk) ? decode_kernel_mask::STRING_NESTED : decode_kernel_mask::STRING; - } else if (page.encoding == Encoding::PLAIN_DICTIONARY || - page.encoding == Encoding::RLE_DICTIONARY) { + } else if (is_dictionary_encoding(page.encoding)) { return is_list(chunk) ? decode_kernel_mask::STRING_DICT_LIST : is_nested(chunk) ? decode_kernel_mask::STRING_DICT_NESTED : decode_kernel_mask::STRING_DICT; @@ -249,8 +248,7 @@ __device__ decode_kernel_mask kernel_mask_for_page(PageInfo const& page, return is_list(chunk) ? decode_kernel_mask::FIXED_WIDTH_NO_DICT_LIST : is_nested(chunk) ? decode_kernel_mask::FIXED_WIDTH_NO_DICT_NESTED : decode_kernel_mask::FIXED_WIDTH_NO_DICT; - } else if (page.encoding == Encoding::PLAIN_DICTIONARY || - page.encoding == Encoding::RLE_DICTIONARY) { + } else if (is_dictionary_encoding(page.encoding)) { return is_list(chunk) ? decode_kernel_mask::FIXED_WIDTH_DICT_LIST : is_nested(chunk) ? decode_kernel_mask::FIXED_WIDTH_DICT_NESTED : decode_kernel_mask::FIXED_WIDTH_DICT; diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index d299cfe7615b..46c34c46e3af 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -84,6 +84,17 @@ CUDF_HOST_DEVICE constexpr bool is_supported_encoding(Encoding enc) } } +/** + * @brief Whether a page encoding references a dictionary page. + * + * Both PLAIN_DICTIONARY (legacy) and RLE_DICTIONARY mark a data page whose values are indices into + * the column chunk's dictionary page. + */ +CUDF_HOST_DEVICE constexpr bool is_dictionary_encoding(Encoding enc) +{ + return enc == Encoding::PLAIN_DICTIONARY or enc == Encoding::RLE_DICTIONARY; +} + /** * @brief Atomically OR `error` into `error_code`. */ @@ -665,7 +676,7 @@ struct EncPage { /** * @brief Test if the given column chunk is in a string column */ -__device__ constexpr bool is_string_col(ColumnChunkDesc const& chunk) +CUDF_HOST_DEVICE constexpr bool is_string_col(ColumnChunkDesc const& chunk) { // return true for non-hashed byte_array and fixed_len_byte_array that isn't representing // a decimal. diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 237172628d69..5358bc43250b 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -31,32 +31,18 @@ namespace cudf::io::parquet::detail { namespace { /** - * @brief Host-side check for whether a column chunk decodes to a plain string column. + * @brief Whether a column chunk is a plain BYTE_ARRAY string chunk. * - * Host-side counterpart of `is_string_col` in `parquet_gpu.hpp` + * Narrows `is_string_col` (parquet_gpu.hpp) to BYTE_ARRAY only: `is_string_col` also accepts + * FIXED_LEN_BYTE_ARRAY, which is typically a binary payload and is excluded from transcode. This is + * a string-type classifier -- one of several inputs to eligibility, not the eligibility decision. * * @param chunk The column chunk descriptor to classify - * @return True if the chunk is a plain BYTE_ARRAY string chunk eligible for transcode + * @return True if the chunk is a plain (non-categorical, non-decimal) BYTE_ARRAY string chunk */ [[nodiscard]] bool is_byte_array_string_chunk(ColumnChunkDesc const& chunk) { - if (chunk.physical_type != Type::BYTE_ARRAY) { return false; } - if (chunk.is_strings_to_cat) { return false; } - if (chunk.logical_type.has_value() and chunk.logical_type->type == LogicalType::DECIMAL) { - return false; - } - return true; -} - -/** - * @brief Whether a data-page encoding that contains a dictionary page. - * - * @param enc The data-page encoding to test - * @return True if the encoding is a dictionary data-page encoding - */ -[[nodiscard]] bool is_dict_data_page_encoding(Encoding enc) -{ - return enc == Encoding::PLAIN_DICTIONARY or enc == Encoding::RLE_DICTIONARY; + return is_string_col(chunk) and chunk.physical_type == Type::BYTE_ARRAY; } /** @@ -139,7 +125,7 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { continue; } auto const chunk_idx = page.chunk_idx; auto const col_idx = pass.chunks[chunk_idx].src_col_index; - if (not is_dict_data_page_encoding(page.encoding)) { elig[col_idx].all_pages_dict = false; } + if (not is_dictionary_encoding(page.encoding)) { elig[col_idx].all_pages_dict = false; } } return elig; From 2e057350f4e8147b25d95886b5f1a72df64269fa Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 17:21:43 -0700 Subject: [PATCH 37/42] Build fixes --- cpp/src/io/parquet/decode_fixed.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 075325d49827..9d67ffcfbaa7 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -105,10 +105,10 @@ __device__ void decode_dict_indices_as_int32( constexpr int num_warps = block_size / cudf::detail::warp_size; constexpr int max_batch_size = num_warps * cudf::detail::warp_size; - int const leaf_level_index = s->col.max_nesting_depth - 1; + int const leaf_level_index = s->setup.col.max_nesting_depth - 1; auto const data_out = s->nesting_info[leaf_level_index].data_out; - int const skipped_leaf_values = s->page.skipped_leaf_values; + int const skipped_leaf_values = s->setup.page.skipped_leaf_values; int pos = start; while (pos < end) { @@ -118,10 +118,10 @@ __device__ void decode_dict_indices_as_int32( int const dst_pos = [&]() { if constexpr (copy_mode_t == copy_mode::DIRECT) { - return thread_pos - s->first_row; + return thread_pos - s->setup.first_row; } else { int dst_pos = sb->nz_idx[rolling_index(thread_pos)]; - if constexpr (!has_lists_t) { dst_pos -= s->first_row; } + if constexpr (!has_lists_t) { dst_pos -= s->setup.first_row; } return dst_pos; } }(); From 176e95a769ec05527a34b301f5d017a018c77c22 Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 18:14:50 -0700 Subject: [PATCH 38/42] Fixed EMPTY DICT test --- cpp/tests/io/parquet_reader_dict_test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index 996a9ffdce06..e7f10cef9e0b 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -221,8 +221,11 @@ TEST_F(ParquetReaderDictTest, EmptyFlatStringDictTranscode) auto const read_col = read_table->view().column(0); if (read_col.type().id() == cudf::type_id::DICTIONARY32) { + // An empty DICTIONARY32 has no keys, so `cudf::dictionary::decode` returns a type-EMPTY empty + // column (there is no key type to recover) rather than an empty STRING. Just confirm the + // round-trip stays empty. auto const decoded = cudf::dictionary::decode(cudf::dictionary_column_view(read_col)); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, decoded->view()); + EXPECT_EQ(decoded->size(), 0); } else { ASSERT_EQ(read_col.type().id(), cudf::type_id::STRING); CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, read_col); From 238992b1c223805cc4489a3dcdcf6858041b452c Mon Sep 17 00:00:00 2001 From: ykiran Date: Fri, 31 Jul 2026 18:29:18 -0700 Subject: [PATCH 39/42] Removed benchmark file --- cpp/benchmarks/CMakeLists.txt | 2 +- .../io/parquet/parquet_reader_dict.cpp | 402 ------------------ 2 files changed, 1 insertion(+), 403 deletions(-) delete mode 100644 cpp/benchmarks/io/parquet/parquet_reader_dict.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 7e062b512d75..1b57e3b23666 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -297,7 +297,7 @@ ConfigureNVBench( # * parquet reader benchmark ---------------------------------------------------------------------- ConfigureNVBench( PARQUET_READER_NVBENCH io/parquet/parquet_reader_input.cpp io/parquet/parquet_reader_options.cpp - io/parquet/parquet_reader_dict.cpp io/parquet/reader_common.cpp + io/parquet/reader_common.cpp ) # ################################################################################################## diff --git a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp b/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp deleted file mode 100644 index c5e32e6dae2d..000000000000 --- a/cpp/benchmarks/io/parquet/parquet_reader_dict.cpp +++ /dev/null @@ -1,402 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Benchmark for the parquet-dictionary -> cudf DICTIONARY32 transcode fast path enabled by -// `parquet_reader_options::output_dict_columns`. A flat fully dictionary-encoded string column is -// read under three modes, selected by the `mode` axis, so the transcode path can be judged against -// both a lower and an upper reference: -// -// - "materialize_string": reader default; the column materializes as STRING. The cheapest -// possible -// read (no dictionary built); serves as the lower-bound reference. -// - "materialize_string_and_encode_dict": materialize as STRING, then `cudf::dictionary::encode` -// to DICTIONARY32. The pre-existing way to obtain a dictionary column and the -// fair apples-to-apples baseline the transcode fast path aims to beat. -// - "direct_dict_transcode": `output_dict_columns=true`; the reader keeps the dictionary -// representation and emits DICTIONARY32 directly, skipping string -// materialization. -// -// Both "materialize_string_and_encode_dict" and "direct_dict_transcode" produce DICTIONARY32 -// output, so their times and peak memory are directly comparable; "materialize_string" shows the -// floor cost of just decoding. A relative comparison table (materialize_string_and_encode_dict = -// 100%%) is printed at program exit (see comparison_collector). -// -// The sweep varies cardinality, rows per row group, and rows per data page at a fixed table size -// (kept modest so the sweep stays light for local/CI runs). A single column (num_cols == 1) is used -// so a row group can hold as many distinct values as possible: the writer picks the dictionary -// index bit width per row group from the distinct values it contains, capped at MAX_DICT_BITS (24). -// Cardinality therefore ranges up to 2^24, the point at which 24-bit indices are required. At high -// distinct-per-row-group counts the writer may abandon dictionary encoding (indices exceed 24 bits, -// or plain encoding is smaller); when that leaves the column ineligible for transcode, that state -// is skipped rather than measured. - -namespace { - -constexpr cudf::size_type num_cols = 1; - -enum class bench_mode { - materialize_string, - materialize_string_and_encode_dict, - direct_dict_transcode -}; - -[[nodiscard]] bench_mode parse_mode(std::string const& mode) -{ - if (mode == "materialize_string") { return bench_mode::materialize_string; } - if (mode == "materialize_string_and_encode_dict") { - return bench_mode::materialize_string_and_encode_dict; - } - if (mode == "direct_dict_transcode") { return bench_mode::direct_dict_transcode; } - CUDF_FAIL("Unknown benchmark mode: " + mode); -} - -// Upper-bound estimate of the dictionary index bit width the writer will use for a row group. The -// width is derived per row group from its distinct value count, which is at most -// min(cardinality, rows in the row group). This over-estimates when the (last) row group is shorter -// than `row_group_size_rows` or when hash collisions reduce distinct counts. -[[nodiscard]] int approx_dict_bits(std::int64_t cardinality, std::int64_t row_group_size_rows) -{ - auto const distinct = std::min(cardinality, row_group_size_rows); - if (distinct <= 1) { return 1; } - return static_cast(std::bit_width(static_cast(distinct - 1))); -} - -// nvbench invokes the benchmark once per axis combination, prints its own results table, and omits -// skipped states from it; there is also no cross-state hook, so a single invocation cannot group -// the three modes of a configuration together. This collector accumulates each run's CPU/GPU mean -// time (and any direct_dict_transcode skip reason), keyed by every setting except `mode`, and -// prints one row per configuration from its destructor -- i.e. at program exit, after nvbench's own -// output -// -- with all three modes in fixed order (materialize_string, materialize_string_and_encode_dict, -// direct_dict_transcode) so each configuration is -// grouped and ordered regardless of nvbench's state ordering or its omission of skipped states. -struct run_settings { - std::int64_t cardinality; - std::int64_t data_size; - std::int64_t row_group_size_rows; - std::int64_t max_page_size_rows; - std::int64_t avg_string_length; - - bool operator<(run_settings const& o) const - { - return std::tie( - cardinality, data_size, row_group_size_rows, max_page_size_rows, avg_string_length) < - std::tie(o.cardinality, - o.data_size, - o.row_group_size_rows, - o.max_page_size_rows, - o.avg_string_length); - } -}; - -struct mode_timing { - double cpu_ms = 0.0; - double gpu_ms = 0.0; - bool present = false; -}; - -class comparison_collector { - public: - void record(run_settings const& key, bench_mode mode, double cpu_ms, double gpu_ms) - { - auto& r = _rows[key]; - auto& slot = (mode == bench_mode::materialize_string_and_encode_dict) - ? r.materialize_string_and_encode_dict - : ((mode == bench_mode::direct_dict_transcode) ? r.direct_dict_transcode - : r.materialize_string); - slot = mode_timing{cpu_ms, gpu_ms, true}; - } - - // Record that direct_dict_transcode was skipped for a configuration, with a short reason shown in - // the table. `direct_dict_transcode` is the only mode this benchmark ever skips. - void record_skip(run_settings const& key, std::string reason) - { - _rows[key].direct_dict_transcode_note = std::move(reason); - } - - ~comparison_collector() { print(); } - - private: - struct row { - mode_timing materialize_string; - mode_timing materialize_string_and_encode_dict; - mode_timing direct_dict_transcode; - // reason shown when `direct_dict_transcode` was skipped as ineligible - std::string direct_dict_transcode_note; - }; - - void print() const - { - if (_rows.empty()) { return; } - - std::printf( - "\n# Per-configuration mode comparison " - "(order: materialize_string, materialize_string_and_encode_dict, " - "direct_dict_transcode)\n\n"); - std::printf( - "| cardinality | ~dict_bits | data_size (MiB) | row_group_size_rows | max_page_size_rows | " - "materialize_string CPU (ms) | materialize_string_and_encode_dict CPU (ms) | " - "direct_dict_transcode CPU (ms) | materialize_string GPU (ms) | " - "materialize_string_and_encode_dict GPU (ms) | direct_dict_transcode GPU (ms) | " - "direct_dict_transcode CPU speedup %% | direct_dict_transcode GPU speedup %% |\n"); - std::printf("|---|---|---|---|---|---|---|---|---|---|---|---|---|\n"); - - auto const num = [](double v) { - std::array buf{}; - std::snprintf(buf.data(), buf.size(), "%.3f", v); - return std::string{buf.data()}; - }; - // Timing cell: the value if the mode ran, else the skip reason (direct_dict_transcode only) or - // "-". - auto const cell = - [&](mode_timing const& t, double mode_timing::* field, std::string const& note) { - if (t.present) { return num(t.*field); } - return note.empty() ? std::string{"-"} : note; - }; - - // Speedup of direct_dict_transcode over the materialize_string_and_encode_dict baseline, as a - // signed percentage of baseline time saved: - // 100 * (materialize_string_and_encode_dict - direct_dict_transcode) / - // materialize_string_and_encode_dict. - // Positive = direct_dict_transcode faster, negative = slower. "-" when either mode is missing. - auto const speedup = - [](mode_timing const& base, mode_timing const& cand, double mode_timing::* field) { - if (not(base.present and cand.present)) { return std::string{"-"}; } - std::array buf{}; - std::snprintf( - buf.data(), buf.size(), "%+.1f%%", 100.0 * (base.*field - cand.*field) / (base.*field)); - return std::string{buf.data()}; - }; - - for (auto const& [key, r] : _rows) { - std::printf( - "| %lld | %d | %lld | %lld | %lld | %s | %s | %s | %s | %s | %s | %s | %s |\n", - static_cast(key.cardinality), - approx_dict_bits(key.cardinality, key.row_group_size_rows), - static_cast(key.data_size >> 20), - static_cast(key.row_group_size_rows), - static_cast(key.max_page_size_rows), - cell(r.materialize_string, &mode_timing::cpu_ms, std::string{}).c_str(), - cell(r.materialize_string_and_encode_dict, &mode_timing::cpu_ms, std::string{}).c_str(), - cell(r.direct_dict_transcode, &mode_timing::cpu_ms, r.direct_dict_transcode_note).c_str(), - cell(r.materialize_string, &mode_timing::gpu_ms, std::string{}).c_str(), - cell(r.materialize_string_and_encode_dict, &mode_timing::gpu_ms, std::string{}).c_str(), - cell(r.direct_dict_transcode, &mode_timing::gpu_ms, r.direct_dict_transcode_note).c_str(), - speedup(r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::cpu_ms) - .c_str(), - speedup(r.materialize_string_and_encode_dict, r.direct_dict_transcode, &mode_timing::gpu_ms) - .c_str()); - } - std::printf("\n"); - } - - std::map _rows; -}; - -comparison_collector g_comparison_collector; - -// The transcode fast path requires every data page of an eligible column to be dictionary-encoded. -// Forcing `dictionary_policy::ALWAYS` maximizes the chance of full dictionary encoding; the writer -// can still fall back to plain when indices exceed MAX_DICT_BITS or plain is smaller, in which case -// the direct_dict_transcode state is skipped by the caller. -void write_dict_encoded_parquet(cudf::table_view const& view, - cuio_source_sink_pair& source_sink, - std::int64_t row_group_size_rows, - std::int64_t max_page_size_rows) -{ - cudf::io::parquet_writer_options write_opts = - cudf::io::parquet_writer_options::builder(source_sink.make_sink_info(), view) - .compression(cudf::io::compression_type::NONE) - .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) - .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN); - if (row_group_size_rows > 0) { - write_opts.set_row_group_size_rows(static_cast(row_group_size_rows)); - } - if (max_page_size_rows > 0) { - write_opts.set_max_page_size_rows(static_cast(max_page_size_rows)); - } - cudf::io::write_parquet(write_opts); -} - -} // namespace - -void BM_parquet_read_dict_transcode(nvbench::state& state) -{ - auto const cardinality = static_cast(state.get_int64("cardinality")); - auto const data_size = static_cast(state.get_int64("data_size")); - auto const rg_size_rows = state.get_int64("row_group_size_rows"); - auto const page_size_rows = state.get_int64("max_page_size_rows"); - auto const mode = parse_mode(state.get_string("mode")); - auto const avg_string_length = static_cast(state.get_int64("avg_string_length")); - auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); - - // corresponds to 3 sigma (full width 6 sigma: 99.7% of range) - auto const half_width = avg_string_length >> 3; - auto const length_min = avg_string_length - half_width; - auto const length_max = avg_string_length + half_width; - - data_profile const profile = - data_profile_builder() - .cardinality(cardinality) - .avg_run_length(1) - .distribution(data_type::STRING, distribution_id::NORMAL, length_min, length_max); - - auto const d_type = get_type_or_group(static_cast(data_type::STRING)); - auto const tbl = - create_random_table(cycle_dtypes(d_type, num_cols), table_size_bytes{data_size}, profile); - auto const view = tbl->view(); - - cuio_source_sink_pair source_sink(source_type); - write_dict_encoded_parquet(view, source_sink, rg_size_rows, page_size_rows); - - cudf::io::parquet_reader_options read_opts = - cudf::io::parquet_reader_options::builder(source_sink.make_source_info()) - .output_dict_columns(mode == bench_mode::direct_dict_transcode); - - // Perform the full work for the selected mode: read, and for `materialize_string_and_encode_dict` - // additionally encode each STRING column to DICTIONARY32. Returns the resulting table so it can - // be reused for both the outside-the-timed-region verification and the timed measurement. - auto const run_mode = [&]() -> std::unique_ptr { - auto result = cudf::io::read_parquet(read_opts); - if (mode == bench_mode::materialize_string_and_encode_dict) { - std::vector> encoded; - encoded.reserve(result.tbl->num_columns()); - for (auto const& col : result.tbl->view()) { - encoded.push_back(cudf::dictionary::encode(col)); - } - return std::make_unique(std::move(encoded)); - } - return std::move(result.tbl); - }; - - // Verification (outside the timed region, run for every mode so warm-up is symmetric). For - // `direct_dict_transcode`, the writer may have fallen back to plain encoding at high cardinality - // / large row groups, leaving the column ineligible for the fast path; in that case skip the - // state rather than silently measuring the plain path or aborting the whole sweep. - { - auto const probe = run_mode(); - // Bind the table_view to a local: `probe->view()` returns a temporary, so calling it separately - // for begin() and end() would yield iterators into two different temporaries (mismatched- - // iterator UB). Iterate a single view instead. - auto const probe_view = probe->view(); - CUDF_EXPECTS(probe_view.num_columns() == num_cols, "Unexpected number of columns"); - auto const all_of_type = [&](cudf::type_id id) { - return std::all_of(probe_view.begin(), probe_view.end(), [id](auto const& col) { - return col.type().id() == id; - }); - }; - auto const actual_type_id = static_cast(probe_view.column(0).type().id()); - if (mode == bench_mode::materialize_string) { - if (not all_of_type(cudf::type_id::STRING)) { - state.skip( - "materialize_string produced unexpected type_id=" + std::to_string(actual_type_id) + - " (expected STRING=" + std::to_string(static_cast(cudf::type_id::STRING)) + ")"); - return; - } - } else if (mode == bench_mode::materialize_string_and_encode_dict) { - if (not all_of_type(cudf::type_id::DICTIONARY32)) { - state.skip("materialize_string_and_encode_dict produced unexpected type_id=" + - std::to_string(actual_type_id) + " (expected DICTIONARY32=" + - std::to_string(static_cast(cudf::type_id::DICTIONARY32)) + ")"); - return; - } - } else if (not all_of_type(cudf::type_id::DICTIONARY32)) { - // Record the skip so the end-of-program per-configuration table can show why - // direct_dict_transcode has no - // timing for this configuration (nvbench omits skipped states from its own table). - g_comparison_collector.record_skip(run_settings{cardinality, - static_cast(data_size), - rg_size_rows, - page_size_rows, - avg_string_length}, - "skipped: plain fallback"); - state.skip( - "direct_dict_transcode did not produce DICTIONARY32: at this cardinality / row-group size " - "the writer " - "fell back to plain encoding, making the column ineligible for the fast path"); - return; - } - } - - auto mem_stats_logger = cudf::memory_stats_logger(); - state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); - state.exec(nvbench::exec_tag::sync | nvbench::exec_tag::timer, - [&](nvbench::launch& launch, auto& timer) { - drop_page_cache_if_enabled(read_opts.get_source().filepaths()); - - timer.start(); - auto const result = run_mode(); - timer.stop(); - - CUDF_EXPECTS(result->num_columns() == num_cols, "Unexpected number of columns"); - }); - - auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); - auto const cpu_time = state.get_summary("nv/cold/time/cpu/mean").get_float64("value"); - state.add_element_count(static_cast(data_size) / time, "bytes_per_second"); - state.add_element_count(static_cast(view.num_rows()) / time, "rows_per_sec"); - state.add_buffer_size( - mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); - state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); - - // Record this run for the end-of-program - // direct_dict_transcode-vs-materialize_string_and_encode_dict comparison table. Times are - // reported by nvbench in seconds; store as milliseconds. - g_comparison_collector.record(run_settings{cardinality, - static_cast(data_size), - rg_size_rows, - page_size_rows, - avg_string_length}, - mode, - cpu_time * 1e3, - time * 1e3); -} - -NVBENCH_BENCH(BM_parquet_read_dict_transcode) - .set_name("parquet_read_dict_transcode") - .add_string_axis("io_type", {"DEVICE_BUFFER"}) - .set_min_samples(4) - .add_string_axis( - "mode", {"materialize_string", "materialize_string_and_encode_dict", "direct_dict_transcode"}) - // Cardinality: low, mid, and 2^24 -- the point at which per-row-group dictionary indices need the - // maximum 24 bits the writer supports (MAX_DICT_BITS); beyond that the writer abandons dictionary - // encoding. Achieved bits = ceil(log2(min(cardinality, rows per row group))). - .add_int64_axis("cardinality", {1 << 10, 1 << 20, 1 << 24}) - // Fixed table size, kept modest so the sweep stays light for local/CI runs (peak memory and - // per-state runtime scale with this). - .add_int64_axis("data_size", {std::int64_t{512} << 20}) - // Rows per row group: small (many row groups -> stresses per-row-group key concatenation), - // default (1M), and very large (>= 2^24 so a single row group can reach the 24-bit dict - // boundary). - .add_int64_axis("row_group_size_rows", {100'000, 1'000'000, 20'000'000}) - // Rows per data page: small and large (page size is a second-order factor for this benchmark). - .add_int64_axis("max_page_size_rows", {20'000, 1'000'000}) - .add_int64_axis("avg_string_length", {16}); From 6b7e3fe5232102d926b2dbdd0d2d543e401d6f6d Mon Sep 17 00:00:00 2001 From: ykiran Date: Mon, 3 Aug 2026 21:04:42 -0700 Subject: [PATCH 40/42] Added fallback for single row gorup path --- .../io/parquet/reader_impl_dict_transcode.cu | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 5358bc43250b..4cfb9b100e0a 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -330,18 +331,27 @@ void reader_impl::assemble_dict_transcoded_columns( "Expected INT32 indices column for dict-transcoded flat string column"); auto indices_owner = std::move(indices_col); - // Single row group fast path: keys are already unique (one dict page), no dedup needed. - // Take ownership of the decoded INT32 indices buffer directly (zero copy), skipping the - // offset/null-count/segment-view work that is only needed for multi-chunk concatenation. - if (chunk_indices.size() == 1) { + // Single row group fast path: the Parquet dictionary page's entries become the keys as-is, + // in page order. If a file carries duplicate dictionary entries, the + // shortcut is skipped and the general multi-chunk path below runs instead: + // `emit_single_row_group_column` returns true when it fell back (keys not distinct), leaving + // `indices_owner` intact for the path below. + auto const emit_single_row_group_column = [&]() -> bool { auto const& chunk = pass.chunks[chunk_indices[0]]; + auto keys = make_keys_column_from_index_pairs( + chunk.str_dict_index, chunk_key_counts[0], _stream, _mr); + auto const num_distinct_keys = cudf::detail::distinct_count( + keys->view(), null_policy::INCLUDE, nan_policy::NAN_IS_VALID, _stream); + if (num_distinct_keys != keys->size()) { return true; } // fall back: dedup below out_columns[out_idx] = - cudf::make_dictionary_column(make_keys_column_from_index_pairs( - chunk.str_dict_index, chunk_key_counts[0], _stream, _mr), - std::move(indices_owner), - _stream, - _mr); - return; + cudf::make_dictionary_column(std::move(keys), std::move(indices_owner), _stream, _mr); + return false; + }; + + if (chunk_indices.size() == 1) { + bool const fallback_used = emit_single_row_group_column(); + if (not fallback_used) { return; } + // Keys were not distinct: fall through to the multi-row-group path, which deduplicates. } // Multi-row-group path: the indices buffer is shared (aliased) by per-chunk DICTIONARY32 From 3802349e8d33cb770ce3ad74183d74aa03217042 Mon Sep 17 00:00:00 2001 From: ykiran Date: Wed, 5 Aug 2026 22:07:20 -0700 Subject: [PATCH 41/42] More cleanup --- cpp/src/io/parquet/decode_fixed.cu | 10 +- cpp/src/io/parquet/reader_impl.cpp | 4 +- cpp/src/io/parquet/reader_impl.hpp | 14 +- .../io/parquet/reader_impl_dict_transcode.cu | 73 ++---- cpp/tests/io/parquet_reader_dict_test.cpp | 225 ++++++++++++++++++ 5 files changed, 258 insertions(+), 68 deletions(-) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 9d67ffcfbaa7..0ac65d070bd7 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -132,8 +132,14 @@ __device__ void decode_dict_indices_as_int32( return thread_pos; }(); - auto* dst = reinterpret_cast(data_out) + dst_pos; - *dst = sb->dict_idx[rolling_index(src_pos)]; + auto* dst = reinterpret_cast(data_out) + dst_pos; + auto const num_keys = static_cast(s->dict_size / sizeof(string_index_pair)); + auto const idx = sb->dict_idx[rolling_index(src_pos)]; + if (idx >= num_keys) { + s->set_error_code(decode_error::DATA_STREAM_OVERRUN); + } else { + *dst = idx; + } } pos += batch_size; diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index bdb81b831e1c..75394523b594 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -716,7 +716,7 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // eligibility and mutate `_output_buffers` / `subpass.pages` before we allocate column buffers // or dispatch decode kernels. This has to happen before `preprocess_chunk_strings` / // `allocate_columns` because those branch on `subpass.kernel_mask` and on `out_buf.type`. - bool const dict_transcode_active = prepare_dict_transcode(mode); + prepare_dict_transcode(mode); // computes: // PageNestingInfo::batch_size for each level of nesting, for each page, taking row bounds into @@ -771,7 +771,7 @@ table_with_metadata reader_impl::read_chunk_internal(read_mode mode) // `prepare_dict_transcode`, the entries in `out_columns` are currently INT32 indices columns. // Assemble them into DICTIONARY32 columns here by attaching per-chunk keys; concatenate // remaps indices to the unified keys child. - if (dict_transcode_active) { assemble_dict_transcoded_columns(out_columns); } + assemble_dict_transcoded_columns(out_columns); out_columns = cudf::structs::detail::enforce_null_consistency(std::move(out_columns), _stream, _mr); diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index f1c9fc93e6e6..46f4769158c4 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -194,20 +194,12 @@ class reader_impl { * * Must be called after `prepare_data()`. Populates `_dict_transcode_eligible` with a bool per * input column indicating whether the column will be assembled as a DICTIONARY32 output later in - * `assemble_dict_transcoded_columns`. + * `assemble_dict_transcoded_columns`. That member is the sole signal of whether the fast path is + * active: `assemble_dict_transcoded_columns` no-ops when no column is eligible. * * @param mode Value indicating if the data sources are read all at once or chunk by chunk - * @return True if dict transcode is active for this read. False otherwise */ - [[nodiscard]] bool prepare_dict_transcode(read_mode mode); - - /** - * @brief Zero-initialize the INT32 output buffers of dict-transcoded columns so that null rows - * carry a well-defined dictionary index (the `DICT_INT32` kernel skips null positions). - * - * Must be called after `allocate_columns` and before `decode_page_data`. - */ - void zero_init_dict_transcoded_index_buffers(); + void prepare_dict_transcode(read_mode mode); /** * @brief Assemble DICTIONARY32 output columns for input columns that were marked eligible by diff --git a/cpp/src/io/parquet/reader_impl_dict_transcode.cu b/cpp/src/io/parquet/reader_impl_dict_transcode.cu index 4cfb9b100e0a..796c48447662 100644 --- a/cpp/src/io/parquet/reader_impl_dict_transcode.cu +++ b/cpp/src/io/parquet/reader_impl_dict_transcode.cu @@ -153,31 +153,31 @@ void update_from_chunk(column_eligibility& e, ColumnChunkDesc const& chunk) } // namespace -bool reader_impl::prepare_dict_transcode(read_mode mode) +void reader_impl::prepare_dict_transcode(read_mode mode) { CUDF_FUNC_RANGE(); _dict_transcode_eligible.assign(_input_columns.size(), false); - if (not _options.output_dict_columns) { return false; } + if (not _options.output_dict_columns) { return; } // The fast path requires the whole column to live in a single subpass. For chunked / multi-pass // reads (non-zero chunk or pass read limit) we skip it and let `finalize_output` produce the // DICTIONARY32 columns via a post-hoc `dictionary::detail::encode` instead. - if (_output_chunk_read_limit != 0 or _input_pass_read_limit != 0) { return false; } + if (_output_chunk_read_limit != 0 or _input_pass_read_limit != 0) { return; } // Skip the fast path if custom row bounds are in effect. - if (uses_custom_row_bounds(mode)) { return false; } + if (uses_custom_row_bounds(mode)) { return; } // AST/JIT filters evaluate predicates on materialized STRING columns, so the direct transcode // fast path cannot run under a filter. Skip it and let `finalize_output` encode the filtered // STRING result to DICTIONARY32 via the post-hoc `dictionary::detail::encode` fallback. - if (_expr_conv.get_converted_expr().has_value()) { return false; } + if (_expr_conv.get_converted_expr().has_value()) { return; } auto& pass = *_pass_itm_data; auto& subpass = *pass.subpass; - if (pass.chunks.empty() or subpass.pages.size() == 0) { return false; } + if (pass.chunks.empty() or subpass.pages.size() == 0) { return; } auto const elig = compute_dict_transcode_eligibility(pass, _input_columns, _output_buffers); std::transform( @@ -187,7 +187,7 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) auto const num_eligible = std::count(_dict_transcode_eligible.begin(), _dict_transcode_eligible.end(), true); - if (num_eligible == 0) { return false; } + if (num_eligible == 0) { return; } auto const num_input_cols = _input_columns.size(); @@ -213,7 +213,12 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) } }); - if (not any_rewritten) { return false; } + // No page was actually rewritten. Clear the eligibility flags so the member reflects the true + // "inactive" state. + if (not any_rewritten) { + _dict_transcode_eligible.assign(_input_columns.size(), false); + return; + } // Push the rewritten `kernel_mask`s back to device so subsequent decode kernels dispatch // correctly. @@ -224,51 +229,6 @@ bool reader_impl::prepare_dict_transcode(read_mode mode) uint32_t{0}, std::bit_or<>{}, [](PageInfo const& page) { return static_cast(page.kernel_mask); }); - return true; -} - -void reader_impl::zero_init_dict_transcoded_index_buffers() -{ - CUDF_FUNC_RANGE(); - - // The `DICT_INT32` kernel only writes to positions with valid definition levels, leaving null - // slots untouched. Since `allocate_columns` does not zero-initialize fixed-width buffers by - // default, the INT32 output buffer for a transcoded column may contain uninitialized bytes at - // null positions. Zero them here so null rows carry a well-defined (valid) index into the - // dictionary keys -- a requirement for `cudf::dictionary::detail::concatenate` to correctly - // remap indices below. - // - // Only nullable columns need this: the kernel decodes non-nullable columns (max definition level - // 0) in DIRECT mode, which writes every output position, so their buffers are fully initialized - // by decode. - if (_pass_itm_data == nullptr) { return; } - auto const& pass = *_pass_itm_data; - std::vector col_nullable(_input_columns.size(), false); - for (auto const& chunk : pass.chunks) { - if (chunk.max_level[level_type::DEFINITION] > 0) { col_nullable[chunk.src_col_index] = true; } - } - - std::vector> index_bufs; - index_bufs.reserve(_input_columns.size()); - std::for_each(cuda::counting_iterator{0}, - cuda::counting_iterator{_input_columns.size()}, - [&](size_t i) { - if (not _dict_transcode_eligible[i]) { return; } - // Non-nullable columns decode in DIRECT mode and write every slot -> no zeroing. - if (not col_nullable[i]) { return; } - auto& out_buf = _output_buffers[_input_columns[i].nesting[0]]; - if (out_buf.type.id() != type_id::INT32) { return; } - if (out_buf.data() == nullptr or out_buf.size == 0) { return; } - index_bufs.emplace_back(static_cast(out_buf.data()), - static_cast(out_buf.size)); - }); - - if (index_bufs.empty()) { return; } - - // Zero all eligible index buffers in a single batched operation instead of one memset per column. - auto const pinned_index_bufs = cudf::detail::make_pinned_vector( - cudf::host_span const>{index_bufs}, _stream); - cudf::detail::batched_memset(pinned_index_bufs, 0, _stream); } void reader_impl::assemble_dict_transcoded_columns( @@ -278,6 +238,13 @@ void reader_impl::assemble_dict_transcoded_columns( if (_pass_itm_data == nullptr) { return; } + // Nothing to assemble unless `prepare_dict_transcode` marked at least one column eligible. + if (std::none_of(_dict_transcode_eligible.begin(), + _dict_transcode_eligible.end(), + [](bool eligible) { return eligible; })) { + return; + } + auto const& pass = *_pass_itm_data; // For each eligible input column, collect its chunks in row-group order, build a per-chunk diff --git a/cpp/tests/io/parquet_reader_dict_test.cpp b/cpp/tests/io/parquet_reader_dict_test.cpp index e7f10cef9e0b..80e2cf6dcf56 100644 --- a/cpp/tests/io/parquet_reader_dict_test.cpp +++ b/cpp/tests/io/parquet_reader_dict_test.cpp @@ -10,12 +10,17 @@ #include #include +#include #include +#include #include #include #include #include +#include +#include #include +#include #include #include @@ -23,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +126,48 @@ cudf::io::table_with_metadata read_parquet_as_dict(std::string const& filepath) return cudf::io::read_parquet(read_opts); } +// A simple INT32 iota column, used as a non-string / filter key column alongside the strings. +cudf::test::fixed_width_column_wrapper make_int_key_column() +{ + std::vector keys(num_rows); + std::iota(keys.begin(), keys.end(), 0); + return cudf::test::fixed_width_column_wrapper(keys.begin(), keys.end()); +} + +// Build a string column whose first half is low-cardinality (a small dictionary that fits the +// writer's dictionary budget) and whose second half is all-distinct (a dictionary too large for +// the budget). cuDF chooses dictionary use per column-chunk, so under an ADAPTIVE policy with a +// small `max_dictionary_size` the resulting column carries a mix of dictionary-encoded and +// PLAIN-encoded chunks -- which makes it ineligible for the direct transcode fast path. +cudf::test::strings_column_wrapper make_mixed_encoding_strings() +{ + std::vector strings(num_rows); + for (cudf::size_type i = 0; i < num_rows; ++i) { + bool const low_cardinality_region = i < num_rows / 2; + int const value = low_cardinality_region ? (i % 16) : i; + strings[i] = make_value_string(value); + } + return cudf::test::strings_column_wrapper(strings.begin(), strings.end()); +} + +// Like `write_parquet`, but with an ADAPTIVE dictionary policy and a caller-supplied dictionary +// budget so the writer falls back to PLAIN encoding for row groups whose dictionary exceeds it. +void write_parquet_adaptive(cudf::table_view const& input, + std::string const& filepath, + size_t max_dict_size) +{ + auto const options = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, input) + .dictionary_policy(cudf::io::dictionary_policy::ADAPTIVE) + .max_dictionary_size(max_dict_size) + .compression(cudf::io::compression_type::NONE) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .row_group_size_rows(row_group_size) + .max_page_fragment_size(row_group_size) + .build(); + cudf::io::write_parquet(options); +} + } // namespace struct ParquetReaderDictTest : public cudf::test::BaseFixture {}; @@ -261,3 +309,180 @@ TEST_F(ParquetReaderDictTest, SlicedFlatStringDictTranscode) auto const decoded_read = cudf::dictionary::decode(cudf::dictionary_column_view(read_col)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(sliced, decoded_read->view()); } + +// Non-happy test (Fast path ineligible): a filter combined with `output_dict_columns`. A filter +// forces the direct transcode fast path off (predicates evaluate on materialized STRING columns); +// `finalize_output` still encodes the surviving rows to DICTIONARY32 after the filter is applied. +// The key column is projected through unchanged. +TEST_F(ParquetReaderDictTest, FilterWithOutputDictColumns) +{ + auto key_col = make_int_key_column(); + auto str_col = make_low_cardinality_strings(); + + auto const input_tbl = cudf::table_view{{key_col, str_col}}; + auto const filepath = temp_env->get_temp_filepath("FilterWithOutputDictColumns.parquet"); + write_parquet(input_tbl, filepath); + + // Filter: key column (col 0) >= num_rows / 2. + auto literal_value = cudf::numeric_scalar(num_rows / 2); + auto literal = cudf::ast::literal(literal_value); + auto col_ref = cudf::ast::column_reference(0); + auto filter_expr = cudf::ast::operation(cudf::ast::ast_operator::GREATER_EQUAL, col_ref, literal); + + // Expected result: apply the same predicate to the input table on host-visible data. + auto const predicate = cudf::compute_column(input_tbl, filter_expr); + auto const expected = cudf::apply_boolean_mask(input_tbl, predicate->view()); + ASSERT_LT(expected->num_rows(), num_rows) << "filter must remove some rows to be meaningful"; + + auto const read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .output_dict_columns(true) + .filter(filter_expr) + .build(); + auto const read_table = cudf::io::read_parquet(read_opts).tbl; + ASSERT_EQ(read_table->num_columns(), 2); + ASSERT_EQ(read_table->num_rows(), expected->num_rows()); + + // Key column: unchanged INT32. + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected->view().column(0), read_table->view().column(0)); + + // String column: DICTIONARY32 via the post-filter fallback encode; decodes to the surviving rows. + auto const read_str = read_table->view().column(1); + ASSERT_EQ(read_str.type().id(), cudf::type_id::DICTIONARY32); + auto const decoded = cudf::dictionary::decode(cudf::dictionary_column_view(read_str)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected->view().column(1), decoded->view()); +} + +// Non-happy test (Fast path ineligible): a column whose chunks mix dictionary and PLAIN data pages. +// cuDF decides dictionary use per column-chunk, so a small dictionary budget over mixed-cardinality +// data yields some dictionary-encoded row groups and some PLAIN-encoded ones. The PLAIN pages +// disqualify the column from the fast path; `output_dict_columns` still produces a correct +// DICTIONARY32 via the fallback encode. +TEST_F(ParquetReaderDictTest, MixedDictAndPlainPagesFallback) +{ + auto input_col = make_mixed_encoding_strings(); + + auto const input_tbl = cudf::table_view{{input_col}}; + auto const filepath = temp_env->get_temp_filepath("MixedDictAndPlainPagesFallback.parquet"); + write_parquet_adaptive(input_tbl, filepath, /*max_dict_size=*/4 * 1024); + + auto const read_table = read_parquet_as_dict(filepath).tbl; + ASSERT_EQ(read_table->num_rows(), num_rows); + ASSERT_EQ(read_table->num_columns(), 1); + + auto const read_col = read_table->view().column(0); + ASSERT_EQ(read_col.type().id(), cudf::type_id::DICTIONARY32) + << "output_dict_columns must still yield DICTIONARY32 via the fallback encode"; + auto const decoded = cudf::dictionary::decode(cudf::dictionary_column_view(read_col)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, decoded->view()); +} + +// Non-happy test (Fast path ineligible): `skip_rows` / `num_rows`. Custom row bounds force the fast +// path off; the fallback still emits a DICTIONARY32 that must decode to exactly the requested row +// window. +TEST_F(ParquetReaderDictTest, SkipRowsNumRowsDictTranscode) +{ + auto input_col = make_low_cardinality_strings(); + + auto const input_tbl = cudf::table_view{{input_col}}; + auto const filepath = temp_env->get_temp_filepath("SkipRowsNumRowsDictTranscode.parquet"); + write_parquet(input_tbl, filepath); + + cudf::size_type const skip = row_group_size + 25; + cudf::size_type const rows = 2 * row_group_size + 40; + + auto const read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .output_dict_columns(true) + .skip_rows(skip) + .num_rows(rows) + .build(); + auto const read_table = cudf::io::read_parquet(read_opts).tbl; + ASSERT_EQ(read_table->num_columns(), 1); + ASSERT_EQ(read_table->num_rows(), rows); + + auto const read_col = read_table->view().column(0); + ASSERT_EQ(read_col.type().id(), cudf::type_id::DICTIONARY32); + auto const decoded = cudf::dictionary::decode(cudf::dictionary_column_view(read_col)); + + auto const expected = + cudf::slice(static_cast(input_col), {skip, skip + rows}).front(); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, decoded->view()); +} + +// Non-happy test (Fast path ineligible): `chunked_parquet_reader`. A chunked read sets an +// output-chunk byte limit, which disables the fast path. Each chunk must come back as DICTIONARY32 +// via the fallback; reassembling the decoded chunks must reproduce the original column. +TEST_F(ParquetReaderDictTest, ChunkedReadDictTranscode) +{ + auto input_col = make_low_cardinality_strings(); + + auto const input_tbl = cudf::table_view{{input_col}}; + auto const filepath = temp_env->get_temp_filepath("ChunkedReadDictTranscode.parquet"); + write_parquet(input_tbl, filepath); + + auto const read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .output_dict_columns(true) + .build(); + // Small byte limit so the read is split across multiple output chunks. + auto reader = cudf::io::chunked_parquet_reader(/*chunk_read_limit=*/16 * 1024, read_opts); + + std::vector> decoded_chunks; + cudf::size_type total_rows = 0; + int num_chunks = 0; + while (reader.has_next()) { + auto chunk = reader.read_chunk(); + ASSERT_EQ(chunk.tbl->num_columns(), 1); + auto const read_col = chunk.tbl->view().column(0); + if (read_col.size() == 0) { continue; } + ASSERT_EQ(read_col.type().id(), cudf::type_id::DICTIONARY32); + decoded_chunks.push_back(cudf::dictionary::decode(cudf::dictionary_column_view(read_col))); + total_rows += read_col.size(); + ++num_chunks; + } + ASSERT_EQ(total_rows, num_rows); + EXPECT_GT(num_chunks, 1) << "byte limit should split the read into multiple chunks"; + + std::vector views; + views.reserve(decoded_chunks.size()); + for (auto const& c : decoded_chunks) { + views.push_back(c->view()); + } + auto const combined = cudf::concatenate(views); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(input_col, combined->view()); +} + +// Non-happy test (Fast path ineligible): a multi-column table mixing eligible and ineligible +// columns. Only the flat string column is transcoded to DICTIONARY32; the LIST column stays +// LIST (flat-only transcode) and the INT32 column is untouched (non-string). This also exercises +// the output-buffer indexing when an eligible column is preceded/followed by columns of differing +// nesting. +TEST_F(ParquetReaderDictTest, MultiColumnMixedEligibility) +{ + auto str_col = make_low_cardinality_strings(); // eligible -> DICTIONARY32 + auto list_col = make_low_cardinality_lists_of_strings(); // ineligible -> LIST + auto key_col = make_int_key_column(); // non-string -> INT32 + + auto const input_tbl = cudf::table_view{{str_col, list_col->view(), key_col}}; + auto const filepath = temp_env->get_temp_filepath("MultiColumnMixedEligibility.parquet"); + write_parquet(input_tbl, filepath); + + auto const read_table = read_parquet_as_dict(filepath).tbl; + ASSERT_EQ(read_table->num_rows(), num_rows); + ASSERT_EQ(read_table->num_columns(), 3); + + // Flat string column: transcoded to DICTIONARY32. + auto const read_str = read_table->view().column(0); + ASSERT_EQ(read_str.type().id(), cudf::type_id::DICTIONARY32); + auto const decoded_str = cudf::dictionary::decode(cudf::dictionary_column_view(read_str)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(str_col, decoded_str->view()); + + // List column: not eligible, remains LIST. + auto const read_list = read_table->view().column(1); + ASSERT_EQ(read_list.type().id(), cudf::type_id::LIST) + << "List must remain LIST when output_dict_columns is on (transcode is flat-only)"; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(list_col->view(), read_list); + + // Non-string column: unchanged INT32. + auto const read_key = read_table->view().column(2); + ASSERT_EQ(read_key.type().id(), cudf::type_id::INT32); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(key_col, read_key); +} From a435cda218bbaf27b48437bbaa0c5bf4d83aa7af Mon Sep 17 00:00:00 2001 From: ykiran Date: Wed, 5 Aug 2026 22:21:54 -0700 Subject: [PATCH 42/42] Bug fix --- cpp/src/io/parquet/decode_fixed.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/io/parquet/decode_fixed.cu b/cpp/src/io/parquet/decode_fixed.cu index 0ac65d070bd7..a46c17ce26f1 100644 --- a/cpp/src/io/parquet/decode_fixed.cu +++ b/cpp/src/io/parquet/decode_fixed.cu @@ -133,7 +133,7 @@ __device__ void decode_dict_indices_as_int32( }(); auto* dst = reinterpret_cast(data_out) + dst_pos; - auto const num_keys = static_cast(s->dict_size / sizeof(string_index_pair)); + auto const num_keys = static_cast(s->stream.dict_size / sizeof(string_index_pair)); auto const idx = sb->dict_idx[rolling_index(src_pos)]; if (idx >= num_keys) { s->set_error_code(decode_error::DATA_STREAM_OVERRUN);