diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 05b2ed621909..622adfbd8c0e 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -67,11 +67,14 @@ class parquet_reader_options_builder; class parquet_reader_options { source_info _source; + // Column selection options. At most one of these may be set at a time. + // Path in schema of column names to read; `nullopt` is all std::optional> _column_names; - // Indices of top-level columns to read; `nullopt` is all (cannot be used alongside - // `_column_names`) + // Indices of top-level columns to read; `nullopt` is all std::optional> _column_indices; + // Parquet field IDs of columns/fields to read; `nullopt` is all + std::optional> _column_field_ids; // List of individual row groups to read (ignored if empty) std::vector> _row_groups; @@ -258,6 +261,13 @@ class parquet_reader_options { */ [[nodiscard]] auto const& get_column_indices() const { return _column_indices; } + /** + * @brief Returns Parquet field IDs of columns/fields to be read, if set. + * + * @return Parquet field IDs of columns/fields to be read; `nullopt` if the option is not set + */ + [[nodiscard]] auto const& get_column_field_ids() const { return _column_field_ids; } + /** * @brief Returns list of individual row groups to be read. * @@ -386,6 +396,8 @@ class parquet_reader_options { { CUDF_EXPECTS(not _column_indices.has_value(), "Cannot select columns by indices and names simultaneously"); + CUDF_EXPECTS(not _column_field_ids.has_value(), + "Cannot select columns by field IDs and names simultaneously"); _column_names = std::move(column_names); } @@ -404,9 +416,26 @@ class parquet_reader_options { { CUDF_EXPECTS(not _column_names.has_value(), "Cannot select columns by indices and names simultaneously"); + CUDF_EXPECTS(not _column_field_ids.has_value(), + "Cannot select columns by field IDs and indices simultaneously"); _column_indices = std::move(col_indices); } + /** + * @brief Sets the Parquet field IDs of columns/fields to be read from all input sources. + * + * @param column_field_ids A vector of Parquet field IDs to attempt to read from each input + * source. + */ + void set_column_field_ids(std::vector column_field_ids) + { + CUDF_EXPECTS(not _column_names.has_value(), + "Cannot select columns by field IDs and names simultaneously"); + CUDF_EXPECTS(not _column_indices.has_value(), + "Cannot select columns by field IDs and indices simultaneously"); + _column_field_ids = std::move(column_field_ids); + } + /** * @brief Specifies which row groups to read from each input source. * @@ -647,6 +676,19 @@ class parquet_reader_options_builder { return *this; } + /** + * @brief Sets the Parquet field IDs of columns/fields to be read from all input sources. + * + * @param column_field_ids A vector of Parquet field IDs to attempt to read from each input + * source. + * @return this for chaining + */ + parquet_reader_options_builder& column_field_ids(std::vector column_field_ids) + { + options.set_column_field_ids(std::move(column_field_ids)); + return *this; + } + /** * @copydoc parquet_reader_options::set_row_groups * @return this for chaining diff --git a/cpp/src/io/parquet/column_path_helpers.cpp b/cpp/src/io/parquet/column_path_helpers.cpp index 240a06ff103e..58d53d9031b8 100644 --- a/cpp/src/io/parquet/column_path_helpers.cpp +++ b/cpp/src/io/parquet/column_path_helpers.cpp @@ -1,19 +1,37 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "column_path_helpers.hpp" +#include + #include #include #include #include +#include #include #include +#include +#include namespace cudf::io::parquet::detail { +std::string column_path_from_index(std::span schema_tree, int schema_idx) +{ + std::vector path; + for (auto idx = schema_idx; idx > 0; idx = schema_tree[idx].parent_idx) { + path.push_back(schema_tree[idx].name); + } + + return std::accumulate( + path.rbegin() + 1, path.rend(), path.back(), [](auto path_so_far, auto const& elem_name) { + return std::move(path_so_far) + "." + elem_name; + }); +} + std::string normalize_column_path(std::string_view col_path, bool case_sensitive_names) { if (case_sensitive_names) { return std::string{col_path}; } diff --git a/cpp/src/io/parquet/column_path_helpers.hpp b/cpp/src/io/parquet/column_path_helpers.hpp index 418721cea0e2..d78365f4fee8 100644 --- a/cpp/src/io/parquet/column_path_helpers.hpp +++ b/cpp/src/io/parquet/column_path_helpers.hpp @@ -1,11 +1,14 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include + #include +#include #include #include #include @@ -13,6 +16,16 @@ namespace cudf::io::parquet::detail { +/** + * @brief Gets the dot-separated path for a schema element. + * + * @param schema_tree The schema tree describing the file structure + * @param schema_idx Index of the schema element + * @return Dot-separated schema path from the root child to the schema element + */ +[[nodiscard]] std::string column_path_from_index(std::span schema_tree, + int schema_idx); + /** * @brief Returns a normalized (lowercased) column name or path when case-insensitive matching is * enabled diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp index 834eacd6494a..d407797166be 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp @@ -266,24 +266,12 @@ std::tuple, aggregate_reader_metadata::select_payload_columns( std::optional> const& payload_column_names, std::optional> const& filter_column_names, - bool include_index, - bool strings_to_categorical, - bool ignore_missing_columns, - type_id timestamp_type_id, - type_id decimal_type_id, - bool case_sensitive_names) + parquet::detail::column_selection_options const& selection_options) { // If neither payload nor filter columns are specified, select all columns if (not payload_column_names.has_value() and not filter_column_names.has_value()) { // Call the base `select_columns()` method without specifying any columns - return select_columns({}, - {}, - include_index, - strings_to_categorical, - ignore_missing_columns, - timestamp_type_id, - decimal_type_id, - case_sensitive_names); + return select_columns({}, {}, selection_options); } std::vector valid_payload_columns; @@ -302,7 +290,7 @@ aggregate_reader_metadata::select_payload_columns( // Remove filter columns from the provided payload column names if (filter_column_names.has_value() and not filter_column_names->empty()) { auto const filter_columns_set = - construct_filter_columns_set(*filter_column_names, case_sensitive_names); + construct_filter_columns_set(*filter_column_names, selection_options.case_sensitive_names); // Remove a payload column name if it is also present in the hash set valid_payload_columns.erase( std::remove_if(valid_payload_columns.begin(), @@ -311,20 +299,13 @@ aggregate_reader_metadata::select_payload_columns( valid_payload_columns.end()); } // Call the base `select_columns()` method with valid payload columns - return select_columns(valid_payload_columns, - {}, - include_index, - strings_to_categorical, - ignore_missing_columns, - timestamp_type_id, - decimal_type_id, - case_sensitive_names); + return select_columns(valid_payload_columns, {}, selection_options); } // Else if only filter columns are specified, select all columns that do not appear in the // filter expression auto const filter_columns_set = - construct_filter_columns_set(*filter_column_names, case_sensitive_names); + construct_filter_columns_set(*filter_column_names, selection_options.case_sensitive_names); std::function add_column_path = [&](std::string path_till_now, int schema_idx) { @@ -341,14 +322,7 @@ aggregate_reader_metadata::select_payload_columns( } } - return select_columns(valid_payload_columns, - {}, - include_index, - strings_to_categorical, - ignore_missing_columns, - timestamp_type_id, - decimal_type_id, - case_sensitive_names); + return select_columns(valid_payload_columns, {}, selection_options); } std::vector> diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp index 07f8bc5ecad0..8d5312601d3d 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp @@ -158,12 +158,7 @@ class aggregate_reader_metadata : public aggregate_reader_metadata_base { * * @param payload_column_names List of paths of select payload column names, if any * @param filter_column_names List of paths of column names present only in filter, if any - * @param include_index Whether to always include the PANDAS index column(s) - * @param strings_to_categorical Type conversion parameter - * @param ignore_missing_columns Whether to ignore non-existent columns - * @param timestamp_type_id Type conversion parameter - * @param decimal_type_id Type conversion parameter - * @param case_sensitive_names Boolean indicating if column names are case sensitive + * @param selection_options Bundled column selection options * * @return input column information, output column buffers, list of output column schema * indices @@ -172,12 +167,7 @@ class aggregate_reader_metadata : public aggregate_reader_metadata_base { tuple, std::vector, std::vector> select_payload_columns(std::optional> const& payload_column_names, std::optional> const& filter_column_names, - bool include_index, - bool strings_to_categorical, - bool ignore_missing_columns, - type_id timestamp_type_id, - type_id decimal_type_id, - bool case_sensitive_names); + parquet::detail::column_selection_options const& selection_options); /** * @brief Filters row groups such that only the row groups that start within the byte range diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp b/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp index 08341df6c314..ee82579ad0b6 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp @@ -99,9 +99,7 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl( parquet_reader_options const& options) { _metadata = std::make_unique( - footer_bytes, - options.is_enabled_use_arrow_schema(), - options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas()); + footer_bytes, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_sources(options)); _extended_metadata = static_cast(_metadata.get()); } @@ -109,10 +107,10 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl( hybrid_scan_reader_impl::hybrid_scan_reader_impl( cudf::host_span parquet_metadatas, parquet_reader_options const& options) { - _metadata = std::make_unique( - parquet_metadatas, - options.is_enabled_use_arrow_schema(), - options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas()); + _metadata = + std::make_unique(parquet_metadatas, + options.is_enabled_use_arrow_schema(), + has_cols_from_mismatched_sources(options)); _extended_metadata = static_cast(_metadata.get()); } @@ -135,6 +133,12 @@ void hybrid_scan_reader_impl::setup_page_indexes( void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode, parquet_reader_options const& options) { + // Initialize reader configuration. + initialize_reader_config(options); + + // Build column selection options directly from the user options. + auto selection_options = make_column_selection_options(options); + if (read_columns_mode == read_columns_mode::ALL_COLUMNS) { if (_is_all_columns_selected) { return; } @@ -142,9 +146,6 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode auto const select_column_names = get_column_projection(options, options.is_enabled_ignore_missing_columns()); - // Initialize column selection related options - initialize_column_selection_options(options); - // Select only columns required by the options and filter. // Using as is from: // https://github.com/rapidsai/cudf/blob/a8b25cd205dc5d04b9918dcb0b3abd6b8c4e4a74/cpp/src/io/parquet/reader_impl.cpp#L556-L569 @@ -155,39 +156,21 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode _num_filter_only_columns = filter_only_columns_names->size(); } std::tie(_input_columns, _output_buffers, _output_column_schemas) = - _metadata->select_columns(select_column_names, - filter_only_columns_names, - options.is_enabled_use_pandas_metadata(), - _strings_to_categorical, - options.is_enabled_ignore_missing_columns(), - _options.timestamp_type.id(), - _options.decimal_width, - _options.case_sensitive_names); + _metadata->select_columns(select_column_names, filter_only_columns_names, selection_options); _is_all_columns_selected = true; _is_filter_columns_selected = false; _is_payload_columns_selected = false; } else if (read_columns_mode == read_columns_mode::FILTER_COLUMNS) { if (_is_filter_columns_selected) { return; } - // Must not ignore missing filter columns - auto constexpr ignore_missing_columns = false; - - // Initialize column selection related options - initialize_column_selection_options(options); + selection_options.ignore_missing_columns = false; _filter_columns_names = cudf::io::parquet::detail::get_column_names_in_expression( options.get_filter(), {}, options, _extended_metadata->get_schema_tree()); // Select only filter columns using the base `select_columns` method std::tie(_input_columns, _output_buffers, _output_column_schemas) = - _extended_metadata->select_columns(_filter_columns_names, - {}, - _use_pandas_metadata, - _strings_to_categorical, - ignore_missing_columns, - _options.timestamp_type.id(), - _options.decimal_width, - _options.case_sensitive_names); + _extended_metadata->select_columns(_filter_columns_names, {}, selection_options); _is_filter_columns_selected = true; _is_payload_columns_selected = false; @@ -195,20 +178,11 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode } else { if (_is_payload_columns_selected) { return; } - // Initialize column selection related options - initialize_column_selection_options(options); - auto select_column_names = get_column_projection(options, options.is_enabled_ignore_missing_columns()); std::tie(_input_columns, _output_buffers, _output_column_schemas) = - _extended_metadata->select_payload_columns(select_column_names, - _filter_columns_names, - _use_pandas_metadata, - _strings_to_categorical, - options.is_enabled_ignore_missing_columns(), - _options.timestamp_type.id(), - _options.decimal_width, - _options.case_sensitive_names); + _extended_metadata->select_payload_columns( + select_column_names, _filter_columns_names, selection_options); _is_payload_columns_selected = true; _is_filter_columns_selected = false; @@ -909,8 +883,7 @@ void hybrid_scan_reader_impl::reset_internal_state() _mr = cudf::get_current_device_resource_ref(); } -void hybrid_scan_reader_impl::initialize_column_selection_options( - parquet_reader_options const& options) +void hybrid_scan_reader_impl::initialize_reader_config(parquet_reader_options const& options) { // Strings may be returned as either string or categorical columns _strings_to_categorical = options.is_enabled_convert_strings_to_categories(); @@ -919,8 +892,6 @@ void hybrid_scan_reader_impl::initialize_column_selection_options( _options.decimal_width = options.get_decimal_width(); _options.use_jit_filter = options.is_enabled_use_jit_filter(); _options.case_sensitive_names = options.is_enabled_case_sensitive_names(); - - _use_pandas_metadata = options.is_enabled_use_pandas_metadata(); } void hybrid_scan_reader_impl::initialize_options(parquet_reader_options const& options, @@ -928,9 +899,6 @@ void hybrid_scan_reader_impl::initialize_options(parquet_reader_options const& o rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - // Initialize column selection related options - initialize_column_selection_options(options); - // Binary columns can be read as binary or strings _reader_column_schema = options.get_column_schema(); diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp index db9dd9acfe41..6c1c23b4b68c 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp @@ -308,11 +308,11 @@ class hybrid_scan_reader_impl : public parquet::detail::reader_impl { enum class read_columns_mode { FILTER_COLUMNS, PAYLOAD_COLUMNS, ALL_COLUMNS }; /** - * @brief Initialize column selection related options + * @brief Populate the reader's `_options` config (and related members) from the user options. * * @param options Reader options */ - void initialize_column_selection_options(parquet_reader_options const& options); + void initialize_reader_config(parquet_reader_options const& options); /** * @brief Initialize the necessary options related internal variables for use later on @@ -557,8 +557,6 @@ class hybrid_scan_reader_impl : public parquet::detail::reader_impl { cudf::size_type _row_mask_offset{0}; bool _output_chunk_produced{false}; - bool _use_pandas_metadata{false}; - bool _is_filter_columns_selected{false}; bool _is_payload_columns_selected{false}; bool _is_all_columns_selected{false}; diff --git a/cpp/src/io/parquet/expression_transform_helpers.cpp b/cpp/src/io/parquet/expression_transform_helpers.cpp index 013cd9c7e3f6..8c9351cba92a 100644 --- a/cpp/src/io/parquet/expression_transform_helpers.cpp +++ b/cpp/src/io/parquet/expression_transform_helpers.cpp @@ -18,6 +18,10 @@ #include +#include +#include +#include + namespace cudf::io::parquet::detail { namespace { @@ -235,17 +239,14 @@ void names_from_expression::visit_operands( [[nodiscard]] std::unordered_map map_column_indices_to_names( cudf::io::parquet_reader_options const& options, - std::vector const& schema_tree, + std::span schema_tree, bool case_sensitive_names) { std::unordered_map column_indices_to_names; - auto const& selected_column_names = options.get_column_names(); - auto const& selected_column_indices = options.get_column_indices(); - - CUDF_EXPECTS( - not(selected_column_names.has_value() and selected_column_indices.has_value()), - "Parquet reader encountered column selection by both names and indices simultaneously"); + auto const& selected_column_names = options.get_column_names(); + auto const& selected_column_indices = options.get_column_indices(); + auto const& selected_column_field_ids = options.get_column_field_ids(); // Map counting indices to the selected column by names if (selected_column_names.has_value()) { @@ -257,32 +258,60 @@ void names_from_expression::visit_operands( return std::make_pair(col_index, normalize_column_path(col_name, case_sensitive_names)); }); - } else { - // Map selected top-level column indices to their names from the schema tree + } + // Map selected top-level column indices to their names from the schema tree + else if (selected_column_indices.has_value()) { auto const& root = schema_tree.front(); - if (selected_column_indices.has_value()) { - std::transform(selected_column_indices->begin(), - selected_column_indices->end(), - cuda::counting_iterator{0}, - std::inserter(column_indices_to_names, column_indices_to_names.end()), - [&](auto selected_col_idx, auto const mapped_col_idx) { - auto const schema_idx = root.children_idx[selected_col_idx]; - return std::make_pair( - mapped_col_idx, - normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)); - }); - } else { - // Map all top-level column indices to their names from the schema tree - std::for_each( - cuda::counting_iterator{0}, - cuda::counting_iterator{static_cast(root.children_idx.size())}, - [&](auto col_idx) { - auto const schema_idx = root.children_idx[col_idx]; - column_indices_to_names.insert( - {col_idx, normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)}); - }); - } + std::transform( + selected_column_indices->begin(), + selected_column_indices->end(), + cuda::counting_iterator{0}, + std::inserter(column_indices_to_names, column_indices_to_names.end()), + [&](auto selected_col_idx, auto const mapped_col_idx) { + CUDF_EXPECTS( + selected_col_idx >= 0 and std::cmp_less(selected_col_idx, root.children_idx.size()), + "Encountered an invalid col index in the top-level column selection", + std::invalid_argument); + auto const schema_idx = root.children_idx[selected_col_idx]; + return std::make_pair( + mapped_col_idx, + normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)); + }); + } + // Map selected field ids to column paths from the schema tree + else if (selected_column_field_ids.has_value()) { + std::transform( + selected_column_field_ids->begin(), + selected_column_field_ids->end(), + cuda::counting_iterator{0}, + std::inserter(column_indices_to_names, column_indices_to_names.end()), + [&](auto const& field_id, auto const mapped_col_idx) { + auto const schema_iter = + std::find_if(schema_tree.begin() + 1, schema_tree.end(), [field_id](auto const& schema) { + return schema.field_id.has_value() and schema.field_id.value() == field_id; + }); + CUDF_EXPECTS(schema_iter != schema_tree.end(), + "Encountered a non-existent Parquet field ID in selected columns", + std::invalid_argument); + auto const schema_idx = static_cast(std::distance(schema_tree.begin(), schema_iter)); + return std::make_pair(mapped_col_idx, + normalize_column_path(column_path_from_index(schema_tree, schema_idx), + case_sensitive_names)); + }); + } + // Map all top-level column indices to their names from the schema tree + else { + auto const& root = schema_tree.front(); + + std::for_each( + cuda::counting_iterator{0}, + cuda::counting_iterator{static_cast(root.children_idx.size())}, + [&](auto col_idx) { + auto const schema_idx = root.children_idx[col_idx]; + column_indices_to_names.insert( + {col_idx, normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)}); + }); } return column_indices_to_names; diff --git a/cpp/src/io/parquet/expression_transform_helpers.hpp b/cpp/src/io/parquet/expression_transform_helpers.hpp index 82cf0fe02eb9..3762c174c941 100644 --- a/cpp/src/io/parquet/expression_transform_helpers.hpp +++ b/cpp/src/io/parquet/expression_transform_helpers.hpp @@ -351,7 +351,7 @@ class offset_column_references : public named_to_reference_converter { */ [[nodiscard]] std::unordered_map map_column_indices_to_names( cudf::io::parquet_reader_options const& options, - std::vector const& schema_tree, + std::span schema_tree, bool case_sensitive_names); /** diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 1c686bcc7067..d9ad3c6d2423 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -5,6 +5,7 @@ #include "reader_impl.hpp" +#include "column_path_helpers.hpp" #include "error.hpp" #include "runtime/context.hpp" @@ -522,16 +523,15 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, // Open and parse the source dataset metadata CUDF_EXPECTS(file_metadatas.empty() or file_metadatas.size() == _sources.size(), "Encountered a mismatch in the number of provided data sources and metadatas"); + _metadata = file_metadatas.empty() ? std::make_unique( _sources, options.is_enabled_use_arrow_schema(), - options.get_column_names().has_value() and - options.is_enabled_allow_mismatched_pq_schemas()) + has_cols_from_mismatched_sources(options)) : std::make_unique( std::forward>(file_metadatas), options.is_enabled_use_arrow_schema(), - options.get_column_names().has_value() and - options.is_enabled_allow_mismatched_pq_schemas()); + has_cols_from_mismatched_sources(options)); // Number of input sources _num_sources = _sources.size(); @@ -547,23 +547,18 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, get_column_projection(options, options.is_enabled_ignore_missing_columns()); std::optional> filter_only_columns_names; - if (options.get_filter().has_value() and - (options.get_column_names().has_value() or options.get_column_indices().has_value())) { + auto const has_column_selection = options.get_column_names().has_value() or + options.get_column_indices().has_value() or + options.get_column_field_ids().has_value(); + if (options.get_filter().has_value() and has_column_selection) { // list, struct, dictionary are not supported by AST filter yet. // extract columns not present in get_column_names() & keep count to remove at end. filter_only_columns_names = get_column_names_in_expression( options.get_filter(), *select_column_names, options, _metadata->get_schema_tree()); _num_filter_only_columns = filter_only_columns_names->size(); } - std::tie(_input_columns, _output_buffers, _output_column_schemas) = - _metadata->select_columns(select_column_names, - filter_only_columns_names, - options.is_enabled_use_pandas_metadata(), - _strings_to_categorical, - options.is_enabled_ignore_missing_columns(), - _options.timestamp_type.id(), - _options.decimal_width, - _options.case_sensitive_names); + std::tie(_input_columns, _output_buffers, _output_column_schemas) = _metadata->select_columns( + select_column_names, filter_only_columns_names, make_column_selection_options(options)); // Save the states of the output buffers for reuse in `chunk_read()`. std::transform( @@ -811,22 +806,16 @@ std::vector reader_impl::calculate_output_num_rows_per_source(size_t con std::optional> reader_impl::get_column_projection( parquet_reader_options const& options, bool ignore_missing_columns) const { - auto const has_column_names = options.get_column_names().has_value(); - auto const has_column_indices = options.get_column_indices().has_value(); - - CUDF_EXPECTS( - not(has_column_names and has_column_indices), - "Parquet reader encountered column selection by both names and indices simultaneously"); - - // No column selection specified. Return nullopt indicating all columns to be selected - if (not has_column_names and not has_column_indices) { - return std::nullopt; - } else if (has_column_names) { - return options.get_column_names(); - } else { + auto const has_column_names = options.get_column_names().has_value(); + auto const has_column_indices = options.get_column_indices().has_value(); + auto const has_column_field_ids = options.get_column_field_ids().has_value(); + + if (has_column_names) { return options.get_column_names(); } + + if (has_column_indices) { std::vector col_names; auto const& top_level_schema_indices = _metadata->get_schema(0).children_idx; - for (auto const index : options.get_column_indices().value_or(std::vector{})) { + for (auto const index : options.get_column_indices().value()) { auto const is_valid_index = std::cmp_greater_equal(index, 0) and std::cmp_less(index, top_level_schema_indices.size()); CUDF_EXPECTS(ignore_missing_columns or is_valid_index, @@ -837,6 +826,28 @@ std::optional> reader_impl::get_column_projection( } return std::make_optional(std::move(col_names)); } + + if (has_column_field_ids) { + std::vector col_names; + auto const& schema_tree = _metadata->get_schema_tree(); + for (auto const field_id : options.get_column_field_ids().value()) { + auto const schema_iter = + std::find_if(schema_tree.cbegin() + 1, schema_tree.cend(), [field_id](auto const& schema) { + return schema.field_id.has_value() and schema.field_id.value() == field_id; + }); + CUDF_EXPECTS(ignore_missing_columns or schema_iter != schema_tree.end(), + "Encountered a non-existent Parquet field ID in selected columns", + std::invalid_argument); + if (schema_iter != schema_tree.end()) { + auto const schema_idx = static_cast(std::distance(schema_tree.cbegin(), schema_iter)); + col_names.emplace_back(column_path_from_index(schema_tree, schema_idx)); + } + } + return std::make_optional(std::move(col_names)); + } + + // No column selection specified. Return nullopt indicating all columns to be selected. + return std::nullopt; } void reader_impl::apply_decimal_width_cast(std::vector>& out_columns) @@ -853,6 +864,31 @@ void reader_impl::apply_decimal_width_cast(std::vector>& } } +column_selection_options reader_impl::make_column_selection_options( + parquet_reader_options const& options) const +{ + auto const selection_mode = [&]() { + if (options.get_column_names().has_value()) { + return column_selection_mode::BY_NAME; + } else if (options.get_column_indices().has_value()) { + return column_selection_mode::BY_INDEX; + } else if (options.get_column_field_ids().has_value()) { + return column_selection_mode::BY_FIELD_ID; + } else { + return column_selection_mode::NONE; + } + }(); + + return column_selection_options{ + .selection_mode = selection_mode, + .include_index = options.is_enabled_use_pandas_metadata(), + .strings_to_categorical = options.is_enabled_convert_strings_to_categories(), + .ignore_missing_columns = options.is_enabled_ignore_missing_columns(), + .timestamp_type_id = options.get_timestamp_type().id(), + .decimal_type_id = options.get_decimal_width(), + .case_sensitive_names = options.is_enabled_case_sensitive_names()}; +} + table_with_metadata reader_impl::finalize_output(read_mode mode, table_metadata& out_metadata, std::vector>& out_columns) diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 666413f95f03..41dc71c24071 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -215,6 +215,18 @@ class reader_impl { */ std::pair> read_column_chunks(); + /** + * @brief Build the `column_selection_options` bundle for `select_columns()`. + * + * Type-conversion and case-sensitivity settings are read from the cached `_options` (which must + * already be populated); selection-only settings are taken from @p options. + * + * @param options Reader options + * @return Column selection options + */ + [[nodiscard]] column_selection_options make_column_selection_options( + parquet_reader_options const& options) const; + /** * @brief Read compressed data and page information for the current pass. */ @@ -363,12 +375,28 @@ class reader_impl { */ void compute_output_chunks_for_subpass(); + /** + * @brief Check if there is more work to be done + */ [[nodiscard]] bool has_more_work() const { return _file_itm_data.num_passes() > 0 && _file_itm_data._current_input_pass < _file_itm_data.num_passes(); } + /** + * @brief Check if the user has specified columns from mismatched sources + * + * @param options Reader options + * @return True if the user has specified columns from mismatched sources + */ + [[nodiscard]] bool has_cols_from_mismatched_sources(parquet_reader_options const& options) + { + return (options.get_column_names().has_value() or + options.get_column_field_ids().has_value()) and + options.is_enabled_allow_mismatched_pq_schemas(); + } + protected: /** * @brief Check if the user has specified custom row bounds diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index b34c79fcd395..570ccd9a097e 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -113,6 +114,118 @@ cuda::std::optional converted_to_logical_type(SchemaElement const& return cuda::std::nullopt; } +/** + * @brief Lookup schema children by name or field ID across multiple sources. + */ +struct schema_child_lookup { + using get_schema_fn = std::function; + + schema_child_lookup(get_schema_fn&& schema_fn, + bool case_sensitive_names, + column_selection_mode selection_mode) + : schema_fn{std::move(schema_fn)}, + case_sensitive_names{case_sensitive_names}, + match_schema_by_field_id{selection_mode == column_selection_mode::BY_FIELD_ID} + { + } + + /** + * @brief Cache of schema indices of all children of a schema element in the specified source + */ + struct schema_idx_cache { + column_path_map by_name; + std::unordered_map by_field_id; + + explicit schema_idx_cache(get_schema_fn const& schema_fn, + bool case_sensitive_names, + int const schema_idx, + int const src_idx) + : by_name{make_column_path_map( + case_sensitive_names, schema_fn(schema_idx, src_idx).children_idx.size())}, + by_field_id{schema_fn(schema_idx, src_idx).children_idx.size()} + { + // Cache all children schema indices of the specified schema element by name and field ID + auto const& schema_elem = schema_fn(schema_idx, src_idx); + for (auto const child_schema_idx : schema_elem.children_idx) { + auto const& child_schema = schema_fn(child_schema_idx, src_idx); + by_name.emplace(child_schema.name, static_cast(child_schema_idx)); + if (child_schema.field_id.has_value()) { + by_field_id.emplace(child_schema.field_id.value(), + static_cast(child_schema_idx)); + } + } + } + }; + + /** + * @brief Find child schema index by name in the specified source + */ + [[nodiscard]] size_type find_schema_child_by_name(int const schema_idx, + std::string_view name, + int const src_idx = 0) + { + auto const& cache = get_schema_idx_cache(schema_idx, src_idx); + auto const child_iter = cache.by_name.find(name); + return child_iter != cache.by_name.end() ? child_iter->second : -1; + } + + /** + * @brief Find matching child schema index in the target source by field ID or name + */ + [[nodiscard]] size_type find_target_schema_child(int const src_schema_idx, + int const dst_schema_idx, + std::string_view name, + int const src_idx) + { + if (match_schema_by_field_id) { + auto const src_child_idx = find_schema_child_by_name(src_schema_idx, name); + CUDF_EXPECTS(src_child_idx != -1, + "Column name not found in the source schema tree", + std::invalid_argument); + + auto const& src_child = schema_fn(src_child_idx, 0); + CUDF_EXPECTS(src_child.field_id.has_value(), + "Column field ID not found in the source schema tree", + std::invalid_argument); + + return find_schema_child_by_field_id(dst_schema_idx, src_child.field_id.value(), src_idx); + } else { + return find_schema_child_by_name(dst_schema_idx, name, src_idx); + } + } + + private: + /** + * @brief Find child schema index by field ID in the specified source + */ + [[nodiscard]] size_type find_schema_child_by_field_id(int const schema_idx, + int32_t field_id, + int const src_idx = 0) + { + auto const& cache = get_schema_idx_cache(schema_idx, src_idx); + auto const child_iter = cache.by_field_id.find(field_id); + return child_iter != cache.by_field_id.end() ? child_iter->second : -1; + } + + /** + * @brief Get or build cached child lookup maps for the specified schema element + */ + [[nodiscard]] schema_idx_cache const& get_schema_idx_cache(int const schema_idx, + int const src_idx = 0) + { + auto const cache_key = + (static_cast(src_idx) << 32) | static_cast(schema_idx); + auto [cache_iter, inserted] = schema_idx_caches.try_emplace( + cache_key, schema_fn, case_sensitive_names, schema_idx, src_idx); + return cache_iter->second; + } + + get_schema_fn schema_fn; + bool case_sensitive_names; + bool match_schema_by_field_id; + std::unordered_map schema_idx_caches; +}; + } // namespace type_id to_type_id(SchemaElement const& schema, @@ -1188,38 +1301,38 @@ aggregate_reader_metadata::get_column_chunk_metadata() const return column_chunk_metadata; } -bool aggregate_reader_metadata::is_schema_index_mapped(int schema_idx, int pfm_idx) const +bool aggregate_reader_metadata::is_schema_index_mapped(int schema_idx, int src_idx) const { - // Check if schema_idx or pfm_idx is invalid + // Check if schema_idx or src_idx is invalid CUDF_EXPECTS( - schema_idx >= 0 and pfm_idx >= 0 and std::cmp_less(pfm_idx, per_file_metadata.size()), - "Parquet reader encountered an invalid schema_idx or pfm_idx", + schema_idx >= 0 and src_idx >= 0 and std::cmp_less(src_idx, per_file_metadata.size()), + "Parquet reader encountered an invalid schema_idx or src_idx", std::out_of_range); // True if root index requested or zeroth file index or schema_idx maps doesn't exist. (i.e. // schemas are identical). - if (schema_idx == 0 or pfm_idx == 0 or schema_idx_maps.empty()) { return true; } + if (schema_idx == 0 or src_idx == 0 or schema_idx_maps.empty()) { return true; } // Check if mapped - auto const& schema_idx_map = schema_idx_maps[pfm_idx - 1]; + auto const& schema_idx_map = schema_idx_maps[src_idx - 1]; return schema_idx_map.find(schema_idx) != schema_idx_map.end(); } -int aggregate_reader_metadata::map_schema_index(int schema_idx, int pfm_idx) const +int aggregate_reader_metadata::map_schema_index(int schema_idx, int src_idx) const { - // Check if schema_idx or pfm_idx is invalid + // Check if schema_idx or src_idx is invalid CUDF_EXPECTS( - schema_idx >= 0 and pfm_idx >= 0 and std::cmp_less(pfm_idx, per_file_metadata.size()), - "Parquet reader encountered an invalid schema_idx or pfm_idx", + schema_idx >= 0 and src_idx >= 0 and std::cmp_less(src_idx, per_file_metadata.size()), + "Parquet reader encountered an invalid schema_idx or src_idx", std::out_of_range); - // Check if pfm_idx is zero or root index requested or schema_idx_maps doesn't exist (i.e. + // Check if src_idx is zero or root index requested or schema_idx_maps doesn't exist (i.e. // schemas are identical). - if (schema_idx == 0 or pfm_idx == 0 or schema_idx_maps.empty()) { return schema_idx; } + if (schema_idx == 0 or src_idx == 0 or schema_idx_maps.empty()) { return schema_idx; } // schema_idx_maps will only have > 0 size when we are reading matching column projection from // mismatched Parquet sources. - auto const& schema_idx_map = schema_idx_maps[pfm_idx - 1]; + auto const& schema_idx_map = schema_idx_maps[src_idx - 1]; CUDF_EXPECTS(schema_idx_map.find(schema_idx) != schema_idx_map.end(), "Unmapped schema index encountered in the specified source tree", std::out_of_range); @@ -1697,27 +1810,23 @@ std::tuple, aggregate_reader_metadata::select_columns( std::optional> const& use_names, std::optional> const& filter_columns_names, - bool include_index, - bool strings_to_categorical, - bool ignore_missing_columns, - type_id timestamp_type_id, - type_id decimal_type_id, - bool case_sensitive_names) + column_selection_options const& selection_options) { - auto const find_schema_child = - [&](SchemaElement const& schema_elem, std::string_view name, int const pfm_idx = 0) { - auto const& col_schema_idx = - std::find_if(schema_elem.children_idx.cbegin(), - schema_elem.children_idx.cend(), - [&](size_t col_schema_idx) { - return are_column_paths_equal( - get_schema(col_schema_idx, pfm_idx).name, name, case_sensitive_names); - }); - - return (col_schema_idx != schema_elem.children_idx.end()) - ? static_cast(*col_schema_idx) - : -1; - }; + auto const include_index = selection_options.include_index; + auto const strings_to_categorical = selection_options.strings_to_categorical; + auto const ignore_missing_columns = selection_options.ignore_missing_columns; + auto const timestamp_type_id = selection_options.timestamp_type_id; + auto const decimal_type_id = selection_options.decimal_type_id; + auto const case_sensitive_names = selection_options.case_sensitive_names; + auto const selection_mode = selection_options.selection_mode; + + // Setup schema lookup helper + auto schema_lookup = + schema_child_lookup{[&](int const schema_idx, int const src_idx) -> SchemaElement const& { + return get_schema(schema_idx, src_idx); + }, + case_sensitive_names, + selection_mode}; std::vector output_columns; std::vector input_columns; @@ -1774,10 +1883,11 @@ aggregate_reader_metadata::select_columns( } } else { for (auto const& idx : col_name_info->children) { - path_is_valid |= build_column(&idx, - find_schema_child(schema_elem, idx.name), - output_col.children, - has_list_parent || col_type == type_id::LIST); + path_is_valid |= + build_column(&idx, + schema_lookup.find_schema_child_by_name(schema_idx, idx.name), + output_col.children, + has_list_parent || col_type == type_id::LIST); } } @@ -1830,23 +1940,30 @@ aggregate_reader_metadata::select_columns( }; // Compares two schema elements to be equal except their number of children - auto const equal_to_except_num_children = [](SchemaElement const& lhs, SchemaElement const& rhs) { + auto const equal_to_except_num_children = [selection_mode](SchemaElement const& lhs, + SchemaElement const& rhs) { + // Match by field ID if enabled, otherwise match by name + auto const match_schema_by_field_id = selection_mode == column_selection_mode::BY_FIELD_ID; + auto const names_match = + (match_schema_by_field_id and lhs.field_id.has_value() and rhs.field_id.has_value()) + ? lhs.field_id == rhs.field_id + : lhs.name == rhs.name; return lhs.type == rhs.type and lhs.converted_type == rhs.converted_type and - lhs.type_length == rhs.type_length and lhs.name == rhs.name and + lhs.type_length == rhs.type_length and names_match and lhs.decimal_scale == rhs.decimal_scale and lhs.decimal_precision == rhs.decimal_precision and lhs.field_id == rhs.field_id; }; // Maps a projected column's schema_idx in the zeroth per_file_metadata (source) to the - // corresponding schema_idx in pfm_idx'th per_file_metadata (destination). The projected + // corresponding schema_idx in src_idx'th per_file_metadata (destination). The projected // column's path must match across sources, else an appropriate exception is thrown. std::function map_column = [&](column_name_info const* col_name_info, int const src_schema_idx, int const dst_schema_idx, - int const pfm_idx) { + int const src_idx) { auto const& src_schema_elem = get_schema(src_schema_idx); - auto const& dst_schema_elem = get_schema(dst_schema_idx, pfm_idx); + auto const& dst_schema_elem = get_schema(dst_schema_idx, src_idx); // Check the schema elements to be equal except their number of children as we only care about // the specific column paths in the schema trees. Raise an invalid_argument error if the @@ -1857,7 +1974,7 @@ aggregate_reader_metadata::select_columns( std::invalid_argument); // Get the schema_idx_map for this data source (pfm) - auto& schema_idx_map = schema_idx_maps[pfm_idx - 1]; + auto& schema_idx_map = schema_idx_maps[src_idx - 1]; // Map the schema index from 0th tree (src) to the one in the current (dst) tree. schema_idx_map[src_schema_idx] = dst_schema_idx; @@ -1873,7 +1990,7 @@ aggregate_reader_metadata::select_columns( return map_column(child_col_name_info, src_schema_elem.children_idx[0], dst_schema_elem.children_idx[0], - pfm_idx); + src_idx); } // The path ends here. If this is a list/struct col (has children), then map all its children @@ -1886,33 +2003,30 @@ aggregate_reader_metadata::select_columns( "column in the selected path", std::out_of_range); - std::for_each(cuda::counting_iterator{0}, - cuda::counting_iterator{src_schema_elem.num_children}, - [&](auto const child_idx) { - map_column(nullptr, - src_schema_elem.children_idx[child_idx], - dst_schema_elem.children_idx[child_idx], - pfm_idx); - }); + for (auto const& child_idx : src_schema_elem.children_idx) { + auto const dst_child_idx = schema_lookup.find_target_schema_child( + src_schema_idx, dst_schema_idx, get_schema(child_idx).name, src_idx); + CUDF_EXPECTS(dst_child_idx != -1, + "Encountered mismatching schema tree depths across data sources", + std::out_of_range); + map_column(nullptr, child_idx, dst_child_idx, src_idx); + } } // The path goes further down to specific child(ren) of this column so map only those // children. else { - std::for_each( - col_name_info->children.cbegin(), - col_name_info->children.cend(), - [&](auto const& child_col_name_info) { - // Ensure that each named child column exists in the destination schema tree for the - // paths to align up. An out_of_range error otherwise. - CUDF_EXPECTS( - find_schema_child(dst_schema_elem, child_col_name_info.name, pfm_idx) != -1, - "Encountered mismatching schema tree depths across data sources", - std::out_of_range); - map_column(&child_col_name_info, - find_schema_child(src_schema_elem, child_col_name_info.name), - find_schema_child(dst_schema_elem, child_col_name_info.name, pfm_idx), - pfm_idx); - }); + for (auto const& child_col_name_info : col_name_info->children) { + // Ensure that each named child column exists in the destination schema tree for the + // paths to align up. An out_of_range error otherwise. + auto const src_child_idx = + schema_lookup.find_schema_child_by_name(src_schema_idx, child_col_name_info.name); + auto const dst_child_idx = schema_lookup.find_target_schema_child( + src_schema_idx, dst_schema_idx, child_col_name_info.name, src_idx); + CUDF_EXPECTS(dst_child_idx != -1, + "Encountered mismatching schema tree depths across data sources", + std::out_of_range); + map_column(&child_col_name_info, src_child_idx, dst_child_idx, src_idx); + } } }; @@ -2055,28 +2169,28 @@ aggregate_reader_metadata::select_columns( } } for (auto& col : selected_columns) { - auto const& top_level_col_schema_idx = find_schema_child(root, col.name); + auto constexpr root_idx = 0; + auto const& top_level_col_schema_idx = + schema_lookup.find_schema_child_by_name(root_idx, col.name); bool const valid_column = build_column(&col, top_level_col_schema_idx, output_columns, false); if (valid_column) { output_column_schemas.push_back(top_level_col_schema_idx); // Map the column's schema_idx across the rest of the data sources if required. if (per_file_metadata.size() > 1 and not schema_idx_maps.empty()) { - std::for_each(cuda::counting_iterator{static_cast(1)}, - cuda::counting_iterator{per_file_metadata.size()}, - [&](auto const pfm_idx) { - auto const& dst_root = get_schema(0, pfm_idx); - // Ensure that each top level column exists in the destination schema - // tree. An out_of_range error is thrown otherwise. - CUDF_EXPECTS( - find_schema_child(dst_root, col.name, pfm_idx) != -1, - "Encountered mismatching schema tree depths across data sources", - std::out_of_range); - map_column(&col, - top_level_col_schema_idx, - find_schema_child(dst_root, col.name, pfm_idx), - pfm_idx); - }); + std::for_each( + cuda::counting_iterator{static_cast(1)}, + cuda::counting_iterator{per_file_metadata.size()}, + [&](auto const src_idx) { + // Ensure that each top level column exists in the destination schema + // tree. An out_of_range error is thrown otherwise. + auto const dst_col_schema_idx = + schema_lookup.find_target_schema_child(root_idx, root_idx, col.name, src_idx); + CUDF_EXPECTS(dst_col_schema_idx != -1, + "Encountered mismatching schema tree depths across data sources", + std::out_of_range); + map_column(&col, top_level_col_schema_idx, dst_col_schema_idx, src_idx); + }); } } } diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index 631293b3789e..2a8938746562 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -143,6 +143,36 @@ struct surviving_row_group_metrics { std::optional after_bloom_filter; // number of surviving row groups after bloom filter }; +/** + * @brief Column selection mode + */ +enum class column_selection_mode : uint8_t { + NONE = 0, // No column selection + BY_NAME = 1, // Select columns by name + BY_INDEX = 2, // Select columns by top-levelindex + BY_FIELD_ID = 3, // Select columns by field ID +}; + +/** + * @brief Bundle of column selection parameters + */ +struct column_selection_options { + // Column selection mode + column_selection_mode selection_mode = column_selection_mode::NONE; + // Whether to always include the PANDAS index column(s) + bool include_index = false; + // Type conversion parameter: convert strings to categorical columns + bool strings_to_categorical = false; + // Whether to ignore non-existent projected columns + bool ignore_missing_columns = false; + // Type conversion parameter for timestamp columns + type_id timestamp_type_id = type_id::EMPTY; + // Type conversion parameter for decimal columns + type_id decimal_type_id = type_id::EMPTY; + // Whether column name matching is case sensitive + bool case_sensitive_names = true; +}; + class aggregate_reader_metadata { protected: std::vector per_file_metadata; @@ -633,11 +663,7 @@ class aggregate_reader_metadata { * @param use_names List of paths of column names to select; `nullopt` if user did not select * columns to read * @param filter_columns_names List of paths of column names that are present only in filter - * @param include_index Whether to always include the PANDAS index column(s) - * @param strings_to_categorical Type conversion parameter - * @param ignore_missing_columns Whether to ignore non-existent projected columns - * @param timestamp_type_id Type conversion parameter - * @param decimal_type_id Type conversion parameter + * @param selection_options Column selection options * * @return input column information, output column buffers, list of output column schema * indices @@ -647,12 +673,7 @@ class aggregate_reader_metadata { std::vector> select_columns(std::optional> const& use_names, std::optional> const& filter_columns_names, - bool include_index, - bool strings_to_categorical, - bool ignore_missing_columns, - type_id timestamp_type_id, - type_id decimal_type_id, - bool case_sensitive_names); + column_selection_options const& selection_options); }; } // namespace cudf::io::parquet::detail diff --git a/cpp/tests/io/parquet_common.cpp b/cpp/tests/io/parquet_common.cpp index a250379ed061..d10dfd7bd591 100644 --- a/cpp/tests/io/parquet_common.cpp +++ b/cpp/tests/io/parquet_common.cpp @@ -27,12 +27,16 @@ cudf::test::TempDirTestEnvironment* const temp_env = std::string write_parquet_temp_file(cudf::table_view const& tbl, std::string_view const filename, - std::vector const& column_names) + std::vector column_names, + std::vector field_ids) { cudf::io::table_input_metadata md{tbl}; for (std::size_t i = 0; i < column_names.size(); ++i) { md.column_metadata[i].set_name(column_names[i]); } + for (std::size_t i = 0; i < field_ids.size(); ++i) { + md.column_metadata[i].set_parquet_field_id(field_ids[i]); + } auto const path = temp_env->get_temp_filepath(std::string{filename}); cudf::io::parquet_writer_options opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{path}, tbl) diff --git a/cpp/tests/io/parquet_common.hpp b/cpp/tests/io/parquet_common.hpp index 0eca6b0605c0..3f35210706bd 100644 --- a/cpp/tests/io/parquet_common.hpp +++ b/cpp/tests/io/parquet_common.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -36,10 +36,10 @@ extern cudf::test::TempDirTestEnvironment* const temp_env; // Writes `tbl` to a temp Parquet file. If `column_names` is non-empty, it sets the top-level // column names in the file metadata. -[[nodiscard]] std::string write_parquet_temp_file( - cudf::table_view const& tbl, - std::string_view const filename, - std::vector const& column_names = {}); +[[nodiscard]] std::string write_parquet_temp_file(cudf::table_view const& tbl, + std::string_view const filename, + std::vector column_names = {}, + std::vector field_ids = {}); // TODO: Replace with `NumericTypes` when unsigned support is added. Issue #5352 using SupportedTypes = cudf::test::Types; diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 90ec77b33d7b..46a8c4362971 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -379,7 +379,20 @@ TEST_F(ParquetReaderTest, ReorderedColumns) auto d = cudf::test::strings_column_wrapper{"ducks", "sheep", "cows", "fish", "birds", "ants"}; cudf::table_view tbl{{a, b, c, d}}; - auto filepath = write_parquet_temp_file(tbl, "ReorderedColumns3.parquet", {"a", "b", "c", "d"}); + auto filepath = temp_env->get_temp_filepath("ReorderedColumns3.parquet"); + cudf::io::table_input_metadata md(tbl); + md.column_metadata[0].set_name("a"); + md.column_metadata[0].set_parquet_field_id(10); + md.column_metadata[1].set_name("b"); + md.column_metadata[1].set_parquet_field_id(11); + md.column_metadata[2].set_name("c"); + md.column_metadata[2].set_parquet_field_id(12); + md.column_metadata[3].set_name("d"); + md.column_metadata[3].set_parquet_field_id(13); + cudf::io::parquet_writer_options opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, tbl) + .metadata(std::move(md)); + cudf::io::write_parquet(opts); { // read them out of order using indices @@ -394,6 +407,29 @@ TEST_F(ParquetReaderTest, ReorderedColumns) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(3), c); } + { + // read them out of order using Parquet field IDs + cudf::io::parquet_reader_options read_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .column_field_ids({13, 10, 11, 12}); + auto result = cudf::io::read_parquet(read_opts); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0), d); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(1), a); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(2), b); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(3), c); + } + + { + // missing Parquet field IDs are errors when ignore_missing_columns is disabled + cudf::io::parquet_reader_options read_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .column_field_ids({999}) + .ignore_missing_columns(false); + + EXPECT_THROW(cudf::io::read_parquet(read_opts), std::invalid_argument); + } + { // read them out of order cudf::io::parquet_reader_options read_opts = @@ -434,6 +470,46 @@ TEST_F(ParquetReaderTest, ReorderedColumns) } } +TEST_F(ParquetReaderTest, SelectColumnByMissingFieldIds) +{ + auto const a = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto const b = cudf::test::fixed_width_column_wrapper{4, 5, 6}; + cudf::table_view const table{{a, b}}; + auto const filepath = write_parquet_temp_file(table, "SelectColumnByMissingFieldIds.parquet"); + + { + auto const options = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .column_field_ids({1}) + .build(); + auto const result = cudf::io::read_parquet(options); + EXPECT_EQ(result.tbl->num_columns(), 0); + } + + { + auto const options = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .column_field_ids({1}) + .ignore_missing_columns(false) + .build(); + EXPECT_THROW(cudf::io::read_parquet(options), std::invalid_argument); + } +} + +TEST_F(ParquetReaderTest, ColumnSelectionModesAreExclusive) +{ + auto selected_by_name = cudf::io::parquet_reader_options{}; + selected_by_name.set_column_names({}); + EXPECT_THROW(selected_by_name.set_column_indices({}), cudf::logic_error); + EXPECT_THROW(selected_by_name.set_column_field_ids({}), cudf::logic_error); + + auto selected_by_index = cudf::io::parquet_reader_options{}; + selected_by_index.set_column_indices({}); + EXPECT_THROW(selected_by_index.set_column_field_ids({}), cudf::logic_error); + + auto selected_by_field_id = cudf::io::parquet_reader_options{}; + selected_by_field_id.set_column_field_ids({}); + EXPECT_THROW(selected_by_field_id.set_column_names({}), cudf::logic_error); +} + TEST_F(ParquetReaderTest, SelectNestedColumn) { // Structget_temp_filepath("SelectNestedColumn.parquet"); cudf::io::parquet_writer_options args = @@ -475,15 +556,8 @@ TEST_F(ParquetReaderTest, SelectNestedColumn) cudf::io::write_parquet(args); { // Test selecting a single leaf from the table - cudf::io::parquet_reader_options read_args = - cudf::io::parquet_reader_options::builder(cudf::io::source_info(filepath)) - .column_names({"being.particulars.age"}); - auto const result = cudf::io::read_parquet(read_args); - - auto expect_ages_col = cudf::test::fixed_width_column_wrapper{ - {48, 27, 25, 31, 351, 351}, {true, true, true, true, true, false}}; auto expect_s_1 = - cudf::test::structs_column_wrapper{{expect_ages_col}, {true, true, true, true, false, true}}; + cudf::test::structs_column_wrapper{{ages_col}, {true, true, true, true, false, true}}; auto expect_s_2 = cudf::test::structs_column_wrapper{{expect_s_1}, {false, true, true, true, true, true}} .release(); @@ -494,24 +568,24 @@ TEST_F(ParquetReaderTest, SelectNestedColumn) expected_metadata.column_metadata[0].child(0).set_name("particulars"); expected_metadata.column_metadata[0].child(0).child(0).set_name("age"); + cudf::io::parquet_reader_options read_args = + cudf::io::parquet_reader_options::builder(cudf::io::source_info(filepath)) + .column_names({"being.particulars.age"}); + auto result = cudf::io::read_parquet(read_args); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + cudf::test::expect_metadata_equal(expected_metadata, result.metadata); + + // Test selecting a single leaf by Parquet field ID + read_args = cudf::io::parquet_reader_options::builder(cudf::io::source_info(filepath)) + .column_field_ids({5}); + result = cudf::io::read_parquet(read_args); CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); cudf::test::expect_metadata_equal(expected_metadata, result.metadata); } { // Test selecting a non-leaf and expecting all hierarchy from that node onwards - cudf::io::parquet_reader_options read_args = - cudf::io::parquet_reader_options::builder(cudf::io::source_info(filepath)) - .column_names({"being.particulars"}); - auto const result = cudf::io::read_parquet(read_args); - - auto expected_weights_col = - cudf::test::fixed_width_column_wrapper{1.1, 2.4, 5.3, 8.0, 9.6, 6.9}; - - auto expected_ages_col = cudf::test::fixed_width_column_wrapper{ - {48, 27, 25, 31, 351, 351}, {true, true, true, true, true, false}}; - - auto expected_s_1 = cudf::test::structs_column_wrapper{ - {expected_weights_col, expected_ages_col}, {true, true, true, true, false, true}}; + auto expected_s_1 = cudf::test::structs_column_wrapper{{weights_col, ages_col}, + {true, true, true, true, false, true}}; auto expect_s_2 = cudf::test::structs_column_wrapper{{expected_s_1}, {false, true, true, true, true, true}} @@ -524,6 +598,17 @@ TEST_F(ParquetReaderTest, SelectNestedColumn) expected_metadata.column_metadata[0].child(0).child(0).set_name("weight"); expected_metadata.column_metadata[0].child(0).child(1).set_name("age"); + cudf::io::parquet_reader_options read_args = + cudf::io::parquet_reader_options::builder(cudf::io::source_info(filepath)) + .column_names({"being.particulars"}); + auto result = cudf::io::read_parquet(read_args); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + cudf::test::expect_metadata_equal(expected_metadata, result.metadata); + + // Test selecting a non-leaf by Parquet field ID + read_args = cudf::io::parquet_reader_options::builder(cudf::io::source_info(filepath)) + .column_field_ids({3}); + result = cudf::io::read_parquet(read_args); CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); cudf::test::expect_metadata_equal(expected_metadata, result.metadata); } @@ -534,19 +619,10 @@ TEST_F(ParquetReaderTest, SelectNestedColumn) .column_names({"being.particulars.age", "being.particulars.weight", "being.human?"}); auto const result = cudf::io::read_parquet(read_args); - auto expected_weights_col = - cudf::test::fixed_width_column_wrapper{1.1, 2.4, 5.3, 8.0, 9.6, 6.9}; - - auto expected_ages_col = cudf::test::fixed_width_column_wrapper{ - {48, 27, 25, 31, 351, 351}, {true, true, true, true, true, false}}; - - auto expected_is_human_col = cudf::test::fixed_width_column_wrapper{ - {true, true, false, false, false, false}, {true, true, false, true, true, false}}; - - auto expect_s_1 = cudf::test::structs_column_wrapper{{expected_ages_col, expected_weights_col}, + auto expect_s_1 = cudf::test::structs_column_wrapper{{ages_col, weights_col}, {true, true, true, true, false, true}}; - auto expect_s_2 = cudf::test::structs_column_wrapper{{expect_s_1, expected_is_human_col}, + auto expect_s_2 = cudf::test::structs_column_wrapper{{expect_s_1, is_human_col}, {false, true, true, true, true, true}} .release(); @@ -564,6 +640,54 @@ TEST_F(ParquetReaderTest, SelectNestedColumn) } } +TEST_F(ParquetReaderTest, SelectMismatchedStructChildByFieldId) +{ + auto x_a = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto y_a = cudf::test::fixed_width_column_wrapper{10, 20, 30}; + auto struct_a = cudf::test::structs_column_wrapper{{x_a, y_a}, {true, true, true}}.release(); + cudf::table_view const table_a{{*struct_a}}; + + auto path_a = temp_env->get_temp_filepath("SelectNestedFieldIdChildOrderA.parquet"); + cudf::io::table_input_metadata metadata_a(table_a); + metadata_a.column_metadata[0].set_name("record").set_parquet_field_id(1); + metadata_a.column_metadata[0].child(0).set_name("x").set_parquet_field_id(2); + metadata_a.column_metadata[0].child(1).set_name("y").set_parquet_field_id(3); + auto write_args_a = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{path_a}, table_a) + .metadata(std::move(metadata_a)); + cudf::io::write_parquet(write_args_a); + + auto y_b = cudf::test::fixed_width_column_wrapper{40, 50}; + auto x_b = cudf::test::fixed_width_column_wrapper{4, 5}; + auto struct_b = cudf::test::structs_column_wrapper{{y_b, x_b}, {true, true}}.release(); + cudf::table_view const table_b{{*struct_b}}; + + auto path_b = temp_env->get_temp_filepath("SelectNestedFieldIdChildOrderB.parquet"); + cudf::io::table_input_metadata metadata_b(table_b); + metadata_b.column_metadata[0].set_name("record").set_parquet_field_id(1); + metadata_b.column_metadata[0].child(0).set_name("y").set_parquet_field_id(3); + metadata_b.column_metadata[0].child(1).set_name("x").set_parquet_field_id(2); + auto write_args_b = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{path_b}, table_b) + .metadata(std::move(metadata_b)); + cudf::io::write_parquet(write_args_b); + + auto expected_x = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5}; + auto expected_y = cudf::test::fixed_width_column_wrapper{10, 20, 30, 40, 50}; + auto expected_struct = + cudf::test::structs_column_wrapper{{expected_x, expected_y}, {true, true, true, true, true}} + .release(); + cudf::table_view const expected{{*expected_struct}}; + + auto const read_args = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) + .allow_mismatched_pq_schemas(true) + .column_field_ids({1}) + .build(); + auto const result = cudf::io::read_parquet(read_args); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); +} + TEST_F(ParquetReaderTest, DecimalRead) { { @@ -1417,8 +1541,11 @@ auto create_parquet_with_stats(std::string const& filename) cudf::io::table_input_metadata expected_metadata(expected); expected_metadata.column_metadata[0].set_name("col_uint32"); + expected_metadata.column_metadata[0].set_parquet_field_id(10); expected_metadata.column_metadata[1].set_name("col_int64"); + expected_metadata.column_metadata[1].set_parquet_field_id(11); expected_metadata.column_metadata[2].set_name("col_double"); + expected_metadata.column_metadata[2].set_parquet_field_id(12); auto const filepath = temp_env->get_temp_filepath(filename); cudf::io::parquet_writer_options const out_opts = @@ -1459,7 +1586,7 @@ TEST_F(ParquetReaderTest, FilterIdentity) TEST_F(ParquetReaderTest, FilterWithColumnProjection) { - // col_uint32, col_int64, col_double + // col_uint32 (field_id: 10), col_int64 (field_id: 11), col_double (field_id: 12) auto [src, filepath] = create_parquet_with_stats("FilterWithColumnProjection.parquet"); auto val = cudf::numeric_scalar{10}; auto lit = cudf::ast::literal{val}; @@ -1490,6 +1617,14 @@ TEST_F(ParquetReaderTest, FilterWithColumnProjection) .filter(read_expr); result = cudf::io::read_parquet(read_opts); CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, *expected); + + // Repeat but select columns using field IDs instead of names + read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .column_field_ids({12}) + .case_sensitive_names(false) + .filter(read_expr); + result = cudf::io::read_parquet(read_opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, *expected); } { // column_reference in parquet filter (indices as per order of column projection) @@ -1510,6 +1645,12 @@ TEST_F(ParquetReaderTest, FilterWithColumnProjection) .column_indices({2, 0}) .filter(read_ref_expr); CUDF_TEST_EXPECT_TABLES_EQUAL(*(cudf::io::read_parquet(read_opts).tbl), *expected); + + // Repeat but select columns using field IDs instead of names + read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .column_field_ids({12, 10}) + .filter(read_ref_expr); + CUDF_TEST_EXPECT_TABLES_EQUAL(*(cudf::io::read_parquet(read_opts).tbl), *expected); } // Error cases @@ -1528,6 +1669,12 @@ TEST_F(ParquetReaderTest, FilterWithColumnProjection) .column_indices({2, 0}) .filter(read_ref_expr); EXPECT_ANY_THROW(cudf::io::read_parquet(read_opts)); + + // Repeat but select columns using field IDs instead of names + read_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .column_field_ids({12, 10}) + .filter(read_ref_expr); + EXPECT_ANY_THROW(cudf::io::read_parquet(read_opts)); } } } @@ -4886,63 +5033,92 @@ TEST_F(ParquetReaderTest, MismatchedSchemaFilterColumnCollision) auto const price_a = column_wrapper{50.0, 150.0, 75.0}; cudf::table_view const table_a{{id_a, price_a}}; auto const path_a = - write_parquet_temp_file(table_a, "MismatchCollisionA.parquet", {"id", "price"}); + write_parquet_temp_file(table_a, "MismatchCollisionA.parquet", {"id", "price"}, {1, 2}); auto const category_b = column_wrapper{"x", "y", "z"}; auto const id_b = column_wrapper{1000, 1001, 1002}; auto const price_b = column_wrapper{40.0, 200.0, 99.0}; cudf::table_view const table_b{{category_b, id_b, price_b}}; - auto const path_b = - write_parquet_temp_file(table_b, "MismatchCollisionB.parquet", {"category", "id", "price"}); + auto const path_b = write_parquet_temp_file( + table_b, "MismatchCollisionB.parquet", {"category", "id", "price"}, {3, 1, 2}); auto value = cudf::numeric_scalar(100.0); auto lit = cudf::ast::literal(value); auto col = cudf::ast::column_name_reference("price"); auto filter = cudf::ast::operation(cudf::ast::ast_operator::LESS, col, lit); - auto const opts = - cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) - .allow_mismatched_pq_schemas(true) - .column_names({"id", "price"}) - .filter(filter) - .build(); auto const exp_id = column_wrapper{1, 3, 1000, 1002}; auto const exp_price = column_wrapper{50.0, 75.0, 40.0, 99.0}; cudf::table_view const expected{{exp_id, exp_price}}; - auto const result = cudf::io::read_parquet(opts); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + + { + auto const opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) + .allow_mismatched_pq_schemas(true) + .column_names({"id", "price"}) + .filter(filter) + .build(); + auto const result = cudf::io::read_parquet(opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + } + { + auto const opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) + .allow_mismatched_pq_schemas(true) + .column_field_ids({1, 2}) + .filter(filter) + .build(); + auto const result = cudf::io::read_parquet(opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + } } TEST_F(ParquetReaderTest, MismatchedSchemaFilterSameTypeCollisionWrongResult) { - // Same-type collision: source 0's int64 `a` lands on source 1's int64 `b`. + // Same-type collision: source 0's int64 `a` (field_id: 1) lands on source 1's int64 `b` + // (field_id: 2). auto const a_a = column_wrapper{1, 2, 3}; auto const b_a = column_wrapper{7, 8, 9}; cudf::table_view const table_a{{a_a, b_a}}; - auto const path_a = write_parquet_temp_file(table_a, "MismatchSameTypeA.parquet", {"a", "b"}); + auto const path_a = + write_parquet_temp_file(table_a, "MismatchSameTypeA.parquet", {"a", "b"}, {1, 2}); auto const b_b = column_wrapper{1000, 2000, 3000}; // B's `b` lands at a's index auto const a_b = column_wrapper{10, 20, 30}; // B's `a` (all < 100) cudf::table_view const table_b{{b_b, a_b}}; - auto const path_b = write_parquet_temp_file(table_b, "MismatchSameTypeB.parquet", {"b", "a"}); + auto const path_b = + write_parquet_temp_file(table_b, "MismatchSameTypeB.parquet", {"b", "a"}, {2, 1}); auto value = cudf::numeric_scalar(100); auto lit = cudf::ast::literal(value); auto col = cudf::ast::column_name_reference("a"); auto filter = cudf::ast::operation(cudf::ast::ast_operator::LESS, col, lit); - auto const opts = - cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) - .allow_mismatched_pq_schemas(true) - .column_names({"a", "b"}) - .filter(filter) - .build(); // All rows have a < 100, so none should be pruned. auto const exp_a = column_wrapper{1, 2, 3, 10, 20, 30}; auto const exp_b = column_wrapper{7, 8, 9, 1000, 2000, 3000}; cudf::table_view const expected{{exp_a, exp_b}}; - auto const result = cudf::io::read_parquet(opts); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + + { + auto const opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) + .allow_mismatched_pq_schemas(true) + .column_names({"a", "b"}) + .filter(filter) + .build(); + auto const result = cudf::io::read_parquet(opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + } + { + auto const opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) + .allow_mismatched_pq_schemas(true) + .column_field_ids({1, 2}) + .filter(filter) + .build(); + auto const result = cudf::io::read_parquet(opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + } } TEST_F(ParquetReaderTest, MismatchedSchemaFilterOnlyColumnCollision) @@ -4952,30 +5128,43 @@ TEST_F(ParquetReaderTest, MismatchedSchemaFilterOnlyColumnCollision) auto const price_a = column_wrapper{10.0, 200.0, 30.0}; cudf::table_view const table_a{{id_a, price_a}}; auto const path_a = - write_parquet_temp_file(table_a, "MismatchFilterOnlyA.parquet", {"id", "price"}); + write_parquet_temp_file(table_a, "MismatchFilterOnlyA.parquet", {"id", "price"}, {10, 11}); auto const category_b = column_wrapper{"x", "y", "z"}; auto const id_b = column_wrapper{1000, 1001, 1002}; auto const price_b = column_wrapper{40.0, 500.0, 60.0}; cudf::table_view const table_b{{category_b, id_b, price_b}}; - auto const path_b = - write_parquet_temp_file(table_b, "MismatchFilterOnlyB.parquet", {"category", "id", "price"}); + auto const path_b = write_parquet_temp_file( + table_b, "MismatchFilterOnlyB.parquet", {"category", "id", "price"}, {12, 10, 11}); auto value = cudf::numeric_scalar(100.0); auto lit = cudf::ast::literal(value); auto col = cudf::ast::column_name_reference("price"); auto filter = cudf::ast::operation(cudf::ast::ast_operator::LESS, col, lit); - auto const opts = - cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) - .allow_mismatched_pq_schemas(true) - .column_names({"id"}) // `price` is filter-only - .filter(filter) - .build(); auto const exp_id = column_wrapper{1, 3, 1000, 1002}; cudf::table_view const expected{{exp_id}}; - auto const result = cudf::io::read_parquet(opts); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + + { + auto const opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) + .allow_mismatched_pq_schemas(true) + .column_names({"id"}) // `price` is filter-only + .filter(filter) + .build(); + auto const result = cudf::io::read_parquet(opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + } + { + auto const opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{path_a, path_b}}) + .allow_mismatched_pq_schemas(true) + .column_field_ids({10}) // `price` is filter-only + .filter(filter) + .build(); + auto const result = cudf::io::read_parquet(opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + } } TEST_F(ParquetReaderTest, RowIndexColumn)