Repository navigation
Select Parquet columns by field ID #22955
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 4 commits
08f1e36
5781078
4241c2d
4b99d07
43235a3
2d26e65
3cc2e46
5048a45
d51c620
09e8735
b006c62
8d95803
cea7b9f
12eedd9
04a2527
fe2485d
545b82f
3dfb09f
8417b0e
8073240
f920a00
12cf311
1335b4a
e66174a
a18925b
86a4b48
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -66,11 +66,14 @@ class parquet_reader_options_builder; | |
| class parquet_reader_options { | ||
| source_info _source; | ||
|
|
||
| // Column selection options. Only one of these must be set at a time. | ||
|
|
||
| // Path in schema of column names to read; `nullopt` is all | ||
| std::optional<std::vector<std::string>> _column_names; | ||
| // Indices of top-level columns to read; `nullopt` is all (cannot be used alongside | ||
| // `_column_names`) | ||
| // Indices of top-level columns to read; `nullopt` is all | ||
| std::optional<std::vector<cudf::size_type>> _column_indices; | ||
| // Parquet field IDs of columns/fields to read; `nullopt` is all | ||
| std::optional<std::vector<int32_t>> _column_field_ids; | ||
|
|
||
| // List of individual row groups to read (ignored if empty) | ||
| std::vector<std::vector<size_type>> _row_groups; | ||
|
|
@@ -253,6 +256,13 @@ class parquet_reader_options { | |
| */ | ||
| [[nodiscard]] auto const& get_column_indices() const { return _column_indices; } | ||
|
|
||
| /** | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Pretty standard setters and getters |
||
| * @brief Returns Parquet field IDs of columns/fields to be read, if set. | ||
| * | ||
| * @return Parquet field IDs of columns/fields to be read; `nullopt` if the option is not set | ||
| */ | ||
| [[nodiscard]] auto const& get_column_field_ids() const { return _column_field_ids; } | ||
|
|
||
| /** | ||
| * @brief Returns list of individual row groups to be read. | ||
| * | ||
|
|
@@ -357,6 +367,8 @@ class parquet_reader_options { | |
| { | ||
| CUDF_EXPECTS(not _column_indices.has_value(), | ||
| "Cannot select columns by indices and names simultaneously"); | ||
| CUDF_EXPECTS(not _column_field_ids.has_value(), | ||
| "Cannot select columns by field IDs and names simultaneously"); | ||
| _column_names = std::move(column_names); | ||
| } | ||
|
|
||
|
|
@@ -375,9 +387,26 @@ class parquet_reader_options { | |
| { | ||
| CUDF_EXPECTS(not _column_names.has_value(), | ||
| "Cannot select columns by indices and names simultaneously"); | ||
| CUDF_EXPECTS(not _column_field_ids.has_value(), | ||
| "Cannot select columns by field IDs and indices simultaneously"); | ||
| _column_indices = std::move(col_indices); | ||
| } | ||
|
|
||
| /** | ||
| * @brief Sets the Parquet field IDs of columns/fields to be read from all input sources. | ||
| * | ||
| * @param column_field_ids A vector of Parquet field IDs to attempt to read from each input | ||
| * source. | ||
| */ | ||
| void set_column_field_ids(std::vector<int32_t> column_field_ids) | ||
| { | ||
| CUDF_EXPECTS(not _column_names.has_value(), | ||
| "Cannot select columns by field IDs and names simultaneously"); | ||
| CUDF_EXPECTS(not _column_indices.has_value(), | ||
| "Cannot select columns by field IDs and indices simultaneously"); | ||
| _column_field_ids = std::move(column_field_ids); | ||
| } | ||
|
|
||
| /** | ||
| * @brief Specifies which row groups to read from each input source. | ||
| * | ||
|
|
@@ -604,6 +633,19 @@ class parquet_reader_options_builder { | |
| return *this; | ||
| } | ||
|
|
||
| /** | ||
| * @brief Sets the Parquet field IDs of columns/fields to be read from all input sources. | ||
| * | ||
| * @param column_field_ids A vector of Parquet field IDs to attempt to read from each input | ||
| * source. | ||
| * @return this for chaining | ||
| */ | ||
| parquet_reader_options_builder& column_field_ids(std::vector<int32_t> column_field_ids) | ||
| { | ||
| options.set_column_field_ids(std::move(column_field_ids)); | ||
| return *this; | ||
| } | ||
|
|
||
| /** | ||
| * @copydoc parquet_reader_options::set_row_groups | ||
| * @return this for chaining | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,15 +5,33 @@ | |
|
|
||
| #include "column_path_helpers.hpp" | ||
|
|
||
| #include <cudf/io/parquet_schema.hpp> | ||
|
|
||
| #include <algorithm> | ||
| #include <cctype> | ||
| #include <cstddef> | ||
| #include <functional> | ||
| #include <numeric> | ||
| #include <string> | ||
| #include <string_view> | ||
| #include <utility> | ||
| #include <vector> | ||
|
|
||
| namespace cudf::io::parquet::detail { | ||
|
|
||
| std::string column_path_from_index(std::span<SchemaElement const> schema_tree, int schema_idx) | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Given a |
||
| { | ||
| std::vector<std::string> path; | ||
| for (auto idx = schema_idx; idx > 0; idx = schema_tree[idx].parent_idx) { | ||
| path.push_back(schema_tree[idx].name); | ||
| } | ||
|
|
||
| return std::accumulate( | ||
| path.rbegin() + 1, path.rend(), path.back(), [](auto path_so_far, auto const& elem_name) { | ||
| return std::move(path_so_far) + "." + elem_name; | ||
| }); | ||
| } | ||
|
|
||
| std::string normalize_column_path(std::string_view col_path, bool case_sensitive_names) | ||
| { | ||
| if (case_sensitive_names) { return std::string{col_path}; } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -271,7 +271,8 @@ aggregate_reader_metadata::select_payload_columns( | |
| bool ignore_missing_columns, | ||
| type_id timestamp_type_id, | ||
| type_id decimal_type_id, | ||
| bool case_sensitive_names) | ||
| bool case_sensitive_names, | ||
| bool match_schema_by_field_id) | ||
| { | ||
| // If neither payload nor filter columns are specified, select all columns | ||
| if (not payload_column_names.has_value() and not filter_column_names.has_value()) { | ||
|
|
@@ -283,7 +284,8 @@ aggregate_reader_metadata::select_payload_columns( | |
| ignore_missing_columns, | ||
| timestamp_type_id, | ||
| decimal_type_id, | ||
| case_sensitive_names); | ||
| case_sensitive_names, | ||
| match_schema_by_field_id); | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just plumbed in a new boolean field
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. these are mutually exclusive right? should you instead switch to an enum to reduce the number of arguments? the api is experimental after all.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
We are calling the base reader's
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sure but can we update that API also to reduce argument bloat everywhere
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done 3dfb09f |
||
| } | ||
|
|
||
| std::vector<std::string> valid_payload_columns; | ||
|
|
@@ -318,7 +320,8 @@ aggregate_reader_metadata::select_payload_columns( | |
| ignore_missing_columns, | ||
| timestamp_type_id, | ||
| decimal_type_id, | ||
| case_sensitive_names); | ||
| case_sensitive_names, | ||
| match_schema_by_field_id); | ||
| } | ||
|
|
||
| // Else if only filter columns are specified, select all columns that do not appear in the | ||
|
|
@@ -348,7 +351,8 @@ aggregate_reader_metadata::select_payload_columns( | |
| ignore_missing_columns, | ||
| timestamp_type_id, | ||
| decimal_type_id, | ||
| case_sensitive_names); | ||
| case_sensitive_names, | ||
| match_schema_by_field_id); | ||
| } | ||
|
|
||
| std::vector<std::vector<cudf::size_type>> | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -68,21 +68,23 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl( | |
| cudf::host_span<cudf::host_span<uint8_t const> const> footer_bytes, | ||
| parquet_reader_options const& options) | ||
| { | ||
| auto const has_cols_from_mismatched_srcs = | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Separate out this logic: We can have mismatched columns if allowed AND selecting columns by names or field IDs |
||
| (options.get_column_names().has_value() or options.get_column_field_ids().has_value()) and | ||
| options.is_enabled_allow_mismatched_pq_schemas(); | ||
| _metadata = std::make_unique<aggregate_reader_metadata>( | ||
| footer_bytes, | ||
| options.is_enabled_use_arrow_schema(), | ||
| options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas()); | ||
| footer_bytes, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_srcs); | ||
|
|
||
| _extended_metadata = static_cast<aggregate_reader_metadata*>(_metadata.get()); | ||
| } | ||
|
|
||
| hybrid_scan_reader_impl::hybrid_scan_reader_impl( | ||
| cudf::host_span<FileMetaData const> parquet_metadatas, parquet_reader_options const& options) | ||
| { | ||
| auto const has_cols_from_mismatched_srcs = | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same here |
||
| (options.get_column_names().has_value() or options.get_column_field_ids().has_value()) and | ||
| options.is_enabled_allow_mismatched_pq_schemas(); | ||
| _metadata = std::make_unique<aggregate_reader_metadata>( | ||
| parquet_metadatas, | ||
| options.is_enabled_use_arrow_schema(), | ||
| options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas()); | ||
| parquet_metadatas, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_srcs); | ||
| _extended_metadata = static_cast<aggregate_reader_metadata*>(_metadata.get()); | ||
| } | ||
|
|
||
|
|
@@ -132,7 +134,8 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode | |
| options.is_enabled_ignore_missing_columns(), | ||
| _options.timestamp_type.id(), | ||
| _options.decimal_width, | ||
| _options.case_sensitive_names); | ||
| _options.case_sensitive_names, | ||
| options.get_column_field_ids().has_value()); | ||
|
|
||
| _is_all_columns_selected = true; | ||
| _is_filter_columns_selected = false; | ||
|
|
@@ -157,7 +160,8 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode | |
| ignore_missing_columns, | ||
| _options.timestamp_type.id(), | ||
| _options.decimal_width, | ||
| _options.case_sensitive_names); | ||
| _options.case_sensitive_names, | ||
| options.get_column_field_ids().has_value()); | ||
|
|
||
| _is_filter_columns_selected = true; | ||
| _is_payload_columns_selected = false; | ||
|
|
@@ -178,7 +182,8 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode | |
| options.is_enabled_ignore_missing_columns(), | ||
| _options.timestamp_type.id(), | ||
| _options.decimal_width, | ||
| _options.case_sensitive_names); | ||
| _options.case_sensitive_names, | ||
| options.get_column_field_ids().has_value()); | ||
|
|
||
| _is_payload_columns_selected = true; | ||
| _is_filter_columns_selected = false; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,6 +18,10 @@ | |
|
|
||
| #include <cuda/iterator> | ||
|
|
||
| #include <algorithm> | ||
| #include <string> | ||
| #include <utility> | ||
|
|
||
| namespace cudf::io::parquet::detail { | ||
|
|
||
| namespace { | ||
|
|
@@ -235,17 +239,20 @@ void names_from_expression::visit_operands( | |
|
|
||
| [[nodiscard]] std::unordered_map<cudf::size_type, std::string> map_column_indices_to_names( | ||
| cudf::io::parquet_reader_options const& options, | ||
| std::vector<SchemaElement> const& schema_tree, | ||
| std::span<SchemaElement const> schema_tree, | ||
| bool case_sensitive_names) | ||
| { | ||
| std::unordered_map<cudf::size_type, std::string> column_indices_to_names; | ||
|
|
||
| auto const& selected_column_names = options.get_column_names(); | ||
| auto const& selected_column_indices = options.get_column_indices(); | ||
| auto const& selected_column_names = options.get_column_names(); | ||
| auto const& selected_column_indices = options.get_column_indices(); | ||
| auto const& selected_column_field_ids = options.get_column_field_ids(); | ||
|
|
||
| CUDF_EXPECTS( | ||
| not(selected_column_names.has_value() and selected_column_indices.has_value()), | ||
| "Parquet reader encountered column selection by both names and indices simultaneously"); | ||
| CUDF_EXPECTS(static_cast<int>(selected_column_names.has_value()) + | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Check to ensure we have at most one column selection mode
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Though it seems a bit too defensive as our
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I feel like we should not be doing this. The options object has the invariant that only one of these is set.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done in e66174a. Removed the redundant downstream validation because |
||
| static_cast<int>(selected_column_indices.has_value()) + | ||
| static_cast<int>(selected_column_field_ids.has_value()) <= | ||
| 1, | ||
| "Parquet reader encountered multiple column selection modes"); | ||
|
|
||
| // Map counting indices to the selected column by names | ||
| if (selected_column_names.has_value()) { | ||
|
|
@@ -257,32 +264,60 @@ void names_from_expression::visit_operands( | |
| return std::make_pair(col_index, | ||
| normalize_column_path(col_name, case_sensitive_names)); | ||
| }); | ||
| } else { | ||
| // Map selected top-level column indices to their names from the schema tree | ||
| } | ||
| // Map selected top-level column indices to their names from the schema tree | ||
| else if (selected_column_indices.has_value()) { | ||
| auto const& root = schema_tree.front(); | ||
|
|
||
| if (selected_column_indices.has_value()) { | ||
| std::transform(selected_column_indices->begin(), | ||
| selected_column_indices->end(), | ||
| cuda::counting_iterator<cudf::size_type>{0}, | ||
| std::inserter(column_indices_to_names, column_indices_to_names.end()), | ||
| [&](auto selected_col_idx, auto const mapped_col_idx) { | ||
| auto const schema_idx = root.children_idx[selected_col_idx]; | ||
| return std::make_pair( | ||
| mapped_col_idx, | ||
| normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)); | ||
| }); | ||
| } else { | ||
| // Map all top-level column indices to their names from the schema tree | ||
| std::for_each( | ||
| cuda::counting_iterator<int32_t>{0}, | ||
| cuda::counting_iterator{static_cast<int32_t>(root.children_idx.size())}, | ||
| [&](auto col_idx) { | ||
| auto const schema_idx = root.children_idx[col_idx]; | ||
| column_indices_to_names.insert( | ||
| {col_idx, normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)}); | ||
| }); | ||
| } | ||
| std::transform( | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Mostly whitespace here (try hiding it in your view). The only interesting part is under |
||
| selected_column_indices->begin(), | ||
| selected_column_indices->end(), | ||
| cuda::counting_iterator<cudf::size_type>{0}, | ||
| std::inserter(column_indices_to_names, column_indices_to_names.end()), | ||
| [&](auto selected_col_idx, auto const mapped_col_idx) { | ||
| CUDF_EXPECTS( | ||
| selected_col_idx > 0 and std::cmp_less(selected_col_idx, root.children_idx.size()), | ||
| "Encountered an invalid col index in the top-level column selection", | ||
| std::invalid_argument); | ||
| auto const schema_idx = root.children_idx[selected_col_idx]; | ||
| return std::make_pair( | ||
| mapped_col_idx, | ||
| normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)); | ||
| }); | ||
| } | ||
| // Map selected field ids to column paths from the schema tree | ||
| else if (selected_column_field_ids.has_value()) { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Simply create a index to name map. Indices are counting iterators and names are derived from the field ID. |
||
| std::transform( | ||
| selected_column_field_ids->begin(), | ||
| selected_column_field_ids->end(), | ||
| cuda::counting_iterator<cudf::size_type>{0}, | ||
| std::inserter(column_indices_to_names, column_indices_to_names.end()), | ||
| [&](auto const& field_id, auto const mapped_col_idx) { | ||
| auto const schema_iter = | ||
| std::find_if(schema_tree.begin() + 1, schema_tree.end(), [field_id](auto const& schema) { | ||
| return schema.field_id.has_value() and schema.field_id.value() == field_id; | ||
| }); | ||
| CUDF_EXPECTS(schema_iter != schema_tree.end(), | ||
| "Encountered a non-existent Parquet field ID in selected columns", | ||
| std::invalid_argument); | ||
| auto const schema_idx = static_cast<int>(std::distance(schema_tree.begin(), schema_iter)); | ||
| return std::make_pair(mapped_col_idx, | ||
| normalize_column_path(column_path_from_index(schema_tree, schema_idx), | ||
| case_sensitive_names)); | ||
| }); | ||
| } | ||
| // Map all top-level column indices to their names from the schema tree | ||
| else { | ||
| auto const& root = schema_tree.front(); | ||
|
|
||
| std::for_each( | ||
| cuda::counting_iterator<int32_t>{0}, | ||
| cuda::counting_iterator{static_cast<int32_t>(root.children_idx.size())}, | ||
| [&](auto col_idx) { | ||
| auto const schema_idx = root.children_idx[col_idx]; | ||
| column_indices_to_names.insert( | ||
| {col_idx, normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)}); | ||
| }); | ||
| } | ||
|
|
||
| return column_indices_to_names; | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.