Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
08f1e36
Select Parquet columns by field ID
mhaseeb123 Jun 24, 2026
5781078
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jun 24, 2026
4241c2d
Apply suggestion from coderabbit
mhaseeb123 Jun 24, 2026
4b99d07
Minor
mhaseeb123 Jun 24, 2026
43235a3
Style
mhaseeb123 Jun 24, 2026
2d26e65
Minor bug fix
mhaseeb123 Jun 24, 2026
3cc2e46
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jun 24, 2026
5048a45
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jun 24, 2026
d51c620
Apply partial suggestions
mhaseeb123 Jun 25, 2026
09e8735
Apply suggestions
mhaseeb123 Jun 25, 2026
b006c62
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jun 26, 2026
8d95803
style
mhaseeb123 Jun 26, 2026
cea7b9f
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jun 26, 2026
12eedd9
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jun 27, 2026
04a2527
Merge branch 'main' into fea/select-pq-cols-by-field-id
vyasr Jun 30, 2026
fe2485d
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jun 30, 2026
545b82f
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jul 1, 2026
3dfb09f
Address reviews
mhaseeb123 Jul 1, 2026
8417b0e
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jul 1, 2026
8073240
style
mhaseeb123 Jul 2, 2026
f920a00
Minor
mhaseeb123 Jul 2, 2026
12cf311
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jul 2, 2026
1335b4a
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jul 9, 2026
e66174a
Address comments
mhaseeb123 Jul 10, 2026
a18925b
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jul 10, 2026
86a4b48
Merge branch 'main' into fea/select-pq-cols-by-field-id
mhaseeb123 Jul 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 44 additions & 2 deletions cpp/include/cudf/io/parquet.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,14 @@ class parquet_reader_options_builder;
class parquet_reader_options {
source_info _source;

// Column selection options. Only one of these must be set at a time.
Comment thread
vuule marked this conversation as resolved.
Resolved

// Path in schema of column names to read; `nullopt` is all
std::optional<std::vector<std::string>> _column_names;
// Indices of top-level columns to read; `nullopt` is all (cannot be used alongside
// `_column_names`)
// Indices of top-level columns to read; `nullopt` is all
std::optional<std::vector<cudf::size_type>> _column_indices;
// Parquet field IDs of columns/fields to read; `nullopt` is all
std::optional<std::vector<int32_t>> _column_field_ids;

// List of individual row groups to read (ignored if empty)
std::vector<std::vector<size_type>> _row_groups;
Expand Down Expand Up @@ -253,6 +256,13 @@ class parquet_reader_options {
*/
[[nodiscard]] auto const& get_column_indices() const { return _column_indices; }

/**

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pretty standard setters and getters

* @brief Returns Parquet field IDs of columns/fields to be read, if set.
*
* @return Parquet field IDs of columns/fields to be read; `nullopt` if the option is not set
*/
[[nodiscard]] auto const& get_column_field_ids() const { return _column_field_ids; }

/**
* @brief Returns list of individual row groups to be read.
*
Expand Down Expand Up @@ -357,6 +367,8 @@ class parquet_reader_options {
{
CUDF_EXPECTS(not _column_indices.has_value(),
"Cannot select columns by indices and names simultaneously");
CUDF_EXPECTS(not _column_field_ids.has_value(),
"Cannot select columns by field IDs and names simultaneously");
_column_names = std::move(column_names);
}

Expand All @@ -375,9 +387,26 @@ class parquet_reader_options {
{
CUDF_EXPECTS(not _column_names.has_value(),
"Cannot select columns by indices and names simultaneously");
CUDF_EXPECTS(not _column_field_ids.has_value(),
"Cannot select columns by field IDs and indices simultaneously");
_column_indices = std::move(col_indices);
}

/**
* @brief Sets the Parquet field IDs of columns/fields to be read from all input sources.
*
* @param column_field_ids A vector of Parquet field IDs to attempt to read from each input
* source.
*/
void set_column_field_ids(std::vector<int32_t> column_field_ids)
{
CUDF_EXPECTS(not _column_names.has_value(),
"Cannot select columns by field IDs and names simultaneously");
CUDF_EXPECTS(not _column_indices.has_value(),
"Cannot select columns by field IDs and indices simultaneously");
_column_field_ids = std::move(column_field_ids);
}

/**
* @brief Specifies which row groups to read from each input source.
*
Expand Down Expand Up @@ -604,6 +633,19 @@ class parquet_reader_options_builder {
return *this;
}

/**
* @brief Sets the Parquet field IDs of columns/fields to be read from all input sources.
*
* @param column_field_ids A vector of Parquet field IDs to attempt to read from each input
* source.
* @return this for chaining
*/
parquet_reader_options_builder& column_field_ids(std::vector<int32_t> column_field_ids)
{
options.set_column_field_ids(std::move(column_field_ids));
return *this;
}

/**
* @copydoc parquet_reader_options::set_row_groups
* @return this for chaining
Expand Down
18 changes: 18 additions & 0 deletions cpp/src/io/parquet/column_path_helpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,33 @@

#include "column_path_helpers.hpp"

#include <cudf/io/parquet_schema.hpp>

#include <algorithm>
#include <cctype>
#include <cstddef>
#include <functional>
#include <numeric>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

namespace cudf::io::parquet::detail {

std::string column_path_from_index(std::span<SchemaElement const> schema_tree, int schema_idx)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given a ColumnChunk field ID, keep going to its parent until root and then construct a dot (.) separated string path

{
std::vector<std::string> path;
for (auto idx = schema_idx; idx > 0; idx = schema_tree[idx].parent_idx) {
path.push_back(schema_tree[idx].name);
}

return std::accumulate(
path.rbegin() + 1, path.rend(), path.back(), [](auto path_so_far, auto const& elem_name) {
return std::move(path_so_far) + "." + elem_name;
});
}

std::string normalize_column_path(std::string_view col_path, bool case_sensitive_names)
{
if (case_sensitive_names) { return std::string{col_path}; }
Expand Down
13 changes: 13 additions & 0 deletions cpp/src/io/parquet/column_path_helpers.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,27 @@

#pragma once

#include <cudf/io/parquet_schema.hpp>

#include <cstddef>
#include <span>
#include <string>
#include <string_view>
#include <unordered_map>
#include <unordered_set>

namespace cudf::io::parquet::detail {

/**
* @brief Gets the dot-separated path for a schema element.
*
* @param schema_tree The schema tree describing the file structure
* @param schema_idx Index of the schema element
* @return Dot-separated schema path from the root child to the schema element
*/
[[nodiscard]] std::string column_path_from_index(std::span<SchemaElement const> schema_tree,
int schema_idx);

/**
* @brief Returns a normalized (lowercased) column name or path when case-insensitive matching is
* enabled
Expand Down
12 changes: 8 additions & 4 deletions cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,8 @@ aggregate_reader_metadata::select_payload_columns(
bool ignore_missing_columns,
type_id timestamp_type_id,
type_id decimal_type_id,
bool case_sensitive_names)
bool case_sensitive_names,
bool match_schema_by_field_id)
{
// If neither payload nor filter columns are specified, select all columns
if (not payload_column_names.has_value() and not filter_column_names.has_value()) {
Expand All @@ -283,7 +284,8 @@ aggregate_reader_metadata::select_payload_columns(
ignore_missing_columns,
timestamp_type_id,
decimal_type_id,
case_sensitive_names);
case_sensitive_names,
match_schema_by_field_id);

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just plumbed in a new boolean field

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these are mutually exclusive right? should you instead switch to an enum to reduce the number of arguments? the api is experimental after all.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these are mutually exclusive right?
Yes

should you instead switch to an enum to reduce the number of arguments? the api is experimental after all.

We are calling the base reader's select_columns API (being updated to take in another bool here).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure but can we update that API also to reduce argument bloat everywhere

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done 3dfb09f

}

std::vector<std::string> valid_payload_columns;
Expand Down Expand Up @@ -318,7 +320,8 @@ aggregate_reader_metadata::select_payload_columns(
ignore_missing_columns,
timestamp_type_id,
decimal_type_id,
case_sensitive_names);
case_sensitive_names,
match_schema_by_field_id);
}

// Else if only filter columns are specified, select all columns that do not appear in the
Expand Down Expand Up @@ -348,7 +351,8 @@ aggregate_reader_metadata::select_payload_columns(
ignore_missing_columns,
timestamp_type_id,
decimal_type_id,
case_sensitive_names);
case_sensitive_names,
match_schema_by_field_id);
}

std::vector<std::vector<cudf::size_type>>
Expand Down
4 changes: 3 additions & 1 deletion cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ class aggregate_reader_metadata : public aggregate_reader_metadata_base {
* @param timestamp_type_id Type conversion parameter
* @param decimal_type_id Type conversion parameter
* @param case_sensitive_names Boolean indicating if column names are case sensitive
* @param match_schema_by_field_id Whether to match schema by field ID
*
* @return input column information, output column buffers, list of output column schema
* indices
Expand All @@ -176,7 +177,8 @@ class aggregate_reader_metadata : public aggregate_reader_metadata_base {
bool ignore_missing_columns,
type_id timestamp_type_id,
type_id decimal_type_id,
bool case_sensitive_names);
bool case_sensitive_names,
bool match_schema_by_field_id);

/**
* @brief Filters row groups such that only the row groups that start within the byte range
Expand Down
23 changes: 14 additions & 9 deletions cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,21 +68,23 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl(
cudf::host_span<cudf::host_span<uint8_t const> const> footer_bytes,
parquet_reader_options const& options)
{
auto const has_cols_from_mismatched_srcs =

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Separate out this logic: We can have mismatched columns if allowed AND selecting columns by names or field IDs

(options.get_column_names().has_value() or options.get_column_field_ids().has_value()) and
options.is_enabled_allow_mismatched_pq_schemas();
_metadata = std::make_unique<aggregate_reader_metadata>(
footer_bytes,
options.is_enabled_use_arrow_schema(),
options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas());
footer_bytes, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_srcs);

_extended_metadata = static_cast<aggregate_reader_metadata*>(_metadata.get());
}

hybrid_scan_reader_impl::hybrid_scan_reader_impl(
cudf::host_span<FileMetaData const> parquet_metadatas, parquet_reader_options const& options)
{
auto const has_cols_from_mismatched_srcs =

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here

(options.get_column_names().has_value() or options.get_column_field_ids().has_value()) and
options.is_enabled_allow_mismatched_pq_schemas();
_metadata = std::make_unique<aggregate_reader_metadata>(
parquet_metadatas,
options.is_enabled_use_arrow_schema(),
options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas());
parquet_metadatas, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_srcs);
_extended_metadata = static_cast<aggregate_reader_metadata*>(_metadata.get());
}

Expand Down Expand Up @@ -132,7 +134,8 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode
options.is_enabled_ignore_missing_columns(),
_options.timestamp_type.id(),
_options.decimal_width,
_options.case_sensitive_names);
_options.case_sensitive_names,
options.get_column_field_ids().has_value());

_is_all_columns_selected = true;
_is_filter_columns_selected = false;
Expand All @@ -157,7 +160,8 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode
ignore_missing_columns,
_options.timestamp_type.id(),
_options.decimal_width,
_options.case_sensitive_names);
_options.case_sensitive_names,
options.get_column_field_ids().has_value());

_is_filter_columns_selected = true;
_is_payload_columns_selected = false;
Expand All @@ -178,7 +182,8 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode
options.is_enabled_ignore_missing_columns(),
_options.timestamp_type.id(),
_options.decimal_width,
_options.case_sensitive_names);
_options.case_sensitive_names,
options.get_column_field_ids().has_value());

_is_payload_columns_selected = true;
_is_filter_columns_selected = false;
Expand Down
95 changes: 65 additions & 30 deletions cpp/src/io/parquet/expression_transform_helpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@

#include <cuda/iterator>

#include <algorithm>
#include <string>
#include <utility>

namespace cudf::io::parquet::detail {

namespace {
Expand Down Expand Up @@ -235,17 +239,20 @@ void names_from_expression::visit_operands(

[[nodiscard]] std::unordered_map<cudf::size_type, std::string> map_column_indices_to_names(
cudf::io::parquet_reader_options const& options,
std::vector<SchemaElement> const& schema_tree,
std::span<SchemaElement const> schema_tree,
bool case_sensitive_names)
{
std::unordered_map<cudf::size_type, std::string> column_indices_to_names;

auto const& selected_column_names = options.get_column_names();
auto const& selected_column_indices = options.get_column_indices();
auto const& selected_column_names = options.get_column_names();
auto const& selected_column_indices = options.get_column_indices();
auto const& selected_column_field_ids = options.get_column_field_ids();

CUDF_EXPECTS(
not(selected_column_names.has_value() and selected_column_indices.has_value()),
"Parquet reader encountered column selection by both names and indices simultaneously");
CUDF_EXPECTS(static_cast<int>(selected_column_names.has_value()) +

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Check to ensure we have at most one column selection mode

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Though it seems a bit too defensive as our parquet_reader_options setters catch this case.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like we should not be doing this. The options object has the invariant that only one of these is set.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in e66174a. Removed the redundant downstream validation because parquet_reader_options enforces this invariant.

static_cast<int>(selected_column_indices.has_value()) +
static_cast<int>(selected_column_field_ids.has_value()) <=
1,
"Parquet reader encountered multiple column selection modes");

// Map counting indices to the selected column by names
if (selected_column_names.has_value()) {
Expand All @@ -257,32 +264,60 @@ void names_from_expression::visit_operands(
return std::make_pair(col_index,
normalize_column_path(col_name, case_sensitive_names));
});
} else {
// Map selected top-level column indices to their names from the schema tree
}
// Map selected top-level column indices to their names from the schema tree
else if (selected_column_indices.has_value()) {
auto const& root = schema_tree.front();

if (selected_column_indices.has_value()) {
std::transform(selected_column_indices->begin(),
selected_column_indices->end(),
cuda::counting_iterator<cudf::size_type>{0},
std::inserter(column_indices_to_names, column_indices_to_names.end()),
[&](auto selected_col_idx, auto const mapped_col_idx) {
auto const schema_idx = root.children_idx[selected_col_idx];
return std::make_pair(
mapped_col_idx,
normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names));
});
} else {
// Map all top-level column indices to their names from the schema tree
std::for_each(
cuda::counting_iterator<int32_t>{0},
cuda::counting_iterator{static_cast<int32_t>(root.children_idx.size())},
[&](auto col_idx) {
auto const schema_idx = root.children_idx[col_idx];
column_indices_to_names.insert(
{col_idx, normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)});
});
}
std::transform(

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mostly whitespace here (try hiding it in your view). The only interesting part is under else if (selected_column_field_ids.has_value()) { block.

selected_column_indices->begin(),
selected_column_indices->end(),
cuda::counting_iterator<cudf::size_type>{0},
std::inserter(column_indices_to_names, column_indices_to_names.end()),
[&](auto selected_col_idx, auto const mapped_col_idx) {
CUDF_EXPECTS(
selected_col_idx > 0 and std::cmp_less(selected_col_idx, root.children_idx.size()),
"Encountered an invalid col index in the top-level column selection",
std::invalid_argument);
auto const schema_idx = root.children_idx[selected_col_idx];
return std::make_pair(
mapped_col_idx,
normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names));
});
}
// Map selected field ids to column paths from the schema tree
else if (selected_column_field_ids.has_value()) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Simply create a index to name map. Indices are counting iterators and names are derived from the field ID.

std::transform(
selected_column_field_ids->begin(),
selected_column_field_ids->end(),
cuda::counting_iterator<cudf::size_type>{0},
std::inserter(column_indices_to_names, column_indices_to_names.end()),
[&](auto const& field_id, auto const mapped_col_idx) {
auto const schema_iter =
std::find_if(schema_tree.begin() + 1, schema_tree.end(), [field_id](auto const& schema) {
return schema.field_id.has_value() and schema.field_id.value() == field_id;
});
CUDF_EXPECTS(schema_iter != schema_tree.end(),
"Encountered a non-existent Parquet field ID in selected columns",
std::invalid_argument);
auto const schema_idx = static_cast<int>(std::distance(schema_tree.begin(), schema_iter));
return std::make_pair(mapped_col_idx,
normalize_column_path(column_path_from_index(schema_tree, schema_idx),
case_sensitive_names));
});
}
// Map all top-level column indices to their names from the schema tree
else {
auto const& root = schema_tree.front();

std::for_each(
cuda::counting_iterator<int32_t>{0},
cuda::counting_iterator{static_cast<int32_t>(root.children_idx.size())},
[&](auto col_idx) {
auto const schema_idx = root.children_idx[col_idx];
column_indices_to_names.insert(
{col_idx, normalize_column_path(schema_tree[schema_idx].name, case_sensitive_names)});
});
}

return column_indices_to_names;
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/io/parquet/expression_transform_helpers.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ class equality_literals_collector : public ast::detail::expression_transformer {
*/
[[nodiscard]] std::unordered_map<cudf::size_type, std::string> map_column_indices_to_names(
cudf::io::parquet_reader_options const& options,
std::vector<SchemaElement> const& schema_tree,
std::span<SchemaElement const> schema_tree,
bool case_sensitive_names);

/**
Expand Down
Loading
Loading