From 051e21d4c7204b10f1ebe223497cf3edfa62a891 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 5 Aug 2026 02:36:06 +0000 Subject: [PATCH 1/9] Add python bindings for HybridScanMetadata and release GIL in HybridScanReader --- .../cudf/io/experimental/hybrid_scan.hpp | 83 ++++ .../io/parquet/experimental/hybrid_scan.cpp | 25 ++ .../parquet/experimental/hybrid_scan_impl.cpp | 12 +- .../parquet/experimental/hybrid_scan_impl.hpp | 11 + cpp/src/io/parquet/reader_impl.cpp | 4 +- cpp/src/io/parquet/reader_impl.hpp | 2 +- .../io/experimental/hybrid_scan_common.cpp | 1 + .../io/experimental/hybrid_scan_test.cpp | 46 +++ .../pylibcudf/io/experimental/__init__.py | 4 +- .../pylibcudf/io/experimental/hybrid_scan.pxd | 9 +- .../pylibcudf/io/experimental/hybrid_scan.pyi | 34 +- .../pylibcudf/io/experimental/hybrid_scan.pyx | 364 +++++++++++++----- .../pylibcudf/libcudf/io/hybrid_scan.pxd | 21 + 13 files changed, 508 insertions(+), 108 deletions(-) diff --git a/cpp/include/cudf/io/experimental/hybrid_scan.hpp b/cpp/include/cudf/io/experimental/hybrid_scan.hpp index 6a3b2059b55d..560ed0af18d0 100644 --- a/cpp/include/cudf/io/experimental/hybrid_scan.hpp +++ b/cpp/include/cudf/io/experimental/hybrid_scan.hpp @@ -31,6 +31,11 @@ namespace cudf::io::parquet::experimental::detail { * Hybrid Scan operation. */ class hybrid_scan_reader_impl; + +/** + * @brief Internal parsed Parquet file metadata for the Hybrid Scan reader. + */ +class aggregate_reader_metadata; } // namespace cudf::io::parquet::experimental::detail //! Using `byte_range_info` from cudf::io::text @@ -52,6 +57,72 @@ enum class use_data_page_mask : bool { NO = false ///< Do not compute or use a data page mask }; +/** + * @brief Shareable, pre-parsed Parquet file metadata for the Hybrid Scan reader. + * + * Parses the Parquet file metadata once so that multiple `hybrid_scan_reader` instances reading + * the same file can borrow it rather than each re-parsing and copying the row group metadata. + * The intended use is to read disjoint row-group ranges of a single file: construct one + * `hybrid_scan_metadata` per file and pass it to as many readers as there are ranges. + * + * @code{.cpp} + * // Parse the metadata once + * auto metadata = std::make_shared(*footer_buffer, + * options); + * // Construct lightweight readers that share it + * auto reader_a = std::make_unique(*metadata); + * auto reader_b = std::make_unique(*metadata); + * @endcode + * + * @note The metadata is immutable after `setup_page_index()` has been called (or immediately after + * construction if page index setup is skipped). Readers sharing one instance may read different + * row-group ranges of the same single file concurrently; overlapping ranges produce duplicate rows. + * This handle does not support multi-source (multi-file) metadata. + */ +class hybrid_scan_metadata { + public: + /** + * @brief Parse and own Parquet file metadata from a span of footer bytes + * + * @param footer_bytes Host span of Parquet file footer bytes + * @param options Parquet reader options + */ + hybrid_scan_metadata(cudf::host_span footer_bytes, + parquet_reader_options const& options); + + /** + * @brief Own Parquet file metadata from a pre-populated `FileMetaData` + * + * @param parquet_metadata Pre-populated Parquet file metadata + * @param options Parquet reader options + */ + hybrid_scan_metadata(FileMetaData const& parquet_metadata, parquet_reader_options const& options); + + /** + * @brief Destructor for the shared Parquet metadata + */ + ~hybrid_scan_metadata(); + + hybrid_scan_metadata(hybrid_scan_metadata const&) = default; ///< Copy constructor + hybrid_scan_metadata(hybrid_scan_metadata&&) = default; ///< Move constructor + + /** + * @brief Copy assignment operator + * @return Reference to this object + */ + hybrid_scan_metadata& operator=(hybrid_scan_metadata const&) = default; + + /** + * @brief Move assignment operator + * @return Reference to this object + */ + hybrid_scan_metadata& operator=(hybrid_scan_metadata&&) = default; + + private: + std::shared_ptr _metadata; + friend class hybrid_scan_reader; +}; + /** * @brief The experimental parquet reader class to optimally read parquet files subject to * highly selective filters, called a Hybrid Scan operation @@ -300,6 +371,18 @@ class hybrid_scan_reader { explicit hybrid_scan_reader(FileMetaData const& parquet_metadata, parquet_reader_options const& options); + /** + * @brief Constructor that borrows shared, pre-parsed Parquet file metadata + * + * Constructs a reader that shares `metadata` instead of parsing and copying the file metadata + * again. Use this to read disjoint row-group ranges of a single file without paying the metadata + * copy per reader. The reader options that govern reading (column selection, filter, ...) are + * supplied per call to the individual read methods. + * + * @param metadata Shared, pre-parsed Parquet file metadata + */ + explicit hybrid_scan_reader(hybrid_scan_metadata const& metadata); + /** * @brief Destructor for the experimental parquet reader class */ diff --git a/cpp/src/io/parquet/experimental/hybrid_scan.cpp b/cpp/src/io/parquet/experimental/hybrid_scan.cpp index 36bc5de06dc4..3ded187fe997 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan.cpp @@ -13,6 +13,26 @@ namespace cudf::io::parquet::experimental { +hybrid_scan_metadata::hybrid_scan_metadata(cudf::host_span footer_bytes, + parquet_reader_options const& options) + : _metadata{std::make_shared( + std::vector>{footer_bytes}, + options.is_enabled_use_arrow_schema(), + options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas())} +{ +} + +hybrid_scan_metadata::hybrid_scan_metadata(FileMetaData const& parquet_metadata, + parquet_reader_options const& options) + : _metadata{std::make_shared( + std::vector{parquet_metadata}, + options.is_enabled_use_arrow_schema(), + options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas())} +{ +} + +hybrid_scan_metadata::~hybrid_scan_metadata() = default; + hybrid_scan_reader::hybrid_scan_reader(cudf::host_span footer_bytes, parquet_reader_options const& options) : _impl{std::make_unique( @@ -27,6 +47,11 @@ hybrid_scan_reader::hybrid_scan_reader(FileMetaData const& parquet_metadata, { } +hybrid_scan_reader::hybrid_scan_reader(hybrid_scan_metadata const& metadata) + : _impl{std::make_unique(metadata._metadata)} +{ +} + hybrid_scan_reader::~hybrid_scan_reader() = default; [[nodiscard]] text::byte_range_info hybrid_scan_reader::page_index_byte_range() const diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp b/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp index aab25c0e648b..8bf93cfe4485 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp @@ -98,7 +98,7 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl( cudf::host_span const> footer_bytes, parquet_reader_options const& options) { - _metadata = std::make_unique( + _metadata = std::make_shared( footer_bytes, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_sources(options)); _extended_metadata = static_cast(_metadata.get()); @@ -108,12 +108,20 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl( cudf::host_span parquet_metadatas, parquet_reader_options const& options) { _metadata = - std::make_unique(parquet_metadatas, + std::make_shared(parquet_metadatas, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_sources(options)); _extended_metadata = static_cast(_metadata.get()); } +hybrid_scan_reader_impl::hybrid_scan_reader_impl( + std::shared_ptr metadata) +{ + CUDF_EXPECTS(metadata != nullptr, "Shared parquet metadata must not be null"); + _metadata = std::move(metadata); + _extended_metadata = static_cast(_metadata.get()); +} + std::vector hybrid_scan_reader_impl::parquet_metadatas() const { return _extended_metadata->parquet_metadatas(); diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp index 3dd241af35ff..7b5a638ff2b5 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp @@ -59,6 +59,17 @@ class hybrid_scan_reader_impl : public parquet::detail::reader_impl { explicit hybrid_scan_reader_impl(cudf::host_span parquet_metadatas, parquet_reader_options const& options); + /** + * @brief Constructor that shares pre-parsed Parquet metadata + * + * Borrows an already-constructed `aggregate_reader_metadata` instead of parsing and copying the + * file metadata again. Multiple single-file readers can share one metadata object, avoiding a + * per-reader copy of the (potentially large) row group metadata. + * + * @param metadata Shared, pre-parsed Parquet file metadata. Must not be null. + */ + explicit hybrid_scan_reader_impl(std::shared_ptr metadata); + /** * @copydoc cudf::io::parquet::experimental::hybrid_scan_multifile::parquet_metadatas */ diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 02485a65637c..4354f486c09e 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -533,11 +533,11 @@ reader_impl::reader_impl(std::size_t chunk_read_limit, CUDF_EXPECTS(file_metadatas.empty() or file_metadatas.size() == _sources.size(), "Encountered a mismatch in the number of provided data sources and metadatas"); - _metadata = file_metadatas.empty() ? std::make_unique( + _metadata = file_metadatas.empty() ? std::make_shared( _sources, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_sources(options)) - : std::make_unique( + : std::make_shared( std::forward>(file_metadatas), options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_sources(options)); diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 76d88f52e310..cffe8329c51c 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -548,7 +548,7 @@ class reader_impl { named_to_reference_converter _expr_conv{std::nullopt, table_metadata{}, true}; std::vector> _sources; - std::unique_ptr _metadata; + std::shared_ptr _metadata; // Number of sources size_t _num_sources{0}; diff --git a/cpp/tests/io/experimental/hybrid_scan_common.cpp b/cpp/tests/io/experimental/hybrid_scan_common.cpp index 339009ecf7e5..8253f11d2159 100644 --- a/cpp/tests/io/experimental/hybrid_scan_common.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_common.cpp @@ -419,6 +419,7 @@ std::pair, std::vector> create_parquet_with_s INSTANTIATE_CREATE_PARQUET_WITH_STATS(T, 1, true, true) INSTANTIATE_CREATE_PARQUET_WITH_STATS(uint32_t, 4, true, false); +INSTANTIATE_CREATE_PARQUET_WITH_STATS(int32_t, 2, true, false); INSTANTIATE_CREATE_PARQUET_WITH_STATS(cudf::timestamp_ms, 2, true, false); INSTANTIATE_CREATE_PARQUET_WITH_STATS(cudf::duration_ms, 2, true, false); diff --git a/cpp/tests/io/experimental/hybrid_scan_test.cpp b/cpp/tests/io/experimental/hybrid_scan_test.cpp index edf43fc6bdff..7e4e633d9be1 100644 --- a/cpp/tests/io/experimental/hybrid_scan_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_test.cpp @@ -985,6 +985,52 @@ TEST_F(HybridScanTest, StructChildFilterColumn) std::invalid_argument); } +TEST_F(HybridScanTest, SharedMetadataReaderMatchesReadParquet) +{ + using T = int32_t; + auto constexpr num_concat = 2; + auto [written_table, parquet_buffer] = create_parquet_with_stats(); + + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + auto const options = cudf::io::parquet_reader_options::builder().build(); + + auto datasource = cudf::io::datasource::create(cudf::host_span( + reinterpret_cast(parquet_buffer.data()), parquet_buffer.size())); + auto const footer_buffer = cudf::io::parquet::fetch_footer_to_host(*datasource); + + // Parse the file metadata once and share it across independent readers. + auto const metadata = std::make_shared( + *footer_buffer, options); + + // Read all columns (single step) through a reader that borrows the shared metadata. + auto const read_all_columns = [&] { + auto const reader = + std::make_unique(*metadata); + auto const row_groups = reader->all_row_groups(options); + auto const chunk_ranges = reader->all_column_chunks_byte_ranges(row_groups, options); + auto [buffers, data, tasks] = + cudf::io::parquet::fetch_byte_ranges_to_device_async(*datasource, chunk_ranges, stream, mr); + tasks.get(); + return reader->materialize_all_columns(row_groups, data, options, stream, mr).tbl; + }; + + // Two readers sharing one metadata instance each produce the same table as the main reader. + auto const table_a = read_all_columns(); + auto const table_b = read_all_columns(); + + auto const expected = + cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info(cudf::host_span(parquet_buffer.data(), parquet_buffer.size()))) + .build(), + stream) + .tbl; + + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected->view(), table_a->view()); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected->view(), table_b->view()); +} + TEST_F(HybridScanTest, AllRowsPrunedReportsInputRowGroups) { using cudf::io::parquet::experimental::use_data_page_mask; diff --git a/python/pylibcudf/pylibcudf/io/experimental/__init__.py b/python/pylibcudf/pylibcudf/io/experimental/__init__.py index 6c64231eb1e9..ef1ca25b7cff 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/__init__.py +++ b/python/pylibcudf/pylibcudf/io/experimental/__init__.py @@ -1,7 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from pylibcudf.io.experimental.hybrid_scan import ( + HybridScanMetadata, HybridScanReader, UseDataPageMask, ) @@ -9,6 +10,7 @@ __all__ = [ "FileMetaData", # backwards compatibility + "HybridScanMetadata", "HybridScanReader", "UseDataPageMask", ] diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pxd b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pxd index a19cd7db8bf8..a999fc5bc7c2 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pxd +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport uint8_t @@ -13,6 +13,7 @@ from pylibcudf.io.parquet cimport ParquetReaderOptions from pylibcudf.io.parquet_metadata cimport FileMetaData as c_FileMetaData from pylibcudf.io.types cimport TableWithMetadata from pylibcudf.libcudf.io.hybrid_scan cimport ( + hybrid_scan_metadata as cpp_hybrid_scan_metadata, hybrid_scan_reader as cpp_hybrid_scan_reader, use_data_page_mask, ) @@ -23,7 +24,13 @@ from pylibcudf.libcudf.utilities.span cimport device_span cdef device_span[const_uint8_t] _get_device_span(object obj) except * +cdef class HybridScanMetadata: + cdef unique_ptr[cpp_hybrid_scan_metadata] c_obj + + cdef class HybridScanReader: cdef unique_ptr[cpp_hybrid_scan_reader] c_obj cdef Stream _stream cdef DeviceMemoryResource mr + cdef object _filter_chunk_data + cdef object _payload_chunk_data diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyi b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyi index f95dc8b054d3..498a458b9f7d 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyi +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyi @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from collections.abc import Sequence from enum import IntEnum from rmm.pylibrmm.memory_resource import DeviceMemoryResource @@ -10,6 +11,7 @@ from pylibcudf.io.parquet import ParquetReaderOptions from pylibcudf.io.parquet_metadata import FileMetaData from pylibcudf.io.text import ByteRangeInfo from pylibcudf.io.types import TableWithMetadata +from pylibcudf.span import Span from pylibcudf.utils import CudaStreamLike try: @@ -21,6 +23,16 @@ class UseDataPageMask(IntEnum): YES = 1 NO = 0 +class HybridScanMetadata: + @staticmethod + def from_footer_bytes( + footer_bytes: Buffer, options: ParquetReaderOptions + ) -> HybridScanMetadata: ... + @staticmethod + def from_parquet_metadata( + metadata: FileMetaData, options: ParquetReaderOptions + ) -> HybridScanMetadata: ... + class HybridScanReader: def __init__( self, footer_bytes: Buffer, options: ParquetReaderOptions @@ -29,6 +41,8 @@ class HybridScanReader: def from_parquet_metadata( metadata: FileMetaData, options: ParquetReaderOptions ) -> HybridScanReader: ... + @staticmethod + def from_metadata(metadata: HybridScanMetadata) -> HybridScanReader: ... def parquet_metadata(self) -> FileMetaData: ... def page_index_byte_range(self) -> ByteRangeInfo: ... def setup_page_index(self, page_index_bytes: Buffer) -> None: ... @@ -48,18 +62,24 @@ class HybridScanReader: ) -> tuple[list[ByteRangeInfo], list[ByteRangeInfo]]: ... def filter_row_groups_with_dictionary_pages( self, - dictionary_page_data: list, + dictionary_page_data: Sequence[Span], row_group_indices: list[int], options: ParquetReaderOptions, stream: CudaStreamLike | None = None, ) -> list[int]: ... def filter_row_groups_with_bloom_filters( self, - bloom_filter_data: list, + bloom_filter_data: Sequence[Span], row_group_indices: list[int], options: ParquetReaderOptions, stream: CudaStreamLike | None = None, ) -> list[int]: ... + def build_all_true_row_mask( + self, + row_group_indices: list[int], + stream: CudaStreamLike | None = None, + mr: DeviceMemoryResource | None = None, + ) -> Column: ... def build_row_mask_with_page_index_stats( self, row_group_indices: list[int], @@ -73,7 +93,7 @@ class HybridScanReader: def materialize_filter_columns( self, row_group_indices: list[int], - column_chunk_data: list, + column_chunk_data: Sequence[Span], row_mask: Column, mask_data_pages: UseDataPageMask, options: ParquetReaderOptions, @@ -86,7 +106,7 @@ class HybridScanReader: def materialize_payload_columns( self, row_group_indices: list[int], - column_chunk_data: list, + column_chunk_data: Sequence[Span], row_mask: Column, mask_data_pages: UseDataPageMask, options: ParquetReaderOptions, @@ -99,7 +119,7 @@ class HybridScanReader: def materialize_all_columns( self, row_group_indices: list[int], - column_chunk_data: list, + column_chunk_data: Sequence[Span], options: ParquetReaderOptions, stream: CudaStreamLike | None = None, mr: DeviceMemoryResource | None = None, @@ -111,7 +131,7 @@ class HybridScanReader: row_group_indices: list[int], row_mask: Column, mask_data_pages: UseDataPageMask, - column_chunk_data: list, + column_chunk_data: Sequence[Span], options: ParquetReaderOptions, stream: CudaStreamLike | None = None, mr: DeviceMemoryResource | None = None, @@ -127,7 +147,7 @@ class HybridScanReader: row_group_indices: list[int], row_mask: Column, mask_data_pages: UseDataPageMask, - column_chunk_data: list, + column_chunk_data: Sequence[Span], options: ParquetReaderOptions, stream: CudaStreamLike | None = None, mr: DeviceMemoryResource | None = None, diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx index 664eb489428c..a81522b71845 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx @@ -1,8 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from cython.operator cimport dereference from libc.stdint cimport uint8_t, uintptr_t from libc.stddef cimport size_t +from libcpp cimport bool from libcpp.memory cimport make_unique, unique_ptr from libcpp.pair cimport pair from libcpp.span cimport span as std_span @@ -15,6 +17,7 @@ from rmm.pylibrmm.stream cimport Stream from pylibcudf.column cimport Column from pylibcudf.io.parquet cimport ParquetReaderOptions from pylibcudf.io.parquet_metadata cimport FileMetaData as c_FileMetaData +from pylibcudf.libcudf.io.parquet_schema cimport FileMetaData as cpp_FileMetaData from pylibcudf.io.text cimport ByteRangeInfo from pylibcudf.io.types cimport TableWithMetadata from pylibcudf.libcudf.column.column cimport column @@ -23,6 +26,7 @@ from pylibcudf.libcudf.io.hybrid_scan cimport ( const_device_span_const_uint8_t, const_size_type, const_uint8_t, + hybrid_scan_metadata as cpp_hybrid_scan_metadata, hybrid_scan_reader as cpp_hybrid_scan_reader, use_data_page_mask as cpp_use_data_page_mask, ) @@ -39,7 +43,7 @@ import pylibcudf.libcudf.io.hybrid_scan UseDataPageMask = pylibcudf.libcudf.io.hybrid_scan.use_data_page_mask -__all__ = ["FileMetaData", "HybridScanReader", "UseDataPageMask"] +__all__ = ["FileMetaData", "HybridScanMetadata", "HybridScanReader", "UseDataPageMask"] cdef device_span[const_uint8_t] _get_device_span(object obj) except *: @@ -53,6 +57,76 @@ cdef device_span[const_uint8_t] _get_device_span(object obj) except *: obj.size) +cdef class HybridScanMetadata: + """Shareable, pre-parsed Parquet file metadata for the hybrid scan reader. + + Parse the metadata of a single Parquet file once, then construct multiple + :class:`HybridScanReader` instances that share it (one per row-group range of the + file) instead of each re-parsing and copying the metadata. + + For details, see :cpp:class:`cudf::io::parquet::experimental::hybrid_scan_metadata` + + Examples + -------- + >>> import pylibcudf as plc + >>> metadata = plc.io.experimental.HybridScanMetadata.from_parquet_metadata( + ... file_metadata, options) + >>> reader = plc.io.experimental.HybridScanReader.from_metadata(metadata) + """ + + @staticmethod + def from_footer_bytes( + const uint8_t[::1] footer_bytes, + ParquetReaderOptions options + ): + """Parse shareable metadata from Parquet footer bytes. + + Parameters + ---------- + footer_bytes : Buffer + Parquet file footer bytes + options : ParquetReaderOptions + Parquet reader options + + Returns + ------- + HybridScanMetadata + """ + cdef HybridScanMetadata self = HybridScanMetadata.__new__(HybridScanMetadata) + cdef const uint8_t* footer_ptr = 0 + if len(footer_bytes) > 0: + footer_ptr = &footer_bytes[0] + with nogil: + self.c_obj = make_unique[cpp_hybrid_scan_metadata]( + host_span[const_uint8_t](footer_ptr, len(footer_bytes)), + options.c_obj + ) + return self + + @staticmethod + def from_parquet_metadata(c_FileMetaData metadata, ParquetReaderOptions options): + """Build shareable metadata from a pre-populated ``FileMetaData``. + + Parameters + ---------- + metadata : FileMetaData + Pre-populated Parquet file metadata + options : ParquetReaderOptions + Parquet reader options + + Returns + ------- + HybridScanMetadata + """ + cdef HybridScanMetadata self = HybridScanMetadata.__new__(HybridScanMetadata) + with nogil: + self.c_obj = make_unique[cpp_hybrid_scan_metadata]( + metadata.c_obj, + options.c_obj + ) + return self + + cdef class HybridScanReader: """Experimental Parquet reader optimized for highly selective filters. @@ -82,10 +156,14 @@ cdef class HybridScanReader: """ def __init__(self, const uint8_t[::1] footer_bytes, ParquetReaderOptions options): - self.c_obj = make_unique[cpp_hybrid_scan_reader]( - host_span[const_uint8_t](&footer_bytes[0], len(footer_bytes)), - options.c_obj - ) + cdef const uint8_t* footer_ptr = 0 + if len(footer_bytes) > 0: + footer_ptr = &footer_bytes[0] + with nogil: + self.c_obj = make_unique[cpp_hybrid_scan_reader]( + host_span[const_uint8_t](footer_ptr, len(footer_bytes)), + options.c_obj + ) @staticmethod def from_parquet_metadata(c_FileMetaData metadata, ParquetReaderOptions options): @@ -103,10 +181,36 @@ cdef class HybridScanReader: HybridScanReader """ cdef HybridScanReader reader = HybridScanReader.__new__(HybridScanReader) - reader.c_obj = make_unique[cpp_hybrid_scan_reader]( - metadata.c_obj, - options.c_obj - ) + with nogil: + reader.c_obj = make_unique[cpp_hybrid_scan_reader]( + metadata.c_obj, + options.c_obj + ) + return reader + + @staticmethod + def from_metadata(HybridScanMetadata metadata): + """Create a HybridScanReader that shares pre-parsed metadata. + + Constructs a lightweight reader that borrows ``metadata`` instead of + re-parsing and copying the file metadata. Use one shared + :class:`HybridScanMetadata` to read row-group ranges of a single file. + Overlapping row-group ranges across readers produce duplicate rows. + + Parameters + ---------- + metadata : HybridScanMetadata + Shared, pre-parsed Parquet file metadata + + Returns + ------- + HybridScanReader + """ + cdef HybridScanReader reader = HybridScanReader.__new__(HybridScanReader) + with nogil: + reader.c_obj = make_unique[cpp_hybrid_scan_reader]( + dereference(metadata.c_obj.get()) + ) return reader def parquet_metadata(self): @@ -117,7 +221,10 @@ cdef class HybridScanReader: FileMetaData Parquet file footer metadata """ - return c_FileMetaData.from_cpp(self.c_obj.get()[0].parquet_metadata()) + cdef cpp_FileMetaData c_result + with nogil: + c_result = self.c_obj.get()[0].parquet_metadata() + return c_FileMetaData.from_cpp(c_result) def page_index_byte_range(self): """Get the byte range of the page index. @@ -127,7 +234,9 @@ cdef class HybridScanReader: ByteRangeInfo Byte range of the page index """ - cdef byte_range_info info = self.c_obj.get()[0].page_index_byte_range() + cdef byte_range_info info + with nogil: + info = self.c_obj.get()[0].page_index_byte_range() return ByteRangeInfo(info.offset(), info.size()) def setup_page_index(self, const uint8_t[::1] page_index_bytes): @@ -138,9 +247,13 @@ cdef class HybridScanReader: page_index_bytes : Buffer Parquet page index buffer bytes """ - self.c_obj.get()[0].setup_page_index( - host_span[const_uint8_t](&page_index_bytes[0], len(page_index_bytes)) - ) + cdef const uint8_t* page_index_ptr = 0 + if len(page_index_bytes) > 0: + page_index_ptr = &page_index_bytes[0] + with nogil: + self.c_obj.get()[0].setup_page_index( + host_span[const_uint8_t](page_index_ptr, len(page_index_bytes)) + ) def all_row_groups(self, ParquetReaderOptions options): """Get all available row groups from the parquet file. @@ -155,9 +268,9 @@ cdef class HybridScanReader: list[int] List of row group indices """ - cdef vector[size_type] row_groups = self.c_obj.get()[0].all_row_groups( - options.c_obj - ) + cdef vector[size_type] row_groups + with nogil: + row_groups = self.c_obj.get()[0].all_row_groups(options.c_obj) return list(row_groups) def total_rows_in_row_groups(self, list row_group_indices): @@ -174,9 +287,12 @@ cdef class HybridScanReader: Total number of top-level rows """ cdef vector[size_type] indices_vec = row_group_indices - return self.c_obj.get()[0].total_rows_in_row_groups( - std_span[const_size_type](indices_vec.data(), indices_vec.size()) - ) + cdef size_type result + with nogil: + result = self.c_obj.get()[0].total_rows_in_row_groups( + std_span[const_size_type](indices_vec.data(), indices_vec.size()) + ) + return result def reset_column_selection(self): """Reset the column selection state. @@ -184,7 +300,8 @@ cdef class HybridScanReader: Resets the internal column selection state forcing re-selection of columns in subsequent filter and read operations """ - self.c_obj.get()[0].reset_column_selection() + with nogil: + self.c_obj.get()[0].reset_column_selection() def filter_row_groups_with_stats( self, @@ -210,15 +327,15 @@ cdef class HybridScanReader: """ cdef Stream _stream = _get_stream(stream) cdef vector[size_type] indices_vec = row_group_indices - cdef vector[size_type] filtered = ( - self.c_obj.get()[0].filter_row_groups_with_stats( + cdef vector[size_type] filtered + with nogil: + filtered = move(self.c_obj.get()[0].filter_row_groups_with_stats( std_span[const_size_type]( indices_vec.data(), indices_vec.size() ), options.c_obj, _stream.view().value() - ) - ) + )) return list(filtered) def secondary_filters_byte_ranges( @@ -241,11 +358,13 @@ cdef class HybridScanReader: Tuple of (bloom_filter_ranges, dictionary_page_ranges) """ cdef vector[size_type] indices_vec = row_group_indices - cdef pair[vector[byte_range_info], vector[byte_range_info]] ranges = \ - self.c_obj.get()[0].secondary_filters_byte_ranges( + cdef pair[vector[byte_range_info], vector[byte_range_info]] ranges + cdef cpp_hybrid_scan_reader* reader_ptr = self.c_obj.get() + with nogil: + ranges = move(reader_ptr.secondary_filters_byte_ranges( std_span[const_size_type](indices_vec.data(), indices_vec.size()), options.c_obj - ) + )) bloom_ranges = [ ByteRangeInfo(r.offset(), r.size()) for r in ranges.first @@ -287,15 +406,16 @@ cdef class HybridScanReader: cdef vector[size_type] indices_vec = row_group_indices - cdef vector[size_type] filtered = \ - self.c_obj.get()[0].filter_row_groups_with_dictionary_pages( + cdef vector[size_type] filtered + with nogil: + filtered = move(self.c_obj.get()[0].filter_row_groups_with_dictionary_pages( std_span[const_device_span_const_uint8_t]( spans_vec.data(), spans_vec.size() ), std_span[const_size_type](indices_vec.data(), indices_vec.size()), options.c_obj, _stream.view().value() - ) + )) return list(filtered) def filter_row_groups_with_bloom_filters( @@ -330,17 +450,52 @@ cdef class HybridScanReader: cdef vector[size_type] indices_vec = row_group_indices - cdef vector[size_type] filtered = \ - self.c_obj.get()[0].filter_row_groups_with_bloom_filters( + cdef vector[size_type] filtered + with nogil: + filtered = move(self.c_obj.get()[0].filter_row_groups_with_bloom_filters( std_span[const_device_span_const_uint8_t]( spans_vec.data(), spans_vec.size() ), std_span[const_size_type](indices_vec.data(), indices_vec.size()), options.c_obj, _stream.view().value() - ) + )) return list(filtered) + def build_all_true_row_mask( + self, + list row_group_indices, + object stream=None, + DeviceMemoryResource mr=None + ): + """Build an all-true boolean survival column for the given row groups. + + Parameters + ---------- + row_group_indices : list[int] + Input row group indices + stream : Stream, optional + CUDA stream + mr : DeviceMemoryResource, optional + Device memory resource + + Returns + ------- + Column + All-true boolean column with one entry per row across all row groups + """ + cdef vector[size_type] indices_vec = row_group_indices + cdef Stream _stream = _get_stream(stream) + mr = _get_memory_resource(mr) + cdef unique_ptr[column] c_result + with nogil: + c_result = move(self.c_obj.get()[0].build_all_true_row_mask( + host_span[const_size_type](indices_vec.data(), indices_vec.size()), + _stream.view().value(), + mr.get_mr() + )) + return Column.from_libcudf(move(c_result), _stream, mr) + def build_row_mask_with_page_index_stats( self, list row_group_indices, @@ -369,13 +524,14 @@ cdef class HybridScanReader: cdef vector[size_type] indices_vec = row_group_indices cdef Stream _stream = _get_stream(stream) mr = _get_memory_resource(mr) - cdef unique_ptr[column] c_result = \ - self.c_obj.get()[0].build_row_mask_with_page_index_stats( + cdef unique_ptr[column] c_result + with nogil: + c_result = move(self.c_obj.get()[0].build_row_mask_with_page_index_stats( std_span[const_size_type](indices_vec.data(), indices_vec.size()), options.c_obj, _stream.view().value(), mr.get_mr() - ) + )) return Column.from_libcudf(move(c_result), _stream, mr) def filter_column_chunks_byte_ranges( @@ -398,11 +554,12 @@ cdef class HybridScanReader: Byte ranges to column chunks of filter columns """ cdef vector[size_type] indices_vec = row_group_indices - cdef vector[byte_range_info] ranges = \ - self.c_obj.get()[0].filter_column_chunks_byte_ranges( + cdef vector[byte_range_info] ranges + with nogil: + ranges = move(self.c_obj.get()[0].filter_column_chunks_byte_ranges( std_span[const_size_type](indices_vec.data(), indices_vec.size()), options.c_obj - ) + )) return [ByteRangeInfo(r.offset(), r.size()) for r in ranges] def materialize_filter_columns( @@ -448,8 +605,9 @@ cdef class HybridScanReader: spans_vec.push_back(_get_device_span(span)) cdef mutable_column_view mask_view = row_mask.mutable_view() - cdef table_with_metadata c_result = \ - self.c_obj.get()[0].materialize_filter_columns( + cdef table_with_metadata c_result + with nogil: + c_result = move(self.c_obj.get()[0].materialize_filter_columns( std_span[const_size_type](indices_vec.data(), indices_vec.size()), std_span[const_device_span_const_uint8_t]( spans_vec.data(), spans_vec.size() @@ -459,7 +617,7 @@ cdef class HybridScanReader: options.c_obj, _stream.view().value(), mr.get_mr() - ) + )) return TableWithMetadata.from_libcudf(c_result, _stream, mr) def payload_column_chunks_byte_ranges( @@ -482,11 +640,12 @@ cdef class HybridScanReader: Byte ranges to column chunks of payload columns """ cdef vector[size_type] indices_vec = row_group_indices - cdef vector[byte_range_info] ranges = \ - self.c_obj.get()[0].payload_column_chunks_byte_ranges( + cdef vector[byte_range_info] ranges + with nogil: + ranges = move(self.c_obj.get()[0].payload_column_chunks_byte_ranges( std_span[const_size_type](indices_vec.data(), indices_vec.size()), options.c_obj - ) + )) return [ByteRangeInfo(r.offset(), r.size()) for r in ranges] def materialize_payload_columns( @@ -532,8 +691,9 @@ cdef class HybridScanReader: spans_vec.push_back(_get_device_span(span)) cdef column_view mask_view = row_mask.view() - cdef table_with_metadata c_result = \ - self.c_obj.get()[0].materialize_payload_columns( + cdef table_with_metadata c_result + with nogil: + c_result = move(self.c_obj.get()[0].materialize_payload_columns( std_span[const_size_type](indices_vec.data(), indices_vec.size()), std_span[const_device_span_const_uint8_t]( spans_vec.data(), spans_vec.size() @@ -543,7 +703,7 @@ cdef class HybridScanReader: options.c_obj, _stream.view().value(), mr.get_mr() - ) + )) return TableWithMetadata.from_libcudf(c_result, _stream, mr) def all_column_chunks_byte_ranges( @@ -566,11 +726,12 @@ cdef class HybridScanReader: Byte ranges to column chunks of all columns """ cdef vector[size_type] indices_vec = row_group_indices - cdef vector[byte_range_info] ranges = \ - self.c_obj.get()[0].all_column_chunks_byte_ranges( + cdef vector[byte_range_info] ranges + with nogil: + ranges = move(self.c_obj.get()[0].all_column_chunks_byte_ranges( std_span[const_size_type](indices_vec.data(), indices_vec.size()), options.c_obj - ) + )) return [ByteRangeInfo(r.offset(), r.size()) for r in ranges] def materialize_all_columns( @@ -608,8 +769,9 @@ cdef class HybridScanReader: mr = _get_memory_resource(mr) for span in column_chunk_data: spans_vec.push_back(_get_device_span(span)) - cdef table_with_metadata c_result = \ - self.c_obj.get()[0].materialize_all_columns( + cdef table_with_metadata c_result + with nogil: + c_result = move(self.c_obj.get()[0].materialize_all_columns( std_span[const_size_type](indices_vec.data(), indices_vec.size()), std_span[const_device_span_const_uint8_t]( spans_vec.data(), spans_vec.size() @@ -617,7 +779,7 @@ cdef class HybridScanReader: options.c_obj, _stream.view().value(), mr.get_mr() - ) + )) return TableWithMetadata.from_libcudf(c_result, _stream, mr) def setup_chunking_for_filter_columns( @@ -663,21 +825,23 @@ cdef class HybridScanReader: self._stream = _get_stream(stream) self.mr = _get_memory_resource(mr) + self._filter_chunk_data = column_chunk_data cdef column_view mask_view = row_mask.view() - self.c_obj.get()[0].setup_chunking_for_filter_columns( - chunk_read_limit, - pass_read_limit, - std_span[const_size_type](indices_vec.data(), indices_vec.size()), - mask_view, - mask_data_pages, - std_span[const_device_span_const_uint8_t]( - spans_vec.data(), spans_vec.size() - ), - options.c_obj, - self._stream.view().value(), - self.mr.get_mr() - ) + with nogil: + self.c_obj.get()[0].setup_chunking_for_filter_columns( + chunk_read_limit, + pass_read_limit, + std_span[const_size_type](indices_vec.data(), indices_vec.size()), + mask_view, + mask_data_pages, + std_span[const_device_span_const_uint8_t]( + spans_vec.data(), spans_vec.size() + ), + options.c_obj, + self._stream.view().value(), + self.mr.get_mr() + ) def materialize_filter_columns_chunk( self, @@ -695,10 +859,12 @@ cdef class HybridScanReader: Table chunk of materialized filter columns and metadata """ cdef mutable_column_view mask_view = row_mask.mutable_view() - cdef table_with_metadata c_result = \ - self.c_obj.get()[0].materialize_filter_columns_chunk( + cdef table_with_metadata c_result + with nogil: + c_result = move(self.c_obj.get()[0].materialize_filter_columns_chunk( mask_view - ) + )) + self._filter_chunk_data = None return TableWithMetadata.from_libcudf( c_result, self._stream, self.mr ) @@ -746,21 +912,23 @@ cdef class HybridScanReader: self._stream = _get_stream(stream) self.mr = _get_memory_resource(mr) + self._payload_chunk_data = column_chunk_data cdef column_view mask_view = row_mask.view() - self.c_obj.get()[0].setup_chunking_for_payload_columns( - chunk_read_limit, - pass_read_limit, - std_span[const_size_type](indices_vec.data(), indices_vec.size()), - mask_view, - mask_data_pages, - std_span[const_device_span_const_uint8_t]( - spans_vec.data(), spans_vec.size() - ), - options.c_obj, - self._stream.view().value(), - self.mr.get_mr() - ) + with nogil: + self.c_obj.get()[0].setup_chunking_for_payload_columns( + chunk_read_limit, + pass_read_limit, + std_span[const_size_type](indices_vec.data(), indices_vec.size()), + mask_view, + mask_data_pages, + std_span[const_device_span_const_uint8_t]( + spans_vec.data(), spans_vec.size() + ), + options.c_obj, + self._stream.view().value(), + self.mr.get_mr() + ) def materialize_payload_columns_chunk( self, @@ -778,10 +946,12 @@ cdef class HybridScanReader: Table chunk of materialized payload columns and metadata """ cdef column_view mask_view = row_mask.view() - cdef table_with_metadata c_result = \ - self.c_obj.get()[0].materialize_payload_columns_chunk( + cdef table_with_metadata c_result + with nogil: + c_result = move(self.c_obj.get()[0].materialize_payload_columns_chunk( mask_view - ) + )) + self._payload_chunk_data = None return TableWithMetadata.from_libcudf( c_result, self._stream, self.mr ) @@ -817,12 +987,15 @@ cdef class HybridScanReader: If ``row_group_indices`` is empty. """ cdef vector[size_type] indices_vec = row_group_indices - return self.c_obj.get()[0].construct_row_group_passes( - std_span[const_size_type]( - indices_vec.data(), indices_vec.size() - ), - pass_read_limit - ) + cdef vector[vector[size_type]] passes + with nogil: + passes = move(self.c_obj.get()[0].construct_row_group_passes( + std_span[const_size_type]( + indices_vec.data(), indices_vec.size() + ), + pass_read_limit + )) + return passes def has_next_table_chunk(self): """Check if there is any parquet data left to read. @@ -832,7 +1005,10 @@ cdef class HybridScanReader: bool True if there is data left to read """ - return self.c_obj.get()[0].has_next_table_chunk() + cdef bool result + with nogil: + result = self.c_obj.get()[0].has_next_table_chunk() + return result UseDataPageMask.__str__ = UseDataPageMask.__repr__ diff --git a/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd b/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd index 36201d545de6..d8b9255a9dab 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd @@ -30,6 +30,17 @@ cdef extern from "cudf/io/experimental/hybrid_scan.hpp" \ YES NO + cdef cppclass hybrid_scan_metadata: + hybrid_scan_metadata( + host_span[const_uint8_t] footer_bytes, + const parquet_reader_options& options + ) except +libcudf_exception_handler + + hybrid_scan_metadata( + const FileMetaData& parquet_metadata, + const parquet_reader_options& options + ) except +libcudf_exception_handler + cdef cppclass hybrid_scan_reader: hybrid_scan_reader( host_span[const_uint8_t] footer_bytes, @@ -41,6 +52,10 @@ cdef extern from "cudf/io/experimental/hybrid_scan.hpp" \ const parquet_reader_options& options ) except +libcudf_exception_handler + hybrid_scan_reader( + const hybrid_scan_metadata& metadata + ) except +libcudf_exception_handler + FileMetaData parquet_metadata() except +libcudf_exception_handler byte_range_info page_index_byte_range() except +libcudf_exception_handler @@ -86,6 +101,12 @@ cdef extern from "cudf/io/experimental/hybrid_scan.hpp" \ cudaStream_t stream ) except +libcudf_exception_handler + unique_ptr[column] build_all_true_row_mask( + host_span[const_size_type] row_group_indices, + cudaStream_t stream, + device_async_resource_ref mr + ) except +libcudf_exception_handler + unique_ptr[column] build_row_mask_with_page_index_stats( std_span[const_size_type] row_group_indices, const parquet_reader_options& options, From 06df8af11a0755a4fbef94713d246208a4453bdc Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 5 Aug 2026 23:12:23 +0000 Subject: [PATCH 2/9] address reviews --- .../io/experimental/hybrid_scan_test.cpp | 49 +++++++++++++++++++ .../pylibcudf/io/experimental/hybrid_scan.pyx | 38 ++++++++------ 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/cpp/tests/io/experimental/hybrid_scan_test.cpp b/cpp/tests/io/experimental/hybrid_scan_test.cpp index 7e4e633d9be1..6e24191a4e5a 100644 --- a/cpp/tests/io/experimental/hybrid_scan_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_test.cpp @@ -1031,6 +1031,55 @@ TEST_F(HybridScanTest, SharedMetadataReaderMatchesReadParquet) CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected->view(), table_b->view()); } +TEST_F(HybridScanTest, SharedMetadataFromFileMetaDataMatchesReadParquet) +{ + using T = int32_t; + auto constexpr num_concat = 2; + auto [written_table, parquet_buffer] = create_parquet_with_stats(); + + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + auto const options = cudf::io::parquet_reader_options::builder().build(); + + auto datasource = cudf::io::datasource::create(cudf::host_span( + reinterpret_cast(parquet_buffer.data()), parquet_buffer.size())); + + // Obtain FileMetaData from an initial reader, then build shared metadata from it. + auto const footer_buffer = cudf::io::parquet::fetch_footer_to_host(*datasource); + auto const seed_reader = + std::make_unique(*footer_buffer, options); + auto const file_metadata = seed_reader->parquet_metadata(); + + auto const metadata = + std::make_shared(file_metadata, options); + + // Two readers sharing the FileMetaData-derived metadata each produce the correct table. + auto const read_all_columns = [&] { + auto const reader = + std::make_unique(*metadata); + auto const row_groups = reader->all_row_groups(options); + auto const chunk_ranges = reader->all_column_chunks_byte_ranges(row_groups, options); + auto [buffers, data, tasks] = + cudf::io::parquet::fetch_byte_ranges_to_device_async(*datasource, chunk_ranges, stream, mr); + tasks.get(); + return reader->materialize_all_columns(row_groups, data, options, stream, mr).tbl; + }; + + auto const table_a = read_all_columns(); + auto const table_b = read_all_columns(); + + auto const expected = + cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info(cudf::host_span(parquet_buffer.data(), parquet_buffer.size()))) + .build(), + stream) + .tbl; + + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected->view(), table_a->view()); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected->view(), table_b->view()); +} + TEST_F(HybridScanTest, AllRowsPrunedReportsInputRowGroups) { using cudf::io::parquet::experimental::use_data_page_mask; diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx index a81522b71845..eb6018e2dc47 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx @@ -376,7 +376,7 @@ cdef class HybridScanReader: def filter_row_groups_with_dictionary_pages( self, - list dictionary_page_data, + object dictionary_page_data, list row_group_indices, ParquetReaderOptions options, object stream=None @@ -385,7 +385,7 @@ cdef class HybridScanReader: Parameters ---------- - dictionary_page_data : list + dictionary_page_data : Sequence Span-like objects containing dictionary page data row_group_indices : list[int] Input row group indices @@ -420,7 +420,7 @@ cdef class HybridScanReader: def filter_row_groups_with_bloom_filters( self, - list bloom_filter_data, + object bloom_filter_data, list row_group_indices, ParquetReaderOptions options, object stream=None @@ -429,7 +429,7 @@ cdef class HybridScanReader: Parameters ---------- - bloom_filter_data : list + bloom_filter_data : Sequence Span-like objects containing bloom filter data row_group_indices : list[int] Input row group indices @@ -565,7 +565,7 @@ cdef class HybridScanReader: def materialize_filter_columns( self, list row_group_indices, - list column_chunk_data, + object column_chunk_data, Column row_mask, cpp_use_data_page_mask mask_data_pages, ParquetReaderOptions options, @@ -578,7 +578,7 @@ cdef class HybridScanReader: ---------- row_group_indices : list[int] Input row group indices - column_chunk_data : list + column_chunk_data : Sequence Span-like objects containing column chunk data of filter columns row_mask : Column Mutable boolean column indicating surviving rows @@ -651,7 +651,7 @@ cdef class HybridScanReader: def materialize_payload_columns( self, list row_group_indices, - list column_chunk_data, + object column_chunk_data, Column row_mask, cpp_use_data_page_mask mask_data_pages, ParquetReaderOptions options, @@ -664,7 +664,7 @@ cdef class HybridScanReader: ---------- row_group_indices : list[int] Input row group indices - column_chunk_data : list + column_chunk_data : Sequence Span-like objects containing column chunk data of payload columns row_mask : Column Boolean column indicating surviving rows @@ -737,7 +737,7 @@ cdef class HybridScanReader: def materialize_all_columns( self, list row_group_indices, - list column_chunk_data, + object column_chunk_data, ParquetReaderOptions options, object stream=None, DeviceMemoryResource mr=None @@ -748,7 +748,7 @@ cdef class HybridScanReader: ---------- row_group_indices : list[int] Input row group indices - column_chunk_data : list + column_chunk_data : Sequence Span-like objects containing column chunk data of all columns options : ParquetReaderOptions Parquet reader options @@ -789,7 +789,7 @@ cdef class HybridScanReader: list row_group_indices, Column row_mask, cpp_use_data_page_mask mask_data_pages, - list column_chunk_data, + object column_chunk_data, ParquetReaderOptions options, object stream=None, DeviceMemoryResource mr=None @@ -808,7 +808,7 @@ cdef class HybridScanReader: Boolean column indicating surviving rows mask_data_pages : UseDataPageMask Whether to use a data page mask - column_chunk_data : list + column_chunk_data : Sequence Span-like objects containing column chunk data of filter columns options : ParquetReaderOptions Parquet reader options @@ -860,11 +860,14 @@ cdef class HybridScanReader: """ cdef mutable_column_view mask_view = row_mask.mutable_view() cdef table_with_metadata c_result + cdef bool more_chunks with nogil: c_result = move(self.c_obj.get()[0].materialize_filter_columns_chunk( mask_view )) - self._filter_chunk_data = None + more_chunks = self.c_obj.get()[0].has_next_table_chunk() + if not more_chunks: + self._filter_chunk_data = None return TableWithMetadata.from_libcudf( c_result, self._stream, self.mr ) @@ -876,7 +879,7 @@ cdef class HybridScanReader: list row_group_indices, Column row_mask, cpp_use_data_page_mask mask_data_pages, - list column_chunk_data, + object column_chunk_data, ParquetReaderOptions options, object stream=None, DeviceMemoryResource mr=None @@ -895,7 +898,7 @@ cdef class HybridScanReader: Boolean column indicating surviving rows mask_data_pages : UseDataPageMask Whether to use a data page mask - column_chunk_data : list + column_chunk_data : Sequence Span-like objects containing column chunk data of payload columns options : ParquetReaderOptions Parquet reader options @@ -947,11 +950,14 @@ cdef class HybridScanReader: """ cdef column_view mask_view = row_mask.view() cdef table_with_metadata c_result + cdef bool more_chunks with nogil: c_result = move(self.c_obj.get()[0].materialize_payload_columns_chunk( mask_view )) - self._payload_chunk_data = None + more_chunks = self.c_obj.get()[0].has_next_table_chunk() + if not more_chunks: + self._payload_chunk_data = None return TableWithMetadata.from_libcudf( c_result, self._stream, self.mr ) From 5627e4076dc50fc5f806d2049dfd56993139dd4c Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 6 Aug 2026 02:17:00 +0000 Subject: [PATCH 3/9] change self to result --- .../pylibcudf/io/experimental/hybrid_scan.pyx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx index eb6018e2dc47..3afe2beb29ef 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx @@ -92,16 +92,16 @@ cdef class HybridScanMetadata: ------- HybridScanMetadata """ - cdef HybridScanMetadata self = HybridScanMetadata.__new__(HybridScanMetadata) + cdef HybridScanMetadata result = HybridScanMetadata.__new__(HybridScanMetadata) cdef const uint8_t* footer_ptr = 0 if len(footer_bytes) > 0: footer_ptr = &footer_bytes[0] with nogil: - self.c_obj = make_unique[cpp_hybrid_scan_metadata]( + result.c_obj = make_unique[cpp_hybrid_scan_metadata]( host_span[const_uint8_t](footer_ptr, len(footer_bytes)), options.c_obj ) - return self + return result @staticmethod def from_parquet_metadata(c_FileMetaData metadata, ParquetReaderOptions options): @@ -118,13 +118,13 @@ cdef class HybridScanMetadata: ------- HybridScanMetadata """ - cdef HybridScanMetadata self = HybridScanMetadata.__new__(HybridScanMetadata) + cdef HybridScanMetadata result = HybridScanMetadata.__new__(HybridScanMetadata) with nogil: - self.c_obj = make_unique[cpp_hybrid_scan_metadata]( + result.c_obj = make_unique[cpp_hybrid_scan_metadata]( metadata.c_obj, options.c_obj ) - return self + return result cdef class HybridScanReader: From be697fe5133baf9cadb581e0ead8822214f41a96 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 6 Aug 2026 12:13:58 +0000 Subject: [PATCH 4/9] address reviews --- cpp/include/cudf/io/experimental/hybrid_scan.hpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cpp/include/cudf/io/experimental/hybrid_scan.hpp b/cpp/include/cudf/io/experimental/hybrid_scan.hpp index 560ed0af18d0..a605530e1974 100644 --- a/cpp/include/cudf/io/experimental/hybrid_scan.hpp +++ b/cpp/include/cudf/io/experimental/hybrid_scan.hpp @@ -75,9 +75,8 @@ enum class use_data_page_mask : bool { * @endcode * * @note The metadata is immutable after `setup_page_index()` has been called (or immediately after - * construction if page index setup is skipped). Readers sharing one instance may read different - * row-group ranges of the same single file concurrently; overlapping ranges produce duplicate rows. - * This handle does not support multi-source (multi-file) metadata. + * construction if page index setup is skipped). Concurrent usage by multiple readers is thread + * safe. This handle does not support multi-source (multi-file) metadata. */ class hybrid_scan_metadata { public: @@ -375,9 +374,7 @@ class hybrid_scan_reader { * @brief Constructor that borrows shared, pre-parsed Parquet file metadata * * Constructs a reader that shares `metadata` instead of parsing and copying the file metadata - * again. Use this to read disjoint row-group ranges of a single file without paying the metadata - * copy per reader. The reader options that govern reading (column selection, filter, ...) are - * supplied per call to the individual read methods. + * again. * * @param metadata Shared, pre-parsed Parquet file metadata */ From 916cd4c34cf779f583a2b7ba1a06ea0d4d66ccff Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 20:06:54 +0000 Subject: [PATCH 5/9] address review --- cpp/include/cudf/io/experimental/hybrid_scan.hpp | 9 ++++----- cpp/src/io/parquet/experimental/hybrid_scan.cpp | 4 ++-- cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp | 6 +----- python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/cpp/include/cudf/io/experimental/hybrid_scan.hpp b/cpp/include/cudf/io/experimental/hybrid_scan.hpp index a605530e1974..bc9fe136b419 100644 --- a/cpp/include/cudf/io/experimental/hybrid_scan.hpp +++ b/cpp/include/cudf/io/experimental/hybrid_scan.hpp @@ -61,7 +61,7 @@ enum class use_data_page_mask : bool { * @brief Shareable, pre-parsed Parquet file metadata for the Hybrid Scan reader. * * Parses the Parquet file metadata once so that multiple `hybrid_scan_reader` instances reading - * the same file can borrow it rather than each re-parsing and copying the row group metadata. + * the same file can share it rather than each re-parsing and copying the row group metadata. * The intended use is to read disjoint row-group ranges of a single file: construct one * `hybrid_scan_metadata` per file and pass it to as many readers as there are ranges. * @@ -371,14 +371,13 @@ class hybrid_scan_reader { parquet_reader_options const& options); /** - * @brief Constructor that borrows shared, pre-parsed Parquet file metadata + * @brief Constructor that takes shared ownership of pre-parsed Parquet file metadata * - * Constructs a reader that shares `metadata` instead of parsing and copying the file metadata - * again. + * Constructs a reader that shares the pre-parsed metadata object. * * @param metadata Shared, pre-parsed Parquet file metadata */ - explicit hybrid_scan_reader(hybrid_scan_metadata const& metadata); + explicit hybrid_scan_reader(hybrid_scan_metadata metadata); /** * @brief Destructor for the experimental parquet reader class diff --git a/cpp/src/io/parquet/experimental/hybrid_scan.cpp b/cpp/src/io/parquet/experimental/hybrid_scan.cpp index 3ded187fe997..8c1734cd8ede 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan.cpp @@ -47,8 +47,8 @@ hybrid_scan_reader::hybrid_scan_reader(FileMetaData const& parquet_metadata, { } -hybrid_scan_reader::hybrid_scan_reader(hybrid_scan_metadata const& metadata) - : _impl{std::make_unique(metadata._metadata)} +hybrid_scan_reader::hybrid_scan_reader(hybrid_scan_metadata metadata) + : _impl{std::make_unique(std::move(metadata._metadata))} { } diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp index 7b5a638ff2b5..b4265e6ace01 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp @@ -60,11 +60,7 @@ class hybrid_scan_reader_impl : public parquet::detail::reader_impl { parquet_reader_options const& options); /** - * @brief Constructor that shares pre-parsed Parquet metadata - * - * Borrows an already-constructed `aggregate_reader_metadata` instead of parsing and copying the - * file metadata again. Multiple single-file readers can share one metadata object, avoiding a - * per-reader copy of the (potentially large) row group metadata. + * @brief Constructor that takes shared ownership of pre-parsed Parquet metadata * * @param metadata Shared, pre-parsed Parquet file metadata. Must not be null. */ diff --git a/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd b/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd index d8b9255a9dab..3ff739a1aab7 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd @@ -53,7 +53,7 @@ cdef extern from "cudf/io/experimental/hybrid_scan.hpp" \ ) except +libcudf_exception_handler hybrid_scan_reader( - const hybrid_scan_metadata& metadata + hybrid_scan_metadata metadata ) except +libcudf_exception_handler FileMetaData parquet_metadata() except +libcudf_exception_handler From bb57d2fee886503443f00888124a8592a7a1fb64 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 20:15:36 +0000 Subject: [PATCH 6/9] add use-after-free comment --- python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx index 3afe2beb29ef..bfd090c7eaa1 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx @@ -825,6 +825,7 @@ cdef class HybridScanReader: self._stream = _get_stream(stream) self.mr = _get_memory_resource(mr) + # keep reference to avoid use-after-free of device spans self._filter_chunk_data = column_chunk_data cdef column_view mask_view = row_mask.view() From 2dfac4052b5e86e4108df6c38307293f2b280bf2 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 20:19:35 +0000 Subject: [PATCH 7/9] bot review --- .../pylibcudf/io/experimental/hybrid_scan.pyx | 10 ++++++++-- python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx index bfd090c7eaa1..54d7d28e3db9 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx @@ -74,6 +74,12 @@ cdef class HybridScanMetadata: >>> reader = plc.io.experimental.HybridScanReader.from_metadata(metadata) """ + def __init__(self): + raise ValueError( + "HybridScanMetadata cannot be constructed directly. " + "Use from_footer_bytes() or from_parquet_metadata()." + ) + @staticmethod def from_footer_bytes( const uint8_t[::1] footer_bytes, @@ -189,7 +195,7 @@ cdef class HybridScanReader: return reader @staticmethod - def from_metadata(HybridScanMetadata metadata): + def from_metadata(HybridScanMetadata metadata not None): """Create a HybridScanReader that shares pre-parsed metadata. Constructs a lightweight reader that borrows ``metadata`` instead of @@ -490,7 +496,7 @@ cdef class HybridScanReader: cdef unique_ptr[column] c_result with nogil: c_result = move(self.c_obj.get()[0].build_all_true_row_mask( - host_span[const_size_type](indices_vec.data(), indices_vec.size()), + std_span[const_size_type](indices_vec.data(), indices_vec.size()), _stream.view().value(), mr.get_mr() )) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd b/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd index 3ff739a1aab7..7a5aec269a56 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/hybrid_scan.pxd @@ -102,7 +102,7 @@ cdef extern from "cudf/io/experimental/hybrid_scan.hpp" \ ) except +libcudf_exception_handler unique_ptr[column] build_all_true_row_mask( - host_span[const_size_type] row_group_indices, + std_span[const_size_type] row_group_indices, cudaStream_t stream, device_async_resource_ref mr ) except +libcudf_exception_handler From 2b09a072cd8bc36d42e93d136d3bc9bd0544c13f Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 18 Aug 2026 16:02:02 +0000 Subject: [PATCH 8/9] simplify docs --- cpp/include/cudf/io/experimental/hybrid_scan.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cpp/include/cudf/io/experimental/hybrid_scan.hpp b/cpp/include/cudf/io/experimental/hybrid_scan.hpp index 62f178bc8ad8..772b65e62fc9 100644 --- a/cpp/include/cudf/io/experimental/hybrid_scan.hpp +++ b/cpp/include/cudf/io/experimental/hybrid_scan.hpp @@ -68,11 +68,10 @@ enum class use_data_page_mask : bool { * * @code{.cpp} * // Parse the metadata once - * auto metadata = std::make_shared(*footer_buffer, - * options); + * auto metadata = parquet::experimental::hybrid_scan_metadata{*footer_buffer, options}; * // Construct lightweight readers that share it - * auto reader_a = std::make_unique(*metadata); - * auto reader_b = std::make_unique(*metadata); + * auto reader_a = std::make_unique(metadata); + * auto reader_b = std::make_unique(metadata); * @endcode * * @note The metadata is immutable after `setup_page_index()` has been called (or immediately after From 035036743cfa661b9467666fcc268d2fc0f9a1df Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Fri, 21 Aug 2026 01:41:36 +0000 Subject: [PATCH 9/9] add a test with multiple hybrid scan readers --- .../io/experimental/hybrid_scan_test.cpp | 84 +++++++++++++++++-- .../pylibcudf/io/experimental/hybrid_scan.pyx | 9 +- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/cpp/tests/io/experimental/hybrid_scan_test.cpp b/cpp/tests/io/experimental/hybrid_scan_test.cpp index 0a5c1ce01321..b4b8c212f7f0 100644 --- a/cpp/tests/io/experimental/hybrid_scan_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_test.cpp @@ -1000,13 +1000,13 @@ TEST_F(HybridScanTest, SharedMetadataReaderMatchesReadParquet) auto const footer_buffer = cudf::io::parquet::fetch_footer_to_host(*datasource); // Parse the file metadata once and share it across independent readers. - auto const metadata = std::make_shared( - *footer_buffer, options); + auto const metadata = + cudf::io::parquet::experimental::hybrid_scan_metadata{*footer_buffer, options}; // Read all columns (single step) through a reader that borrows the shared metadata. auto const read_all_columns = [&] { auto const reader = - std::make_unique(*metadata); + std::make_unique(metadata); auto const row_groups = reader->all_row_groups(options); auto const chunk_ranges = reader->all_column_chunks_byte_ranges(row_groups, options); auto [buffers, data, tasks] = @@ -1051,12 +1051,12 @@ TEST_F(HybridScanTest, SharedMetadataFromFileMetaDataMatchesReadParquet) auto const file_metadata = seed_reader->parquet_metadata(); auto const metadata = - std::make_shared(file_metadata, options); + cudf::io::parquet::experimental::hybrid_scan_metadata{file_metadata, options}; // Two readers sharing the FileMetaData-derived metadata each produce the correct table. auto const read_all_columns = [&] { auto const reader = - std::make_unique(*metadata); + std::make_unique(metadata); auto const row_groups = reader->all_row_groups(options); auto const chunk_ranges = reader->all_column_chunks_byte_ranges(row_groups, options); auto [buffers, data, tasks] = @@ -1080,6 +1080,80 @@ TEST_F(HybridScanTest, SharedMetadataFromFileMetaDataMatchesReadParquet) CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected->view(), table_b->view()); } +TEST_F(HybridScanTest, SharedMetadataConcurrentReadersMatchReadParquet) +{ + using T = uint32_t; + auto constexpr num_concat = 4; + auto [written_table, parquet_buffer] = create_parquet_with_stats(); + + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + auto const options = cudf::io::parquet_reader_options::builder().build(); + + auto datasource = cudf::io::datasource::create(cudf::host_span( + reinterpret_cast(parquet_buffer.data()), parquet_buffer.size())); + auto const footer_buffer = cudf::io::parquet::fetch_footer_to_host(*datasource); + + // Parse the metadata once and share it, by value, across two readers that are alive at the + // same time, each responsible for a disjoint range of the file's row groups. + auto const metadata = + cudf::io::parquet::experimental::hybrid_scan_metadata{*footer_buffer, options}; + auto const reader_a = + std::make_unique(metadata); + auto const reader_b = + std::make_unique(metadata); + + // Only one of the readers sharing this metadata sets up the page index. + auto const page_index_byte_range = reader_a->page_index_byte_range(); + ASSERT_FALSE(page_index_byte_range.is_empty()); + auto const page_index_buffer = + cudf::io::parquet::fetch_page_index_to_host(*datasource, page_index_byte_range); + reader_a->setup_page_index(*page_index_buffer); + + // The page index materialized through `reader_a` must be visible through `reader_b` since both + // readers share the same underlying metadata. + auto const metadata_from_b = reader_b->parquet_metadata(); + ASSERT_GT(metadata_from_b.row_groups.size(), 1); + for (auto const& row_group : metadata_from_b.row_groups) { + for (auto const& column_chunk : row_group.columns) { + EXPECT_TRUE(column_chunk.column_index.has_value()); + EXPECT_TRUE(column_chunk.offset_index.has_value()); + } + } + + // Split the row groups into two disjoint ranges and read each range through a different reader + // sharing the metadata, with both readers alive and used concurrently. + auto const all_row_groups = reader_a->all_row_groups(options); + auto const split = all_row_groups.size() / 2; + auto const row_groups_a = + std::vector(all_row_groups.begin(), all_row_groups.begin() + split); + auto const row_groups_b = + std::vector(all_row_groups.begin() + split, all_row_groups.end()); + + auto const materialize = [&](auto const& reader, auto const& row_group_indices) { + auto const chunk_ranges = reader->all_column_chunks_byte_ranges(row_group_indices, options); + auto [buffers, data, tasks] = + cudf::io::parquet::fetch_byte_ranges_to_device_async(*datasource, chunk_ranges, stream, mr); + tasks.get(); + return reader->materialize_all_columns(row_group_indices, data, options, stream, mr).tbl; + }; + + auto const table_a = materialize(reader_a, row_groups_a); + auto const table_b = materialize(reader_b, row_groups_b); + + auto const table = + cudf::concatenate(std::vector{table_a->view(), table_b->view()}); + auto const expected = + cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info(cudf::host_span(parquet_buffer.data(), parquet_buffer.size()))) + .build(), + stream) + .tbl; + + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected->view(), table->view()); +} + TEST_F(HybridScanTest, AllRowsPrunedReportsInputRowGroups) { using cudf::io::parquet::experimental::use_data_page_mask; diff --git a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx index 1e88bd2de5a0..4cfc0214f4d8 100644 --- a/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx +++ b/python/pylibcudf/pylibcudf/io/experimental/hybrid_scan.pyx @@ -66,9 +66,10 @@ cdef device_span[const_uint8_t] _get_device_span(object obj) except *: cdef class HybridScanMetadata: """Shareable, pre-parsed Parquet file metadata for the hybrid scan reader. - Parse the metadata of a single Parquet file once, then construct multiple - :class:`HybridScanReader` instances that share it (one per row-group range of the - file) instead of each re-parsing and copying the metadata. + This class enables parsing the metadata of a Parquet file once, then + constructing multiple :class:`HybridScanReader` instances that share it + (one per row-group range of the file) instead of each re-parsing and copying + the metadata. For details, see :cpp:class:`cudf::io::parquet::experimental::hybrid_scan_metadata` @@ -133,7 +134,7 @@ cdef class HybridScanMetadata: cdef HybridScanMetadata result = HybridScanMetadata.__new__(HybridScanMetadata) with nogil: result.c_obj = make_unique[cpp_hybrid_scan_metadata]( - metadata.c_obj, + dereference(metadata.c_obj), options.c_obj ) return result