From 1d41f3e27c339c679a20b11b740d9dd5a0941904 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Tue, 2 Sep 2025 14:03:56 -0700 Subject: [PATCH 1/5] Step 1. Add into_builder function to convert WriterProperties into WriterPropertiesBuilder. --- parquet/src/file/properties.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index 96e3706e27d7..bc819a9592be 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -20,7 +20,7 @@ use crate::basic::{Compression, Encoding}; use crate::compression::{CodecOptions, CodecOptionsBuilder}; #[cfg(feature = "encryption")] use crate::encryption::encrypt::FileEncryptionProperties; -use crate::file::metadata::KeyValue; +use crate::file::metadata::{KeyValue, RowGroupMetaData, RowGroupMetaDataBuilder}; use crate::format::SortingColumn; use crate::schema::types::ColumnPath; use std::str::FromStr; @@ -193,6 +193,30 @@ impl WriterProperties { WriterPropertiesBuilder::default() } + /// Converts this [`RowGroupMetaData`] into a [`RowGroupMetaDataBuilder`] + pub fn into_builder(self) -> WriterPropertiesBuilder { + WriterPropertiesBuilder { + data_page_size_limit: self.data_page_size_limit, + data_page_row_count_limit: self.data_page_row_count_limit, + write_batch_size: self.write_batch_size, + max_row_group_size: self.max_row_group_size, + bloom_filter_position: self.bloom_filter_position, + writer_version: self.writer_version, + created_by: self.created_by, + offset_index_disabled: self.offset_index_disabled, + key_value_metadata: self.key_value_metadata, + default_column_properties: self.default_column_properties, + column_properties: self.column_properties, + sorting_columns: self.sorting_columns, + column_index_truncate_length: self.column_index_truncate_length, + statistics_truncate_length: self.statistics_truncate_length, + coerce_types: self.coerce_types, + #[cfg(feature = "encryption")] + file_encryption_properties: self.file_encryption_properties, + } + } + + /// Returns data page size limit. /// /// Note: this is a best effort limit based on the write batch size From e120327d6e49a49c73288ed7d914ef372c8d7cd2 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Tue, 2 Sep 2025 14:18:33 -0700 Subject: [PATCH 2/5] Add simple test, formatting, basic docs. --- parquet/src/file/properties.rs | 99 ++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index bc819a9592be..dbf0feac1b34 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -20,7 +20,7 @@ use crate::basic::{Compression, Encoding}; use crate::compression::{CodecOptions, CodecOptionsBuilder}; #[cfg(feature = "encryption")] use crate::encryption::encrypt::FileEncryptionProperties; -use crate::file::metadata::{KeyValue, RowGroupMetaData, RowGroupMetaDataBuilder}; +use crate::file::metadata::KeyValue; use crate::format::SortingColumn; use crate::schema::types::ColumnPath; use std::str::FromStr; @@ -193,7 +193,8 @@ impl WriterProperties { WriterPropertiesBuilder::default() } - /// Converts this [`RowGroupMetaData`] into a [`RowGroupMetaDataBuilder`] + /// Converts this [`WriterProperties`] into a [`WriterPropertiesBuilder`] + /// Used for mutating existing property settings pub fn into_builder(self) -> WriterPropertiesBuilder { WriterPropertiesBuilder { data_page_size_limit: self.data_page_size_limit, @@ -216,7 +217,6 @@ impl WriterProperties { } } - /// Returns data page size limit. /// /// Note: this is a best effort limit based on the write batch size @@ -1401,50 +1401,59 @@ mod tests { .set_column_bloom_filter_fpp(ColumnPath::from("col"), 0.1) .build(); - assert_eq!(props.writer_version(), WriterVersion::PARQUET_2_0); - assert_eq!(props.data_page_size_limit(), 10); - assert_eq!(props.dictionary_page_size_limit(), 20); - assert_eq!(props.write_batch_size(), 30); - assert_eq!(props.max_row_group_size(), 40); - assert_eq!(props.created_by(), "default"); - assert_eq!( - props.key_value_metadata(), - Some(&vec![ - KeyValue::new("key".to_string(), "value".to_string(),) - ]) - ); + fn test_props(props: &WriterProperties) { + assert_eq!(props.writer_version(), WriterVersion::PARQUET_2_0); + assert_eq!(props.data_page_size_limit(), 10); + assert_eq!(props.dictionary_page_size_limit(), 20); + assert_eq!(props.write_batch_size(), 30); + assert_eq!(props.max_row_group_size(), 40); + assert_eq!(props.created_by(), "default"); + assert_eq!( + props.key_value_metadata(), + Some(&vec![ + KeyValue::new("key".to_string(), "value".to_string(),) + ]) + ); - assert_eq!( - props.encoding(&ColumnPath::from("a")), - Some(Encoding::DELTA_BINARY_PACKED) - ); - assert_eq!( - props.compression(&ColumnPath::from("a")), - Compression::GZIP(Default::default()) - ); - assert!(!props.dictionary_enabled(&ColumnPath::from("a"))); - assert_eq!( - props.statistics_enabled(&ColumnPath::from("a")), - EnabledStatistics::None - ); + assert_eq!( + props.encoding(&ColumnPath::from("a")), + Some(Encoding::DELTA_BINARY_PACKED) + ); + assert_eq!( + props.compression(&ColumnPath::from("a")), + Compression::GZIP(Default::default()) + ); + assert!(!props.dictionary_enabled(&ColumnPath::from("a"))); + assert_eq!( + props.statistics_enabled(&ColumnPath::from("a")), + EnabledStatistics::None + ); - assert_eq!( - props.encoding(&ColumnPath::from("col")), - Some(Encoding::RLE) - ); - assert_eq!( - props.compression(&ColumnPath::from("col")), - Compression::SNAPPY - ); - assert!(props.dictionary_enabled(&ColumnPath::from("col"))); - assert_eq!( - props.statistics_enabled(&ColumnPath::from("col")), - EnabledStatistics::Chunk - ); - assert_eq!( - props.bloom_filter_properties(&ColumnPath::from("col")), - Some(&BloomFilterProperties { fpp: 0.1, ndv: 100 }) - ); + assert_eq!( + props.encoding(&ColumnPath::from("col")), + Some(Encoding::RLE) + ); + assert_eq!( + props.compression(&ColumnPath::from("col")), + Compression::SNAPPY + ); + assert!(props.dictionary_enabled(&ColumnPath::from("col"))); + assert_eq!( + props.statistics_enabled(&ColumnPath::from("col")), + EnabledStatistics::Chunk + ); + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("col")), + Some(&BloomFilterProperties { fpp: 0.1, ndv: 100 }) + ); + } + + // Test direct build of properties + test_props(&props); + + // Test that into_builder() gives the same result + let props_into_builder_and_back = props.into_builder().build(); + test_props(&props_into_builder_and_back); } #[test] From 1973996dd1324aacde5b8c4ca4002cf0fecc5146 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Tue, 2 Sep 2025 17:18:54 -0700 Subject: [PATCH 3/5] Move to_builder implementaion to From trait. --- parquet/src/file/properties.rs | 46 ++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index dbf0feac1b34..283cd0e82d71 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -196,25 +196,8 @@ impl WriterProperties { /// Converts this [`WriterProperties`] into a [`WriterPropertiesBuilder`] /// Used for mutating existing property settings pub fn into_builder(self) -> WriterPropertiesBuilder { - WriterPropertiesBuilder { - data_page_size_limit: self.data_page_size_limit, - data_page_row_count_limit: self.data_page_row_count_limit, - write_batch_size: self.write_batch_size, - max_row_group_size: self.max_row_group_size, - bloom_filter_position: self.bloom_filter_position, - writer_version: self.writer_version, - created_by: self.created_by, - offset_index_disabled: self.offset_index_disabled, - key_value_metadata: self.key_value_metadata, - default_column_properties: self.default_column_properties, - column_properties: self.column_properties, - sorting_columns: self.sorting_columns, - column_index_truncate_length: self.column_index_truncate_length, - statistics_truncate_length: self.statistics_truncate_length, - coerce_types: self.coerce_types, - #[cfg(feature = "encryption")] - file_encryption_properties: self.file_encryption_properties, - } + let builder: WriterPropertiesBuilder = self.into(); + builder } /// Returns data page size limit. @@ -459,6 +442,7 @@ impl WriterProperties { /// Builder for [`WriterProperties`] Parquet writer configuration. /// /// See example on [`WriterProperties`] +#[derive(Debug, Clone)] pub struct WriterPropertiesBuilder { data_page_size_limit: usize, data_page_row_count_limit: usize, @@ -958,6 +942,30 @@ impl WriterPropertiesBuilder { } } +impl From for WriterPropertiesBuilder { + fn from(props: WriterProperties) -> Self { + WriterPropertiesBuilder { + data_page_size_limit: props.data_page_size_limit, + data_page_row_count_limit: props.data_page_row_count_limit, + write_batch_size: props.write_batch_size, + max_row_group_size: props.max_row_group_size, + bloom_filter_position: props.bloom_filter_position, + writer_version: props.writer_version, + created_by: props.created_by, + offset_index_disabled: props.offset_index_disabled, + key_value_metadata: props.key_value_metadata, + default_column_properties: props.default_column_properties, + column_properties: props.column_properties, + sorting_columns: props.sorting_columns, + column_index_truncate_length: props.column_index_truncate_length, + statistics_truncate_length: props.statistics_truncate_length, + coerce_types: props.coerce_types, + #[cfg(feature = "encryption")] + file_encryption_properties: props.file_encryption_properties, + } + } +} + /// Controls the level of statistics to be computed by the writer and stored in /// the parquet file. /// From 16d44a59f680d84ff384dbd6d9ec5927dddc5153 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Tue, 2 Sep 2025 17:24:29 -0700 Subject: [PATCH 4/5] clippy + cargo fmt. --- parquet/src/file/properties.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index 283cd0e82d71..0bffb015fdb5 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -963,7 +963,7 @@ impl From for WriterPropertiesBuilder { #[cfg(feature = "encryption")] file_encryption_properties: props.file_encryption_properties, } - } + } } /// Controls the level of statistics to be computed by the writer and stored in From d24146ea87c8357f07fed7a60cbaa26d6b38f417 Mon Sep 17 00:00:00 2001 From: Corwin Joy Date: Thu, 4 Sep 2025 16:38:55 -0700 Subject: [PATCH 5/5] Update parquet/src/file/properties.rs Co-authored-by: Andrew Lamb --- parquet/src/file/properties.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index 0bffb015fdb5..603db6660f45 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -196,8 +196,7 @@ impl WriterProperties { /// Converts this [`WriterProperties`] into a [`WriterPropertiesBuilder`] /// Used for mutating existing property settings pub fn into_builder(self) -> WriterPropertiesBuilder { - let builder: WriterPropertiesBuilder = self.into(); - builder + self.into() } /// Returns data page size limit.