From 582e9e083a7a1d7c33c09fae11a43747bb2a89af Mon Sep 17 00:00:00 2001 From: Vinish Reddy Date: Tue, 1 Apr 2025 14:55:07 -0700 Subject: [PATCH 1/3] Add getColumnStats method which supports list of columns --- .../common/table/view/NoOpTableMetadata.java | 5 + .../hudi/metadata/BaseTableMetadata.java | 99 ++++++++++++------- .../FileSystemBackedTableMetadata.java | 5 + .../hudi/metadata/HoodieTableMetadata.java | 12 +++ .../functional/TestHoodieBackedMetadata.java | 63 ++++++++++++ 5 files changed, 149 insertions(+), 35 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java index ae1e36ac39e4a..6323f49e3a706 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java @@ -96,6 +96,11 @@ public Map, HoodieMetadataColumnStats> getColumnStats(List< throw new HoodieMetadataException("Unsupported operation: getColumnsStats!"); } + @Override + public Map, List> getColumnStats(List> partitionNameFileNameList, List columnNames) throws HoodieMetadataException { + throw new HoodieMetadataException("Unsupported operation: getColumnsStats!"); + } + @Override public Map readRecordIndex(List recordKeys) { throw new HoodieMetadataException("Unsupported operation: readRecordIndex!"); diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java index 671ff75ab5c66..bb984f857e953 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java @@ -237,46 +237,21 @@ public Map, BloomFilter> getBloomFilters(final List, HoodieMetadataColumnStats> getColumnStats(final List> partitionNameFileNameList, final String columnName) throws HoodieMetadataException { + Map, List> partitionNameFileToColStats = getColumnStats(partitionNameFileNameList, Collections.singletonList(columnName)); + ValidationUtils.checkArgument(partitionNameFileToColStats.isEmpty() || partitionNameFileToColStats.values().stream().anyMatch(stats -> stats.size() == 1)); + return partitionNameFileToColStats.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().get(0))); + } + + @Override + public Map, List> getColumnStats(List> partitionNameFileNameList, List columnNames) + throws HoodieMetadataException { if (!dataMetaClient.getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.COLUMN_STATS)) { LOG.error("Metadata column stats index is disabled!"); return Collections.emptyMap(); } - Map> columnStatKeyToFileNameMap = new HashMap<>(); - Set columnStatKeyset = new HashSet<>(); - final ColumnIndexID columnIndexID = new ColumnIndexID(columnName); - for (Pair partitionNameFileNamePair : partitionNameFileNameList) { - final String columnStatsIndexKey = HoodieMetadataPayload.getColumnStatsIndexKey( - new PartitionIndexID(HoodieTableMetadataUtil.getColumnStatsIndexPartitionIdentifier(partitionNameFileNamePair.getLeft())), - new FileIndexID(partitionNameFileNamePair.getRight()), - columnIndexID); - columnStatKeyset.add(columnStatsIndexKey); - columnStatKeyToFileNameMap.put(columnStatsIndexKey, partitionNameFileNamePair); - } - - List columnStatKeylist = new ArrayList<>(columnStatKeyset); - HoodieTimer timer = HoodieTimer.start(); - Map> hoodieRecords = - getRecordsByKeys(columnStatKeylist, MetadataPartitionType.COLUMN_STATS.getPartitionPath()); - metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR, timer.endTimer())); - metrics.ifPresent(m -> m.setMetric(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_FILE_COUNT_STR, columnStatKeylist.size())); - - Map, HoodieMetadataColumnStats> fileToColumnStatMap = new HashMap<>(); - for (final Map.Entry> entry : hoodieRecords.entrySet()) { - final Option columnStatMetadata = - entry.getValue().getData().getColumnStatMetadata(); - if (columnStatMetadata.isPresent()) { - if (!columnStatMetadata.get().getIsDeleted()) { - ValidationUtils.checkState(columnStatKeyToFileNameMap.containsKey(entry.getKey())); - final Pair partitionFileNamePair = columnStatKeyToFileNameMap.get(entry.getKey()); - ValidationUtils.checkState(!fileToColumnStatMap.containsKey(partitionFileNamePair)); - fileToColumnStatMap.put(partitionFileNamePair, columnStatMetadata.get()); - } - } else { - LOG.error("Meta index column stats missing for: {}", entry.getKey()); - } - } - return fileToColumnStatMap; + Map> columnStatKeyToFileNameMap = computeColStatKeyToFileName(partitionNameFileNameList, columnNames); + return computeFileToColumnStatsMap(columnStatKeyToFileNameMap); } /** @@ -427,6 +402,60 @@ Map> fetchAllFilesInPartitionPaths(List> computeColStatKeyToFileName( + final List> partitionNameFileNameList, + final List columnNames) { + Map> columnStatKeyToFileNameMap = new HashMap<>(); + for (String columnName : columnNames) { + final ColumnIndexID columnIndexID = new ColumnIndexID(columnName); + for (Pair partitionNameFileNamePair : partitionNameFileNameList) { + final String columnStatsIndexKey = HoodieMetadataPayload.getColumnStatsIndexKey( + new PartitionIndexID(HoodieTableMetadataUtil.getColumnStatsIndexPartitionIdentifier(partitionNameFileNamePair.getLeft())), + new FileIndexID(partitionNameFileNamePair.getRight()), + columnIndexID); + columnStatKeyToFileNameMap.put(columnStatsIndexKey, partitionNameFileNamePair); + } + } + return columnStatKeyToFileNameMap; + } + + /** + * Computes the map from partition and file name pair to HoodieMetadataColumnStats record. + * + * @param columnStatKeyToFileNameMap - A map from col-stats key to partition and file name pair. + */ + private Map, List> computeFileToColumnStatsMap(Map> columnStatKeyToFileNameMap) { + List columnStatKeylist = new ArrayList<>(columnStatKeyToFileNameMap.keySet()); + HoodieTimer timer = HoodieTimer.start(); + Map> hoodieRecords = + getRecordsByKeys(columnStatKeylist, MetadataPartitionType.COLUMN_STATS.getPartitionPath()); + metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR, timer.endTimer())); + Map, List> fileToColumnStatMap = new HashMap<>(); + for (final Map.Entry> entry : hoodieRecords.entrySet()) { + final Option columnStatMetadata = + entry.getValue().getData().getColumnStatMetadata(); + if (columnStatMetadata.isPresent()) { + if (!columnStatMetadata.get().getIsDeleted()) { + ValidationUtils.checkState(columnStatKeyToFileNameMap.containsKey(entry.getKey())); + final Pair partitionFileNamePair = columnStatKeyToFileNameMap.get(entry.getKey()); + if (!fileToColumnStatMap.containsKey(partitionFileNamePair)) { + fileToColumnStatMap.put(partitionFileNamePair, new ArrayList<>()); + } + fileToColumnStatMap.get(partitionFileNamePair).add(columnStatMetadata.get()); + } + } else { + LOG.error("Meta index column stats missing for: " + entry.getKey()); + } + } + return fileToColumnStatMap; + } + /** * Handle spurious deletes. Depending on config, throw an exception or log a warn msg. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java index 1677b4b4b88bd..62711dd77ba40 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java @@ -303,6 +303,11 @@ public Map, HoodieMetadataColumnStats> getColumnStats(final throw new HoodieMetadataException("Unsupported operation: getColumnsStats!"); } + @Override + public Map, List> getColumnStats(List> partitionNameFileNameList, List columnNames) throws HoodieMetadataException { + throw new HoodieMetadataException("Unsupported operation: getColumnsStats!"); + } + @Override public HoodieData> getRecordsByKeyPrefixes(List keyPrefixes, String partitionName, boolean shouldLoadInMemory) { throw new HoodieMetadataException("Unsupported operation: getRecordsByKeyPrefixes!"); diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java index ff8eae014e146..ec3ce260102f8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java @@ -254,6 +254,18 @@ Map, BloomFilter> getBloomFilters(final List, HoodieMetadataColumnStats> getColumnStats(final List> partitionNameFileNameList, final String columnName) throws HoodieMetadataException; + + /** + * Get column stats for files from the metadata table index. + * + * @param partitionNameFileNameList - List of partition and file name pair for which bloom filters need to be retrieved. + * @param columnNames - List of column name for which stats are needed. + * @return Map of partition and file name pair to a list of column stats. + * @throws HoodieMetadataException + */ + Map, List> getColumnStats(List> partitionNameFileNameList, List columnNames) + throws HoodieMetadataException; + /** * Returns the location of record keys which are found in the record index. * Records that are not found are ignored and wont be part of map object that is returned. diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index 26d8d5519e392..e0bbea25a25a4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -31,12 +31,14 @@ import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.config.LockConfiguration; import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.model.HoodieColumnRangeMetadata; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieFileFormat; @@ -92,6 +94,7 @@ import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase; +import org.apache.hudi.io.storage.HoodieIOFactory; import org.apache.hudi.metadata.FileSystemBackedTableMetadata; import org.apache.hudi.metadata.HoodieBackedTableMetadata; import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter; @@ -121,6 +124,7 @@ import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -152,11 +156,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.apache.hudi.avro.HoodieAvroUtils.unwrapAvroValueWrapper; import static org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_PATH_PROP_KEY; import static org.apache.hudi.common.model.HoodieTableType.COPY_ON_WRITE; import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ; @@ -3140,6 +3146,63 @@ public void testNonPartitionedColStats() throws Exception { } } + @Test + public void testColStatsMultipleColumns() throws Exception { + initPath(); + Properties properties = new TypedProperties(); + properties.put(HoodieTableConfig.PARTITION_FIELDS.key(), "partition_path"); + HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, false) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexColumnStats(true) + .withProperties(properties) + .build()).build(); + init(HoodieTableType.COPY_ON_WRITE, writeConfig); + initMetaClient(writeConfig.getProps()); + HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc); + + HoodieTestDataGenerator partitionedGenerator = new HoodieTestDataGenerator(); + try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, writeConfig)) { + // Write 1 (Bulk insert) + String newCommitTime = "0000001"; + List records = partitionedGenerator.generateInserts(newCommitTime, 10); + client.startCommitWithTime(newCommitTime); + client.bulkInsert(jsc.parallelize(records, 1), newCommitTime).collect(); + + HoodieTableMetadata tableMetadata = metadata(client); + List metadataPartitions = tableMetadata.getAllPartitionPaths(); + List> partitionNameFileNameList = new ArrayList<>(); + for (String metadataPartition : metadataPartitions) { + partitionNameFileNameList.addAll( + tableMetadata.getAllFilesInPartitions(singletonList(String.format("%s/%s",basePath, metadataPartition))).entrySet().stream() + .flatMap(entry -> entry.getValue().stream().map(storagePathInfo -> Pair.of(metadataPartition, storagePathInfo.getPath().getName()))) + .collect(Collectors.toList()) + ); + } + List columns = Arrays.asList("begin_lat", "end_lat", "distance_in_meters", "weight", "seconds_since_epoch", "nation"); + Map, List> colStatsFromMetadata = tableMetadata.getColumnStats(partitionNameFileNameList, columns); + assertEquals(partitionNameFileNameList.size(), colStatsFromMetadata.size()); + // Assert stats from parquet footer same as metadata. + colStatsFromMetadata.forEach(((partitionAndFileName, stats) -> { + StoragePath fullFilePath = new StoragePath(basePath, String.format("%s/%s", partitionAndFileName.getLeft(), partitionAndFileName.getRight())); + Map> columnRangesMap = + HoodieIOFactory.getIOFactory(metaClient.getStorage()) + .getFileFormatUtils(HoodieFileFormat.PARQUET) + .readColumnStatsFromMetadata(metaClient.getStorage(), fullFilePath, columns) + .stream() + .collect(Collectors.toMap(HoodieColumnRangeMetadata::getColumnName, Function.identity())); + Map columnStatsMap = stats.stream().collect(Collectors.toMap(HoodieMetadataColumnStats::getColumnName, Function.identity())); + Assertions.assertEquals(columnRangesMap.size(), columnStatsMap.size()); + for (String column : columns) { + // Assert getColumnStats returns same data. + Assertions.assertEquals(columnStatsMap.get(column), tableMetadata.getColumnStats(Collections.singletonList(partitionAndFileName), column).get(partitionAndFileName)); + Assertions.assertEquals(columnRangesMap.get(column).getNullCount(), columnStatsMap.get(column).getNullCount()); + Assertions.assertEquals(columnRangesMap.get(column).getValueCount(),columnStatsMap.get(column).getValueCount()); + Assertions.assertEquals(columnRangesMap.get(column).getMaxValue(), unwrapAvroValueWrapper(columnStatsMap.get(column).getMaxValue())); + Assertions.assertEquals(columnRangesMap.get(column).getMinValue(), unwrapAvroValueWrapper(columnStatsMap.get(column).getMinValue())); + } + })); + } + } + /** * Test various metrics published by metadata table. */ From 8efb5c1548d9852938421fa0c57a26cb46e9904b Mon Sep 17 00:00:00 2001 From: Vinish Reddy Date: Mon, 28 Apr 2025 14:20:56 -0700 Subject: [PATCH 2/3] Fix build errors --- .../apache/hudi/client/functional/TestHoodieBackedMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index e0bbea25a25a4..bbb98cbb4d60a 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -3167,7 +3167,7 @@ public void testColStatsMultipleColumns() throws Exception { client.startCommitWithTime(newCommitTime); client.bulkInsert(jsc.parallelize(records, 1), newCommitTime).collect(); - HoodieTableMetadata tableMetadata = metadata(client); + HoodieTableMetadata tableMetadata = metadata(client, metaClient.getStorage()); List metadataPartitions = tableMetadata.getAllPartitionPaths(); List> partitionNameFileNameList = new ArrayList<>(); for (String metadataPartition : metadataPartitions) { From 1d60c0411143ad8b7c27db55d78e2cc5675ab780 Mon Sep 17 00:00:00 2001 From: Vinish Reddy Date: Tue, 29 Apr 2025 15:30:52 -0700 Subject: [PATCH 3/3] Adddress comments --- .../hudi/metadata/BaseTableMetadata.java | 15 +++++-------- .../functional/TestHoodieBackedMetadata.java | 22 +++++++++++++------ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java index bb984f857e953..01f7c6c845915 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java @@ -440,17 +440,12 @@ private Map, List> computeFileTo for (final Map.Entry> entry : hoodieRecords.entrySet()) { final Option columnStatMetadata = entry.getValue().getData().getColumnStatMetadata(); - if (columnStatMetadata.isPresent()) { - if (!columnStatMetadata.get().getIsDeleted()) { - ValidationUtils.checkState(columnStatKeyToFileNameMap.containsKey(entry.getKey())); - final Pair partitionFileNamePair = columnStatKeyToFileNameMap.get(entry.getKey()); - if (!fileToColumnStatMap.containsKey(partitionFileNamePair)) { - fileToColumnStatMap.put(partitionFileNamePair, new ArrayList<>()); - } - fileToColumnStatMap.get(partitionFileNamePair).add(columnStatMetadata.get()); - } + if (columnStatMetadata.isPresent() && !columnStatMetadata.get().getIsDeleted()) { + ValidationUtils.checkState(columnStatKeyToFileNameMap.containsKey(entry.getKey())); + final Pair partitionFileNamePair = columnStatKeyToFileNameMap.get(entry.getKey()); + fileToColumnStatMap.computeIfAbsent(partitionFileNamePair, k -> new ArrayList<>()).add(columnStatMetadata.get()); } else { - LOG.error("Meta index column stats missing for: " + entry.getKey()); + LOG.error("Meta index column stats missing for {}", entry.getKey()); } } return fileToColumnStatMap; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index bbb98cbb4d60a..8b7ea41803b65 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -3152,7 +3152,9 @@ public void testColStatsMultipleColumns() throws Exception { Properties properties = new TypedProperties(); properties.put(HoodieTableConfig.PARTITION_FIELDS.key(), "partition_path"); HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, false) - .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexColumnStats(true) + .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true) + .withMetadataIndexColumnStats(true) + .withColumnStatsIndexForColumns("begin_lat,end_lat,distance_in_meters,weight") .withProperties(properties) .build()).build(); init(HoodieTableType.COPY_ON_WRITE, writeConfig); @@ -3183,21 +3185,27 @@ public void testColStatsMultipleColumns() throws Exception { // Assert stats from parquet footer same as metadata. colStatsFromMetadata.forEach(((partitionAndFileName, stats) -> { StoragePath fullFilePath = new StoragePath(basePath, String.format("%s/%s", partitionAndFileName.getLeft(), partitionAndFileName.getRight())); - Map> columnRangesMap = + Map> parquetStatsMap = HoodieIOFactory.getIOFactory(metaClient.getStorage()) .getFileFormatUtils(HoodieFileFormat.PARQUET) .readColumnStatsFromMetadata(metaClient.getStorage(), fullFilePath, columns) .stream() .collect(Collectors.toMap(HoodieColumnRangeMetadata::getColumnName, Function.identity())); Map columnStatsMap = stats.stream().collect(Collectors.toMap(HoodieMetadataColumnStats::getColumnName, Function.identity())); - Assertions.assertEquals(columnRangesMap.size(), columnStatsMap.size()); + List columnsWithoutStats = Arrays.asList("seconds_since_epoch", "nation"); + Assertions.assertEquals(parquetStatsMap.size() - columnsWithoutStats.size(), columnStatsMap.size()); for (String column : columns) { + if (columnsWithoutStats.contains(column)) { + // Assert columnsWithoutStats are not present in MDT result. + Assertions.assertFalse(columnStatsMap.containsKey(column)); + continue; + } // Assert getColumnStats returns same data. Assertions.assertEquals(columnStatsMap.get(column), tableMetadata.getColumnStats(Collections.singletonList(partitionAndFileName), column).get(partitionAndFileName)); - Assertions.assertEquals(columnRangesMap.get(column).getNullCount(), columnStatsMap.get(column).getNullCount()); - Assertions.assertEquals(columnRangesMap.get(column).getValueCount(),columnStatsMap.get(column).getValueCount()); - Assertions.assertEquals(columnRangesMap.get(column).getMaxValue(), unwrapAvroValueWrapper(columnStatsMap.get(column).getMaxValue())); - Assertions.assertEquals(columnRangesMap.get(column).getMinValue(), unwrapAvroValueWrapper(columnStatsMap.get(column).getMinValue())); + Assertions.assertEquals(parquetStatsMap.get(column).getNullCount(), columnStatsMap.get(column).getNullCount()); + Assertions.assertEquals(parquetStatsMap.get(column).getValueCount(),columnStatsMap.get(column).getValueCount()); + Assertions.assertEquals(parquetStatsMap.get(column).getMaxValue(), unwrapAvroValueWrapper(columnStatsMap.get(column).getMaxValue())); + Assertions.assertEquals(parquetStatsMap.get(column).getMinValue(), unwrapAvroValueWrapper(columnStatsMap.get(column).getMinValue())); } })); }