diff --git a/parquet-arrow/pom.xml b/parquet-arrow/pom.xml index d033b5e200..3b59ae57ff 100644 --- a/parquet-arrow/pom.xml +++ b/parquet-arrow/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml index d46df7c967..58ab779b66 100644 --- a/parquet-avro/pom.xml +++ b/parquet-avro/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml index 4c5ba16560..1920a17230 100644 --- a/parquet-benchmarks/pom.xml +++ b/parquet-benchmarks/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-cascading-deprecated/pom.xml b/parquet-cascading-deprecated/pom.xml index f1db451b3a..7ba330216b 100644 --- a/parquet-cascading-deprecated/pom.xml +++ b/parquet-cascading-deprecated/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-cascading3-deprecated/pom.xml b/parquet-cascading3-deprecated/pom.xml index dbb0b44ddf..432d98db17 100644 --- a/parquet-cascading3-deprecated/pom.xml +++ b/parquet-cascading3-deprecated/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml index 700a6673f9..5863476640 100644 --- a/parquet-cli/pom.xml +++ b/parquet-cli/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index eb9bb6fb4d..02f861ff79 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml index 7ab59886ef..849ac5af82 100644 --- a/parquet-common/pom.xml +++ b/parquet-common/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml index 1c2ef36be9..52daf8e58a 100644 --- a/parquet-encoding/pom.xml +++ b/parquet-encoding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml index 1709f85863..ab7489f71d 100644 --- a/parquet-format-structures/pom.xml +++ b/parquet-format-structures/pom.xml @@ -24,7 +24,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 parquet-format-structures diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml index da0dfd8c3c..67fd36b5cd 100644 --- a/parquet-generator/pom.xml +++ b/parquet-generator/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml index 929bebca1c..99b7d41a3a 100644 --- a/parquet-hadoop-bundle/pom.xml +++ b/parquet-hadoop-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index ae520f3739..b73654b6cf 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/QueryMetrics.java b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/QueryMetrics.java index ac6d90b616..ed9a547b3f 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/QueryMetrics.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/QueryMetrics.java @@ -24,6 +24,9 @@ public class QueryMetrics { private long skipBloomBlocks = 0; private long skipBloomRows = 0; private long totalBloomBlocks = 0; + private long totalPagesCount = 0; + private long filteredPagesCount = 0; + private long afterFilterPagesCount = 0; public String getSkipBloomFilter() { return skipBloomFilter; @@ -56,4 +59,34 @@ public long getTotalBloomBlocks() { public void setTotalBloomBlocks(long totalBloomBlocks) { this.totalBloomBlocks = totalBloomBlocks; } + + public long getTotalPagesCount() { + return totalPagesCount; + } + + public void setTotalPagesCount(long totalPagesCount) { + this.totalPagesCount = totalPagesCount; + } + + public long getFilteredPagesCount() { + return filteredPagesCount; + } + + public void setFilteredPagesCount(long filteredPagesCount) { + this.filteredPagesCount = filteredPagesCount; + } + + public long getAfterFilterPagesCount() { + return afterFilterPagesCount; + } + + public void setAfterFilterPagesCount(long afterFilterPagesCount) { + this.afterFilterPagesCount = afterFilterPagesCount; + } + + public void logParquetPageFilter(long total, long afterFilter) { + totalPagesCount += total; + filteredPagesCount += (total - afterFilter); + afterFilterPagesCount += afterFilter; + } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index e63d934b46..8fe871982d 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -998,6 +998,7 @@ public PageReadStore readNextFilteredRowGroup() throws IOException { OffsetIndex filteredOffsetIndex = filterOffsetIndex(offsetIndex, rowRanges, block.getRowCount()); + this.queryMetrics.logParquetPageFilter(offsetIndex.getPageCount(), filteredOffsetIndex.getPageCount()); for (OffsetRange range : calculateOffsetRanges(filteredOffsetIndex, mc, offsetIndex.getOffset(0))) { BenchmarkCounter.incrementTotalBytes(range.getLength()); long startingPos = range.getOffset(); diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml index 7ec8e8a54e..394afaebbc 100644 --- a/parquet-jackson/pom.xml +++ b/parquet-jackson/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml index 0bf63be30f..a701362192 100644 --- a/parquet-pig-bundle/pom.xml +++ b/parquet-pig-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml index bf48b28dcc..b761eed2fc 100644 --- a/parquet-pig/pom.xml +++ b/parquet-pig/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml index f1144e5230..6cb6abd485 100644 --- a/parquet-protobuf/pom.xml +++ b/parquet-protobuf/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml index 37ff4cdb50..e7bb40f9e2 100644 --- a/parquet-scala/pom.xml +++ b/parquet-scala/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-scrooge-deprecated/pom.xml b/parquet-scrooge-deprecated/pom.xml index 5682a6df17..3f16505389 100644 --- a/parquet-scrooge-deprecated/pom.xml +++ b/parquet-scrooge-deprecated/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml index ac2b219996..e2d2affdb1 100644 --- a/parquet-thrift/pom.xml +++ b/parquet-thrift/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/parquet-tools-deprecated/pom.xml b/parquet-tools-deprecated/pom.xml index 48611db8a4..11f30fe7e4 100644 --- a/parquet-tools-deprecated/pom.xml +++ b/parquet-tools-deprecated/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 4.0.0 diff --git a/pom.xml b/pom.xml index b1f1130b0d..496667b965 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.apache.parquet parquet - 1.12.2-kylin-r4 + 1.12.2-kylin-r5 pom Apache Parquet MR