diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java index c352516809..12ed7b4f87 100644 --- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java +++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java @@ -102,4 +102,9 @@ int compareMinValues(PrimitiveComparator comparator, int index1, int ind int compareMaxValues(PrimitiveComparator comparator, int index1, int index2) { return comparator.compare(maxValues.get(index1), maxValues.get(index2)); } + + @Override + int sizeOf(Object value) { + return ((Binary) value).length(); + } } diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BooleanColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BooleanColumnIndexBuilder.java index 9a4ea89c6a..3053f784f0 100644 --- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BooleanColumnIndexBuilder.java +++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BooleanColumnIndexBuilder.java @@ -103,4 +103,9 @@ int compareMinValues(PrimitiveComparator comparator, int index1, int ind int compareMaxValues(PrimitiveComparator comparator, int index1, int index2) { return comparator.compare(maxValues.get(index1), maxValues.get(index2)); } + + @Override + int sizeOf(Object value) { + return 1; + } } diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java index 6edd753550..aa0502ba16 100644 --- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java +++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java @@ -195,6 +195,11 @@ int compareMinValues(PrimitiveComparator comparator, int index1, int ind int compareMaxValues(PrimitiveComparator comparator, int index1, int index2) { return 0; } + + @Override + int sizeOf(Object value) { + return 0; + } }; private static final Map BUILDERS = new EnumMap<>(PrimitiveTypeName.class); @@ -202,6 +207,7 @@ int compareMaxValues(PrimitiveComparator comparator, int index1, int ind private PrimitiveType type; private final BooleanList nullPages = new BooleanArrayList(); private final LongList nullCounts = new LongArrayList(); + private long minMaxSize; /** * @return a no-op builder that does not collect statistics objects and therefore returns {@code null} at @@ -293,7 +299,11 @@ public static ColumnIndex build( public void add(Statistics stats) { if (stats.hasNonNullValue()) { nullPages.add(false); - addMinMax(stats.genericGetMin(), stats.genericGetMax()); + Object min = stats.genericGetMin(); + Object max = stats.genericGetMax(); + addMinMax(min, max); + minMaxSize += sizeOf(min); + minMaxSize += sizeOf(max); } else { nullPages.add(true); addMinMax(null, null); @@ -316,7 +326,7 @@ private void fill(List nullPages, List nullCounts, List nullPages, List nullCounts, List comparator, int index1, int ind int compareMaxValues(PrimitiveComparator comparator, int index1, int index2) { return comparator.compare(maxValues.get(index1), maxValues.get(index2)); } + + @Override + int sizeOf(Object value) { + return Double.BYTES; + } } diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java index 24c911fae2..f170662452 100644 --- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java +++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java @@ -67,7 +67,7 @@ private static float convert(ByteBuffer buffer) { } private static ByteBuffer convert(float value) { - return ByteBuffer.allocate(Float.SIZE / 8).order(LITTLE_ENDIAN).putFloat(0, value); + return ByteBuffer.allocate(Float.BYTES).order(LITTLE_ENDIAN).putFloat(0, value); } @Override @@ -105,4 +105,9 @@ int compareMinValues(PrimitiveComparator comparator, int index1, int ind int compareMaxValues(PrimitiveComparator comparator, int index1, int index2) { return comparator.compare(maxValues.get(index1), maxValues.get(index2)); } + + @Override + int sizeOf(Object value) { + return Float.BYTES; + } } diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IntColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IntColumnIndexBuilder.java index e4a117c6f9..f6bd94bf1f 100644 --- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IntColumnIndexBuilder.java +++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IntColumnIndexBuilder.java @@ -67,7 +67,7 @@ private static int convert(ByteBuffer buffer) { } private static ByteBuffer convert(int value) { - return ByteBuffer.allocate(Integer.SIZE / 8).order(LITTLE_ENDIAN).putInt(0, value); + return ByteBuffer.allocate(Integer.BYTES).order(LITTLE_ENDIAN).putInt(0, value); } @Override @@ -105,4 +105,9 @@ int compareMinValues(PrimitiveComparator comparator, int index1, int ind int compareMaxValues(PrimitiveComparator comparator, int index1, int index2) { return comparator.compare(maxValues.get(index1), maxValues.get(index2)); } + + @Override + int sizeOf(Object value) { + return Integer.BYTES; + } } diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/LongColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/LongColumnIndexBuilder.java index 94e7e0f274..696602d70b 100644 --- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/LongColumnIndexBuilder.java +++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/LongColumnIndexBuilder.java @@ -67,7 +67,7 @@ private static long convert(ByteBuffer buffer) { } private static ByteBuffer convert(long value) { - return ByteBuffer.allocate(Long.SIZE / 8).order(LITTLE_ENDIAN).putLong(0, value); + return ByteBuffer.allocate(Long.BYTES).order(LITTLE_ENDIAN).putLong(0, value); } @Override @@ -105,4 +105,9 @@ int compareMinValues(PrimitiveComparator comparator, int index1, int ind int compareMaxValues(PrimitiveComparator comparator, int index1, int index2) { return comparator.compare(maxValues.get(index1), maxValues.get(index2)); } + + @Override + int sizeOf(Object value) { + return Long.BYTES; + } } diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java index f1706a1e09..5acae97c9c 100644 --- a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java +++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java @@ -69,14 +69,17 @@ public void testBuildBinaryDecimal() { assertThat(builder, instanceOf(BinaryColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, null, null)); - builder.add(stats(type, decimalBinary("-0.17"), decimalBinary("1234567890.12"))); - builder.add(stats(type, decimalBinary("-234.23"), null, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, decimalBinary("-9999293.23"), decimalBinary("2348978.45"))); - builder.add(stats(type, null, null, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, decimalBinary("87656273"))); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, decimalBinary("-0.17"), decimalBinary("1234567890.12"))); + builder.add(sb.stats(type, decimalBinary("-234.23"), null, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, decimalBinary("-9999293.23"), decimalBinary("2348978.45"))); + builder.add(sb.stats(type, null, null, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, decimalBinary("87656273"))); + assertEquals(8, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 0, 3, 3, 0, 4, 2, 0); @@ -101,14 +104,17 @@ public void testBuildBinaryDecimal() { decimalBinary("87656273")); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null, null, null)); - builder.add(stats(type, decimalBinary("-9999293.23"), decimalBinary("-234.23"))); - builder.add(stats(type, decimalBinary("-0.17"), decimalBinary("87656273"))); - builder.add(stats(type, null, null)); - builder.add(stats(type, decimalBinary("87656273"))); - builder.add(stats(type, null, null)); - builder.add(stats(type, decimalBinary("1234567890.12"), null, null, null)); - builder.add(stats(type, null, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null, null, null)); + builder.add(sb.stats(type, decimalBinary("-9999293.23"), decimalBinary("-234.23"))); + builder.add(sb.stats(type, decimalBinary("-0.17"), decimalBinary("87656273"))); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, decimalBinary("87656273"))); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, decimalBinary("1234567890.12"), null, null, null)); + builder.add(sb.stats(type, null, null, null)); + assertEquals(8, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 4, 0, 0, 2, 0, 2, 3, 3); @@ -133,14 +139,17 @@ public void testBuildBinaryDecimal() { null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, decimalBinary("1234567890.12"), null, null, null)); - builder.add(stats(type, null, null, null, null)); - builder.add(stats(type, decimalBinary("1234567890.12"), decimalBinary("87656273"))); - builder.add(stats(type, decimalBinary("987656273"), decimalBinary("-0.17"))); - builder.add(stats(type, null, null)); - builder.add(stats(type, decimalBinary("-234.23"), decimalBinary("-9999293.23"))); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, decimalBinary("1234567890.12"), null, null, null)); + builder.add(sb.stats(type, null, null, null, null)); + builder.add(sb.stats(type, decimalBinary("1234567890.12"), decimalBinary("87656273"))); + builder.add(sb.stats(type, decimalBinary("987656273"), decimalBinary("-0.17"))); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, decimalBinary("-234.23"), decimalBinary("-9999293.23"))); + assertEquals(8, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 3, 2, 3, 4, 0, 0, 2, 0); @@ -172,14 +181,17 @@ public void testBuildBinaryUtf8() { assertThat(builder, instanceOf(BinaryColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, null, null)); - builder.add(stats(type, stringBinary("Jeltz"), stringBinary("Slartibartfast"), null, null)); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, stringBinary("Beeblebrox"), stringBinary("Prefect"))); - builder.add(stats(type, stringBinary("Dent"), stringBinary("Trilian"), null)); - builder.add(stats(type, stringBinary("Beeblebrox"))); - builder.add(stats(type, null, null)); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, stringBinary("Jeltz"), stringBinary("Slartibartfast"), null, null)); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, stringBinary("Beeblebrox"), stringBinary("Prefect"))); + builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Trilian"), null)); + builder.add(sb.stats(type, stringBinary("Beeblebrox"))); + builder.add(sb.stats(type, null, null)); + assertEquals(8, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 2, 5, 2, 0, 1, 0, 2); @@ -204,14 +216,17 @@ public void testBuildBinaryUtf8() { null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, stringBinary("Beeblebrox"), stringBinary("Dent"), null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, stringBinary("Dent"), stringBinary("Jeltz"))); - builder.add(stats(type, stringBinary("Dent"), stringBinary("Prefect"), null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, stringBinary("Slartibartfast"))); - builder.add(stats(type, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, stringBinary("Beeblebrox"), stringBinary("Dent"), null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Jeltz"))); + builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Prefect"), null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, stringBinary("Slartibartfast"))); + builder.add(sb.stats(type, null, null)); + assertEquals(8, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 2, 5, 0, 1, 2, 0, 2); @@ -236,14 +251,17 @@ public void testBuildBinaryUtf8() { null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, stringBinary("Slartibartfast"))); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, stringBinary("Prefect"), stringBinary("Jeltz"), null)); - builder.add(stats(type, stringBinary("Dent"), stringBinary("Dent"))); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, stringBinary("Dent"), stringBinary("Beeblebrox"), null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, stringBinary("Slartibartfast"))); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, stringBinary("Prefect"), stringBinary("Jeltz"), null)); + builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Dent"))); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Beeblebrox"), null, null)); + assertEquals(8, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 0, 5, 1, 0, 2, 2, 2); @@ -323,11 +341,15 @@ public void testBuildBoolean() { assertThat(builder, instanceOf(BooleanColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, false, true)); - builder.add(stats(type, true, false, null)); - builder.add(stats(type, true, true, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, false, false)); + builder = ColumnIndexBuilder.getBuilder(type); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, false, true)); + builder.add(sb.stats(type, true, false, null)); + builder.add(sb.stats(type, true, true, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, false, false)); + assertEquals(5, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0); @@ -336,13 +358,16 @@ public void testBuildBoolean() { assertCorrectValues(columnIndex.getMinValues(), false, false, true, null, false); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, false, false)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, null, null, null, null)); - builder.add(stats(type, false, true, null)); - builder.add(stats(type, false, true, null, null)); - builder.add(stats(type, null, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, false, false)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, null, null, null, null)); + builder.add(sb.stats(type, false, true, null)); + builder.add(sb.stats(type, false, true, null, null)); + builder.add(sb.stats(type, null, null, null)); + assertEquals(7, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 0, 3, 4, 1, 2, 3); @@ -351,13 +376,16 @@ public void testBuildBoolean() { assertCorrectValues(columnIndex.getMinValues(), null, false, null, null, false, false, null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, true, true)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, null, null, null, null)); - builder.add(stats(type, true, false, null)); - builder.add(stats(type, false, false, null, null)); - builder.add(stats(type, null, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, true, true)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, null, null, null, null)); + builder.add(sb.stats(type, true, false, null)); + builder.add(sb.stats(type, false, false, null, null)); + builder.add(sb.stats(type, null, null, null)); + assertEquals(7, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 0, 3, 4, 1, 2, 3); @@ -389,12 +417,15 @@ public void testBuildDouble() { assertThat(builder, instanceOf(DoubleColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, -4.2, -4.1)); - builder.add(stats(type, -11.7, 7.0, null)); - builder.add(stats(type, 2.2, 2.2, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 1.9, 2.32)); - builder.add(stats(type, -21.0, 8.1)); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, -4.2, -4.1)); + builder.add(sb.stats(type, -11.7, 7.0, null)); + builder.add(sb.stats(type, 2.2, 2.2, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 1.9, 2.32)); + builder.add(sb.stats(type, -21.0, 8.1)); + assertEquals(6, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0); @@ -403,15 +434,18 @@ public void testBuildDouble() { assertCorrectValues(columnIndex.getMinValues(), -4.2, -11.7, 2.2, null, 1.9, -21.0); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, -532.3, -345.2, null, null)); - builder.add(stats(type, -234.7, -234.6, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, -234.6, 2.99999)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 3.0, 42.83)); - builder.add(stats(type, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -532.3, -345.2, null, null)); + builder.add(sb.stats(type, -234.7, -234.6, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, -234.6, 2.99999)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 3.0, 42.83)); + builder.add(sb.stats(type, null, null)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2); @@ -420,15 +454,18 @@ public void testBuildDouble() { assertCorrectValues(columnIndex.getMinValues(), null, -532.3, -234.7, null, null, -234.6, null, 3.0, null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, 532.3, 345.2)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 234.7, 234.6, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 234.69, -2.99999)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, -3.0, -42.83)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, 532.3, 345.2)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 234.7, 234.6, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 234.69, -2.99999)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -3.0, -42.83)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0); @@ -460,12 +497,15 @@ public void testBuildFloat() { assertThat(builder, instanceOf(FloatColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, -4.2f, -4.1f)); - builder.add(stats(type, -11.7f, 7.0f, null)); - builder.add(stats(type, 2.2f, 2.2f, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 1.9f, 2.32f)); - builder.add(stats(type, -21.0f, 8.1f)); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, -4.2f, -4.1f)); + builder.add(sb.stats(type, -11.7f, 7.0f, null)); + builder.add(sb.stats(type, 2.2f, 2.2f, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 1.9f, 2.32f)); + builder.add(sb.stats(type, -21.0f, 8.1f)); + assertEquals(6, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0); @@ -474,15 +514,18 @@ public void testBuildFloat() { assertCorrectValues(columnIndex.getMinValues(), -4.2f, -11.7f, 2.2f, null, 1.9f, -21.0f); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, -532.3f, -345.2f, null, null)); - builder.add(stats(type, -300.6f, -234.7f, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, -234.6f, 2.99999f)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 3.0f, 42.83f)); - builder.add(stats(type, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -532.3f, -345.2f, null, null)); + builder.add(sb.stats(type, -300.6f, -234.7f, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, -234.6f, 2.99999f)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 3.0f, 42.83f)); + builder.add(sb.stats(type, null, null)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2); @@ -491,15 +534,18 @@ public void testBuildFloat() { assertCorrectValues(columnIndex.getMinValues(), null, -532.3f, -300.6f, null, null, -234.6f, null, 3.0f, null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, 532.3f, 345.2f)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 234.7f, 234.6f, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 234.6f, -2.99999f)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, -3.0f, -42.83f)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, 532.3f, 345.2f)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 234.7f, 234.6f, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 234.6f, -2.99999f)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -3.0f, -42.83f)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0); @@ -531,12 +577,15 @@ public void testBuildInt32() { assertThat(builder, instanceOf(IntColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, -4, 10)); - builder.add(stats(type, -11, 7, null)); - builder.add(stats(type, 2, 2, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 1, 2)); - builder.add(stats(type, -21, 8)); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, -4, 10)); + builder.add(sb.stats(type, -11, 7, null)); + builder.add(sb.stats(type, 2, 2, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 1, 2)); + builder.add(sb.stats(type, -21, 8)); + assertEquals(6, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0); @@ -545,15 +594,18 @@ public void testBuildInt32() { assertCorrectValues(columnIndex.getMinValues(), -4, -11, 2, null, 1, -21); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, -532, -345, null, null)); - builder.add(stats(type, -500, -42, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, -42, 2)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 3, 42)); - builder.add(stats(type, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -532, -345, null, null)); + builder.add(sb.stats(type, -500, -42, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, -42, 2)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 3, 42)); + builder.add(sb.stats(type, null, null)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2); @@ -562,15 +614,18 @@ public void testBuildInt32() { assertCorrectValues(columnIndex.getMinValues(), null, -532, -500, null, null, -42, null, 3, null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, 532, 345)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 234, 42, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 42, -2)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, -3, -42)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, 532, 345)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 234, 42, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 42, -2)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -3, -42)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0); @@ -602,12 +657,15 @@ public void testBuildUInt8() { assertThat(builder, instanceOf(IntColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, 4, 10)); - builder.add(stats(type, 11, 17, null)); - builder.add(stats(type, 2, 2, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 1, 0xFF)); - builder.add(stats(type, 0xEF, 0xFA)); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, 4, 10)); + builder.add(sb.stats(type, 11, 17, null)); + builder.add(sb.stats(type, 2, 2, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 1, 0xFF)); + builder.add(sb.stats(type, 0xEF, 0xFA)); + assertEquals(6, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0); @@ -616,15 +674,18 @@ public void testBuildUInt8() { assertCorrectValues(columnIndex.getMinValues(), 4, 11, 2, null, 1, 0xEF); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, 0, 0, null, null)); - builder.add(stats(type, 0, 42, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 42, 0xEE)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 0xEF, 0xFF)); - builder.add(stats(type, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 0, 0, null, null)); + builder.add(sb.stats(type, 0, 42, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 42, 0xEE)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 0xEF, 0xFF)); + builder.add(sb.stats(type, null, null)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2); @@ -633,15 +694,18 @@ public void testBuildUInt8() { assertCorrectValues(columnIndex.getMinValues(), null, 0, 0, null, null, 42, null, 0xEF, null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, 0xFF, 0xFF)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 0xEF, 0xEA, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 0xEE, 42)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 41, 0)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, 0xFF, 0xFF)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 0xEF, 0xEA, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 0xEE, 42)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 41, 0)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0); @@ -657,12 +721,15 @@ public void testBuildInt64() { assertThat(builder, instanceOf(LongColumnIndexBuilder.class)); assertNull(builder.build()); - builder.add(stats(type, -4l, 10l)); - builder.add(stats(type, -11l, 7l, null)); - builder.add(stats(type, 2l, 2l, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 1l, 2l)); - builder.add(stats(type, -21l, 8l)); + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(type, -4l, 10l)); + builder.add(sb.stats(type, -11l, 7l, null)); + builder.add(sb.stats(type, 2l, 2l, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 1l, 2l)); + builder.add(sb.stats(type, -21l, 8l)); + assertEquals(6, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); ColumnIndex columnIndex = builder.build(); assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 0l, 1l, 2l, 3l, 0l, 0l); @@ -671,15 +738,18 @@ public void testBuildInt64() { assertCorrectValues(columnIndex.getMinValues(), -4l, -11l, 2l, null, 1l, -21l); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null)); - builder.add(stats(type, -532l, -345l, null, null)); - builder.add(stats(type, -234l, -42l, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, -42l, 2l)); - builder.add(stats(type, null, null)); - builder.add(stats(type, -3l, 42l)); - builder.add(stats(type, null, null)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -532l, -345l, null, null)); + builder.add(sb.stats(type, -234l, -42l, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, -42l, 2l)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -3l, 42l)); + builder.add(sb.stats(type, null, null)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2); @@ -688,15 +758,18 @@ public void testBuildInt64() { assertCorrectValues(columnIndex.getMinValues(), null, -532l, -234l, null, null, -42l, null, -3l, null); builder = ColumnIndexBuilder.getBuilder(type); - builder.add(stats(type, null, null, null, null, null)); - builder.add(stats(type, 532l, 345l)); - builder.add(stats(type, null, null, null)); - builder.add(stats(type, 234l, 42l, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, 42l, -2l)); - builder.add(stats(type, null, null)); - builder.add(stats(type, null, null)); - builder.add(stats(type, -3l, -42l)); + sb = new StatsBuilder(); + builder.add(sb.stats(type, null, null, null, null, null)); + builder.add(sb.stats(type, 532l, 345l)); + builder.add(sb.stats(type, null, null, null)); + builder.add(sb.stats(type, 234l, 42l, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, 42l, -2l)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, null, null)); + builder.add(sb.stats(type, -3l, -42l)); + assertEquals(9, builder.getPageCount()); + assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize()); columnIndex = builder.build(); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0); @@ -724,12 +797,15 @@ public void testStaticBuildInt64() { @Test public void testNoOpBuilder() { ColumnIndexBuilder builder = ColumnIndexBuilder.getNoOpBuilder(); - builder.add(stats(Types.required(BINARY).as(UTF8).named("test_binary_utf8"), stringBinary("Jeltz"), + StatsBuilder sb = new StatsBuilder(); + builder.add(sb.stats(Types.required(BINARY).as(UTF8).named("test_binary_utf8"), stringBinary("Jeltz"), stringBinary("Slartibartfast"), null, null)); - builder.add(stats(Types.required(BOOLEAN).named("test_boolean"), true, true, null, null)); - builder.add(stats(Types.required(DOUBLE).named("test_double"), null, null, null)); - builder.add(stats(Types.required(INT32).named("test_int32"), null, null)); - builder.add(stats(Types.required(INT64).named("test_int64"), -234l, -42l, null)); + builder.add(sb.stats(Types.required(BOOLEAN).named("test_boolean"), true, true, null, null)); + builder.add(sb.stats(Types.required(DOUBLE).named("test_double"), null, null, null)); + builder.add(sb.stats(Types.required(INT32).named("test_int32"), null, null)); + builder.add(sb.stats(Types.required(INT64).named("test_int64"), -234l, -42l, null)); + assertEquals(0, builder.getPageCount()); + assertEquals(0, builder.getMinMaxSize()); assertNull(builder.build()); } @@ -912,38 +988,50 @@ private static void assertCorrectNullPages(ColumnIndex columnIndex, boolean... e } } - private static Statistics stats(PrimitiveType type, Object... values) { - Statistics stats = Statistics.createStats(type); - for (Object value : values) { - if (value == null) { - stats.incrementNumNulls(); - continue; + private static class StatsBuilder { + private long minMaxSize; + + Statistics stats(PrimitiveType type, Object... values) { + Statistics stats = Statistics.createStats(type); + for (Object value : values) { + if (value == null) { + stats.incrementNumNulls(); + continue; + } + switch (type.getPrimitiveTypeName()) { + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + case INT96: + stats.updateStats((Binary) value); + break; + case BOOLEAN: + stats.updateStats((boolean) value); + break; + case DOUBLE: + stats.updateStats((double) value); + break; + case FLOAT: + stats.updateStats((float) value); + break; + case INT32: + stats.updateStats((int) value); + break; + case INT64: + stats.updateStats((long) value); + break; + default: + fail("Unsupported value type for stats: " + value.getClass()); + } } - switch (type.getPrimitiveTypeName()) { - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - case INT96: - stats.updateStats((Binary) value); - break; - case BOOLEAN: - stats.updateStats((boolean) value); - break; - case DOUBLE: - stats.updateStats((double) value); - break; - case FLOAT: - stats.updateStats((float) value); - break; - case INT32: - stats.updateStats((int) value); - break; - case INT64: - stats.updateStats((long) value); - break; - default: - fail("Unsupported value type for stats: " + value.getClass()); + if (stats.hasNonNullValue()) { + minMaxSize += stats.getMinBytes().length; + minMaxSize += stats.getMaxBytes().length; } + return stats; + } + + long getMinMaxSize() { + return minMaxSize; } - return stats; } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index bd0f6835e4..3c85b02ec2 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -19,6 +19,7 @@ package org.apache.parquet.hadoop; import static org.apache.parquet.format.Util.writeFileMetaData; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.MAX_STATS_SIZE; import static org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE; import static org.apache.parquet.hadoop.ParquetWriter.MAX_PADDING_SIZE_DEFAULT; @@ -576,7 +577,11 @@ void writeColumnChunk(ColumnDescriptor descriptor, public void endColumn() throws IOException { state = state.endColumn(); LOG.debug("{}: end column", out.getPos()); - currentColumnIndexes.add(columnIndexBuilder.build()); + if (columnIndexBuilder.getMinMaxSize() > columnIndexBuilder.getPageCount() * MAX_STATS_SIZE) { + currentColumnIndexes.add(null); + } else { + currentColumnIndexes.add(columnIndexBuilder.build()); + } currentOffsetIndexes.add(offsetIndexBuilder.build(firstPageOffset)); currentBlock.addColumn(ColumnChunkMetaData.get( currentChunkPath, diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java index a8de38c38f..917ad57910 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java @@ -65,6 +65,7 @@ import static org.junit.Assert.*; import static org.apache.parquet.column.Encoding.BIT_PACKED; import static org.apache.parquet.column.Encoding.PLAIN; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.MAX_STATS_SIZE; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; import static org.apache.parquet.schema.Type.Repetition.*; import static org.apache.parquet.hadoop.TestUtils.enforceEmptyDir; @@ -813,7 +814,12 @@ public void testColumnIndexWriteRead() throws Exception { w.endBlock(); w.startBlock(4); w.startColumn(C1, 7, CODEC); - w.writeDataPage(7, 4, BytesInput.from(BYTES3), EMPTY_STATS, BIT_PACKED, BIT_PACKED, PLAIN); + w.writeDataPage(7, 4, BytesInput.from(BYTES3), + // Creating huge stats so the column index will reach the limit and won't be written + statsC1( + Binary.fromConstantByteArray(new byte[(int) MAX_STATS_SIZE]), + Binary.fromConstantByteArray(new byte[1])), + 4, BIT_PACKED, BIT_PACKED, PLAIN); w.endColumn(); w.startColumn(C2, 8, CODEC); w.writeDataPage(8, 4, BytesInput.from(BYTES4), EMPTY_STATS, BIT_PACKED, BIT_PACKED, PLAIN); @@ -876,6 +882,8 @@ public void testColumnIndexWriteRead() throws Exception { assertEquals(0, offsetIndex.getFirstRowIndex(0)); assertEquals(1, offsetIndex.getFirstRowIndex(1)); assertEquals(3, offsetIndex.getFirstRowIndex(2)); + + assertNull(reader.readColumnIndex(footer.getBlocks().get(2).getColumns().get(0))); } }