Skip to content

More Parquet MetadataReader tests #11793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
findepi merged 3 commits into from
Apr 6, 2022
Merged

More Parquet MetadataReader tests #11793

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d397c10
Add Float and Double reading test
findepi Mar 19, 2021
fadd259
Test when min or max is missing on file level
findepi Mar 19, 2021
c25512f
empty
findepi Apr 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestMetadataReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

import com.google.common.io.BaseEncoding;
import org.apache.parquet.column.statistics.BinaryStatistics;
import org.apache.parquet.column.statistics.DoubleStatistics;
import org.apache.parquet.column.statistics.FloatStatistics;
import org.apache.parquet.column.statistics.IntStatistics;
import org.apache.parquet.column.statistics.LongStatistics;
import org.apache.parquet.format.Statistics;
Expand All @@ -28,6 +30,8 @@
import static io.trino.testing.assertions.Assert.assertEquals;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
import static org.apache.parquet.schema.Type.Repetition.OPTIONAL;
Expand Down Expand Up @@ -94,6 +98,48 @@ public void testReadStatsInt64(Optional<String> fileCreatedBy)
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsFloat(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMin(fromHex("1234ABCD"));
statistics.setMax(fromHex("12340000"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, FLOAT, "Test column")))
.isInstanceOfSatisfying(FloatStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), -3.59039552E8f);
assertEquals(columnStatistics.getMax(), 1.868E-41f);
assertEquals(columnStatistics.genericGetMin(), -3.59039552E8f);
assertEquals(columnStatistics.genericGetMax(), 1.868E-41f);
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsDouble(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMin(fromHex("001234ABCD000000"));
statistics.setMax(fromHex("000000000000E043"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, DOUBLE, "Test column")))
.isInstanceOfSatisfying(DoubleStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), 4.36428250013E-312);
assertEquals(columnStatistics.getMax(), 9.223372036854776E18);
assertEquals(columnStatistics.genericGetMin(), 4.36428250013E-312);
assertEquals(columnStatistics.genericGetMax(), 9.223372036854776E18);
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsInt64WithoutNullCount(Optional<String> fileCreatedBy)
{
Expand All @@ -114,6 +160,126 @@ public void testReadStatsInt64WithoutNullCount(Optional<String> fileCreatedBy)
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsInt64WithoutMin(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMax(fromHex("3AA4000000000000"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, INT64, "Test column")))
.isInstanceOfSatisfying(LongStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), 0);
assertEquals(columnStatistics.getMax(), 0); // file statistics indicate 42042
assertEquals(columnStatistics.genericGetMin(), (Long) 0L);
assertEquals(columnStatistics.genericGetMax(), (Long) 0L); // file statistics indicate 42042
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsInt64WithoutMax(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMin(fromHex("F6FFFFFFFFFFFFFF"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, INT64, "Test column")))
.isInstanceOfSatisfying(LongStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), 0); // file statistics indicate -10
assertEquals(columnStatistics.getMax(), 0);
assertEquals(columnStatistics.genericGetMin(), (Long) 0L); // file statistics indicate -10
assertEquals(columnStatistics.genericGetMax(), (Long) 0L);
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsFloatWithoutMin(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMax(fromHex("12340000"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, FLOAT, "Test column")))
.isInstanceOfSatisfying(FloatStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), 0f);
assertEquals(columnStatistics.getMax(), 0f);
assertEquals(columnStatistics.genericGetMin(), 0f);
assertEquals(columnStatistics.genericGetMax(), 0f);
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsFloatWithoutMax(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMin(fromHex("1234ABCD"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, FLOAT, "Test column")))
.isInstanceOfSatisfying(FloatStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), 0f);
assertEquals(columnStatistics.getMax(), 0f);
assertEquals(columnStatistics.genericGetMin(), 0f);
assertEquals(columnStatistics.genericGetMax(), 0f);
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsDoubleWithoutMin(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMax(fromHex("3AA4000000000000"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, DOUBLE, "Test column")))
.isInstanceOfSatisfying(DoubleStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), 0d);
assertEquals(columnStatistics.getMax(), 0d);
assertEquals(columnStatistics.genericGetMin(), 0d);
assertEquals(columnStatistics.genericGetMax(), 0d);
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsDoubleWithoutMax(Optional<String> fileCreatedBy)
{
Statistics statistics = new Statistics();
statistics.setNull_count(13);
statistics.setMin(fromHex("F6FFFFFFFFFFFFFF"));
assertThat(MetadataReader.readStats(fileCreatedBy, Optional.of(statistics), new PrimitiveType(OPTIONAL, DOUBLE, "Test column")))
.isInstanceOfSatisfying(DoubleStatistics.class, columnStatistics -> {
assertFalse(columnStatistics.isEmpty());

assertTrue(columnStatistics.isNumNullsSet());
assertEquals(columnStatistics.getNumNulls(), 13);

assertEquals(columnStatistics.getMin(), 0d);
assertEquals(columnStatistics.getMax(), 0d);
assertEquals(columnStatistics.genericGetMin(), 0d);
assertEquals(columnStatistics.genericGetMax(), 0d);
});
}

@Test(dataProvider = "allCreatedBy")
public void testReadStatsBinary(Optional<String> fileCreatedBy)
{
Expand Down