Skip to content

PARQUET-1310: Column indexes: Filtering #509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gszadovszky merged 10 commits into from
Aug 17, 2018
Merged

PARQUET-1310: Column indexes: Filtering #509

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
465cca6
PARQUET-1310: New options for column index based filtering
gszadovszky Jun 5, 2018
c00b2c0
PARQUET-1310: Prepare for skipping pages at reading
gszadovszky Jun 21, 2018
3c9fe3e
PARQUET-1310: Simple implementation of page filtering
gszadovszky Jul 3, 2018
c471c72
PARQUET-1310: Glue the whole stuff together; implement proper tests
gszadovszky Jul 26, 2018
c0ce0f3
PARQUET-1310: Modifications according to zi's comments
gszadovszky Aug 8, 2018
7d5faea
PARQUET-1310: Implement binary search for ASCENDING/DESCENDING orders
gszadovszky Aug 15, 2018
f9b6ecc
PARQUET-1310: Modifications according to zi's comments
gszadovszky Aug 16, 2018
e82b9d5
PARQUET-1310: Update terminology of column index based filtering
gszadovszky Aug 16, 2018
325e5f6
PARQUET-1310: Fix exception naming
gszadovszky Aug 17, 2018
659c1a0
PARQUET-1310: Fix issue introduced at exception renaming
gszadovszky Aug 17, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions parquet-hadoop/src/main/java/org/apache/parquet/HadoopReadOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

import java.util.Map;

import static org.apache.parquet.hadoop.ParquetInputFormat.COLUMN_INDEX_FILTERING_ENABLED;
import static org.apache.parquet.hadoop.ParquetInputFormat.DICTIONARY_FILTERING_ENABLED;
import static org.apache.parquet.hadoop.ParquetInputFormat.RECORD_FILTERING_ENABLED;
import static org.apache.parquet.hadoop.ParquetInputFormat.STATS_FILTERING_ENABLED;
Expand All @@ -43,6 +44,7 @@ private HadoopReadOptions(boolean useSignedStringMinMax,
boolean useStatsFilter,
boolean useDictionaryFilter,
boolean useRecordFilter,
boolean useColumnIndexFilter,
FilterCompat.Filter recordFilter,
MetadataFilter metadataFilter,
CompressionCodecFactory codecFactory,
Expand All @@ -51,8 +53,8 @@ private HadoopReadOptions(boolean useSignedStringMinMax,
Map<String, String> properties,
Configuration conf) {
super(
useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, recordFilter,
metadataFilter, codecFactory, allocator, maxAllocationSize, properties
useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties
);
this.conf = conf;
}
Expand Down Expand Up @@ -83,6 +85,7 @@ public Builder(Configuration conf) {
useDictionaryFilter(conf.getBoolean(STATS_FILTERING_ENABLED, true));
useStatsFilter(conf.getBoolean(DICTIONARY_FILTERING_ENABLED, true));
useRecordFilter(conf.getBoolean(RECORD_FILTERING_ENABLED, true));
useColumnIndexFilter(conf.getBoolean(COLUMN_INDEX_FILTERING_ENABLED, true));
withCodecFactory(HadoopCodecs.newFactory(conf, 0));
withRecordFilter(getFilter(conf));
withMaxAllocationInBytes(conf.getInt(ALLOCATION_SIZE, 8388608));
Expand All @@ -95,7 +98,7 @@ public Builder(Configuration conf) {
@Override
public ParquetReadOptions build() {
return new HadoopReadOptions(
useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter,
useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties,
conf);
}
Expand Down
20 changes: 19 additions & 1 deletion parquet-hadoop/src/main/java/org/apache/parquet/ParquetReadOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,14 @@ public class ParquetReadOptions {
private static final boolean RECORD_FILTERING_ENABLED_DEFAULT = true;
private static final boolean STATS_FILTERING_ENABLED_DEFAULT = true;
private static final boolean DICTIONARY_FILTERING_ENABLED_DEFAULT = true;
private static final boolean COLUMN_INDEX_FILTERING_ENABLED_DEFAULT = true;
private static final int ALLOCATION_SIZE_DEFAULT = 8388608; // 8MB

private final boolean useSignedStringMinMax;
private final boolean useStatsFilter;
private final boolean useDictionaryFilter;
private final boolean useRecordFilter;
private final boolean useColumnIndexFilter;
private final FilterCompat.Filter recordFilter;
private final ParquetMetadataConverter.MetadataFilter metadataFilter;
private final CompressionCodecFactory codecFactory;
Expand All @@ -55,6 +57,7 @@ public class ParquetReadOptions {
boolean useStatsFilter,
boolean useDictionaryFilter,
boolean useRecordFilter,
boolean useColumnIndexFilter,
FilterCompat.Filter recordFilter,
ParquetMetadataConverter.MetadataFilter metadataFilter,
CompressionCodecFactory codecFactory,
Expand All @@ -65,6 +68,7 @@ public class ParquetReadOptions {
this.useStatsFilter = useStatsFilter;
this.useDictionaryFilter = useDictionaryFilter;
this.useRecordFilter = useRecordFilter;
this.useColumnIndexFilter = useColumnIndexFilter;
this.recordFilter = recordFilter;
this.metadataFilter = metadataFilter;
this.codecFactory = codecFactory;
Expand All @@ -89,6 +93,10 @@ public boolean useRecordFilter() {
return useRecordFilter;
}

public boolean useColumnIndexFilter() {
return useColumnIndexFilter;
}

public FilterCompat.Filter getRecordFilter() {
return recordFilter;
}
Expand Down Expand Up @@ -134,6 +142,7 @@ public static class Builder {
protected boolean useStatsFilter = STATS_FILTERING_ENABLED_DEFAULT;
protected boolean useDictionaryFilter = DICTIONARY_FILTERING_ENABLED_DEFAULT;
protected boolean useRecordFilter = RECORD_FILTERING_ENABLED_DEFAULT;
protected boolean useColumnIndexFilter = COLUMN_INDEX_FILTERING_ENABLED_DEFAULT;
protected FilterCompat.Filter recordFilter = null;
protected ParquetMetadataConverter.MetadataFilter metadataFilter = NO_FILTER;
// the page size parameter isn't used when only using the codec factory to get decompressors
Expand Down Expand Up @@ -182,6 +191,15 @@ public Builder useRecordFilter() {
return this;
}

public Builder useColumnIndexFilter(boolean useColumnIndexFilter) {
this.useColumnIndexFilter = useColumnIndexFilter;
return this;
}

public Builder useColumnIndexFilter() {

@zivanfi zivanfi Aug 3, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(nit) I would remove this convenience method as it is not only superfluous but also unnecessary on the "convenience" path, since true is already the default. With this method we have 3 ways of setting the value to true: not doing anything, calling useColumnIndexFilter(true) and calling useColumnIndexFilter().

@gszadovszky gszadovszky Aug 7, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've followed the pattern of the other options (e.g. useRecordFilter(boolean) and useRecordFilter() etc.). I think, it is better to be consistent.

return useColumnIndexFilter(true);
}

public Builder withRecordFilter(FilterCompat.Filter rowGroupFilter) {
this.recordFilter = rowGroupFilter;
return this;
Expand Down Expand Up @@ -239,7 +257,7 @@ public Builder copy(ParquetReadOptions options) {

public ParquetReadOptions build() {
return new ParquetReadOptions(
useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter,
useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties);
}
}
Expand Down
5 changes: 5 additions & 0 deletions parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,11 @@ public class ParquetInputFormat<T> extends FileInputFormat<Void, T> {
*/
public static final String DICTIONARY_FILTERING_ENABLED = "parquet.filter.dictionary.enabled";

/**
* key to configure whether column index filtering of pages is enabled
*/
public static final String COLUMN_INDEX_FILTERING_ENABLED = "parquet.filter.columnindex.enabled";

/**
* key to turn on or off task side metadata loading (default true)
* if true then metadata is read on the task side and some tasks may finish immediately.
Expand Down