Skip to content

Pushdown dereferences in ORC Reader #3445

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martint merged 3 commits into from
Apr 23, 2020
Merged

Pushdown dereferences in ORC Reader #3445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7130621
Add test for dereferences on data containing null rows
phd3 Apr 16, 2020
cfc6df1
Pushdown dereference projections in ORC reader
phd3 Apr 16, 2020
83b3d52
Implement predicate pushdown for nested columns in ORC reader
phd3 Jan 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 79 additions & 5 deletions presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import io.prestosql.plugin.hive.FileFormatDataSourceStats;
import io.prestosql.plugin.hive.HdfsEnvironment;
import io.prestosql.plugin.hive.HiveColumnHandle;
import io.prestosql.plugin.hive.HiveColumnProjectionInfo;
import io.prestosql.plugin.hive.HivePageSourceFactory;
import io.prestosql.plugin.hive.ReaderProjections;
import io.prestosql.plugin.hive.orc.OrcPageSource.ColumnAdaptation;
Expand Down Expand Up @@ -58,12 +59,16 @@
import java.util.Optional;
import java.util.Properties;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.nullToEmpty;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.Maps.uniqueIndex;
import static io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext;
import static io.prestosql.orc.OrcReader.INITIAL_BATCH_SIZE;
import static io.prestosql.orc.OrcReader.ProjectedLayout.createProjectedLayout;
import static io.prestosql.orc.OrcReader.ProjectedLayout.fullyProjectedLayout;
import static io.prestosql.orc.metadata.OrcType.OrcTypeKind.INT;
import static io.prestosql.orc.metadata.OrcType.OrcTypeKind.LONG;
import static io.prestosql.orc.metadata.OrcType.OrcTypeKind.STRUCT;
Expand All @@ -90,6 +95,8 @@
import static java.lang.String.format;
import static java.util.Locale.ENGLISH;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.ql.io.AcidUtils.isFullAcidTable;

public class OrcPageSourceFactory
Expand Down Expand Up @@ -149,7 +156,6 @@ public Optional<ReaderPageSourceWithProjections> createPageSource(
}

Optional<ReaderProjections> projectedReaderColumns = projectBaseColumns(columns);
effectivePredicate = effectivePredicate.transform(column -> column.isBaseColumn() ? column : null);

ConnectorPageSource orcPageSource = createOrcPageSource(
hdfsEnvironment,
Expand All @@ -162,6 +168,7 @@ public Optional<ReaderPageSourceWithProjections> createPageSource(
projectedReaderColumns
.map(ReaderProjections::getReaderColumns)
.orElse(columns),
columns,
isUseOrcColumnNames(session),
isFullAcidTable(Maps.fromProperties(schema)),
effectivePredicate,
Expand Down Expand Up @@ -190,6 +197,7 @@ private static OrcPageSource createOrcPageSource(
long length,
long fileSize,
List<HiveColumnHandle> columns,
List<HiveColumnHandle> projections,
boolean useOrcColumnNames,
boolean isFullAcid,
TupleDomain<HiveColumnHandle> effectivePredicate,
Expand Down Expand Up @@ -229,17 +237,23 @@ private static OrcPageSource createOrcPageSource(
List<OrcColumn> fileColumns = reader.getRootColumn().getNestedColumns();
List<OrcColumn> fileReadColumns = new ArrayList<>(columns.size() + (isFullAcid ? 3 : 0));
List<Type> fileReadTypes = new ArrayList<>(columns.size() + (isFullAcid ? 3 : 0));
List<OrcReader.ProjectedLayout> fileReadLayouts = new ArrayList<>(columns.size() + (isFullAcid ? 3 : 0));
if (isFullAcid) {
verifyAcidSchema(reader, path);
Map<String, OrcColumn> acidColumnsByName = uniqueIndex(fileColumns, orcColumn -> orcColumn.getColumnName().toLowerCase(ENGLISH));
fileColumns = acidColumnsByName.get(ACID_COLUMN_ROW_STRUCT.toLowerCase(ENGLISH)).getNestedColumns();

fileReadColumns.add(acidColumnsByName.get(ACID_COLUMN_ORIGINAL_TRANSACTION.toLowerCase(ENGLISH)));
fileReadTypes.add(BIGINT);
fileReadLayouts.add(fullyProjectedLayout());

fileReadColumns.add(acidColumnsByName.get(ACID_COLUMN_BUCKET.toLowerCase(ENGLISH)));
fileReadTypes.add(INTEGER);
fileReadLayouts.add(fullyProjectedLayout());

fileReadColumns.add(acidColumnsByName.get(ACID_COLUMN_ROW_ID.toLowerCase(ENGLISH)));
fileReadTypes.add(BIGINT);
fileReadLayouts.add(fullyProjectedLayout());
}

Map<String, OrcColumn> fileColumnsByName = ImmutableMap.of();
Expand All @@ -250,18 +264,53 @@ private static OrcPageSource createOrcPageSource(
fileColumnsByName = uniqueIndex(fileColumns, orcColumn -> orcColumn.getColumnName().toLowerCase(ENGLISH));
}

Map<String, List<List<String>>> projectionsByColumnName = ImmutableMap.of();
Map<Integer, List<List<String>>> projectionsByColumnIndex = ImmutableMap.of();
if (useOrcColumnNames || isFullAcid) {
projectionsByColumnName = projections.stream()
.collect(Collectors.groupingBy(
HiveColumnHandle::getBaseColumnName,
mapping(
column -> column.getHiveColumnProjectionInfo().map(HiveColumnProjectionInfo::getDereferenceNames).orElse(ImmutableList.<String>of()),
toList())));
}
else {
projectionsByColumnIndex = projections.stream()
.collect(Collectors.groupingBy(
HiveColumnHandle::getBaseHiveColumnIndex,
mapping(
column -> column.getHiveColumnProjectionInfo().map(HiveColumnProjectionInfo::getDereferenceNames).orElse(ImmutableList.<String>of()),
toList())));
}

TupleDomainOrcPredicateBuilder predicateBuilder = TupleDomainOrcPredicate.builder()
.setBloomFiltersEnabled(options.isBloomFiltersEnabled());
Map<HiveColumnHandle, Domain> effectivePredicateDomains = effectivePredicate.getDomains()
.orElseThrow(() -> new IllegalArgumentException("Effective predicate is none"));
List<ColumnAdaptation> columnAdaptations = new ArrayList<>(columns.size());
for (HiveColumnHandle column : columns) {
OrcColumn orcColumn = null;
OrcReader.ProjectedLayout projectedLayout = null;
Map<Optional<HiveColumnProjectionInfo>, Domain> columnDomains = null;

if (useOrcColumnNames || isFullAcid) {
orcColumn = fileColumnsByName.get(column.getName().toLowerCase(ENGLISH));
String columnName = column.getName().toLowerCase(ENGLISH);
orcColumn = fileColumnsByName.get(columnName);
if (orcColumn != null) {
projectedLayout = createProjectedLayout(orcColumn, projectionsByColumnName.get(columnName));
columnDomains = effectivePredicateDomains.entrySet().stream()
.filter(columnDomain -> columnDomain.getKey().getBaseColumnName().toLowerCase(ENGLISH).equals(columnName))
.collect(toImmutableMap(columnDomain -> columnDomain.getKey().getHiveColumnProjectionInfo(), Map.Entry::getValue));
}
}
else if (column.getBaseHiveColumnIndex() < fileColumns.size()) {
orcColumn = fileColumns.get(column.getBaseHiveColumnIndex());
if (orcColumn != null) {
projectedLayout = createProjectedLayout(orcColumn, projectionsByColumnIndex.get(column.getBaseHiveColumnIndex()));
columnDomains = effectivePredicateDomains.entrySet().stream()
.filter(columnDomain -> columnDomain.getKey().getBaseHiveColumnIndex() == column.getBaseHiveColumnIndex())
.collect(toImmutableMap(columnDomain -> columnDomain.getKey().getHiveColumnProjectionInfo(), Map.Entry::getValue));
}
}

Type readType = column.getType();
Expand All @@ -270,10 +319,14 @@ else if (column.getBaseHiveColumnIndex() < fileColumns.size()) {
columnAdaptations.add(ColumnAdaptation.sourceColumn(sourceIndex));
fileReadColumns.add(orcColumn);
fileReadTypes.add(readType);
fileReadLayouts.add(projectedLayout);

Domain domain = effectivePredicateDomains.get(column);
if (domain != null) {
predicateBuilder.addColumn(orcColumn.getColumnId(), domain);
// Add predicates on top-level and nested columns
for (Map.Entry<Optional<HiveColumnProjectionInfo>, Domain> columnDomain : columnDomains.entrySet()) {
OrcColumn nestedColumn = getNestedColumn(orcColumn, columnDomain.getKey());
if (nestedColumn != null) {
predicateBuilder.addColumn(nestedColumn.getColumnId(), columnDomain.getValue());
}
}
}
else {
Expand All @@ -284,6 +337,7 @@ else if (column.getBaseHiveColumnIndex() < fileColumns.size()) {
OrcRecordReader recordReader = reader.createRecordReader(
fileReadColumns,
fileReadTypes,
fileReadLayouts,
predicateBuilder.build(),
start,
length,
Expand Down Expand Up @@ -364,4 +418,24 @@ private static void verifyAcidColumn(OrcReader orcReader, int columnIndex, Strin
throw new PrestoException(HIVE_BAD_DATA, format("ORC ACID file %s column should be type %s: %s", columnName, columnType, path));
}
}

private static OrcColumn getNestedColumn(OrcColumn baseColumn, Optional<HiveColumnProjectionInfo> projectionInfo)
{
if (!projectionInfo.isPresent()) {
return baseColumn;
}

OrcColumn current = baseColumn;
for (String field : projectionInfo.get().getDereferenceNames()) {
Optional<OrcColumn> orcColumn = current.getNestedColumns().stream()
.filter(column -> column.getColumnName().toLowerCase(ENGLISH).equals(field))
.findFirst();

if (!orcColumn.isPresent()) {
return null;
}
current = orcColumn.get();
}
return current;
}
}
44 changes: 44 additions & 0 deletions presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3017,6 +3017,50 @@ private void testRows(Session session, HiveStorageFormat format)
assertUpdate(session, "DROP TABLE " + tableName);
}

@Test
public void testRowsWithNulls()
{
testRowsWithNulls(getSession(), HiveStorageFormat.ORC);
testRowsWithNulls(getSession(), HiveStorageFormat.PARQUET);
}

private void testRowsWithNulls(Session session, HiveStorageFormat format)
{
String tableName = "test_dereferences_with_nulls";
@Language("SQL") String createTable = "" +
"CREATE TABLE " + tableName + "\n" +
"(col0 BIGINT, col1 row(f0 BIGINT, f1 BIGINT), col2 row(f0 BIGINT, f1 ROW(f0 BIGINT, f1 BIGINT)))\n" +
"WITH (format = '" + format + "')";

assertUpdate(session, createTable);

@Language("SQL") String insertTable = "" +
"INSERT INTO " + tableName + " VALUES \n" +
"row(1, row(2, 3), row(4, row(5, 6))),\n" +
"row(7, row(8, 9), row(10, row(11, NULL))),\n" +
"row(NULL, NULL, row(12, NULL)),\n" +
"row(13, row(NULL, 14), NULL),\n" +
"row(15, row(16, NULL), row(NULL, row(17, 18)))";

assertUpdate(session, insertTable, 5);

assertQuery(
session,
format("SELECT col0, col1.f0, col2.f1.f1 FROM %s", tableName),
"SELECT * FROM \n" +
" (SELECT 1, 2, 6) UNION\n" +
" (SELECT 7, 8, NULL) UNION\n" +
" (SELECT NULL, NULL, NULL) UNION\n" +
" (SELECT 13, NULL, NULL) UNION\n" +
" (SELECT 15, 16, 18)");

assertQuery(session, format("SELECT col0 FROM %s WHERE col2.f1.f1 IS NOT NULL", tableName), "SELECT * FROM UNNEST(array[1, 15])");

assertQuery(session, format("SELECT col0, col1.f0, col1.f1 FROM %s WHERE col2.f1.f1 = 18", tableName), "SELECT 15, 16, NULL");

assertUpdate(session, "DROP TABLE " + tableName);
}

@Test
public void testComplex()
{
Expand Down
Loading