diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 4478d74815072..7a1b65370d482 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -126,6 +126,7 @@ import java.util.stream.IntStream; import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.and; import static com.facebook.presto.hive.HiveAnalyzeProperties.getPartitionList; import static com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics; import static com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics; @@ -1739,24 +1740,26 @@ public ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, Co return new ConnectorPushdownFilterResult(getTableLayout(session, currentLayoutHandle.get()), TRUE_CONSTANT); } - if (currentLayoutHandle.isPresent()) { - throw new UnsupportedOperationException("Partial filter pushdown is not supported"); - } - // Split the filter into 3 groups of conjuncts: // - range filters that apply to entire columns, // - range filters that apply to subfields, - // - the rest - ExtractionResult decomposedFilter = rowExpressionService.getDomainTranslator().fromPredicate( - session, - filter, - new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor()); + // - the rest. Intersect these with possibly pre-existing filters. + ExtractionResult decomposedFilter = rowExpressionService.getDomainTranslator() + .fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor()); + if (currentLayoutHandle.isPresent()) { + HiveTableLayoutHandle currentHiveLayout = (HiveTableLayoutHandle) currentLayoutHandle.get(); + decomposedFilter = intersectExtractionResult(decomposedFilter, new ExtractionResult(currentHiveLayout.getDomainPredicate(), currentHiveLayout.getRemainingPredicate())); + } Map columnHandles = getColumnHandles(session, tableHandle); TupleDomain entireColumnDomain = decomposedFilter.getTupleDomain() .transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null) .transform(columnHandles::get); - // TODO Extract deterministic conjuncts that apply to partition columns and specify these as Contraint#predicate + if (currentLayoutHandle.isPresent()) { + entireColumnDomain = entireColumnDomain.intersect(((HiveTableLayoutHandle) (currentLayoutHandle.get())).getPartitionColumnPredicate()); + } + + // TODO Extract deterministic conjuncts that apply to partition columns and specify these as Constraint#predicate HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, new Constraint<>(entireColumnDomain), session); TupleDomain domainPredicate = withColumnDomains(ImmutableMap.builder() @@ -1804,6 +1807,21 @@ public ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, Co TRUE_CONSTANT); } + private static ExtractionResult intersectExtractionResult(ExtractionResult left, ExtractionResult right) + { + RowExpression newRemainingExpression; + if (right.getRemainingExpression().equals(TRUE_CONSTANT)) { + newRemainingExpression = left.getRemainingExpression(); + } + else if (left.getRemainingExpression().equals(TRUE_CONSTANT)) { + newRemainingExpression = right.getRemainingExpression(); + } + else { + newRemainingExpression = and(left.getRemainingExpression(), right.getRemainingExpression()); + } + return new ExtractionResult(left.getTupleDomain().intersect(right.getTupleDomain()), newRemainingExpression); + } + private String createTableLayoutString( ConnectorSession session, SchemaTableName tableName, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java index 38e79fdfe2f77..ebe4677943fae 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java @@ -581,6 +581,19 @@ public void testFilterFunctions() assertFilterProject("nested_keys[1][5] > 0 AND orderkey % 5 > 10", "keys"); } + @Test + public void testPushdownComposition() + { + // Tests composing two pushdowns each with a range filter and filter function. + assertQuery( + "WITH data AS (" + + " SELECT l.suppkey, l.linenumber, l.shipmode, MAX(o.orderdate)" + + " FROM lineitem l, orders o WHERE" + + " o.orderkey = l.orderkey AND linenumber IN (2, 3, 4, 6) AND shipmode LIKE '%AIR%'" + + " GROUP BY l.suppkey, l.linenumber, l.shipmode)" + + "SELECT COUNT(*) FROM data WHERE suppkey BETWEEN 10 AND 30 AND shipmode LIKE '%REG%'"); + } + @Test public void testPartitionColumns() {