diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/SqlEarlyOutJoinsBenchmarks.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/SqlEarlyOutJoinsBenchmarks.java new file mode 100644 index 0000000000000..036eaefa18a7c --- /dev/null +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/SqlEarlyOutJoinsBenchmarks.java @@ -0,0 +1,88 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.testing.LocalQueryRunner; +import com.google.common.collect.ImmutableMap; +import org.intellij.lang.annotations.Language; + +import java.util.Map; + +import static com.facebook.presto.SystemSessionProperties.EXPLOIT_CONSTRAINTS; +import static com.facebook.presto.SystemSessionProperties.IN_PREDICATES_AS_INNER_JOINS_ENABLED; +import static com.facebook.presto.SystemSessionProperties.PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD; + +public class SqlEarlyOutJoinsBenchmarks + extends AbstractSqlBenchmark +{ + private static final Logger LOGGER = Logger.get(SqlEarlyOutJoinsBenchmarks.class); + + private static Map disableOptimization = ImmutableMap.of(IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.toString(false), + EXPLOIT_CONSTRAINTS, Boolean.toString(true)); + private static Map enableOptimization = ImmutableMap.of(IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.toString(true), + EXPLOIT_CONSTRAINTS, Boolean.toString(true)); + + public SqlEarlyOutJoinsBenchmarks(LocalQueryRunner localQueryRunner, @Language("SQL") String sql) + { + super(localQueryRunner, "early_out_joins", 10, 10, sql); + } + + public static void main(String[] args) + { + benchmarkTransformDistinctInnerJoinToLeftEarlyOutJoin(); + benchmarkTransformDistinctInnerJoinToRightEarlyOutJoin(); + benchmarkRewriteOfInPredicateToDistinctInnerJoin(); + } + + private static void benchmarkTransformDistinctInnerJoinToLeftEarlyOutJoin() + { + LOGGER.info("benchmarkTransformDistinctInnerJoinToLeftEarlyOutJoin"); + String sql = "select distinct orderkey from lineitem, nation where orderkey=nationkey"; + LOGGER.info("Without optimization"); + new SqlEarlyOutJoinsBenchmarks(BenchmarkQueryRunner.createLocalQueryRunner(disableOptimization), sql).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out)); + LOGGER.info("With optimization"); + new SqlEarlyOutJoinsBenchmarks(BenchmarkQueryRunner.createLocalQueryRunner(enableOptimization), sql).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out)); + } + + private static void benchmarkTransformDistinctInnerJoinToRightEarlyOutJoin() + { + LOGGER.info("benchmarkTransformDistinctInnerJoinToRightEarlyOutJoin"); + String sql = "select distinct l.orderkey, l.comment from lineitem l, orders o where l.orderkey = o.orderkey"; + LOGGER.info("Without optimization"); + new SqlEarlyOutJoinsBenchmarks(BenchmarkQueryRunner.createLocalQueryRunner(disableOptimization), sql).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out)); + LOGGER.info("With optimization"); + new SqlEarlyOutJoinsBenchmarks(BenchmarkQueryRunner.createLocalQueryRunner(enableOptimization), sql).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out)); + } + + private static void benchmarkRewriteOfInPredicateToDistinctInnerJoin() + { + LOGGER.info("benchmarkInPredicateToDistinctInnerJoin"); + LOGGER.info("Case 1: Rewrite IN predicate to distinct + inner join"); + String sql = " explain select * from region where regionkey in (select orderkey from lineitem)"; + LOGGER.info("Without optimization"); + new SqlEarlyOutJoinsBenchmarks(BenchmarkQueryRunner.createLocalQueryRunner(disableOptimization), sql).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out)); + LOGGER.info("With optimization: case 1"); + new SqlEarlyOutJoinsBenchmarks(BenchmarkQueryRunner.createLocalQueryRunner(enableOptimization), sql).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out)); + + LOGGER.info("Case 2: Rewrite IN predicate to distinct + inner join and then push aggregation down into the probe of the join"); + //Use same query as previous and change the byte reduction threshold + LOGGER.info("With optimization: case 2"); + Map alteredByteReductionThreshold = ImmutableMap.builder() + .putAll(enableOptimization) + .put(PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD, "0.001") + .build(); + new SqlEarlyOutJoinsBenchmarks(BenchmarkQueryRunner.createLocalQueryRunner(enableOptimization), sql).runBenchmark(new SimpleLineBenchmarkResultWriter(System.out)); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 17c169f441e8f..157f675db3fcb 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -241,6 +241,8 @@ public final class SystemSessionProperties public static final String PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID = "push_remote_exchange_through_group_id"; public static final String OPTIMIZE_MULTIPLE_APPROX_PERCENTILE_ON_SAME_FIELD = "optimize_multiple_approx_percentile_on_same_field"; public static final String RANDOMIZE_OUTER_JOIN_NULL_KEY = "randomize_outer_join_null_key"; + public static final String IN_PREDICATES_AS_INNER_JOINS_ENABLED = "in_predicates_as_inner_joins_enabled"; + public static final String PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD = "push_aggregation_below_join_byte_reduction_threshold"; public static final String KEY_BASED_SAMPLING_ENABLED = "key_based_sampling_enabled"; public static final String KEY_BASED_SAMPLING_PERCENTAGE = "key_based_sampling_percentage"; public static final String KEY_BASED_SAMPLING_FUNCTION = "key_based_sampling_function"; @@ -1421,6 +1423,15 @@ public SystemSessionProperties( REMOVE_REDUNDANT_DISTINCT_AGGREGATION_ENABLED, "Enable removing distinct aggregation node if input is already distinct", featuresConfig.isRemoveRedundantDistinctAggregationEnabled(), + false), + booleanProperty(IN_PREDICATES_AS_INNER_JOINS_ENABLED, + "Enable transformation of IN predicates to inner joins", + featuresConfig.isInPredicatesAsInnerJoinsEnabled(), + false), + doubleProperty( + PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD, + "Byte reduction ratio threshold at which to disable pushdown of aggregation below inner join", + featuresConfig.getPushAggregationBelowJoinByteReductionThreshold(), false)); } @@ -2381,4 +2392,14 @@ public static boolean isRemoveRedundantDistinctAggregationEnabled(Session sessio { return session.getSystemProperty(REMOVE_REDUNDANT_DISTINCT_AGGREGATION_ENABLED, Boolean.class); } + + public static boolean isInPredicatesAsInnerJoinsEnabled(Session session) + { + return session.getSystemProperty(IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.class); + } + + public static double getPushAggregationBelowJoinByteReductionThreshold(Session session) + { + return session.getSystemProperty(PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD, Double.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 83b6714c35df8..9a3f15b9d02f5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -241,6 +241,8 @@ public class FeaturesConfig private boolean randomizeOuterJoinNullKey; private boolean isOptimizeConditionalAggregationEnabled; private boolean isRemoveRedundantDistinctAggregationEnabled = true; + private boolean inPredicatesAsInnerJoinsEnabled; + private double pushAggregationBelowJoinByteReductionThreshold = 1; public enum PartitioningPrecisionStrategy { @@ -2287,4 +2289,30 @@ public FeaturesConfig setRemoveRedundantDistinctAggregationEnabled(boolean isRem this.isRemoveRedundantDistinctAggregationEnabled = isRemoveRedundantDistinctAggregationEnabled; return this; } + + public boolean isInPredicatesAsInnerJoinsEnabled() + { + return inPredicatesAsInnerJoinsEnabled; + } + + @Config("optimizer.in-predicates-as-inner-joins-enabled") + @ConfigDescription("Enable rewrite of In predicates to INNER joins") + public FeaturesConfig setInPredicatesAsInnerJoinsEnabled(boolean inPredicatesAsInnerJoinsEnabled) + { + this.inPredicatesAsInnerJoinsEnabled = inPredicatesAsInnerJoinsEnabled; + return this; + } + + public double getPushAggregationBelowJoinByteReductionThreshold() + { + return pushAggregationBelowJoinByteReductionThreshold; + } + + @Config("optimizer.push-aggregation-below-join-byte-reduction-threshold") + @ConfigDescription("Byte reduction ratio threshold at which to disable pushdown of aggregation below inner join") + public FeaturesConfig setPushAggregationBelowJoinByteReductionThreshold(double pushAggregationBelowJoinByteReductionThreshold) + { + this.pushAggregationBelowJoinByteReductionThreshold = pushAggregationBelowJoinByteReductionThreshold; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 5637d769e691a..5bdd1ad797781 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -122,7 +122,10 @@ import com.facebook.presto.sql.planner.iterative.rule.TransformCorrelatedScalarAggregationToJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformCorrelatedScalarSubquery; import com.facebook.presto.sql.planner.iterative.rule.TransformCorrelatedSingleRowSubqueryToProject; +import com.facebook.presto.sql.planner.iterative.rule.TransformDistinctInnerJoinToLeftEarlyOutJoin; +import com.facebook.presto.sql.planner.iterative.rule.TransformDistinctInnerJoinToRightEarlyOutJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformExistsApplyToLateralNode; +import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedLateralToJoin; import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; @@ -395,6 +398,7 @@ public PlanOptimizers( ImmutableSet.of( new RemoveUnreferencedScalarLateralNodes(), new TransformUncorrelatedLateralToJoin(), + new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin(), new TransformUncorrelatedInPredicateSubqueryToSemiJoin(), new TransformCorrelatedScalarAggregationToJoin(metadata.getFunctionAndTypeManager()), new TransformCorrelatedLateralJoinToJoin())), @@ -599,6 +603,24 @@ public PlanOptimizers( // We run it again to mark this for intermediate join nodes. builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new HistoricalStatisticsEquivalentPlanMarkingOptimizer(statsCalculator))); + // Run this set of join transformations after ReorderJoins, but before DetermineJoinDistributionType + builder.add(new IterativeOptimizer( + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + Optional.of(new LogicalPropertiesProviderImpl(new FunctionResolution(metadata.getFunctionAndTypeManager()))), + ImmutableSet.of( + new TransformDistinctInnerJoinToLeftEarlyOutJoin(), + new TransformDistinctInnerJoinToRightEarlyOutJoin(), + new RemoveRedundantDistinct(), + new RemoveRedundantTopN(), + new RemoveRedundantSort(), + new RemoveRedundantLimit(), + new RemoveRedundantDistinctLimit(), + new RemoveRedundantAggregateDistinct(), + new RemoveRedundantIdentityProjections(), + new PushAggregationThroughOuterJoin(metadata.getFunctionAndTypeManager())))); + builder.add(new IterativeOptimizer( ruleStats, statsCalculator, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java index 747968ca4729f..e86b800b418ff 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java @@ -39,12 +39,15 @@ import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.InPredicate; import com.facebook.presto.sql.tree.LambdaArgumentDeclaration; +import com.facebook.presto.sql.tree.LogicalBinaryExpression; import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.NodeRef; import com.facebook.presto.sql.tree.NotExpression; import com.facebook.presto.sql.tree.QuantifiedComparisonExpression; import com.facebook.presto.sql.tree.QuantifiedComparisonExpression.Quantifier; import com.facebook.presto.sql.tree.Query; +import com.facebook.presto.sql.tree.SearchedCaseExpression; +import com.facebook.presto.sql.tree.SimpleCaseExpression; import com.facebook.presto.sql.tree.SubqueryExpression; import com.facebook.presto.sql.tree.SymbolReference; import com.facebook.presto.util.MorePredicates; @@ -69,6 +72,7 @@ import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.EQUAL; +import static com.facebook.presto.sql.tree.LogicalBinaryExpression.Operator.OR; import static com.facebook.presto.sql.util.AstUtils.nodeContains; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -132,10 +136,11 @@ public PlanBuilder handleSubqueries(PlanBuilder builder, Expression expression, private PlanBuilder handleSubqueries(PlanBuilder builder, Expression expression, Node node, boolean correlationAllowed, SqlPlannerContext context) { - builder = appendInPredicateApplyNodes(builder, collectInPredicateSubqueries(expression, node), correlationAllowed, node, context); + boolean mayParticipateInAntiJoin = detectAntiJoinInExpression(expression); + builder = appendInPredicateApplyNodes(builder, collectInPredicateSubqueries(expression, node), correlationAllowed, mayParticipateInAntiJoin, node, context); builder = appendScalarSubqueryApplyNodes(builder, collectScalarSubqueries(expression, node), correlationAllowed, context); - builder = appendExistsSubqueryApplyNodes(builder, collectExistsSubqueries(expression, node), correlationAllowed, context); - builder = appendQuantifiedComparisonApplyNodes(builder, collectQuantifiedComparisonSubqueries(expression, node), correlationAllowed, node, context); + builder = appendExistsSubqueryApplyNodes(builder, collectExistsSubqueries(expression, node), correlationAllowed, mayParticipateInAntiJoin, context); + builder = appendQuantifiedComparisonApplyNodes(builder, collectQuantifiedComparisonSubqueries(expression, node), correlationAllowed, mayParticipateInAntiJoin, node, context); return builder; } @@ -171,15 +176,15 @@ public Set collectQuantifiedComparisonSubqueries .collect(toImmutableSet()); } - private PlanBuilder appendInPredicateApplyNodes(PlanBuilder subPlan, Set inPredicates, boolean correlationAllowed, Node node, SqlPlannerContext context) + private PlanBuilder appendInPredicateApplyNodes(PlanBuilder subPlan, Set inPredicates, boolean correlationAllowed, boolean mayParticipateInAntiJoin, Node node, SqlPlannerContext context) { for (InPredicate inPredicate : inPredicates) { - subPlan = appendInPredicateApplyNode(subPlan, inPredicate, correlationAllowed, node, context); + subPlan = appendInPredicateApplyNode(subPlan, inPredicate, correlationAllowed, mayParticipateInAntiJoin, node, context); } return subPlan; } - private PlanBuilder appendInPredicateApplyNode(PlanBuilder subPlan, InPredicate inPredicate, boolean correlationAllowed, Node node, SqlPlannerContext context) + private PlanBuilder appendInPredicateApplyNode(PlanBuilder subPlan, InPredicate inPredicate, boolean correlationAllowed, boolean mayParticipateInAntiJoin, Node node, SqlPlannerContext context) { if (subPlan.canTranslate(inPredicate)) { // given subquery is already appended @@ -204,7 +209,7 @@ private PlanBuilder appendInPredicateApplyNode(PlanBuilder subPlan, InPredicate subPlan.getTranslations().put(inPredicate, inPredicateSubqueryVariable); - return appendApplyNode(subPlan, inPredicate, subqueryPlan.getRoot(), Assignments.of(inPredicateSubqueryVariable, castToRowExpression(inPredicateSubqueryExpression)), correlationAllowed); + return appendApplyNode(subPlan, inPredicate, subqueryPlan.getRoot(), Assignments.of(inPredicateSubqueryVariable, castToRowExpression(inPredicateSubqueryExpression)), correlationAllowed, mayParticipateInAntiJoin); } private PlanBuilder appendScalarSubqueryApplyNodes(PlanBuilder builder, Set scalarSubqueries, boolean correlationAllowed, SqlPlannerContext context) @@ -261,10 +266,10 @@ public PlanBuilder appendLateralJoin(PlanBuilder subPlan, PlanBuilder subqueryPl subQueryNotSupportedError(query, "Given correlated subquery"))); } - private PlanBuilder appendExistsSubqueryApplyNodes(PlanBuilder builder, Set existsPredicates, boolean correlationAllowed, SqlPlannerContext context) + private PlanBuilder appendExistsSubqueryApplyNodes(PlanBuilder builder, Set existsPredicates, boolean correlationAllowed, boolean mayParticipateInAntiJoin, SqlPlannerContext context) { for (ExistsPredicate existsPredicate : existsPredicates) { - builder = appendExistSubqueryApplyNode(builder, existsPredicate, correlationAllowed, context); + builder = appendExistSubqueryApplyNode(builder, existsPredicate, correlationAllowed, mayParticipateInAntiJoin, context); } return builder; } @@ -278,7 +283,7 @@ private PlanBuilder appendExistsSubqueryApplyNodes(PlanBuilder builder, Set */ - private PlanBuilder appendExistSubqueryApplyNode(PlanBuilder subPlan, ExistsPredicate existsPredicate, boolean correlationAllowed, SqlPlannerContext context) + private PlanBuilder appendExistSubqueryApplyNode(PlanBuilder subPlan, ExistsPredicate existsPredicate, boolean correlationAllowed, boolean mayParticipateInAntiJoin, SqlPlannerContext context) { if (subPlan.canTranslate(existsPredicate)) { // given subquery is already appended @@ -304,18 +309,19 @@ private PlanBuilder appendExistSubqueryApplyNode(PlanBuilder subPlan, ExistsPred existsPredicate.getSubquery(), subqueryNode, Assignments.of(exists, castToRowExpression(rewrittenExistsPredicate)), - correlationAllowed); + correlationAllowed, + mayParticipateInAntiJoin); } - private PlanBuilder appendQuantifiedComparisonApplyNodes(PlanBuilder subPlan, Set quantifiedComparisons, boolean correlationAllowed, Node node, SqlPlannerContext context) + private PlanBuilder appendQuantifiedComparisonApplyNodes(PlanBuilder subPlan, Set quantifiedComparisons, boolean correlationAllowed, boolean mayParticipateInAntiJoin, Node node, SqlPlannerContext context) { for (QuantifiedComparisonExpression quantifiedComparison : quantifiedComparisons) { - subPlan = appendQuantifiedComparisonApplyNode(subPlan, quantifiedComparison, correlationAllowed, node, context); + subPlan = appendQuantifiedComparisonApplyNode(subPlan, quantifiedComparison, correlationAllowed, mayParticipateInAntiJoin, node, context); } return subPlan; } - private PlanBuilder appendQuantifiedComparisonApplyNode(PlanBuilder subPlan, QuantifiedComparisonExpression quantifiedComparison, boolean correlationAllowed, Node node, SqlPlannerContext context) + private PlanBuilder appendQuantifiedComparisonApplyNode(PlanBuilder subPlan, QuantifiedComparisonExpression quantifiedComparison, boolean correlationAllowed, boolean mayParticipateInAntiJoin, Node node, SqlPlannerContext context) { if (subPlan.canTranslate(quantifiedComparison)) { // given subquery is already appended @@ -325,12 +331,12 @@ private PlanBuilder appendQuantifiedComparisonApplyNode(PlanBuilder subPlan, Qua case EQUAL: switch (quantifiedComparison.getQuantifier()) { case ALL: - return planQuantifiedApplyNode(subPlan, quantifiedComparison, correlationAllowed, context); + return planQuantifiedApplyNode(subPlan, quantifiedComparison, correlationAllowed, mayParticipateInAntiJoin, context); case ANY: case SOME: // A = ANY B <=> A IN B InPredicate inPredicate = new InPredicate(quantifiedComparison.getValue(), quantifiedComparison.getSubquery()); - subPlan = appendInPredicateApplyNode(subPlan, inPredicate, correlationAllowed, node, context); + subPlan = appendInPredicateApplyNode(subPlan, inPredicate, correlationAllowed, mayParticipateInAntiJoin, node, context); subPlan.getTranslations().put(quantifiedComparison, subPlan.translate(inPredicate)); return subPlan; } @@ -349,7 +355,7 @@ private PlanBuilder appendQuantifiedComparisonApplyNode(PlanBuilder subPlan, Qua // "A <> ALL B" is equivalent to "NOT (A = ANY B)" so add a rewrite for the initial quantifiedComparison to notAny subPlan.getTranslations().put(quantifiedComparison, subPlan.getTranslations().rewrite(notAny)); // now plan "A = ANY B" part by calling ourselves for rewrittenAny - return appendQuantifiedComparisonApplyNode(subPlan, rewrittenAny, correlationAllowed, node, context); + return appendQuantifiedComparisonApplyNode(subPlan, rewrittenAny, correlationAllowed, true, node, context); case ANY: case SOME: // A <> ANY B <=> min B <> max B || A <> min B <=> !(min B = max B && A = min B) <=> !(A = ALL B) @@ -362,7 +368,7 @@ private PlanBuilder appendQuantifiedComparisonApplyNode(PlanBuilder subPlan, Qua // "A <> ANY B" is equivalent to "NOT (A = ALL B)" so add a rewrite for the initial quantifiedComparison to notAll subPlan.getTranslations().put(quantifiedComparison, subPlan.getTranslations().rewrite(notAll)); // now plan "A = ALL B" part by calling ourselves for rewrittenAll - return appendQuantifiedComparisonApplyNode(subPlan, rewrittenAll, correlationAllowed, node, context); + return appendQuantifiedComparisonApplyNode(subPlan, rewrittenAll, correlationAllowed, true, node, context); } break; @@ -370,14 +376,14 @@ private PlanBuilder appendQuantifiedComparisonApplyNode(PlanBuilder subPlan, Qua case LESS_THAN_OR_EQUAL: case GREATER_THAN: case GREATER_THAN_OR_EQUAL: - return planQuantifiedApplyNode(subPlan, quantifiedComparison, correlationAllowed, context); + return planQuantifiedApplyNode(subPlan, quantifiedComparison, correlationAllowed, mayParticipateInAntiJoin, context); } // all cases are checked, so this exception should never be thrown throw new IllegalArgumentException( format("Unexpected quantified comparison: '%s %s'", quantifiedComparison.getOperator().getValue(), quantifiedComparison.getQuantifier())); } - private PlanBuilder planQuantifiedApplyNode(PlanBuilder subPlan, QuantifiedComparisonExpression quantifiedComparison, boolean correlationAllowed, SqlPlannerContext context) + private PlanBuilder planQuantifiedApplyNode(PlanBuilder subPlan, QuantifiedComparisonExpression quantifiedComparison, boolean correlationAllowed, boolean mayParticipateInAntiJoin, SqlPlannerContext context) { subPlan = subPlan.appendProjections(ImmutableList.of(quantifiedComparison.getValue()), variableAllocator, idAllocator); @@ -402,7 +408,8 @@ private PlanBuilder planQuantifiedApplyNode(PlanBuilder subPlan, QuantifiedCompa quantifiedComparison.getSubquery(), subqueryPlan.getRoot(), Assignments.of(coercedQuantifiedComparisonVariable, castToRowExpression(coercedQuantifiedComparison)), - correlationAllowed); + correlationAllowed, + mayParticipateInAntiJoin); } private static boolean isAggregationWithEmptyGroupBy(PlanNode planNode) @@ -434,7 +441,7 @@ private List coercionsFor(Expression expression) .collect(toImmutableList()); } - private PlanBuilder appendApplyNode(PlanBuilder subPlan, Node subquery, PlanNode subqueryNode, Assignments subqueryAssignments, boolean correlationAllowed) + private PlanBuilder appendApplyNode(PlanBuilder subPlan, Node subquery, PlanNode subqueryNode, Assignments subqueryAssignments, boolean correlationAllowed, boolean mayParticipateInAntiJoin) { Map correlation = extractCorrelation(subPlan, subqueryNode); if (!correlationAllowed && !correlation.isEmpty()) { @@ -454,7 +461,8 @@ private PlanBuilder appendApplyNode(PlanBuilder subPlan, Node subquery, PlanNode subqueryNode, subqueryAssignments, ImmutableList.copyOf(VariablesExtractor.extractUnique(correlation.values(), variableAllocator.getTypes())), - subQueryNotSupportedError(subquery, "Given correlated subquery"))); + subQueryNotSupportedError(subquery, "Given correlated subquery"), + mayParticipateInAntiJoin)); } private Map extractCorrelation(PlanBuilder subPlan, PlanNode subquery) @@ -607,4 +615,55 @@ public PlanNode visitValues(ValuesNode node, RewriteContext context) node.getValuesNodeLabel()); } } + + private boolean detectAntiJoinInExpression(Expression expression) + { + AntiJoinDetector antiJoinDetector = new AntiJoinDetector(); + antiJoinDetector.process(expression); + return antiJoinDetector.getMayContainAntiJoin(); + } + + private static class AntiJoinDetector + extends DefaultExpressionTraversalVisitor + { + private boolean mayContainAntiJoin; + + @Override + protected Void visitNotExpression(NotExpression node, Void context) + { + mayContainAntiJoin = true; + return null; + } + + @Override + protected Void visitLogicalBinaryExpression(LogicalBinaryExpression node, Void context) + { + if (node.getOperator() == OR) { + mayContainAntiJoin = true; + return null; + } + process(node.getRight(), context); + process(node.getLeft(), context); + return null; + } + + @Override + protected Void visitSearchedCaseExpression(SearchedCaseExpression node, Void context) + { + mayContainAntiJoin = true; + return null; + } + + @Override + protected Void visitSimpleCaseExpression(SimpleCaseExpression node, Void context) + { + mayContainAntiJoin = true; + return null; + } + + public boolean getMayContainAntiJoin() + { + return mayContainAntiJoin; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java index 632b7a04518f2..0f1e83938ae5b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java @@ -293,6 +293,12 @@ public WarningCollector getWarningCollector() { return context.warningCollector; } + + @Override + public Optional getLogicalPropertiesProvider() + { + return logicalPropertiesProvider; + } }; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/Rule.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/Rule.java index 3d3a569774dd8..b9e2392912c3a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/Rule.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/Rule.java @@ -19,6 +19,7 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.LogicalPropertiesProvider; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.sql.planner.PlanVariableAllocator; @@ -58,6 +59,8 @@ interface Context void checkTimeoutNotExhausted(); WarningCollector getWarningCollector(); + + Optional getLogicalPropertiesProvider(); } final class Result diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java index cff1f7120b153..e9e2a12dc4336 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java @@ -17,7 +17,6 @@ import com.facebook.presto.spi.plan.LogicalProperties; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.AssignmentUtils; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.relational.FunctionResolution; @@ -164,6 +163,27 @@ public boolean isAtMost(long n) return maxCardProperty.isAtMost(n); } + /** + * Determines whether one set of expressions (expressions) can be realized/rewritten + * in terms of the other (targetVariables) using EquivalenceClasses + * + * @param expressions + * @param targetVariables + * @return True if all expressions can be realized in terms of targetVariables + */ + public boolean canBeHomogenized(Set expressions, Set targetVariables) + { + Set expressionsInTermsOfEquivalenceClassHeads = expressions.stream() + .map(equivalenceClassProperty::getEquivalenceClassHead) + .collect(Collectors.toSet()); + + Set targetVariablesInTermsOfEquivalenceClassHeads = targetVariables.stream() + .map(equivalenceClassProperty::getEquivalenceClassHead) + .collect(Collectors.toSet()); + + return targetVariablesInTermsOfEquivalenceClassHeads.containsAll(expressionsInTermsOfEquivalenceClassHeads); + } + private boolean keyRequirementSatisfied(Key keyRequirement) { if (maxCardProperty.isAtMostOne()) { @@ -273,10 +293,10 @@ public static LogicalPropertiesImpl propagateAndLimitProperties(LogicalPropertie * This logical properties builder should be used by PlanNode's that propagate their source * properties and add a unique key. For example, an AggregationNode with a single grouping key * propagates it's input properties and adds the grouping key attributes as a new unique key. - * The resulting properties are projected by the provided output variables. + * All computed logical properties are propagated (including ones that are projected out) */ - public static LogicalPropertiesImpl aggregationProperties(LogicalPropertiesImpl sourceProperties, Set keyVariables, List outputVariables) + public static LogicalPropertiesImpl aggregationProperties(LogicalPropertiesImpl sourceProperties, Set keyVariables) { checkArgument(!keyVariables.isEmpty(), "keyVariables is empty"); @@ -292,18 +312,20 @@ public static LogicalPropertiesImpl aggregationProperties(LogicalPropertiesImpl KeyProperty keyProperty = new KeyProperty(sourceProperties.keyProperty); resultProperties = new LogicalPropertiesImpl(equivalenceClassProperty, maxCardProperty, keyProperty); } - //project the properties using the output variables to ensure only the interesting constraints propagate - return projectProperties(resultProperties, AssignmentUtils.identityAssignments(outputVariables)); + + // Emit all interesting constraints, including ones that may be projected out + // Some optimizations (e.g. canBeHomogenized) may utilize these + return resultProperties; } /** * This logical properties builder should be used by PlanNode's that propagate their source * properties, add a unique key, and also limit the result. For example, a DistinctLimitNode. */ - public static LogicalPropertiesImpl distinctLimitProperties(LogicalPropertiesImpl sourceProperties, Set keyVariables, Long limit, List outputVariables) + public static LogicalPropertiesImpl distinctLimitProperties(LogicalPropertiesImpl sourceProperties, Set keyVariables, Long limit) { checkArgument(!keyVariables.isEmpty(), "keyVariables is empty"); - LogicalPropertiesImpl aggregationProperties = aggregationProperties(sourceProperties, keyVariables, outputVariables); + LogicalPropertiesImpl aggregationProperties = aggregationProperties(sourceProperties, keyVariables); return propagateAndLimitProperties(aggregationProperties, limit); } @@ -350,7 +372,6 @@ public static LogicalPropertiesImpl joinProperties(LogicalPropertiesImpl leftPro List equijoinPredicates, JoinNode.Type joinType, Optional filterPredicate, - List outputVariables, FunctionResolution functionResolution) { MaxCardProperty maxCardProperty = new MaxCardProperty(); @@ -412,10 +433,9 @@ public static LogicalPropertiesImpl joinProperties(LogicalPropertiesImpl leftPro } //since we likely merged equivalence class from left and right source we will normalize the key property - LogicalPropertiesImpl resultProperties = normalizeKeyPropertyAndSetMaxCard(keyProperty, maxCardProperty, equivalenceClassProperty); - - //project the resulting properties by the output variables - return projectProperties(resultProperties, AssignmentUtils.identityAssignments(outputVariables)); + // Emit all interesting constraints, including ones that may be projected out + // Some optimizations (e.g. canBeHomogenized) may utilize these + return normalizeKeyPropertyAndSetMaxCard(keyProperty, maxCardProperty, equivalenceClassProperty); } /** diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesProviderImpl.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesProviderImpl.java index 7b6e7fbe2cf61..37405227d61af 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesProviderImpl.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesProviderImpl.java @@ -164,7 +164,7 @@ public LogicalProperties getJoinProperties(PlanNode node) LogicalPropertiesImpl leftProps = (LogicalPropertiesImpl) ((GroupReference) joinNode.getLeft()).getLogicalProperties().get(); LogicalPropertiesImpl rightProps = (LogicalPropertiesImpl) ((GroupReference) joinNode.getRight()).getLogicalProperties().get(); - return joinProperties(leftProps, rightProps, joinNode.getCriteria(), joinNode.getType(), joinNode.getFilter(), joinNode.getOutputVariables(), functionResolution); + return joinProperties(leftProps, rightProps, joinNode.getCriteria(), joinNode.getType(), joinNode.getFilter(), functionResolution); } /** @@ -214,8 +214,7 @@ public LogicalProperties getAggregationProperties(AggregationNode aggregationNod } else { return aggregationProperties(sourceProperties, - aggregationNode.getGroupingKeys().stream().collect(Collectors.toSet()), - aggregationNode.getOutputVariables()); + aggregationNode.getGroupingKeys().stream().collect(Collectors.toSet())); } } @@ -244,7 +243,7 @@ public LogicalProperties getAssignUniqueIdProperties(PlanNode node) LogicalPropertiesImpl sourceProperties = (LogicalPropertiesImpl) ((GroupReference) assignUniqueIdNode.getSource()).getLogicalProperties().get(); Set key = new HashSet<>(); key.add(assignUniqueIdNode.getIdVariable()); - return aggregationProperties(sourceProperties, key, assignUniqueIdNode.getOutputVariables()); + return aggregationProperties(sourceProperties, key); } /** @@ -264,8 +263,7 @@ public LogicalProperties getDistinctLimitProperties(DistinctLimitNode distinctLi LogicalPropertiesImpl sourceProperties = (LogicalPropertiesImpl) ((GroupReference) distinctLimitNode.getSource()).getLogicalProperties().get(); return distinctLimitProperties(sourceProperties, distinctLimitNode.getDistinctVariables().stream().collect(Collectors.toSet()), - distinctLimitNode.getLimit(), - distinctLimitNode.getOutputVariables()); + distinctLimitNode.getLimit()); } /** diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java index 6a1f67c009336..f3a4241dc006c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java @@ -343,7 +343,8 @@ public Result apply(ApplyNode applyNode, Captures captures, Context context) applyNode.getSubquery(), subqueryAssignments, applyNode.getCorrelation(), - applyNode.getOriginSubqueryError())); + applyNode.getOriginSubqueryError(), + applyNode.getMayParticipateInAntiJoin())); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java index 0310e3ba92ef0..27de8b19f1252 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java @@ -346,7 +346,8 @@ public Result apply(ApplyNode applyNode, Captures captures, Context context) applyNode.getSubquery(), rewrittenAssignments.get(), applyNode.getCorrelation(), - applyNode.getOriginSubqueryError())); + applyNode.getOriginSubqueryError(), + applyNode.getMayParticipateInAntiJoin())); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToLeftEarlyOutJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToLeftEarlyOutJoin.java new file mode 100644 index 0000000000000..6abb575612c2d --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToLeftEarlyOutJoin.java @@ -0,0 +1,187 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LogicalProperties; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.SystemSessionProperties.getJoinReorderingStrategy; +import static com.facebook.presto.SystemSessionProperties.isExploitConstraints; +import static com.facebook.presto.SystemSessionProperties.isInPredicatesAsInnerJoinsEnabled; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; +import static com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.sql.planner.plan.Patterns.Join.type; +import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; +import static com.facebook.presto.sql.planner.plan.Patterns.join; +import static com.facebook.presto.sql.planner.plan.Patterns.source; +import static com.google.common.collect.Iterators.getOnlyElement; + +/** + * This optimizer looks for a distinct aggregation above an inner join and + * determines whether it can be pushed down into the right input of the join. + * This effectively converts the inner join into a semi join + *

+ * Plan before optimizer: + * 

+ * Aggregation (distinct)
+ *   Join (inner)
+ *     left
+ *     right
+ *
+ *

+ * Plan after optimizer: + * 

+ * Aggregation (distinct)
+ *   Project
+ *     Filter (semijoinvariable)
+ *     SemiJoin
+ *       source: left
+ *       filteringSource: right
+ *       semijoinOutput: semijoinvariable
+ *
+ */ +public class TransformDistinctInnerJoinToLeftEarlyOutJoin + implements Rule +{ + private static final Capture JOIN = newCapture(); + private static final Pattern PATTERN = aggregation() + .matching(AggregationNode::isDistinct) + .with(source().matching( + join() + .capturedAs(JOIN) + .with(type() + .matching(type -> type == INNER)))); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isInPredicatesAsInnerJoinsEnabled(session) && + isExploitConstraints(session) && + getJoinReorderingStrategy(session) == AUTOMATIC; + } + + @Override + public Result apply(AggregationNode aggregationNode, Captures captures, Context context) + { + JoinNode innerJoin = captures.get(JOIN); + + if (!canAggregationBePushedDown(aggregationNode, innerJoin, context)) { + return Result.empty(); + } + + EquiJoinClause equiJoinClause = getOnlyElement(innerJoin.getCriteria().listIterator()); + VariableReferenceExpression sourceJoinVariable = equiJoinClause.getLeft(); + VariableReferenceExpression filteringSourceJoinVariable = equiJoinClause.getRight(); + VariableReferenceExpression semiJoinVariable = context.getVariableAllocator().newVariable("semijoinvariable", BOOLEAN, "eoj"); + + SemiJoinNode semiJoinNode = new SemiJoinNode( + innerJoin.getSourceLocation(), + context.getIdAllocator().getNextId(), + innerJoin.getLeft(), + innerJoin.getRight(), + sourceJoinVariable, + filteringSourceJoinVariable, + semiJoinVariable, + Optional.empty(), + Optional.empty(), + Optional.empty(), + ImmutableMap.of()); + + FilterNode filterNode = new FilterNode( + semiJoinNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + semiJoinNode, + semiJoinVariable); + + Assignments.Builder assignments = Assignments.builder(); + filterNode.getOutputVariables() + .stream() + .filter(variable -> !variable.equals(semiJoinVariable)) + .forEach(variable -> assignments.put(variable, variable)); + + ProjectNode projectNode = new ProjectNode( + filterNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + filterNode, + assignments.build(), + LOCAL); + + AggregationNode newAggregationNode = new AggregationNode( + aggregationNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + projectNode, + aggregationNode.getAggregations(), + aggregationNode.getGroupingSets(), + aggregationNode.getPreGroupedVariables(), + aggregationNode.getStep(), + aggregationNode.getHashVariable(), + aggregationNode.getGroupIdVariable()); + + return Result.ofPlanNode(newAggregationNode); + } + + private boolean canAggregationBePushedDown(AggregationNode aggregationNode, JoinNode joinNode, Context context) + { + if (!context.getLogicalPropertiesProvider().isPresent()) { + return false; + } + + // Semijoin can have only one filtering condition + if (joinNode.isCrossJoin() || joinNode.getCriteria().size() != 1) { + return false; + } + + Set groupingVariables = ImmutableSet.copyOf(aggregationNode.getGroupingKeys()); + Set joinInputVariablesRight = ImmutableSet.copyOf(joinNode.getRight().getOutputVariables()); + Set joinInputVariablesLeft = ImmutableSet.copyOf(joinNode.getLeft().getOutputVariables()); + Set joinOutputVariables = ImmutableSet.copyOf(joinNode.getOutputVariables()); + + LogicalProperties aggregationNodelogicalProperties = context.getLogicalPropertiesProvider().get().getAggregationProperties(aggregationNode); + if (!aggregationNodelogicalProperties.canBeHomogenized(joinInputVariablesRight, groupingVariables)) { + return false; + } + + if (!joinInputVariablesLeft.equals(joinOutputVariables)) { + return false; + } + + return true; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToRightEarlyOutJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToRightEarlyOutJoin.java new file mode 100644 index 0000000000000..01b1b69425237 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToRightEarlyOutJoin.java @@ -0,0 +1,182 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.cost.StatsProvider; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.LogicalProperties; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.GroupReference; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.SystemSessionProperties.getJoinReorderingStrategy; +import static com.facebook.presto.SystemSessionProperties.getPushAggregationBelowJoinByteReductionThreshold; +import static com.facebook.presto.SystemSessionProperties.isExploitConstraints; +import static com.facebook.presto.SystemSessionProperties.isInPredicatesAsInnerJoinsEnabled; +import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.sql.planner.plan.Patterns.Join.type; +import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; +import static com.facebook.presto.sql.planner.plan.Patterns.join; +import static com.facebook.presto.sql.planner.plan.Patterns.source; + +/** + * This optimizer looks for a Distinct aggregation above an inner join and + * determines whether it can be pushed down into the left input of the join. + *

+ * Plan before optimizer: + * 

+ * Aggregation (distinct)
+ *   Join (inner)
+ *     left
+ *     right
+ *
+ *

+ * Plan after optimizer: + * 

+ * Aggregation (distinct)
+ *   Join (inner)
+ *     Aggregation (distinct)
+ *       left
+ *     right
+ *
+ */ +public class TransformDistinctInnerJoinToRightEarlyOutJoin + implements Rule +{ + private static final Capture JOIN = newCapture(); + private static final Pattern PATTERN = aggregation() + .matching(AggregationNode::isDistinct) + .with(source().matching( + join() + .capturedAs(JOIN) + .with(type() + .matching(type -> type == INNER)))); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isInPredicatesAsInnerJoinsEnabled(session) && + isExploitConstraints(session) && + getJoinReorderingStrategy(session) == AUTOMATIC; + } + + @Override + public Result apply(AggregationNode aggregationNode, Captures captures, Context context) + { + JoinNode innerJoin = captures.get(JOIN); + + if (!canAggregationBePushedDown(aggregationNode, innerJoin, context)) { + return Result.empty(); + } + + AggregationNode aggregationBelowJoin = new AggregationNode( + innerJoin.getLeft().getSourceLocation(), + context.getIdAllocator().getNextId(), + innerJoin.getLeft(), + ImmutableMap.of(), + singleGroupingSet(innerJoin.getLeft().getOutputVariables()), + ImmutableList.of(), + SINGLE, + Optional.empty(), + Optional.empty()); + + JoinNode newInnerJoin = new JoinNode( + innerJoin.getSourceLocation(), + context.getIdAllocator().getNextId(), + innerJoin.getType(), + aggregationBelowJoin, + innerJoin.getRight(), + innerJoin.getCriteria(), + innerJoin.getOutputVariables(), + innerJoin.getFilter(), + innerJoin.getLeftHashVariable(), + innerJoin.getRightHashVariable(), + innerJoin.getDistributionType(), + innerJoin.getDynamicFilters()); + + AggregationNode newDistinctNode = new AggregationNode( + aggregationNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + newInnerJoin, + aggregationNode.getAggregations(), + aggregationNode.getGroupingSets(), + aggregationNode.getPreGroupedVariables(), + aggregationNode.getStep(), + aggregationNode.getHashVariable(), + aggregationNode.getGroupIdVariable()); + + return Result.ofPlanNode(newDistinctNode); + } + + private boolean canAggregationBePushedDown(AggregationNode aggregationNode, JoinNode joinNode, Context context) + { + if (!context.getLogicalPropertiesProvider().isPresent() || + !((GroupReference) joinNode.getLeft()).getLogicalProperties().isPresent()) { + return false; + } + + if (joinNode.isCrossJoin() || isJoinCardinalityReducing(joinNode, context)) { + return false; + } + + Set groupingVariables = ImmutableSet.copyOf(aggregationNode.getGroupingKeys()); + Set joinLeftInputVariables = ImmutableSet.copyOf(joinNode.getLeft().getOutputVariables()); + + LogicalProperties joinLeftInputLogicalProperties = ((GroupReference) joinNode.getLeft()).getLogicalProperties().get(); + LogicalProperties aggregationNodelogicalProperties = context.getLogicalPropertiesProvider().get().getAggregationProperties(aggregationNode); + + if (!aggregationNodelogicalProperties.canBeHomogenized(joinLeftInputVariables, groupingVariables)) { + return false; + } + + if (joinLeftInputLogicalProperties.isDistinct(joinLeftInputVariables)) { + return false; + } + + return true; + } + + private boolean isJoinCardinalityReducing(JoinNode joinNode, Context context) + { + StatsProvider stats = context.getStatsProvider(); + PlanNodeStatsEstimate joinStats = stats.getStats(joinNode); + PlanNodeStatsEstimate leftStats = stats.getStats(joinNode.getLeft()); + + double inputBytes = leftStats.getOutputSizeInBytes(joinNode.getLeft().getOutputVariables()); + double outputBytes = joinStats.getOutputSizeInBytes(joinNode.getOutputVariables()); + return outputBytes <= inputBytes * getPushAggregationBelowJoinByteReductionThreshold(context.getSession()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java new file mode 100644 index 0000000000000..d78d139f008e5 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java @@ -0,0 +1,180 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.GroupReference; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.ApplyNode; +import com.facebook.presto.sql.planner.plan.AssignUniqueId; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.InPredicate; +import com.facebook.presto.sql.tree.SymbolReference; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.getJoinReorderingStrategy; +import static com.facebook.presto.SystemSessionProperties.isExploitConstraints; +import static com.facebook.presto.SystemSessionProperties.isInPredicatesAsInnerJoinsEnabled; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.matching.Pattern.empty; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; +import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identitiesAsSymbolReferences; +import static com.facebook.presto.sql.planner.plan.Patterns.Apply.correlation; +import static com.facebook.presto.sql.planner.plan.Patterns.applyNode; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.facebook.presto.sql.tree.BooleanLiteral.TRUE_LITERAL; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.Iterables.getOnlyElement; + +/** + * This optimizer looks for InPredicate expressions in ApplyNodes and replaces the nodes with Distinct + Inner Joins. + *

+ * Plan before optimizer: + * 

+ * Filter(a IN b):
+ *   Apply
+ *     - correlation: []  // empty
+ *     - input: some plan A producing symbol a
+ *     - subquery: some plan B producing symbol b
+ *
+ *

+ * Plan after optimizer: + * 

+ * Aggregate (Distinct unique, a):
+ *   InnerJoin (a=b)
+ *     -source AssignUniqueId (plan A) -> producing uniqueId, a
+ *     - plan B producing symbol b
+ *
+ */ +public class TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin + implements Rule +{ + private static final Pattern PATTERN = applyNode() + .with(empty(correlation())); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isExploitConstraints(session) && + getJoinReorderingStrategy(session) == AUTOMATIC && + isInPredicatesAsInnerJoinsEnabled(session); + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) + { + if (applyNode.getMayParticipateInAntiJoin()) { + return Result.empty(); + } + + Assignments subqueryAssignments = applyNode.getSubqueryAssignments(); + if (subqueryAssignments.size() != 1) { + return Result.empty(); + } + + Expression expression = castToExpression(getOnlyElement(subqueryAssignments.getExpressions())); + if (!(expression instanceof InPredicate)) { + return Result.empty(); + } + + InPredicate inPredicate = (InPredicate) expression; + VariableReferenceExpression inPredicateOutputVariable = getOnlyElement(subqueryAssignments.getVariables()); + + PlanNode leftInput = applyNode.getInput(); + // Add unique id column if the set of columns do not form a unique key already + if (!((GroupReference) leftInput).getLogicalProperties().isPresent() || + !((GroupReference) leftInput).getLogicalProperties().get().isDistinct(ImmutableSet.copyOf(leftInput.getOutputVariables()))) { + VariableReferenceExpression uniqueKeyVariable = context.getVariableAllocator().newVariable("unique", BIGINT); + leftInput = new AssignUniqueId( + applyNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + leftInput, + uniqueKeyVariable); + } + + checkArgument(inPredicate.getValue() instanceof SymbolReference, "Unexpected expression: %s", inPredicate.getValue()); + VariableReferenceExpression leftVariableReference = context.getVariableAllocator().toVariableReference(inPredicate.getValue()); + checkArgument(inPredicate.getValueList() instanceof SymbolReference, "Unexpected expression: %s", inPredicate.getValueList()); + VariableReferenceExpression rightVariableReference = context.getVariableAllocator().toVariableReference(inPredicate.getValueList()); + + JoinNode innerJoin = new JoinNode( + applyNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + JoinNode.Type.INNER, + leftInput, + applyNode.getSubquery(), + ImmutableList.of(new EquiJoinClause( + leftVariableReference, + rightVariableReference)), + ImmutableList.builder() + .addAll(leftInput.getOutputVariables()) + .build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + ImmutableMap.of()); + + AggregationNode distinctNode = new AggregationNode( + innerJoin.getSourceLocation(), + context.getIdAllocator().getNextId(), + innerJoin, + ImmutableMap.of(), + singleGroupingSet(ImmutableList.builder() + .addAll(innerJoin.getOutputVariables()) + .build()), + ImmutableList.of(), + SINGLE, + Optional.empty(), + Optional.empty()); + + ImmutableList referencedOutputs = ImmutableList.builder() + .addAll(applyNode.getInput().getOutputVariables()) + .add(inPredicateOutputVariable) + .build(); + + ProjectNode finalProjectNdde = new ProjectNode( + context.getIdAllocator().getNextId(), + distinctNode, + Assignments.builder() + .putAll(identitiesAsSymbolReferences(distinctNode.getOutputVariables())) + .put(inPredicateOutputVariable, castToRowExpression(TRUE_LITERAL)) + .build() + .filter(referencedOutputs)); + + return Result.ofPlanNode(finalProjectNdde); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java index f0179f011af89..6d8bd4986add0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java @@ -895,7 +895,7 @@ public PlanNode visitApply(ApplyNode node, RewriteContext context) Assignments assignments = canonicalize(node.getSubqueryAssignments()); verifySubquerySupported(assignments); - return new ApplyNode(node.getSourceLocation(), node.getId(), source, subquery, assignments, canonicalCorrelation, node.getOriginSubqueryError()); + return new ApplyNode(node.getSourceLocation(), node.getId(), source, subquery, assignments, canonicalCorrelation, node.getOriginSubqueryError(), node.getMayParticipateInAntiJoin()); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java index 534621600b810..5d168fa6c7aeb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java @@ -69,6 +69,12 @@ public class ApplyNode */ private final String originSubqueryError; + /** + * Indicates whether this apply node appears inside a NOT/OR expression + * i.e. anti-join + */ + private final boolean mayParticipateInAntiJoin; + @JsonCreator public ApplyNode( Optional sourceLocation, @@ -77,9 +83,10 @@ public ApplyNode( @JsonProperty("subquery") PlanNode subquery, @JsonProperty("subqueryAssignments") Assignments subqueryAssignments, @JsonProperty("correlation") List correlation, - @JsonProperty("originSubqueryError") String originSubqueryError) + @JsonProperty("originSubqueryError") String originSubqueryError, + @JsonProperty("mayParticipateInAntiJoin") boolean mayParticipateInAntiJoin) { - this(sourceLocation, id, Optional.empty(), input, subquery, subqueryAssignments, correlation, originSubqueryError); + this(sourceLocation, id, Optional.empty(), input, subquery, subqueryAssignments, correlation, originSubqueryError, mayParticipateInAntiJoin); } public ApplyNode( @@ -90,7 +97,8 @@ public ApplyNode( PlanNode subquery, Assignments subqueryAssignments, List correlation, - String originSubqueryError) + String originSubqueryError, + boolean mayParticipateInAntiJoin) { super(sourceLocation, id, statsEquivalentPlanNode); checkArgument(input.getOutputVariables().containsAll(correlation), "Input does not contain symbols from correlation"); @@ -101,6 +109,7 @@ public ApplyNode( this.subqueryAssignments = requireNonNull(subqueryAssignments, "assignments is null"); this.correlation = ImmutableList.copyOf(requireNonNull(correlation, "correlation is null")); this.originSubqueryError = requireNonNull(originSubqueryError, "originSubqueryError is null"); + this.mayParticipateInAntiJoin = mayParticipateInAntiJoin; } @JsonProperty @@ -133,6 +142,12 @@ public String getOriginSubqueryError() return originSubqueryError; } + @JsonProperty + public boolean getMayParticipateInAntiJoin() + { + return mayParticipateInAntiJoin; + } + @Override public List getSources() { @@ -158,12 +173,12 @@ public R accept(InternalPlanVisitor visitor, C context) public PlanNode replaceChildren(List newChildren) { checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); - return new ApplyNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), newChildren.get(0), newChildren.get(1), subqueryAssignments, correlation, originSubqueryError); + return new ApplyNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), newChildren.get(0), newChildren.get(1), subqueryAssignments, correlation, originSubqueryError, mayParticipateInAntiJoin); } @Override public PlanNode assignStatsEquivalentPlanNode(Optional statsEquivalentPlanNode) { - return new ApplyNode(getSourceLocation(), getId(), statsEquivalentPlanNode, input, subquery, subqueryAssignments, correlation, originSubqueryError); + return new ApplyNode(getSourceLocation(), getId(), statsEquivalentPlanNode, input, subquery, subqueryAssignments, correlation, originSubqueryError, mayParticipateInAntiJoin); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 16a715fd8e721..5256468ac6e35 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -212,7 +212,9 @@ public void testDefaults() .setNativeExecutionExecutablePath("./presto_server") .setRandomizeOuterJoinNullKeyEnabled(false) .setOptimizeConditionalAggregationEnabled(false) - .setRemoveRedundantDistinctAggregationEnabled(true)); + .setRemoveRedundantDistinctAggregationEnabled(true) + .setInPredicatesAsInnerJoinsEnabled(false) + .setPushAggregationBelowJoinByteReductionThreshold(1)); } @Test @@ -375,6 +377,8 @@ public void testExplicitPropertyMappings() .put("optimizer.randomize-outer-join-null-key", "true") .put("optimizer.optimize-conditional-aggregation-enabled", "true") .put("optimizer.remove-redundant-distinct-aggregation-enabled", "false") + .put("optimizer.in-predicates-as-inner-joins-enabled", "true") + .put("optimizer.push-aggregation-below-join-byte-reduction-threshold", "0.9") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -534,7 +538,9 @@ public void testExplicitPropertyMappings() .setNativeExecutionExecutablePath("/bin/echo") .setRandomizeOuterJoinNullKeyEnabled(true) .setOptimizeConditionalAggregationEnabled(true) - .setRemoveRedundantDistinctAggregationEnabled(false); + .setRemoveRedundantDistinctAggregationEnabled(false) + .setInPredicatesAsInnerJoinsEnabled(true) + .setPushAggregationBelowJoinByteReductionThreshold(0.9); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEarlyOutJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEarlyOutJoins.java new file mode 100644 index 0000000000000..266fdf92a7e00 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEarlyOutJoins.java @@ -0,0 +1,343 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.Session; +import com.facebook.presto.sql.planner.assertions.BasePlanTest; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.SystemSessionProperties.EXPLOIT_CONSTRAINTS; +import static com.facebook.presto.SystemSessionProperties.IN_PREDICATES_AS_INNER_JOINS_ENABLED; +import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static com.facebook.presto.SystemSessionProperties.PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; + +public class TestEarlyOutJoins + extends BasePlanTest +{ + private ImmutableMap nationColumns = ImmutableMap.builder() + .put("regionkey", "regionkey") + .put("nationkey", "nationkey") + .put("name", "name") + .put("comment", "comment") + .build(); + + private ImmutableMap orderColumns = ImmutableMap.builder() + .put("orderpriority", "orderpriority") + .put("orderstatus", "orderstatus") + .put("totalprice", "totalprice") + .put("orderkey", "orderkey") + .put("custkey", "custkey") + .put("orderdate", "orderdate") + .put("comment", "comment") + .put("shippriority", "shippriority") + .put("clerk", "clerk") + .build(); + + public TestEarlyOutJoins() + { + super(ImmutableMap.of(EXPLOIT_CONSTRAINTS, Boolean.toString(true), + IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.toString(true), + JOIN_REORDERING_STRATEGY, AUTOMATIC.name())); + } + + @Test + public void testDistinctInnerRewrite() + { + // Rewrite to distinct + inner join and then reorder the join (orders >> nation) + + String query = "select * from nation where nationkey in (select custkey from orders)"; + assertPlan(query, + output( + project( + aggregation(ImmutableMap.of(), + anyTree( + join(INNER, + ImmutableList.of(equiJoinClause("custkey", "nationkey")), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))), + anyTree( + assignUniqueId( + "unique", + tableScan("nation", nationColumns))))))))); + } + + @Test + public void testDistinctInnerToLeftEarlyOutRewrite() + { + // Rewrite inner join to semi join + String query = "select distinct o.custkey, o.totalprice from orders o, nation n where o.custkey = n.nationkey"; + assertPlan(query, + output( + anyTree( + semiJoin("custkey", "nationkey", "semijoinvariable$eoj", + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey", "totalprice", "totalprice"))), + anyTree( + tableScan("nation", ImmutableMap.of("nationkey", "nationkey"))))))); + + // Negative test - too many columns read from nation + query = "select distinct o.custkey, o.totalprice, n.nationkey, n.name from orders o, nation n where o.custkey = n.nationkey"; + assertPlan(query, + output( + anyTree( + join(INNER, + ImmutableList.of(equiJoinClause("custkey", "nationkey")), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey", "totalprice", "totalprice"))), + anyTree( + tableScan("nation", ImmutableMap.of("nationkey", "nationkey", "name", "name"))))))); + + // Trasnform to distinct + inner join and then transform back to semi join + // The join inputs were not reordered and the join was cardinality reducing + query = "select * from orders where custkey in (select custkey from customer where name = 'Customer#000156251')"; + assertPlan(query, + output( + project( + anyTree( + semiJoin("custkey", "custkey_1", "semijoinvariable$eoj", + anyTree( + tableScan("orders", orderColumns)), + anyTree( + tableScan("customer", ImmutableMap.of("custkey_1", "custkey", "name", "name")))))))); + } + + @Test + public void testDistinctInnerToRightEarlyOutRewrite() + { + // Rewrite to distinct + inner join and then push aggregation below the left input + // The join inputs were reordered and the join was not cardinality reducing + String query = "select orderkey from orders where orderkey in (select orderkey from lineitem)"; + assertPlan(query, + output( + join(INNER, + ImmutableList.of(equiJoinClause("orderkey_1", "orderkey")), + project( + aggregation(ImmutableMap.of(), + tableScan("lineitem", ImmutableMap.of("orderkey_1", "orderkey")))), + anyTree( + tableScan("orders", ImmutableMap.of("orderkey", "orderkey")))))); + + Session sessionWithIncreasedByteReductionThreshold = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty(PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD, Double.toString(2)) + .build(); + + // Same as previous query, except the aggregation is not pushed below the join since the join is not + // considered cardinality reducing due to the altered value for parameter EARLY_OUT_JOIN_TRANSFORMATION_BYTE_REDUCTION_THRESHOLD + assertPlanWithSession(query, + sessionWithIncreasedByteReductionThreshold, + false, + output( + anyTree( + aggregation(ImmutableMap.of(), + anyTree( + join(INNER, + ImmutableList.of(equiJoinClause("orderkey_1", "orderkey")), + anyTree( + tableScan("lineitem", ImmutableMap.of("orderkey_1", "orderkey"))), + anyTree( + assignUniqueId("unique", + tableScan("orders", ImmutableMap.of("orderkey", "orderkey")))))))))); + + // Aggregation pushed down the left input of the join, but not the right since the output contains o.custkey (not a join key in the output) + query = "select distinct l.orderkey, l.partkey, o.custkey from lineitem l, orders o where l.orderkey = o.orderkey"; + assertPlan(query, + output( + anyTree( + join(INNER, + ImmutableList.of(equiJoinClause("orderkey", "orderkey_0")), + anyTree( + aggregation(ImmutableMap.of(), + anyTree( + tableScan("lineitem", ImmutableMap.of("partkey", "partkey", "orderkey", "orderkey"))))), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey", "orderkey_0", "orderkey"))))))); + } + + @Test + public void testAntiJoinScenarios() + { + // No NOT's or OR's or CASE's + + String query = "select * from nation where nationkey not in (select custkey from orders)"; + assertPlan(query, + output( + anyTree( + semiJoin("nationkey", "custkey", "expr_9", + anyTree( + tableScan("nation", nationColumns)), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))))))); + + query = "select * from nation where nationkey in (select custkey from orders) or nationkey in (select orderkey from lineitem)"; + assertPlan(query, + output( + anyTree( + semiJoin("nationkey", "orderkey_11", "expr_21", + semiJoin("nationkey", "custkey", "expr_9", + anyTree( + tableScan("nation", nationColumns)), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey")))), + anyTree( + tableScan("lineitem", ImmutableMap.of("orderkey_11", "orderkey"))))))); + + query = "select * from nation where not (nationkey = any (select custkey from orders))"; + assertPlan(query, + output( + anyTree( + semiJoin("nationkey", "custkey", "expr_9", + anyTree( + tableScan("nation", nationColumns)), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))))))); + + query = "select case when (nationkey in (select custkey from orders)) then 1 else 2 end from nation"; + assertPlan(query, + output( + anyTree( + semiJoin("nationkey", "custkey", "expr_9", + anyTree( + tableScan("nation", ImmutableMap.of("nationkey", "nationkey"))), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))))))); + } + + @Test + void testCombinationsOfSubqueries() + { + // Rewrite to distinct + inner join for both subqueries and then find optimal order for the 3-way join + String query = "select * from nation where nationkey in (select custkey from orders) and nationkey in (select orderkey from lineitem)"; + assertPlan(query, + output( + anyTree( + aggregation(ImmutableMap.of(), + anyTree( + join(INNER, + ImmutableList.of(equiJoinClause("orderkey_9", "nationkey")), + anyTree( + tableScan("lineitem", ImmutableMap.of("orderkey_9", "orderkey"))), + anyTree( + aggregation(ImmutableMap.of(), + project( + join(INNER, + ImmutableList.of(equiJoinClause("custkey", "nationkey")), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))), + anyTree(assignUniqueId("unique_34", + tableScan("nation", nationColumns))))))))))))); + + // Disjoint conjunctions also transformed similarly + query = "select * from nation where nationkey in (select custkey from orders) and regionkey in (select orderkey from lineitem)"; + assertPlan(query, + output( + anyTree( + aggregation(ImmutableMap.of(), + project( + join(INNER, + ImmutableList.of(equiJoinClause("orderkey_9", "regionkey")), + anyTree( + tableScan("lineitem", ImmutableMap.of("orderkey_9", "orderkey"))), + assignUniqueId("unique", + project( + aggregation(ImmutableMap.of(), + anyTree( + join(INNER, + ImmutableList.of(equiJoinClause("custkey", "nationkey")), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))), + anyTree( + assignUniqueId("unique_34", + tableScan("nation", nationColumns)))))))))))))); + } + + @Test + void testComplexQueries() + { + // Subquery produces distinct output -> is rewritten to inner join, inputs are reordered and extraneous aggregations are removed + String query = "select * from nation where nationkey in (select custkey from orders group by custkey)"; + assertPlan(query, + output( + join(INNER, + ImmutableList.of(equiJoinClause("custkey", "nationkey")), + anyTree( + aggregation(ImmutableMap.of(), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))))), + anyTree( + tableScan("nation", nationColumns))))); + + // In predicates in the having clause + query = "select nationkey, name from nation having nationkey in (select custkey from orders)"; + assertPlan(query, + output( + project( + aggregation(ImmutableMap.of(), + anyTree( + join(INNER, + ImmutableList.of(equiJoinClause("custkey", "nationkey")), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))), + anyTree( + assignUniqueId("unique", + tableScan("nation", ImmutableMap.of("nationkey", "nationkey", "name", "name")))))))))); + + query = "select nationkey, name from nation having nationkey in (select custkey from orders) and nationkey in (select orderkey from lineitem)"; + assertPlan(query, + output( + project( + aggregation(ImmutableMap.of(), + project( + join(INNER, + ImmutableList.of(equiJoinClause("orderkey_9", "nationkey")), + anyTree( + tableScan("lineitem", ImmutableMap.of("orderkey_9", "orderkey"))), + assignUniqueId("unique", + anyTree( + aggregation(ImmutableMap.of(), + project( + join(INNER, + ImmutableList.of(equiJoinClause("custkey", "nationkey")), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey"))), + anyTree( + assignUniqueId("unique_26", + tableScan("nation", ImmutableMap.of("nationkey", "nationkey", "name", "name"))))))))))))))); + + query = "select nationkey, name from nation having nationkey in (select custkey from orders) or nationkey in (select orderkey from lineitem)"; + assertPlan(query, + output( + anyTree( + semiJoin("nationkey", "orderkey_9", "expr_17", + semiJoin("nationkey", "custkey", "expr_7", + anyTree( + tableScan("nation", ImmutableMap.of("name", "name", "nationkey", "nationkey"))), + anyTree( + tableScan("orders", ImmutableMap.of("custkey", "custkey")))), + anyTree( + tableScan("lineitem", ImmutableMap.of("orderkey_9", "orderkey"))))))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java index 8506739bba4af..87c1280a64674 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java @@ -29,6 +29,7 @@ import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantIdentityProjections; import com.facebook.presto.sql.planner.iterative.rule.SimplifyRowExpressions; +import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; @@ -144,7 +145,7 @@ private List getMinimalOptimizers() new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), - ImmutableSet.of(new TransformUncorrelatedInPredicateSubqueryToSemiJoin(), new RemoveRedundantIdentityProjections())), + ImmutableSet.of(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin(), new TransformUncorrelatedInPredicateSubqueryToSemiJoin(), new RemoveRedundantIdentityProjections())), getExpressionTranslator(), new IterativeOptimizer( new RuleStatsRecorder(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java index b794a4ac237c1..c1163c8e94607 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java @@ -23,6 +23,7 @@ import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.LogicalPropertiesProvider; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -182,6 +183,12 @@ public WarningCollector getWarningCollector() { return WarningCollector.NOOP; } + + @Override + public Optional getLogicalPropertiesProvider() + { + return Optional.empty(); + } }; } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java index 5f34f5219ae35..6a5b04b631035 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java @@ -1655,14 +1655,16 @@ public void testAggregationNodeLogicalProperties() .matches(expectedLogicalProperties); // Test propagation of equivalence classes through aggregation. - // None of the equivalence classes from aggregation's source node should be propagated since none of the - // members are projected by the aggregation node. // Key property (shippriority, linenumber) which form the group by keys and maxcard=6 should be propagated. + EquivalenceClassProperty equivalenceClassProperty2 = new EquivalenceClassProperty(ImmutableMap.of(customerCustKeyVariable, ordersOrderKeyVariable), + ImmutableMap.of(customerCustKeyVariable, ImmutableList.of(ordersCustKeyVariable), ordersOrderKeyVariable, ImmutableList.of(lineitemOrderkeyVariable))); expectedLogicalProperties = new LogicalPropertiesImpl( - new EquivalenceClassProperty(), + equivalenceClassProperty2, new MaxCardProperty(), - new KeyProperty(ImmutableSet.of(new Key(ImmutableSet.of(shipPriorityVariable, lineitemLinenumberVariable))))); + new KeyProperty(ImmutableSet.of( + new Key(ImmutableSet.of(shipPriorityVariable, lineitemLinenumberVariable)), + new Key(ImmutableSet.of(ordersOrderKeyVariable, lineitemLinenumberVariable))))); tester().assertThat(new NoOpRule(), logicalPropertiesProvider) .on(p -> { @@ -1712,7 +1714,7 @@ public void testAggregationNodeLogicalProperties() // A variation to the above case, where in groupby keys are (l_lineitem,o_orderkey,shippriority). Since // (o_orderkey, l_lineitem) are already a key, the key should be normalized to have only (o_orderkey, l_lineitem). expectedLogicalProperties = new LogicalPropertiesImpl( - new EquivalenceClassProperty(), + equivalenceClassProperty2, new MaxCardProperty(), new KeyProperty(ImmutableSet.of(new Key(ImmutableSet.of(ordersOrderKeyVariable, lineitemLinenumberVariable))))); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToLeftEarlyOutJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToLeftEarlyOutJoin.java new file mode 100644 index 0000000000000..6c42da8a1423e --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToLeftEarlyOutJoin.java @@ -0,0 +1,174 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.cost.VariableStatsEstimate; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.properties.LogicalPropertiesProviderImpl; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; +import java.util.function.Function; + +import static com.facebook.presto.SystemSessionProperties.EXPLOIT_CONSTRAINTS; +import static com.facebook.presto.SystemSessionProperties.IN_PREDICATES_AS_INNER_JOINS_ENABLED; +import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.sql.relational.Expressions.variable; +import static java.util.Collections.emptyList; + +public class TestTransformDistinctInnerJoinToLeftEarlyOutJoin + extends BaseRuleTest +{ + @BeforeClass + public final void setUp() + { + tester = new RuleTester(emptyList(), + ImmutableMap.of(IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.toString(true), + EXPLOIT_CONSTRAINTS, Boolean.toString(true), + JOIN_REORDERING_STRATEGY, AUTOMATIC.name())); + } + + @Test + public void testAggregationPushedDown() + { + tester().assertThat(new TransformDistinctInnerJoinToLeftEarlyOutJoin(), new LogicalPropertiesProviderImpl(new FunctionResolution(getFunctionManager()))) + .on(p -> { + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression b = p.variable("b", BIGINT); + VariableReferenceExpression unique = p.variable("unique", BIGINT); + return p.aggregation(agg -> agg + .step(SINGLE) + .singleGroupingSet(unique, a) + .source(p.join( + INNER, + p.assignUniqueId(unique, + p.values(new PlanNodeId("valuesA"), 1000, a)), + p.values(new PlanNodeId("valuesB"), 100, b), + ImmutableList.of(new JoinNode.EquiJoinClause(a, b)), + ImmutableList.of(unique, a), + Optional.empty()))); + }) + .overrideStats("valuesA", PlanNodeStatsEstimate.builder() + .setOutputRowCount(1000) + .setConfident(true) + .addVariableStatistics(variable("a", BIGINT), new VariableStatsEstimate(0, 1000, 0, 8, 100)) + .build()) + .overrideStats("valuesB", PlanNodeStatsEstimate.builder() + .setOutputRowCount(100) + .setConfident(true) + .addVariableStatistics(variable("b", BIGINT), new VariableStatsEstimate(0, 1000, 0, 8, 10)) + .build()) + .matches(aggregation(ImmutableMap.of(), + SINGLE, + project( + filter("semijoinvariable", + semiJoin("a", + "b", + "semijoinvariable", + assignUniqueId("unique", + values("a")), + values("b")))))); + + // Negative test + // Join output contains columns from B that are not part of the join key + tester().assertThat(new TransformDistinctInnerJoinToLeftEarlyOutJoin(), new LogicalPropertiesProviderImpl(new FunctionResolution(getFunctionManager()))) + .on(p -> { + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression b = p.variable("b", BIGINT); + VariableReferenceExpression c = p.variable("c", BIGINT); + VariableReferenceExpression unique = p.variable("unique", BIGINT); + return p.aggregation(agg -> agg + .step(SINGLE) + .singleGroupingSet(unique, a, c) + .source(p.join( + INNER, + p.assignUniqueId(unique, + p.values(new PlanNodeId("valuesA"), 1000, a)), + p.values(new PlanNodeId("valuesBC"), 100, b, c), + ImmutableList.of(new JoinNode.EquiJoinClause(a, b)), + ImmutableList.of(unique, a, c), + Optional.empty()))); + }) + .overrideStats("valuesA", PlanNodeStatsEstimate.builder() + .setOutputRowCount(1000) + .setConfident(true) + .addVariableStatistics(variable("a", BIGINT), new VariableStatsEstimate(0, 1000, 0, 8, 100)) + .build()) + .overrideStats("valuesB", PlanNodeStatsEstimate.builder() + .setOutputRowCount(100) + .setConfident(true) + .addVariableStatistics(variable("b", BIGINT), new VariableStatsEstimate(0, 1000, 0, 8, 10)) + .addVariableStatistics(variable("c", BIGINT), new VariableStatsEstimate(0, 1000, 0, 8, 10)) + .build()) + .doesNotFire(); + } + + @Test + public void testFeatureDisabled() + { + Function planProvider = p -> { + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression b = p.variable("b", BIGINT); + VariableReferenceExpression unique = p.variable("unique", BIGINT); + return p.project( + assignment(a, a), + p.aggregation(agg -> agg + .step(SINGLE) + .singleGroupingSet(unique, a) + .source(p.join( + INNER, + p.values(new PlanNodeId("valuesB"), b), + p.assignUniqueId(unique, + p.values(new PlanNodeId("valuesA"), a)), + new JoinNode.EquiJoinClause(b, a))))); + }; + + tester().assertThat(new TransformDistinctInnerJoinToLeftEarlyOutJoin()) + .setSystemProperty(IN_PREDICATES_AS_INNER_JOINS_ENABLED, "false") + .on(planProvider) + .doesNotFire(); + + tester().assertThat(new TransformDistinctInnerJoinToLeftEarlyOutJoin()) + .setSystemProperty(EXPLOIT_CONSTRAINTS, "false") + .on(planProvider) + .doesNotFire(); + + tester().assertThat(new TransformDistinctInnerJoinToLeftEarlyOutJoin()) + .setSystemProperty(JOIN_REORDERING_STRATEGY, "NONE") + .on(planProvider) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToRightEarlyOutJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToRightEarlyOutJoin.java new file mode 100644 index 0000000000000..6e7ae2de4ce2b --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToRightEarlyOutJoin.java @@ -0,0 +1,136 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.cost.VariableStatsEstimate; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.properties.LogicalPropertiesProviderImpl; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.function.Function; + +import static com.facebook.presto.SystemSessionProperties.EXPLOIT_CONSTRAINTS; +import static com.facebook.presto.SystemSessionProperties.IN_PREDICATES_AS_INNER_JOINS_ENABLED; +import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.sql.relational.Expressions.variable; +import static java.util.Collections.emptyList; + +public class TestTransformDistinctInnerJoinToRightEarlyOutJoin + extends BaseRuleTest +{ + @BeforeClass + public final void setUp() + { + tester = new RuleTester(emptyList(), + ImmutableMap.of(IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.toString(true), + EXPLOIT_CONSTRAINTS, Boolean.toString(true), + JOIN_REORDERING_STRATEGY, AUTOMATIC.name())); + } + + @Test + public void testAggregationPushedDown() + { + tester().assertThat(new TransformDistinctInnerJoinToRightEarlyOutJoin(), new LogicalPropertiesProviderImpl(new FunctionResolution(getFunctionManager()))) + .on(p -> { + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression b = p.variable("b", BIGINT); + VariableReferenceExpression unique = p.variable("unique", BIGINT); + return p.aggregation(agg -> agg + .step(SINGLE) + .singleGroupingSet(b) + .source(p.join( + INNER, + p.values(new PlanNodeId("valuesB"), 100, b), + p.assignUniqueId(unique, + p.values(new PlanNodeId("valuesA"), 1000, a)), + new JoinNode.EquiJoinClause(b, a)))); + }) + .overrideStats("valuesA", PlanNodeStatsEstimate.builder() + .setOutputRowCount(1000) + .setConfident(true) + .addVariableStatistics(variable("a", BIGINT), new VariableStatsEstimate(0, 1000, 0, 8, 100)) + .build()) + .overrideStats("valuesB", PlanNodeStatsEstimate.builder() + .setOutputRowCount(100) + .setConfident(true) + .addVariableStatistics(variable("b", BIGINT), new VariableStatsEstimate(0, 1000, 0, 8, 10)) + .build()) + .matches(aggregation(ImmutableMap.of(), + SINGLE, + join(INNER, + ImmutableList.of(equiJoinClause("b", "a")), + aggregation(ImmutableMap.of(), + SINGLE, + values("b")), + assignUniqueId("unique", + values("a"))))); + } + + @Test + public void testFeatureDisabled() + { + Function planProvider = p -> { + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression b = p.variable("b", BIGINT); + VariableReferenceExpression unique = p.variable("unique", BIGINT); + return p.project( + assignment(a, a), + p.aggregation(agg -> agg + .step(SINGLE) + .singleGroupingSet(unique, a) + .source(p.join( + INNER, + p.values(new PlanNodeId("valuesB"), b), + p.assignUniqueId(unique, + p.values(new PlanNodeId("valuesA"), a)), + new JoinNode.EquiJoinClause(b, a))))); + }; + + tester().assertThat(new TransformDistinctInnerJoinToRightEarlyOutJoin()) + .setSystemProperty(IN_PREDICATES_AS_INNER_JOINS_ENABLED, "false") + .on(planProvider) + .doesNotFire(); + + tester().assertThat(new TransformDistinctInnerJoinToRightEarlyOutJoin()) + .setSystemProperty(EXPLOIT_CONSTRAINTS, "false") + .on(planProvider) + .doesNotFire(); + + tester().assertThat(new TransformDistinctInnerJoinToRightEarlyOutJoin()) + .setSystemProperty(JOIN_REORDERING_STRATEGY, "NONE") + .on(planProvider) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java new file mode 100644 index 0000000000000..32448070209da --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java @@ -0,0 +1,207 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.sql.planner.iterative.properties.LogicalPropertiesProviderImpl; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.tree.ExistsPredicate; +import com.facebook.presto.sql.tree.InPredicate; +import com.facebook.presto.sql.tree.LongLiteral; +import com.facebook.presto.sql.tree.SymbolReference; +import com.facebook.presto.tpch.TpchConnectorFactory; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.EXPLOIT_CONSTRAINTS; +import static com.facebook.presto.SystemSessionProperties.IN_PREDICATES_AS_INNER_JOINS_ENABLED; +import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static java.util.Collections.emptyList; + +public class TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin + extends BaseRuleTest +{ + @BeforeClass + public final void setUp() + { + tester = new RuleTester(emptyList(), + ImmutableMap.of(IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.toString(true), + EXPLOIT_CONSTRAINTS, Boolean.toString(true), + JOIN_REORDERING_STRATEGY, AUTOMATIC.name()), + Optional.of(1), + new TpchConnectorFactory(1)); + } + + @Test + public void testDoesNotFireOnCorrelation() + { + tester().assertThat(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()) + .on(p -> p.apply( + assignment( + p.variable("x"), + new InPredicate( + new SymbolReference("y"), + new SymbolReference("z"))), + ImmutableList.of(p.variable("y")), + p.values(p.variable("y")), + p.values())) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnNonInPredicateSubquery() + { + tester().assertThat(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()) + .on(p -> p.apply( + assignment(p.variable("x"), new ExistsPredicate(new LongLiteral("1"))), + emptyList(), + p.values(), + p.values())) + .doesNotFire(); + } + + @Test + public void testFiresForInPredicate() + { + tester().assertThat(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()) + .on(p -> p.apply( + assignment( + p.variable("x"), + new InPredicate( + new SymbolReference("y"), + new SymbolReference("z"))), + emptyList(), + p.values(p.variable("y")), + p.values(p.variable("z")))) + .matches(project( + aggregation(ImmutableMap.of(), + SINGLE, + join(INNER, + ImmutableList.of(equiJoinClause("y", "z")), + assignUniqueId("unique", values("y")), + values("z"))))); + } + + @Test + public void testDoesNotFiresForInPredicateThatMayParticipateInAntiJoin() + { + tester().assertThat(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()) + .on(p -> p.apply( + assignment( + p.variable("x"), + new InPredicate( + new SymbolReference("y"), + new SymbolReference("z"))), + emptyList(), + p.values(p.variable("y")), + p.values(p.variable("z")), + true)) + .doesNotFire(); + } + + @Test + public void testSimpleSemijoins() + { + tester().assertThat(ImmutableSet.of(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()), new LogicalPropertiesProviderImpl(new FunctionResolution(getFunctionManager()))) + .on("SELECT * FROM nation WHERE regionkey IN (SELECT regionkey FROM region)") + .matches(output(anyTree( + aggregation( + ImmutableMap.of(), + join(INNER, + ImmutableList.of(equiJoinClause("regionkey", "regionkey_1")), + assignUniqueId("unique", + tableScan("nation", ImmutableMap.of("regionkey", "regionkey", "nationkey", "nationkey", "name", "name", "comment", "comment"))), + tableScan("region", ImmutableMap.of("regionkey_1", "regionkey"))))))); + + tester().assertThat(ImmutableSet.of(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()), new LogicalPropertiesProviderImpl(new FunctionResolution(getFunctionManager()))) + .on("SELECT * FROM nation WHERE regionkey IN (SELECT regionkey FROM region) AND name IN (SELECT name FROM region)") + .matches(output(anyTree( + aggregation( + ImmutableMap.of(), + join(INNER, + ImmutableList.of(equiJoinClause("name", "name_12")), + assignUniqueId("unique", + (anyTree( + aggregation( + ImmutableMap.of(), + join(INNER, + ImmutableList.of(equiJoinClause("regionkey", "regionkey_1")), + assignUniqueId("unique_37", + tableScan("nation", ImmutableMap.of("regionkey", "regionkey", "nationkey", "nationkey", "name", "name", "comment", "comment"))), + tableScan("region", ImmutableMap.of("regionkey_1", "regionkey"))))))), + tableScan("region", ImmutableMap.of("name_12", "name"))))))); + } + + @Test + public void testFeatureDisabled() + { + tester().assertThat(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()) + .setSystemProperty(IN_PREDICATES_AS_INNER_JOINS_ENABLED, "false") + .on(p -> p.apply( + assignment( + p.variable("x"), + new InPredicate( + new SymbolReference("y"), + new SymbolReference("z"))), + emptyList(), + p.values(p.variable("y")), + p.values(p.variable("z")))) + .doesNotFire(); + + tester().assertThat(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()) + .setSystemProperty(EXPLOIT_CONSTRAINTS, "false") + .on(p -> p.apply( + assignment( + p.variable("x"), + new InPredicate( + new SymbolReference("y"), + new SymbolReference("z"))), + emptyList(), + p.values(p.variable("y")), + p.values(p.variable("z")))) + .doesNotFire(); + + tester().assertThat(new TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin()) + .setSystemProperty(JOIN_REORDERING_STRATEGY, "NONE") + .on(p -> p.apply( + assignment( + p.variable("x"), + new InPredicate( + new SymbolReference("y"), + new SymbolReference("z"))), + emptyList(), + p.values(p.variable("y")), + p.values(p.variable("z")))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index 0de8649256e22..a2676dd8fff7a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -493,9 +493,14 @@ protected AggregationNode build() } public ApplyNode apply(Assignments subqueryAssignments, List correlation, PlanNode input, PlanNode subquery) + { + return apply(subqueryAssignments, correlation, input, subquery, false); + } + + public ApplyNode apply(Assignments subqueryAssignments, List correlation, PlanNode input, PlanNode subquery, boolean mayParticipateInAntiJoin) { verifySubquerySupported(subqueryAssignments); - return new ApplyNode(subquery.getSourceLocation(), idAllocator.getNextId(), input, subquery, subqueryAssignments, correlation, ""); + return new ApplyNode(subquery.getSourceLocation(), idAllocator.getNextId(), input, subquery, subqueryAssignments, correlation, "", mayParticipateInAntiJoin); } public AssignUniqueId assignUniqueId(VariableReferenceExpression variable, PlanNode source) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java index 029876dfa6a98..7978c3a21a122 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java @@ -43,7 +43,9 @@ import com.facebook.presto.sql.planner.iterative.PlanNodeMatcher; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.iterative.properties.LogicalPropertiesImpl; +import com.facebook.presto.sql.planner.iterative.properties.LogicalPropertiesProviderImpl; import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableSet; @@ -265,6 +267,7 @@ private Rule.Context ruleContext(StatsCalculator statsCalculator, CostCalculator { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, Optional.of(memo), lookup, session, variableAllocator.getTypes()); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.of(memo), session); + LogicalPropertiesProvider logicalPropertiesProvider = new LogicalPropertiesProviderImpl(new FunctionResolution(metadata.getFunctionAndTypeManager())); return new Rule.Context() { @@ -312,6 +315,12 @@ public WarningCollector getWarningCollector() { return WarningCollector.NOOP; } + + @Override + public Optional getLogicalPropertiesProvider() + { + return Optional.of(logicalPropertiesProvider); + } }; } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java b/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java index 876104c484f7f..41d06b76bdc9a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java @@ -39,7 +39,7 @@ class QueryAssertions implements Closeable { - private final QueryRunner runner; + protected QueryRunner runner; public QueryAssertions() { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java index 86fbfd0eb326c..e7508e716af0e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java @@ -48,12 +48,14 @@ public class TestSubqueries { private static final String UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG = "line .*: Given correlated subquery is not supported"; - private QueryAssertions assertions; + protected QueryAssertions assertions; + protected QueryAssertions tpchAssertions; @BeforeClass public void init() { assertions = new QueryAssertions(); + tpchAssertions = new TpchQueryAssertions(ImmutableMap.of()); } @AfterClass(alwaysRun = true) @@ -242,6 +244,54 @@ public void testCorrelatedSubqueryWithExplicitCoercion() "VALUES 1"); } + @Test + public void testEarlyOutJoins() + { + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM nation WHERE nationkey IN (SELECT custkey FROM orders)", + "VALUES BIGINT '16'"); + + tpchAssertions.assertQuery( + "SELECT COUNT (DISTINCT o.custkey) FROM orders o, nation n WHERE o.custkey = n.nationkey", + "VALUES BIGINT '16'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM orders WHERE custkey IN (SELECT custkey FROM customer WHERE name = 'unknown')", + "VALUES BIGINT '0'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM (SELECT orderkey FROM orders WHERE orderkey IN (SELECT orderkey FROM lineitem))", + "VALUES BIGINT '15000'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM (SELECT DISTINCT l.orderkey, l.partkey, o.custkey FROM lineitem l, orders o WHERE l.orderkey = o.orderkey)", + "VALUES BIGINT '60113'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM nation WHERE nationkey IN (SELECT custkey FROM orders) AND nationkey IN (SELECT orderkey FROM lineitem)", + "VALUES BIGINT '5'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM nation WHERE nationkey IN (SELECT custkey FROM orders) AND regionkey IN (SELECT orderkey FROM lineitem)", + "VALUES BIGINT '13'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM nation WHERE nationkey IN (SELECT custkey FROM orders GROUP BY custkey)", + "VALUES BIGINT '16'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM (SELECT nationkey, name FROM nation HAVING nationkey IN (SELECT custkey FROM orders))", + "VALUES BIGINT '16'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM (SELECT nationkey, name FROM nation HAVING nationkey IN (SELECT custkey FROM orders) AND nationkey IN (SELECT orderkey FROM lineitem))", + "VALUES BIGINT '5'"); + + tpchAssertions.assertQuery( + "SELECT COUNT(*) FROM (SELECT nationkey, name FROM nation HAVING nationkey IN (SELECT custkey FROM orders) OR nationkey IN (SELECT orderkey FROM lineitem))", + "VALUES BIGINT '18'"); + } + private void assertExistsRewrittenToAggregationBelowJoin(@Language("SQL") String actual, @Language("SQL") String expected, boolean extraAggregation) { PlanMatchPattern source = node(ValuesNode.class); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueriesWithEarlyOutJoinTransformationsEnabled.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueriesWithEarlyOutJoinTransformationsEnabled.java new file mode 100644 index 0000000000000..9e1277f726e19 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueriesWithEarlyOutJoinTransformationsEnabled.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.query; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; + +import java.util.Map; + +import static com.facebook.presto.SystemSessionProperties.EXPLOIT_CONSTRAINTS; +import static com.facebook.presto.SystemSessionProperties.IN_PREDICATES_AS_INNER_JOINS_ENABLED; + +public class TestSubqueriesWithEarlyOutJoinTransformationsEnabled + extends TestSubqueries +{ + private static final Map sessionProperties = ImmutableMap.of(EXPLOIT_CONSTRAINTS, Boolean.toString(true), IN_PREDICATES_AS_INNER_JOINS_ENABLED, Boolean.toString(true)); + + @Override + @BeforeClass + public void init() + { + assertions = new QueryAssertions(sessionProperties); + tpchAssertions = new TpchQueryAssertions(sessionProperties); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TpchQueryAssertions.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TpchQueryAssertions.java new file mode 100644 index 0000000000000..5fd231f7042bb --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TpchQueryAssertions.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.query; + +import com.facebook.presto.Session; +import com.facebook.presto.testing.LocalQueryRunner; +import com.facebook.presto.tpch.TpchConnectorFactory; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; + +class TpchQueryAssertions + extends QueryAssertions +{ + public TpchQueryAssertions(Map systemProperties) + { + Session.SessionBuilder builder = testSessionBuilder() + .setCatalog("tpch") + .setSchema(TINY_SCHEMA_NAME); + systemProperties.forEach(builder::setSystemProperty); + runner = new LocalQueryRunner(builder.build()); + ((LocalQueryRunner) runner).createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of()); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/LogicalProperties.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/LogicalProperties.java index a7e060c7c6ede..53f47873a64fd 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/plan/LogicalProperties.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/LogicalProperties.java @@ -53,4 +53,14 @@ public interface LogicalProperties * @return True if there is provably at most one tuple or false otherwise. */ boolean isAtMost(long n); + + /** + * Determines whether one set of expressions (expressions) can be realized/rewritten + * in terms of the other (targetVariables) using EquivalenceClasses + * + * @param expressions + * @param targetVariables + * @return True if all expressions can be realized in terms of targetVariables + */ + boolean canBeHomogenized(Set expressions, Set targetVariables); }