diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 1afbc272f89b3..84b0d7030f31a 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -364,6 +364,7 @@ public final class SystemSessionProperties public static final String OPTIMIZER_USE_HISTOGRAMS = "optimizer_use_histograms"; public static final String WARN_ON_COMMON_NAN_PATTERNS = "warn_on_common_nan_patterns"; public static final String INLINE_PROJECTIONS_ON_VALUES = "inline_projections_on_values"; + public static final String TRANSFORM_IN_VALUES_TO_IN_FILTER = "transform_in_values_to_in_filter"; private final List> sessionProperties; @@ -2038,6 +2039,10 @@ public SystemSessionProperties( booleanProperty(INLINE_PROJECTIONS_ON_VALUES, "Whether to evaluate project node on values node", featuresConfig.getInlineProjectionsOnValues(), + false), + booleanProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, + "Transform in values to in filter instead of semijoin whenever possible", + featuresConfig.getTransformInValuesToInFilter(), false)); } @@ -3370,4 +3375,9 @@ public static boolean isInlineProjectionsOnValues(Session session) { return session.getSystemProperty(INLINE_PROJECTIONS_ON_VALUES, Boolean.class); } + + public static boolean isTransformInValuesToInFilterEnabled(Session session) + { + return session.getSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, Boolean.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index c88271e97dd55..a4d7dc6b7a033 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -295,6 +295,7 @@ public class FeaturesConfig private boolean useHistograms; private boolean isInlineProjectionsOnValuesEnabled; + private boolean transformInValuesToInFilterEnabled; public enum PartitioningPrecisionStrategy { @@ -2969,4 +2970,17 @@ public FeaturesConfig setInlineProjectionsOnValues(boolean isInlineProjectionsOn this.isInlineProjectionsOnValuesEnabled = isInlineProjectionsOnValuesEnabled; return this; } + + public boolean getTransformInValuesToInFilter() + { + return transformInValuesToInFilterEnabled; + } + + @Config("optimizer.transform_in_values_to_in_filter") + @ConfigDescription("Transform the in values form to in filter instead of semijoin whenever possible") + public FeaturesConfig setTransformInValuesToInFilter(boolean transformInValuesToInFilterEnabled) + { + this.transformInValuesToInFilterEnabled = transformInValuesToInFilterEnabled; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 32184097720da..f27b5551905d7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -140,6 +140,7 @@ import com.facebook.presto.sql.planner.iterative.rule.TransformDistinctInnerJoinToLeftEarlyOutJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformDistinctInnerJoinToRightEarlyOutJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformExistsApplyToLateralNode; +import com.facebook.presto.sql.planner.iterative.rule.TransformInValuesToInFilter; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedLateralToJoin; @@ -485,7 +486,14 @@ public PlanOptimizers( estimatedExchangesCostCalculator, ImmutableSet.>builder() .add(new InlineProjectionsOnValues(metadata.getFunctionAndTypeManager())) - .addAll(new SimplifyRowExpressions(metadata).rules()) + .addAll(new SimplifyRowExpressions(metadata).rules()).build()), + new IterativeOptimizer( + metadata, + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + ImmutableSet.>builder() + .add(new TransformInValuesToInFilter()) .build()), new IterativeOptimizer( metadata, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformInValuesToInFilter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformInValuesToInFilter.java new file mode 100644 index 0000000000000..6dfc074028315 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformInValuesToInFilter.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.google.common.collect.ImmutableList; + +import java.util.Iterator; + +import static com.facebook.presto.SystemSessionProperties.isTransformInValuesToInFilterEnabled; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IN; +import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments; +import static com.facebook.presto.sql.planner.plan.Patterns.SemiJoin.filteringSource; +import static com.facebook.presto.sql.planner.plan.Patterns.semiJoin; +import static com.facebook.presto.sql.planner.plan.Patterns.values; +import static com.facebook.presto.sql.relational.Expressions.specialForm; + +/** + * This optimizer looks for SemiJoinNode whose filteringSource has only one column, then transform the SemijoinNode into ProjectNode with predicate variable for filtering. + *

+ * Plan before optimizer: + * 

+ * SemiJoinNode (semiJoinOutput variable c):
+ *   - source (sourceJoinVariable a)
+ *   - filteringSource (one column variable b)
+ *       - ProjectNode
+ *            - ValuesNode
+ *
+ *

+ * Plan after optimizer: + * 

+ * ProjectNode:
+ *   - source
+ *   - assignments(identityAssignments of source output,c in b)
+ *
+ */ +public class TransformInValuesToInFilter + implements Rule +{ + private static final Pattern PATTERN = semiJoin().with(filteringSource().matching(values())); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isTransformInValuesToInFilterEnabled(session); + } + + @Override + public Result apply(SemiJoinNode semiJoinNode, Captures captures, Context context) + { + PlanNode source = semiJoinNode.getSource(); + PlanNode planNode = context.getLookup().resolveGroup(semiJoinNode.getFilteringSource()).findFirst().get(); + if (!(planNode instanceof ValuesNode)) { + return Result.empty(); + } + ValuesNode valuesNode = (ValuesNode) planNode; + if (valuesNode.getRows().stream().anyMatch(row -> row.size() > 1)) { + return Result.empty(); + } + Iterator iter = valuesNode.getRows().stream().map(row -> row.get(0)).iterator(); + RowExpression predicate = specialForm(IN, BOOLEAN, ImmutableList.builder().add(semiJoinNode.getSourceJoinVariable()).addAll(iter).build()); + Assignments.Builder builder = Assignments.builder(); + builder.putAll(identityAssignments(source.getOutputVariables())) + .put(semiJoinNode.getSemiJoinOutput(), predicate); + ProjectNode projectNode = new ProjectNode(context.getIdAllocator().getNextId(), source, builder.build()); + return Result.ofPlanNode(projectNode); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java index e60ab4f29a128..4490dc290ff24 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java @@ -233,6 +233,14 @@ public static Property source() Optional.empty()); } + public static class SemiJoin + { + public static Property filteringSource() + { + return optionalProperty("filteringSource", node -> Optional.of(node.getFilteringSource())); + } + } + public static Property> sources() { return property("sources", PlanNode::getSources); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 01ddec6dee152..340e35e161c7f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -255,7 +255,8 @@ public void testDefaults() .setPrintEstimatedStatsFromCache(false) .setRemoveCrossJoinWithSingleConstantRow(true) .setUseHistograms(false) - .setInlineProjectionsOnValues(false)); + .setInlineProjectionsOnValues(false) + .setTransformInValuesToInFilter(false)); } @Test @@ -460,6 +461,7 @@ public void testExplicitPropertyMappings() .put("optimizer.remove-cross-join-with-single-constant-row", "false") .put("optimizer.use-histograms", "true") .put("optimizer.inline-projections-on-values", "true") + .put("optimizer.transform_in_values_to_in_filter", "true") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -661,7 +663,8 @@ public void testExplicitPropertyMappings() .setPrintEstimatedStatsFromCache(true) .setRemoveCrossJoinWithSingleConstantRow(false) .setUseHistograms(true) - .setInlineProjectionsOnValues(true); + .setInlineProjectionsOnValues(true) + .setTransformInValuesToInFilter(true); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformInValuesToInFilter.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformInValuesToInFilter.java new file mode 100644 index 0000000000000..958e380d40a9c --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformInValuesToInFilter.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.TRANSFORM_IN_VALUES_TO_IN_FILTER; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; + +public class TestTransformInValuesToInFilter + extends BaseRuleTest +{ + @Test + public void testDoesNotFireOnWithTwoColumns() + { + tester().assertThat(new TransformInValuesToInFilter()) + .setSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, "true") + .on(p -> p.semiJoin(p.variable("x"), + p.variable("c"), + p.variable("d"), + Optional.empty(), + Optional.empty(), + p.values(p.variable("x"), p.variable("y")), + p.project(p.values(p.getIdAllocator().getNextId(), 0, p.variable("a"), p.variable("b")), assignment(p.variable("c"), p.variable("a"))))) + .doesNotFire(); + } + + @Test + public void testFiresOnOneColumn() + { + tester().assertThat(new TransformInValuesToInFilter()) + .setSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, "true") + .on(p -> p.semiJoin(p.variable("x"), + p.variable("c"), + p.variable("d"), + Optional.empty(), + Optional.empty(), + p.values(p.variable("x"), p.variable("y")), + p.values(p.getIdAllocator().getNextId(), 2, p.variable("c")))) + .matches(node(ProjectNode.class, values("x", "y"))); + } +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index 8e6d34006c2c9..76e99a665a7a0 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -89,6 +89,7 @@ import static com.facebook.presto.SystemSessionProperties.REWRITE_LEFT_JOIN_ARRAY_CONTAINS_TO_EQUI_JOIN; import static com.facebook.presto.SystemSessionProperties.REWRITE_LEFT_JOIN_NULL_FILTER_TO_SEMI_JOIN; import static com.facebook.presto.SystemSessionProperties.SIMPLIFY_PLAN_WITH_EMPTY_INPUT; +import static com.facebook.presto.SystemSessionProperties.TRANSFORM_IN_VALUES_TO_IN_FILTER; import static com.facebook.presto.SystemSessionProperties.USE_DEFAULTS_FOR_CORRELATED_AGGREGATION_PUSHDOWN_THROUGH_OUTER_JOINS; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -7874,4 +7875,24 @@ public void testEvaluateProjectOnValues() "SELECT a * 2, a - 1 FROM (SELECT x * 2 as a FROM (VALUES 15) t(x))", "SELECT * FROM (VALUES (60, 29))"); } + + @Test + public void testTransformInValuesToInFilter() + { + Session session = Session.builder(getSession()) + .setSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, "true") + .setSystemProperty(INLINE_PROJECTIONS_ON_VALUES, "true") + .build(); + assertQuery(session, "SELECT * FROM " + + "(VALUES " + + "(1, 5), " + + "(2, 6), " + + "(3, 7)) " + + "t(k, v) " + + "WHERE k in (Values 1, 2)", + "SELECT * FROM " + + "(VALUES " + + "(1, 5), " + + "(2, 6))"); + } }