Transform in values to in filter

presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java

@@ -364,6 +364,7 @@ public final class SystemSessionProperties
     public static final String OPTIMIZER_USE_HISTOGRAMS = "optimizer_use_histograms";
     public static final String WARN_ON_COMMON_NAN_PATTERNS = "warn_on_common_nan_patterns";
     public static final String INLINE_PROJECTIONS_ON_VALUES = "inline_projections_on_values";
+    public static final String TRANSFORM_IN_VALUES_TO_IN_FILTER = "transform_in_values_to_in_filter";
 
     private final List<PropertyMetadata<?>> sessionProperties;
 
@@ -2038,6 +2039,10 @@ public SystemSessionProperties(
                 booleanProperty(INLINE_PROJECTIONS_ON_VALUES,
                         "Whether to evaluate project node on values node",
                         featuresConfig.getInlineProjectionsOnValues(),
+                        false),
+                booleanProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER,
+                        "Transform in values to in filter instead of semijoin whenever possible",
+                        featuresConfig.getTransformInValuesToInFilter(),
                         false));
     }
 
@@ -3370,4 +3375,9 @@ public static boolean isInlineProjectionsOnValues(Session session)
     {
         return session.getSystemProperty(INLINE_PROJECTIONS_ON_VALUES, Boolean.class);
     }
+
+    public static boolean isTransformInValuesToInFilterEnabled(Session session)
+    {
+        return session.getSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, Boolean.class);
+    }
 }

presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java

@@ -295,6 +295,7 @@ public class FeaturesConfig
     private boolean useHistograms;
 
     private boolean isInlineProjectionsOnValuesEnabled;
+    private boolean transformInValuesToInFilterEnabled;
 
     public enum PartitioningPrecisionStrategy
     {
@@ -2969,4 +2970,17 @@ public FeaturesConfig setInlineProjectionsOnValues(boolean isInlineProjectionsOn
         this.isInlineProjectionsOnValuesEnabled = isInlineProjectionsOnValuesEnabled;
         return this;
     }
+
+    public boolean getTransformInValuesToInFilter()
+    {
+        return transformInValuesToInFilterEnabled;
+    }
+
+    @Config("optimizer.transform_in_values_to_in_filter")
+    @ConfigDescription("Transform the in values form to in filter instead of semijoin whenever possible")
+    public FeaturesConfig setTransformInValuesToInFilter(boolean transformInValuesToInFilterEnabled)
+    {
+        this.transformInValuesToInFilterEnabled = transformInValuesToInFilterEnabled;
+        return this;
+    }
 }

presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java

@@ -140,6 +140,7 @@
 import com.facebook.presto.sql.planner.iterative.rule.TransformDistinctInnerJoinToLeftEarlyOutJoin;
 import com.facebook.presto.sql.planner.iterative.rule.TransformDistinctInnerJoinToRightEarlyOutJoin;
 import com.facebook.presto.sql.planner.iterative.rule.TransformExistsApplyToLateralNode;
+import com.facebook.presto.sql.planner.iterative.rule.TransformInValuesToInFilter;
 import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin;
 import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin;
 import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedLateralToJoin;
@@ -485,7 +486,14 @@ public PlanOptimizers(
                         estimatedExchangesCostCalculator,
                         ImmutableSet.<Rule<?>>builder()
                                 .add(new InlineProjectionsOnValues(metadata.getFunctionAndTypeManager()))
-                                .addAll(new SimplifyRowExpressions(metadata).rules())
+                                .addAll(new SimplifyRowExpressions(metadata).rules()).build()),
+                new IterativeOptimizer(
+                        metadata,
+                        ruleStats,
+                        statsCalculator,
+                        estimatedExchangesCostCalculator,
+                        ImmutableSet.<Rule<?>>builder()
+                                .add(new TransformInValuesToInFilter())
                                 .build()),
                 new IterativeOptimizer(
                         metadata,

presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformInValuesToInFilter.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.sql.planner.iterative.rule;
+
+import com.facebook.presto.Session;
+import com.facebook.presto.matching.Captures;
+import com.facebook.presto.matching.Pattern;
+import com.facebook.presto.spi.plan.Assignments;
+import com.facebook.presto.spi.plan.PlanNode;
+import com.facebook.presto.spi.plan.ProjectNode;
+import com.facebook.presto.spi.plan.ValuesNode;
+import com.facebook.presto.spi.relation.RowExpression;
+import com.facebook.presto.sql.planner.iterative.Rule;
+import com.facebook.presto.sql.planner.plan.SemiJoinNode;
+import com.google.common.collect.ImmutableList;
+
+import java.util.Iterator;
+
+import static com.facebook.presto.SystemSessionProperties.isTransformInValuesToInFilterEnabled;
+import static com.facebook.presto.common.type.BooleanType.BOOLEAN;
+import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IN;
+import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments;
+import static com.facebook.presto.sql.planner.plan.Patterns.SemiJoin.filteringSource;
+import static com.facebook.presto.sql.planner.plan.Patterns.semiJoin;
+import static com.facebook.presto.sql.planner.plan.Patterns.values;
+import static com.facebook.presto.sql.relational.Expressions.specialForm;
+
+/**
+ * This optimizer looks for SemiJoinNode whose filteringSource has only one column, then transform the SemijoinNode into ProjectNode with predicate variable for filtering.
+ * <p/>
+ * Plan before optimizer:
+ * <pre>
+ * SemiJoinNode (semiJoinOutput variable c):
+ *   - source (sourceJoinVariable a)
+ *   - filteringSource (one column variable b)
+ *       - ProjectNode
+ *            - ValuesNode
+ * </pre>
+ * <p/>
+ * Plan after optimizer:
+ * <pre>
+ * ProjectNode:
+ *   - source
+ *   - assignments(identityAssignments of source output,c in b)
+ * </pre>
+ */
+public class TransformInValuesToInFilter
+        implements Rule<SemiJoinNode>
+{
+    private static final Pattern<SemiJoinNode> PATTERN = semiJoin().with(filteringSource().matching(values()));
+
+    @Override
+    public Pattern<SemiJoinNode> getPattern()
+    {
+        return PATTERN;
+    }
+
+    @Override
+    public boolean isEnabled(Session session)
+    {
+        return isTransformInValuesToInFilterEnabled(session);
+    }
+
+    @Override
+    public Result apply(SemiJoinNode semiJoinNode, Captures captures, Context context)
+    {
+        PlanNode source = semiJoinNode.getSource();
+        PlanNode planNode = context.getLookup().resolveGroup(semiJoinNode.getFilteringSource()).findFirst().get();
+        if (!(planNode instanceof ValuesNode)) {
+            return Result.empty();
+        }
+        ValuesNode valuesNode = (ValuesNode) planNode;
+        if (valuesNode.getRows().stream().anyMatch(row -> row.size() > 1)) {
+            return Result.empty();
+        }
+        Iterator<RowExpression> iter = valuesNode.getRows().stream().map(row -> row.get(0)).iterator();
+        RowExpression predicate = specialForm(IN, BOOLEAN, ImmutableList.<RowExpression>builder().add(semiJoinNode.getSourceJoinVariable()).addAll(iter).build());
+        Assignments.Builder builder = Assignments.builder();
+        builder.putAll(identityAssignments(source.getOutputVariables()))
+                .put(semiJoinNode.getSemiJoinOutput(), predicate);
+        ProjectNode projectNode = new ProjectNode(context.getIdAllocator().getNextId(), source, builder.build());
+        return Result.ofPlanNode(projectNode);
+    }
+}

presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java

@@ -233,6 +233,14 @@ public static Property<PlanNode, PlanNode> source()
                 Optional.empty());
     }
 
+    public static class SemiJoin
+    {
+        public static Property<SemiJoinNode, PlanNode> filteringSource()
+        {
+            return optionalProperty("filteringSource", node -> Optional.of(node.getFilteringSource()));
+        }
+    }
+
     public static Property<PlanNode, List<PlanNode>> sources()
     {
         return property("sources", PlanNode::getSources);

presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java

@@ -255,7 +255,8 @@ public void testDefaults()
                 .setPrintEstimatedStatsFromCache(false)
                 .setRemoveCrossJoinWithSingleConstantRow(true)
                 .setUseHistograms(false)
-                .setInlineProjectionsOnValues(false));
+                .setInlineProjectionsOnValues(false)
+                .setTransformInValuesToInFilter(false));
     }
 
     @Test
@@ -460,6 +461,7 @@ public void testExplicitPropertyMappings()
                 .put("optimizer.remove-cross-join-with-single-constant-row", "false")
                 .put("optimizer.use-histograms", "true")
                 .put("optimizer.inline-projections-on-values", "true")
+                .put("optimizer.transform_in_values_to_in_filter", "true")
                 .build();
 
         FeaturesConfig expected = new FeaturesConfig()
@@ -661,7 +663,8 @@ public void testExplicitPropertyMappings()
                 .setPrintEstimatedStatsFromCache(true)
                 .setRemoveCrossJoinWithSingleConstantRow(false)
                 .setUseHistograms(true)
-                .setInlineProjectionsOnValues(true);
+                .setInlineProjectionsOnValues(true)
+                .setTransformInValuesToInFilter(true);
         assertFullMapping(properties, expected);
     }
 

presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformInValuesToInFilter.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.sql.planner.iterative.rule;
+
+import com.facebook.presto.spi.plan.ProjectNode;
+import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest;
+import org.testng.annotations.Test;
+
+import java.util.Optional;
+
+import static com.facebook.presto.SystemSessionProperties.TRANSFORM_IN_VALUES_TO_IN_FILTER;
+import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node;
+import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values;
+import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment;
+
+public class TestTransformInValuesToInFilter
+        extends BaseRuleTest
+{
+    @Test
+    public void testDoesNotFireOnWithTwoColumns()
+    {
+        tester().assertThat(new TransformInValuesToInFilter())
+                .setSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, "true")
+                .on(p -> p.semiJoin(p.variable("x"),
+                        p.variable("c"),
+                        p.variable("d"),
+                        Optional.empty(),
+                        Optional.empty(),
+                        p.values(p.variable("x"), p.variable("y")),
+                        p.project(p.values(p.getIdAllocator().getNextId(), 0, p.variable("a"), p.variable("b")), assignment(p.variable("c"), p.variable("a")))))
+                .doesNotFire();
+    }
+
+    @Test
+    public void testFiresOnOneColumn()
+    {
+        tester().assertThat(new TransformInValuesToInFilter())
+                .setSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, "true")
+                .on(p -> p.semiJoin(p.variable("x"),
+                        p.variable("c"),
+                        p.variable("d"),
+                        Optional.empty(),
+                        Optional.empty(),
+                        p.values(p.variable("x"), p.variable("y")),
+                        p.values(p.getIdAllocator().getNextId(), 2, p.variable("c"))))
+                .matches(node(ProjectNode.class, values("x", "y")));
+    }
+}

presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java

@@ -89,6 +89,7 @@
 import static com.facebook.presto.SystemSessionProperties.REWRITE_LEFT_JOIN_ARRAY_CONTAINS_TO_EQUI_JOIN;
 import static com.facebook.presto.SystemSessionProperties.REWRITE_LEFT_JOIN_NULL_FILTER_TO_SEMI_JOIN;
 import static com.facebook.presto.SystemSessionProperties.SIMPLIFY_PLAN_WITH_EMPTY_INPUT;
+import static com.facebook.presto.SystemSessionProperties.TRANSFORM_IN_VALUES_TO_IN_FILTER;
 import static com.facebook.presto.SystemSessionProperties.USE_DEFAULTS_FOR_CORRELATED_AGGREGATION_PUSHDOWN_THROUGH_OUTER_JOINS;
 import static com.facebook.presto.common.type.BigintType.BIGINT;
 import static com.facebook.presto.common.type.BooleanType.BOOLEAN;
@@ -7874,4 +7875,24 @@ public void testEvaluateProjectOnValues()
                 "SELECT a * 2, a - 1 FROM (SELECT x * 2 as a FROM (VALUES 15) t(x))",
                 "SELECT * FROM (VALUES (60, 29))");
     }
+
+    @Test
+    public void testTransformInValuesToInFilter()
+    {
+        Session session = Session.builder(getSession())
+                .setSystemProperty(TRANSFORM_IN_VALUES_TO_IN_FILTER, "true")
+                .setSystemProperty(INLINE_PROJECTIONS_ON_VALUES, "true")
+                .build();
+        assertQuery(session, "SELECT * FROM " +
+                        "(VALUES " +
+                        "(1, 5), " +
+                        "(2, 6), " +
+                        "(3, 7)) " +
+                        "t(k, v) " +
+                        "WHERE k in (Values 1, 2)",
+                "SELECT * FROM " +
+                        "(VALUES " +
+                        "(1, 5), " +
+                        "(2, 6))");
+    }
 }