diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 29dfc118762a3..f9cbf5e68f6fd 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -352,6 +352,7 @@ public final class SystemSessionProperties public static final String JOIN_PREFILTER_BUILD_SIDE = "join_prefilter_build_side"; public static final String OPTIMIZER_USE_HISTOGRAMS = "optimizer_use_histograms"; public static final String WARN_ON_COMMON_NAN_PATTERNS = "warn_on_common_nan_patterns"; + public static final String INLINE_PROJECTIONS_ON_VALUES = "inline_projections_on_values"; private final List> sessionProperties; @@ -1970,6 +1971,10 @@ public SystemSessionProperties( booleanProperty(WARN_ON_COMMON_NAN_PATTERNS, "Whether to give a warning for some common issues relating to NaNs", featuresConfig.getWarnOnCommonNanPatterns(), + false), + booleanProperty(INLINE_PROJECTIONS_ON_VALUES, + "Whether to evaluate project node on values node", + featuresConfig.getInlineProjectionsOnValues(), false)); } @@ -3287,4 +3292,9 @@ public static boolean warnOnCommonNanPatterns(Session session) { return session.getSystemProperty(WARN_ON_COMMON_NAN_PATTERNS, Boolean.class); } + + public static boolean isInlineProjectionsOnValues(Session session) + { + return session.getSystemProperty(INLINE_PROJECTIONS_ON_VALUES, Boolean.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index d43fa24a14ab8..603cf787437ae 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -317,6 +317,7 @@ public class FeaturesConfig private boolean useNewNanDefinition = true; private boolean warnOnPossibleNans; + private boolean isInlineProjectionsOnValuesEnabled; public enum PartitioningPrecisionStrategy { @@ -3200,4 +3201,17 @@ public FeaturesConfig setWarnOnCommonNanPatterns(boolean warnOnPossibleNans) this.warnOnPossibleNans = warnOnPossibleNans; return this; } + + public boolean getInlineProjectionsOnValues() + { + return isInlineProjectionsOnValuesEnabled; + } + + @Config("optimizer.inline-projections-on-values") + @ConfigDescription("Inline deterministic projections on values input") + public FeaturesConfig setInlineProjectionsOnValues(boolean isInlineProjectionsOnValuesEnabled) + { + this.isInlineProjectionsOnValuesEnabled = isInlineProjectionsOnValuesEnabled; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 6bff688e2a892..87c239e778e1f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -45,6 +45,7 @@ import com.facebook.presto.sql.planner.iterative.rule.ImplementFilteredAggregations; import com.facebook.presto.sql.planner.iterative.rule.ImplementOffset; import com.facebook.presto.sql.planner.iterative.rule.InlineProjections; +import com.facebook.presto.sql.planner.iterative.rule.InlineProjectionsOnValues; import com.facebook.presto.sql.planner.iterative.rule.InlineSqlFunctions; import com.facebook.presto.sql.planner.iterative.rule.LeftJoinNullFilterToSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.LeftJoinWithArrayContainsToEquiJoinCondition; @@ -477,6 +478,15 @@ public PlanOptimizers( new TransformUncorrelatedInPredicateSubqueryToSemiJoin(), new TransformCorrelatedScalarAggregationToJoin(metadata.getFunctionAndTypeManager()), new TransformCorrelatedLateralJoinToJoin(metadata.getFunctionAndTypeManager()))), + new IterativeOptimizer( + metadata, + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + ImmutableSet.>builder() + .add(new InlineProjectionsOnValues(metadata.getFunctionAndTypeManager())) + .addAll(new SimplifyRowExpressions(metadata).rules()) + .build()), new IterativeOptimizer( metadata, ruleStats, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/InlineProjectionsOnValues.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/InlineProjectionsOnValues.java new file mode 100644 index 0000000000000..1ad9f9cda384b --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/InlineProjectionsOnValues.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.spi.relation.DeterminismEvaluator; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Streams; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.isInlineProjectionsOnValues; +import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.sql.planner.RowExpressionVariableInliner.inlineVariables; +import static com.facebook.presto.sql.planner.plan.Patterns.project; +import static com.facebook.presto.sql.planner.plan.Patterns.source; +import static com.facebook.presto.sql.planner.plan.Patterns.values; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; + +/** + * This optimizer looks for ProjectNode followed by a ValuesNode and get the ProjectNode Evaluated. + * When this rule is used on iterative optimizer, the rule could apply iteratively. + *

+ * Plan before optimizer: + *

+ * ProjectNode (outputVariables)
+ *   - ValuesNode
+ * 
+ *

+ * Plan after optimizer: + *

+ * ValuesNode (outputVariables)
+ * 
+ */ +public class InlineProjectionsOnValues + implements Rule +{ + private static final Capture CHILD = newCapture(); + + private static final Pattern PATTERN = project() + .with(source().matching(values().capturedAs(CHILD))); + + private final FunctionAndTypeManager functionAndTypeManager; + + public InlineProjectionsOnValues(FunctionAndTypeManager functionAndTypeManager) + { + this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionManager is null"); + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isInlineProjectionsOnValues(session); + } + + @Override + public Result apply(ProjectNode projectNode, Captures captures, Context context) + { + ValuesNode source = captures.get(CHILD); + List> rows = source.getRows(); + List valuesOutputVariables = source.getOutputVariables(); + List projectOutputVariables = projectNode.getOutputVariables(); + List projectRowExpressions = projectNode.getAssignments() + .getExpressions() + .stream() + .collect(toImmutableList()); + + // exclude non-deterministic function + DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager); + if (!projectRowExpressions.stream().allMatch(determinismEvaluator::isDeterministic)) { + return Result.empty(); + } + if (!rows.stream().allMatch(row -> row.stream() + .allMatch(determinismEvaluator::isDeterministic))) { + return Result.empty(); + } + + //rewrite ProjectNode assignment expressions + ImmutableList.Builder> rowExpressionsListBuilder = ImmutableList.builder(); + for (List rowExpressions : rows) { + verify(rowExpressions.size() == valuesOutputVariables.size(), "Output variable does not match its value in ValuesNode"); + Map mapping = Streams.zip( + valuesOutputVariables.stream(), + rowExpressions.stream(), + SimpleImmutableEntry::new) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + List rowExpressionsInProject = projectRowExpressions.stream() + .map(expression -> inlineVariables(mapping, expression)) + .collect(toImmutableList()); + rowExpressionsListBuilder.add(rowExpressionsInProject); + } + + ValuesNode updatedProject = new ValuesNode( + source.getSourceLocation(), + context.getIdAllocator().getNextId(), + projectOutputVariables, + rowExpressionsListBuilder.build(), + Optional.empty()); + + return Result.ofPlanNode(updatedProject); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 49121da5fdc8f..f787d32660241 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -277,7 +277,8 @@ public void testDefaults() .setRemoveCrossJoinWithSingleConstantRow(true) .setUseHistograms(false) .setUseNewNanDefinition(true) - .setWarnOnCommonNanPatterns(false)); + .setWarnOnCommonNanPatterns(false) + .setInlineProjectionsOnValues(false)); } @Test @@ -499,6 +500,7 @@ public void testExplicitPropertyMappings() .put("optimizer.use-histograms", "true") .put("use-new-nan-definition", "false") .put("warn-on-common-nan-patterns", "true") + .put("optimizer.inline-projections-on-values", "true") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -717,7 +719,8 @@ public void testExplicitPropertyMappings() .setRemoveCrossJoinWithSingleConstantRow(false) .setUseHistograms(true) .setUseNewNanDefinition(false) - .setWarnOnCommonNanPatterns(true); + .setWarnOnCommonNanPatterns(true) + .setInlineProjectionsOnValues(true); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestInlineProjectionsOnValues.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestInlineProjectionsOnValues.java new file mode 100644 index 0000000000000..9e98913961105 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestInlineProjectionsOnValues.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import org.testng.annotations.Test; + +import static com.facebook.presto.SystemSessionProperties.INLINE_PROJECTIONS_ON_VALUES; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; +import static com.facebook.presto.sql.relational.Expressions.call; + +public class TestInlineProjectionsOnValues + extends BaseRuleTest +{ + @Test + public void testDoesNotFireOn() + { + tester().assertThat(new InlineProjectionsOnValues(tester.getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(INLINE_PROJECTIONS_ON_VALUES, "true") + .on(p -> p.project(p.project(p.values(p.getIdAllocator().getNextId(), p.variable("a")), + assignment(p.variable("c"), p.variable("a"))), + assignment(p.variable("d"), p.variable("c")))) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnWithNonDeterministicFunction() + { + tester().assertThat(new InlineProjectionsOnValues(tester.getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(INLINE_PROJECTIONS_ON_VALUES, "true") + .on(p -> p.project(p.values(p.getIdAllocator().getNextId(), p.variable("a")), + assignment(p.variable("b"), call(tester.getMetadata().getFunctionAndTypeManager(), "random", BIGINT, p.variable("a"))))) + .doesNotFire(); + } + + @Test + public void testFireOnProjectFollowedByValues() + { + tester().assertThat(new InlineProjectionsOnValues(tester.getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(INLINE_PROJECTIONS_ON_VALUES, "true") + // Form the input planNode: ProjectNode -> ValuesNode + .on(p -> p.project(p.values(p.getIdAllocator().getNextId(), p.variable("a")), + assignment(p.variable("c"), p.variable("a")))) + // Ensure the PlanNode is optimized to a ValuesNode + .matches(node(ValuesNode.class)); + } +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index 7f891f0eba173..a56847e972477 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -55,6 +55,7 @@ import static com.facebook.presto.SystemSessionProperties.FIELD_NAMES_IN_JSON_CAST_ENABLED; import static com.facebook.presto.SystemSessionProperties.GENERATE_DOMAIN_FILTERS; import static com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT; +import static com.facebook.presto.SystemSessionProperties.INLINE_PROJECTIONS_ON_VALUES; import static com.facebook.presto.SystemSessionProperties.ITERATIVE_OPTIMIZER_TIMEOUT; import static com.facebook.presto.SystemSessionProperties.JOIN_PREFILTER_BUILD_SIDE; import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_ENABLED; @@ -7713,4 +7714,36 @@ public void testRealDistinctPositiveAndNegativeZero(String optimizeHashGeneratio .build(); assertQuery(session, "SELECT DISTINCT x FROM (VALUES (REAL '0.0'), (REAL '-0.0')) t(x)", "SELECT CAST(0.0 AS REAL)"); } + + @Test + public void testEvaluateProjectOnValues() + { + Session session = Session.builder(getSession()) + .setSystemProperty(INLINE_PROJECTIONS_ON_VALUES, "true") + .build(); + assertQuery(session, + "SELECT MAP(ARRAY[1,2,3],ARRAY['one','two','three'])[x] from (values 1,2) as t(x)", + "SELECT * FROM (VALUES 'one','two')"); + assertQuery(session, + "SELECT CASE WHEN x