diff --git a/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java b/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java index 6b94af599656f..84c71f1111688 100644 --- a/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java +++ b/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java @@ -36,6 +36,7 @@ import java.nio.file.Paths; import java.util.stream.Stream; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.testing.TestngUtils.toDataProvider; @@ -121,7 +122,7 @@ private String generateQueryPlan(String query) String sql = query.replaceAll("\\s+;\\s+$", "") .replace("${database}.${schema}.", "") .replace("\"${database}\".\"${schema}\".\"${prefix}", "\""); - Plan plan = plan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); + Plan plan = plan(sql, OPTIMIZED_AND_VALIDATED, false); JoinOrderPrinter joinOrderPrinter = new JoinOrderPrinter(); plan.getRoot().accept(joinOrderPrinter, 0); diff --git a/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java b/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java index f84faa15764c1..e45a67f5ab822 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java +++ b/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java @@ -40,6 +40,7 @@ private RuntimeMetricName() public static final String GET_COLUMN_METADATA_TIME_NANOS = "getColumnMetadataTimeNanos"; public static final String GET_SPLITS_TIME_NANOS = "getSplitsTimeNanos"; public static final String LOGICAL_PLANNER_TIME_NANOS = "logicalPlannerTimeNanos"; + public static final String OPTIMIZER_TIME_NANOS = "optimizerTimeNanos"; public static final String FRAGMENT_PLAN_TIME_NANOS = "fragmentPlanTimeNanos"; public static final String GET_LAYOUT_TIME_NANOS = "getLayoutTimeNanos"; public static final String REWRITE_AGGREGATION_IF_TO_FILTER_APPLIED = "rewriteAggregationIfToFilterApplied"; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java index fe56e9c8ec543..6c5abac27c48f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java @@ -38,11 +38,13 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.resourceGroups.ResourceGroupQueryLimits; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.split.CloseableSplitSourceProvider; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; @@ -88,10 +90,12 @@ import static com.facebook.presto.SystemSessionProperties.isUseLegacyScheduler; import static com.facebook.presto.common.RuntimeMetricName.FRAGMENT_PLAN_TIME_NANOS; import static com.facebook.presto.common.RuntimeMetricName.LOGICAL_PLANNER_TIME_NANOS; +import static com.facebook.presto.common.RuntimeMetricName.OPTIMIZER_TIME_NANOS; import static com.facebook.presto.execution.buffer.OutputBuffers.BROADCAST_PARTITION_ID; import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; import static com.facebook.presto.execution.buffer.OutputBuffers.createSpoolingOutputBuffers; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; import static com.facebook.presto.sql.planner.PlanNodeCanonicalInfo.getCanonicalInfo; import static com.google.common.base.Preconditions.checkArgument; @@ -523,20 +527,34 @@ private PlanRoot createLogicalPlan() stateMachine.beginAnalysis(); // plan query + final PlanVariableAllocator planVariableAllocator = new PlanVariableAllocator(); LogicalPlanner logicalPlanner = new LogicalPlanner( - false, stateMachine.getSession(), - planOptimizers, idAllocator, metadata, + planVariableAllocator); + + PlanNode planNode = getSession().getRuntimeStats().profileNanos( + LOGICAL_PLANNER_TIME_NANOS, + () -> logicalPlanner.plan(analysis)); + + Optimizer optimizer = new Optimizer( + stateMachine.getSession(), + metadata, + planOptimizers, + planChecker, sqlParser, + planVariableAllocator, + idAllocator, + stateMachine.getWarningCollector(), statsCalculator, costCalculator, - stateMachine.getWarningCollector(), - planChecker); + false); + Plan plan = getSession().getRuntimeStats().profileNanos( - LOGICAL_PLANNER_TIME_NANOS, - () -> logicalPlanner.plan(analysis)); + OPTIMIZER_TIME_NANOS, + () -> optimizer.validateAndOptimizePlan(planNode, OPTIMIZED_AND_VALIDATED)); + queryPlan.set(plan); stateMachine.setPlanStatsAndCosts(plan.getStatsAndCosts()); stateMachine.setPlanCanonicalInfo(getCanonicalInfo(getSession(), plan.getRoot(), planCanonicalInfoProvider)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java new file mode 100644 index 0000000000000..525b6bced2542 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql; + +import com.facebook.presto.Session; +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.cost.CachingCostProvider; +import com.facebook.presto.cost.CachingStatsProvider; +import com.facebook.presto.cost.CostCalculator; +import com.facebook.presto.cost.CostProvider; +import com.facebook.presto.cost.StatsAndCosts; +import com.facebook.presto.cost.StatsCalculator; +import com.facebook.presto.cost.StatsProvider; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.Plan; +import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; +import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.sanity.PlanChecker; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery; +import static com.facebook.presto.common.RuntimeUnit.NANO; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class Optimizer +{ + public enum PlanStage + { + CREATED, OPTIMIZED, OPTIMIZED_AND_VALIDATED + } + + private final List planOptimizers; + private final PlanChecker planChecker; + private final Session session; + private final Metadata metadata; + private final SqlParser sqlParser; + private final PlanVariableAllocator variableAllocator; + private final PlanNodeIdAllocator idAllocator; + private final WarningCollector warningCollector; + private final StatsCalculator statsCalculator; + private final CostCalculator costCalculator; + private final boolean explain; + + public Optimizer( + Session session, + Metadata metadata, + List planOptimizers, + PlanChecker planChecker, + SqlParser sqlParser, + PlanVariableAllocator variableAllocator, + PlanNodeIdAllocator idAllocator, + WarningCollector warningCollector, + StatsCalculator statsCalculator, + CostCalculator costCalculator, + boolean explain) + { + this.session = requireNonNull(session, "session is null"); + this.planOptimizers = requireNonNull(planOptimizers, "planOptimizers is null"); + this.planChecker = requireNonNull(planChecker, "planChecker is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.variableAllocator = requireNonNull(variableAllocator, "variableAllocator is null"); + this.idAllocator = requireNonNull(idAllocator, "idAllocator is null"); + this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); + this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator is null"); + this.costCalculator = requireNonNull(costCalculator, "costCalculator is null"); + this.explain = explain; + } + + public Plan validateAndOptimizePlan(PlanNode root, PlanStage stage) + { + planChecker.validateIntermediatePlan(root, session, metadata, sqlParser, variableAllocator.getTypes(), warningCollector); + + boolean enableVerboseRuntimeStats = SystemSessionProperties.isVerboseRuntimeStatsEnabled(session); + if (stage.ordinal() >= OPTIMIZED.ordinal()) { + for (PlanOptimizer optimizer : planOptimizers) { + long start = System.nanoTime(); + root = optimizer.optimize(root, session, variableAllocator.getTypes(), variableAllocator, idAllocator, warningCollector); + requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName())); + if (enableVerboseRuntimeStats) { + session.getRuntimeStats().addMetricValue(String.format("optimizer%sTimeNanos", optimizer.getClass().getSimpleName()), NANO, System.nanoTime() - start); + } + } + } + + if (stage.ordinal() >= OPTIMIZED_AND_VALIDATED.ordinal()) { + // make sure we produce a valid plan after optimizations run. This is mainly to catch programming errors + planChecker.validateFinalPlan(root, session, metadata, sqlParser, variableAllocator.getTypes(), warningCollector); + } + + TypeProvider types = variableAllocator.getTypes(); + return new Plan(root, types, computeStats(root, types)); + } + + private StatsAndCosts computeStats(PlanNode root, TypeProvider types) + { + if (explain || isPrintStatsForNonJoinQuery(session) || + PlanNodeSearcher.searchFrom(root).where(node -> + (node instanceof JoinNode) || (node instanceof SemiJoinNode)).matches()) { + StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); + CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session); + return StatsAndCosts.create(root, statsProvider, costProvider); + } + return StatsAndCosts.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java index 847d454b5bf67..e8062ddf3377d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java @@ -20,13 +20,16 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LogicalPlanner; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.PlanFragmenter; import com.facebook.presto.sql.planner.PlanOptimizers; +import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.SubPlan; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter; @@ -43,7 +46,10 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.common.RuntimeMetricName.LOGICAL_PLANNER_TIME_NANOS; +import static com.facebook.presto.common.RuntimeMetricName.OPTIMIZER_TIME_NANOS; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; import static com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.textIOPlan; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.graphvizDistributedPlan; @@ -194,19 +200,34 @@ public Plan getLogicalPlan(Session session, Statement statement, List logicalPlanner.plan(analysis)); + + Optimizer optimizer = new Optimizer( + session, + metadata, + planOptimizers, + planChecker, sqlParser, + planVariableAllocator, + idAllocator, + warningCollector, statsCalculator, costCalculator, - warningCollector, - planChecker); - return logicalPlanner.plan(analysis); + true); + + return session.getRuntimeStats().profileNanos( + OPTIMIZER_TIME_NANOS, + () -> optimizer.validateAndOptimizePlan(planNode, OPTIMIZED_AND_VALIDATED)); } public SubPlan getDistributedPlan(Session session, Statement statement, List parameters, WarningCollector warningCollector) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java index 51bb314234bc4..bf2de4623eeed 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java @@ -14,17 +14,9 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.Session; -import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.type.Type; -import com.facebook.presto.cost.CachingCostProvider; -import com.facebook.presto.cost.CachingStatsProvider; -import com.facebook.presto.cost.CostCalculator; -import com.facebook.presto.cost.CostProvider; -import com.facebook.presto.cost.StatsAndCosts; -import com.facebook.presto.cost.StatsCalculator; -import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.NewTableLayout; import com.facebook.presto.spi.ColumnHandle; @@ -34,7 +26,6 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableMetadata; -import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.LimitNode; @@ -50,21 +41,15 @@ import com.facebook.presto.sql.analyzer.RelationId; import com.facebook.presto.sql.analyzer.RelationType; import com.facebook.presto.sql.analyzer.Scope; -import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation; -import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; -import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; -import com.facebook.presto.sql.planner.sanity.PlanChecker; import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.CreateTableAsSelect; @@ -93,8 +78,6 @@ import java.util.Optional; import java.util.Set; -import static com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery; -import static com.facebook.presto.common.RuntimeUnit.NANO; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.metadata.MetadataUtil.toSchemaTableName; @@ -117,96 +100,31 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Streams.zip; -import static java.lang.String.format; import static java.util.Objects.requireNonNull; - public class LogicalPlanner { - public enum Stage - { - CREATED, OPTIMIZED, OPTIMIZED_AND_VALIDATED - } - - private final boolean explain; private final PlanNodeIdAllocator idAllocator; private final Session session; - private final List planOptimizers; - private final PlanChecker planChecker; - private final PlanVariableAllocator variableAllocator = new PlanVariableAllocator(); + private final PlanVariableAllocator variableAllocator; private final Metadata metadata; - private final SqlParser sqlParser; private final StatisticsAggregationPlanner statisticsAggregationPlanner; - private final StatsCalculator statsCalculator; - private final CostCalculator costCalculator; - private final WarningCollector warningCollector; public LogicalPlanner( - boolean explain, Session session, - List planOptimizers, PlanNodeIdAllocator idAllocator, Metadata metadata, - SqlParser sqlParser, - StatsCalculator statsCalculator, - CostCalculator costCalculator, - WarningCollector warningCollector, - PlanChecker planChecker) + PlanVariableAllocator variableAllocator) { - this.explain = explain; this.session = requireNonNull(session, "session is null"); - this.planOptimizers = requireNonNull(planOptimizers, "planOptimizers is null"); this.idAllocator = requireNonNull(idAllocator, "idAllocator is null"); this.metadata = requireNonNull(metadata, "metadata is null"); - this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.variableAllocator = requireNonNull(variableAllocator, "variableAllocator is null"); this.statisticsAggregationPlanner = new StatisticsAggregationPlanner(variableAllocator, metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver()); - this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator is null"); - this.costCalculator = requireNonNull(costCalculator, "costCalculator is null"); - this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); - this.planChecker = requireNonNull(planChecker, "planChecker is null"); } - public Plan plan(Analysis analysis) + public PlanNode plan(Analysis analysis) { - return plan(analysis, Stage.OPTIMIZED_AND_VALIDATED); - } - - public Plan plan(Analysis analysis, Stage stage) - { - PlanNode root = planStatement(analysis, analysis.getStatement()); - - planChecker.validateIntermediatePlan(root, session, metadata, sqlParser, variableAllocator.getTypes(), warningCollector); - - boolean enableVerboseRuntimeStats = SystemSessionProperties.isVerboseRuntimeStatsEnabled(session); - if (stage.ordinal() >= Stage.OPTIMIZED.ordinal()) { - for (PlanOptimizer optimizer : planOptimizers) { - long start = System.nanoTime(); - root = optimizer.optimize(root, session, variableAllocator.getTypes(), variableAllocator, idAllocator, warningCollector); - requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName())); - if (enableVerboseRuntimeStats) { - session.getRuntimeStats().addMetricValue(String.format("optimizer%sTimeNanos", optimizer.getClass().getSimpleName()), NANO, System.nanoTime() - start); - } - } - } - - if (stage.ordinal() >= Stage.OPTIMIZED_AND_VALIDATED.ordinal()) { - // make sure we produce a valid plan after optimizations run. This is mainly to catch programming errors - planChecker.validateFinalPlan(root, session, metadata, sqlParser, variableAllocator.getTypes(), warningCollector); - } - - TypeProvider types = variableAllocator.getTypes(); - return new Plan(root, types, computeStats(root, types)); - } - - private StatsAndCosts computeStats(PlanNode root, TypeProvider types) - { - if (explain || isPrintStatsForNonJoinQuery(session) || - PlanNodeSearcher.searchFrom(root).where(node -> - (node instanceof JoinNode) || (node instanceof SemiJoinNode)).matches()) { - StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); - CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session); - return StatsAndCosts.create(root, statsProvider, costProvider); - } - return StatsAndCosts.empty(); + return planStatement(analysis, analysis.getStatement()); } public PlanNode planStatement(Analysis analysis, Statement statement) diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index c05663c420995..bc9328d75881c 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -148,6 +148,7 @@ import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; import com.facebook.presto.split.SplitSource; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; import com.facebook.presto.sql.analyzer.AnalyzerProviderManager; @@ -174,6 +175,7 @@ import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.PlanFragmenter; import com.facebook.presto.sql.planner.PlanOptimizers; +import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.RemoteSourceFactory; import com.facebook.presto.sql.planner.SubPlan; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; @@ -247,6 +249,8 @@ import static com.facebook.presto.SystemSessionProperties.getQueryMaxTotalMemoryPerNode; import static com.facebook.presto.SystemSessionProperties.isHeapDumpOnExceededMemoryLimitEnabled; import static com.facebook.presto.SystemSessionProperties.isVerboseExceededMemoryLimitErrorsEnabled; +import static com.facebook.presto.common.RuntimeMetricName.LOGICAL_PLANNER_TIME_NANOS; +import static com.facebook.presto.common.RuntimeMetricName.OPTIMIZER_TIME_NANOS; import static com.facebook.presto.cost.StatsCalculatorModule.createNewStatsCalculator; import static com.facebook.presto.execution.scheduler.StreamingPlanSection.extractStreamingSections; import static com.facebook.presto.execution.scheduler.TableWriteInfo.createTableWriteInfo; @@ -254,6 +258,7 @@ import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.REWINDABLE_GROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static com.facebook.presto.sql.testing.TreeAssertions.assertFormattedSql; @@ -989,15 +994,15 @@ public PlanNodeId getNextId() @Override public Plan createPlan(Session session, @Language("SQL") String sql, WarningCollector warningCollector) { - return createPlan(session, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, warningCollector); + return createPlan(session, sql, OPTIMIZED_AND_VALIDATED, warningCollector); } - public Plan createPlan(Session session, @Language("SQL") String sql, LogicalPlanner.Stage stage, WarningCollector warningCollector) + public Plan createPlan(Session session, @Language("SQL") String sql, Optimizer.PlanStage stage, WarningCollector warningCollector) { return createPlan(session, sql, stage, true, warningCollector); } - public Plan createPlan(Session session, @Language("SQL") String sql, LogicalPlanner.Stage stage, boolean forceSingleNode, WarningCollector warningCollector) + public Plan createPlan(Session session, @Language("SQL") String sql, Optimizer.PlanStage stage, boolean forceSingleNode, WarningCollector warningCollector) { AnalyzerOptions analyzerOptions = createAnalyzerOptions(session, warningCollector); BuiltInPreparedQuery preparedQuery = new BuiltInQueryPreparer(sqlParser).prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); @@ -1030,10 +1035,10 @@ public List getPlanOptimizers(boolean forceSingleNode) public Plan createPlan(Session session, @Language("SQL") String sql, List optimizers, WarningCollector warningCollector) { - return createPlan(session, sql, optimizers, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, warningCollector); + return createPlan(session, sql, optimizers, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, warningCollector); } - public Plan createPlan(Session session, @Language("SQL") String sql, List optimizers, LogicalPlanner.Stage stage, WarningCollector warningCollector) + public Plan createPlan(Session session, @Language("SQL") String sql, List optimizers, Optimizer.PlanStage stage, WarningCollector warningCollector) { AnalyzerOptions analyzerOptions = createAnalyzerOptions(session, warningCollector); BuiltInPreparedQuery preparedQuery = new BuiltInQueryPreparer(sqlParser).prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); @@ -1053,10 +1058,35 @@ public Plan createPlan(Session session, @Language("SQL") String sql, List logicalPlanner.plan(analysis)); + + Optimizer optimizer = new Optimizer( + session, + metadata, + optimizers, + singleNodePlanChecker, + sqlParser, + planVariableAllocator, + idAllocator, + warningCollector, + statsCalculator, + costCalculator, + preparedQuery.getWrappedStatement() instanceof Explain); + + return session.getRuntimeStats().profileNanos( + OPTIMIZER_TIME_NANOS, + () -> optimizer.validateAndOptimizePlan(planNode, stage)); } private static List getNextBatch(SplitSource splitSource) diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java index 6bb7203439126..8196062636ecb 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java @@ -22,7 +22,7 @@ import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntry; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.statistics.PlanStatistics; -import com.facebook.presto.sql.planner.LogicalPlanner; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.assertions.PlanAssert; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; @@ -85,10 +85,10 @@ public void testHistoryBasedStatsCalculator() private void assertPlan(String sql, PlanMatchPattern pattern) { - assertPlan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern); + assertPlan(sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, pattern); } - private void assertPlan(String sql, LogicalPlanner.Stage stage, PlanMatchPattern pattern) + private void assertPlan(String sql, Optimizer.PlanStage stage, PlanMatchPattern pattern) { queryRunner.inTransaction(transactionSession -> { Plan actualPlan = queryRunner.createPlan(transactionSession, sql, stage, WarningCollector.NOOP); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestStatsCalculator.java index 242aba6050b28..3b7f04df494c5 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestStatsCalculator.java @@ -15,7 +15,7 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.TableScanNode; -import com.facebook.presto.sql.planner.LogicalPlanner; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.assertions.PlanAssert; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; @@ -58,10 +58,10 @@ public void testStatsCalculatorUsesLayout() private void assertPlan(String sql, PlanMatchPattern pattern) { - assertPlan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern); + assertPlan(sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, pattern); } - private void assertPlan(String sql, LogicalPlanner.Stage stage, PlanMatchPattern pattern) + private void assertPlan(String sql, Optimizer.PlanStage stage, PlanMatchPattern pattern) { queryRunner.inTransaction(transactionSession -> { Plan actualPlan = queryRunner.createPlan(transactionSession, sql, stage, WarningCollector.NOOP); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java b/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java index 93241fd368266..40a0dc933d4b0 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java @@ -23,8 +23,8 @@ import com.facebook.presto.spi.WarningCode; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.analyzer.SemanticException; -import com.facebook.presto.sql.planner.LogicalPlanner; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; @@ -104,7 +104,7 @@ public static void assertPlannerWarnings(LocalQueryRunner queryRunner, @Language createPlan(queryRunner, transactionSession, sql, warningCollector, rules.get()); } else { - queryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.CREATED, false, warningCollector); + queryRunner.createPlan(transactionSession, sql, Optimizer.PlanStage.CREATED, false, warningCollector); } return null; }); @@ -138,7 +138,7 @@ private static Plan createPlan(LocalQueryRunner queryRunner, Session session, St queryRunner.getCostCalculator(), ImmutableSet.copyOf(new TranslateExpressions(queryRunner.getMetadata(), queryRunner.getSqlParser()).rules())); - return queryRunner.createPlan(session, sql, ImmutableList.of(optimizer, expressionTranslator), LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, warningCollector); + return queryRunner.createPlan(session, sql, ImmutableList.of(optimizer, expressionTranslator), Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, warningCollector); } public static List createTestWarnings(int numberOfWarnings) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/BenchmarkPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/BenchmarkPlanner.java index 53cd52ae41a36..1f82058b33a6d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/BenchmarkPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/BenchmarkPlanner.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.tpch.ColumnNaming; import com.facebook.presto.tpch.TpchConnectorFactory; @@ -67,7 +68,7 @@ public static class BenchmarkData private String iterativeOptimizerEnabled = "true"; @Param({"optimized", "created"}) - private String stage = LogicalPlanner.Stage.OPTIMIZED.toString(); + private String stage = Optimizer.PlanStage.OPTIMIZED.toString(); private LocalQueryRunner queryRunner; private List queries; @@ -117,7 +118,7 @@ public String readResource(String resource) public List planQueries(BenchmarkData benchmarkData) { return benchmarkData.queryRunner.inTransaction(transactionSession -> { - LogicalPlanner.Stage stage = LogicalPlanner.Stage.valueOf(benchmarkData.stage.toUpperCase()); + Optimizer.PlanStage stage = Optimizer.PlanStage.valueOf(benchmarkData.stage.toUpperCase()); return benchmarkData.queries.stream() .map(query -> benchmarkData.queryRunner.createPlan(transactionSession, query, stage, false, WarningCollector.NOOP)) .collect(toImmutableList()); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java index aa6a5d0219411..50f7868b54b1d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java @@ -17,6 +17,7 @@ import com.facebook.presto.common.plan.PlanCanonicalizationStrategy; import com.facebook.presto.cost.HistoryBasedPlanStatisticsCalculator; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import org.testng.annotations.Test; @@ -35,7 +36,7 @@ public void testCache() { Session session = createSession(); String sql = "SELECT O.totalprice, C.name FROM orders O, customer C WHERE C.custkey = O.custkey LIMIT 10"; - PlanNode root = plan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, session).getRoot(); + PlanNode root = plan(sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, session).getRoot(); assertTrue(root.getStatsEquivalentPlanNode().isPresent()); CachingPlanCanonicalInfoProvider planCanonicalInfoProvider = (CachingPlanCanonicalInfoProvider) ((HistoryBasedPlanStatisticsCalculator) getQueryRunner().getStatsCalculator()).getPlanCanonicalInfoProvider(); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java index 1180ddfeed474..e4ff184d38b7e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java @@ -39,9 +39,9 @@ import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.CONNECTOR; import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.REMOVE_SAFE_CONSTANTS; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.planner.CanonicalPlanGenerator.generateCanonicalPlan; import static com.facebook.presto.sql.planner.CanonicalPlanGenerator.generateCanonicalPlanFragment; -import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED; import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java index 2e395d2c68297..ef337d476511f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -358,7 +359,7 @@ private void assertDifferentPlanHash(String sql1, String sql2, PlanCanonicalizat private List getStatsEquivalentPlanHashes(String sql) { Session session = createSession(); - PlanNode root = plan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, session).getRoot(); + PlanNode root = plan(sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, session).getRoot(); assertTrue(root.getStatsEquivalentPlanNode().isPresent()); ImmutableList.Builder result = ImmutableList.builder(); @@ -372,7 +373,7 @@ private String getPlanHash(String sql, PlanCanonicalizationStrategy strategy) throws Exception { Session session = createSession(); - PlanNode plan = plan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, session).getRoot(); + PlanNode plan = plan(sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, session).getRoot(); assertTrue(plan.getStatsEquivalentPlanNode().isPresent()); return getObjectMapper().writeValueAsString(generateCanonicalPlan(plan.getStatsEquivalentPlanNode().get(), strategy, getObjectMapper()).get()); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalDynamicFilter.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalDynamicFilter.java index 77aea21131c0d..9a35a1530cfd6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalDynamicFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalDynamicFilter.java @@ -19,6 +19,7 @@ import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.expressions.DynamicFilters.DynamicFilterPlaceholder; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; import com.facebook.presto.sql.planner.plan.JoinNode; @@ -197,7 +198,7 @@ public void testCreateSingleColumn() SubPlan subplan = subplan( "SELECT count() FROM lineitem, orders WHERE lineitem.orderkey = orders.orderkey " + "AND orders.custkey < 10", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); JoinNode joinNode = searchJoins(subplan.getChildren().get(0).getFragment()).findOnlyElement(); LocalDynamicFilter filter = LocalDynamicFilter.create(joinNode, 1).orElseThrow(NoSuchElementException::new); @@ -218,7 +219,7 @@ public void testCreateDistributedJoin() .build(); SubPlan subplan = subplan( "SELECT count() FROM nation, region WHERE nation.regionkey = region.regionkey " + "AND region.comment = 'abc'", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false, session); JoinNode joinNode = searchJoins(subplan.getChildren().get(0).getFragment()).findOnlyElement(); @@ -234,7 +235,7 @@ public void testCreateMultipleCriteria() "SELECT count() FROM lineitem, partsupp " + "WHERE lineitem.partkey = partsupp.partkey AND lineitem.suppkey = partsupp.suppkey " + "AND partsupp.availqty < 10", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); JoinNode joinNode = searchJoins(subplan.getChildren().get(0).getFragment()).findOnlyElement(); LocalDynamicFilter filter = LocalDynamicFilter.create(joinNode, 1).orElseThrow(NoSuchElementException::new); @@ -263,7 +264,7 @@ public void testCreateMultipleJoins() "SELECT count() FROM lineitem, orders, part " + "WHERE lineitem.orderkey = orders.orderkey AND lineitem.partkey = part.partkey " + "AND orders.custkey < 10 AND part.name = 'abc'", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); List joinNodes = searchJoins(subplan.getChildren().get(0).getFragment()).findAll(); assertEquals(joinNodes.size(), 2); @@ -288,7 +289,7 @@ public void testCreateProbeSideUnion() "((SELECT partkey FROM part) UNION (SELECT suppkey FROM supplier)) " + "SELECT count() FROM union_table, nation WHERE union_table.key = nation.nationkey " + "AND nation.comment = 'abc'", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, true); JoinNode joinNode = searchJoins(subplan.getFragment()).findOnlyElement(); LocalDynamicFilter filter = LocalDynamicFilter.create(joinNode, 1).orElseThrow(NoSuchElementException::new); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java index db0cfc4d34e70..b984046ac98d8 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java @@ -82,7 +82,7 @@ import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.execution.TaskTestUtils.createTestingPlanner; import static com.facebook.presto.spi.StandardErrorCode.COMPILER_ERROR; -import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index b68186f3d772f..0ee411e8517ec 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -77,9 +77,9 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; import static com.facebook.presto.sql.TestExpressionInterpreter.SQUARE_UDF_CPP; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; -import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.OPTIMIZED; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.any; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyNot; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java index 5ef5824a89295..96104c9510be5 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java @@ -25,8 +25,8 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spiller.NodeSpillConfig; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.planner.LogicalPlanner; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.SubPlan; @@ -53,7 +53,7 @@ import java.util.function.Predicate; import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; -import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.CREATED; +import static com.facebook.presto.sql.Optimizer.PlanStage.CREATED; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS; import static com.google.common.base.Strings.nullToEmpty; @@ -146,20 +146,20 @@ protected LocalQueryRunner getQueryRunner() protected void assertPlan(String sql, PlanMatchPattern pattern) { - assertPlan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern); + assertPlan(sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, pattern); } protected void assertPlan(String sql, Session session, PlanMatchPattern pattern) { - assertPlan(sql, session, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern, queryRunner.getPlanOptimizers(true)); + assertPlan(sql, session, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, pattern, queryRunner.getPlanOptimizers(true)); } protected void assertPlan(String sql, Session session, PlanMatchPattern pattern, boolean forceSingleNode) { - assertPlan(sql, session, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern, queryRunner.getPlanOptimizers(forceSingleNode)); + assertPlan(sql, session, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, pattern, queryRunner.getPlanOptimizers(forceSingleNode)); } - protected void assertPlan(String sql, LogicalPlanner.Stage stage, PlanMatchPattern pattern) + protected void assertPlan(String sql, Optimizer.PlanStage stage, PlanMatchPattern pattern) { List optimizers = queryRunner.getPlanOptimizers(true); @@ -168,10 +168,10 @@ protected void assertPlan(String sql, LogicalPlanner.Stage stage, PlanMatchPatte protected void assertPlan(String sql, PlanMatchPattern pattern, List optimizers) { - assertPlan(sql, queryRunner.getDefaultSession(), LogicalPlanner.Stage.OPTIMIZED, pattern, optimizers); + assertPlan(sql, queryRunner.getDefaultSession(), Optimizer.PlanStage.OPTIMIZED, pattern, optimizers); } - protected void assertPlan(String sql, LogicalPlanner.Stage stage, PlanMatchPattern pattern, Predicate optimizerPredicate) + protected void assertPlan(String sql, Optimizer.PlanStage stage, PlanMatchPattern pattern, Predicate optimizerPredicate) { List optimizers = queryRunner.getPlanOptimizers(true).stream() .filter(optimizerPredicate) @@ -180,7 +180,7 @@ protected void assertPlan(String sql, LogicalPlanner.Stage stage, PlanMatchPatte assertPlan(sql, queryRunner.getDefaultSession(), stage, pattern, optimizers); } - protected void assertPlan(String sql, Session session, LogicalPlanner.Stage stage, PlanMatchPattern pattern, List optimizers) + protected void assertPlan(String sql, Session session, Optimizer.PlanStage stage, PlanMatchPattern pattern, List optimizers) { queryRunner.inTransaction(session, transactionSession -> { Plan actualPlan = queryRunner.createPlan( @@ -218,13 +218,13 @@ protected void assertMinimallyOptimizedPlan(@Language("SQL") String sql, PlanMat ImmutableSet.of(new RemoveRedundantIdentityProjections())), getExpressionTranslator()); // To avoid assert plan failure not printing out plan (#12885) - assertPlan(sql, queryRunner.getDefaultSession(), LogicalPlanner.Stage.OPTIMIZED, pattern, optimizers); + assertPlan(sql, queryRunner.getDefaultSession(), Optimizer.PlanStage.OPTIMIZED, pattern, optimizers); } protected void assertPlanWithSession(@Language("SQL") String sql, Session session, boolean forceSingleNode, PlanMatchPattern pattern) { queryRunner.inTransaction(session, transactionSession -> { - Plan actualPlan = queryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, forceSingleNode, WarningCollector.NOOP); + Plan actualPlan = queryRunner.createPlan(transactionSession, sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, forceSingleNode, WarningCollector.NOOP); PlanAssert.assertPlan(transactionSession, queryRunner.getMetadata(), queryRunner.getStatsCalculator(), actualPlan, pattern); return null; }); @@ -233,7 +233,7 @@ protected void assertPlanWithSession(@Language("SQL") String sql, Session sessio protected void assertPlanWithSession(@Language("SQL") String sql, Session session, boolean forceSingleNode, PlanMatchPattern pattern, Consumer planValidator) { queryRunner.inTransaction(session, transactionSession -> { - Plan actualPlan = queryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, forceSingleNode, WarningCollector.NOOP); + Plan actualPlan = queryRunner.createPlan(transactionSession, sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, forceSingleNode, WarningCollector.NOOP); PlanAssert.assertPlan(transactionSession, queryRunner.getMetadata(), queryRunner.getStatsCalculator(), actualPlan, pattern); planValidator.accept(actualPlan); return null; @@ -243,7 +243,7 @@ protected void assertPlanWithSession(@Language("SQL") String sql, Session sessio protected void assertPlanValidatorWithSession(@Language("SQL") String sql, Session session, boolean forceSingleNode, Consumer planValidator) { queryRunner.inTransaction(session, transactionSession -> { - Plan actualPlan = queryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, forceSingleNode, WarningCollector.NOOP); + Plan actualPlan = queryRunner.createPlan(transactionSession, sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, forceSingleNode, WarningCollector.NOOP); planValidator.accept(actualPlan); return null; }); @@ -274,15 +274,15 @@ protected void assertPlanSucceeded(String sql, Session session) protected Plan plan(String sql) { - return plan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED); + return plan(sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED); } - protected Plan plan(String sql, LogicalPlanner.Stage stage) + protected Plan plan(String sql, Optimizer.PlanStage stage) { return plan(sql, stage, true); } - protected Plan plan(String sql, LogicalPlanner.Stage stage, boolean forceSingleNode) + protected Plan plan(String sql, Optimizer.PlanStage stage, boolean forceSingleNode) { try { return queryRunner.inTransaction(transactionSession -> queryRunner.createPlan(transactionSession, sql, stage, forceSingleNode, WarningCollector.NOOP)); @@ -292,12 +292,12 @@ protected Plan plan(String sql, LogicalPlanner.Stage stage, boolean forceSingleN } } - protected Plan plan(String sql, LogicalPlanner.Stage stage, Session session) + protected Plan plan(String sql, Optimizer.PlanStage stage, Session session) { return plan(sql, stage, true, session); } - protected Plan plan(String sql, LogicalPlanner.Stage stage, boolean forceSingleNode, Session session) + protected Plan plan(String sql, Optimizer.PlanStage stage, boolean forceSingleNode, Session session) { try { return queryRunner.inTransaction(session, transactionSession -> queryRunner.createPlan(transactionSession, sql, stage, forceSingleNode, WarningCollector.NOOP)); @@ -307,12 +307,12 @@ protected Plan plan(String sql, LogicalPlanner.Stage stage, boolean forceSingleN } } - protected SubPlan subplan(String sql, LogicalPlanner.Stage stage, boolean forceSingleNode) + protected SubPlan subplan(String sql, Optimizer.PlanStage stage, boolean forceSingleNode) { return subplan(sql, stage, forceSingleNode, getQueryRunner().getDefaultSession()); } - protected SubPlan subplan(String sql, LogicalPlanner.Stage stage, boolean forceSingleNode, Session session) + protected SubPlan subplan(String sql, Optimizer.PlanStage stage, boolean forceSingleNode, Session session) { try { return queryRunner.inTransaction(session, transactionSession -> { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java index cdb214201ea1c..f45bc2a14b79c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java @@ -21,7 +21,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; -import com.facebook.presto.sql.planner.LogicalPlanner; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.RuleStatsRecorder; @@ -93,7 +93,7 @@ public OptimizerAssert on(String sql) checkState(plan == null, "plan has already been set"); //get an initial plan and apply a minimal set of optimizers in preparation foor applying the specific rules to be tested - Plan result = queryRunner.inTransaction(session -> queryRunner.createPlan(session, sql, getMinimalOptimizers(), LogicalPlanner.Stage.OPTIMIZED, WarningCollector.NOOP)); + Plan result = queryRunner.inTransaction(session -> queryRunner.createPlan(session, sql, getMinimalOptimizers(), Optimizer.PlanStage.OPTIMIZED, WarningCollector.NOOP)); plan = result.getRoot(); types = result.getTypes(); return this; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java index a32174a1f0a81..8a5ceaa756328 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java @@ -35,7 +35,7 @@ import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.TASK_CONCURRENCY; -import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.OPTIMIZED; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java index 29878344bcad0..803fdd076baf9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java @@ -16,7 +16,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.TopNNode; -import com.facebook.presto.sql.planner.LogicalPlanner; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.plan.ExchangeNode; @@ -53,7 +53,7 @@ public void testSimpleUnion() { Plan plan = plan( "SELECT suppkey FROM supplier UNION ALL SELECT nationkey FROM nation", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); List remotes = searchFrom(plan.getRoot()) @@ -75,7 +75,7 @@ public void testUnionUnderTopN() " SELECT nationkey FROM nation" + ") t(a) " + "ORDER BY a LIMIT 1", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); List remotes = searchFrom(plan.getRoot()) @@ -102,7 +102,7 @@ public void testUnionOverSingleNodeAggregationAndUnion() " SELECT 1 FROM nation " + " UNION ALL " + " SELECT 1 FROM nation))", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); List remotes = searchFrom(plan.getRoot()) @@ -119,7 +119,7 @@ public void testPartialAggregationsWithUnion() { Plan plan = plan( "SELECT orderstatus, sum(orderkey) FROM (SELECT orderkey, orderstatus FROM orders UNION ALL SELECT orderkey, orderstatus FROM orders) x GROUP BY (orderstatus)", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); assertAtMostOneAggregationBetweenRemoteExchanges(plan); assertPlanIsFullyDistributed(plan); @@ -130,7 +130,7 @@ public void testPartialRollupAggregationsWithUnion() { Plan plan = plan( "SELECT orderstatus, sum(orderkey) FROM (SELECT orderkey, orderstatus FROM orders UNION ALL SELECT orderkey, orderstatus FROM orders) x GROUP BY ROLLUP (orderstatus)", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); assertAtMostOneAggregationBetweenRemoteExchanges(plan); assertPlanIsFullyDistributed(plan); @@ -141,7 +141,7 @@ public void testAggregationWithUnionAndValues() { Plan plan = plan( "SELECT regionkey, count(*) FROM (SELECT regionkey FROM nation UNION ALL SELECT * FROM (VALUES 2, 100) t(regionkey)) GROUP BY regionkey", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); assertAtMostOneAggregationBetweenRemoteExchanges(plan); // TODO: Enable this check once distributed UNION can handle both partitioned and single node sources at the same time @@ -153,7 +153,7 @@ public void testUnionOnProbeSide() { Plan plan = plan( "SELECT * FROM (SELECT * FROM nation UNION ALL SELECT * from nation) n, region r WHERE n.regionkey=r.regionkey", - LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, + Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false); assertPlanIsFullyDistributed(plan); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TestFilteredAggregations.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TestFilteredAggregations.java index 08aa8f3a6e26b..2a5caa62f294b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/TestFilteredAggregations.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TestFilteredAggregations.java @@ -14,7 +14,7 @@ package com.facebook.presto.sql.query; import com.facebook.presto.spi.plan.FilterNode; -import com.facebook.presto.sql.planner.LogicalPlanner; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.google.common.collect.ImmutableMap; import org.testng.annotations.AfterClass; @@ -280,7 +280,7 @@ public void testNoFilterAddedForConstantValueFilters() private void assertPlanContainsNoFilter(String sql) { assertFalse( - searchFrom(plan(sql, LogicalPlanner.Stage.OPTIMIZED).getRoot()) + searchFrom(plan(sql, Optimizer.PlanStage.OPTIMIZED).getRoot()) .where(isInstanceOfAny(FilterNode.class)) .matches(), "Unexpected node for query: " + sql); diff --git a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java index 6a0ce40616128..53c296aa3aaa9 100644 --- a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java +++ b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java @@ -20,7 +20,7 @@ import com.facebook.presto.plugin.memory.MemoryConnectorFactory; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.sql.planner.LogicalPlanner; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.JoinNode; @@ -213,7 +213,7 @@ private void assertInvalidSpatialPartitioning(Session session, String sql, Strin LocalQueryRunner queryRunner = getQueryRunner(); try { queryRunner.inTransaction(session, transactionSession -> { - queryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false, WarningCollector.NOOP); + queryRunner.createPlan(transactionSession, sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, false, WarningCollector.NOOP); return null; }); fail(format("Expected query to fail: %s", sql)); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java index 60bedc57c645b..6511a1ccfd00f 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java @@ -25,8 +25,10 @@ import com.facebook.presto.spark.PrestoSparkPhysicalResourceCalculator; import com.facebook.presto.spark.PrestoSparkSourceStatsCollector; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; @@ -39,6 +41,7 @@ import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.PlanCanonicalInfoProvider; import com.facebook.presto.sql.planner.PlanOptimizers; +import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.plan.OutputNode; import com.facebook.presto.sql.planner.sanity.PlanChecker; import com.google.common.collect.ImmutableList; @@ -50,9 +53,11 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.common.RuntimeMetricName.LOGICAL_PLANNER_TIME_NANOS; +import static com.facebook.presto.common.RuntimeMetricName.OPTIMIZER_TIME_NANOS; +import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; import static com.facebook.presto.sql.analyzer.utils.StatementUtils.getQueryType; -import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.planner.PlanNodeCanonicalInfo.getCanonicalInfo; import static java.util.Objects.requireNonNull; @@ -105,20 +110,36 @@ public PlanAndMore createQueryPlan(Session session, BuiltInPreparedQuery prepare parameterExtractor(preparedQuery.getStatement(), preparedQuery.getParameters()), warningCollector); + Analysis analysis = analyzer.analyze(preparedQuery.getStatement()); + + final PlanVariableAllocator planVariableAllocator = new PlanVariableAllocator(); LogicalPlanner logicalPlanner = new LogicalPlanner( - false, session, - optimizers.getPlanningTimeOptimizers(), idAllocator, metadata, + planVariableAllocator); + + PlanNode planNode = session.getRuntimeStats().profileNanos( + LOGICAL_PLANNER_TIME_NANOS, + () -> logicalPlanner.plan(analysis)); + + Optimizer optimizer = new Optimizer( + session, + metadata, + optimizers.getPlanningTimeOptimizers(), + planChecker, sqlParser, + planVariableAllocator, + idAllocator, + warningCollector, statsCalculator, costCalculator, - warningCollector, - planChecker); + false); + + Plan plan = session.getRuntimeStats().profileNanos( + OPTIMIZER_TIME_NANOS, + () -> optimizer.validateAndOptimizePlan(planNode, OPTIMIZED_AND_VALIDATED)); - Analysis analysis = analyzer.analyze(preparedQuery.getStatement()); - Plan plan = logicalPlanner.plan(analysis, OPTIMIZED_AND_VALIDATED); List inputs = new InputExtractor(metadata, session).extractInputs(plan.getRoot()); Optional output = new OutputExtractor().extractOutput(plan.getRoot()); Optional queryType = getQueryType(preparedQuery.getStatement().getClass()); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java b/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java index 9fd433a10d21c..f4dd018d58edf 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java @@ -15,7 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.sql.planner.LogicalPlanner; +import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.planPrinter.PlanPrinter; import com.facebook.presto.testing.LocalQueryRunner; @@ -62,7 +62,7 @@ public void checkPlanIsDeterministic(Session session, String sql) private String getPlanText(Session session, String sql) { return localQueryRunner.inTransaction(session, transactionSession -> { - Plan plan = localQueryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, WarningCollector.NOOP); + Plan plan = localQueryRunner.createPlan(transactionSession, sql, Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED, WarningCollector.NOOP); return PlanPrinter.textLogicalPlan( plan.getRoot(), plan.getTypes(),