diff --git a/presto-main/src/main/java/com/facebook/presto/Session.java b/presto-main/src/main/java/com/facebook/presto/Session.java index f3a10056b3c0b..634912e8d7629 100644 --- a/presto-main/src/main/java/com/facebook/presto/Session.java +++ b/presto-main/src/main/java/com/facebook/presto/Session.java @@ -33,6 +33,7 @@ import com.facebook.presto.spi.session.SessionPropertyConfigurationManager.SystemSessionPropertyConfiguration; import com.facebook.presto.spi.tracing.Tracer; import com.facebook.presto.sql.planner.optimizations.OptimizerInformationCollector; +import com.facebook.presto.sql.planner.optimizations.OptimizerResultCollector; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -93,6 +94,7 @@ public final class Session private final RuntimeStats runtimeStats = new RuntimeStats(); private final OptimizerInformationCollector optimizerInformationCollector = new OptimizerInformationCollector(); + private final OptimizerResultCollector optimizerResultCollector = new OptimizerResultCollector(); public Session( QueryId queryId, @@ -319,6 +321,11 @@ public OptimizerInformationCollector getOptimizerInformationCollector() return optimizerInformationCollector; } + public OptimizerResultCollector getOptimizerResultCollector() + { + return optimizerResultCollector; + } + public Session beginTransactionId(TransactionId transactionId, TransactionManager transactionManager, AccessControl accessControl) { requireNonNull(transactionId, "transactionId is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index ca047611a6966..37bf65fe33687 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -239,6 +239,7 @@ public final class SystemSessionProperties public static final String DISTRIBUTED_TRACING_MODE = "distributed_tracing_mode"; public static final String VERBOSE_RUNTIME_STATS_ENABLED = "verbose_runtime_stats_enabled"; public static final String VERBOSE_OPTIMIZER_INFO_ENABLED = "verbose_optimizer_info_enabled"; + public static final String VERBOSE_OPTIMIZER_RESULTS = "verbose_optimizer_results"; public static final String STREAMING_FOR_PARTIAL_AGGREGATION_ENABLED = "streaming_for_partial_aggregation_enabled"; public static final String MAX_STAGE_COUNT_FOR_EAGER_SCHEDULING = "max_stage_count_for_eager_scheduling"; public static final String HYPERLOGLOG_STANDARD_ERROR_WARNING_THRESHOLD = "hyperloglog_standard_error_warning_threshold"; @@ -1302,6 +1303,16 @@ public SystemSessionProperties( "Enable logging of verbose information about applied optimizations", featuresConfig.isVerboseOptimizerInfoEnabled(), false), + /**/ + new PropertyMetadata<>( + VERBOSE_OPTIMIZER_RESULTS, + "Print result of selected optimizer(s), allowed values are ALL | NONE | [,...]", + VARCHAR, + VerboseOptimizerResultsProperty.class, + VerboseOptimizerResultsProperty.disabled(), + false, + value -> VerboseOptimizerResultsProperty.valueOf((String) value), + object -> object), booleanProperty( STREAMING_FOR_PARTIAL_AGGREGATION_ENABLED, "Enable streaming for partial aggregation", @@ -2485,6 +2496,16 @@ public static boolean isVerboseRuntimeStatsEnabled(Session session) return session.getSystemProperty(VERBOSE_RUNTIME_STATS_ENABLED, Boolean.class); } + public static boolean isVerboseOptimizerResults(Session session) + { + return session.getSystemProperty(VERBOSE_OPTIMIZER_RESULTS, VerboseOptimizerResultsProperty.class).isEnabled(); + } + + public static boolean isVerboseOptimizerResults(Session session, String optimizer) + { + return session.getSystemProperty(VERBOSE_OPTIMIZER_RESULTS, VerboseOptimizerResultsProperty.class).containsOptimizer(optimizer); + } + public static boolean isVerboseOptimizerInfoEnabled(Session session) { return session.getSystemProperty(VERBOSE_OPTIMIZER_INFO_ENABLED, Boolean.class); diff --git a/presto-main/src/main/java/com/facebook/presto/VerboseOptimizerResultsProperty.java b/presto-main/src/main/java/com/facebook/presto/VerboseOptimizerResultsProperty.java new file mode 100644 index 0000000000000..f4796d92c793d --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/VerboseOptimizerResultsProperty.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto; + +import com.google.common.base.Splitter; + +import java.util.ArrayList; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class VerboseOptimizerResultsProperty +{ + private boolean isEnabled; + private boolean isDetailedAllOptimizerResults; + private List optimizerNames; + + public VerboseOptimizerResultsProperty(boolean isEnabled, boolean isDetailedAllOptimizerResults, List optimizerNames) + { + this.isEnabled = isEnabled; + this.isDetailedAllOptimizerResults = isDetailedAllOptimizerResults; + this.optimizerNames = requireNonNull(optimizerNames, "optimizerNames is null"); + } + + public boolean isEnabled() + { + return isEnabled; + } + + public boolean isDetailedAllOptimizerResults() + { + return isDetailedAllOptimizerResults; + } + + public boolean containsOptimizer(String optimizerName) + { + return isDetailedAllOptimizerResults || optimizerNames.contains(optimizerName); + } + + public static VerboseOptimizerResultsProperty disabled() + { + return new VerboseOptimizerResultsProperty(false, false, new ArrayList<>()); + } + + public static VerboseOptimizerResultsProperty valueOf(String value) + { + if (value.equalsIgnoreCase("none")) { + return disabled(); + } + if (value.equalsIgnoreCase("all")) { + return new VerboseOptimizerResultsProperty(true, true, new ArrayList<>()); + } + + List optimizerNames = Splitter.on(',').trimResults().omitEmptyStrings().splitToList(value); + return new VerboseOptimizerResultsProperty(true, false, optimizerNames); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java index 543e79fcbae0f..4c6e8c8aff4da 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.Plan; +import com.facebook.presto.sql.planner.PlannerUtils; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; @@ -45,6 +46,7 @@ import static com.facebook.presto.SystemSessionProperties.getQueryAnalyzerTimeout; import static com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery; import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerResults; import static com.facebook.presto.common.RuntimeUnit.NANO; import static com.facebook.presto.spi.StandardErrorCode.QUERY_PLANNING_TIMEOUT; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; @@ -113,7 +115,9 @@ public Plan validateAndOptimizePlan(PlanNode root, PlanStage stage) if (enableVerboseRuntimeStats) { session.getRuntimeStats().addMetricValue(String.format("optimizer%sTimeNanos", optimizer.getClass().getSimpleName()), NANO, System.nanoTime() - start); } - collectOptimizerInformation(optimizer, root, newRoot); + TypeProvider types = TypeProvider.viewOf(variableAllocator.getVariables()); + + collectOptimizerInformation(optimizer, root, newRoot, types); root = newRoot; } } @@ -139,13 +143,14 @@ private StatsAndCosts computeStats(PlanNode root, TypeProvider types) return StatsAndCosts.empty(); } - private void collectOptimizerInformation(PlanOptimizer optimizer, PlanNode oldNode, PlanNode newNode) + private void collectOptimizerInformation(PlanOptimizer optimizer, PlanNode oldNode, PlanNode newNode, TypeProvider types) { if (optimizer instanceof IterativeOptimizer) { // iterative optimizers do their own recording of what rules got triggered return; } + String optimizerName = optimizer.getClass().getSimpleName(); boolean isTriggered = (oldNode != newNode); boolean isApplicable = isTriggered || @@ -153,7 +158,13 @@ private void collectOptimizerInformation(PlanOptimizer optimizer, PlanNode oldNo optimizer.isApplicable(oldNode, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector); if (isTriggered || isApplicable) { - session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(optimizer.getClass().getSimpleName(), isTriggered, Optional.of(isApplicable))); + session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(optimizerName, isTriggered, Optional.of(isApplicable))); + } + + if (isTriggered && isVerboseOptimizerResults(session, optimizerName)) { + String oldNodeStr = PlannerUtils.getPlanString(oldNode, session, types, metadata, false); + String newNodeStr = PlannerUtils.getPlanString(newNode, session, types, metadata, false); + session.getOptimizerResultCollector().addOptimizerResult(optimizerName, oldNodeStr, newNodeStr); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java index fd89621f3fabc..9b3b441ab5dfe 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java @@ -46,6 +46,7 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; import static com.facebook.presto.common.RuntimeMetricName.LOGICAL_PLANNER_TIME_NANOS; import static com.facebook.presto.common.RuntimeMetricName.OPTIMIZER_TIME_NANOS; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; @@ -133,7 +134,7 @@ public String getPlan(Session session, Statement statement, Type planType, List< switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); - return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session, 0, verbose); + return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session, 0, verbose, isVerboseOptimizerInfoEnabled(session)); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); return PlanPrinter.textDistributedPlan(subPlan, metadata.getFunctionAndTypeManager(), session, verbose); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 10846a6bdf8ac..7cbf2135f5d48 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -270,6 +270,7 @@ public PlanOptimizers( new PruneTableScanColumns()); IterativeOptimizer inlineProjections = new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -279,6 +280,7 @@ public PlanOptimizers( new RemoveIdentityProjectionsBelowProjection())); IterativeOptimizer projectionPushDown = new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -287,6 +289,7 @@ public PlanOptimizers( new PushProjectionThroughExchange())); IterativeOptimizer simplifyRowExpressionOptimizer = new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -296,12 +299,14 @@ public PlanOptimizers( .build()); IterativeOptimizer caseExpressionPredicateRewriter = new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, new RewriteCaseExpressionPredicate(metadata.getFunctionAndTypeManager()).rules()); IterativeOptimizer caseToMapRewriter = new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -312,6 +317,7 @@ public PlanOptimizers( builder.add( new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -321,11 +327,13 @@ public PlanOptimizers( .addAll(new SimplifyCardinalityMap(metadata.getFunctionAndTypeManager()).rules()) .build()), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new EvaluateZeroLimit())), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -358,6 +366,7 @@ public PlanOptimizers( new RewriteSpatialPartitioningAggregation(metadata))) .build()), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -367,6 +376,7 @@ public PlanOptimizers( simplifyRowExpressionOptimizer, new UnaliasSymbolReferences(metadata.getFunctionAndTypeManager()), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -377,17 +387,20 @@ public PlanOptimizers( inlineProjections, new LimitPushDown(), // Run the LimitPushDown after flattening set operators to make it easier to do the set flattening new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, columnPruningRules), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new TransformExistsApplyToLateralNode(metadata.getFunctionAndTypeManager()))), new TransformQuantifiedComparisonApplyToLateralJoin(metadata.getFunctionAndTypeManager()), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -399,6 +412,7 @@ public PlanOptimizers( new TransformCorrelatedScalarAggregationToJoin(metadata.getFunctionAndTypeManager()), new TransformCorrelatedLateralJoinToJoin(metadata.getFunctionAndTypeManager()))), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -409,6 +423,7 @@ public PlanOptimizers( new TransformCorrelatedLateralJoinToJoin(metadata.getFunctionAndTypeManager()), new ImplementFilteredAggregations(metadata.getFunctionAndTypeManager()))), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -418,6 +433,7 @@ public PlanOptimizers( new TransformCorrelatedSingleRowSubqueryToProject())), new CheckSubqueryNodesAreRewritten(), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -425,6 +441,7 @@ public PlanOptimizers( new PullConstantsAboveGroupBy()))); builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -434,6 +451,7 @@ public PlanOptimizers( // identity assignments). Hence we need to simplify projection assignments to combine/inline expressions in assignments so as to identify the candidate IF expressions. builder.add( new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -447,12 +465,14 @@ public PlanOptimizers( caseToMapRewriter, caseExpressionPredicateRewriter, new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new RewriteAggregationIfToFilter(metadata.getFunctionAndTypeManager()))), predicatePushDown, new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -461,18 +481,21 @@ public PlanOptimizers( new CrossJoinWithOrFilterToInnerJoin(metadata.getFunctionAndTypeManager()), new CrossJoinWithArrayContainsToInnerJoin(metadata.getFunctionAndTypeManager()))), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new LeftJoinNullFilterToSemiJoin(metadata.getFunctionAndTypeManager()))), new KeyBasedSampler(metadata, sqlParser), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, new PickTableLayout(metadata).rules()), new PruneUnreferencedOutputs(), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -495,6 +518,7 @@ public PlanOptimizers( new PruneUnreferencedOutputs(), // Make sure to run this before index join. Filtered projections may not have all the columns. new IndexJoinOptimizer(metadata), // Run this after projections and filters have been fully simplified and pushed down new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -503,6 +527,7 @@ public PlanOptimizers( new WindowFilterPushDown(metadata), // This must run after PredicatePushDown and LimitPushDown so that it squashes any successive filter nodes and limits prefilterForLimitingAggregation, new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -514,6 +539,7 @@ public PlanOptimizers( inlineProjections, new PruneUnreferencedOutputs(), // Make sure to run this at the end to help clean the plan for logging/execution and not remove info that other optimizers might need at an earlier point new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -521,6 +547,7 @@ public PlanOptimizers( builder.add( new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -529,6 +556,7 @@ public PlanOptimizers( simplifyRowExpressionOptimizer); // Should always run simplifyOptimizer after predicatePushDown builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -538,6 +566,7 @@ public PlanOptimizers( new PruneUnreferencedOutputs()); builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -546,6 +575,7 @@ public PlanOptimizers( // PlanRemoteProjections only handles RowExpression so this need to run after TranslateExpressions // Rules applied after this need to handle locality of ProjectNode properly. builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -564,6 +594,7 @@ public PlanOptimizers( new PruneUnreferencedOutputs()); builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -578,6 +609,7 @@ public PlanOptimizers( // This can pull up Filter and Project nodes from between Joins, so we need to push them down again builder.add( new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -587,6 +619,7 @@ public PlanOptimizers( builder.add(new OptimizeMixedDistinctAggregations(metadata)); builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -599,6 +632,7 @@ public PlanOptimizers( builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new HistoricalStatisticsEquivalentPlanMarkingOptimizer(statsCalculator))); builder.add(new IterativeOptimizer( + metadata, // Because ReorderJoins runs only once, // PredicatePushDown, PruneUnreferencedOutputs and RemoveRedundantIdentityProjections // need to run beforehand in order to produce an optimal join order @@ -614,6 +648,7 @@ public PlanOptimizers( // Run this set of join transformations after ReorderJoins, but before DetermineJoinDistributionType builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -631,6 +666,7 @@ public PlanOptimizers( new PushAggregationThroughOuterJoin(metadata.getFunctionAndTypeManager())))); builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -645,6 +681,7 @@ public PlanOptimizers( if (!forceSingleNode) { builder.add(new ReplicateSemiJoinInDelete()); // Must run before AddExchanges builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -656,6 +693,7 @@ public PlanOptimizers( builder.add(new RandomizeNullKeyInOuterJoin(metadata.getFunctionAndTypeManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -665,6 +703,7 @@ public PlanOptimizers( new RemoveRedundantIdentityProjections()))); builder.add( new IterativeOptimizer( + metadata, ruleStats, statsCalculator, estimatedExchangesCostCalculator, @@ -677,6 +716,7 @@ public PlanOptimizers( builder.add( new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -690,6 +730,7 @@ public PlanOptimizers( builder.add(new UnaliasSymbolReferences(metadata.getFunctionAndTypeManager())); // Run unalias after merging projections to simplify projections more efficiently builder.add(new PruneUnreferencedOutputs()); builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -711,6 +752,7 @@ public PlanOptimizers( // Optimizers above this do not need to care about aggregations with the type other than SINGLE // This optimizer must be run after all exchange-related optimizers builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -720,6 +762,7 @@ public PlanOptimizers( // MergePartialAggregationsWithFilter should immediately follow PushPartialAggregationThroughExchange new MergePartialAggregationsWithFilter(metadata.getFunctionAndTypeManager()), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -727,6 +770,7 @@ public PlanOptimizers( new PruneJoinColumns()))); builder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -737,6 +781,7 @@ public PlanOptimizers( builder.add( new ApplyConnectorOptimization(() -> planOptimizerManager.getOptimizers(PHYSICAL)), new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, @@ -756,6 +801,7 @@ public PlanOptimizers( // Add runtime cost-based optimizers ImmutableList.Builder runtimeBuilder = ImmutableList.builder(); runtimeBuilder.add(new IterativeOptimizer( + metadata, ruleStats, statsCalculator, costCalculator, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java index 32f9d0679b55b..74599e6faa2fb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java @@ -326,9 +326,9 @@ else if (planNode instanceof ProjectNode) { return null; } - public static String getPlanString(PlanNode planNode, Session session, TypeProvider types, Metadata metadata) + public static String getPlanString(PlanNode planNode, Session session, TypeProvider types, Metadata metadata, boolean isVerboseOptimizerInfoEnabled) { - return PlanPrinter.textLogicalPlan(planNode, types, StatsAndCosts.empty(), metadata.getFunctionAndTypeManager(), session, 0); + return PlanPrinter.textLogicalPlan(planNode, types, StatsAndCosts.empty(), metadata.getFunctionAndTypeManager(), session, 0, false, isVerboseOptimizerInfoEnabled); } private static String getNameHint(Expression expression) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java index 027a56f2c4fc4..8aa03f35252f8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java @@ -23,6 +23,7 @@ import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.matching.Match; import com.facebook.presto.matching.Matcher; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; @@ -30,12 +31,14 @@ import com.facebook.presto.spi.plan.LogicalPropertiesProvider; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.PlannerUtils; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.google.common.collect.ImmutableList; import io.airlift.units.Duration; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -55,6 +58,7 @@ public class IterativeOptimizer implements PlanOptimizer { + private final Metadata metadata; private final RuleStatsRecorder stats; private final StatsCalculator statsCalculator; private final CostCalculator costCalculator; @@ -62,23 +66,24 @@ public class IterativeOptimizer private final RuleIndex ruleIndex; private final Optional logicalPropertiesProvider; - public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, Set> rules) + public IterativeOptimizer(Metadata metadata, RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, Set> rules) { - this(stats, statsCalculator, costCalculator, ImmutableList.of(), Optional.empty(), rules); + this(metadata, stats, statsCalculator, costCalculator, ImmutableList.of(), Optional.empty(), rules); } - public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, Optional logicalPropertiesProvider, Set> rules) + public IterativeOptimizer(Metadata metadata, RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, Optional logicalPropertiesProvider, Set> rules) { - this(stats, statsCalculator, costCalculator, ImmutableList.of(), logicalPropertiesProvider, rules); + this(metadata, stats, statsCalculator, costCalculator, ImmutableList.of(), logicalPropertiesProvider, rules); } - public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, List legacyRules, Set> newRules) + public IterativeOptimizer(Metadata metadata, RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, List legacyRules, Set> newRules) { - this(stats, statsCalculator, costCalculator, legacyRules, Optional.empty(), newRules); + this(metadata, stats, statsCalculator, costCalculator, legacyRules, Optional.empty(), newRules); } - public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, List legacyRules, Optional logicalPropertiesProvider, Set> newRules) + public IterativeOptimizer(Metadata metadata, RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, List legacyRules, Optional logicalPropertiesProvider, Set> newRules) { + this.metadata = requireNonNull(metadata, "metadata is null"); this.stats = requireNonNull(stats, "stats is null"); this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator is null"); this.costCalculator = requireNonNull(costCalculator, "costCalculator is null"); @@ -122,7 +127,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Var session, TypeProvider.viewOf(variableAllocator.getVariables())); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.of(memo), session); - Context context = new Context(memo, lookup, idAllocator, variableAllocator, System.nanoTime(), timeout.toMillis(), session, warningCollector, costProvider, statsProvider); + Context context = new Context(memo, lookup, idAllocator, variableAllocator, System.nanoTime(), timeout.toMillis(), session, warningCollector, costProvider, statsProvider, metadata, types); boolean planChanged = exploreGroup(memo.getRootGroup(), context, matcher); context.collectOptimizerInformation(); if (!planChanged) { @@ -188,8 +193,8 @@ private boolean exploreNode(int group, Context context, Matcher matcher) transformedNode = transformedNode.assignStatsEquivalentPlanNode(node.getStatsEquivalentPlanNode()); } } + context.addRulesTriggered(rule.getClass().getSimpleName(), node, transformedNode); node = context.memo.replace(group, transformedNode, rule.getClass().getName()); - context.addRulesTriggered(rule.getClass().getSimpleName()); done = false; progress = true; @@ -315,6 +320,35 @@ public Optional getLogicalPropertiesProvider() }; } + private static class RuleTriggered + { + private final String rule; + private final Optional oldNode; + private final Optional newNode; + + public RuleTriggered(String rule, Optional oldNode, Optional newNode) + { + this.rule = requireNonNull(rule, "rule is null"); + this.oldNode = requireNonNull(oldNode, "oldNode is null"); + this.newNode = requireNonNull(newNode, "newNode is null"); + } + + public String getRule() + { + return rule; + } + + public Optional getOldNode() + { + return oldNode; + } + + public Optional getNewNode() + { + return newNode; + } + } + private static class Context { private final Memo memo; @@ -327,8 +361,10 @@ private static class Context private final WarningCollector warningCollector; private final CostProvider costProvider; private final StatsProvider statsProvider; - private final Set rulesTriggered; + private final List rulesTriggered; private final Set rulesApplicable; + private final Metadata metadata; + private final TypeProvider types; public Context( Memo memo, @@ -340,7 +376,9 @@ public Context( Session session, WarningCollector warningCollector, CostProvider costProvider, - StatsProvider statsProvider) + StatsProvider statsProvider, + Metadata metadata, + TypeProvider types) { checkArgument(timeoutInMilliseconds >= 0, "Timeout has to be a non-negative number [milliseconds]"); @@ -354,7 +392,9 @@ public Context( this.warningCollector = warningCollector; this.costProvider = costProvider; this.statsProvider = statsProvider; - this.rulesTriggered = new HashSet<>(); + this.metadata = metadata; + this.types = types; + this.rulesTriggered = new ArrayList<>(); this.rulesApplicable = new HashSet<>(); } @@ -365,9 +405,17 @@ public void checkTimeoutNotExhausted() } } - public void addRulesTriggered(String rule) + public void addRulesTriggered(String rule, PlanNode oldNode, PlanNode newNode) { - rulesTriggered.add(rule); + Optional before = Optional.empty(); + Optional after = Optional.empty(); + + if (SystemSessionProperties.isVerboseOptimizerResults(session, rule)) { + before = Optional.of(PlannerUtils.getPlanString(oldNode, session, types, metadata, false)); + after = Optional.of(PlannerUtils.getPlanString(newNode, session, types, metadata, false)); + } + + rulesTriggered.add(new RuleTriggered(rule, before, after)); } public void addRulesApplicable(String rule) @@ -377,7 +425,10 @@ public void addRulesApplicable(String rule) public void collectOptimizerInformation() { - rulesTriggered.forEach(x -> session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(x, true, Optional.empty()))); + rulesTriggered.stream().map(x -> x.getRule()).distinct().forEach(rule -> session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(rule, true, Optional.empty()))); + if (SystemSessionProperties.isVerboseOptimizerResults(session)) { + rulesTriggered.stream().filter(x -> x.getNewNode().isPresent()).forEach(x -> session.getOptimizerResultCollector().addOptimizerResult(x.getRule(), x.getOldNode().get(), x.getNewNode().get())); + } rulesApplicable.forEach(x -> session.getOptimizerInformationCollector().addInformation(new PlanOptimizerInformation(x, false, Optional.of(true)))); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizerResult.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizerResult.java new file mode 100644 index 0000000000000..7d029343c8ef4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizerResult.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.sql.planner.optimizations; + +import static java.util.Objects.requireNonNull; + +public class OptimizerResult +{ + private final String optimizer; + private final String oldNode; + private final String newNode; + + public OptimizerResult(String optimizer, String oldNode, String newNode) + { + this.optimizer = requireNonNull(optimizer, "rule is null"); + this.oldNode = requireNonNull(oldNode, "oldNode is null"); + this.newNode = requireNonNull(newNode, "newNode is null"); + } + + public String getOptimizer() + { + return optimizer; + } + + public String getOldNode() + { + return oldNode; + } + + public String getNewNode() + { + return newNode; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizerResultCollector.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizerResultCollector.java new file mode 100644 index 0000000000000..dc5dc85dd6aaa --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizerResultCollector.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.optimizations; + +import com.google.common.collect.ImmutableList; + +import java.util.LinkedList; +import java.util.List; + +public class OptimizerResultCollector +{ + private final List optimizerResults = new LinkedList<>(); + + public void addOptimizerResult(String optimizer, String oldNode, String newNode) + { + this.optimizerResults.add(new OptimizerResult(optimizer, oldNode, newNode)); + } + + public List getOptimizerResults() + { + return ImmutableList.copyOf(optimizerResults); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index fc82888063f3e..16bcaef9f9a53 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -180,7 +180,13 @@ private PlanPrinter( .mapToLong(planNode -> planNode.getPlanNodeScheduledTime().toMillis()) .sum(), MILLISECONDS)); - this.representation = new PlanRepresentation(planRoot, types, totalCpuTime, totalScheduledTime, session.getOptimizerInformationCollector().getOptimizationInfo()); + this.representation = new PlanRepresentation( + planRoot, + types, + totalCpuTime, + totalScheduledTime, + session.getOptimizerInformationCollector().getOptimizationInfo(), + session.getOptimizerResultCollector().getOptimizerResults()); RowExpressionFormatter rowExpressionFormatter = new RowExpressionFormatter(functionAndTypeManager); ConnectorSession connectorSession = requireNonNull(session, "session is null").toConnectorSession(); @@ -225,9 +231,10 @@ public static String textLogicalPlan( FunctionAndTypeManager functionAndTypeManager, Session session, int level, - boolean verbose) + boolean verbose, + boolean verboseOptimizerInfoEnabled) { - return textLogicalPlan(plan, types, Optional.empty(), estimatedStatsAndCosts, Optional.empty(), functionAndTypeManager, session, level, verbose); + return textLogicalPlan(plan, types, Optional.empty(), estimatedStatsAndCosts, Optional.empty(), functionAndTypeManager, session, level, verbose, verboseOptimizerInfoEnabled); } public static String textLogicalPlan( @@ -239,9 +246,10 @@ public static String textLogicalPlan( FunctionAndTypeManager functionAndTypeManager, Session session, int level, - boolean verbose) + boolean verbose, + boolean verboseOptimizerInfoEnabled) { - return new PlanPrinter(plan, types, stageExecutionStrategy, estimatedStatsAndCosts, stats, functionAndTypeManager, session).toText(verbose, level, isVerboseOptimizerInfoEnabled(session)); + return new PlanPrinter(plan, types, stageExecutionStrategy, estimatedStatsAndCosts, stats, functionAndTypeManager, session).toText(verbose, level, verboseOptimizerInfoEnabled); } public static String textDistributedPlan(StageInfo outputStageInfo, FunctionAndTypeManager functionAndTypeManager, Session session, boolean verbose) @@ -419,7 +427,8 @@ private static String formatFragment( functionAndTypeManager, session, 1, - verbose)) + verbose, + isVerboseOptimizerInfoEnabled(session))) .append("\n"); return builder.toString(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java index 94fcbbd5a5186..c247d6bf0c8cb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.optimizations.OptimizerResult; import io.airlift.units.Duration; import java.util.HashMap; @@ -35,14 +36,22 @@ class PlanRepresentation private final Map nodeInfo = new HashMap<>(); private final List planOptimizerInfo; + private final List planOptimizerResults; - public PlanRepresentation(PlanNode root, TypeProvider types, Optional totalCpuTime, Optional totalScheduledTime, List planOptimizerInfo) + public PlanRepresentation( + PlanNode root, + TypeProvider types, + Optional totalCpuTime, + Optional totalScheduledTime, + List planOptimizerInfo, + List planOptimizerResults) { this.root = requireNonNull(root, "root is null"); this.totalCpuTime = requireNonNull(totalCpuTime, "totalCpuTime is null"); this.types = requireNonNull(types, "types is null"); this.totalScheduledTime = requireNonNull(totalScheduledTime, "totalScheduledTime is null"); this.planOptimizerInfo = requireNonNull(planOptimizerInfo, "planOptimizerInfo is null"); + this.planOptimizerResults = requireNonNull(planOptimizerResults, "planOptimizerResults is null"); } public NodeRepresentation getRoot() @@ -87,4 +96,9 @@ public List getPlanOptimizerInfo() { return planOptimizerInfo; } + + public List getPlanOptimizerResults() + { + return planOptimizerResults; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java index bc40b586e6d9f..76f81a28c2af3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java @@ -16,6 +16,7 @@ import com.facebook.presto.cost.PlanCostEstimate; import com.facebook.presto.cost.PlanNodeStatsEstimate; import com.facebook.presto.spi.eventlistener.PlanOptimizerInformation; +import com.facebook.presto.sql.planner.optimizations.OptimizerResult; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; @@ -56,7 +57,9 @@ public String render(PlanRepresentation plan) if (verboseOptimizerInfo) { String optimizerInfo = optimizerInfoToText(plan.getPlanOptimizerInfo()); + String optimizerResults = optimizerResultsToText(plan.getPlanOptimizerResults()); result += optimizerInfo; + result += optimizerResults; } return result; } @@ -298,4 +301,18 @@ private String optimizerInfoToText(List planOptimizerI String.join(", ", applicableOptimizers) + "]\n"; return triggered + applicable; } + + private String optimizerResultsToText(List optimizerResults) + { + StringBuilder builder = new StringBuilder(); + + optimizerResults.forEach(opt -> { + builder.append(opt.getOptimizer() + " (before):\n"); + builder.append(opt.getOldNode() + "\n"); + builder.append(opt.getOptimizer() + " (after):\n"); + builder.append(opt.getNewNode() + "\n"); + }); + + return builder.toString(); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 0c0808b8405d1..51963e8e8ac7d 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -252,6 +252,7 @@ import static com.facebook.presto.SystemSessionProperties.getQueryMaxTotalMemoryPerNode; import static com.facebook.presto.SystemSessionProperties.isHeapDumpOnExceededMemoryLimitEnabled; import static com.facebook.presto.SystemSessionProperties.isVerboseExceededMemoryLimitErrorsEnabled; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; import static com.facebook.presto.common.RuntimeMetricName.LOGICAL_PLANNER_TIME_NANOS; import static com.facebook.presto.common.RuntimeMetricName.OPTIMIZER_TIME_NANOS; import static com.facebook.presto.cost.StatsCalculatorModule.createNewStatsCalculator; @@ -906,7 +907,7 @@ public List createDrivers(Session session, @Language("SQL") String sql, private List createDrivers(Session session, Plan plan, OutputFactory outputFactory, TaskContext taskContext) { if (printPlan) { - System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session, 0, false)); + System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session, 0, false, isVerboseOptimizerInfoEnabled(session))); } SubPlan subplan = createSubPlans(session, plan, true); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java b/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java index 5929a597ab06d..97c0f1dd55c71 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java @@ -125,6 +125,7 @@ private static Plan createPlan(LocalQueryRunner queryRunner, Session session, St { // Warnings from testing rules will be added PlanOptimizer optimizer = new IterativeOptimizer( + queryRunner.getMetadata(), new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java index 919a8c0fb45ff..adda76a469254 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java @@ -77,6 +77,7 @@ public void testDuplicatesInWindowOrderBy() ImmutableList.of( new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java index 58517c08d09a3..b2bdd7006f80a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java @@ -209,6 +209,7 @@ protected void assertMinimallyOptimizedPlan(@Language("SQL") String sql, PlanMat new UnaliasSymbolReferences(queryRunner.getMetadata().getFunctionAndTypeManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java index 99bb80c0f06f5..72d4eeade8928 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java @@ -160,11 +160,13 @@ private List getMinimalOptimizers() new UnaliasSymbolReferences(queryRunner.getMetadata().getFunctionAndTypeManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( + queryRunner.getMetadata(), new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), rules), new IterativeOptimizer( + queryRunner.getMetadata(), new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java index 2f0c2c3c756d1..48d4ff29c8140 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java @@ -72,6 +72,7 @@ public void tearDown() public void optimizerTimeoutsOnNonConvergingPlan() { PlanOptimizer optimizer = new IterativeOptimizer( + queryRunner.getMetadata(), new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java index 358b925739dfa..1a68954602654 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java @@ -51,6 +51,7 @@ import java.util.function.Function; import java.util.stream.Stream; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; import static com.facebook.presto.sql.planner.assertions.PlanAssert.assertPlan; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.textLogicalPlan; import static com.facebook.presto.transaction.TransactionBuilder.transaction; @@ -239,7 +240,7 @@ private String formatPlan(PlanNode plan, TypeProvider types) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, session); - return inTransaction(session -> textLogicalPlan(plan, types, StatsAndCosts.create(plan, statsProvider, costProvider), metadata.getFunctionAndTypeManager(), session, 2, false)); + return inTransaction(session -> textLogicalPlan(plan, types, StatsAndCosts.create(plan, statsProvider, costProvider), metadata.getFunctionAndTypeManager(), session, 2, false, isVerboseOptimizerInfoEnabled(session))); } private T inTransaction(Function transactionSessionConsumer) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java index 2f1e8be070417..0914213e6ce3a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java @@ -128,6 +128,7 @@ public RuleAssert assertThat(Rule rule, LogicalPropertiesProvider logicalPropert public OptimizerAssert assertThat(Set> rules) { PlanOptimizer optimizer = new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), @@ -138,6 +139,7 @@ public OptimizerAssert assertThat(Set> rules) public OptimizerAssert assertThat(Set> rules, LogicalPropertiesProvider logicalPropertiesProvider) { PlanOptimizer optimizer = new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java index f2b1c1591befd..fb80a31ccc7ec 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java @@ -136,6 +136,7 @@ public void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern pattern new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getCostCalculator(), @@ -145,6 +146,7 @@ public void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern pattern new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java index f048a23a3cad3..dfa2eb4a01e1b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java @@ -586,6 +586,7 @@ private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern patter List optimizers = ImmutableList.of( new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getEstimatedExchangesCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java index e6d9a3cb61a92..88b2c8bb2bb3e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java @@ -116,6 +116,7 @@ private void assertUnitPlan(String sql, PlanMatchPattern pattern) List optimizers = ImmutableList.of( new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getEstimatedExchangesCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java index 1a73a7a95de8c..f43d32d44f4ef 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java @@ -324,6 +324,7 @@ private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern patter new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser()), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getEstimatedExchangesCostCalculator(), @@ -334,6 +335,7 @@ private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern patter new GatherAndMergeWindows.SwapAdjacentWindowsBySpecifications(2))), new PruneUnreferencedOutputs(), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getEstimatedExchangesCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java index c76d5613332be..cc3c5a5cbf0ef 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java @@ -129,6 +129,7 @@ public void assertPlan(String sql, PlanMatchPattern pattern) new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( + getMetadata(), new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getEstimatedExchangesCostCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java index 9fe9945ed5378..4f0377c27a0e6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java @@ -88,6 +88,7 @@ private PlanRepresentation getPlanRepresentation(PlanNode root) TypeProvider.viewOf(VARIABLE_ALLOCATOR.getVariables()), Optional.empty(), Optional.empty(), + ImmutableList.of(), ImmutableList.of()); } diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/AdaptivePlanOptimizers.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/AdaptivePlanOptimizers.java index c186b3a164102..2466bd9002140 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/AdaptivePlanOptimizers.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/AdaptivePlanOptimizers.java @@ -52,7 +52,7 @@ public AdaptivePlanOptimizers( CostCalculator costCalculator) { this.exporter = exporter; - this.adaptiveOptimizers = ImmutableList.of(new IterativeOptimizer(ruleStats, statsCalculator, costCalculator, ImmutableSet.of(new PickJoinSides(metadata, sqlParser)))); + this.adaptiveOptimizers = ImmutableList.of(new IterativeOptimizer(metadata, ruleStats, statsCalculator, costCalculator, ImmutableSet.of(new PickJoinSides(metadata, sqlParser)))); } @PostConstruct diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java b/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java index f4dd018d58edf..f07c129ac1106 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java @@ -23,6 +23,7 @@ import java.util.function.Function; import java.util.stream.IntStream; +import static com.facebook.presto.SystemSessionProperties.isVerboseOptimizerInfoEnabled; import static org.testng.Assert.assertEquals; public class PlanDeterminismChecker @@ -70,7 +71,8 @@ private String getPlanText(Session session, String sql) localQueryRunner.getMetadata().getFunctionAndTypeManager(), transactionSession, 0, - false); + false, + isVerboseOptimizerInfoEnabled(session)); }); } } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java index a0d53f26ce515..42d7ff158ab32 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java @@ -30,10 +30,13 @@ import java.util.regex.Pattern; import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_PAYLOAD_JOINS; +import static com.facebook.presto.SystemSessionProperties.VERBOSE_OPTIMIZER_INFO_ENABLED; +import static com.facebook.presto.SystemSessionProperties.VERBOSE_OPTIMIZER_RESULTS; import static com.facebook.presto.testing.TestingSession.TESTING_CATALOG; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.google.common.collect.Iterables.getOnlyElement; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; public class TestVerboseOptimizerInfo @@ -74,7 +77,7 @@ public static LocalQueryRunner createLocalQueryRunner() public void testApplicableOptimizers() { Session session = Session.builder(getSession()) - .setSystemProperty("verbose_optimizer_info_enabled", "true") + .setSystemProperty(VERBOSE_OPTIMIZER_INFO_ENABLED, "true") .build(); String query = "SELECT o.orderkey FROM part p, orders o, lineitem l WHERE p.partkey = l.partkey AND l.orderkey = o.orderkey AND p.partkey <> o.orderkey AND p.name < l.comment"; MaterializedResult materializedResult = computeActual(session, "explain " + query); @@ -98,6 +101,29 @@ public void testApplicableOptimizers() checkOptimizerInfo(explainPayloadJoinQuery, true, ImmutableList.of("PayloadJoinOptimizer")); } + @Test + public void testVerboseOptimizerResults() + { + Session sessionPrintAll = Session.builder(getSession()) + .setSystemProperty(VERBOSE_OPTIMIZER_INFO_ENABLED, "true") + .setSystemProperty(VERBOSE_OPTIMIZER_RESULTS, "all") + .setSystemProperty(OPTIMIZE_PAYLOAD_JOINS, "true") + .build(); + String query = "SELECT l.* FROM (select *, map(ARRAY[1,3], ARRAY[2,4]) as m1 from lineitem) l left join orders o on (l.orderkey = o.orderkey) left join part p on (l.partkey=p.partkey)"; + MaterializedResult materializedResult = computeActual(sessionPrintAll, "explain " + query); + String explain = (String) getOnlyElement(materializedResult.getOnlyColumnAsSet()); + + checkOptimizerResults(explain, ImmutableList.of("PayloadJoinOptimizer", "RemoveRedundantIdentityProjections", "PruneUnreferencedOutputs"), ImmutableList.of()); + + Session sessionPrintSome = Session.builder(sessionPrintAll) + .setSystemProperty(VERBOSE_OPTIMIZER_RESULTS, "PayloadJoinOptimizer,RemoveRedundantIdentityProjections") + .build(); + materializedResult = computeActual(sessionPrintSome, "explain " + query); + explain = (String) getOnlyElement(materializedResult.getOnlyColumnAsSet()); + + checkOptimizerResults(explain, ImmutableList.of("PayloadJoinOptimizer", "RemoveRedundantIdentityProjections"), ImmutableList.of("PruneUnreferencedOutputs")); + } + private void checkOptimizerInfo(String explain, boolean checkTriggered, List optimizers) { String regex = checkTriggered ? "Triggered optimizers.*" : "Applicable optimizers.*"; @@ -110,4 +136,16 @@ private void checkOptimizerInfo(String explain, boolean checkTriggered, List includedOptimizers, List excludedOptimizers) + { + for (String opt : includedOptimizers) { + assertTrue(explain.contains(opt + " (before):")); + assertTrue(explain.contains(opt + " (after):")); + } + + for (String opt : excludedOptimizers) { + assertFalse(explain.contains(opt + " (before):")); + } + } }