From e5d3a52283454980b19407f8875c65806bac70b0 Mon Sep 17 00:00:00 2001 From: George Wang Date: Sun, 13 Nov 2022 23:41:35 -0800 Subject: [PATCH 1/3] Add estimates to graphviz of the plan --- .../java/com/facebook/presto/sql/analyzer/QueryExplainer.java | 2 +- .../facebook/presto/sql/planner/planPrinter/PlanPrinter.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java index 28fe99ef26176..ab2fe78040751 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java @@ -153,7 +153,7 @@ public String getGraphvizPlan(Session session, Statement statement, Type planTyp switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); - return graphvizLogicalPlan(plan.getRoot(), plan.getTypes(), session, metadata.getFunctionAndTypeManager()); + return graphvizLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), session, metadata.getFunctionAndTypeManager()); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); return graphvizDistributedPlan(subPlan, session, metadata.getFunctionAndTypeManager()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index 4616567985a6c..b60acfbba790c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -417,7 +417,7 @@ private static String formatFragment( return builder.toString(); } - public static String graphvizLogicalPlan(PlanNode plan, TypeProvider types, Session session, FunctionAndTypeManager functionAndTypeManager) + public static String graphvizLogicalPlan(PlanNode plan, TypeProvider types, StatsAndCosts estimatedStatsAndCosts, Session session, FunctionAndTypeManager functionAndTypeManager) { // TODO: This should move to something like GraphvizRenderer PlanFragment fragment = new PlanFragment( @@ -429,7 +429,7 @@ public static String graphvizLogicalPlan(PlanNode plan, TypeProvider types, Sess new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), plan.getOutputVariables()), StageExecutionDescriptor.ungroupedExecution(), false, - StatsAndCosts.empty(), + estimatedStatsAndCosts, Optional.empty()); return GraphvizPrinter.printLogical(ImmutableList.of(fragment), session, functionAndTypeManager); } From f300623de497a4c040c3539b6ded587478a7480f Mon Sep 17 00:00:00 2001 From: George Wang Date: Mon, 14 Nov 2022 22:30:28 -0800 Subject: [PATCH 2/3] Add estimates to json format of the plan --- .../hive/TestHiveIntegrationSmokeTest.java | 2 +- .../presto/cost/PlanNodeStatsEstimate.java | 3 + .../facebook/presto/event/QueryMonitor.java | 21 ++- .../scheduler/LegacySqlQueryScheduler.java | 6 +- .../scheduler/SqlQueryScheduler.java | 6 +- .../presto/sql/analyzer/QueryExplainer.java | 8 +- .../sql/planner/BasePlanFragmenter.java | 2 +- .../sql/planner/planPrinter/JsonRenderer.java | 70 +++++++- .../sql/planner/planPrinter/PlanPrinter.java | 64 ++++--- .../presto/testing/LocalQueryRunner.java | 2 +- .../facebook/presto/util/GraphvizPrinter.java | 38 ++-- .../sql/planner/assertions/PlanAssert.java | 4 +- .../iterative/rule/test/RuleAssert.java | 4 +- .../planner/planPrinter/TestJsonRenderer.java | 165 ++++++++++++++++++ .../presto/util/TestGraphvizPrinter.java | 14 +- .../presto/tests/PlanDeterminismChecker.java | 2 +- .../framework/ExplainVerification.java | 21 ++- 17 files changed, 359 insertions(+), 73 deletions(-) create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index fb30e590a2b40..0eae676f58656 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -4302,7 +4302,7 @@ public static Consumer assertRemoteExchangesCount(int expectedRemoteExchan .size(); if (actualRemoteExchangesCount != expectedRemoteExchangesCount) { Metadata metadata = queryRunner.getCoordinator().getMetadata(); - String formattedPlan = textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionAndTypeManager(), StatsAndCosts.empty(), session, 0); + String formattedPlan = textLogicalPlan(plan.getRoot(), plan.getTypes(), StatsAndCosts.empty(), metadata.getFunctionAndTypeManager(), session, 0); throw new AssertionError(format( "Expected [\n%s\n] remote exchanges but found [\n%s\n] remote exchanges. Actual plan is [\n\n%s\n]", expectedRemoteExchangesCount, diff --git a/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java b/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java index 5263edb393806..8cd8d225a74d7 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java @@ -24,8 +24,10 @@ import com.facebook.presto.spi.statistics.PlanStatistics; import com.facebook.presto.spi.statistics.PlanStatisticsWithSourceInfo; import com.facebook.presto.spi.statistics.SourceInfo; +import com.facebook.presto.sql.Serialization; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.ImmutableMap; import org.pcollections.HashTreePMap; import org.pcollections.PMap; @@ -194,6 +196,7 @@ public VariableStatsEstimate getVariableStatistics(VariableReferenceExpression v return variableStatistics.getOrDefault(variable, VariableStatsEstimate.unknown()); } + @JsonSerialize(keyUsing = Serialization.VariableReferenceExpressionSerializer.class) @JsonProperty public Map getVariableStatistics() { diff --git a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java index b8c31fd7f1010..2bc35655cdb75 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java @@ -75,6 +75,7 @@ import static com.facebook.presto.execution.QueryState.QUEUED; import static com.facebook.presto.execution.StageInfo.getAllStages; +import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.graphvizDistributedPlan; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonDistributedPlan; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.textDistributedPlan; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -403,7 +404,8 @@ private Optional createJsonQueryPlan(QueryInfo queryInfo) try { if (queryInfo.getOutputStage().isPresent()) { return Optional.of(jsonDistributedPlan( - queryInfo.getOutputStage().get())); + queryInfo.getOutputStage().get(), + functionAndTypeManager)); } } catch (Exception e) { @@ -413,6 +415,23 @@ private Optional createJsonQueryPlan(QueryInfo queryInfo) return Optional.empty(); } + private Optional createGraphvizQueryPlan(QueryInfo queryInfo) + { + try { + if (queryInfo.getOutputStage().isPresent()) { + return Optional.of(graphvizDistributedPlan( + queryInfo.getOutputStage().get(), + functionAndTypeManager, + queryInfo.getSession().toSession(sessionPropertyManager))); + } + } + catch (Exception e) { + // Don't fail to create event if the graphviz plan can not be created + log.warn(e, "Error creating graphviz plan for query %s: %s", queryInfo.getQueryId(), e); + } + return Optional.empty(); + } + private static QueryIOMetadata getQueryIOMetadata(QueryInfo queryInfo) { ImmutableList.Builder inputs = ImmutableList.builder(); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/LegacySqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/LegacySqlQueryScheduler.java index b3a7d061df5e9..05c4bec459f76 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/LegacySqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/LegacySqlQueryScheduler.java @@ -17,6 +17,7 @@ import com.facebook.airlift.log.Logger; import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.Session; +import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.execution.BasicStageExecutionStats; import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.PartialResultQueryManager; @@ -609,6 +610,7 @@ private Optional performRuntimeOptimizations(StreamingSubPlan subP newRoot = optimizer.optimize(newRoot, session, variableAllocator.getTypes(), variableAllocator, idAllocator, warningCollector); } if (newRoot != fragment.getRoot()) { + StatsAndCosts estimatedStatsAndCosts = fragment.getStatsAndCosts(); return Optional.of( // The partitioningScheme should stay the same // even if the root's outputVariable layout is changed. @@ -621,8 +623,8 @@ private Optional performRuntimeOptimizations(StreamingSubPlan subP fragment.getPartitioningScheme(), fragment.getStageExecutionDescriptor(), fragment.isOutputTableWriterFragment(), - fragment.getStatsAndCosts(), - Optional.of(jsonFragmentPlan(newRoot, fragment.getVariables(), functionAndTypeManager, session)))); + estimatedStatsAndCosts, + Optional.of(jsonFragmentPlan(newRoot, fragment.getVariables(), estimatedStatsAndCosts, functionAndTypeManager, session)))); } return Optional.empty(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java index 760d0ab46bf62..e7f2b21089b5a 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java @@ -17,6 +17,7 @@ import com.facebook.airlift.log.Logger; import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.Session; +import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.execution.BasicStageExecutionStats; import com.facebook.presto.execution.ExecutionFailureInfo; import com.facebook.presto.execution.LocationFactory; @@ -497,6 +498,7 @@ private Optional performRuntimeOptimizations(StreamingSubPlan subP newRoot = optimizer.optimize(newRoot, session, variableAllocator.getTypes(), variableAllocator, idAllocator, warningCollector); } if (newRoot != fragment.getRoot()) { + StatsAndCosts estimatedStatsAndCosts = fragment.getStatsAndCosts(); return Optional.of( // The partitioningScheme should stay the same // even if the root's outputVariable layout is changed. @@ -509,8 +511,8 @@ private Optional performRuntimeOptimizations(StreamingSubPlan subP fragment.getPartitioningScheme(), fragment.getStageExecutionDescriptor(), fragment.isOutputTableWriterFragment(), - fragment.getStatsAndCosts(), - Optional.of(jsonFragmentPlan(newRoot, fragment.getVariables(), functionAndTypeManager, session)))); + estimatedStatsAndCosts, + Optional.of(jsonFragmentPlan(newRoot, fragment.getVariables(), estimatedStatsAndCosts, functionAndTypeManager, session)))); } return Optional.empty(); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java index ab2fe78040751..b4fd9ceb10200 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java @@ -127,7 +127,7 @@ public String getPlan(Session session, Statement statement, Type planType, List< switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); - return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionAndTypeManager(), plan.getStatsAndCosts(), session, 0, verbose); + return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session, 0, verbose); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); return PlanPrinter.textDistributedPlan(subPlan, metadata.getFunctionAndTypeManager(), session, verbose); @@ -153,10 +153,10 @@ public String getGraphvizPlan(Session session, Statement statement, Type planTyp switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); - return graphvizLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), session, metadata.getFunctionAndTypeManager()); + return graphvizLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); - return graphvizDistributedPlan(subPlan, session, metadata.getFunctionAndTypeManager()); + return graphvizDistributedPlan(subPlan, metadata.getFunctionAndTypeManager(), session); } throw new IllegalArgumentException("Unhandled plan type: " + planType); } @@ -179,7 +179,7 @@ public String getJsonPlan(Session session, Statement statement, Type planType, L return jsonLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionAndTypeManager(), plan.getStatsAndCosts(), session); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); - return jsonDistributedPlan(subPlan); + return jsonDistributedPlan(subPlan, metadata.getFunctionAndTypeManager()); default: throw new PrestoException(NOT_SUPPORTED, format("Unsupported explain plan type %s for JSON format", planType)); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java index c68c89230a36c..8aceee5628086 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java @@ -193,7 +193,7 @@ private SubPlan buildFragment(PlanNode root, FragmentProperties properties, Plan StageExecutionDescriptor.ungroupedExecution(), outputTableWriterFragment, statsAndCosts.getForSubplan(root), - Optional.of(jsonFragmentPlan(root, fragmentVariableTypes, metadata.getFunctionAndTypeManager(), session))); + Optional.of(jsonFragmentPlan(root, fragmentVariableTypes, statsAndCosts.getForSubplan(root), metadata.getFunctionAndTypeManager(), session))); return new SubPlan(fragment, properties.getChildren()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java index e57079b039ad8..d622275940790 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java @@ -14,14 +14,23 @@ package com.facebook.presto.sql.planner.planPrinter; import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.JsonObjectMapperProvider; +import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.spi.SourceLocation; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.Serialization; import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRawValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -30,21 +39,33 @@ public class JsonRenderer implements Renderer { - private static final JsonCodec CODEC = JsonCodec.jsonCodec(JsonRenderedNode.class); - private static final JsonCodec> PLAN_MAP_CODEC = JsonCodec.mapJsonCodec(PlanFragmentId.class, JsonPlanFragment.class); + private static JsonCodec codec = JsonCodec.jsonCodec(JsonRenderedNode.class); + private static JsonCodec> planMapCodec = JsonCodec.mapJsonCodec(PlanFragmentId.class, JsonPlanFragment.class); + + public JsonRenderer(FunctionAndTypeManager functionAndTypeManager) + { + JsonObjectMapperProvider provider = new JsonObjectMapperProvider(); + provider.setJsonSerializers(ImmutableMap.of(VariableReferenceExpression.class, new Serialization.VariableReferenceExpressionSerializer())); + provider.setKeyDeserializers(ImmutableMap.of(VariableReferenceExpression.class, new Serialization.VariableReferenceExpressionDeserializer(functionAndTypeManager))); + + JsonCodecFactory codecFactory = new JsonCodecFactory(provider, true); + this.codec = codecFactory.jsonCodec(JsonRenderedNode.class); + this.planMapCodec = codecFactory.mapJsonCodec(PlanFragmentId.class, JsonPlanFragment.class); + } @Override public String render(PlanRepresentation plan) { - return CODEC.toJson(renderJson(plan, plan.getRoot())); + return codec.toJson(renderJson(plan, plan.getRoot())); } public String render(Map fragmentJsonMap) { - return PLAN_MAP_CODEC.toJson(fragmentJsonMap); + return planMapCodec.toJson(fragmentJsonMap); } - private JsonRenderedNode renderJson(PlanRepresentation plan, NodeRepresentation node) + @VisibleForTesting + public JsonRenderedNode renderJson(PlanRepresentation plan, NodeRepresentation node) { List children = node.getChildren().stream() .map(plan::getNode) @@ -62,7 +83,8 @@ private JsonRenderedNode renderJson(PlanRepresentation plan, NodeRepresentation children, node.getRemoteSources().stream() .map(PlanFragmentId::toString) - .collect(toImmutableList())); + .collect(toImmutableList()), + node.getEstimatedStats()); } public static class JsonRenderedNode @@ -74,9 +96,10 @@ public static class JsonRenderedNode private final String details; private final List children; private final List remoteSources; + private final List estimates; @JsonCreator - public JsonRenderedNode(Optional sourceLocation, String id, String name, String identifier, String details, List children, List remoteSources) + public JsonRenderedNode(Optional sourceLocation, String id, String name, String identifier, String details, List children, List remoteSources, List estimates) { this.sourceLocation = sourceLocation; this.id = requireNonNull(id, "id is null"); @@ -85,6 +108,7 @@ public JsonRenderedNode(Optional sourceLocation, String id, Stri this.details = requireNonNull(details, "details is null"); this.children = requireNonNull(children, "children is null"); this.remoteSources = requireNonNull(remoteSources, "id is null"); + this.estimates = requireNonNull(estimates, "estimate is null"); } @JsonProperty @@ -128,6 +152,38 @@ public List getRemoteSources() { return remoteSources; } + + @JsonProperty + public List getEstimates() + { + return estimates; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + JsonRenderedNode other = (JsonRenderedNode) obj; + return Objects.equals(this.name, other.name) && + Objects.equals(this.id, other.id) && + Objects.equals(this.sourceLocation, other.sourceLocation) && + Objects.equals(this.identifier, other.identifier) && + Objects.equals(this.details, other.details) && + Objects.equals(this.children, other.children) && + Objects.equals(this.estimates, other.estimates) && + Objects.equals(this.remoteSources, other.remoteSources); + } + + @Override + public int hashCode() + { + return Objects.hash(name, id, sourceLocation, identifier, details, children, estimates, remoteSources); + } } public static class JsonPlanFragment diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index b60acfbba790c..a17fb4e8715c4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -112,6 +112,7 @@ import io.airlift.units.Duration; import java.util.ArrayList; +import java.util.Comparator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -132,6 +133,7 @@ import static com.facebook.presto.sql.planner.planPrinter.TextRenderer.formatDouble; import static com.facebook.presto.sql.planner.planPrinter.TextRenderer.formatPositions; import static com.facebook.presto.sql.planner.planPrinter.TextRenderer.indentString; +import static com.facebook.presto.util.GraphvizPrinter.printDistributedFromFragments; import static com.google.common.base.CaseFormat.UPPER_UNDERSCORE; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -153,10 +155,10 @@ private PlanPrinter( PlanNode planRoot, TypeProvider types, Optional stageExecutionStrategy, - FunctionAndTypeManager functionAndTypeManager, StatsAndCosts estimatedStatsAndCosts, - Session session, - Optional> stats) + Optional> stats, + FunctionAndTypeManager functionAndTypeManager, + Session session) { requireNonNull(planRoot, "planRoot is null"); requireNonNull(types, "types is null"); @@ -195,45 +197,45 @@ public String toText(boolean verbose, int level) public String toJson() { - return new JsonRenderer().render(representation); + return new JsonRenderer(functionAndTypeManager).render(representation); } - public static String jsonFragmentPlan(PlanNode root, Set variables, FunctionAndTypeManager functionAndTypeManager, Session session) + public static String jsonFragmentPlan(PlanNode root, Set variables, StatsAndCosts estimatedStatsAndCosts, FunctionAndTypeManager functionAndTypeManager, Session session) { TypeProvider typeProvider = TypeProvider.fromVariables(variables); - return new PlanPrinter(root, typeProvider, Optional.empty(), functionAndTypeManager, StatsAndCosts.empty(), session, Optional.empty()).toJson(); + return new PlanPrinter(root, typeProvider, Optional.empty(), estimatedStatsAndCosts, Optional.empty(), functionAndTypeManager, session).toJson(); } - public static String textLogicalPlan(PlanNode plan, TypeProvider types, FunctionAndTypeManager functionAndTypeManager, StatsAndCosts estimatedStatsAndCosts, Session session, int level) + public static String textLogicalPlan(PlanNode plan, TypeProvider types, StatsAndCosts estimatedStatsAndCosts, FunctionAndTypeManager functionAndTypeManager, Session session, int level) { - return new PlanPrinter(plan, types, Optional.empty(), functionAndTypeManager, estimatedStatsAndCosts, session, Optional.empty()).toText(false, level); + return new PlanPrinter(plan, types, Optional.empty(), estimatedStatsAndCosts, Optional.empty(), functionAndTypeManager, session).toText(false, level); } public static String textLogicalPlan( PlanNode plan, TypeProvider types, - FunctionAndTypeManager functionAndTypeManager, StatsAndCosts estimatedStatsAndCosts, + FunctionAndTypeManager functionAndTypeManager, Session session, int level, boolean verbose) { - return textLogicalPlan(plan, types, Optional.empty(), functionAndTypeManager, estimatedStatsAndCosts, session, Optional.empty(), level, verbose); + return textLogicalPlan(plan, types, Optional.empty(), estimatedStatsAndCosts, Optional.empty(), functionAndTypeManager, session, level, verbose); } public static String textLogicalPlan( PlanNode plan, TypeProvider types, Optional stageExecutionStrategy, - FunctionAndTypeManager functionAndTypeManager, StatsAndCosts estimatedStatsAndCosts, - Session session, Optional> stats, + FunctionAndTypeManager functionAndTypeManager, + Session session, int level, boolean verbose) { - return new PlanPrinter(plan, types, stageExecutionStrategy, functionAndTypeManager, estimatedStatsAndCosts, session, stats).toText(verbose, level); + return new PlanPrinter(plan, types, stageExecutionStrategy, estimatedStatsAndCosts, stats, functionAndTypeManager, session).toText(verbose, level); } public static String textDistributedPlan(StageInfo outputStageInfo, FunctionAndTypeManager functionAndTypeManager, Session session, boolean verbose) @@ -300,21 +302,21 @@ public static String jsonLogicalPlan( Session session, Optional> stats) { - return new PlanPrinter(plan, types, stageExecutionStrategy, functionAndTypeManager, estimatedStatsAndCosts, session, stats).toJson(); + return new PlanPrinter(plan, types, stageExecutionStrategy, estimatedStatsAndCosts, stats, functionAndTypeManager, session).toJson(); } - public static String jsonDistributedPlan(StageInfo outputStageInfo) + public static String jsonDistributedPlan(StageInfo outputStageInfo, FunctionAndTypeManager functionAndTypeManager) { List allFragments = getAllStages(Optional.of(outputStageInfo)).stream() .map(StageInfo::getPlan) .map(Optional::get) .collect(toImmutableList()); - return formatJsonFragmentList(allFragments); + return formatJsonFragmentList(allFragments, functionAndTypeManager); } - public static String jsonDistributedPlan(SubPlan plan) + public static String jsonDistributedPlan(SubPlan plan, FunctionAndTypeManager functionAndTypeManager) { - return formatJsonFragmentList(plan.getAllFragments()); + return formatJsonFragmentList(plan.getAllFragments(), functionAndTypeManager); } private String formatSourceLocation(Optional sourceLocation1, Optional sourceLocation2) @@ -335,7 +337,7 @@ private String formatSourceLocation(Optional sourceLocation) return ""; } - private static String formatJsonFragmentList(List fragments) + private static String formatJsonFragmentList(List fragments, FunctionAndTypeManager functionAndTypeManager) { ImmutableSortedMap.Builder fragmentJsonMap = ImmutableSortedMap.naturalOrder(); for (PlanFragment fragment : fragments) { @@ -343,7 +345,7 @@ private static String formatJsonFragmentList(List fragments) JsonPlanFragment jsonPlanFragment = new JsonPlanFragment(fragment.getJsonRepresentation().get()); fragmentJsonMap.put(fragmentId, jsonPlanFragment); } - return new JsonRenderer().render(fragmentJsonMap.build()); + return new JsonRenderer(functionAndTypeManager).render(fragmentJsonMap.build()); } private static String formatFragment( @@ -406,10 +408,10 @@ private static String formatFragment( fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), - functionAndTypeManager, fragment.getStatsAndCosts(), - session, planNodeStats, + functionAndTypeManager, + session, 1, verbose)) .append("\n"); @@ -417,7 +419,7 @@ private static String formatFragment( return builder.toString(); } - public static String graphvizLogicalPlan(PlanNode plan, TypeProvider types, StatsAndCosts estimatedStatsAndCosts, Session session, FunctionAndTypeManager functionAndTypeManager) + public static String graphvizLogicalPlan(PlanNode plan, TypeProvider types, StatsAndCosts estimatedStatsAndCosts, FunctionAndTypeManager functionAndTypeManager, Session session) { // TODO: This should move to something like GraphvizRenderer PlanFragment fragment = new PlanFragment( @@ -431,12 +433,22 @@ public static String graphvizLogicalPlan(PlanNode plan, TypeProvider types, Stat false, estimatedStatsAndCosts, Optional.empty()); - return GraphvizPrinter.printLogical(ImmutableList.of(fragment), session, functionAndTypeManager); + return GraphvizPrinter.printLogical(ImmutableList.of(fragment), functionAndTypeManager, session); } - public static String graphvizDistributedPlan(SubPlan plan, Session session, FunctionAndTypeManager functionAndTypeManager) + public static String graphvizDistributedPlan(SubPlan plan, FunctionAndTypeManager functionAndTypeManager, Session session) { - return GraphvizPrinter.printDistributed(plan, session, functionAndTypeManager); + return GraphvizPrinter.printDistributed(plan, functionAndTypeManager, session); + } + + public static String graphvizDistributedPlan(StageInfo stageInfo, FunctionAndTypeManager functionAndTypeManager, Session session) + { + List allFragments = getAllStages(Optional.of(stageInfo)).stream() + .map(StageInfo::getPlan) + .map(Optional::get) + .sorted(Comparator.comparing(PlanFragment::getId)) + .collect(toImmutableList()); + return printDistributedFromFragments(allFragments, functionAndTypeManager, session); } private class Visitor diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 8cf9f440d4d0f..0de60ce2de35a 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -838,7 +838,7 @@ public List createDrivers(Session session, @Language("SQL") String sql, private List createDrivers(Session session, Plan plan, OutputFactory outputFactory, TaskContext taskContext) { if (printPlan) { - System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionAndTypeManager(), plan.getStatsAndCosts(), session, 0, false)); + System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session, 0, false)); } SubPlan subplan = createSubPlans(session, plan, true); diff --git a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java index 813398229379c..6549d718b2308 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java @@ -149,7 +149,7 @@ private enum NodeType private GraphvizPrinter() {} - public static String printLogical(List fragments, Session session, FunctionAndTypeManager functionAndTypeManager) + public static String printLogical(List fragments, FunctionAndTypeManager functionAndTypeManager, Session session) { Map fragmentsById = Maps.uniqueIndex(fragments, PlanFragment::getId); PlanNodeIdGenerator idGenerator = new PlanNodeIdGenerator(); @@ -158,7 +158,7 @@ public static String printLogical(List fragments, Session session, output.append("digraph logical_plan {\n"); for (PlanFragment fragment : fragments) { - printFragmentNodes(output, fragment, idGenerator, session, functionAndTypeManager); + printFragmentNodes(output, fragment, idGenerator, functionAndTypeManager, session); } for (PlanFragment fragment : fragments) { @@ -170,7 +170,7 @@ public static String printLogical(List fragments, Session session, return output.toString(); } - public static String printDistributed(SubPlan plan, Session session, FunctionAndTypeManager functionAndTypeManager) + public static String printDistributed(SubPlan plan, FunctionAndTypeManager functionAndTypeManager, Session session) { List fragments = plan.getAllFragments(); Map fragmentsById = Maps.uniqueIndex(fragments, PlanFragment::getId); @@ -179,31 +179,47 @@ public static String printDistributed(SubPlan plan, Session session, FunctionAnd StringBuilder output = new StringBuilder(); output.append("digraph distributed_plan {\n"); - printSubPlan(plan, fragmentsById, idGenerator, output, session, functionAndTypeManager); + printSubPlan(plan, fragmentsById, idGenerator, output, functionAndTypeManager, session); output.append("}\n"); return output.toString(); } + public static String printDistributedFromFragments(List allFragments, FunctionAndTypeManager functionAndTypeManager, Session session) + { + PlanNodeIdGenerator idGenerator = new PlanNodeIdGenerator(); + Map fragmentsById = Maps.uniqueIndex(allFragments, PlanFragment::getId); + + StringBuilder output = new StringBuilder(); + output.append("digraph distributed_plan {\n"); + + for (PlanFragment planFragment : allFragments) { + printFragmentNodes(output, planFragment, idGenerator, functionAndTypeManager, session); + planFragment.getRoot().accept(new EdgePrinter(output, fragmentsById, idGenerator), null); + } + + output.append("}\n"); + return output.toString(); + } private static void printSubPlan( SubPlan plan, Map fragmentsById, PlanNodeIdGenerator idGenerator, StringBuilder output, - Session session, - FunctionAndTypeManager functionAndTypeManager) + FunctionAndTypeManager functionAndTypeManager, + Session session) { PlanFragment fragment = plan.getFragment(); - printFragmentNodes(output, fragment, idGenerator, session, functionAndTypeManager); + printFragmentNodes(output, fragment, idGenerator, functionAndTypeManager, session); fragment.getRoot().accept(new EdgePrinter(output, fragmentsById, idGenerator), null); for (SubPlan child : plan.getChildren()) { - printSubPlan(child, fragmentsById, idGenerator, output, session, functionAndTypeManager); + printSubPlan(child, fragmentsById, idGenerator, output, functionAndTypeManager, session); } } - private static void printFragmentNodes(StringBuilder output, PlanFragment fragment, PlanNodeIdGenerator idGenerator, Session session, FunctionAndTypeManager functionAndTypeManager) + private static void printFragmentNodes(StringBuilder output, PlanFragment fragment, PlanNodeIdGenerator idGenerator, FunctionAndTypeManager functionAndTypeManager, Session session) { String clusterId = "cluster_" + fragment.getId(); output.append("subgraph ") @@ -215,7 +231,7 @@ private static void printFragmentNodes(StringBuilder output, PlanFragment fragme .append('\n'); PlanNode plan = fragment.getRoot(); - plan.accept(new NodePrinter(output, idGenerator, session, functionAndTypeManager, fragment.getStatsAndCosts()), null); + plan.accept(new NodePrinter(output, idGenerator, fragment.getStatsAndCosts(), functionAndTypeManager, session), null); output.append("}") .append('\n'); @@ -230,7 +246,7 @@ private static class NodePrinter private final Function formatter; StatsAndCosts estimatedStatsAndCosts; - public NodePrinter(StringBuilder output, PlanNodeIdGenerator idGenerator, Session session, FunctionAndTypeManager functionAndTypeManager, StatsAndCosts estimatedStatsAndCosts) + public NodePrinter(StringBuilder output, PlanNodeIdGenerator idGenerator, StatsAndCosts estimatedStatsAndCosts, FunctionAndTypeManager functionAndTypeManager, Session session) { this.output = output; this.idGenerator = idGenerator; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java index 79aa47c1e187f..194839b290210 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java @@ -52,7 +52,7 @@ public static void assertPlan(Session session, Metadata metadata, StatsProvider // TODO (Issue #13231) add back printing unresolved plan once we have no need to translate OriginalExpression to RowExpression if (!matches.isMatch()) { PlanNode resolvedPlan = resolveGroupReferences(actual.getRoot(), lookup); - String resolvedFormattedPlan = textLogicalPlan(planSanitizer.apply(resolvedPlan), actual.getTypes(), metadata.getFunctionAndTypeManager(), StatsAndCosts.empty(), session, 0); + String resolvedFormattedPlan = textLogicalPlan(planSanitizer.apply(resolvedPlan), actual.getTypes(), StatsAndCosts.empty(), metadata.getFunctionAndTypeManager(), session, 0); throw new AssertionError(format( "Plan does not match, expected [\n\n%s\n] but found [\n\n%s\n]", pattern, @@ -66,7 +66,7 @@ public static void assertPlanDoesNotMatch(Session session, Metadata metadata, St // TODO (Issue #13231) add back printing unresolved plan once we have no need to translate OriginalExpression to RowExpression if (matches.isMatch()) { PlanNode resolvedPlan = resolveGroupReferences(actual.getRoot(), lookup); - String resolvedFormattedPlan = textLogicalPlan(planSanitizer.apply(resolvedPlan), actual.getTypes(), metadata.getFunctionAndTypeManager(), StatsAndCosts.empty(), session, 0); + String resolvedFormattedPlan = textLogicalPlan(planSanitizer.apply(resolvedPlan), actual.getTypes(), StatsAndCosts.empty(), metadata.getFunctionAndTypeManager(), session, 0); throw new AssertionError(format( "Plan unexpectedly matches the pattern, pattern is [\n\n%s\n] and plan is [\n\n%s\n]", pattern, diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java index 7978c3a21a122..fc1e760968ccd 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java @@ -147,7 +147,7 @@ public void doesNotFire() fail(String.format( "Expected %s to not fire for:\n%s", rule.getClass().getName(), - inTransaction(session -> textLogicalPlan(plan, ruleApplication.types, metadata.getFunctionAndTypeManager(), StatsAndCosts.empty(), session, 2)))); + inTransaction(session -> textLogicalPlan(plan, ruleApplication.types, StatsAndCosts.empty(), metadata.getFunctionAndTypeManager(), session, 2)))); } } @@ -243,7 +243,7 @@ private String formatPlan(PlanNode plan, TypeProvider types) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, session); - return inTransaction(session -> textLogicalPlan(translateExpressions(plan, types), types, metadata.getFunctionAndTypeManager(), StatsAndCosts.create(plan, statsProvider, costProvider), session, 2, false)); + return inTransaction(session -> textLogicalPlan(translateExpressions(plan, types), types, StatsAndCosts.create(plan, statsProvider, costProvider), metadata.getFunctionAndTypeManager(), session, 2, false)); } private T inTransaction(Function transactionSessionConsumer) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java new file mode 100644 index 0000000000000..e886124dad606 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/planPrinter/TestJsonRenderer.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.planPrinter; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.JsonObjectMapperProvider; +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.Serialization; +import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.testing.TestingHandle; +import com.facebook.presto.testing.TestingMetadata; +import com.facebook.presto.testing.TestingTransactionHandle; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.metadata.AbstractMockMetadata.dummyMetadata; +import static com.facebook.presto.testing.TestingEnvironment.FUNCTION_AND_TYPE_MANAGER; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.testng.Assert.assertEquals; + +public class TestJsonRenderer +{ + private static final PlanBuilder PLAN_BUILDER = new PlanBuilder(TEST_SESSION, new PlanNodeIdAllocator(), dummyMetadata()); + private static final VariableReferenceExpression COLUMN_VARIABLE = new VariableReferenceExpression(Optional.empty(), "column", VARCHAR); + private static final ColumnHandle COLUMN_HANDLE = new TestingMetadata.TestingColumnHandle("column"); + private static final PlanVariableAllocator VARIABLE_ALLOCATOR = new PlanVariableAllocator(); + private static final JsonRenderer JSON_RENDERER = new JsonRenderer(FUNCTION_AND_TYPE_MANAGER); + private static final TableHandle TABLE_HANDLE_WITH_LAYOUT = new TableHandle( + new ConnectorId("testConnector"), + new TestingMetadata.TestingTableHandle(), + TestingTransactionHandle.create(), + Optional.of(TestingHandle.INSTANCE)); + private static final JsonCodec PLAN_CODEC; + + static { + JsonObjectMapperProvider provider = new JsonObjectMapperProvider(); + provider.setKeyDeserializers(ImmutableMap.of(VariableReferenceExpression.class, new Serialization.VariableReferenceExpressionDeserializer(FUNCTION_AND_TYPE_MANAGER))); + JsonCodecFactory codecFactory = new JsonCodecFactory(provider, true); + PLAN_CODEC = codecFactory.jsonCodec(JsonRenderer.JsonRenderedNode.class); + } + + @Test + public void testRoundtrip() + { + Map mapper = buildSimpleNodePlan(); + verify(mapper); + mapper = buildMultiNodePlan(); + verify(mapper); + } + + private PlanRepresentation getPlanRepresentation(PlanNode root) + { + return new PlanRepresentation( + root, + VARIABLE_ALLOCATOR.getTypes(), + Optional.empty(), + Optional.empty()); + } + + private NodeRepresentation getNodeRepresentation(PlanNode root, List planNodeIds) + { + return new NodeRepresentation( + Optional.empty(), + root.getId(), + root.getClass().getName(), + root.getClass().getSimpleName(), + "", + root.getOutputVariables(), + Optional.empty(), + ImmutableList.of(), + ImmutableList.of(), + planNodeIds, + ImmutableList.of()); + } + + private Map buildPlanResult(PlanNode root) + { + ImmutableMap.Builder result = ImmutableMap.builder(); + List childrenIds = root.getSources().stream().map(PlanNode::getId).collect(toImmutableList()); + PlanRepresentation key = getPlanRepresentation(root); + NodeRepresentation nodeRepresentation = getNodeRepresentation(root, childrenIds); + key.addNode(nodeRepresentation); + // deserialize string to json + JsonRenderer.JsonRenderedNode value = JSON_RENDERER.renderJson(key, nodeRepresentation); + // put json as value for verify + result.put(key, value); + return result.build(); + } + + private Map buildSimpleNodePlan() + { + TableScanNode scanNode = PLAN_BUILDER.tableScan( + TABLE_HANDLE_WITH_LAYOUT, + ImmutableList.of(COLUMN_VARIABLE), + ImmutableMap.of(COLUMN_VARIABLE, COLUMN_HANDLE), + TupleDomain.all(), + TupleDomain.all()); + + List childrenIds = scanNode.getSources().stream().map(PlanNode::getId).collect(toImmutableList()); + PlanRepresentation key = getPlanRepresentation(scanNode); + NodeRepresentation nodeRepresentation = getNodeRepresentation(scanNode, childrenIds); + key.addNode(nodeRepresentation); + + return buildPlanResult(scanNode); + } + + private Map buildMultiNodePlan() + { + ImmutableMap.Builder result = ImmutableMap.builder(); + ImmutableMap map = ImmutableMap.of( + COLUMN_VARIABLE, + COLUMN_VARIABLE); + TableScanNode scan = PLAN_BUILDER.tableScan( + TABLE_HANDLE_WITH_LAYOUT, + ImmutableList.of(COLUMN_VARIABLE), + ImmutableMap.of(COLUMN_VARIABLE, COLUMN_HANDLE)); + LimitNode limit = PLAN_BUILDER.limit(10, scan); + ProjectNode root = PLAN_BUILDER.project(new Assignments(map), limit); + + return buildPlanResult(root); + } + + private void verify(Map mapper) + { + for (PlanRepresentation key : mapper.keySet()) { + // serialize json to string + String originKey = JSON_RENDERER.render(key); + JsonRenderer.JsonRenderedNode result = PLAN_CODEC.fromJson(originKey); + JsonRenderer.JsonRenderedNode originValue = mapper.get(key); + assertEquals(result, originValue); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java b/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java index 26466ad7f6161..51995db86d0e8 100644 --- a/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java +++ b/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java @@ -39,10 +39,10 @@ import java.util.Collections; import java.util.Optional; -import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; import static com.facebook.presto.operator.StageExecutionDescriptor.ungroupedExecution; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; +import static com.facebook.presto.testing.TestingEnvironment.FUNCTION_AND_TYPE_MANAGER; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.util.GraphvizPrinter.printDistributed; import static com.facebook.presto.util.GraphvizPrinter.printLogical; @@ -74,8 +74,8 @@ public void testPrintLogical() { String actual = printLogical( ImmutableList.of(createTestPlanFragment(0, TEST_TABLE_SCAN_NODE)), - testSessionBuilder().build(), - createTestFunctionAndTypeManager()); + FUNCTION_AND_TYPE_MANAGER, + testSessionBuilder().build()); String expected = join( System.lineSeparator(), "digraph logical_plan {", @@ -99,8 +99,8 @@ public void testPrintDistributed() ImmutableList.of(tableScanNodeSubPlan)); String actualNestedSubPlan = printDistributed( nestedSubPlan, - testSessionBuilder().build(), - createTestFunctionAndTypeManager()); + FUNCTION_AND_TYPE_MANAGER, + testSessionBuilder().build()); String expectedNestedSubPlan = join( System.lineSeparator(), "digraph distributed_plan {", @@ -141,8 +141,8 @@ public void testPrintLogicalForJoinNode() String actual = printLogical( ImmutableList.of(createTestPlanFragment(0, node)), - testSessionBuilder().build(), - createTestFunctionAndTypeManager()); + FUNCTION_AND_TYPE_MANAGER, + testSessionBuilder().build()); String expected = "digraph logical_plan {\n" + "subgraph cluster_0 {\n" + diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java b/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java index 2e67625c12cea..9fd433a10d21c 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/PlanDeterminismChecker.java @@ -66,8 +66,8 @@ private String getPlanText(Session session, String sql) return PlanPrinter.textLogicalPlan( plan.getRoot(), plan.getTypes(), - localQueryRunner.getMetadata().getFunctionAndTypeManager(), plan.getStatsAndCosts(), + localQueryRunner.getMetadata().getFunctionAndTypeManager(), transactionSession, 0, false); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainVerification.java index 6d266437d17dd..6cd36ebd33c7e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ExplainVerification.java @@ -14,6 +14,10 @@ package com.facebook.presto.verifier.framework; import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.JsonObjectMapperProvider; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.Serialization; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.planPrinter.JsonRenderer.JsonRenderedNode; import com.facebook.presto.sql.tree.Explain; @@ -25,13 +29,14 @@ import com.facebook.presto.verifier.prestoaction.QueryActions; import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListeningExecutorService; import java.util.Objects; import java.util.Optional; -import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.sql.tree.ExplainFormat.Type.JSON; +import static com.facebook.presto.testing.TestingEnvironment.FUNCTION_AND_TYPE_MANAGER; import static com.facebook.presto.verifier.framework.ExplainMatchResult.MatchType; import static com.facebook.presto.verifier.framework.ExplainMatchResult.MatchType.DETAILS_MISMATCH; import static com.facebook.presto.verifier.framework.ExplainMatchResult.MatchType.MATCH; @@ -45,7 +50,7 @@ public class ExplainVerification extends AbstractVerification { private static final ResultSetConverter QUERY_PLAN_RESULT_SET_CONVERTER = resultSet -> Optional.of(resultSet.getString("Query Plan")); - private static final JsonCodec PLAN_CODEC = jsonCodec(JsonRenderedNode.class); + private static JsonCodec planCodec; private final SqlParser sqlParser; public ExplainVerification( @@ -59,6 +64,11 @@ public ExplainVerification( { super(queryActions, sourceQuery, exceptionClassifier, verificationContext, Optional.of(QUERY_PLAN_RESULT_SET_CONVERTER), verifierConfig, executor); this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + JsonObjectMapperProvider provider = new JsonObjectMapperProvider(); + provider.setJsonSerializers(ImmutableMap.of(VariableReferenceExpression.class, new Serialization.VariableReferenceExpressionSerializer())); + provider.setKeyDeserializers(ImmutableMap.of(VariableReferenceExpression.class, new Serialization.VariableReferenceExpressionDeserializer(FUNCTION_AND_TYPE_MANAGER))); + JsonCodecFactory codecFactory = new JsonCodecFactory(provider, true); + planCodec = codecFactory.jsonCodec(JsonRenderedNode.class); } @Override @@ -81,8 +91,8 @@ protected ExplainMatchResult verify( checkArgument(controlQueryResult.isPresent(), "control query plan is missing"); checkArgument(testQueryResult.isPresent(), "test query plan is missing"); - JsonRenderedNode controlPlan = PLAN_CODEC.fromJson(getOnlyElement(controlQueryResult.get().getResults())); - JsonRenderedNode testPlan = PLAN_CODEC.fromJson(getOnlyElement(testQueryResult.get().getResults())); + JsonRenderedNode controlPlan = planCodec.fromJson(getOnlyElement(controlQueryResult.get().getResults())); + JsonRenderedNode testPlan = planCodec.fromJson(getOnlyElement(testQueryResult.get().getResults())); return new ExplainMatchResult(match(controlPlan, testPlan)); } @@ -114,7 +124,8 @@ private MatchType match(JsonRenderedNode controlPlan, JsonRenderedNode testPlan) return STRUCTURE_MISMATCH; } - boolean detailsMismatched = !Objects.equals(controlPlan.getDetails(), testPlan.getDetails()); + boolean detailsMismatched = !Objects.equals(controlPlan.getDetails(), testPlan.getDetails()) + || !Objects.equals(controlPlan.getEstimates(), testPlan.getEstimates()); for (int i = 0; i < controlPlan.getChildren().size(); i++) { MatchType childMatchType = match(controlPlan.getChildren().get(i), testPlan.getChildren().get(i)); if (childMatchType == STRUCTURE_MISMATCH) { From da88672ac51fac344f51e721dbe67dbc1a0f1cdf Mon Sep 17 00:00:00 2001 From: George Wang Date: Wed, 16 Nov 2022 09:05:12 -0800 Subject: [PATCH 3/3] Add Graphviz query plan monitoring for instrumentation --- .../facebook/presto/event/QueryMonitor.java | 3 ++ .../facebook/presto/util/GraphvizPrinter.java | 10 +++++++ .../presto/util/TestGraphvizPrinter.java | 28 +++++++++++++++++++ .../spi/eventlistener/QueryMetadata.java | 9 ++++++ .../presto/execution/TestEventListener.java | 15 ++++++++++ 5 files changed, 65 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java index 2bc35655cdb75..c314e114e1c6a 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java @@ -145,6 +145,7 @@ public void queryCreatedEvent(BasicQueryInfo queryInfo) Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), ImmutableList.of(), queryInfo.getSession().getTraceToken()))); } @@ -168,6 +169,7 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), ImmutableList.of(), queryInfo.getSession().getTraceToken()), new QueryStatistics( @@ -271,6 +273,7 @@ private QueryMetadata createQueryMetadata(QueryInfo queryInfo) queryInfo.getSelf(), createTextQueryPlan(queryInfo), createJsonQueryPlan(queryInfo), + createGraphvizQueryPlan(queryInfo), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, maxJsonLimit)), queryInfo.getRuntimeOptimizedStages().orElse(ImmutableList.of()).stream() .map(stageId -> String.valueOf(stageId.getId())) diff --git a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java index 6549d718b2308..1470be6752b11 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java @@ -41,6 +41,7 @@ import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; +import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; @@ -116,6 +117,7 @@ private enum NodeType INDEX_SOURCE, UNNEST, ANALYZE_FINISH, + EXPLAIN_ANALYZE, } private static final Map NODE_COLORS = immutableEnumMap(ImmutableMap.builder() @@ -141,6 +143,7 @@ private enum NodeType .put(NodeType.UNNEST, "crimson") .put(NodeType.SAMPLE, "goldenrod4") .put(NodeType.ANALYZE_FINISH, "plum") + .put(NodeType.EXPLAIN_ANALYZE, "cadetblue1") .build()); static { @@ -294,6 +297,13 @@ public Void visitTableFinish(TableFinishNode node, Void context) return node.getSource().accept(this, context); } + @Override + public Void visitExplainAnalyze(ExplainAnalyzeNode node, Void context) + { + printNode(node, format("ExplainAnalyze[%s]", Joiner.on(", ").join(node.getOutputVariables())), NODE_COLORS.get(NodeType.EXPLAIN_ANALYZE)); + return node.getSource().accept(this, context); + } + @Override public Void visitSample(SampleNode node, Void context) { diff --git a/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java b/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java index 51995db86d0e8..00a3554da6e98 100644 --- a/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java +++ b/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java @@ -36,7 +36,9 @@ import com.google.common.collect.ImmutableSet; import org.testng.annotations.Test; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Optional; import static com.facebook.presto.operator.StageExecutionDescriptor.ungroupedExecution; @@ -45,6 +47,7 @@ import static com.facebook.presto.testing.TestingEnvironment.FUNCTION_AND_TYPE_MANAGER; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.util.GraphvizPrinter.printDistributed; +import static com.facebook.presto.util.GraphvizPrinter.printDistributedFromFragments; import static com.facebook.presto.util.GraphvizPrinter.printLogical; import static java.lang.String.format; import static java.lang.String.join; @@ -117,6 +120,31 @@ public void testPrintDistributed() assertEquals(actualNestedSubPlan, expectedNestedSubPlan); } + @Test + public void testPrintDistributedFromFragments() + { + List allFragments = new ArrayList<>(); + allFragments.add(createTestPlanFragment(0, TEST_TABLE_SCAN_NODE)); + allFragments.add(createTestPlanFragment(1, TEST_TABLE_SCAN_NODE)); + String actual = printDistributedFromFragments( + allFragments, + FUNCTION_AND_TYPE_MANAGER, + testSessionBuilder().build()); + String expected = "digraph distributed_plan {\n" + + "subgraph cluster_0 {\n" + + "label = \"SOURCE\"\n" + + "plannode_1[label=\"{TableScan | [TableHandle \\{connectorId='connector_id', connectorHandle='com.facebook.presto.testing.TestingMetadata$TestingTableHandle@1af56f7', layout='Optional.empty'\\}]|Estimates: \\{rows: ? (0B), cpu: ?, memory: ?, network: ?\\}\n" + + "}\", style=\"rounded, filled\", shape=record, fillcolor=deepskyblue];\n" + + "}\n" + + "subgraph cluster_1 {\n" + + "label = \"SOURCE\"\n" + + "plannode_1[label=\"{TableScan | [TableHandle \\{connectorId='connector_id', connectorHandle='com.facebook.presto.testing.TestingMetadata$TestingTableHandle@1af56f7', layout='Optional.empty'\\}]|Estimates: \\{rows: ? (0B), cpu: ?, memory: ?, network: ?\\}\n" + + "}\", style=\"rounded, filled\", shape=record, fillcolor=deepskyblue];\n" + + "}\n" + + "}\n"; + assertEquals(actual, expected); + } + @Test public void testPrintLogicalForJoinNode() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java index 62659970e5cb6..ca96cdba0bb44 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java @@ -38,6 +38,7 @@ public class QueryMetadata private final Optional jsonPlan; + private final Optional graphvizPlan; private final Optional payload; private final List runtimeOptimizedStages; @@ -52,6 +53,7 @@ public QueryMetadata( URI uri, Optional plan, Optional jsonPlan, + Optional graphvizPlan, Optional payload, List runtimeOptimizedStages, Optional tracingId) @@ -65,6 +67,7 @@ public QueryMetadata( this.uri = requireNonNull(uri, "uri is null"); this.plan = requireNonNull(plan, "plan is null"); this.jsonPlan = requireNonNull(jsonPlan, "jsonPlan is null"); + this.graphvizPlan = requireNonNull(graphvizPlan, "graphvizPlan is null"); this.payload = requireNonNull(payload, "payload is null"); this.runtimeOptimizedStages = requireNonNull(runtimeOptimizedStages, "runtimeOptimizedStages is null"); this.tracingId = requireNonNull(tracingId, "tracingId is null"); @@ -124,6 +127,12 @@ public Optional getJsonPlan() return jsonPlan; } + @JsonProperty + public Optional getGraphvizPlan() + { + return graphvizPlan; + } + @JsonProperty public Optional getPayload() { diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java index 66470b3c26459..27200966c919e 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java @@ -265,6 +265,21 @@ public void testOutputStats() assertEquals(1L, queryCompletedEvent.getStatistics().getOutputRows()); } + @Test + public void testGraphvizQueryPlanOutput() + throws Exception + { + int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1; + String query = "EXPLAIN (type distributed, format graphviz) SELECT * FROM LINEITEM limit 1"; + Session sessionForEventLoggingWithStats = Session.builder(session) + .setSystemProperty("print_stats_for_non_join_query", "true") + .build(); + runQueryAndWaitForEvents("SELECT * FROM lineitem limit 1", expectedEvents, sessionForEventLoggingWithStats); + QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents()); + MaterializedResult expected = runQueryAndWaitForEvents(query, expectedEvents); + assertEquals(queryCompletedEvent.getMetadata().getGraphvizPlan().get(), getOnlyElement(expected.getOnlyColumnAsSet())); + } + static class EventsBuilder { private ImmutableList.Builder queryCreatedEvents;