diff --git a/presto-docs/src/main/sphinx/sql/explain-analyze.rst b/presto-docs/src/main/sphinx/sql/explain-analyze.rst index 8ae2fa9171cb6..9111aa0b02355 100644 --- a/presto-docs/src/main/sphinx/sql/explain-analyze.rst +++ b/presto-docs/src/main/sphinx/sql/explain-analyze.rst @@ -7,7 +7,7 @@ Synopsis .. code-block:: none - EXPLAIN ANALYZE [VERBOSE] statement + EXPLAIN ANALYZE [VERBOSE] [(format )] statement Description ----------- @@ -17,6 +17,8 @@ along with the cost of each operation. The ``VERBOSE`` option will give more detailed information and low-level statistics; understanding these may require knowledge of Presto internals and implementation details. +The format of the output can be set by the user with the ``format`` option. The default +output format is ``TEXT``. .. note:: diff --git a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java index f20761e50c903..2be4564990f9a 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java @@ -472,7 +472,8 @@ private Optional createJsonQueryPlan(QueryInfo queryInfo) if (queryInfo.getOutputStage().isPresent()) { return Optional.of(jsonDistributedPlan( queryInfo.getOutputStage().get(), - functionAndTypeManager)); + functionAndTypeManager, + queryInfo.getSession().toSession(sessionPropertyManager))); } } catch (Exception e) { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java index 6203e823d194d..956d64b58b924 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java @@ -20,13 +20,17 @@ import com.facebook.presto.execution.StageId; import com.facebook.presto.execution.StageInfo; import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.sql.tree.ExplainFormat; import com.google.common.collect.ImmutableList; import java.util.List; import java.util.concurrent.TimeUnit; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonDistributedPlan; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.textDistributedPlan; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -42,6 +46,7 @@ public static class ExplainAnalyzeOperatorFactory private final QueryPerformanceFetcher queryPerformanceFetcher; private final FunctionAndTypeManager functionAndTypeManager; private final boolean verbose; + private final ExplainFormat.Type format; private boolean closed; public ExplainAnalyzeOperatorFactory( @@ -49,13 +54,15 @@ public ExplainAnalyzeOperatorFactory( PlanNodeId planNodeId, QueryPerformanceFetcher queryPerformanceFetcher, FunctionAndTypeManager functionAndTypeManager, - boolean verbose) + boolean verbose, + ExplainFormat.Type format) { this.operatorId = operatorId; this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); this.queryPerformanceFetcher = requireNonNull(queryPerformanceFetcher, "queryPerformanceFetcher is null"); this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionManager is null"); this.verbose = verbose; + this.format = requireNonNull(format, "format is null"); } @Override @@ -63,7 +70,7 @@ public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, ExplainAnalyzeOperator.class.getSimpleName()); - return new ExplainAnalyzeOperator(operatorContext, queryPerformanceFetcher, functionAndTypeManager, verbose); + return new ExplainAnalyzeOperator(operatorContext, queryPerformanceFetcher, functionAndTypeManager, verbose, format); } @Override @@ -75,7 +82,7 @@ public void noMoreOperators() @Override public OperatorFactory duplicate() { - return new ExplainAnalyzeOperatorFactory(operatorId, planNodeId, queryPerformanceFetcher, functionAndTypeManager, verbose); + return new ExplainAnalyzeOperatorFactory(operatorId, planNodeId, queryPerformanceFetcher, functionAndTypeManager, verbose, format); } } @@ -83,6 +90,7 @@ public OperatorFactory duplicate() private final QueryPerformanceFetcher queryPerformanceFetcher; private final FunctionAndTypeManager functionAndTypeManager; private final boolean verbose; + private final ExplainFormat.Type format; private boolean finishing; private boolean outputConsumed; @@ -90,12 +98,14 @@ public ExplainAnalyzeOperator( OperatorContext operatorContext, QueryPerformanceFetcher queryPerformanceFetcher, FunctionAndTypeManager functionAndTypeManager, - boolean verbose) + boolean verbose, + ExplainFormat.Type format) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.queryPerformanceFetcher = requireNonNull(queryPerformanceFetcher, "queryPerformanceFetcher is null"); this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionManager is null"); this.verbose = verbose; + this.format = requireNonNull(format, "format is null"); } @Override @@ -144,8 +154,18 @@ public Page getOutput() if (!hasFinalStageInfo(queryInfo.getOutputStage().get())) { return null; } + String plan; + switch (format) { + case TEXT: + plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionAndTypeManager, operatorContext.getSession(), verbose); + break; + case JSON: + plan = jsonDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionAndTypeManager, operatorContext.getSession()); + break; + default: + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Explain format not supported: " + format); + } - String plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionAndTypeManager, operatorContext.getSession(), verbose); BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1); VARCHAR.writeString(builder, plan); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/queryplan/JsonPrestoQueryPlanFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/queryplan/JsonPrestoQueryPlanFunctions.java index 9b5fcd4f78fbd..b854b817d3dd1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/queryplan/JsonPrestoQueryPlanFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/queryplan/JsonPrestoQueryPlanFunctions.java @@ -38,6 +38,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -197,7 +198,7 @@ private static JsonRenderedNode scrubJsonPlan(JsonRenderedNode node) List newChildren = node.getChildren().stream().map(x -> scrubJsonPlan(x)).collect(Collectors.toList()); String newDetails = scrubDetails(node.getDetails()); - return new JsonRenderedNode(node.getSourceLocation(), "PLANID", newName, newIdentifier, newDetails, newChildren, node.getRemoteSources(), ImmutableList.of()); + return new JsonRenderedNode(node.getSourceLocation(), "PLANID", newName, newIdentifier, newDetails, newChildren, node.getRemoteSources(), ImmutableList.of(), Optional.empty()); } private static String scrubName(String name) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java index 9b3b441ab5dfe..a2cc76cbc2acd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java @@ -186,7 +186,7 @@ public String getJsonPlan(Session session, Statement statement, Type planType, L return jsonLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionAndTypeManager(), plan.getStatsAndCosts(), session); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); - return jsonDistributedPlan(subPlan, metadata.getFunctionAndTypeManager()); + return jsonDistributedPlan(subPlan, metadata.getFunctionAndTypeManager(), session); default: throw new PrestoException(NOT_SUPPORTED, format("Unsupported explain plan type %s for JSON format", planType)); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index 3621b0346cb3c..d0703c0eccf0a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -94,6 +94,7 @@ import com.facebook.presto.sql.tree.Except; import com.facebook.presto.sql.tree.Execute; import com.facebook.presto.sql.tree.Explain; +import com.facebook.presto.sql.tree.ExplainFormat; import com.facebook.presto.sql.tree.ExplainType; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.ExpressionRewriter; @@ -273,6 +274,8 @@ import static com.facebook.presto.sql.planner.ExpressionDeterminismEvaluator.isDeterministic; import static com.facebook.presto.sql.planner.ExpressionInterpreter.evaluateConstantExpression; import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionOptimizer; +import static com.facebook.presto.sql.tree.ExplainFormat.Type.JSON; +import static com.facebook.presto.sql.tree.ExplainFormat.Type.TEXT; import static com.facebook.presto.sql.tree.ExplainType.Type.DISTRIBUTED; import static com.facebook.presto.sql.tree.FrameBound.Type.CURRENT_ROW; import static com.facebook.presto.sql.tree.FrameBound.Type.FOLLOWING; @@ -1135,9 +1138,20 @@ protected Scope visitExplain(Explain node, Optional scope) throws SemanticException { checkState(node.isAnalyze(), "Non analyze explain should be rewritten to Query"); - if (node.getOptions().stream().anyMatch(option -> !option.equals(new ExplainType(DISTRIBUTED)))) { - throw new SemanticException(NOT_SUPPORTED, node, "EXPLAIN ANALYZE only supports TYPE DISTRIBUTED option"); - } + List formats = node.getOptions().stream() + .filter(option -> option instanceof ExplainFormat) + .map(ExplainFormat.class::cast) + .map(ExplainFormat::getType) + .collect(Collectors.toList()); + checkState(formats.size() <= 1, "only a single format option is supported in EXPLAIN ANALYZE"); + formats.stream().findFirst().ifPresent(format -> checkState(format.equals(TEXT) || format.equals(JSON), + "only TEXT and JSON formats are supported in EXPLAIN ANALYZE")); + checkState(node.getOptions().stream() + .filter(option -> option instanceof ExplainType) + .findFirst() + .map(ExplainType.class::cast) + .map(ExplainType::getType) + .orElse(DISTRIBUTED).equals(DISTRIBUTED), "only DISTRIBUTED type is supported in EXPLAIN ANALYZE"); process(node.getStatement(), scope); analysis.setUpdateType(null); return createAndAssignScope(node, scope, Field.newUnqualified(node.getLocation(), "Query Plan", VARCHAR)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index 7802bf9b0af85..2a66962ad4cc7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -998,7 +998,8 @@ public PhysicalOperation visitExplainAnalyze(ExplainAnalyzeNode node, LocalExecu node.getId(), analyzeContext.getQueryPerformanceFetcher(), metadata.getFunctionAndTypeManager(), - node.isVerbose()); + node.isVerbose(), + node.getFormat()); return new PhysicalOperation(operatorFactory, makeLayout(node), context, source); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java index 2c08570a1d7cb..1763247be121e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java @@ -59,6 +59,7 @@ import com.facebook.presto.sql.tree.CreateTableAsSelect; import com.facebook.presto.sql.tree.Delete; import com.facebook.presto.sql.tree.Explain; +import com.facebook.presto.sql.tree.ExplainFormat; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.Insert; @@ -101,6 +102,7 @@ import static com.facebook.presto.sql.planner.plan.TableWriterNode.UpdateTarget; import static com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.tree.ExplainFormat.Type.TEXT; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -196,7 +198,12 @@ private RelationPlan createExplainAnalyzePlan(Analysis analysis, Explain stateme PlanNode root = underlyingPlan.getRoot(); Scope scope = analysis.getScope(statement); VariableReferenceExpression outputVariable = newVariable(variableAllocator, scope.getRelationType().getFieldByIndex(0)); - root = new ExplainAnalyzeNode(getSourceLocation(statement), idAllocator.getNextId(), root, outputVariable, statement.isVerbose()); + ExplainFormat.Type type = statement.getOptions() + .stream().filter(option -> option instanceof ExplainFormat) + .findFirst().map(ExplainFormat.class::cast) + .map(ExplainFormat::getType) + .orElse(TEXT); + root = new ExplainAnalyzeNode(getSourceLocation(statement), idAllocator.getNextId(), root, outputVariable, statement.isVerbose(), type); return new RelationPlan(root, scope, ImmutableList.of(outputVariable)); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java index 197c4c0ee4562..54449ed14b207 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -215,7 +215,7 @@ public PlanNode visitGroupId(GroupIdNode node, RewriteContext context) public PlanNode visitExplainAnalyze(ExplainAnalyzeNode node, RewriteContext context) { PlanNode source = context.rewrite(node.getSource()); - return new ExplainAnalyzeNode(node.getSourceLocation(), node.getId(), source, canonicalize(node.getOutputVariable()), node.isVerbose()); + return new ExplainAnalyzeNode(node.getSourceLocation(), node.getId(), source, canonicalize(node.getOutputVariable()), node.isVerbose(), node.getFormat()); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExplainAnalyzeNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExplainAnalyzeNode.java index 1c58e1d0beced..26c3260154e01 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExplainAnalyzeNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExplainAnalyzeNode.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.tree.ExplainFormat.Type; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -36,6 +37,7 @@ public class ExplainAnalyzeNode private final PlanNode source; private final VariableReferenceExpression outputVariable; private final boolean verbose; + private final Type format; @JsonCreator public ExplainAnalyzeNode( @@ -43,9 +45,10 @@ public ExplainAnalyzeNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, @JsonProperty("outputVariable") VariableReferenceExpression outputVariable, - @JsonProperty("verbose") boolean verbose) + @JsonProperty("verbose") boolean verbose, + @JsonProperty("format") Type format) { - this(sourceLocation, id, Optional.empty(), source, outputVariable, verbose); + this(sourceLocation, id, Optional.empty(), source, outputVariable, verbose, format); } public ExplainAnalyzeNode( @@ -54,12 +57,14 @@ public ExplainAnalyzeNode( Optional statsEquivalentPlanNode, PlanNode source, VariableReferenceExpression outputVariable, - boolean verbose) + boolean verbose, + Type format) { super(sourceLocation, id, statsEquivalentPlanNode); this.source = requireNonNull(source, "source is null"); this.outputVariable = requireNonNull(outputVariable, "outputVariable is null"); this.verbose = verbose; + this.format = requireNonNull(format, "options is null"); } @JsonProperty("outputVariable") @@ -80,6 +85,12 @@ public boolean isVerbose() return verbose; } + @JsonProperty("format") + public Type getFormat() + { + return format; + } + @Override public List getOutputVariables() { @@ -101,12 +112,12 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new ExplainAnalyzeNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), Iterables.getOnlyElement(newChildren), outputVariable, isVerbose()); + return new ExplainAnalyzeNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), Iterables.getOnlyElement(newChildren), outputVariable, isVerbose(), format); } @Override public PlanNode assignStatsEquivalentPlanNode(Optional statsEquivalentPlanNode) { - return new ExplainAnalyzeNode(getSourceLocation(), getId(), statsEquivalentPlanNode, source, outputVariable, isVerbose()); + return new ExplainAnalyzeNode(getSourceLocation(), getId(), statsEquivalentPlanNode, source, outputVariable, isVerbose(), format); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java index 06f6f1faeaaea..fc2d099ef5189 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java @@ -36,12 +36,17 @@ public HashCollisionPlanNodeStats( PlanNodeId planNodeId, Duration planNodeScheduledTime, Duration planNodeCpuTime, + Duration planNodeBlockedWallTime, + Duration planNodeAddInputWallTime, + Duration planNodeGetOutputWallTime, + Duration planNodeFinishWallTime, long planNodeInputPositions, DataSize planNodeInputDataSize, long planNodeRawInputPositions, DataSize planNodeRawInputDataSize, long planNodeOutputPositions, DataSize planNodeOutputDataSize, + DataSize planNodePeakMemorySize, Map operatorInputStats, long planNodeNullJoinBuildKeyCount, long planNodeJoinBuildKeyCount, @@ -50,9 +55,9 @@ public HashCollisionPlanNodeStats( Optional dynamicFilterStats, Map operatorHashCollisionsStats) { - super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, - planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount, planNodeNullJoinProbeKeyCount, - planNodeJoinProbeKeyCount, dynamicFilterStats); + super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeBlockedWallTime, planNodeAddInputWallTime, planNodeGetOutputWallTime, planNodeFinishWallTime, + planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, + planNodePeakMemorySize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount, planNodeNullJoinProbeKeyCount, planNodeJoinProbeKeyCount, dynamicFilterStats); this.operatorHashCollisionsStats = requireNonNull(operatorHashCollisionsStats, "operatorHashCollisionsStats is null"); } @@ -101,12 +106,17 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.getPlanNodeId(), merged.getPlanNodeScheduledTime(), merged.getPlanNodeCpuTime(), + merged.getPlanNodeBlockedWallTime(), + merged.getPlanNodeAddInputWallTime(), + merged.getPlanNodeGetOutputWallTime(), + merged.getPlanNodeFinishWallTime(), merged.getPlanNodeInputPositions(), merged.getPlanNodeInputDataSize(), merged.getPlanNodeRawInputPositions(), merged.getPlanNodeRawInputDataSize(), merged.getPlanNodeOutputPositions(), merged.getPlanNodeOutputDataSize(), + merged.getPlanNodePeakMemorySize(), merged.operatorInputStats, merged.getPlanNodeNullJoinBuildKeyCount(), merged.getPlanNodeJoinBuildKeyCount(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java index ba75d07a02894..603cbe42c80b6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java @@ -41,6 +41,7 @@ public class JsonRenderer { private final JsonCodec> planMapCodec; private final JsonCodec codec; + private final JsonCodec> deserializationCodec; public JsonRenderer(FunctionAndTypeManager functionAndTypeManager) { @@ -51,6 +52,12 @@ public JsonRenderer(FunctionAndTypeManager functionAndTypeManager) JsonCodecFactory codecFactory = new JsonCodecFactory(provider, true); this.codec = codecFactory.jsonCodec(JsonRenderedNode.class); this.planMapCodec = codecFactory.mapJsonCodec(PlanFragmentId.class, JsonPlanFragment.class); + this.deserializationCodec = codecFactory.mapJsonCodec(PlanFragmentId.class, JsonPlan.class); + } + + public Map deserialize(String serialized) + { + return deserializationCodec.fromJson(serialized); } @Override @@ -84,7 +91,8 @@ public JsonRenderedNode renderJson(PlanRepresentation plan, NodeRepresentation n node.getRemoteSources().stream() .map(PlanFragmentId::toString) .collect(toImmutableList()), - node.getEstimatedStats()); + node.getEstimatedStats(), + node.getStats()); } public static class JsonRenderedNode @@ -97,9 +105,10 @@ public static class JsonRenderedNode private final List children; private final List remoteSources; private final List estimates; + private final Optional stats; @JsonCreator - public JsonRenderedNode(Optional sourceLocation, String id, String name, String identifier, String details, List children, List remoteSources, List estimates) + public JsonRenderedNode(Optional sourceLocation, String id, String name, String identifier, String details, List children, List remoteSources, List estimates, Optional stats) { this.sourceLocation = sourceLocation; this.id = requireNonNull(id, "id is null"); @@ -109,6 +118,7 @@ public JsonRenderedNode(Optional sourceLocation, String id, Stri this.children = requireNonNull(children, "children is null"); this.remoteSources = requireNonNull(remoteSources, "id is null"); this.estimates = requireNonNull(estimates, "estimate is null"); + this.stats = requireNonNull(stats, "stats is null"); } @JsonProperty @@ -159,6 +169,12 @@ public List getEstimates() return estimates; } + @JsonProperty + public Optional getStats() + { + return stats; + } + @Override public boolean equals(Object obj) { @@ -176,13 +192,14 @@ public boolean equals(Object obj) Objects.equals(this.details, other.details) && Objects.equals(this.children, other.children) && Objects.equals(this.estimates, other.estimates) && - Objects.equals(this.remoteSources, other.remoteSources); + Objects.equals(this.remoteSources, other.remoteSources) && + Objects.equals(this.stats, other.stats); } @Override public int hashCode() { - return Objects.hash(name, id, sourceLocation, identifier, details, children, estimates, remoteSources); + return Objects.hash(name, id, sourceLocation, identifier, details, children, estimates, remoteSources, stats); } } @@ -203,4 +220,25 @@ public String getPlan() return this.plan; } } + + /** + * Used for deserializing rendered JSON plan + */ + public static class JsonPlan + { + @JsonProperty + private final JsonRenderedNode plan; + + @JsonCreator + public JsonPlan(@JsonProperty("plan") JsonRenderedNode plan) + { + this.plan = plan; + } + + @JsonProperty + public JsonRenderedNode getPlan() + { + return plan; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java index e89098af0b429..0ba5e5d4725ed 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java @@ -13,29 +13,39 @@ */ package com.facebook.presto.sql.planner.planPrinter; -class OperatorInputStats +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class OperatorInputStats { private final long totalDrivers; private final long inputPositions; private final double sumSquaredInputPositions; - public OperatorInputStats(long totalDrivers, long inputPositions, double sumSquaredInputPositions) + @JsonCreator + public OperatorInputStats( + @JsonProperty("totalDrivers") long totalDrivers, + @JsonProperty("inputPositions") long inputPositions, + @JsonProperty("sumSquaredInputPositions") double sumSquaredInputPositions) { this.totalDrivers = totalDrivers; this.inputPositions = inputPositions; this.sumSquaredInputPositions = sumSquaredInputPositions; } + @JsonProperty public long getTotalDrivers() { return totalDrivers; } + @JsonProperty public long getInputPositions() { return inputPositions; } + @JsonProperty public double getSumSquaredInputPositions() { return sumSquaredInputPositions; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java index 9b3da2d9ed900..676231c237053 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java @@ -16,9 +16,12 @@ import com.facebook.presto.operator.DynamicFilterStats; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.util.Mergeable; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -39,6 +42,10 @@ public class PlanNodeStats private final Duration planNodeScheduledTime; private final Duration planNodeCpuTime; + private final Duration planNodeBlockedWallTime; + private final Duration planNodeAddInputWallTime; + private final Duration planNodeGetOutputWallTime; + private final Duration planNodeFinishWallTime; private final long planNodeInputPositions; private final DataSize planNodeInputDataSize; private final long planNodeRawInputPositions; @@ -46,6 +53,7 @@ public class PlanNodeStats private final long planNodeOutputPositions; private final DataSize planNodeOutputDataSize; + private final DataSize planNodePeakMemorySize; protected final Map operatorInputStats; private final long planNodeNullJoinBuildKeyCount; private final long planNodeJoinBuildKeyCount; @@ -53,27 +61,37 @@ public class PlanNodeStats private final long planNodeJoinProbeKeyCount; private final Optional dynamicFilterStats; - PlanNodeStats( - PlanNodeId planNodeId, - Duration planNodeScheduledTime, - Duration planNodeCpuTime, - long planNodeInputPositions, - DataSize planNodeInputDataSize, - long planNodeRawInputPositions, - DataSize planNodeRawInputDataSize, - long planNodeOutputPositions, - DataSize planNodeOutputDataSize, - Map operatorInputStats, - long planNodeNullJoinBuildKeyCount, - long planNodeJoinBuildKeyCount, - long planNodeNullJoinProbeKeyCount, - long planNodeJoinProbeKeyCount, - Optional dynamicFilterStats) + @JsonCreator + public PlanNodeStats( + @JsonProperty("planNodeId") PlanNodeId planNodeId, + @JsonProperty("planNodeScheduledTime") Duration planNodeScheduledTime, + @JsonProperty("planNodeCpuTime") Duration planNodeCpuTime, + @JsonProperty("planNodeBlockedWallTime") Duration planNodeBlockedWallTime, + @JsonProperty("planNodeAddInputWallTime") Duration planNodeAddInputWallTime, + @JsonProperty("planNodeGetOutputWallTime") Duration planNodeGetOutputWallTime, + @JsonProperty("planNodeFinishWallTime") Duration planNodeFinishWallTime, + @JsonProperty("planNodeInputPositions") long planNodeInputPositions, + @JsonProperty("planNodeInputDataSize") DataSize planNodeInputDataSize, + @JsonProperty("planNodeRawInputPositions") long planNodeRawInputPositions, + @JsonProperty("planNodeRawInputDataSize") DataSize planNodeRawInputDataSize, + @JsonProperty("planNodeOutputPositions") long planNodeOutputPositions, + @JsonProperty("planNodeOutputDataSize") DataSize planNodeOutputDataSize, + @JsonProperty("planNodePeakMemorySize") DataSize planNodePeakMemorySize, + @JsonProperty("operatorInputStats") Map operatorInputStats, + @JsonProperty("planNodeNullJoinBuildKeyCount") long planNodeNullJoinBuildKeyCount, + @JsonProperty("planNodeJoinBuildKeyCount") long planNodeJoinBuildKeyCount, + @JsonProperty("planNodeNullJoinProbeKeyCount") long planNodeNullJoinProbeKeyCount, + @JsonProperty("planNodeJoinProbeKeyCount") long planNodeJoinProbeKeyCount, + @JsonProperty("dynamicFilterStats") Optional dynamicFilterStats) { this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); this.planNodeScheduledTime = requireNonNull(planNodeScheduledTime, "planNodeScheduledTime is null"); this.planNodeCpuTime = requireNonNull(planNodeCpuTime, "planNodeCpuTime is null"); + this.planNodeBlockedWallTime = requireNonNull(planNodeBlockedWallTime, "planNodeBlockedWallTime is null"); + this.planNodeAddInputWallTime = requireNonNull(planNodeAddInputWallTime, "planNodeAddInputWallTime is null"); + this.planNodeGetOutputWallTime = requireNonNull(planNodeGetOutputWallTime, "planNodeGetOutputWallTime is null"); + this.planNodeFinishWallTime = requireNonNull(planNodeFinishWallTime, "planNodeFinishWallTime is null"); this.planNodeInputPositions = planNodeInputPositions; this.planNodeInputDataSize = planNodeInputDataSize; this.planNodeRawInputPositions = planNodeRawInputPositions; @@ -82,6 +100,7 @@ public class PlanNodeStats this.planNodeOutputDataSize = planNodeOutputDataSize; this.operatorInputStats = requireNonNull(operatorInputStats, "operatorInputStats is null"); + this.planNodePeakMemorySize = planNodePeakMemorySize; this.planNodeNullJoinBuildKeyCount = planNodeNullJoinBuildKeyCount; this.planNodeJoinBuildKeyCount = planNodeJoinBuildKeyCount; this.planNodeNullJoinProbeKeyCount = planNodeNullJoinProbeKeyCount; @@ -97,51 +116,91 @@ private static double computedStdDev(double sumSquared, double sum, long n) return sqrt(max(variance, 0d)); } + @JsonProperty public PlanNodeId getPlanNodeId() { return planNodeId; } + @JsonProperty public Duration getPlanNodeScheduledTime() { return planNodeScheduledTime; } + @JsonProperty public Duration getPlanNodeCpuTime() { return planNodeCpuTime; } + @JsonProperty + public Duration getPlanNodeBlockedWallTime() + { + return planNodeBlockedWallTime; + } + + @JsonProperty + public Duration getPlanNodeAddInputWallTime() + { + return planNodeAddInputWallTime; + } + + @JsonProperty + public Duration getPlanNodeGetOutputWallTime() + { + return planNodeGetOutputWallTime; + } + + @JsonProperty + public Duration getPlanNodeFinishWallTime() + { + return planNodeFinishWallTime; + } + + @JsonProperty + public Map getOperatorInputStats() + { + // no need to copy, just prevent modifications + return Collections.unmodifiableMap(operatorInputStats); + } + public Set getOperatorTypes() { return operatorInputStats.keySet(); } + @JsonProperty public long getPlanNodeInputPositions() { return planNodeInputPositions; } + @JsonProperty public DataSize getPlanNodeInputDataSize() { return planNodeInputDataSize; } + @JsonProperty public long getPlanNodeRawInputPositions() { return planNodeRawInputPositions; } + @JsonProperty public DataSize getPlanNodeRawInputDataSize() { return planNodeRawInputDataSize; } + @JsonProperty public long getPlanNodeOutputPositions() { return planNodeOutputPositions; } + @JsonProperty public DataSize getPlanNodeOutputDataSize() { return planNodeOutputDataSize; @@ -166,21 +225,31 @@ public Map getOperatorInputPositionsStdDevs() entry.getValue().getTotalDrivers()))); } + @JsonProperty + public DataSize getPlanNodePeakMemorySize() + { + return planNodePeakMemorySize; + } + + @JsonProperty public long getPlanNodeNullJoinBuildKeyCount() { return planNodeNullJoinBuildKeyCount; } + @JsonProperty public long getPlanNodeJoinBuildKeyCount() { return planNodeJoinBuildKeyCount; } + @JsonProperty public long getPlanNodeNullJoinProbeKeyCount() { return planNodeNullJoinProbeKeyCount; } + @JsonProperty public long getPlanNodeJoinProbeKeyCount() { return planNodeJoinProbeKeyCount; @@ -216,6 +285,7 @@ public PlanNodeStats mergeWith(PlanNodeStats other) DataSize planNodeRawInputDataSize = succinctBytes((long) ((double) this.planNodeRawInputDataSize.toBytes() + (double) other.planNodeRawInputDataSize.toBytes())); long planNodeOutputPositions = this.planNodeOutputPositions + other.planNodeOutputPositions; DataSize planNodeOutputDataSize = succinctBytes((long) ((double) this.planNodeOutputDataSize.toBytes() + (double) other.planNodeOutputDataSize.toBytes())); + DataSize planNodePeakMemorySize = succinctBytes(Math.max(this.planNodePeakMemorySize.toBytes(), other.planNodePeakMemorySize.toBytes())); Map operatorInputStats = mergeMaps(this.operatorInputStats, other.operatorInputStats, OperatorInputStats::merge); long planNodeNullJoinBuildKeyCount = this.planNodeNullJoinBuildKeyCount + other.planNodeNullJoinBuildKeyCount; @@ -228,10 +298,16 @@ public PlanNodeStats mergeWith(PlanNodeStats other) planNodeId, new Duration(planNodeScheduledTime.toMillis() + other.getPlanNodeScheduledTime().toMillis(), MILLISECONDS), new Duration(planNodeCpuTime.toMillis() + other.getPlanNodeCpuTime().toMillis(), MILLISECONDS), + new Duration(planNodeBlockedWallTime.toMillis() + other.getPlanNodeBlockedWallTime().toMillis(), MILLISECONDS), + new Duration(planNodeAddInputWallTime.toMillis() + other.getPlanNodeAddInputWallTime().toMillis(), MILLISECONDS), + new Duration(planNodeGetOutputWallTime.toMillis() + other.getPlanNodeGetOutputWallTime().toMillis(), MILLISECONDS), + new Duration(planNodeFinishWallTime.toMillis() + other.getPlanNodeFinishWallTime().toMillis(), MILLISECONDS), planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, + planNodePeakMemorySize, operatorInputStats, + planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount, planNodeNullJoinProbeKeyCount, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java index 3e7c4fbc819f4..364320c21ad14 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java @@ -82,6 +82,11 @@ private static List getPlanNodeStats(TaskStats taskStats) Map planNodeOutputBytes = new HashMap<>(); Map planNodeScheduledMillis = new HashMap<>(); Map planNodeCpuMillis = new HashMap<>(); + Map planNodePeakMemory = new HashMap<>(); + Map planNodeBlockedMillis = new HashMap<>(); + Map planNodeAddInputMillis = new HashMap<>(); + Map planNodeGetOutputMillis = new HashMap<>(); + Map planNodeFinishMillis = new HashMap<>(); Map planNodeNullJoinBuildKeyCount = new HashMap<>(); Map planNodeJoinBuildKeyCount = new HashMap<>(); Map planNodeNullJoinProbeKeyCount = new HashMap<>(); @@ -114,6 +119,11 @@ private static List getPlanNodeStats(TaskStats taskStats) long cpuMillis = operatorStats.getAddInputCpu().toMillis() + operatorStats.getGetOutputCpu().toMillis() + operatorStats.getFinishCpu().toMillis(); planNodeCpuMillis.merge(planNodeId, cpuMillis, Long::sum); + planNodeBlockedMillis.merge(planNodeId, operatorStats.getBlockedWall().toMillis(), Long::sum); + planNodeAddInputMillis.merge(planNodeId, operatorStats.getAddInputWall().toMillis(), Long::sum); + planNodeFinishMillis.merge(planNodeId, operatorStats.getFinishWall().toMillis(), Long::sum); + planNodeGetOutputMillis.merge(planNodeId, operatorStats.getGetOutputWall().toMillis(), Long::sum); + planNodePeakMemory.merge(planNodeId, operatorStats.getPeakTotalMemoryReservation().toBytes(), Math::max); // A pipeline like hash build before join might link to another "internal" pipelines which provide actual input for this plan node if (operatorStats.getPlanNodeId().equals(inputPlanNode) && !pipelineStats.isInputPipeline()) { @@ -212,12 +222,17 @@ private static List getPlanNodeStats(TaskStats taskStats) planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeBlockedMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeAddInputMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeGetOutputMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeFinishMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), planNodeRawInputPositions.get(planNodeId), succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), + succinctDataSize(planNodePeakMemory.get(planNodeId), BYTE), operatorInputStats.get(planNodeId), planNodeNullJoinBuildKeyCount.get(planNodeId), planNodeJoinBuildKeyCount.get(planNodeId), @@ -231,12 +246,17 @@ else if (windowNodeStats.containsKey(planNodeId)) { planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeBlockedMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeAddInputMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeGetOutputMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeFinishMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), planNodeRawInputPositions.get(planNodeId), succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), + succinctDataSize(planNodePeakMemory.get(planNodeId), BYTE), operatorInputStats.get(planNodeId), planNodeNullJoinBuildKeyCount.get(planNodeId), planNodeJoinBuildKeyCount.get(planNodeId), @@ -250,12 +270,17 @@ else if (windowNodeStats.containsKey(planNodeId)) { planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeBlockedMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeAddInputMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeGetOutputMillis.get(planNodeId), MILLISECONDS), + new Duration(planNodeFinishMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), planNodeRawInputPositions.get(planNodeId), succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), + succinctDataSize(planNodePeakMemory.get(planNodeId), BYTE), operatorInputStats.get(planNodeId), planNodeNullJoinBuildKeyCount.get(planNodeId), planNodeJoinBuildKeyCount.get(planNodeId), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index 2c7fd3bb99dc9..368bc68a45226 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -326,18 +326,20 @@ public static String jsonLogicalPlan( return new PlanPrinter(plan, types, stageExecutionStrategy, estimatedStatsAndCosts, stats, functionAndTypeManager, session).toJson(); } - public static String jsonDistributedPlan(StageInfo outputStageInfo, FunctionAndTypeManager functionAndTypeManager) + public static String jsonDistributedPlan(StageInfo outputStageInfo, FunctionAndTypeManager functionAndTypeManager, Session session) { + List allStages = getAllStages(Optional.of(outputStageInfo)); + Map aggregatedStats = aggregateStageStats(allStages); List allFragments = getAllStages(Optional.of(outputStageInfo)).stream() .map(StageInfo::getPlan) .map(Optional::get) .collect(toImmutableList()); - return formatJsonFragmentList(allFragments, functionAndTypeManager); + return formatJsonFragmentList(allFragments, Optional.of(aggregatedStats), functionAndTypeManager, session); } - public static String jsonDistributedPlan(SubPlan plan, FunctionAndTypeManager functionAndTypeManager) + public static String jsonDistributedPlan(SubPlan plan, FunctionAndTypeManager functionAndTypeManager, Session session) { - return formatJsonFragmentList(plan.getAllFragments(), functionAndTypeManager); + return formatJsonFragmentList(plan.getAllFragments(), Optional.empty(), functionAndTypeManager, session); } private String formatSourceLocation(Optional sourceLocation1, Optional sourceLocation2) @@ -358,12 +360,14 @@ private String formatSourceLocation(Optional sourceLocation) return ""; } - private static String formatJsonFragmentList(List fragments, FunctionAndTypeManager functionAndTypeManager) + private static String formatJsonFragmentList(List fragments, Optional> executionStats, FunctionAndTypeManager functionAndTypeManager, Session session) { ImmutableSortedMap.Builder fragmentJsonMap = ImmutableSortedMap.naturalOrder(); for (PlanFragment fragment : fragments) { PlanFragmentId fragmentId = fragment.getId(); - JsonPlanFragment jsonPlanFragment = new JsonPlanFragment(fragment.getJsonRepresentation().get()); + TypeProvider typeProvider = TypeProvider.fromVariables(fragment.getVariables()); + PlanPrinter printer = new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), fragment.getStatsAndCosts().orElse(StatsAndCosts.empty()), executionStats, functionAndTypeManager, session); + JsonPlanFragment jsonPlanFragment = new JsonPlanFragment(printer.toJson()); fragmentJsonMap.put(fragmentId, jsonPlanFragment); } return new JsonRenderer(functionAndTypeManager).render(fragmentJsonMap.build()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java index 51fb29ce5d5a4..3e1d41e99a1f4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanRepresentation.java @@ -28,7 +28,7 @@ import static java.util.Objects.requireNonNull; -class PlanRepresentation +public class PlanRepresentation { private final PlanNode root; private final Optional totalCpuTime; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java index 258be3f78fdf2..f373525c004f3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java @@ -32,12 +32,17 @@ public WindowPlanNodeStats( PlanNodeId planNodeId, Duration planNodeScheduledTime, Duration planNodeCpuTime, + Duration planNodeBlockedWallTime, + Duration planNodeAddInputWallTime, + Duration planNodeGetOutputWallTime, + Duration planNodeFinishWallTime, long planNodeInputPositions, DataSize planNodeInputDataSize, long planNodeRawInputPositions, DataSize planNodeRawInputDataSize, long planNodeOutputPositions, DataSize planNodeOutputDataSize, + DataSize planNodePeakMemorySize, Map operatorInputStats, long planNodeNullJoinBuildKeyCount, long planNodeJoinBuildKeyCount, @@ -46,9 +51,8 @@ public WindowPlanNodeStats( Optional dynamicFilterStats, WindowOperatorStats windowOperatorStats) { - super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, - planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount, planNodeNullJoinProbeKeyCount, - planNodeJoinProbeKeyCount, dynamicFilterStats); + super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeBlockedWallTime, planNodeAddInputWallTime, planNodeGetOutputWallTime, planNodeFinishWallTime, planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, + planNodeOutputPositions, planNodeOutputDataSize, planNodePeakMemorySize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount, planNodeNullJoinProbeKeyCount, planNodeJoinProbeKeyCount, dynamicFilterStats); this.windowOperatorStats = windowOperatorStats; } @@ -67,12 +71,17 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.getPlanNodeId(), merged.getPlanNodeScheduledTime(), merged.getPlanNodeCpuTime(), + merged.getPlanNodeBlockedWallTime(), + merged.getPlanNodeAddInputWallTime(), + merged.getPlanNodeGetOutputWallTime(), + merged.getPlanNodeFinishWallTime(), merged.getPlanNodeInputPositions(), merged.getPlanNodeInputDataSize(), merged.getPlanNodeRawInputPositions(), merged.getPlanNodeRawInputDataSize(), merged.getPlanNodeOutputPositions(), merged.getPlanNodeOutputDataSize(), + merged.getPlanNodePeakMemorySize(), merged.operatorInputStats, merged.getPlanNodeNullJoinBuildKeyCount(), merged.getPlanNodeJoinBuildKeyCount(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java index 35a64c26fc1a7..cb00036bffc8d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java @@ -86,6 +86,7 @@ import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @Test(singleThreaded = true) @@ -846,6 +847,31 @@ public void testExplainAnalyze() analyze("EXPLAIN ANALYZE SELECT * FROM t1"); } + @Test + public void testExplainAnalyzeFormatJson() + { + analyze("EXPLAIN ANALYZE (format JSON) SELECT * FROM t1"); + } + + public void testExplainAnalyzeFormatJsonHasStats() + { + analyze("EXPLAIN ANALYZE (format JSON) SELECT * FROM t1"); + } + + @Test + public void testExplainAnalyzeFormatJsonTypeDistributed() + { + analyze("EXPLAIN ANALYZE (format JSON, type DISTRIBUTED) SELECT * FROM t1"); + } + + @Test + public void testExplainAnalyzeIllegalArgs() + { + assertThrows(IllegalStateException.class, () -> analyze("EXPLAIN ANALYZE (type LOGICAL) SELECT * FROM t1")); + assertThrows(IllegalStateException.class, () -> analyze("EXPLAIN ANALYZE (format TEXT, type LOGICAL) SELECT * FROM t1")); + assertThrows(IllegalStateException.class, () -> analyze("EXPLAIN ANALYZE (format JSON, format TEXT) SELECT * FROM t1")); + } + @Test public void testInsert() { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java index 21ef47ef203cf..c503478de7de7 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; +import com.facebook.presto.sql.tree.ExplainFormat; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -67,7 +68,8 @@ public void testValidateFailed() Assignments.of() ), ImmutableList.of(), ImmutableList.of() ), new VariableReferenceExpression(Optional.empty(), "a", BIGINT), - false), + false, + ExplainFormat.Type.TEXT), ImmutableList.of(), ImmutableList.of()); new VerifyOnlyOneOutputNode().validate(root, null, null, null, null, WarningCollector.NOOP); } diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java index 02da574d1eaed..a4e3edfc6fe5b 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java @@ -2015,6 +2015,20 @@ public void testExplainAnalyzeTypeDistributed() new Explain(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("t"))), true, false, ImmutableList.of(new ExplainType(ExplainType.Type.DISTRIBUTED)))); } + @Test + public void testExplainAnalyzeFormatJson() + { + assertStatement("EXPLAIN ANALYZE (format JSON) SELECT * FROM t", + new Explain(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("t"))), true, false, ImmutableList.of(new ExplainFormat(ExplainFormat.Type.JSON)))); + } + + @Test + public void testExplainAnalyzeFormatJsonTypeDistributed() + { + assertStatement("EXPLAIN ANALYZE (format JSON, type DISTRIBUTED) SELECT * FROM t", + new Explain(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("t"))), true, false, ImmutableList.of(new ExplainFormat(ExplainFormat.Type.JSON), new ExplainType(ExplainType.Type.DISTRIBUTED)))); + } + @Test public void testExplainAnalyzeVerbose() { @@ -2029,6 +2043,13 @@ public void testExplainAnalyzeVerboseTypeDistributed() new Explain(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("t"))), true, true, ImmutableList.of(new ExplainType(ExplainType.Type.DISTRIBUTED)))); } + @Test + public void testExplainAnalyzeVerboseFormatJson() + { + assertStatement("EXPLAIN ANALYZE VERBOSE (format JSON) SELECT * FROM t", + new Explain(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("t"))), true, true, ImmutableList.of(new ExplainFormat(ExplainFormat.Type.JSON)))); + } + @Test public void testJoinPrecedence() { diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java index b35e523fafe5d..6ed791e333ad1 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java @@ -21,6 +21,8 @@ import com.facebook.presto.execution.QueryManager; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.sql.planner.plan.PlanFragmentId; +import com.facebook.presto.sql.planner.planPrinter.JsonRenderer; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.TestingSession; @@ -35,6 +37,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; import java.util.regex.Matcher; @@ -367,6 +370,24 @@ public void testExplainAnalyzeVerbose() assertExplainAnalyze("EXPLAIN ANALYZE VERBOSE SELECT rank() OVER (PARTITION BY orderkey ORDER BY clerk DESC) FROM orders WHERE orderkey < 0"); } + private static void assertJsonNodesHaveStats(JsonRenderer.JsonRenderedNode node) + { + assertTrue(node.getStats().isPresent()); + node.getChildren().forEach(AbstractTestDistributedQueries::assertJsonNodesHaveStats); + } + + @Test + public void testExplainAnalyzeFormatJson() + { + JsonRenderer renderer = new JsonRenderer(getQueryRunner().getMetadata().getFunctionAndTypeManager()); + Map fragments = renderer.deserialize((String) computeActual("EXPLAIN ANALYZE (format JSON) SELECT * FROM orders").getOnlyValue()); + fragments.values().forEach(planFragment -> assertJsonNodesHaveStats(planFragment.getPlan())); + fragments = renderer.deserialize((String) computeActual("EXPLAIN ANALYZE (format JSON) SELECT rank() OVER (PARTITION BY orderkey ORDER BY clerk DESC) FROM orders").getOnlyValue()); + fragments.values().forEach(planFragment -> assertJsonNodesHaveStats(planFragment.getPlan())); + fragments = renderer.deserialize((String) computeActual("EXPLAIN ANALYZE (format JSON) SELECT rank() OVER (PARTITION BY orderkey ORDER BY clerk DESC) FROM orders WHERE orderkey < 0").getOnlyValue()); + fragments.values().forEach(planFragment -> assertJsonNodesHaveStats(planFragment.getPlan())); + } + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "EXPLAIN ANALYZE doesn't support statement type: DropTable") public void testExplainAnalyzeDDL() {