From c69a4499a75487ddc9b1a9cb94f4dec06b640861 Mon Sep 17 00:00:00 2001 From: feilong-liu Date: Fri, 20 Oct 2023 13:18:26 -0700 Subject: [PATCH] Add null stats for join probe Collect number of null keys for join, and use it to trigger NULL salt optimization with HBO --- .../HistoryBasedPlanStatisticsTracker.java | 4 +- .../presto/cost/JoinNodeStatsEstimate.java | 32 ++++++- .../presto/cost/PlanNodeStatsEstimate.java | 8 +- .../facebook/presto/operator/JoinProbe.java | 12 +++ .../presto/operator/LookupJoinOperator.java | 8 ++ .../presto/operator/OperatorContext.java | 18 +++- .../presto/operator/OperatorStats.java | 40 ++++++++- .../presto/server/TaskResourceUtils.java | 8 +- .../RandomizeNullKeyInOuterJoin.java | 17 ++-- .../HashCollisionPlanNodeStats.java | 6 +- .../planner/planPrinter/PlanNodeStats.java | 24 ++++- .../planPrinter/PlanNodeStatsSummarizer.java | 12 ++- .../planPrinter/WindowPlanNodeStats.java | 6 +- .../presto/execution/TestQueryStats.java | 8 ++ .../presto/operator/TestOperatorStats.java | 4 + .../planner/assertions/PlanMatchPattern.java | 4 +- ...her.java => StatsJoinKeyCountMatcher.java} | 15 +++- .../spi/statistics/JoinNodeStatistics.java | 33 ++++++- .../TestHistoryBasedStatsTracking.java | 90 +++++++++++++++++-- 19 files changed, 308 insertions(+), 41 deletions(-) rename presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/{StatsJoinBuildKeyCountMatcher.java => StatsJoinKeyCountMatcher.java} (65%) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java index 3ce0e02f4090b..56b7226b43771 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java @@ -148,10 +148,12 @@ public Map getQueryStats(QueryIn double outputBytes = adjustedOutputBytes(planNode, planNodeStats); double nullJoinBuildKeyCount = planNodeStats.getPlanNodeNullJoinBuildKeyCount(); double joinBuildKeyCount = planNodeStats.getPlanNodeJoinBuildKeyCount(); + double nullJoinProbeKeyCount = planNodeStats.getPlanNodeNullJoinProbeKeyCount(); + double joinProbeKeyCount = planNodeStats.getPlanNodeJoinProbeKeyCount(); JoinNodeStatistics joinNodeStatistics = JoinNodeStatistics.empty(); if (planNode instanceof JoinNode) { - joinNodeStatistics = new JoinNodeStatistics(Estimate.of(nullJoinBuildKeyCount), Estimate.of(joinBuildKeyCount)); + joinNodeStatistics = new JoinNodeStatistics(Estimate.of(nullJoinBuildKeyCount), Estimate.of(joinBuildKeyCount), Estimate.of(nullJoinProbeKeyCount), Estimate.of(joinProbeKeyCount)); } TableWriterNodeStatistics tableWriterNodeStatistics = TableWriterNodeStatistics.empty(); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/JoinNodeStatsEstimate.java b/presto-main/src/main/java/com/facebook/presto/cost/JoinNodeStatsEstimate.java index 917cc6e25c951..c35753f420215 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/JoinNodeStatsEstimate.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/JoinNodeStatsEstimate.java @@ -23,16 +23,24 @@ public class JoinNodeStatsEstimate { - private static final JoinNodeStatsEstimate UNKNOWN = new JoinNodeStatsEstimate(NaN, NaN); + private static final JoinNodeStatsEstimate UNKNOWN = new JoinNodeStatsEstimate(NaN, NaN, NaN, NaN); private final double nullJoinBuildKeyCount; private final double joinBuildKeyCount; + private final double nullJoinProbeKeyCount; + private final double joinProbeKeyCount; @JsonCreator - public JoinNodeStatsEstimate(@JsonProperty("nullJoinBuildKeyCount") double nullJoinBuildKeyCount, @JsonProperty("joinBuildKeyCount") double joinBuildKeyCount) + public JoinNodeStatsEstimate( + @JsonProperty("nullJoinBuildKeyCount") double nullJoinBuildKeyCount, + @JsonProperty("joinBuildKeyCount") double joinBuildKeyCount, + @JsonProperty("nullJoinProbeKeyCount") double nullJoinProbeKeyCount, + @JsonProperty("joinProbeKeyCount") double joinProbeKeyCount) { this.nullJoinBuildKeyCount = nullJoinBuildKeyCount; this.joinBuildKeyCount = joinBuildKeyCount; + this.nullJoinProbeKeyCount = nullJoinProbeKeyCount; + this.joinProbeKeyCount = joinProbeKeyCount; } public static JoinNodeStatsEstimate unknown() @@ -52,12 +60,26 @@ public double getJoinBuildKeyCount() return joinBuildKeyCount; } + @JsonProperty + public double getNullJoinProbeKeyCount() + { + return nullJoinProbeKeyCount; + } + + @JsonProperty + public double getJoinProbeKeyCount() + { + return joinProbeKeyCount; + } + @Override public String toString() { return toStringHelper(this) .add("nullJoinBuildKeyCount", nullJoinBuildKeyCount) .add("joinBuildKeyCount", joinBuildKeyCount) + .add("nullJoinProbeKeyCount", nullJoinProbeKeyCount) + .add("joinProbeKeyCount", joinProbeKeyCount) .toString(); } @@ -72,12 +94,14 @@ public boolean equals(Object o) } JoinNodeStatsEstimate that = (JoinNodeStatsEstimate) o; return Double.compare(nullJoinBuildKeyCount, that.nullJoinBuildKeyCount) == 0 && - Double.compare(joinBuildKeyCount, that.joinBuildKeyCount) == 0; + Double.compare(joinBuildKeyCount, that.joinBuildKeyCount) == 0 && + Double.compare(nullJoinProbeKeyCount, that.nullJoinProbeKeyCount) == 0 && + Double.compare(joinProbeKeyCount, that.joinProbeKeyCount) == 0; } @Override public int hashCode() { - return Objects.hash(nullJoinBuildKeyCount, joinBuildKeyCount); + return Objects.hash(nullJoinBuildKeyCount, joinBuildKeyCount, nullJoinProbeKeyCount, joinProbeKeyCount); } } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java b/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java index 34b0d529beffd..8293de807d585 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java @@ -257,7 +257,9 @@ public PlanNodeStatsEstimate combineStats(PlanStatistics planStatistics, SourceI planStatistics.getJoinNodeStatistics().isEmpty() ? getJoinNodeStatsEstimate() : new JoinNodeStatsEstimate( planStatistics.getJoinNodeStatistics().getNullJoinBuildKeyCount().getValue(), - planStatistics.getJoinNodeStatistics().getJoinBuildKeyCount().getValue()), + planStatistics.getJoinNodeStatistics().getJoinBuildKeyCount().getValue(), + planStatistics.getJoinNodeStatistics().getNullJoinProbeKeyCount().getValue(), + planStatistics.getJoinNodeStatistics().getJoinProbeKeyCount().getValue()), planStatistics.getTableWriterNodeStatistics().isEmpty() ? getTableWriterNodeStatsEstimate() : new TableWriterNodeStatsEstimate(planStatistics.getTableWriterNodeStatistics().getTaskCountIfScaledWriter().getValue())); } @@ -309,7 +311,9 @@ public PlanStatisticsWithSourceInfo toPlanStatisticsWithSourceInfo(PlanNodeId id sourceInfo.isConfident() ? 1 : 0, new JoinNodeStatistics( Estimate.estimateFromDouble(joinNodeStatsEstimate.getNullJoinBuildKeyCount()), - Estimate.estimateFromDouble(joinNodeStatsEstimate.getJoinBuildKeyCount())), + Estimate.estimateFromDouble(joinNodeStatsEstimate.getJoinBuildKeyCount()), + Estimate.estimateFromDouble(joinNodeStatsEstimate.getNullJoinProbeKeyCount()), + Estimate.estimateFromDouble(joinNodeStatsEstimate.getJoinProbeKeyCount())), new TableWriterNodeStatistics(Estimate.estimateFromDouble(tableWriterNodeStatsEstimate.getTaskCountIfScaledWriter()))), sourceInfo); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/JoinProbe.java b/presto-main/src/main/java/com/facebook/presto/operator/JoinProbe.java index 37ebf777df5af..245176d8988e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/JoinProbe.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/JoinProbe.java @@ -55,6 +55,7 @@ public JoinProbe createJoinProbe(Page page) private final boolean probeMayHaveNull; private int position = -1; + private int nullRowCount; private JoinProbe(int[] probeOutputChannels, Page page, Page probePage, @Nullable Block probeHashBlock) { @@ -79,6 +80,7 @@ public boolean advanceNextPosition() public long getCurrentJoinPosition(LookupSource lookupSource) { if (probeMayHaveNull && currentRowContainsNull()) { + ++nullRowCount; return -1; } if (probeHashBlock != null) { @@ -117,4 +119,14 @@ private static boolean probeMayHaveNull(Page probePage) } return false; } + + public int getNullRowCount() + { + return nullRowCount; + } + + public int getPositionCount() + { + return positionCount; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java index 9bdc3e27c79f7..276a0b428f760 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java @@ -96,6 +96,8 @@ public class LookupJoinOperator private Optional>> unspilledLookupSource = Optional.empty(); private Iterator unspilledInputPages = emptyIterator(); private final boolean optimizeProbeForEmptyBuild; + private long nullProbeRowCount; + private long inputProbeRowCount; public LookupJoinOperator( OperatorContext operatorContext, @@ -404,6 +406,8 @@ private void tryUnspillNext() lookupSourceProvider = null; } spiller.ifPresent(PartitioningSpiller::verifyAllPartitionsRead); + operatorContext.recordJoinProbeKeyCount(inputProbeRowCount); + operatorContext.recordNullJoinProbeKeyCount(nullProbeRowCount); finished = true; } @@ -715,6 +719,10 @@ private void buildPage() private void clearProbe() { // Before updating the probe flush the current page + if (probe != null) { + nullProbeRowCount += probe.getNullRowCount(); + inputProbeRowCount += probe.getPositionCount(); + } buildPage(); probe = null; } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java b/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java index 46f0a6055c8ba..7b6af9a9a2b76 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java @@ -77,6 +77,10 @@ public class OperatorContext private final AtomicLong nullJoinBuildKeyCount = new AtomicLong(); // Number of elements in hash table for join operator private final AtomicLong joinBuildKeyCount = new AtomicLong(); + // Number of NULL probe rows for join operator + private final AtomicLong nullJoinProbeKeyCount = new AtomicLong(); + // Number of probe rows for join operator + private final AtomicLong joinProbeKeyCount = new AtomicLong(); private final AtomicLong additionalCpuNanos = new AtomicLong(); @@ -236,6 +240,16 @@ public void recordJoinBuildKeyCount(long positions) joinBuildKeyCount.getAndAdd(positions); } + public void recordNullJoinProbeKeyCount(long positions) + { + nullJoinProbeKeyCount.getAndAdd(positions); + } + + public void recordJoinProbeKeyCount(long positions) + { + joinProbeKeyCount.getAndAdd(positions); + } + public void recordPhysicalWrittenData(long sizeInBytes) { physicalWrittenDataSize.getAndAdd(sizeInBytes); @@ -562,7 +576,9 @@ public OperatorStats getOperatorStats() info, runtimeStats, nullJoinBuildKeyCount.get(), - joinBuildKeyCount.get()); + joinBuildKeyCount.get(), + nullJoinProbeKeyCount.get(), + joinProbeKeyCount.get()); } public R accept(QueryContextVisitor visitor, C context) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java b/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java index 3c2950f083981..f0069f6773c1e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java @@ -97,6 +97,8 @@ public class OperatorStats private final long nullJoinBuildKeyCount; private final long joinBuildKeyCount; + private final long nullJoinProbeKeyCount; + private final long joinProbeKeyCount; @JsonCreator public OperatorStats( @@ -151,7 +153,9 @@ public OperatorStats( @JsonProperty("info") OperatorInfo info, @JsonProperty("runtimeStats") RuntimeStats runtimeStats, @JsonProperty("nullJoinBuildKeyCount") long nullJoinBuildKeyCount, - @JsonProperty("joinBuildKeyCount") long joinBuildKeyCount) + @JsonProperty("joinBuildKeyCount") long joinBuildKeyCount, + @JsonProperty("nullJoinProbeKeyCount") long nullJoinProbeKeyCount, + @JsonProperty("joinProbeKeyCount") long joinProbeKeyCount) { this.stageId = stageId; this.stageExecutionId = stageExecutionId; @@ -211,6 +215,8 @@ public OperatorStats( this.infoUnion = null; this.nullJoinBuildKeyCount = nullJoinBuildKeyCount; this.joinBuildKeyCount = joinBuildKeyCount; + this.nullJoinProbeKeyCount = nullJoinProbeKeyCount; + this.joinProbeKeyCount = joinProbeKeyCount; } @ThriftConstructor @@ -266,7 +272,9 @@ public OperatorStats( @Nullable OperatorInfoUnion infoUnion, long nullJoinBuildKeyCount, - long joinBuildKeyCount) + long joinBuildKeyCount, + long nullJoinProbeKeyCount, + long joinProbeKeyCount) { this.stageId = stageId; this.stageExecutionId = stageExecutionId; @@ -326,6 +334,8 @@ public OperatorStats( this.info = null; this.nullJoinBuildKeyCount = nullJoinBuildKeyCount; this.joinBuildKeyCount = joinBuildKeyCount; + this.nullJoinProbeKeyCount = nullJoinProbeKeyCount; + this.joinProbeKeyCount = joinProbeKeyCount; } @JsonProperty @@ -623,6 +633,20 @@ public long getJoinBuildKeyCount() return joinBuildKeyCount; } + @JsonProperty + @ThriftField(42) + public long getNullJoinProbeKeyCount() + { + return nullJoinProbeKeyCount; + } + + @JsonProperty + @ThriftField(43) + public long getJoinProbeKeyCount() + { + return joinProbeKeyCount; + } + public OperatorStats add(OperatorStats operatorStats) { return add(ImmutableList.of(operatorStats)); @@ -674,6 +698,8 @@ public OperatorStats add(Iterable operators) long nullJoinBuildKeyCount = this.nullJoinBuildKeyCount; long joinBuildKeyCount = this.joinBuildKeyCount; + long nullJoinProbeKeyCount = this.nullJoinProbeKeyCount; + long joinProbeKeyCount = this.joinProbeKeyCount; Mergeable base = getMergeableInfoOrNull(info); for (OperatorStats operator : operators) { @@ -731,6 +757,8 @@ public OperatorStats add(Iterable operators) nullJoinBuildKeyCount += operator.getNullJoinBuildKeyCount(); joinBuildKeyCount += operator.getJoinBuildKeyCount(); + nullJoinProbeKeyCount += operator.getNullJoinProbeKeyCount(); + joinProbeKeyCount += operator.getJoinProbeKeyCount(); } return new OperatorStats( @@ -784,7 +812,9 @@ public OperatorStats add(Iterable operators) (OperatorInfo) base, runtimeStats, nullJoinBuildKeyCount, - joinBuildKeyCount); + joinBuildKeyCount, + nullJoinProbeKeyCount, + joinProbeKeyCount); } @SuppressWarnings("unchecked") @@ -850,6 +880,8 @@ public OperatorStats summarize() info, runtimeStats, nullJoinBuildKeyCount, - joinBuildKeyCount); + joinBuildKeyCount, + nullJoinProbeKeyCount, + joinProbeKeyCount); } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java b/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java index 4e02e731459ea..34463273b7eb7 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/server/TaskResourceUtils.java @@ -250,7 +250,9 @@ private static OperatorStats convertToThriftOperatorStats(OperatorStats operator operatorStats.getRuntimeStats(), convertToOperatorInfoUnion(operatorStats.getInfo()), operatorStats.getNullJoinBuildKeyCount(), - operatorStats.getJoinBuildKeyCount()); + operatorStats.getJoinBuildKeyCount(), + operatorStats.getNullJoinProbeKeyCount(), + operatorStats.getJoinProbeKeyCount()); } private static MetadataUpdates convertToThriftMetadataUpdates( @@ -482,7 +484,9 @@ private static OperatorStats convertFromThriftOperatorStats(OperatorStats thrift convertToOperatorInfo(thriftOperatorStats.getInfoUnion()), thriftOperatorStats.getRuntimeStats(), thriftOperatorStats.getNullJoinBuildKeyCount(), - thriftOperatorStats.getJoinBuildKeyCount()); + thriftOperatorStats.getJoinBuildKeyCount(), + thriftOperatorStats.getNullJoinProbeKeyCount(), + thriftOperatorStats.getJoinProbeKeyCount()); } private static MetadataUpdates convertFromThriftMetadataUpdates( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java index 7599a39bbcf12..37c53f118f708 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java @@ -193,6 +193,7 @@ private static class Rewriter extends SimplePlanRewriter> { private static final double NULL_BUILD_KEY_COUNT_THRESHOLD = 100_000; + private static final double NULL_PROBE_KEY_COUNT_THRESHOLD = 100_000; private static final String LEFT_PREFIX = "l"; private static final String RIGHT_PREFIX = "r"; private final Session session; @@ -286,11 +287,7 @@ public PlanNode visitJoin(JoinNode joinNode, RewriteContext NULL_BUILD_KEY_COUNT_THRESHOLD - && joinEstimate.getNullJoinBuildKeyCount() / joinEstimate.getJoinBuildKeyCount() > getRandomizeOuterJoinNullKeyNullRatioThreshold(session); - + boolean enabledByCostModel = strategy.equals(COST_BASED) && hasNullSkew(statsProvider.getStats(joinNode).getJoinNodeStatsEstimate()); List candidateEquiJoinClauses = joinNode.getCriteria().stream() .filter(x -> isSupportedType(x.getLeft()) && isSupportedType(x.getRight())) .filter(x -> enabledByCostModel || strategy.equals(ALWAYS) || enabledForJoinKeyFromOuterJoin(context.get(), x)) @@ -382,6 +379,16 @@ public PlanNode visitJoin(JoinNode joinNode, RewriteContext NULL_BUILD_KEY_COUNT_THRESHOLD + && joinEstimate.getNullJoinBuildKeyCount() / joinEstimate.getJoinBuildKeyCount() > getRandomizeOuterJoinNullKeyNullRatioThreshold(session)) + || (joinEstimate.getNullJoinProbeKeyCount() > NULL_PROBE_KEY_COUNT_THRESHOLD + && joinEstimate.getNullJoinProbeKeyCount() / joinEstimate.getJoinProbeKeyCount() > getRandomizeOuterJoinNullKeyNullRatioThreshold(session))); + } + private RowExpression randomizeJoinKey(RowExpression keyExpression, String prefix) { int partitionCount = getHashPartitionCount(session); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java index cc4b7c3b536d0..55a0a4b352ecb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java @@ -43,10 +43,12 @@ public HashCollisionPlanNodeStats( Map operatorInputStats, long planNodeNullJoinBuildKeyCount, long planNodeJoinBuildKeyCount, + long planNodeNullJoinProbeKeyCount, + long planNodeJoinProbeKeyCount, Map operatorHashCollisionsStats) { super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, - planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount); + planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount, planNodeNullJoinProbeKeyCount, planNodeJoinProbeKeyCount); this.operatorHashCollisionsStats = requireNonNull(operatorHashCollisionsStats, "operatorHashCollisionsStats is null"); } @@ -104,6 +106,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.operatorInputStats, merged.getPlanNodeNullJoinBuildKeyCount(), merged.getPlanNodeJoinBuildKeyCount(), + merged.getPlanNodeNullJoinProbeKeyCount(), + merged.getPlanNodeJoinProbeKeyCount(), operatorHashCollisionsStats); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java index a23071ce7d09e..b3d0d290294bd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java @@ -47,6 +47,8 @@ public class PlanNodeStats protected final Map operatorInputStats; private final long planNodeNullJoinBuildKeyCount; private final long planNodeJoinBuildKeyCount; + private final long planNodeNullJoinProbeKeyCount; + private final long planNodeJoinProbeKeyCount; PlanNodeStats( PlanNodeId planNodeId, @@ -60,7 +62,9 @@ public class PlanNodeStats DataSize planNodeOutputDataSize, Map operatorInputStats, long planNodeNullJoinBuildKeyCount, - long planNodeJoinBuildKeyCount) + long planNodeJoinBuildKeyCount, + long planNodeNullJoinProbeKeyCount, + long planNodeJoinProbeKeyCount) { this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); @@ -76,6 +80,8 @@ public class PlanNodeStats this.operatorInputStats = requireNonNull(operatorInputStats, "operatorInputStats is null"); this.planNodeNullJoinBuildKeyCount = planNodeNullJoinBuildKeyCount; this.planNodeJoinBuildKeyCount = planNodeJoinBuildKeyCount; + this.planNodeNullJoinProbeKeyCount = planNodeNullJoinProbeKeyCount; + this.planNodeJoinProbeKeyCount = planNodeJoinProbeKeyCount; } private static double computedStdDev(double sumSquared, double sum, long n) @@ -165,6 +171,16 @@ public long getPlanNodeJoinBuildKeyCount() return planNodeJoinBuildKeyCount; } + public long getPlanNodeNullJoinProbeKeyCount() + { + return planNodeNullJoinProbeKeyCount; + } + + public long getPlanNodeJoinProbeKeyCount() + { + return planNodeJoinProbeKeyCount; + } + @Override public PlanNodeStats mergeWith(PlanNodeStats other) { @@ -180,6 +196,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) Map operatorInputStats = mergeMaps(this.operatorInputStats, other.operatorInputStats, OperatorInputStats::merge); long planNodeNullJoinBuildKeyCount = this.planNodeNullJoinBuildKeyCount + other.planNodeNullJoinBuildKeyCount; long planNodeJoinBuildKeyCount = this.planNodeJoinBuildKeyCount + other.planNodeJoinBuildKeyCount; + long planNodeNullJoinProbeKeyCount = this.planNodeNullJoinProbeKeyCount + other.planNodeNullJoinProbeKeyCount; + long planNodeJoinProbeKeyCount = this.planNodeJoinProbeKeyCount + other.planNodeJoinProbeKeyCount; return new PlanNodeStats( planNodeId, @@ -190,6 +208,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, planNodeNullJoinBuildKeyCount, - planNodeJoinBuildKeyCount); + planNodeJoinBuildKeyCount, + planNodeNullJoinProbeKeyCount, + planNodeJoinProbeKeyCount); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java index f1a7d5d660565..f4aab51181d0c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java @@ -82,6 +82,8 @@ private static List getPlanNodeStats(TaskStats taskStats) Map planNodeCpuMillis = new HashMap<>(); Map planNodeNullJoinBuildKeyCount = new HashMap<>(); Map planNodeJoinBuildKeyCount = new HashMap<>(); + Map planNodeNullJoinProbeKeyCount = new HashMap<>(); + Map planNodeJoinProbeKeyCount = new HashMap<>(); Map> operatorInputStats = new HashMap<>(); Map> operatorHashCollisionsStats = new HashMap<>(); @@ -153,6 +155,8 @@ private static List getPlanNodeStats(TaskStats taskStats) planNodeNullJoinBuildKeyCount.merge(planNodeId, operatorStats.getNullJoinBuildKeyCount(), Long::sum); planNodeJoinBuildKeyCount.merge(planNodeId, operatorStats.getJoinBuildKeyCount(), Long::sum); + planNodeNullJoinProbeKeyCount.merge(planNodeId, operatorStats.getNullJoinProbeKeyCount(), Long::sum); + planNodeJoinProbeKeyCount.merge(planNodeId, operatorStats.getJoinProbeKeyCount(), Long::sum); processedNodes.add(planNodeId); } @@ -212,6 +216,8 @@ private static List getPlanNodeStats(TaskStats taskStats) operatorInputStats.get(planNodeId), planNodeNullJoinBuildKeyCount.get(planNodeId), planNodeJoinBuildKeyCount.get(planNodeId), + planNodeNullJoinProbeKeyCount.get(planNodeId), + planNodeJoinProbeKeyCount.get(planNodeId), operatorHashCollisionsStats.get(planNodeId)); } else if (windowNodeStats.containsKey(planNodeId)) { @@ -228,6 +234,8 @@ else if (windowNodeStats.containsKey(planNodeId)) { operatorInputStats.get(planNodeId), planNodeNullJoinBuildKeyCount.get(planNodeId), planNodeJoinBuildKeyCount.get(planNodeId), + planNodeNullJoinProbeKeyCount.get(planNodeId), + planNodeJoinProbeKeyCount.get(planNodeId), windowNodeStats.get(planNodeId)); } else { @@ -243,7 +251,9 @@ else if (windowNodeStats.containsKey(planNodeId)) { succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId), planNodeNullJoinBuildKeyCount.get(planNodeId), - planNodeJoinBuildKeyCount.get(planNodeId)); + planNodeJoinBuildKeyCount.get(planNodeId), + planNodeNullJoinProbeKeyCount.get(planNodeId), + planNodeJoinProbeKeyCount.get(planNodeId)); } stats.add(nodeStats); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java index 0bf88149d0884..6ce97a3764be5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java @@ -39,10 +39,12 @@ public WindowPlanNodeStats( Map operatorInputStats, long planNodeNullJoinBuildKeyCount, long planNodeJoinBuildKeyCount, + long planNodeNullJoinProbeKeyCount, + long planNodeJoinProbeKeyCount, WindowOperatorStats windowOperatorStats) { super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, - planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount); + planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, planNodeNullJoinBuildKeyCount, planNodeJoinBuildKeyCount, planNodeNullJoinProbeKeyCount, planNodeJoinProbeKeyCount); this.windowOperatorStats = windowOperatorStats; } @@ -70,6 +72,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.operatorInputStats, merged.getPlanNodeNullJoinBuildKeyCount(), merged.getPlanNodeJoinBuildKeyCount(), + merged.getPlanNodeNullJoinProbeKeyCount(), + merged.getPlanNodeJoinProbeKeyCount(), windowOperatorStats); } } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java index ea17019d11673..d81043faf51c2 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java @@ -99,6 +99,8 @@ public class TestQueryStats null, new RuntimeStats(ImmutableMap.of(TEST_METRIC_NAME, RuntimeMetric.copyOf(TEST_RUNTIME_METRIC_1))), 0, + 0, + 0, 0), new OperatorStats( 20, @@ -141,6 +143,8 @@ public class TestQueryStats null, new RuntimeStats(ImmutableMap.of(TEST_METRIC_NAME, RuntimeMetric.copyOf(TEST_RUNTIME_METRIC_2))), 0, + 0, + 0, 0), new OperatorStats( 30, @@ -183,6 +187,8 @@ public class TestQueryStats null, new RuntimeStats(), 0, + 0, + 0, 0)); static final QueryStats EXPECTED = new QueryStats( @@ -545,6 +551,8 @@ private static OperatorStats createOperatorStats(int stageId, int stageExecution null, new RuntimeStats(ImmutableMap.of(TEST_METRIC_NAME, RuntimeMetric.copyOf(TEST_RUNTIME_METRIC_1))), 0, + 0, + 0, 0); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java index 6b4e27de005b4..1c86855c14b84 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java @@ -89,6 +89,8 @@ public class TestOperatorStats NON_MERGEABLE_INFO, new RuntimeStats(ImmutableMap.of(TEST_METRIC_NAME, RuntimeMetric.copyOf(TEST_RUNTIME_METRIC_1))), 0, + 0, + 0, 0); public static final OperatorStats MERGEABLE = new OperatorStats( @@ -139,6 +141,8 @@ public class TestOperatorStats MERGEABLE_INFO, new RuntimeStats(ImmutableMap.of(TEST_METRIC_NAME, RuntimeMetric.copyOf(TEST_RUNTIME_METRIC_2))), 0, + 0, + 0, 0); @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java index c65094af82ca4..17d6e766c3e38 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java @@ -761,9 +761,9 @@ public PlanMatchPattern withOutputSize(double expectedOutputSize) return this; } - public PlanMatchPattern withJoinBuildKeyStatistics(double expectedJoinBuildKeyCount, double expectedNullJoinBuildKeyCount) + public PlanMatchPattern withJoinStatistics(double expectedJoinBuildKeyCount, double expectedNullJoinBuildKeyCount, double expectedJoinProbeKeyCount, double expectedNullJoinProbeKeyCount) { - matchers.add(new StatsJoinBuildKeyCountMatcher(expectedJoinBuildKeyCount, expectedNullJoinBuildKeyCount)); + matchers.add(new StatsJoinKeyCountMatcher(expectedJoinBuildKeyCount, expectedNullJoinBuildKeyCount, expectedJoinProbeKeyCount, expectedNullJoinProbeKeyCount)); return this; } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StatsJoinBuildKeyCountMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StatsJoinKeyCountMatcher.java similarity index 65% rename from presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StatsJoinBuildKeyCountMatcher.java rename to presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StatsJoinKeyCountMatcher.java index e09ffe4169905..0ad9da706888b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StatsJoinBuildKeyCountMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StatsJoinKeyCountMatcher.java @@ -18,16 +18,20 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.plan.PlanNode; -public class StatsJoinBuildKeyCountMatcher +public class StatsJoinKeyCountMatcher implements Matcher { private final double expectedJoinBuildKeyCount; private final double expectedNullJoinBuildKeyCount; + private final double expectedJoinProbeKeyCount; + private final double expectedNullJoinProbeKeyCount; - StatsJoinBuildKeyCountMatcher(double expectedJoinBuildKeyCount, double expectedNullJoinBuildKeyCount) + StatsJoinKeyCountMatcher(double expectedJoinBuildKeyCount, double expectedNullJoinBuildKeyCount, double expectedJoinProbeKeyCount, double expectedNullJoinProbeKeyCount) { this.expectedJoinBuildKeyCount = expectedJoinBuildKeyCount; this.expectedNullJoinBuildKeyCount = expectedNullJoinBuildKeyCount; + this.expectedJoinProbeKeyCount = expectedJoinProbeKeyCount; + this.expectedNullJoinProbeKeyCount = expectedNullJoinProbeKeyCount; } @Override @@ -40,12 +44,15 @@ public boolean shapeMatches(PlanNode node) public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) { return new MatchResult(Double.compare(stats.getStats(node).getJoinNodeStatsEstimate().getJoinBuildKeyCount(), expectedJoinBuildKeyCount) == 0 - && Double.compare(stats.getStats(node).getJoinNodeStatsEstimate().getNullJoinBuildKeyCount(), expectedNullJoinBuildKeyCount) == 0); + && Double.compare(stats.getStats(node).getJoinNodeStatsEstimate().getNullJoinBuildKeyCount(), expectedNullJoinBuildKeyCount) == 0 + && Double.compare(stats.getStats(node).getJoinNodeStatsEstimate().getJoinProbeKeyCount(), expectedJoinProbeKeyCount) == 0 + && Double.compare(stats.getStats(node).getJoinNodeStatsEstimate().getNullJoinProbeKeyCount(), expectedNullJoinProbeKeyCount) == 0); } @Override public String toString() { - return "expectedJoinBuildKeyCount(" + expectedJoinBuildKeyCount + ")" + " expectedNullJoinBuildKeyCount(" + expectedNullJoinBuildKeyCount + ")"; + return "expectedJoinBuildKeyCount(" + expectedJoinBuildKeyCount + ")" + " expectedNullJoinBuildKeyCount(" + expectedNullJoinBuildKeyCount + ")" + + " expectedNullJoinProbeKeyCount(" + expectedNullJoinProbeKeyCount + ")" + " expectedNullJoinProbeKeyCount(" + expectedNullJoinProbeKeyCount + ")"; } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/JoinNodeStatistics.java b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/JoinNodeStatistics.java index 7123ee0991041..8a49b1bf2898f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/JoinNodeStatistics.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/JoinNodeStatistics.java @@ -26,20 +26,28 @@ @ThriftStruct public class JoinNodeStatistics { - private static final JoinNodeStatistics EMPTY = new JoinNodeStatistics(Estimate.unknown(), Estimate.unknown()); + private static final JoinNodeStatistics EMPTY = new JoinNodeStatistics(Estimate.unknown(), Estimate.unknown(), Estimate.unknown(), Estimate.unknown()); // Number of input rows from build side of a join which has at least one join column to be NULL private final Estimate nullJoinBuildKeyCount; // Number of input rows from build side of a join private final Estimate joinBuildKeyCount; + // Number of input rows from probe side of a join which has at least one join column to be NULL + private final Estimate nullJoinProbeKeyCount; + // Number of input rows from probe side of a join + private final Estimate joinProbeKeyCount; @JsonCreator @ThriftConstructor public JoinNodeStatistics( @JsonProperty("nullJoinBuildKeyCount") Estimate nullJoinBuildKeyCount, - @JsonProperty("joinBuildKeyCount") Estimate joinBuildKeyCount) + @JsonProperty("joinBuildKeyCount") Estimate joinBuildKeyCount, + @JsonProperty("nullJoinProbeKeyCount") Estimate nullJoinProbeKeyCount, + @JsonProperty("joinProbeKeyCount") Estimate joinProbeKeyCount) { this.nullJoinBuildKeyCount = requireNonNull(nullJoinBuildKeyCount, "nullJoinBuildKeyCount is null"); this.joinBuildKeyCount = requireNonNull(joinBuildKeyCount, "joinBuildKeyCount is null"); + this.nullJoinProbeKeyCount = requireNonNull(nullJoinProbeKeyCount, "nullJoinProbeKeyCount is null"); + this.joinProbeKeyCount = requireNonNull(joinProbeKeyCount, "joinProbeKeyCount is null"); } public static JoinNodeStatistics empty() @@ -66,6 +74,20 @@ public Estimate getJoinBuildKeyCount() return joinBuildKeyCount; } + @JsonProperty + @ThriftField(3) + public Estimate getNullJoinProbeKeyCount() + { + return nullJoinProbeKeyCount; + } + + @JsonProperty + @ThriftField(4) + public Estimate getJoinProbeKeyCount() + { + return joinProbeKeyCount; + } + @Override public boolean equals(Object o) { @@ -76,13 +98,14 @@ public boolean equals(Object o) return false; } JoinNodeStatistics that = (JoinNodeStatistics) o; - return Objects.equals(nullJoinBuildKeyCount, that.nullJoinBuildKeyCount) && Objects.equals(joinBuildKeyCount, that.joinBuildKeyCount); + return Objects.equals(nullJoinBuildKeyCount, that.nullJoinBuildKeyCount) && Objects.equals(joinBuildKeyCount, that.joinBuildKeyCount) + && Objects.equals(nullJoinProbeKeyCount, that.nullJoinProbeKeyCount) && Objects.equals(joinProbeKeyCount, that.joinProbeKeyCount); } @Override public int hashCode() { - return Objects.hash(nullJoinBuildKeyCount, joinBuildKeyCount); + return Objects.hash(nullJoinBuildKeyCount, joinBuildKeyCount, nullJoinProbeKeyCount, joinProbeKeyCount); } @Override @@ -91,6 +114,8 @@ public String toString() return "JoinNodeStatistics{" + "nullJoinBuildKeyCount=" + nullJoinBuildKeyCount + ", joinBuildKeyCount=" + joinBuildKeyCount + + ", nullJoinProbeKeyCount=" + nullJoinProbeKeyCount + + ", joinProbeKeyCount=" + joinProbeKeyCount + '}'; } } diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java index 88424a4a104a5..f0cbcf4bb4b5a 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java @@ -252,18 +252,18 @@ public void testJoin() { assertPlan( "SELECT N.name, O.totalprice, C.name FROM orders O, customer C, nation N WHERE N.nationkey = C.nationkey and C.custkey = O.custkey and year(O.orderdate) = 1995 AND substr(N.name, 1, 1) >= 'C'", - anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(Double.NaN).withJoinBuildKeyStatistics(Double.NaN, Double.NaN))); + anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(Double.NaN).withJoinStatistics(Double.NaN, Double.NaN, Double.NaN, Double.NaN))); assertPlan( "SELECT N.name, O.totalprice, C.name FROM orders O, customer C, nation N WHERE N.nationkey = C.nationkey and C.custkey = O.custkey and year(O.orderdate) = 1995 AND substr(N.name, 1, 1) >= 'C'", - anyTree(node(JoinNode.class, anyTree(anyTree(any()), anyTree(any())), anyTree(any())).withOutputRowCount(Double.NaN).withJoinBuildKeyStatistics(Double.NaN, Double.NaN))); + anyTree(node(JoinNode.class, anyTree(anyTree(any()), anyTree(any())), anyTree(any())).withOutputRowCount(Double.NaN).withJoinStatistics(Double.NaN, Double.NaN, Double.NaN, Double.NaN))); executeAndTrackHistory("SELECT N.name, O.totalprice, C.name FROM orders O, customer C, nation N WHERE N.nationkey = C.nationkey and C.custkey = O.custkey and year(O.orderdate) = 1995 AND substr(N.name, 1, 1) >= 'C'"); assertPlan( "SELECT N.name, O.totalprice, C.name FROM orders O, customer C, nation N WHERE N.nationkey = C.nationkey and C.custkey = O.custkey and year(O.orderdate) = 1995 AND substr(N.name, 1, 1) >= 'C'", - anyTree(node(JoinNode.class, anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(2204).withJoinBuildKeyStatistics(1500, 0)), anyTree(any())))); + anyTree(node(JoinNode.class, anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(2204).withJoinStatistics(1500, 0, 2204, 0)), anyTree(any())))); assertPlan( "SELECT N.name, O.totalprice, C.name FROM orders O, customer C, nation N WHERE N.nationkey = C.nationkey and C.custkey = O.custkey and year(O.orderdate) = 1995 AND substr(N.name, 1, 1) >= 'C'", - anyTree(node(JoinNode.class, anyTree(anyTree(any()), anyTree(any())), anyTree(any())).withOutputRowCount(1915).withJoinBuildKeyStatistics(22, 0))); + anyTree(node(JoinNode.class, anyTree(anyTree(any()), anyTree(any())), anyTree(any())).withOutputRowCount(1915).withJoinStatistics(22, 0, 2204, 0))); // Check that output size doesn't include hash variables assertPlan( "SELECT N.name, O.totalprice, C.name FROM orders O, customer C, nation N WHERE N.nationkey = C.nationkey and C.custkey = O.custkey and year(O.orderdate) = 1995 AND substr(N.name, 1, 1) >= 'C'", @@ -271,16 +271,92 @@ public void testJoin() } @Test - public void testJoinWithNull() + public void testJoinWithBuildNull() { assertPlan( "SELECT O.totalprice, C.name FROM orders O JOIN (SELECT name, custkey FROM customer UNION ALL SELECT * FROM (VALUES ('unknown', NULL)) t(name, custkey)) C ON C.custkey = O.custkey AND YEAR(O.orderdate) = 1995", - anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(Double.NaN).withJoinBuildKeyStatistics(Double.NaN, Double.NaN))); + anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(Double.NaN).withJoinStatistics(Double.NaN, Double.NaN, Double.NaN, Double.NaN))); executeAndTrackHistory("SELECT O.totalprice, C.name FROM orders O JOIN (SELECT name, custkey FROM customer UNION ALL SELECT * FROM (VALUES ('unknown', NULL)) t(name, custkey)) C ON C.custkey = O.custkey AND YEAR(O.orderdate) = 1995"); assertPlan( "SELECT O.totalprice, C.name FROM orders O JOIN (SELECT name, custkey FROM customer UNION ALL SELECT * FROM (VALUES ('unknown', NULL)) t(name, custkey)) C ON C.custkey = O.custkey AND YEAR(O.orderdate) = 1995", - anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(2204).withJoinBuildKeyStatistics(1501, 1))); + anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(2204).withJoinStatistics(1501, 1, 2204, 0))); + } + + @Test + public void testJoinWithProbeNull() + { + String sql = "SELECT\n" + + " O.totalprice,\n" + + " C.name\n" + + "FROM (\n" + + " SELECT\n" + + " totalprice,\n" + + " custkey,\n" + + " orderdate\n" + + " FROM orders\n" + + "\n" + + " UNION ALL\n" + + "\n" + + " SELECT\n" + + " *\n" + + " FROM (\n" + + " VALUES\n" + + " (10.0, NULL, date '1995-09-20')\n" + + " )\n" + + ") O\n" + + "JOIN customer C\n" + + " ON C.custkey = O.custkey\n" + + " AND YEAR(O.orderdate) = 1995"; + assertPlan(sql, anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(Double.NaN).withJoinStatistics(Double.NaN, Double.NaN, Double.NaN, Double.NaN))); + + executeAndTrackHistory(sql); + assertPlan(sql, anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(2204).withJoinStatistics(1500, 0, 2205, 1))); + } + + @Test + public void testJoinNull() + { + String sql = "SELECT\n" + + " O.totalprice,\n" + + " C.name\n" + + "FROM (\n" + + " SELECT\n" + + " totalprice,\n" + + " custkey,\n" + + " orderdate\n" + + " FROM orders\n" + + "\n" + + " UNION ALL\n" + + "\n" + + " SELECT\n" + + " *\n" + + " FROM (\n" + + " VALUES\n" + + " (10.0, NULL, DATE '1995-09-20')\n" + + " )\n" + + ") O\n" + + "JOIN (\n" + + " SELECT\n" + + " name,\n" + + " custkey\n" + + " FROM customer\n" + + "\n" + + " UNION ALL\n" + + "\n" + + " SELECT\n" + + " *\n" + + " FROM (\n" + + " VALUES\n" + + " ('unknown', NULL)\n" + + " ) t(name, custkey)\n" + + ") C\n" + + " ON C.custkey = O.custkey\n" + + " AND YEAR(O.orderdate) = 1995"; + assertPlan(sql, anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(Double.NaN).withJoinStatistics(Double.NaN, Double.NaN, Double.NaN, Double.NaN))); + + executeAndTrackHistory(sql); + assertPlan(sql, anyTree(node(JoinNode.class, anyTree(any()), anyTree(any())).withOutputRowCount(2204).withJoinStatistics(1501, 1, 2205, 1))); } @Test