From 1708e561d8425f2c4d8530e3750d143c1a080894 Mon Sep 17 00:00:00 2001 From: "tanel.kiis@gmail.com" Date: Tue, 5 Jan 2021 19:18:06 +0200 Subject: [PATCH 1/2] Ignore Distinct if it is the right child of a left semi or anti join --- .../sql/catalyst/optimizer/Optimizer.scala | 7 +- .../optimizer/ReplaceOperatorSuite.scala | 24 ++ .../approved-plans-v1_4/q38.sf100/explain.txt | 254 ++++++++---------- .../q38.sf100/simplified.txt | 106 ++++---- .../approved-plans-v1_4/q38/explain.txt | 216 ++++++--------- .../approved-plans-v1_4/q38/simplified.txt | 70 +++-- .../approved-plans-v1_4/q87.sf100/explain.txt | 254 ++++++++---------- .../q87.sf100/simplified.txt | 106 ++++---- .../approved-plans-v1_4/q87/explain.txt | 216 ++++++--------- .../approved-plans-v1_4/q87/simplified.txt | 70 +++-- 10 files changed, 571 insertions(+), 752 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 47260cfb59bb..fde03f7d64fe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -176,7 +176,7 @@ abstract class Optimizer(catalogManager: CatalogManager) // to enforce idempotence on it and we change this batch from Once to FixedPoint(1). Batch("Subquery", FixedPoint(1), OptimizeSubqueries) :: - Batch("Replace Operators", fixedPoint, + Batch("Replace Operators", Once, RewriteExceptAll, RewriteIntersectAll, ReplaceIntersectWithSemiJoin, @@ -1612,9 +1612,12 @@ object ConvertToLocalRelation extends Rule[LogicalPlan] { * {{{ * SELECT DISTINCT f1, f2 FROM t ==> SELECT f1, f2 FROM t GROUP BY f1, f2 * }}} + * + * The [[Distinct]] can be ignored if it is the right child of a left semi or anti join. */ object ReplaceDistinctWithAggregate extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transform { + def apply(plan: LogicalPlan): LogicalPlan = plan transformDown { + case j @ Join(_, Distinct(right), LeftSemiOrAnti(_), _, _) => j.copy(right = right) case Distinct(child) => Aggregate(child.output, child.output, child) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala index 9bf864f5201f..1148beb9e3d9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala @@ -191,6 +191,30 @@ class ReplaceOperatorSuite extends PlanTest { comparePlans(optimized, correctAnswer) } + test("SPARK-34014: Ignore Distinct if it is the right child of a left semi join") { + val x = LocalRelation('x1.int, 'x2.int) + val y = LocalRelation('y1.int, 'y2.int) + + val query = x.join(Distinct(y), LeftSemi, Some('x1 === 'y1)) + val optimized = Optimize.execute(query.analyze) + + val correctAnswer = x.join(y, LeftSemi, Some('x1 === 'y1)).analyze + + comparePlans(optimized, correctAnswer) + } + + test("SPARK-34014: Ignore Distinct if it is the right child of a left anti join") { + val x = LocalRelation('x1.int, 'x2.int) + val y = LocalRelation('y1.int, 'y2.int) + + val query = x.join(Distinct(y), LeftAnti, Some('x1 === 'y1)) + val optimized = Optimize.execute(query.analyze) + + val correctAnswer = x.join(y, LeftAnti, Some('x1 === 'y1)).analyze + + comparePlans(optimized, correctAnswer) + } + test("replace batch Deduplicate with Aggregate") { val input = LocalRelation('a.int, 'b.int) val attrA = input.output(0) diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt index 7465ddae84e8..698d1dc08eb7 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* HashAggregate (67) -+- Exchange (66) - +- * HashAggregate (65) - +- * HashAggregate (64) - +- * HashAggregate (63) - +- * HashAggregate (62) - +- * HashAggregate (61) - +- * HashAggregate (60) - +- Exchange (59) - +- * HashAggregate (58) - +- SortMergeJoin LeftSemi (57) - :- SortMergeJoin LeftSemi (39) +* HashAggregate (61) ++- Exchange (60) + +- * HashAggregate (59) + +- * HashAggregate (58) + +- * HashAggregate (57) + +- * HashAggregate (56) + +- * HashAggregate (55) + +- * HashAggregate (54) + +- Exchange (53) + +- * HashAggregate (52) + +- SortMergeJoin LeftSemi (51) + :- SortMergeJoin LeftSemi (36) : :- * Sort (21) : : +- Exchange (20) : : +- * Project (19) @@ -32,40 +32,34 @@ : : +- * Filter (15) : : +- * ColumnarToRow (14) : : +- Scan parquet default.customer (13) - : +- * Sort (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- Exchange (35) - : +- * HashAggregate (34) - : +- * Project (33) - : +- * SortMergeJoin Inner (32) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Filter (24) - : : : +- * ColumnarToRow (23) - : : : +- Scan parquet default.catalog_sales (22) - : : +- ReusedExchange (25) - : +- * Sort (31) - : +- ReusedExchange (30) - +- * Sort (56) - +- Exchange (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- * Project (51) - +- * SortMergeJoin Inner (50) - :- * Sort (47) - : +- Exchange (46) - : +- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet default.web_sales (40) - : +- ReusedExchange (43) - +- * Sort (49) - +- ReusedExchange (48) + : +- * Sort (35) + : +- Exchange (34) + : +- * Project (33) + : +- * SortMergeJoin Inner (32) + : :- * Sort (29) + : : +- Exchange (28) + : : +- * Project (27) + : : +- * BroadcastHashJoin Inner BuildRight (26) + : : :- * Filter (24) + : : : +- * ColumnarToRow (23) + : : : +- Scan parquet default.catalog_sales (22) + : : +- ReusedExchange (25) + : +- * Sort (31) + : +- ReusedExchange (30) + +- * Sort (50) + +- Exchange (49) + +- * Project (48) + +- * SortMergeJoin Inner (47) + :- * Sort (44) + : +- Exchange (43) + : +- * Project (42) + : +- * BroadcastHashJoin Inner BuildRight (41) + : :- * Filter (39) + : : +- * ColumnarToRow (38) + : : +- Scan parquet default.web_sales (37) + : +- ReusedExchange (40) + +- * Sort (46) + +- ReusedExchange (45) (1) Scan parquet default.store_sales @@ -115,7 +109,7 @@ Input [4]: [ss_sold_date_sk#1, ss_customer_sk#2, d_date_sk#3, d_date#4] (11) Exchange Input [2]: [ss_customer_sk#2, d_date#4] -Arguments: hashpartitioning(ss_customer_sk#2, 5), true, [id=#7] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#7] (12) Sort [codegen id : 3] Input [2]: [ss_customer_sk#2, d_date#4] @@ -137,7 +131,7 @@ Condition : isnotnull(c_customer_sk#8) (16) Exchange Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10] -Arguments: hashpartitioning(c_customer_sk#8, 5), true, [id=#11] +Arguments: hashpartitioning(c_customer_sk#8, 5), ENSURE_REQUIREMENTS, [id=#11] (17) Sort [codegen id : 5] Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10] @@ -154,7 +148,7 @@ Input [5]: [ss_customer_sk#2, d_date#4, c_customer_sk#8, c_first_name#9, c_last_ (20) Exchange Input [3]: [d_date#4, c_first_name#9, c_last_name#10] -Arguments: hashpartitioning(coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4), 5), true, [id=#12] +Arguments: hashpartitioning(coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4), 5), ENSURE_REQUIREMENTS, [id=#12] (21) Sort [codegen id : 7] Input [3]: [d_date#4, c_first_name#9, c_last_name#10] @@ -188,7 +182,7 @@ Input [4]: [cs_sold_date_sk#13, cs_bill_customer_sk#14, d_date_sk#15, d_date#16] (28) Exchange Input [2]: [cs_bill_customer_sk#14, d_date#16] -Arguments: hashpartitioning(cs_bill_customer_sk#14, 5), true, [id=#17] +Arguments: hashpartitioning(cs_bill_customer_sk#14, 5), ENSURE_REQUIREMENTS, [id=#17] (29) Sort [codegen id : 10] Input [2]: [cs_bill_customer_sk#14, d_date#16] @@ -210,179 +204,143 @@ Join condition: None Output [3]: [c_last_name#20, c_first_name#19, d_date#16] Input [5]: [cs_bill_customer_sk#14, d_date#16, c_customer_sk#18, c_first_name#19, c_last_name#20] -(34) HashAggregate [codegen id : 13] +(34) Exchange Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#20, c_first_name#19, d_date#16] - -(35) Exchange -Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Arguments: hashpartitioning(c_last_name#20, c_first_name#19, d_date#16, 5), true, [id=#21] - -(36) HashAggregate [codegen id : 14] -Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#20, c_first_name#19, d_date#16] - -(37) Exchange -Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Arguments: hashpartitioning(coalesce(c_last_name#20, ), isnull(c_last_name#20), coalesce(c_first_name#19, ), isnull(c_first_name#19), coalesce(d_date#16, 0), isnull(d_date#16), 5), true, [id=#22] +Arguments: hashpartitioning(coalesce(c_last_name#20, ), isnull(c_last_name#20), coalesce(c_first_name#19, ), isnull(c_first_name#19), coalesce(d_date#16, 0), isnull(d_date#16), 5), ENSURE_REQUIREMENTS, [id=#21] -(38) Sort [codegen id : 15] +(35) Sort [codegen id : 14] Input [3]: [c_last_name#20, c_first_name#19, d_date#16] Arguments: [coalesce(c_last_name#20, ) ASC NULLS FIRST, isnull(c_last_name#20) ASC NULLS FIRST, coalesce(c_first_name#19, ) ASC NULLS FIRST, isnull(c_first_name#19) ASC NULLS FIRST, coalesce(d_date#16, 0) ASC NULLS FIRST, isnull(d_date#16) ASC NULLS FIRST], false, 0 -(39) SortMergeJoin +(36) SortMergeJoin Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4)] Right keys [6]: [coalesce(c_last_name#20, ), isnull(c_last_name#20), coalesce(c_first_name#19, ), isnull(c_first_name#19), coalesce(d_date#16, 0), isnull(d_date#16)] Join condition: None -(40) Scan parquet default.web_sales -Output [2]: [ws_sold_date_sk#23, ws_bill_customer_sk#24] +(37) Scan parquet default.web_sales +Output [2]: [ws_sold_date_sk#22, ws_bill_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_sold_date_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 17] -Input [2]: [ws_sold_date_sk#23, ws_bill_customer_sk#24] +(38) ColumnarToRow [codegen id : 16] +Input [2]: [ws_sold_date_sk#22, ws_bill_customer_sk#23] -(42) Filter [codegen id : 17] -Input [2]: [ws_sold_date_sk#23, ws_bill_customer_sk#24] -Condition : (isnotnull(ws_sold_date_sk#23) AND isnotnull(ws_bill_customer_sk#24)) +(39) Filter [codegen id : 16] +Input [2]: [ws_sold_date_sk#22, ws_bill_customer_sk#23] +Condition : (isnotnull(ws_sold_date_sk#22) AND isnotnull(ws_bill_customer_sk#23)) -(43) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#25, d_date#26] +(40) ReusedExchange [Reuses operator id: 8] +Output [2]: [d_date_sk#24, d_date#25] -(44) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#23] -Right keys [1]: [d_date_sk#25] +(41) BroadcastHashJoin [codegen id : 16] +Left keys [1]: [ws_sold_date_sk#22] +Right keys [1]: [d_date_sk#24] Join condition: None -(45) Project [codegen id : 17] -Output [2]: [ws_bill_customer_sk#24, d_date#26] -Input [4]: [ws_sold_date_sk#23, ws_bill_customer_sk#24, d_date_sk#25, d_date#26] +(42) Project [codegen id : 16] +Output [2]: [ws_bill_customer_sk#23, d_date#25] +Input [4]: [ws_sold_date_sk#22, ws_bill_customer_sk#23, d_date_sk#24, d_date#25] -(46) Exchange -Input [2]: [ws_bill_customer_sk#24, d_date#26] -Arguments: hashpartitioning(ws_bill_customer_sk#24, 5), true, [id=#27] +(43) Exchange +Input [2]: [ws_bill_customer_sk#23, d_date#25] +Arguments: hashpartitioning(ws_bill_customer_sk#23, 5), ENSURE_REQUIREMENTS, [id=#26] -(47) Sort [codegen id : 18] -Input [2]: [ws_bill_customer_sk#24, d_date#26] -Arguments: [ws_bill_customer_sk#24 ASC NULLS FIRST], false, 0 +(44) Sort [codegen id : 17] +Input [2]: [ws_bill_customer_sk#23, d_date#25] +Arguments: [ws_bill_customer_sk#23 ASC NULLS FIRST], false, 0 -(48) ReusedExchange [Reuses operator id: 16] -Output [3]: [c_customer_sk#28, c_first_name#29, c_last_name#30] +(45) ReusedExchange [Reuses operator id: 16] +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -(49) Sort [codegen id : 20] -Input [3]: [c_customer_sk#28, c_first_name#29, c_last_name#30] -Arguments: [c_customer_sk#28 ASC NULLS FIRST], false, 0 +(46) Sort [codegen id : 19] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27 ASC NULLS FIRST], false, 0 -(50) SortMergeJoin [codegen id : 21] -Left keys [1]: [ws_bill_customer_sk#24] -Right keys [1]: [c_customer_sk#28] +(47) SortMergeJoin [codegen id : 20] +Left keys [1]: [ws_bill_customer_sk#23] +Right keys [1]: [c_customer_sk#27] Join condition: None -(51) Project [codegen id : 21] -Output [3]: [c_last_name#30, c_first_name#29, d_date#26] -Input [5]: [ws_bill_customer_sk#24, d_date#26, c_customer_sk#28, c_first_name#29, c_last_name#30] +(48) Project [codegen id : 20] +Output [3]: [c_last_name#29, c_first_name#28, d_date#25] +Input [5]: [ws_bill_customer_sk#23, d_date#25, c_customer_sk#27, c_first_name#28, c_last_name#29] -(52) HashAggregate [codegen id : 21] -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#30, c_first_name#29, d_date#26] +(49) Exchange +Input [3]: [c_last_name#29, c_first_name#28, d_date#25] +Arguments: hashpartitioning(coalesce(c_last_name#29, ), isnull(c_last_name#29), coalesce(c_first_name#28, ), isnull(c_first_name#28), coalesce(d_date#25, 0), isnull(d_date#25), 5), ENSURE_REQUIREMENTS, [id=#30] -(53) Exchange -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Arguments: hashpartitioning(c_last_name#30, c_first_name#29, d_date#26, 5), true, [id=#31] +(50) Sort [codegen id : 21] +Input [3]: [c_last_name#29, c_first_name#28, d_date#25] +Arguments: [coalesce(c_last_name#29, ) ASC NULLS FIRST, isnull(c_last_name#29) ASC NULLS FIRST, coalesce(c_first_name#28, ) ASC NULLS FIRST, isnull(c_first_name#28) ASC NULLS FIRST, coalesce(d_date#25, 0) ASC NULLS FIRST, isnull(d_date#25) ASC NULLS FIRST], false, 0 -(54) HashAggregate [codegen id : 22] -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#30, c_first_name#29, d_date#26] - -(55) Exchange -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Arguments: hashpartitioning(coalesce(c_last_name#30, ), isnull(c_last_name#30), coalesce(c_first_name#29, ), isnull(c_first_name#29), coalesce(d_date#26, 0), isnull(d_date#26), 5), true, [id=#32] - -(56) Sort [codegen id : 23] -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Arguments: [coalesce(c_last_name#30, ) ASC NULLS FIRST, isnull(c_last_name#30) ASC NULLS FIRST, coalesce(c_first_name#29, ) ASC NULLS FIRST, isnull(c_first_name#29) ASC NULLS FIRST, coalesce(d_date#26, 0) ASC NULLS FIRST, isnull(d_date#26) ASC NULLS FIRST], false, 0 - -(57) SortMergeJoin +(51) SortMergeJoin Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#30, ), isnull(c_last_name#30), coalesce(c_first_name#29, ), isnull(c_first_name#29), coalesce(d_date#26, 0), isnull(d_date#26)] +Right keys [6]: [coalesce(c_last_name#29, ), isnull(c_last_name#29), coalesce(c_first_name#28, ), isnull(c_first_name#28), coalesce(d_date#25, 0), isnull(d_date#25)] Join condition: None -(58) HashAggregate [codegen id : 24] +(52) HashAggregate [codegen id : 22] Input [3]: [d_date#4, c_first_name#9, c_last_name#10] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(59) Exchange +(53) Exchange Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), true, [id=#33] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, [id=#31] -(60) HashAggregate [codegen id : 25] +(54) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(61) HashAggregate [codegen id : 25] +(55) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(62) HashAggregate [codegen id : 25] +(56) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(63) HashAggregate [codegen id : 25] +(57) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(64) HashAggregate [codegen id : 25] +(58) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results: [] -(65) HashAggregate [codegen id : 25] +(59) HashAggregate [codegen id : 23] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#34] -Results [1]: [count#35] +Aggregate Attributes [1]: [count#32] +Results [1]: [count#33] -(66) Exchange -Input [1]: [count#35] -Arguments: SinglePartition, true, [id=#36] +(60) Exchange +Input [1]: [count#33] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#34] -(67) HashAggregate [codegen id : 26] -Input [1]: [count#35] +(61) HashAggregate [codegen id : 24] +Input [1]: [count#33] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#37] -Results [1]: [count(1)#37 AS count(1)#38] +Aggregate Attributes [1]: [count(1)#35] +Results [1]: [count(1)#35 AS count(1)#36] diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt index 8dd59340cf06..150806201ff4 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt @@ -1,8 +1,8 @@ -WholeStageCodegen (26) +WholeStageCodegen (24) HashAggregate [count] [count(1),count(1),count] InputAdapter Exchange #1 - WholeStageCodegen (25) + WholeStageCodegen (23) HashAggregate [count,count] HashAggregate [c_last_name,c_first_name,d_date] HashAggregate [c_last_name,c_first_name,d_date] @@ -11,7 +11,7 @@ WholeStageCodegen (26) HashAggregate [c_last_name,c_first_name,d_date] InputAdapter Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (24) + WholeStageCodegen (22) HashAggregate [c_last_name,c_first_name,d_date] InputAdapter SortMergeJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] @@ -53,65 +53,55 @@ WholeStageCodegen (26) ColumnarToRow InputAdapter Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name] - WholeStageCodegen (15) + WholeStageCodegen (14) Sort [c_last_name,c_first_name,d_date] InputAdapter Exchange [c_last_name,c_first_name,d_date] #7 - WholeStageCodegen (14) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (13) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (10) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #9 - WholeStageCodegen (9) - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_sold_date_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #5 - InputAdapter - WholeStageCodegen (12) - Sort [c_customer_sk] + WholeStageCodegen (13) + Project [c_last_name,c_first_name,d_date] + SortMergeJoin [cs_bill_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (10) + Sort [cs_bill_customer_sk] + InputAdapter + Exchange [cs_bill_customer_sk] #8 + WholeStageCodegen (9) + Project [cs_bill_customer_sk,d_date] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + Filter [cs_sold_date_sk,cs_bill_customer_sk] + ColumnarToRow + InputAdapter + Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 - WholeStageCodegen (23) + ReusedExchange [d_date_sk,d_date] #5 + InputAdapter + WholeStageCodegen (12) + Sort [c_customer_sk] + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 + WholeStageCodegen (21) Sort [c_last_name,c_first_name,d_date] InputAdapter - Exchange [c_last_name,c_first_name,d_date] #10 - WholeStageCodegen (22) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #11 - WholeStageCodegen (21) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #12 - WholeStageCodegen (17) - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_sold_date_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #5 - InputAdapter - WholeStageCodegen (20) - Sort [c_customer_sk] + Exchange [c_last_name,c_first_name,d_date] #9 + WholeStageCodegen (20) + Project [c_last_name,c_first_name,d_date] + SortMergeJoin [ws_bill_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (17) + Sort [ws_bill_customer_sk] + InputAdapter + Exchange [ws_bill_customer_sk] #10 + WholeStageCodegen (16) + Project [ws_bill_customer_sk,d_date] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + Filter [ws_sold_date_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 + ReusedExchange [d_date_sk,d_date] #5 + InputAdapter + WholeStageCodegen (19) + Sort [c_customer_sk] + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt index 74454cf32afd..22af52bffb38 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* HashAggregate (54) -+- Exchange (53) - +- * HashAggregate (52) - +- * HashAggregate (51) - +- * HashAggregate (50) - +- * HashAggregate (49) - +- * HashAggregate (48) - +- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * BroadcastHashJoin LeftSemi BuildRight (44) - :- * BroadcastHashJoin LeftSemi BuildRight (30) +* HashAggregate (48) ++- Exchange (47) + +- * HashAggregate (46) + +- * HashAggregate (45) + +- * HashAggregate (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * BroadcastHashJoin LeftSemi BuildRight (38) + :- * BroadcastHashJoin LeftSemi BuildRight (27) : :- * Project (16) : : +- * BroadcastHashJoin Inner BuildRight (15) : : :- * Project (10) @@ -27,32 +27,26 @@ : : +- * Filter (13) : : +- * ColumnarToRow (12) : : +- Scan parquet default.customer (11) - : +- BroadcastExchange (29) - : +- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Filter (19) - : : : +- * ColumnarToRow (18) - : : : +- Scan parquet default.catalog_sales (17) - : : +- ReusedExchange (20) - : +- ReusedExchange (23) - +- BroadcastExchange (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Filter (33) - : : +- * ColumnarToRow (32) - : : +- Scan parquet default.web_sales (31) - : +- ReusedExchange (34) - +- ReusedExchange (37) + : +- BroadcastExchange (26) + : +- * Project (25) + : +- * BroadcastHashJoin Inner BuildRight (24) + : :- * Project (22) + : : +- * BroadcastHashJoin Inner BuildRight (21) + : : :- * Filter (19) + : : : +- * ColumnarToRow (18) + : : : +- Scan parquet default.catalog_sales (17) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- BroadcastExchange (37) + +- * Project (36) + +- * BroadcastHashJoin Inner BuildRight (35) + :- * Project (33) + : +- * BroadcastHashJoin Inner BuildRight (32) + : :- * Filter (30) + : : +- * ColumnarToRow (29) + : : +- Scan parquet default.web_sales (28) + : +- ReusedExchange (31) + +- ReusedExchange (34) (1) Scan parquet default.store_sales @@ -62,10 +56,10 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 11] +(2) ColumnarToRow [codegen id : 9] Input [2]: [ss_sold_date_sk#1, ss_customer_sk#2] -(3) Filter [codegen id : 11] +(3) Filter [codegen id : 9] Input [2]: [ss_sold_date_sk#1, ss_customer_sk#2] Condition : (isnotnull(ss_sold_date_sk#1) AND isnotnull(ss_customer_sk#2)) @@ -91,12 +85,12 @@ Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Input [2]: [d_date_sk#3, d_date#4] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#6] -(9) BroadcastHashJoin [codegen id : 11] +(9) BroadcastHashJoin [codegen id : 9] Left keys [1]: [ss_sold_date_sk#1] Right keys [1]: [d_date_sk#3] Join condition: None -(10) Project [codegen id : 11] +(10) Project [codegen id : 9] Output [2]: [ss_customer_sk#2, d_date#4] Input [4]: [ss_sold_date_sk#1, ss_customer_sk#2, d_date_sk#3, d_date#4] @@ -118,12 +112,12 @@ Condition : isnotnull(c_customer_sk#7) Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10] -(15) BroadcastHashJoin [codegen id : 11] +(15) BroadcastHashJoin [codegen id : 9] Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#7] Join condition: None -(16) Project [codegen id : 11] +(16) Project [codegen id : 9] Output [3]: [d_date#4, c_first_name#8, c_last_name#9] Input [5]: [ss_customer_sk#2, d_date#4, c_customer_sk#7, c_first_name#8, c_last_name#9] @@ -165,159 +159,123 @@ Join condition: None Output [3]: [c_last_name#17, c_first_name#16, d_date#14] Input [5]: [cs_bill_customer_sk#12, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] -(26) HashAggregate [codegen id : 5] +(26) BroadcastExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#17, c_first_name#16, d_date#14] - -(27) Exchange -Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), true, [id=#18] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#18] -(28) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#17, c_first_name#16, d_date#14] - -(29) BroadcastExchange -Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#19] - -(30) BroadcastHashJoin [codegen id : 11] +(27) BroadcastHashJoin [codegen id : 9] Left keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#8, ), isnull(c_first_name#8), coalesce(d_date#4, 0), isnull(d_date#4)] Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 0), isnull(d_date#14)] Join condition: None -(31) Scan parquet default.web_sales -Output [2]: [ws_sold_date_sk#20, ws_bill_customer_sk#21] +(28) Scan parquet default.web_sales +Output [2]: [ws_sold_date_sk#19, ws_bill_customer_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_sold_date_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 9] -Input [2]: [ws_sold_date_sk#20, ws_bill_customer_sk#21] +(29) ColumnarToRow [codegen id : 8] +Input [2]: [ws_sold_date_sk#19, ws_bill_customer_sk#20] -(33) Filter [codegen id : 9] -Input [2]: [ws_sold_date_sk#20, ws_bill_customer_sk#21] -Condition : (isnotnull(ws_sold_date_sk#20) AND isnotnull(ws_bill_customer_sk#21)) +(30) Filter [codegen id : 8] +Input [2]: [ws_sold_date_sk#19, ws_bill_customer_sk#20] +Condition : (isnotnull(ws_sold_date_sk#19) AND isnotnull(ws_bill_customer_sk#20)) -(34) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#22, d_date#23] +(31) ReusedExchange [Reuses operator id: 8] +Output [2]: [d_date_sk#21, d_date#22] -(35) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#20] -Right keys [1]: [d_date_sk#22] +(32) BroadcastHashJoin [codegen id : 8] +Left keys [1]: [ws_sold_date_sk#19] +Right keys [1]: [d_date_sk#21] Join condition: None -(36) Project [codegen id : 9] -Output [2]: [ws_bill_customer_sk#21, d_date#23] -Input [4]: [ws_sold_date_sk#20, ws_bill_customer_sk#21, d_date_sk#22, d_date#23] +(33) Project [codegen id : 8] +Output [2]: [ws_bill_customer_sk#20, d_date#22] +Input [4]: [ws_sold_date_sk#19, ws_bill_customer_sk#20, d_date_sk#21, d_date#22] -(37) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +(34) ReusedExchange [Reuses operator id: 14] +Output [3]: [c_customer_sk#23, c_first_name#24, c_last_name#25] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_bill_customer_sk#21] -Right keys [1]: [c_customer_sk#24] +(35) BroadcastHashJoin [codegen id : 8] +Left keys [1]: [ws_bill_customer_sk#20] +Right keys [1]: [c_customer_sk#23] Join condition: None -(39) Project [codegen id : 9] -Output [3]: [c_last_name#26, c_first_name#25, d_date#23] -Input [5]: [ws_bill_customer_sk#21, d_date#23, c_customer_sk#24, c_first_name#25, c_last_name#26] +(36) Project [codegen id : 8] +Output [3]: [c_last_name#25, c_first_name#24, d_date#22] +Input [5]: [ws_bill_customer_sk#20, d_date#22, c_customer_sk#23, c_first_name#24, c_last_name#25] -(40) HashAggregate [codegen id : 9] -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Keys [3]: [c_last_name#26, c_first_name#25, d_date#23] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#26, c_first_name#25, d_date#23] - -(41) Exchange -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Arguments: hashpartitioning(c_last_name#26, c_first_name#25, d_date#23, 5), true, [id=#27] - -(42) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Keys [3]: [c_last_name#26, c_first_name#25, d_date#23] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#26, c_first_name#25, d_date#23] - -(43) BroadcastExchange -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#28] +(37) BroadcastExchange +Input [3]: [c_last_name#25, c_first_name#24, d_date#22] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#26] -(44) BroadcastHashJoin [codegen id : 11] +(38) BroadcastHashJoin [codegen id : 9] Left keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#8, ), isnull(c_first_name#8), coalesce(d_date#4, 0), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#26, ), isnull(c_last_name#26), coalesce(c_first_name#25, ), isnull(c_first_name#25), coalesce(d_date#23, 0), isnull(d_date#23)] +Right keys [6]: [coalesce(c_last_name#25, ), isnull(c_last_name#25), coalesce(c_first_name#24, ), isnull(c_first_name#24), coalesce(d_date#22, 0), isnull(d_date#22)] Join condition: None -(45) HashAggregate [codegen id : 11] +(39) HashAggregate [codegen id : 9] Input [3]: [d_date#4, c_first_name#8, c_last_name#9] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(46) Exchange +(40) Exchange Input [3]: [c_last_name#9, c_first_name#8, d_date#4] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#4, 5), true, [id=#29] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#4, 5), ENSURE_REQUIREMENTS, [id=#27] -(47) HashAggregate [codegen id : 12] +(41) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(48) HashAggregate [codegen id : 12] +(42) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(49) HashAggregate [codegen id : 12] +(43) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(50) HashAggregate [codegen id : 12] +(44) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(51) HashAggregate [codegen id : 12] +(45) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results: [] -(52) HashAggregate [codegen id : 12] +(46) HashAggregate [codegen id : 10] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#30] -Results [1]: [count#31] +Aggregate Attributes [1]: [count#28] +Results [1]: [count#29] -(53) Exchange -Input [1]: [count#31] -Arguments: SinglePartition, true, [id=#32] +(47) Exchange +Input [1]: [count#29] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#30] -(54) HashAggregate [codegen id : 13] -Input [1]: [count#31] +(48) HashAggregate [codegen id : 11] +Input [1]: [count#29] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#33] -Results [1]: [count(1)#33 AS count(1)#34] +Aggregate Attributes [1]: [count(1)#31] +Results [1]: [count(1)#31 AS count(1)#32] diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt index a5b57a4ac945..29fd421729ea 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt @@ -1,8 +1,8 @@ -WholeStageCodegen (13) +WholeStageCodegen (11) HashAggregate [count] [count(1),count(1),count] InputAdapter Exchange #1 - WholeStageCodegen (12) + WholeStageCodegen (10) HashAggregate [count,count] HashAggregate [c_last_name,c_first_name,d_date] HashAggregate [c_last_name,c_first_name,d_date] @@ -11,7 +11,7 @@ WholeStageCodegen (13) HashAggregate [c_last_name,c_first_name,d_date] InputAdapter Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (11) + WholeStageCodegen (9) HashAggregate [c_last_name,c_first_name,d_date] BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] @@ -40,41 +40,31 @@ WholeStageCodegen (13) Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #5 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (5) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_sold_date_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_sold_date_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 + WholeStageCodegen (5) + Project [c_last_name,c_first_name,d_date] + BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] + Project [cs_bill_customer_sk,d_date] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + Filter [cs_sold_date_sk,cs_bill_customer_sk] + ColumnarToRow InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] + InputAdapter + ReusedExchange [d_date_sk,d_date] #3 + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (8) + Project [c_last_name,c_first_name,d_date] + BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] + Project [ws_bill_customer_sk,d_date] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + Filter [ws_sold_date_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] + InputAdapter + ReusedExchange [d_date_sk,d_date] #3 + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt index 377bd36f520e..1f1d836d5b4e 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* HashAggregate (67) -+- Exchange (66) - +- * HashAggregate (65) - +- * HashAggregate (64) - +- * HashAggregate (63) - +- * HashAggregate (62) - +- * HashAggregate (61) - +- * HashAggregate (60) - +- Exchange (59) - +- * HashAggregate (58) - +- SortMergeJoin LeftAnti (57) - :- SortMergeJoin LeftAnti (39) +* HashAggregate (61) ++- Exchange (60) + +- * HashAggregate (59) + +- * HashAggregate (58) + +- * HashAggregate (57) + +- * HashAggregate (56) + +- * HashAggregate (55) + +- * HashAggregate (54) + +- Exchange (53) + +- * HashAggregate (52) + +- SortMergeJoin LeftAnti (51) + :- SortMergeJoin LeftAnti (36) : :- * Sort (21) : : +- Exchange (20) : : +- * Project (19) @@ -32,40 +32,34 @@ : : +- * Filter (15) : : +- * ColumnarToRow (14) : : +- Scan parquet default.customer (13) - : +- * Sort (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- Exchange (35) - : +- * HashAggregate (34) - : +- * Project (33) - : +- * SortMergeJoin Inner (32) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Filter (24) - : : : +- * ColumnarToRow (23) - : : : +- Scan parquet default.catalog_sales (22) - : : +- ReusedExchange (25) - : +- * Sort (31) - : +- ReusedExchange (30) - +- * Sort (56) - +- Exchange (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- * Project (51) - +- * SortMergeJoin Inner (50) - :- * Sort (47) - : +- Exchange (46) - : +- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet default.web_sales (40) - : +- ReusedExchange (43) - +- * Sort (49) - +- ReusedExchange (48) + : +- * Sort (35) + : +- Exchange (34) + : +- * Project (33) + : +- * SortMergeJoin Inner (32) + : :- * Sort (29) + : : +- Exchange (28) + : : +- * Project (27) + : : +- * BroadcastHashJoin Inner BuildRight (26) + : : :- * Filter (24) + : : : +- * ColumnarToRow (23) + : : : +- Scan parquet default.catalog_sales (22) + : : +- ReusedExchange (25) + : +- * Sort (31) + : +- ReusedExchange (30) + +- * Sort (50) + +- Exchange (49) + +- * Project (48) + +- * SortMergeJoin Inner (47) + :- * Sort (44) + : +- Exchange (43) + : +- * Project (42) + : +- * BroadcastHashJoin Inner BuildRight (41) + : :- * Filter (39) + : : +- * ColumnarToRow (38) + : : +- Scan parquet default.web_sales (37) + : +- ReusedExchange (40) + +- * Sort (46) + +- ReusedExchange (45) (1) Scan parquet default.store_sales @@ -115,7 +109,7 @@ Input [4]: [ss_sold_date_sk#1, ss_customer_sk#2, d_date_sk#3, d_date#4] (11) Exchange Input [2]: [ss_customer_sk#2, d_date#4] -Arguments: hashpartitioning(ss_customer_sk#2, 5), true, [id=#7] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [id=#7] (12) Sort [codegen id : 3] Input [2]: [ss_customer_sk#2, d_date#4] @@ -137,7 +131,7 @@ Condition : isnotnull(c_customer_sk#8) (16) Exchange Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10] -Arguments: hashpartitioning(c_customer_sk#8, 5), true, [id=#11] +Arguments: hashpartitioning(c_customer_sk#8, 5), ENSURE_REQUIREMENTS, [id=#11] (17) Sort [codegen id : 5] Input [3]: [c_customer_sk#8, c_first_name#9, c_last_name#10] @@ -154,7 +148,7 @@ Input [5]: [ss_customer_sk#2, d_date#4, c_customer_sk#8, c_first_name#9, c_last_ (20) Exchange Input [3]: [d_date#4, c_first_name#9, c_last_name#10] -Arguments: hashpartitioning(coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4), 5), true, [id=#12] +Arguments: hashpartitioning(coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4), 5), ENSURE_REQUIREMENTS, [id=#12] (21) Sort [codegen id : 7] Input [3]: [d_date#4, c_first_name#9, c_last_name#10] @@ -188,7 +182,7 @@ Input [4]: [cs_sold_date_sk#13, cs_bill_customer_sk#14, d_date_sk#15, d_date#16] (28) Exchange Input [2]: [cs_bill_customer_sk#14, d_date#16] -Arguments: hashpartitioning(cs_bill_customer_sk#14, 5), true, [id=#17] +Arguments: hashpartitioning(cs_bill_customer_sk#14, 5), ENSURE_REQUIREMENTS, [id=#17] (29) Sort [codegen id : 10] Input [2]: [cs_bill_customer_sk#14, d_date#16] @@ -210,179 +204,143 @@ Join condition: None Output [3]: [c_last_name#20, c_first_name#19, d_date#16] Input [5]: [cs_bill_customer_sk#14, d_date#16, c_customer_sk#18, c_first_name#19, c_last_name#20] -(34) HashAggregate [codegen id : 13] +(34) Exchange Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#20, c_first_name#19, d_date#16] - -(35) Exchange -Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Arguments: hashpartitioning(c_last_name#20, c_first_name#19, d_date#16, 5), true, [id=#21] - -(36) HashAggregate [codegen id : 14] -Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#20, c_first_name#19, d_date#16] - -(37) Exchange -Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -Arguments: hashpartitioning(coalesce(c_last_name#20, ), isnull(c_last_name#20), coalesce(c_first_name#19, ), isnull(c_first_name#19), coalesce(d_date#16, 0), isnull(d_date#16), 5), true, [id=#22] +Arguments: hashpartitioning(coalesce(c_last_name#20, ), isnull(c_last_name#20), coalesce(c_first_name#19, ), isnull(c_first_name#19), coalesce(d_date#16, 0), isnull(d_date#16), 5), ENSURE_REQUIREMENTS, [id=#21] -(38) Sort [codegen id : 15] +(35) Sort [codegen id : 14] Input [3]: [c_last_name#20, c_first_name#19, d_date#16] Arguments: [coalesce(c_last_name#20, ) ASC NULLS FIRST, isnull(c_last_name#20) ASC NULLS FIRST, coalesce(c_first_name#19, ) ASC NULLS FIRST, isnull(c_first_name#19) ASC NULLS FIRST, coalesce(d_date#16, 0) ASC NULLS FIRST, isnull(d_date#16) ASC NULLS FIRST], false, 0 -(39) SortMergeJoin +(36) SortMergeJoin Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4)] Right keys [6]: [coalesce(c_last_name#20, ), isnull(c_last_name#20), coalesce(c_first_name#19, ), isnull(c_first_name#19), coalesce(d_date#16, 0), isnull(d_date#16)] Join condition: None -(40) Scan parquet default.web_sales -Output [2]: [ws_sold_date_sk#23, ws_bill_customer_sk#24] +(37) Scan parquet default.web_sales +Output [2]: [ws_sold_date_sk#22, ws_bill_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_sold_date_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 17] -Input [2]: [ws_sold_date_sk#23, ws_bill_customer_sk#24] +(38) ColumnarToRow [codegen id : 16] +Input [2]: [ws_sold_date_sk#22, ws_bill_customer_sk#23] -(42) Filter [codegen id : 17] -Input [2]: [ws_sold_date_sk#23, ws_bill_customer_sk#24] -Condition : (isnotnull(ws_sold_date_sk#23) AND isnotnull(ws_bill_customer_sk#24)) +(39) Filter [codegen id : 16] +Input [2]: [ws_sold_date_sk#22, ws_bill_customer_sk#23] +Condition : (isnotnull(ws_sold_date_sk#22) AND isnotnull(ws_bill_customer_sk#23)) -(43) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#25, d_date#26] +(40) ReusedExchange [Reuses operator id: 8] +Output [2]: [d_date_sk#24, d_date#25] -(44) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#23] -Right keys [1]: [d_date_sk#25] +(41) BroadcastHashJoin [codegen id : 16] +Left keys [1]: [ws_sold_date_sk#22] +Right keys [1]: [d_date_sk#24] Join condition: None -(45) Project [codegen id : 17] -Output [2]: [ws_bill_customer_sk#24, d_date#26] -Input [4]: [ws_sold_date_sk#23, ws_bill_customer_sk#24, d_date_sk#25, d_date#26] +(42) Project [codegen id : 16] +Output [2]: [ws_bill_customer_sk#23, d_date#25] +Input [4]: [ws_sold_date_sk#22, ws_bill_customer_sk#23, d_date_sk#24, d_date#25] -(46) Exchange -Input [2]: [ws_bill_customer_sk#24, d_date#26] -Arguments: hashpartitioning(ws_bill_customer_sk#24, 5), true, [id=#27] +(43) Exchange +Input [2]: [ws_bill_customer_sk#23, d_date#25] +Arguments: hashpartitioning(ws_bill_customer_sk#23, 5), ENSURE_REQUIREMENTS, [id=#26] -(47) Sort [codegen id : 18] -Input [2]: [ws_bill_customer_sk#24, d_date#26] -Arguments: [ws_bill_customer_sk#24 ASC NULLS FIRST], false, 0 +(44) Sort [codegen id : 17] +Input [2]: [ws_bill_customer_sk#23, d_date#25] +Arguments: [ws_bill_customer_sk#23 ASC NULLS FIRST], false, 0 -(48) ReusedExchange [Reuses operator id: 16] -Output [3]: [c_customer_sk#28, c_first_name#29, c_last_name#30] +(45) ReusedExchange [Reuses operator id: 16] +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -(49) Sort [codegen id : 20] -Input [3]: [c_customer_sk#28, c_first_name#29, c_last_name#30] -Arguments: [c_customer_sk#28 ASC NULLS FIRST], false, 0 +(46) Sort [codegen id : 19] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27 ASC NULLS FIRST], false, 0 -(50) SortMergeJoin [codegen id : 21] -Left keys [1]: [ws_bill_customer_sk#24] -Right keys [1]: [c_customer_sk#28] +(47) SortMergeJoin [codegen id : 20] +Left keys [1]: [ws_bill_customer_sk#23] +Right keys [1]: [c_customer_sk#27] Join condition: None -(51) Project [codegen id : 21] -Output [3]: [c_last_name#30, c_first_name#29, d_date#26] -Input [5]: [ws_bill_customer_sk#24, d_date#26, c_customer_sk#28, c_first_name#29, c_last_name#30] +(48) Project [codegen id : 20] +Output [3]: [c_last_name#29, c_first_name#28, d_date#25] +Input [5]: [ws_bill_customer_sk#23, d_date#25, c_customer_sk#27, c_first_name#28, c_last_name#29] -(52) HashAggregate [codegen id : 21] -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#30, c_first_name#29, d_date#26] +(49) Exchange +Input [3]: [c_last_name#29, c_first_name#28, d_date#25] +Arguments: hashpartitioning(coalesce(c_last_name#29, ), isnull(c_last_name#29), coalesce(c_first_name#28, ), isnull(c_first_name#28), coalesce(d_date#25, 0), isnull(d_date#25), 5), ENSURE_REQUIREMENTS, [id=#30] -(53) Exchange -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Arguments: hashpartitioning(c_last_name#30, c_first_name#29, d_date#26, 5), true, [id=#31] +(50) Sort [codegen id : 21] +Input [3]: [c_last_name#29, c_first_name#28, d_date#25] +Arguments: [coalesce(c_last_name#29, ) ASC NULLS FIRST, isnull(c_last_name#29) ASC NULLS FIRST, coalesce(c_first_name#28, ) ASC NULLS FIRST, isnull(c_first_name#28) ASC NULLS FIRST, coalesce(d_date#25, 0) ASC NULLS FIRST, isnull(d_date#25) ASC NULLS FIRST], false, 0 -(54) HashAggregate [codegen id : 22] -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#30, c_first_name#29, d_date#26] - -(55) Exchange -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Arguments: hashpartitioning(coalesce(c_last_name#30, ), isnull(c_last_name#30), coalesce(c_first_name#29, ), isnull(c_first_name#29), coalesce(d_date#26, 0), isnull(d_date#26), 5), true, [id=#32] - -(56) Sort [codegen id : 23] -Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -Arguments: [coalesce(c_last_name#30, ) ASC NULLS FIRST, isnull(c_last_name#30) ASC NULLS FIRST, coalesce(c_first_name#29, ) ASC NULLS FIRST, isnull(c_first_name#29) ASC NULLS FIRST, coalesce(d_date#26, 0) ASC NULLS FIRST, isnull(d_date#26) ASC NULLS FIRST], false, 0 - -(57) SortMergeJoin +(51) SortMergeJoin Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 0), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#30, ), isnull(c_last_name#30), coalesce(c_first_name#29, ), isnull(c_first_name#29), coalesce(d_date#26, 0), isnull(d_date#26)] +Right keys [6]: [coalesce(c_last_name#29, ), isnull(c_last_name#29), coalesce(c_first_name#28, ), isnull(c_first_name#28), coalesce(d_date#25, 0), isnull(d_date#25)] Join condition: None -(58) HashAggregate [codegen id : 24] +(52) HashAggregate [codegen id : 22] Input [3]: [d_date#4, c_first_name#9, c_last_name#10] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(59) Exchange +(53) Exchange Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), true, [id=#33] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, [id=#31] -(60) HashAggregate [codegen id : 25] +(54) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(61) HashAggregate [codegen id : 25] +(55) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(62) HashAggregate [codegen id : 25] +(56) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(63) HashAggregate [codegen id : 25] +(57) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#10, c_first_name#9, d_date#4] -(64) HashAggregate [codegen id : 25] +(58) HashAggregate [codegen id : 23] Input [3]: [c_last_name#10, c_first_name#9, d_date#4] Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] Aggregate Attributes: [] Results: [] -(65) HashAggregate [codegen id : 25] +(59) HashAggregate [codegen id : 23] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#34] -Results [1]: [count#35] +Aggregate Attributes [1]: [count#32] +Results [1]: [count#33] -(66) Exchange -Input [1]: [count#35] -Arguments: SinglePartition, true, [id=#36] +(60) Exchange +Input [1]: [count#33] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#34] -(67) HashAggregate [codegen id : 26] -Input [1]: [count#35] +(61) HashAggregate [codegen id : 24] +Input [1]: [count#33] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#37] -Results [1]: [count(1)#37 AS count(1)#38] +Aggregate Attributes [1]: [count(1)#35] +Results [1]: [count(1)#35 AS count(1)#36] diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt index 8dd59340cf06..150806201ff4 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt @@ -1,8 +1,8 @@ -WholeStageCodegen (26) +WholeStageCodegen (24) HashAggregate [count] [count(1),count(1),count] InputAdapter Exchange #1 - WholeStageCodegen (25) + WholeStageCodegen (23) HashAggregate [count,count] HashAggregate [c_last_name,c_first_name,d_date] HashAggregate [c_last_name,c_first_name,d_date] @@ -11,7 +11,7 @@ WholeStageCodegen (26) HashAggregate [c_last_name,c_first_name,d_date] InputAdapter Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (24) + WholeStageCodegen (22) HashAggregate [c_last_name,c_first_name,d_date] InputAdapter SortMergeJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] @@ -53,65 +53,55 @@ WholeStageCodegen (26) ColumnarToRow InputAdapter Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name] - WholeStageCodegen (15) + WholeStageCodegen (14) Sort [c_last_name,c_first_name,d_date] InputAdapter Exchange [c_last_name,c_first_name,d_date] #7 - WholeStageCodegen (14) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (13) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (10) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #9 - WholeStageCodegen (9) - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_sold_date_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #5 - InputAdapter - WholeStageCodegen (12) - Sort [c_customer_sk] + WholeStageCodegen (13) + Project [c_last_name,c_first_name,d_date] + SortMergeJoin [cs_bill_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (10) + Sort [cs_bill_customer_sk] + InputAdapter + Exchange [cs_bill_customer_sk] #8 + WholeStageCodegen (9) + Project [cs_bill_customer_sk,d_date] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + Filter [cs_sold_date_sk,cs_bill_customer_sk] + ColumnarToRow + InputAdapter + Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 - WholeStageCodegen (23) + ReusedExchange [d_date_sk,d_date] #5 + InputAdapter + WholeStageCodegen (12) + Sort [c_customer_sk] + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 + WholeStageCodegen (21) Sort [c_last_name,c_first_name,d_date] InputAdapter - Exchange [c_last_name,c_first_name,d_date] #10 - WholeStageCodegen (22) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #11 - WholeStageCodegen (21) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #12 - WholeStageCodegen (17) - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_sold_date_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #5 - InputAdapter - WholeStageCodegen (20) - Sort [c_customer_sk] + Exchange [c_last_name,c_first_name,d_date] #9 + WholeStageCodegen (20) + Project [c_last_name,c_first_name,d_date] + SortMergeJoin [ws_bill_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (17) + Sort [ws_bill_customer_sk] + InputAdapter + Exchange [ws_bill_customer_sk] #10 + WholeStageCodegen (16) + Project [ws_bill_customer_sk,d_date] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + Filter [ws_sold_date_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 + ReusedExchange [d_date_sk,d_date] #5 + InputAdapter + WholeStageCodegen (19) + Sort [c_customer_sk] + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #6 diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt index 3d59a670b7e8..fb758a15a4be 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* HashAggregate (54) -+- Exchange (53) - +- * HashAggregate (52) - +- * HashAggregate (51) - +- * HashAggregate (50) - +- * HashAggregate (49) - +- * HashAggregate (48) - +- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * BroadcastHashJoin LeftAnti BuildRight (44) - :- * BroadcastHashJoin LeftAnti BuildRight (30) +* HashAggregate (48) ++- Exchange (47) + +- * HashAggregate (46) + +- * HashAggregate (45) + +- * HashAggregate (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * BroadcastHashJoin LeftAnti BuildRight (38) + :- * BroadcastHashJoin LeftAnti BuildRight (27) : :- * Project (16) : : +- * BroadcastHashJoin Inner BuildRight (15) : : :- * Project (10) @@ -27,32 +27,26 @@ : : +- * Filter (13) : : +- * ColumnarToRow (12) : : +- Scan parquet default.customer (11) - : +- BroadcastExchange (29) - : +- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Filter (19) - : : : +- * ColumnarToRow (18) - : : : +- Scan parquet default.catalog_sales (17) - : : +- ReusedExchange (20) - : +- ReusedExchange (23) - +- BroadcastExchange (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Filter (33) - : : +- * ColumnarToRow (32) - : : +- Scan parquet default.web_sales (31) - : +- ReusedExchange (34) - +- ReusedExchange (37) + : +- BroadcastExchange (26) + : +- * Project (25) + : +- * BroadcastHashJoin Inner BuildRight (24) + : :- * Project (22) + : : +- * BroadcastHashJoin Inner BuildRight (21) + : : :- * Filter (19) + : : : +- * ColumnarToRow (18) + : : : +- Scan parquet default.catalog_sales (17) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- BroadcastExchange (37) + +- * Project (36) + +- * BroadcastHashJoin Inner BuildRight (35) + :- * Project (33) + : +- * BroadcastHashJoin Inner BuildRight (32) + : :- * Filter (30) + : : +- * ColumnarToRow (29) + : : +- Scan parquet default.web_sales (28) + : +- ReusedExchange (31) + +- ReusedExchange (34) (1) Scan parquet default.store_sales @@ -62,10 +56,10 @@ Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 11] +(2) ColumnarToRow [codegen id : 9] Input [2]: [ss_sold_date_sk#1, ss_customer_sk#2] -(3) Filter [codegen id : 11] +(3) Filter [codegen id : 9] Input [2]: [ss_sold_date_sk#1, ss_customer_sk#2] Condition : (isnotnull(ss_sold_date_sk#1) AND isnotnull(ss_customer_sk#2)) @@ -91,12 +85,12 @@ Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Input [2]: [d_date_sk#3, d_date#4] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#6] -(9) BroadcastHashJoin [codegen id : 11] +(9) BroadcastHashJoin [codegen id : 9] Left keys [1]: [ss_sold_date_sk#1] Right keys [1]: [d_date_sk#3] Join condition: None -(10) Project [codegen id : 11] +(10) Project [codegen id : 9] Output [2]: [ss_customer_sk#2, d_date#4] Input [4]: [ss_sold_date_sk#1, ss_customer_sk#2, d_date_sk#3, d_date#4] @@ -118,12 +112,12 @@ Condition : isnotnull(c_customer_sk#7) Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [id=#10] -(15) BroadcastHashJoin [codegen id : 11] +(15) BroadcastHashJoin [codegen id : 9] Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#7] Join condition: None -(16) Project [codegen id : 11] +(16) Project [codegen id : 9] Output [3]: [d_date#4, c_first_name#8, c_last_name#9] Input [5]: [ss_customer_sk#2, d_date#4, c_customer_sk#7, c_first_name#8, c_last_name#9] @@ -165,159 +159,123 @@ Join condition: None Output [3]: [c_last_name#17, c_first_name#16, d_date#14] Input [5]: [cs_bill_customer_sk#12, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] -(26) HashAggregate [codegen id : 5] +(26) BroadcastExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#17, c_first_name#16, d_date#14] - -(27) Exchange -Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), true, [id=#18] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#18] -(28) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#17, c_first_name#16, d_date#14] - -(29) BroadcastExchange -Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#19] - -(30) BroadcastHashJoin [codegen id : 11] +(27) BroadcastHashJoin [codegen id : 9] Left keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#8, ), isnull(c_first_name#8), coalesce(d_date#4, 0), isnull(d_date#4)] Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 0), isnull(d_date#14)] Join condition: None -(31) Scan parquet default.web_sales -Output [2]: [ws_sold_date_sk#20, ws_bill_customer_sk#21] +(28) Scan parquet default.web_sales +Output [2]: [ws_sold_date_sk#19, ws_bill_customer_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_sold_date_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 9] -Input [2]: [ws_sold_date_sk#20, ws_bill_customer_sk#21] +(29) ColumnarToRow [codegen id : 8] +Input [2]: [ws_sold_date_sk#19, ws_bill_customer_sk#20] -(33) Filter [codegen id : 9] -Input [2]: [ws_sold_date_sk#20, ws_bill_customer_sk#21] -Condition : (isnotnull(ws_sold_date_sk#20) AND isnotnull(ws_bill_customer_sk#21)) +(30) Filter [codegen id : 8] +Input [2]: [ws_sold_date_sk#19, ws_bill_customer_sk#20] +Condition : (isnotnull(ws_sold_date_sk#19) AND isnotnull(ws_bill_customer_sk#20)) -(34) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#22, d_date#23] +(31) ReusedExchange [Reuses operator id: 8] +Output [2]: [d_date_sk#21, d_date#22] -(35) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#20] -Right keys [1]: [d_date_sk#22] +(32) BroadcastHashJoin [codegen id : 8] +Left keys [1]: [ws_sold_date_sk#19] +Right keys [1]: [d_date_sk#21] Join condition: None -(36) Project [codegen id : 9] -Output [2]: [ws_bill_customer_sk#21, d_date#23] -Input [4]: [ws_sold_date_sk#20, ws_bill_customer_sk#21, d_date_sk#22, d_date#23] +(33) Project [codegen id : 8] +Output [2]: [ws_bill_customer_sk#20, d_date#22] +Input [4]: [ws_sold_date_sk#19, ws_bill_customer_sk#20, d_date_sk#21, d_date#22] -(37) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +(34) ReusedExchange [Reuses operator id: 14] +Output [3]: [c_customer_sk#23, c_first_name#24, c_last_name#25] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_bill_customer_sk#21] -Right keys [1]: [c_customer_sk#24] +(35) BroadcastHashJoin [codegen id : 8] +Left keys [1]: [ws_bill_customer_sk#20] +Right keys [1]: [c_customer_sk#23] Join condition: None -(39) Project [codegen id : 9] -Output [3]: [c_last_name#26, c_first_name#25, d_date#23] -Input [5]: [ws_bill_customer_sk#21, d_date#23, c_customer_sk#24, c_first_name#25, c_last_name#26] +(36) Project [codegen id : 8] +Output [3]: [c_last_name#25, c_first_name#24, d_date#22] +Input [5]: [ws_bill_customer_sk#20, d_date#22, c_customer_sk#23, c_first_name#24, c_last_name#25] -(40) HashAggregate [codegen id : 9] -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Keys [3]: [c_last_name#26, c_first_name#25, d_date#23] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#26, c_first_name#25, d_date#23] - -(41) Exchange -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Arguments: hashpartitioning(c_last_name#26, c_first_name#25, d_date#23, 5), true, [id=#27] - -(42) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Keys [3]: [c_last_name#26, c_first_name#25, d_date#23] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#26, c_first_name#25, d_date#23] - -(43) BroadcastExchange -Input [3]: [c_last_name#26, c_first_name#25, d_date#23] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#28] +(37) BroadcastExchange +Input [3]: [c_last_name#25, c_first_name#24, d_date#22] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 0), isnull(input[2, date, true])),false), [id=#26] -(44) BroadcastHashJoin [codegen id : 11] +(38) BroadcastHashJoin [codegen id : 9] Left keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#8, ), isnull(c_first_name#8), coalesce(d_date#4, 0), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#26, ), isnull(c_last_name#26), coalesce(c_first_name#25, ), isnull(c_first_name#25), coalesce(d_date#23, 0), isnull(d_date#23)] +Right keys [6]: [coalesce(c_last_name#25, ), isnull(c_last_name#25), coalesce(c_first_name#24, ), isnull(c_first_name#24), coalesce(d_date#22, 0), isnull(d_date#22)] Join condition: None -(45) HashAggregate [codegen id : 11] +(39) HashAggregate [codegen id : 9] Input [3]: [d_date#4, c_first_name#8, c_last_name#9] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(46) Exchange +(40) Exchange Input [3]: [c_last_name#9, c_first_name#8, d_date#4] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#4, 5), true, [id=#29] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#4, 5), ENSURE_REQUIREMENTS, [id=#27] -(47) HashAggregate [codegen id : 12] +(41) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(48) HashAggregate [codegen id : 12] +(42) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(49) HashAggregate [codegen id : 12] +(43) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(50) HashAggregate [codegen id : 12] +(44) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results [3]: [c_last_name#9, c_first_name#8, d_date#4] -(51) HashAggregate [codegen id : 12] +(45) HashAggregate [codegen id : 10] Input [3]: [c_last_name#9, c_first_name#8, d_date#4] Keys [3]: [c_last_name#9, c_first_name#8, d_date#4] Functions: [] Aggregate Attributes: [] Results: [] -(52) HashAggregate [codegen id : 12] +(46) HashAggregate [codegen id : 10] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#30] -Results [1]: [count#31] +Aggregate Attributes [1]: [count#28] +Results [1]: [count#29] -(53) Exchange -Input [1]: [count#31] -Arguments: SinglePartition, true, [id=#32] +(47) Exchange +Input [1]: [count#29] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#30] -(54) HashAggregate [codegen id : 13] -Input [1]: [count#31] +(48) HashAggregate [codegen id : 11] +Input [1]: [count#29] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#33] -Results [1]: [count(1)#33 AS count(1)#34] +Aggregate Attributes [1]: [count(1)#31] +Results [1]: [count(1)#31 AS count(1)#32] diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt index a5b57a4ac945..29fd421729ea 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt @@ -1,8 +1,8 @@ -WholeStageCodegen (13) +WholeStageCodegen (11) HashAggregate [count] [count(1),count(1),count] InputAdapter Exchange #1 - WholeStageCodegen (12) + WholeStageCodegen (10) HashAggregate [count,count] HashAggregate [c_last_name,c_first_name,d_date] HashAggregate [c_last_name,c_first_name,d_date] @@ -11,7 +11,7 @@ WholeStageCodegen (13) HashAggregate [c_last_name,c_first_name,d_date] InputAdapter Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (11) + WholeStageCodegen (9) HashAggregate [c_last_name,c_first_name,d_date] BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] @@ -40,41 +40,31 @@ WholeStageCodegen (13) Scan parquet default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #5 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (5) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_sold_date_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_sold_date_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 + WholeStageCodegen (5) + Project [c_last_name,c_first_name,d_date] + BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] + Project [cs_bill_customer_sk,d_date] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + Filter [cs_sold_date_sk,cs_bill_customer_sk] + ColumnarToRow InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + Scan parquet default.catalog_sales [cs_sold_date_sk,cs_bill_customer_sk] + InputAdapter + ReusedExchange [d_date_sk,d_date] #3 + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (8) + Project [c_last_name,c_first_name,d_date] + BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] + Project [ws_bill_customer_sk,d_date] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + Filter [ws_sold_date_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + Scan parquet default.web_sales [ws_sold_date_sk,ws_bill_customer_sk] + InputAdapter + ReusedExchange [d_date_sk,d_date] #3 + InputAdapter + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 From 3bb7cd59433498c6b3d1d6df14d166b082e7d0ce Mon Sep 17 00:00:00 2001 From: "tanel.kiis@gmail.com" Date: Wed, 6 Jan 2021 19:31:08 +0200 Subject: [PATCH 2/2] Address comments --- .../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 4a7ebd36d698..9077c90c923f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -1613,11 +1613,10 @@ object ConvertToLocalRelation extends Rule[LogicalPlan] { * {{{ * SELECT DISTINCT f1, f2 FROM t ==> SELECT f1, f2 FROM t GROUP BY f1, f2 * }}} - * - * The [[Distinct]] can be ignored if it is the right child of a left semi or anti join. */ object ReplaceDistinctWithAggregate extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformDown { + // The [[Distinct]] can be ignored if it is the right child of a left semi or anti join. case j @ Join(_, Distinct(right), LeftSemiOrAnti(_), _, _) => j.copy(right = right) case Distinct(child) => Aggregate(child.output, child.output, child) }