From 45fe128edd866e167562372040ad2dea42ce74d7 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Tue, 14 Jul 2020 23:22:58 +0800 Subject: [PATCH 01/24] [SPARK-32290][SQL] NotInSubquery SingleColumn Optimize Normally, a NotInSubquery will be planed into BroadcastNestedLoopJoin which is very time consuming, for instance, in TPCH Query 16. ``` select p_brand, p_type, p_size, count(distinct ps_suppkey) as supplier_cnt from partsupp, part where p_partkey = ps_partkey and p_brand <> 'Brand#45' and p_type not like 'MEDIUM POLISHED%' and p_size in (49, 14, 23, 45, 19, 3, 36, 9) and ps_suppkey not in ( select s_suppkey from supplier where s_comment like '%Customer%Complaints%' ) group by p_brand, p_type, p_size order by supplier_cnt desc, p_brand, p_type, p_size ``` In above not in subquery, will planed into LeftAnti condition Or((ps_suppkey=s_suppkey), IsNull(ps_suppkey=s_suppkey)) Inside BroadcastNestedLoopJoinExec will perform M*N, if buildSide is small enough, we can always change buildSide into a HashSet, and streamedSide just need to lookup in the HashSet, then the calculation will be optimized into M*log(N). But this optimize is only targeting on NotInSubquery with single column case. After apply this patch, the TPCH Query 16 performance decrease from 41mins to 30s TPCH is a common benchmark for distributed compute engine, all other 21 Query works fine on Spark, except for Query 16, apply this patch will make Spark more competitive among all these popular engine. BTW, this patch has restricted rules and only apply on NotInSubquery Single Column case, which is safe enough. No. 1. Manually run org.apache.spark.sql.SQLQueryTestSuite. 2. Compare performance before and after applying this patch against TPCH Query 16. Change-Id: I5fcb186e352e09b7e85b26353cf13d90b372205d --- .../apache/spark/sql/internal/SQLConf.scala | 24 +++ .../joins/BroadcastNestedLoopJoinExec.scala | 140 +++++++++++++++++- 2 files changed, 163 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index c68d7ccab4d10..cdc78c8e3e97a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2672,6 +2672,23 @@ object SQLConf { .checkValue(_ >= 0, "The value must be non-negative.") .createWithDefault(8) + val NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ENABLED = + buildConf("spark.sql.notInSubquery.singleColumn.optimize.enabled") + .internal() + .doc("When true, single column not in subquery execution in BroadcastNestedLoopJoinExec " + + "will be optimized from M*N calculation into M*log(N) calculation using HashMap lookup " + + "instead of Looping lookup.") + .booleanConf + .createWithDefault(false) + + val NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ROW_COUNT_THRESHOLD = + buildConf("spark.sql.notInSubquery.singleColumn.optimize.rowCountThreshold") + .internal() + .doc("Build side rowCount in BroadcastNestedLoopJoinExec must be less than this value " + + "before spark.sql.notInSubquery.singleColumn.optimize actually works.") + .intConf + .createWithDefault(10000) + /** * Holds information about keys that have been deprecated. * @@ -3277,6 +3294,13 @@ class SQLConf extends Serializable with Logging { def coalesceBucketsInJoinMaxBucketRatio: Int = getConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO) + def notInSubquerySingleColumnOptimizeEnabled: Boolean = + getConf(SQLConf.NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ENABLED) + + def notInSubquerySingleColumnOptimizeRowCountThreshold: Int = { + getConf(SQLConf.NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ROW_COUNT_THRESHOLD) + } + /** ********************** SQLConf functionality methods ************ */ /** Set Spark SQL configuration properties. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala index 52b476f9cf134..fc012067578b5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.execution.joins +import scala.collection.mutable + import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow @@ -26,6 +28,7 @@ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical._ import org.apache.spark.sql.execution.{ExplainUtils, SparkPlan} import org.apache.spark.sql.execution.metric.SQLMetrics +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.collection.{BitSet, CompactBuffer} case class BroadcastNestedLoopJoinExec( @@ -97,6 +100,24 @@ case class BroadcastNestedLoopJoinExec( } } + + /** + * See. [SPARK-32290] + * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join + * which is very time consuming because it's a M*N calculation. But if it's a single column + * NotInSubquery, and buildSide data is small enough, M*N calculation could be optimized into + * M*log(N) using HashMap to lookup build side rows. + * Meet following condition the optimize will works: + * NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ENABLED is true + * BuildSideRowCount <= NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ROW_COUNT_THRESHOLD + */ + + // NotInSubquery Key in StreamedSide + private var notInSubquerySingleColumnOptimizeStreamedKey: Attribute = _ + + // NotInSubquery Key Index in StreamedSide + private var notInSubquerySingleColumnOptimizeStreamedKeyIndex: Int = _ + /** * The implementation for InnerJoin. */ @@ -205,6 +226,117 @@ case class BroadcastNestedLoopJoinExec( } } + case class NotInSubquerySingleColumnOptimizeParams( + buildSideHashSet: mutable.HashSet[AnyRef], + isNullExists: Boolean, + isBuildRowsEmpty: Boolean) + + private def notInSubquerySingleColumnOptimizeEnabled: Boolean = { + if (SQLConf.get.notInSubquerySingleColumnOptimizeEnabled && right.output.length == 1) { + // buildSide must be single column + // and condition must be either of following pattern + // or(a=b,isnull(a=b)) + // or(isnull(a=b),a=b) + condition.get match { + case _@Or(_@EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), + _@IsNull(_@EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference))) + if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => + notInSubquerySingleColumnOptimizeSetStreamedKey(leftAttr, rightAttr) + if (notInSubquerySingleColumnOptimizeStreamedKeyIndex != -1) { + true + } else { + logWarning(s"failed to find notInSubquerySingleColumnOptimizeStreamedKeyIndex," + + s" fallback to leftExistenceJoin.") + false + } + case _@Or(_@IsNull(_@EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference)), + _@EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference)) + if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => + notInSubquerySingleColumnOptimizeSetStreamedKey(leftAttr, rightAttr) + if (notInSubquerySingleColumnOptimizeStreamedKeyIndex != -1) { + true + } else { + logWarning(s"failed to find notInSubquerySingleColumnOptimizeStreamedKeyIndex," + + s" fallback to leftExistenceJoin.") + false + } + case _ => false + } + } else { + false + } + } + + private def notInSubquerySingleColumnOptimizeBuildParams( + buildRows: Array[InternalRow]): NotInSubquerySingleColumnOptimizeParams = { + val buildRowsHashSet = mutable.HashSet[AnyRef]() + val dataType = right.output.head.dataType + var isNullExist = false + buildRows.foreach(row => { + if (row.isNullAt(0)) { + isNullExist = true + } + // put null as value, because we only leverage HashMap keys + buildRowsHashSet.add(row.get(0, dataType)) + }) + NotInSubquerySingleColumnOptimizeParams(buildRowsHashSet, isNullExist, buildRows.isEmpty) + } + + private def notInSubquerySingleColumnOptimizeSetStreamedKey( + leftAttr: AttributeReference, + rightAttr: AttributeReference): Unit = { + val leftNotInKeyFound = left.output.find(_.semanticEquals(leftAttr)) + if (leftNotInKeyFound.isDefined) { + notInSubquerySingleColumnOptimizeStreamedKey = leftAttr + notInSubquerySingleColumnOptimizeStreamedKeyIndex = + AttributeSeq(left.output).indexOf(leftAttr.exprId) + } else { + notInSubquerySingleColumnOptimizeStreamedKey = rightAttr + notInSubquerySingleColumnOptimizeStreamedKeyIndex = + AttributeSeq(left.output).indexOf(rightAttr.exprId) + } + } + + /** + * Optimized version implementation for LeftAnti + * which converted from single column NotInSubquery + * @param relation + * @return + */ + private def notInSubquerySingleColumnOptimize( + relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = { + assert(buildSide == BuildRight) + val buildRows = relation.value + if (buildRows.length > SQLConf.get.notInSubquerySingleColumnOptimizeRowCountThreshold) { + logWarning(s"BuildSide row count ${buildRows.length} exceeded " + + s"notInSubquerySingleColumnOptimizeRowCountThreshold ${SQLConf.get + .notInSubquerySingleColumnOptimizeRowCountThreshold}, fallback to leftExistenceJoin.") + leftExistenceJoin(relation, false) + } else { + val params = notInSubquerySingleColumnOptimizeBuildParams(buildRows) + streamed.execute().mapPartitionsInternal { streamedIter => + streamedIter.filter(row => { + // See. not-in-unit-tests-single-column.sql for detail filter rules + if (params.isBuildRowsEmpty) { + true + } else { + val streamedRowIsNull = row.isNullAt(notInSubquerySingleColumnOptimizeStreamedKeyIndex) + val streamedRowNotInKey = row.get( + notInSubquerySingleColumnOptimizeStreamedKeyIndex, + notInSubquerySingleColumnOptimizeStreamedKey.dataType + ) + val notInKeyEqual = params.buildSideHashSet.contains(streamedRowNotInKey) + if (!streamedRowIsNull && !params.isNullExists && !notInKeyEqual) { + true + } else { + false + } + } + }) + } + } + } + private def existenceJoin(relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = { assert(buildSide == BuildRight) streamed.execute().mapPartitionsInternal { streamedIter => @@ -357,7 +489,13 @@ case class BroadcastNestedLoopJoinExec( case (LeftSemi, BuildRight) => leftExistenceJoin(broadcastedRelation, exists = true) case (LeftAnti, BuildRight) => - leftExistenceJoin(broadcastedRelation, exists = false) + // if LeftAnti is planed from Not In Subquery with single column + // use HashMap implementation to further optimize + if (notInSubquerySingleColumnOptimizeEnabled) { + notInSubquerySingleColumnOptimize(broadcastedRelation) + } else { + leftExistenceJoin(broadcastedRelation, exists = false) + } case (j: ExistenceJoin, BuildRight) => existenceJoin(broadcastedRelation) case _ => From 25fcb789663ea18c25d6450da8caabc4a07dcc62 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Wed, 15 Jul 2020 19:51:51 +0800 Subject: [PATCH 02/24] [SPARK-32290][SQL] NotInSubquery SingleColumn Optimize sub commit 1. change HashSet into HashedRelation. 2. remove duplicate code, Or(EqualTo(a, b), IsNull(EqualTo(a, b))) will be the only option. 3. code style refine. 4. run org.apache.spark.sql.SQLQueryTestSuite with switch on and off, both passed. Change-Id: If0e803c8a23764b678c5d64091119f8a9f06c4a6 --- .../joins/BroadcastNestedLoopJoinExec.scala | 67 ++++++++----------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala index fc012067578b5..54be558e61503 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala @@ -17,8 +17,6 @@ package org.apache.spark.sql.execution.joins -import scala.collection.mutable - import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow @@ -227,31 +225,19 @@ case class BroadcastNestedLoopJoinExec( } case class NotInSubquerySingleColumnOptimizeParams( - buildSideHashSet: mutable.HashSet[AnyRef], + buildSideHashedRelation: HashedRelation, isNullExists: Boolean, isBuildRowsEmpty: Boolean) private def notInSubquerySingleColumnOptimizeEnabled: Boolean = { if (SQLConf.get.notInSubquerySingleColumnOptimizeEnabled && right.output.length == 1) { - // buildSide must be single column - // and condition must be either of following pattern - // or(a=b,isnull(a=b)) - // or(isnull(a=b),a=b) + // BuildSide must be single column, condition must be the following pattern + // Or(EqualTo(a, b), IsNull(EqualTo(a, b))) condition.get match { - case _@Or(_@EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), - _@IsNull(_@EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference))) - if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => - notInSubquerySingleColumnOptimizeSetStreamedKey(leftAttr, rightAttr) - if (notInSubquerySingleColumnOptimizeStreamedKeyIndex != -1) { - true - } else { - logWarning(s"failed to find notInSubquerySingleColumnOptimizeStreamedKeyIndex," + - s" fallback to leftExistenceJoin.") - false - } - case _@Or(_@IsNull(_@EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference)), - _@EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference)) - if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => + case _ @ Or( + _ @ EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), + _ @ IsNull(_ @ EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference))) + if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => notInSubquerySingleColumnOptimizeSetStreamedKey(leftAttr, rightAttr) if (notInSubquerySingleColumnOptimizeStreamedKeyIndex != -1) { true @@ -269,24 +255,20 @@ case class BroadcastNestedLoopJoinExec( private def notInSubquerySingleColumnOptimizeBuildParams( buildRows: Array[InternalRow]): NotInSubquerySingleColumnOptimizeParams = { - val buildRowsHashSet = mutable.HashSet[AnyRef]() - val dataType = right.output.head.dataType - var isNullExist = false - buildRows.foreach(row => { - if (row.isNullAt(0)) { - isNullExist = true - } - // put null as value, because we only leverage HashMap keys - buildRowsHashSet.add(row.get(0, dataType)) - }) - NotInSubquerySingleColumnOptimizeParams(buildRowsHashSet, isNullExist, buildRows.isEmpty) + NotInSubquerySingleColumnOptimizeParams( + HashedRelation(buildRows.iterator, + BindReferences.bindReferences[Expression]( + Seq(right.output.head), AttributeSeq(right.output)), + buildRows.length), + buildRows.exists(row => row.isNullAt(0)), + buildRows.isEmpty) } private def notInSubquerySingleColumnOptimizeSetStreamedKey( leftAttr: AttributeReference, rightAttr: AttributeReference): Unit = { - val leftNotInKeyFound = left.output.find(_.semanticEquals(leftAttr)) - if (leftNotInKeyFound.isDefined) { + val leftNotInKeyFound = left.output.exists(_.semanticEquals(leftAttr)) + if (leftNotInKeyFound) { notInSubquerySingleColumnOptimizeStreamedKey = leftAttr notInSubquerySingleColumnOptimizeStreamedKeyIndex = AttributeSeq(left.output).indexOf(leftAttr.exprId) @@ -315,17 +297,24 @@ case class BroadcastNestedLoopJoinExec( } else { val params = notInSubquerySingleColumnOptimizeBuildParams(buildRows) streamed.execute().mapPartitionsInternal { streamedIter => + val keyGenerator = + UnsafeProjection.create( + BindReferences.bindReferences[Expression]( + Seq(notInSubquerySingleColumnOptimizeStreamedKey), + AttributeSeq(left.output)) + ) streamedIter.filter(row => { // See. not-in-unit-tests-single-column.sql for detail filter rules if (params.isBuildRowsEmpty) { true } else { val streamedRowIsNull = row.isNullAt(notInSubquerySingleColumnOptimizeStreamedKeyIndex) - val streamedRowNotInKey = row.get( - notInSubquerySingleColumnOptimizeStreamedKeyIndex, - notInSubquerySingleColumnOptimizeStreamedKey.dataType - ) - val notInKeyEqual = params.buildSideHashSet.contains(streamedRowNotInKey) + val lookupRow: UnsafeRow = keyGenerator(row) + val notInKeyEqual = params.buildSideHashedRelation.get(lookupRow) match { + case null => false + case _ => true + } + if (!streamedRowIsNull && !params.isNullExists && !notInKeyEqual) { true } else { From 7a02c0ba24bf402587655cb4513ce10df3d62365 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Fri, 17 Jul 2020 09:20:12 +0800 Subject: [PATCH 03/24] [SPARK-32290][SQL] NotInSubquery SingleColumn Optimize sub commit 1. move code into BroadcastHashJoinExec. 2. just for review purpose, wait for commiter's approval. Change-Id: I5c9f8fb0c3315e5ee8189e756392d22106cd3e31 --- .../joins/BroadcastHashJoinExec.scala | 88 ++++++++++++++----- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 71faad9829a42..7648f9689a228 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -49,6 +49,8 @@ case class BroadcastHashJoinExec( right: SparkPlan) extends HashJoin with CodegenSupport { + var singleColumnNullAware: Boolean = false + override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) @@ -453,29 +455,75 @@ case class BroadcastHashJoinExec( val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) val (matched, checkCondition, _) = getJoinCondition(ctx, input) val numOutput = metricTerm(ctx, "numOutputRows") + val (buildSideNullExists, buildSideEmpty) = if (singleColumnNullAware) { + val singleColumnNullRow: UnsafeRow = new UnsafeRow(1) + singleColumnNullRow.setNullAt(0) + (broadcastRelation.value.get(singleColumnNullRow) != null, + broadcastRelation.value.keys().isEmpty) + } else { + // should not be reach if singleColumnNullAware = false + (false, false) + } if (uniqueKeyCodePath) { val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin + if (singleColumnNullAware) { + assert(checkCondition == "") + if (buildSideEmpty) { + s""" + |// singleColumnNullAware buildSideEmpty(true) accept all + |$numOutput.add(1); + |${consume(ctx, input)} + """.stripMargin + } else if (buildSideNullExists) { + s""" + |// singleColumnNullAware buildSideEmpty(false) buildSideNullExists(true) reject all + """.stripMargin + } else { + s""" + |// singleColumnNullAware buildSideEmpty(false) buildSideNullExists(false) + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | $found = true; + | } + |} else { + | $found = true; + |} + | + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin + } + } else { + s""" + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | // Evaluate the condition. + | $checkCondition { + | $found = true; + | } + | } + |} + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin + } } else { val matches = ctx.freshName("matches") val iteratorCls = classOf[Iterator[UnsafeRow]].getName From ec66df6de408ed112ab171deabe324eb672920be Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Fri, 17 Jul 2020 17:17:22 +0800 Subject: [PATCH 04/24] [SPARK-32290][SQL] NotInSubquery SingleColumn Optimize sub commit 1. revert code in BroadcastHashJoinExec. 2. revert code in BroadcastNestedLoopJoinExec. 3. move code into new ExecNode call BroadcastNullAwareHashJoinExec, currently this ExecNode only handle SingleColumn NotInSubquery case, it might be extended into supporting multi column. 4. add NotInSubqueryHashJoinTestSuite with NOT_IN_SUBQUERY_HASH_JOIN_ENABLED = true to run E2E SQL Coverage cases for all subquery case. 5. resolve other commiter's comment. Test. * NotInSubqueryHashJoinTestSuite passed. * SQLQueryTestSuite passed. * Local E2E with NOT_IN_SUBQUERY_HASH_JOIN_ENABLED = true, passed and verified BroadcastNullAwareHashJoinExec applied. * Local E2E with NOT_IN_SUBQUERY_HASH_JOIN_ENABLED = true, spark.sql.adaptive.enabled = true, passed and verified BroadcastNullAwareHashJoinExec applied. * TPCH Query 16. Change-Id: I29c22d3db6d0684595ca494fd33f768b4d9a2297 --- .../apache/spark/sql/internal/SQLConf.scala | 28 +-- .../spark/sql/execution/SparkStrategies.scala | 32 +++- .../adaptive/LogicalQueryStageStrategy.scala | 35 +++- .../joins/BroadcastHashJoinExec.scala | 88 +++------- .../joins/BroadcastNestedLoopJoinExec.scala | 129 +------------- .../BroadcastNullAwareHashJoinExec.scala | 163 ++++++++++++++++++ .../sql/NotInSubqueryHashJoinTestSuite.scala | 64 +++++++ 7 files changed, 317 insertions(+), 222 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/NotInSubqueryHashJoinTestSuite.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index cdc78c8e3e97a..3312487b1ddb2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2672,23 +2672,17 @@ object SQLConf { .checkValue(_ >= 0, "The value must be non-negative.") .createWithDefault(8) - val NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ENABLED = - buildConf("spark.sql.notInSubquery.singleColumn.optimize.enabled") + val NOT_IN_SUBQUERY_HASH_JOIN_ENABLED = + buildConf("spark.sql.notInSubquery.hashJoin.enabled") .internal() - .doc("When true, single column not in subquery execution in BroadcastNestedLoopJoinExec " + - "will be optimized from M*N calculation into M*log(N) calculation using HashMap lookup " + - "instead of Looping lookup.") + .doc("When true, not in subquery execution will be planed into " + + "BroadcastNullAwareHashJoinExec, " + + "optimized from O(M*N) calculation into O(M) calculation " + + "using Hash lookup instead of Looping lookup." + + "Only support for singleColumn not in subquery for now.") .booleanConf .createWithDefault(false) - val NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ROW_COUNT_THRESHOLD = - buildConf("spark.sql.notInSubquery.singleColumn.optimize.rowCountThreshold") - .internal() - .doc("Build side rowCount in BroadcastNestedLoopJoinExec must be less than this value " + - "before spark.sql.notInSubquery.singleColumn.optimize actually works.") - .intConf - .createWithDefault(10000) - /** * Holds information about keys that have been deprecated. * @@ -3294,12 +3288,8 @@ class SQLConf extends Serializable with Logging { def coalesceBucketsInJoinMaxBucketRatio: Int = getConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO) - def notInSubquerySingleColumnOptimizeEnabled: Boolean = - getConf(SQLConf.NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ENABLED) - - def notInSubquerySingleColumnOptimizeRowCountThreshold: Int = { - getConf(SQLConf.NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ROW_COUNT_THRESHOLD) - } + def notInSubqueryHashJoinEnabled: Boolean = + getConf(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED) /** ********************** SQLConf functionality methods ************ */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala index 78aa258387daa..705714ecd2a92 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala @@ -259,6 +259,31 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { if (canBuildLeft(joinType)) BuildLeft else BuildRight } + /** + * See. [SPARK-32290] + * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join, + * which is very time consuming because it's an O(M*N) calculation. + * But if it's a single column NotInSubquery, and buildSide data is small enough, + * O(M*N) calculation could be optimized into O(M) using hash lookup instead of loop lookup. + */ + def createBroadcastNullAwareHashJoin() = { + if (conf.notInSubqueryHashJoinEnabled && + joinType == LeftAnti && + canBroadcastBySize(right, conf) && + right.output.length == 1) { + val (matched, _, _) = + joins.NotInSubqueryConditionPattern.singleColumnPatternMatch(condition) + if (matched) { + Some(Seq(joins.BroadcastNullAwareHashJoinExec( + planLater(left), planLater(right), BuildRight, LeftAnti, condition))) + } else { + None + } + } else { + None + } + } + def createBroadcastNLJoin(buildLeft: Boolean, buildRight: Boolean) = { val maybeBuildSide = if (buildLeft && buildRight) { Some(desiredBuildSide) @@ -294,9 +319,10 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { } } - createBroadcastNLJoin(hintToBroadcastLeft(hint), hintToBroadcastRight(hint)) - .orElse { if (hintToShuffleReplicateNL(hint)) createCartesianProduct() else None } - .getOrElse(createJoinWithoutHint()) + createBroadcastNullAwareHashJoin().orElse { + createBroadcastNLJoin(hintToBroadcastLeft(hint), hintToBroadcastRight(hint)) + .orElse { if (hintToShuffleReplicateNL(hint)) createCartesianProduct() else None } + }.getOrElse(createJoinWithoutHint()) // --- Cases where this strategy does not apply --------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala index ac98342277bc0..e1aa6b7418545 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala @@ -21,9 +21,11 @@ import org.apache.spark.sql.Strategy import org.apache.spark.sql.catalyst.expressions.PredicateHelper import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys +import org.apache.spark.sql.catalyst.plans.LeftAnti import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan} -import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.{joins, SparkPlan} import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec} +import org.apache.spark.sql.internal.SQLConf /** * Strategy for plans containing [[LogicalQueryStage]] nodes: @@ -50,9 +52,34 @@ object LogicalQueryStageStrategy extends Strategy with PredicateHelper { case j @ Join(left, right, joinType, condition, _) if isBroadcastStage(left) || isBroadcastStage(right) => - val buildSide = if (isBroadcastStage(left)) BuildLeft else BuildRight - BroadcastNestedLoopJoinExec( - planLater(left), planLater(right), buildSide, joinType, condition) :: Nil + def createBroadcastNLJoin() = { + val buildSide = if (isBroadcastStage(left)) BuildLeft else BuildRight + BroadcastNestedLoopJoinExec( + planLater(left), planLater(right), buildSide, joinType, condition) :: Nil + } + + /** + * See. [SPARK-32290] + * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join, + * which is very time consuming because it's an O(M*N) calculation. + * But if it's a single column NotInSubquery, and buildSide data is small enough, + * O(M*N) calculation could be optimized into O(M) using hash lookup instead of loop lookup. + */ + if (SQLConf.get.notInSubqueryHashJoinEnabled && + joinType == LeftAnti && + isBroadcastStage(right) && + right.output.length == 1) { + val (matched, _, _) = + joins.NotInSubqueryConditionPattern.singleColumnPatternMatch(condition) + if (matched) { + Seq(joins.BroadcastNullAwareHashJoinExec( + planLater(left), planLater(right), BuildRight, LeftAnti, condition)) + } else { + createBroadcastNLJoin() + } + } else { + createBroadcastNLJoin() + } case q: LogicalQueryStage => q.physicalPlan :: Nil diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 7648f9689a228..71faad9829a42 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -49,8 +49,6 @@ case class BroadcastHashJoinExec( right: SparkPlan) extends HashJoin with CodegenSupport { - var singleColumnNullAware: Boolean = false - override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) @@ -455,75 +453,29 @@ case class BroadcastHashJoinExec( val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) val (matched, checkCondition, _) = getJoinCondition(ctx, input) val numOutput = metricTerm(ctx, "numOutputRows") - val (buildSideNullExists, buildSideEmpty) = if (singleColumnNullAware) { - val singleColumnNullRow: UnsafeRow = new UnsafeRow(1) - singleColumnNullRow.setNullAt(0) - (broadcastRelation.value.get(singleColumnNullRow) != null, - broadcastRelation.value.keys().isEmpty) - } else { - // should not be reach if singleColumnNullAware = false - (false, false) - } if (uniqueKeyCodePath) { val found = ctx.freshName("found") - if (singleColumnNullAware) { - assert(checkCondition == "") - if (buildSideEmpty) { - s""" - |// singleColumnNullAware buildSideEmpty(true) accept all - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } else if (buildSideNullExists) { - s""" - |// singleColumnNullAware buildSideEmpty(false) buildSideNullExists(true) reject all - """.stripMargin - } else { - s""" - |// singleColumnNullAware buildSideEmpty(false) buildSideNullExists(false) - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | $found = true; - | } - |} else { - | $found = true; - |} - | - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } - } else { - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } + s""" + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | // Evaluate the condition. + | $checkCondition { + | $found = true; + | } + | } + |} + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin } else { val matches = ctx.freshName("matches") val iteratorCls = classOf[Iterator[UnsafeRow]].getName diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala index 54be558e61503..52b476f9cf134 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala @@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical._ import org.apache.spark.sql.execution.{ExplainUtils, SparkPlan} import org.apache.spark.sql.execution.metric.SQLMetrics -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.collection.{BitSet, CompactBuffer} case class BroadcastNestedLoopJoinExec( @@ -98,24 +97,6 @@ case class BroadcastNestedLoopJoinExec( } } - - /** - * See. [SPARK-32290] - * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join - * which is very time consuming because it's a M*N calculation. But if it's a single column - * NotInSubquery, and buildSide data is small enough, M*N calculation could be optimized into - * M*log(N) using HashMap to lookup build side rows. - * Meet following condition the optimize will works: - * NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ENABLED is true - * BuildSideRowCount <= NOT_IN_SUBQUERY_SINGLE_COLUMN_OPTIMIZE_ROW_COUNT_THRESHOLD - */ - - // NotInSubquery Key in StreamedSide - private var notInSubquerySingleColumnOptimizeStreamedKey: Attribute = _ - - // NotInSubquery Key Index in StreamedSide - private var notInSubquerySingleColumnOptimizeStreamedKeyIndex: Int = _ - /** * The implementation for InnerJoin. */ @@ -224,108 +205,6 @@ case class BroadcastNestedLoopJoinExec( } } - case class NotInSubquerySingleColumnOptimizeParams( - buildSideHashedRelation: HashedRelation, - isNullExists: Boolean, - isBuildRowsEmpty: Boolean) - - private def notInSubquerySingleColumnOptimizeEnabled: Boolean = { - if (SQLConf.get.notInSubquerySingleColumnOptimizeEnabled && right.output.length == 1) { - // BuildSide must be single column, condition must be the following pattern - // Or(EqualTo(a, b), IsNull(EqualTo(a, b))) - condition.get match { - case _ @ Or( - _ @ EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), - _ @ IsNull(_ @ EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference))) - if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => - notInSubquerySingleColumnOptimizeSetStreamedKey(leftAttr, rightAttr) - if (notInSubquerySingleColumnOptimizeStreamedKeyIndex != -1) { - true - } else { - logWarning(s"failed to find notInSubquerySingleColumnOptimizeStreamedKeyIndex," + - s" fallback to leftExistenceJoin.") - false - } - case _ => false - } - } else { - false - } - } - - private def notInSubquerySingleColumnOptimizeBuildParams( - buildRows: Array[InternalRow]): NotInSubquerySingleColumnOptimizeParams = { - NotInSubquerySingleColumnOptimizeParams( - HashedRelation(buildRows.iterator, - BindReferences.bindReferences[Expression]( - Seq(right.output.head), AttributeSeq(right.output)), - buildRows.length), - buildRows.exists(row => row.isNullAt(0)), - buildRows.isEmpty) - } - - private def notInSubquerySingleColumnOptimizeSetStreamedKey( - leftAttr: AttributeReference, - rightAttr: AttributeReference): Unit = { - val leftNotInKeyFound = left.output.exists(_.semanticEquals(leftAttr)) - if (leftNotInKeyFound) { - notInSubquerySingleColumnOptimizeStreamedKey = leftAttr - notInSubquerySingleColumnOptimizeStreamedKeyIndex = - AttributeSeq(left.output).indexOf(leftAttr.exprId) - } else { - notInSubquerySingleColumnOptimizeStreamedKey = rightAttr - notInSubquerySingleColumnOptimizeStreamedKeyIndex = - AttributeSeq(left.output).indexOf(rightAttr.exprId) - } - } - - /** - * Optimized version implementation for LeftAnti - * which converted from single column NotInSubquery - * @param relation - * @return - */ - private def notInSubquerySingleColumnOptimize( - relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = { - assert(buildSide == BuildRight) - val buildRows = relation.value - if (buildRows.length > SQLConf.get.notInSubquerySingleColumnOptimizeRowCountThreshold) { - logWarning(s"BuildSide row count ${buildRows.length} exceeded " + - s"notInSubquerySingleColumnOptimizeRowCountThreshold ${SQLConf.get - .notInSubquerySingleColumnOptimizeRowCountThreshold}, fallback to leftExistenceJoin.") - leftExistenceJoin(relation, false) - } else { - val params = notInSubquerySingleColumnOptimizeBuildParams(buildRows) - streamed.execute().mapPartitionsInternal { streamedIter => - val keyGenerator = - UnsafeProjection.create( - BindReferences.bindReferences[Expression]( - Seq(notInSubquerySingleColumnOptimizeStreamedKey), - AttributeSeq(left.output)) - ) - streamedIter.filter(row => { - // See. not-in-unit-tests-single-column.sql for detail filter rules - if (params.isBuildRowsEmpty) { - true - } else { - val streamedRowIsNull = row.isNullAt(notInSubquerySingleColumnOptimizeStreamedKeyIndex) - val lookupRow: UnsafeRow = keyGenerator(row) - val notInKeyEqual = params.buildSideHashedRelation.get(lookupRow) match { - case null => false - case _ => true - } - - if (!streamedRowIsNull && !params.isNullExists && !notInKeyEqual) { - true - } else { - false - } - } - }) - } - } - } - private def existenceJoin(relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = { assert(buildSide == BuildRight) streamed.execute().mapPartitionsInternal { streamedIter => @@ -478,13 +357,7 @@ case class BroadcastNestedLoopJoinExec( case (LeftSemi, BuildRight) => leftExistenceJoin(broadcastedRelation, exists = true) case (LeftAnti, BuildRight) => - // if LeftAnti is planed from Not In Subquery with single column - // use HashMap implementation to further optimize - if (notInSubquerySingleColumnOptimizeEnabled) { - notInSubquerySingleColumnOptimize(broadcastedRelation) - } else { - leftExistenceJoin(broadcastedRelation, exists = false) - } + leftExistenceJoin(broadcastedRelation, exists = false) case (j: ExistenceJoin, BuildRight) => existenceJoin(broadcastedRelation) case _ => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala new file mode 100644 index 0000000000000..a0025e42da568 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.joins + +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.optimizer.{BuildRight, BuildSide} +import org.apache.spark.sql.catalyst.plans._ +import org.apache.spark.sql.catalyst.plans.physical._ +import org.apache.spark.sql.execution.{ExplainUtils, SparkPlan} +import org.apache.spark.sql.execution.metric.SQLMetrics +import org.apache.spark.sql.internal.SQLConf + +private case class NotInSubqueryHashJoinParams( + buildSideRelation: HashedRelation, + buildSideIsNullExists: Boolean, + buildSideIsEmpty: Boolean, + streamedSideKey: Attribute, + streamedSideKeyIndex: Int) + +object NotInSubqueryConditionPattern { + def singleColumnPatternMatch(condition: Option[Expression]): (Boolean, Attribute, Attribute) = { + condition.get match { + case Or(EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), + IsNull(EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference))) + if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => + (true, leftAttr, rightAttr) + case _ => (false, null, null) + } + } +} + +case class BroadcastNullAwareHashJoinExec( + left: SparkPlan, + right: SparkPlan, + buildSide: BuildSide, + joinType: JoinType, + condition: Option[Expression]) extends BaseJoinExec { + + // TODO support multi column not in subquery in future. + // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6 + // multi-column null aware anti join is much more complicated than single column ones. + require(right.output.length == 1, "not in subquery hash join optimize only single column.") + require(joinType == LeftAnti, "joinType must be LeftAnti.") + require(buildSide == BuildRight, "buildSide must be BuildRight.") + require(SQLConf.get.notInSubqueryHashJoinEnabled, + "notInSubqueryHashJoinEnabled must turn on for BroadcastNullAwareHashJoinExec.") + + override def leftKeys: Seq[Expression] = Nil + override def rightKeys: Seq[Expression] = Nil + + override lazy val metrics = Map( + "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) + + private val (streamed, broadcast) = (left, right) + + override def simpleStringWithNodeId(): String = { + val opId = ExplainUtils.getOpId(this) + s"$nodeName $joinType ${buildSide} ($opId)".trim + } + + override def requiredChildDistribution: Seq[Distribution] = { + UnspecifiedDistribution :: BroadcastDistribution(IdentityBroadcastMode) :: Nil + } + + private[this] def genResultProjection: UnsafeProjection = { + UnsafeProjection.create(output, output) + } + + override def output: Seq[Attribute] = { + left.output + } + + private def buildParams(buildSideRows: Array[InternalRow]): NotInSubqueryHashJoinParams = { + val (_, leftAttr, rightAttr) = + NotInSubqueryConditionPattern.singleColumnPatternMatch(condition) + + val leftNotInKeyFound = left.output.exists(_.semanticEquals(leftAttr)) + val (streamedKey, streamedKeyIndex) = if (leftNotInKeyFound) { + (leftAttr, AttributeSeq(left.output).indexOf(leftAttr.exprId)) + } else { + (rightAttr, AttributeSeq(left.output).indexOf(rightAttr.exprId)) + } + + NotInSubqueryHashJoinParams( + HashedRelation(buildSideRows.iterator, + BindReferences.bindReferences[Expression]( + Seq(right.output.head), AttributeSeq(right.output)), + buildSideRows.length), + buildSideRows.exists(row => row.isNullAt(0)), + buildSideRows.isEmpty, + streamedKey, + streamedKeyIndex + ) + } + + private def leftAntiNullAwareJoin( + relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = { + val params = buildParams(relation.value) + streamed.execute().mapPartitionsInternal { streamedIter => + if (params.buildSideIsEmpty) { + streamedIter + } else if (params.buildSideIsNullExists) { + Iterator.empty + } else { + val keyGenerator = UnsafeProjection.create( + BindReferences.bindReferences[Expression]( + Seq(params.streamedSideKey), + AttributeSeq(left.output)) + ) + streamedIter.filter(row => { + val streamedRowIsNull = row.isNullAt(params.streamedSideKeyIndex) + val lookupRow: UnsafeRow = keyGenerator(row) + val notInKeyEqual = params.buildSideRelation.get(lookupRow) != null + if (!streamedRowIsNull && !notInKeyEqual) { + true + } else { + false + } + }) + } + } + } + + protected override def doExecute(): RDD[InternalRow] = { + val broadcastedRelation = broadcast.executeBroadcast[Array[InternalRow]]() + + val resultRdd = (joinType, buildSide) match { + case (LeftAnti, BuildRight) => + leftAntiNullAwareJoin(broadcastedRelation) + case _ => + throw new IllegalArgumentException( + s"BroadcastNullAwareHashJoinExec support (LeftAnti + BuildRight) only for now.") + } + + val numOutputRows = longMetric("numOutputRows") + resultRdd.mapPartitionsWithIndexInternal { (index, iter) => + val resultProj = genResultProjection + resultProj.initialize(index) + iter.map { r => + numOutputRows += 1 + resultProj(r) + } + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/NotInSubqueryHashJoinTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/NotInSubqueryHashJoinTestSuite.scala new file mode 100644 index 0000000000000..94bee6ca653fa --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/NotInSubqueryHashJoinTestSuite.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import java.io.File + +import org.apache.spark.SparkConf +import org.apache.spark.sql.internal.SQLConf + +/** + * End-to-end test cases for subquery SQL queries coverage with + * NOT_IN_SUBQUERY_HASH_JOIN_ENABLED = true. + * + * Each case is loaded from a file in + * "spark/sql/core/src/test/resources/sql-tests/inputs/subquery". + * Each case has a golden result file in + * "spark/sql/core/src/test/resources/sql-tests/results/subquery". + * + * To run the entire test suite: + * {{{ + * build/sbt "sql/test-only *NotInSubqueryHashJoinTestSuite" + * }}} + * + */ +class NotInSubqueryHashJoinTestSuite extends SQLQueryTestSuite { + + protected override def sparkConf: SparkConf = super.sparkConf + // Fewer shuffle partitions to speed up testing. + // enable NOT_IN_SUBQUERY_HASH_JOIN_ENABLED for subquery case coverage. + .set(SQLConf.SHUFFLE_PARTITIONS, 4) + .set(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED, true) + + override lazy val listTestCases: Seq[TestCase] = { + listFilesRecursively(new File(inputFilePath)).flatMap { file => + val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".out" + val absPath = file.getAbsolutePath + val testCaseName = + s"notInSubqueryHashJoin@" + + s"${absPath.stripPrefix(inputFilePath).stripPrefix(File.separator)}" + + if (file.getAbsolutePath.startsWith( + s"$inputFilePath${File.separator}subquery")) { + RegularTestCase(testCaseName, absPath, resultFile) :: Nil + } else { + Seq.empty + } + } + } +} From b00abfea109c69dee2ad6e8fd7973d8d19718c9f Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 20 Jul 2020 10:27:48 +0800 Subject: [PATCH 05/24] [SPARK-32290][SQL] NotInSubquery SingleColumn Optimize sub commit 1. move extractor into patterns.scala. 2. add cases in JoinSuite. 3. code refine. 4. BroadcastNullAwareHashJoinExec add codegen support. 5. HashedRelation & BroadcaseHashJoinExec prototype for BHJ implementation (might be removed later) Change-Id: Iba0b4743ff3d06d3d4dd85f5dc45e3e4c288fb74 --- .../spark/unsafe/map/BytesToBytesMap.java | 23 +++ .../sql/catalyst/planning/patterns.scala | 35 +++++ .../spark/sql/execution/SparkStrategies.scala | 42 ++--- .../adaptive/LogicalQueryStageStrategy.scala | 38 +---- .../joins/BroadcastHashJoinExec.scala | 145 ++++++++++++------ .../BroadcastNullAwareHashJoinExec.scala | 122 +++++++++++---- .../sql/execution/joins/HashedRelation.scala | 27 ++++ .../org/apache/spark/sql/JoinSuite.scala | 27 ++++ 8 files changed, 325 insertions(+), 134 deletions(-) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index 6e028886f2318..4189e006c88f0 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -171,6 +171,29 @@ public final class BytesToBytesMap extends MemoryConsumer { private volatile MapIterator destructiveIterator = null; private LinkedList spillWriters = new LinkedList<>(); + private boolean inputIsEmpty = false; + private boolean anyNullKeyExists = false; + + public boolean isInputIsEmpty() + { + return inputIsEmpty; + } + + public void setInputIsEmpty(boolean inputIsEmpty) + { + this.inputIsEmpty = inputIsEmpty; + } + + public boolean isAnyNullKeyExists() + { + return anyNullKeyExists; + } + + public void setAnyNullKeyExists(boolean anyNullKeyExists) + { + this.anyNullKeyExists = anyNullKeyExists; + } + public BytesToBytesMap( TaskMemoryManager taskMemoryManager, BlockManager blockManager, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index 415ce46788119..fd5cfc555e4f6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -21,8 +21,10 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression +import org.apache.spark.sql.catalyst.optimizer.JoinSelectionHelper import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.internal.SQLConf trait OperationHelper { type ReturnType = (Seq[NamedExpression], Seq[Expression], LogicalPlan) @@ -388,3 +390,36 @@ object PhysicalWindow { case _ => None } } + +object ExtractSingleColumnNotInSubquery extends JoinSelectionHelper { + + // SingleColumnNotInSubquery + // streamedSideKeys, buildSideKeys + // currently these two return Seq[Expression] should have only one element + private type ReturnType = + (Seq[Expression], Seq[Expression]) + + /** + * See. [SPARK-32290] + * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join, + * which is very time consuming because it's an O(M*N) calculation. + * But if it's a single column NotInSubquery, and buildSide data is small enough, + * O(M*N) calculation could be optimized into O(M) using hash lookup instead of loop lookup. + */ + def unapply(join: Join): Option[ReturnType] = join match { + case Join(left, right, LeftAnti, + Some(Or(EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), + IsNull(EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference)))), _) + if SQLConf.get.notInSubqueryHashJoinEnabled && + leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) && + canBroadcastBySize(right, SQLConf.get) && + right.output.length == 1 => + val leftNotInKeyFound = left.output.exists(_.semanticEquals(leftAttr)) + if (leftNotInKeyFound) { + Some(Seq(leftAttr), Seq(rightAttr)) + } else { + Some(Seq(rightAttr), Seq(leftAttr)) + } + case _ => None + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala index 705714ecd2a92..2d9b7ec31ce99 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala @@ -232,6 +232,16 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { .orElse { if (hintToShuffleReplicateNL(hint)) createCartesianProduct() else None } .getOrElse(createJoinWithoutHint()) + case j @ ExtractSingleColumnNotInSubquery(leftKeys, rightKeys) => + Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, + planLater(j.left), planLater(j.right), BuildRight, LeftAnti, j.condition)) + + // for BHJ Prototype +// val bhj = joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, +// None, planLater(j.left), planLater(j.right)) +// bhj.nullAwareJoin = true +// Seq(bhj) + // If it is not an equi-join, we first look at the join hints w.r.t. the following order: // 1. broadcast hint: pick broadcast nested loop join. If both sides have the broadcast // hints, choose the smaller side (based on stats) to broadcast for inner and full joins, @@ -259,31 +269,6 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { if (canBuildLeft(joinType)) BuildLeft else BuildRight } - /** - * See. [SPARK-32290] - * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join, - * which is very time consuming because it's an O(M*N) calculation. - * But if it's a single column NotInSubquery, and buildSide data is small enough, - * O(M*N) calculation could be optimized into O(M) using hash lookup instead of loop lookup. - */ - def createBroadcastNullAwareHashJoin() = { - if (conf.notInSubqueryHashJoinEnabled && - joinType == LeftAnti && - canBroadcastBySize(right, conf) && - right.output.length == 1) { - val (matched, _, _) = - joins.NotInSubqueryConditionPattern.singleColumnPatternMatch(condition) - if (matched) { - Some(Seq(joins.BroadcastNullAwareHashJoinExec( - planLater(left), planLater(right), BuildRight, LeftAnti, condition))) - } else { - None - } - } else { - None - } - } - def createBroadcastNLJoin(buildLeft: Boolean, buildRight: Boolean) = { val maybeBuildSide = if (buildLeft && buildRight) { Some(desiredBuildSide) @@ -319,10 +304,9 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { } } - createBroadcastNullAwareHashJoin().orElse { - createBroadcastNLJoin(hintToBroadcastLeft(hint), hintToBroadcastRight(hint)) - .orElse { if (hintToShuffleReplicateNL(hint)) createCartesianProduct() else None } - }.getOrElse(createJoinWithoutHint()) + createBroadcastNLJoin(hintToBroadcastLeft(hint), hintToBroadcastRight(hint)) + .orElse { if (hintToShuffleReplicateNL(hint)) createCartesianProduct() else None } + .getOrElse(createJoinWithoutHint()) // --- Cases where this strategy does not apply --------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala index e1aa6b7418545..704d4b385dc22 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala @@ -20,12 +20,11 @@ package org.apache.spark.sql.execution.adaptive import org.apache.spark.sql.Strategy import org.apache.spark.sql.catalyst.expressions.PredicateHelper import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} -import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys +import org.apache.spark.sql.catalyst.planning.{ExtractEquiJoinKeys, ExtractSingleColumnNotInSubquery} import org.apache.spark.sql.catalyst.plans.LeftAnti import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan} import org.apache.spark.sql.execution.{joins, SparkPlan} import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec} -import org.apache.spark.sql.internal.SQLConf /** * Strategy for plans containing [[LogicalQueryStage]] nodes: @@ -50,36 +49,15 @@ object LogicalQueryStageStrategy extends Strategy with PredicateHelper { Seq(BroadcastHashJoinExec( leftKeys, rightKeys, joinType, buildSide, condition, planLater(left), planLater(right))) + case j @ ExtractSingleColumnNotInSubquery(leftKeys, rightKeys) => + Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, + planLater(j.left), planLater(j.right), BuildRight, LeftAnti, j.condition)) + case j @ Join(left, right, joinType, condition, _) if isBroadcastStage(left) || isBroadcastStage(right) => - def createBroadcastNLJoin() = { - val buildSide = if (isBroadcastStage(left)) BuildLeft else BuildRight - BroadcastNestedLoopJoinExec( - planLater(left), planLater(right), buildSide, joinType, condition) :: Nil - } - - /** - * See. [SPARK-32290] - * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join, - * which is very time consuming because it's an O(M*N) calculation. - * But if it's a single column NotInSubquery, and buildSide data is small enough, - * O(M*N) calculation could be optimized into O(M) using hash lookup instead of loop lookup. - */ - if (SQLConf.get.notInSubqueryHashJoinEnabled && - joinType == LeftAnti && - isBroadcastStage(right) && - right.output.length == 1) { - val (matched, _, _) = - joins.NotInSubqueryConditionPattern.singleColumnPatternMatch(condition) - if (matched) { - Seq(joins.BroadcastNullAwareHashJoinExec( - planLater(left), planLater(right), BuildRight, LeftAnti, condition)) - } else { - createBroadcastNLJoin() - } - } else { - createBroadcastNLJoin() - } + val buildSide = if (isBroadcastStage(left)) BuildLeft else BuildRight + BroadcastNestedLoopJoinExec( + planLater(left), planLater(right), buildSide, joinType, condition) :: Nil case q: LogicalQueryStage => q.physicalPlan :: Nil diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 71faad9829a42..695be355eca27 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical.{BroadcastDistribution, Distribution, HashPartitioning, Partitioning, PartitioningCollection, UnspecifiedDistribution} import org.apache.spark.sql.execution.{CodegenSupport, SparkPlan} import org.apache.spark.sql.execution.metric.SQLMetrics +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{BooleanType, LongType} /** @@ -49,6 +50,8 @@ case class BroadcastHashJoinExec( right: SparkPlan) extends HashJoin with CodegenSupport { + var nullAwareJoin: Boolean = false + override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) @@ -454,55 +457,101 @@ case class BroadcastHashJoinExec( val (matched, checkCondition, _) = getJoinCondition(ctx, input) val numOutput = metricTerm(ctx, "numOutputRows") - if (uniqueKeyCodePath) { - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin + if (nullAwareJoin) { + require(leftKeys.length == 1, "leftKeys length should be 1") + require(rightKeys.length == 1, "rightKeys length should be 1") + require(right.output.length == 1, "not in subquery hash join optimize only single column.") + require(joinType == LeftAnti, "joinType must be LeftAnti.") + require(buildSide == BuildRight, "buildSide must be BuildRight.") + require(SQLConf.get.notInSubqueryHashJoinEnabled, + "notInSubqueryHashJoinEnabled must turn on for BroadcastNullAwareHashJoinExec.") + require(checkCondition == "", "not in subquery hash join optimize empty condition.") + + if (broadcastRelation.value.inputIsEmpty) { + s""" + |// singleColumnNullAware buildSideIsEmpty(true) accept all + |$numOutput.add(1); + |${consume(ctx, input)} + """.stripMargin + } else if (broadcastRelation.value.anyNullKeyExists) { + s""" + |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(true) reject all + """.stripMargin + } else { + val found = ctx.freshName("found") + s""" + |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(false) + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | $found = true; + | } + |} else { + | $found = true; + |} + | + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin + } } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); - | if ($matches != null) { - | // Evaluate the condition. - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin + if (uniqueKeyCodePath) { + val found = ctx.freshName("found") + s""" + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | // Evaluate the condition. + | $checkCondition { + | $found = true; + | } + | } + |} + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin + } else { + val matches = ctx.freshName("matches") + val iteratorCls = classOf[Iterator[UnsafeRow]].getName + val found = ctx.freshName("found") + s""" + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); + | if ($matches != null) { + | // Evaluate the condition. + | while (!$found && $matches.hasNext()) { + | UnsafeRow $matched = (UnsafeRow) $matches.next(); + | $checkCondition { + | $found = true; + | } + | } + | } + |} + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin + } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala index a0025e42da568..5740dbb8e9789 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala @@ -21,12 +21,14 @@ import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, GenerateUnsafeProjection} import org.apache.spark.sql.catalyst.optimizer.{BuildRight, BuildSide} import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical._ -import org.apache.spark.sql.execution.{ExplainUtils, SparkPlan} +import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, SparkPlan} import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.LongType private case class NotInSubqueryHashJoinParams( buildSideRelation: HashedRelation, @@ -35,37 +37,26 @@ private case class NotInSubqueryHashJoinParams( streamedSideKey: Attribute, streamedSideKeyIndex: Int) -object NotInSubqueryConditionPattern { - def singleColumnPatternMatch(condition: Option[Expression]): (Boolean, Attribute, Attribute) = { - condition.get match { - case Or(EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), - IsNull(EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference))) - if leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => - (true, leftAttr, rightAttr) - case _ => (false, null, null) - } - } -} - case class BroadcastNullAwareHashJoinExec( + leftKeys: Seq[Expression], + rightKeys: Seq[Expression], left: SparkPlan, right: SparkPlan, buildSide: BuildSide, joinType: JoinType, - condition: Option[Expression]) extends BaseJoinExec { + condition: Option[Expression]) extends BaseJoinExec with CodegenSupport { // TODO support multi column not in subquery in future. // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6 // multi-column null aware anti join is much more complicated than single column ones. + require(leftKeys.length == 1, "leftKeys length should be 1") + require(rightKeys.length == 1, "rightKeys length should be 1") require(right.output.length == 1, "not in subquery hash join optimize only single column.") require(joinType == LeftAnti, "joinType must be LeftAnti.") require(buildSide == BuildRight, "buildSide must be BuildRight.") require(SQLConf.get.notInSubqueryHashJoinEnabled, "notInSubqueryHashJoinEnabled must turn on for BroadcastNullAwareHashJoinExec.") - override def leftKeys: Seq[Expression] = Nil - override def rightKeys: Seq[Expression] = Nil - override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) @@ -89,8 +80,8 @@ case class BroadcastNullAwareHashJoinExec( } private def buildParams(buildSideRows: Array[InternalRow]): NotInSubqueryHashJoinParams = { - val (_, leftAttr, rightAttr) = - NotInSubqueryConditionPattern.singleColumnPatternMatch(condition) + val leftAttr = leftKeys.head.asInstanceOf[AttributeReference] + val rightAttr = rightKeys.head.asInstanceOf[AttributeReference] val leftNotInKeyFound = left.output.exists(_.semanticEquals(leftAttr)) val (streamedKey, streamedKeyIndex) = if (leftNotInKeyFound) { @@ -129,25 +120,21 @@ case class BroadcastNullAwareHashJoinExec( val streamedRowIsNull = row.isNullAt(params.streamedSideKeyIndex) val lookupRow: UnsafeRow = keyGenerator(row) val notInKeyEqual = params.buildSideRelation.get(lookupRow) != null - if (!streamedRowIsNull && !notInKeyEqual) { - true - } else { - false - } + !streamedRowIsNull && !notInKeyEqual }) } } } protected override def doExecute(): RDD[InternalRow] = { - val broadcastedRelation = broadcast.executeBroadcast[Array[InternalRow]]() + val broadcastRelation = broadcast.executeBroadcast[Array[InternalRow]]() val resultRdd = (joinType, buildSide) match { case (LeftAnti, BuildRight) => - leftAntiNullAwareJoin(broadcastedRelation) + leftAntiNullAwareJoin(broadcastRelation) case _ => throw new IllegalArgumentException( - s"BroadcastNullAwareHashJoinExec support (LeftAnti + BuildRight) only for now.") + s"BroadcastNullAwareHashJoinExec only supports (LeftAnti + BuildRight) for now.") } val numOutputRows = longMetric("numOutputRows") @@ -160,4 +147,85 @@ case class BroadcastNullAwareHashJoinExec( } } } + + override def needCopyResult: Boolean = + streamed.asInstanceOf[CodegenSupport].needCopyResult + + override def inputRDDs(): Seq[RDD[InternalRow]] = { + streamed.asInstanceOf[CodegenSupport].inputRDDs() + } + + override def doProduce(ctx: CodegenContext): String = { + streamed.asInstanceOf[CodegenSupport].produce(ctx, this) + } + + private def genStreamSideJoinKey( + ctx: CodegenContext, + input: Seq[ExprCode]): (ExprCode, String) = { + ctx.currentVars = input + if (leftKeys.length == 1 && leftKeys.head.dataType == LongType) { + // generate the join key as Long + val ev = + BindReferences.bindReference[Expression](leftKeys.head, left.output).genCode(ctx) + (ev, ev.isNull) + } else { + // generate the join key as UnsafeRow + val ev = GenerateUnsafeProjection.createCode(ctx, + BindReferences.bindReferences[Expression](leftKeys, left.output)) + (ev, s"${ev.value}.anyNull()") + } + } + + override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { + val broadcastRelation = broadcast.executeBroadcast[Array[InternalRow]]() + val params = buildParams(broadcastRelation.value) + val broadcastRef = ctx.addReferenceObj("broadcast", params.buildSideRelation) + val clsName = params.buildSideRelation.getClass.getName + + // Inline mutable state since not many join operations in a task + val relationTerm = ctx.addMutableState(clsName, "relation", + v => s""" + | $v = (($clsName) $broadcastRef).asReadOnlyCopy(); + | incPeakExecutionMemory($v.estimatedSize()); + """.stripMargin, forceInline = true) + + val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) + val matched = ctx.freshName("matched") + val numOutput = metricTerm(ctx, "numOutputRows") + val found = ctx.freshName("found") + + if (params.buildSideIsEmpty) { + s""" + |// singleColumnNullAware buildSideIsEmpty(true) accept all + |$numOutput.add(1); + |${consume(ctx, input)} + """.stripMargin + } else if (params.buildSideIsNullExists) { + s""" + |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(true) reject all + """.stripMargin + } else { + s""" + |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(false) + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | $found = true; + | } + |} else { + | $found = true; + |} + | + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin + } + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 13180d6b20902..08e5fb14f71fa 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -71,6 +71,16 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { */ def keyIsUnique: Boolean + /** + * is input: Iterator[InternalRow] empty + */ + def inputIsEmpty: Boolean + + /** + * anyNull key exists in input + */ + def anyNullKeyExists: Boolean + /** * Returns an iterator for keys of InternalRow type. */ @@ -302,6 +312,10 @@ private[joins] class UnsafeHashedRelation( override def read(kryo: Kryo, in: Input): Unit = Utils.tryOrIOException { read(() => in.readInt(), () => in.readLong(), in.readBytes) } + + override def inputIsEmpty: Boolean = binaryMap.isInputIsEmpty + + override def anyNullKeyExists: Boolean = binaryMap.isAnyNullKeyExists } private[joins] object UnsafeHashedRelation { @@ -323,6 +337,7 @@ private[joins] object UnsafeHashedRelation { // Create a mapping of buildKeys -> rows val keyGenerator = UnsafeProjection.create(key) var numFields = 0 + binaryMap.setInputIsEmpty(input.isEmpty) while (input.hasNext) { val row = input.next().asInstanceOf[UnsafeRow] numFields = row.numFields() @@ -338,6 +353,8 @@ private[joins] object UnsafeHashedRelation { throw new SparkOutOfMemoryError("There is not enough memory to build hash map") // scalastyle:on throwerror } + } else { + binaryMap.setAnyNullKeyExists(true) } } @@ -382,6 +399,9 @@ private[joins] object UnsafeHashedRelation { private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, capacity: Int) extends MemoryConsumer(mm) with Externalizable with KryoSerializable { + var inputIsEmpty = false + var anyNullKeyExists = false + // Whether the keys are stored in dense mode or not. private var isDense = false @@ -879,6 +899,10 @@ class LongHashedRelation( * Returns an iterator for keys of InternalRow type. */ override def keys(): Iterator[InternalRow] = map.keys() + + override def inputIsEmpty: Boolean = map.inputIsEmpty + + override def anyNullKeyExists: Boolean = map.anyNullKeyExists } /** @@ -896,6 +920,7 @@ private[joins] object LongHashedRelation { // Create a mapping of key -> rows var numFields = 0 + map.inputIsEmpty = input.isEmpty while (input.hasNext) { val unsafeRow = input.next().asInstanceOf[UnsafeRow] numFields = unsafeRow.numFields() @@ -903,6 +928,8 @@ private[joins] object LongHashedRelation { if (!rowKey.isNullAt(0)) { val key = rowKey.getLong(0) map.append(key, unsafeRow) + } else { + map.anyNullKeyExists = true } } map.optimize() diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index c42d4c6f74a93..3fcd8fa042341 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -82,6 +82,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan case j: ShuffledHashJoinExec => j case j: CartesianProductExec => j case j: BroadcastNestedLoopJoinExec => j + case j: BroadcastNullAwareHashJoinExec => j case j: SortMergeJoinExec => j } @@ -1147,4 +1148,30 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan }) } } + + test("SPARK-32290: NotInSubquery SingleColumn Optimize positive") { + withSQLConf(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED.key -> "true", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString) { + assertJoin(( + "select * from testData where key not in (select a from testData2)", + classOf[BroadcastNullAwareHashJoinExec])) + // multi column not supported + assertJoin(( + "select * from testData where (key, key + 1) not in (select * from testData2)", + classOf[BroadcastNestedLoopJoinExec])) + } + } + + test("SPARK-32290: NotInSubquery SingleColumn Optimize negative") { + withSQLConf(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED.key -> "true", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "0") { + assertJoin(( + "select * from testData where key not in (select a from testData2)", + classOf[BroadcastNestedLoopJoinExec])) + // multi column not supported + assertJoin(( + "select * from testData where (key, key + 1) not in (select * from testData2)", + classOf[BroadcastNestedLoopJoinExec])) + } + } } From f338a4732db855a883726c4d86ab43a077ab6bd3 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 20 Jul 2020 10:29:03 +0800 Subject: [PATCH 06/24] [SPARK-32290][SQL] NotInSubquery SingleColumn Optimize typo. Change-Id: I6dc1a1039e2ccd5bfd0e8b7ccee02eabf9801bd0 --- .../sql/execution/joins/BroadcastNullAwareHashJoinExec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala index 5740dbb8e9789..ab15bfef06a4c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala @@ -118,8 +118,8 @@ case class BroadcastNullAwareHashJoinExec( ) streamedIter.filter(row => { val streamedRowIsNull = row.isNullAt(params.streamedSideKeyIndex) - val lookupRow: UnsafeRow = keyGenerator(row) - val notInKeyEqual = params.buildSideRelation.get(lookupRow) != null + val lookupKey: UnsafeRow = keyGenerator(row) + val notInKeyEqual = params.buildSideRelation.get(lookupKey) != null !streamedRowIsNull && !notInKeyEqual }) } From 519acd8969dff3ef75e84a6a294ed0d62bad545a Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 20 Jul 2020 20:07:04 +0800 Subject: [PATCH 07/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. find bugs in BroadcastNullAwareHashJoinExec with codegen off. 2. change singleColumnNotInSubquery desc into SingleColumn Null Aware Anti Join Optimize. 3. code refine. 4. rename suiteName into NullAwareAntiJoinSQLQueryTestSuite. 5. BroadcastHashJoinExec implemented with non-codegen version. NullAwareAntiJoinSQLQueryTestSuite passed. org.apache.spark.sql.JoinSuite passed. Manually Test with the following config combination with not-in-unit-tests-single-column.sql all passed. ``` Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "false", "spark.sql.codegen.wholeStage" -> "false", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "false" ), Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "false", "spark.sql.codegen.wholeStage" -> "true", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "false" ), Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "true", "spark.sql.codegen.wholeStage" -> "false", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "false" ), Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "true", "spark.sql.codegen.wholeStage" -> "true", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "false" ), Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "false", "spark.sql.codegen.wholeStage" -> "false", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "true" ), Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "false", "spark.sql.codegen.wholeStage" -> "true", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "true" ), Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "true", "spark.sql.codegen.wholeStage" -> "false", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "true" ), Map( "spark.sql.nullAware.antiJoin.optimize.enabled" -> "true", "spark.sql.adaptive.enabled" -> "true", "spark.sql.codegen.wholeStage" -> "true", "spark.sql.nullAware.antiJoin.optimize.use.bhj" -> "true" ) ``` Change-Id: I11f937af7743f2d51c13015a51bae8deaa376f20 --- .../spark/unsafe/map/BytesToBytesMap.java | 10 +-- .../sql/catalyst/planning/patterns.scala | 11 +-- .../apache/spark/sql/internal/SQLConf.scala | 23 ++++-- .../spark/sql/execution/SparkStrategies.scala | 18 ++--- .../adaptive/LogicalQueryStageStrategy.scala | 16 +++- .../PlanDynamicPruningFilters.scala | 4 +- .../joins/BroadcastHashJoinExec.scala | 48 +++++++---- .../BroadcastNullAwareHashJoinExec.scala | 80 +++++++------------ .../sql/execution/joins/HashedRelation.scala | 33 +++++--- .../sql/DynamicPartitionPruningSuite.scala | 2 +- .../org/apache/spark/sql/JoinSuite.scala | 16 +++- ... NullAwareAntiJoinSQLQueryTestSuite.scala} | 12 +-- .../execution/WholeStageCodegenSuite.scala | 4 +- 13 files changed, 156 insertions(+), 121 deletions(-) rename sql/core/src/test/scala/org/apache/spark/sql/{NotInSubqueryHashJoinTestSuite.scala => NullAwareAntiJoinSQLQueryTestSuite.scala} (85%) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index 4189e006c88f0..d359c7bcc4578 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -171,17 +171,17 @@ public final class BytesToBytesMap extends MemoryConsumer { private volatile MapIterator destructiveIterator = null; private LinkedList spillWriters = new LinkedList<>(); - private boolean inputIsEmpty = false; + private boolean inputEmpty = false; private boolean anyNullKeyExists = false; - public boolean isInputIsEmpty() + public boolean isInputEmpty() { - return inputIsEmpty; + return inputEmpty; } - public void setInputIsEmpty(boolean inputIsEmpty) + public void setInputEmpty(boolean inputEmpty) { - this.inputIsEmpty = inputIsEmpty; + this.inputEmpty = inputEmpty; } public boolean isAnyNullKeyExists() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index fd5cfc555e4f6..6aa7da1c8ad80 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -391,9 +391,9 @@ object PhysicalWindow { } } -object ExtractSingleColumnNotInSubquery extends JoinSelectionHelper { +object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper { - // SingleColumnNotInSubquery + // SingleColumn NullAwareAntiJoin // streamedSideKeys, buildSideKeys // currently these two return Seq[Expression] should have only one element private type ReturnType = @@ -401,16 +401,17 @@ object ExtractSingleColumnNotInSubquery extends JoinSelectionHelper { /** * See. [SPARK-32290] - * Not in Subquery will almost certainly be planned as a Broadcast Nested Loop join, + * LeftAnti(condition: Or(EqualTo(a=b), IsNull(EqualTo(a=b))) + * will almost certainly be planned as a Broadcast Nested Loop join, * which is very time consuming because it's an O(M*N) calculation. - * But if it's a single column NotInSubquery, and buildSide data is small enough, + * But if it's a single column case, and buildSide data is small enough, * O(M*N) calculation could be optimized into O(M) using hash lookup instead of loop lookup. */ def unapply(join: Join): Option[ReturnType] = join match { case Join(left, right, LeftAnti, Some(Or(EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), IsNull(EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference)))), _) - if SQLConf.get.notInSubqueryHashJoinEnabled && + if SQLConf.get.nullAwareAntiJoinOptimizeEnabled && leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) && canBroadcastBySize(right, SQLConf.get) && right.output.length == 1 => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 3312487b1ddb2..9231634536093 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2672,14 +2672,22 @@ object SQLConf { .checkValue(_ >= 0, "The value must be non-negative.") .createWithDefault(8) - val NOT_IN_SUBQUERY_HASH_JOIN_ENABLED = - buildConf("spark.sql.notInSubquery.hashJoin.enabled") + val NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED = + buildConf("spark.sql.nullAware.antiJoin.optimize.enabled") .internal() - .doc("When true, not in subquery execution will be planed into " + + .doc("When true, NULL-aware anti join execution will be planed into " + "BroadcastNullAwareHashJoinExec, " + "optimized from O(M*N) calculation into O(M) calculation " + "using Hash lookup instead of Looping lookup." + - "Only support for singleColumn not in subquery for now.") + "Only support for singleColumn NAAJ for now.") + .booleanConf + .createWithDefault(false) + + val NULL_AWARE_ANTI_JOIN_OPTIMIZE_USE_BHJ = + buildConf("spark.sql.nullAware.antiJoin.optimize.use.bhj") + .internal() + .doc("When true, NULL-aware anti join execution will be planed into " + + "BroadcastHashJoinExec with special execution.") .booleanConf .createWithDefault(false) @@ -3288,8 +3296,11 @@ class SQLConf extends Serializable with Logging { def coalesceBucketsInJoinMaxBucketRatio: Int = getConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO) - def notInSubqueryHashJoinEnabled: Boolean = - getConf(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED) + def nullAwareAntiJoinOptimizeEnabled: Boolean = + getConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED) + + def nullAwareAntiJoinOptimizeUseBHJ: Boolean = + getConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_USE_BHJ) /** ********************** SQLConf functionality methods ************ */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala index 2d9b7ec31ce99..3dcc87bc3391a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala @@ -232,15 +232,15 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { .orElse { if (hintToShuffleReplicateNL(hint)) createCartesianProduct() else None } .getOrElse(createJoinWithoutHint()) - case j @ ExtractSingleColumnNotInSubquery(leftKeys, rightKeys) => - Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, - planLater(j.left), planLater(j.right), BuildRight, LeftAnti, j.condition)) - - // for BHJ Prototype -// val bhj = joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, -// None, planLater(j.left), planLater(j.right)) -// bhj.nullAwareJoin = true -// Seq(bhj) + case j @ ExtractSingleColumnNullAwareAntiJoin(leftKeys, rightKeys) => + if (SQLConf.get.nullAwareAntiJoinOptimizeUseBHJ) { + // for BHJ Prototype + Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, + None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) + } else { + Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, + planLater(j.left), planLater(j.right), BuildRight, LeftAnti, None)) + } // If it is not an equi-join, we first look at the join hints w.r.t. the following order: // 1. broadcast hint: pick broadcast nested loop join. If both sides have the broadcast diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala index 704d4b385dc22..36f4cdd18c399 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala @@ -20,11 +20,12 @@ package org.apache.spark.sql.execution.adaptive import org.apache.spark.sql.Strategy import org.apache.spark.sql.catalyst.expressions.PredicateHelper import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} -import org.apache.spark.sql.catalyst.planning.{ExtractEquiJoinKeys, ExtractSingleColumnNotInSubquery} +import org.apache.spark.sql.catalyst.planning.{ExtractEquiJoinKeys, ExtractSingleColumnNullAwareAntiJoin} import org.apache.spark.sql.catalyst.plans.LeftAnti import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan} import org.apache.spark.sql.execution.{joins, SparkPlan} import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec} +import org.apache.spark.sql.internal.SQLConf /** * Strategy for plans containing [[LogicalQueryStage]] nodes: @@ -49,9 +50,16 @@ object LogicalQueryStageStrategy extends Strategy with PredicateHelper { Seq(BroadcastHashJoinExec( leftKeys, rightKeys, joinType, buildSide, condition, planLater(left), planLater(right))) - case j @ ExtractSingleColumnNotInSubquery(leftKeys, rightKeys) => - Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, - planLater(j.left), planLater(j.right), BuildRight, LeftAnti, j.condition)) + case j @ ExtractSingleColumnNullAwareAntiJoin(leftKeys, rightKeys) + if isBroadcastStage(j.right) => + if (SQLConf.get.nullAwareAntiJoinOptimizeUseBHJ) { + // for BHJ Prototype + Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, + None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) + } else { + Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, + planLater(j.left), planLater(j.right), BuildRight, LeftAnti, None)) + } case j @ Join(left, right, joinType, condition, _) if isBroadcastStage(left) || isBroadcastStage(right) => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala index cfc653a23840d..098576d72f540 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala @@ -59,9 +59,9 @@ case class PlanDynamicPruningFilters(sparkSession: SparkSession) // the first to be applied (apart from `InsertAdaptiveSparkPlan`). val canReuseExchange = SQLConf.get.exchangeReuseEnabled && buildKeys.nonEmpty && plan.find { - case BroadcastHashJoinExec(_, _, _, BuildLeft, _, left, _) => + case BroadcastHashJoinExec(_, _, _, BuildLeft, _, left, _, _) => left.sameResult(sparkPlan) - case BroadcastHashJoinExec(_, _, _, BuildRight, _, _, right) => + case BroadcastHashJoinExec(_, _, _, BuildRight, _, _, right, _) => right.sameResult(sparkPlan) case _ => false }.isDefined diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 695be355eca27..d85b4eace1495 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -47,11 +47,9 @@ case class BroadcastHashJoinExec( buildSide: BuildSide, condition: Option[Expression], left: SparkPlan, - right: SparkPlan) + right: SparkPlan, isNullAwareAntiJoin: Boolean = false) extends HashJoin with CodegenSupport { - var nullAwareJoin: Boolean = false - override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) @@ -136,10 +134,32 @@ case class BroadcastHashJoinExec( val numOutputRows = longMetric("numOutputRows") val broadcastRelation = buildPlan.executeBroadcast[HashedRelation]() - streamedPlan.execute().mapPartitions { streamedIter => - val hashed = broadcastRelation.value.asReadOnlyCopy() - TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize) - join(streamedIter, hashed, numOutputRows) + if (isNullAwareAntiJoin) { + streamedPlan.execute().mapPartitionsInternal { streamedIter => + if (broadcastRelation.value.inputEmpty) { + streamedIter + } else if (broadcastRelation.value.anyNullKeyExists) { + Iterator.empty + } else { + val keyGenerator = UnsafeProjection.create( + BindReferences.bindReferences[Expression]( + leftKeys, + AttributeSeq(left.output)) + ) + streamedIter.filter(row => { + val lookupKey: UnsafeRow = keyGenerator(row) + val streamedRowIsNull = lookupKey.isNullAt(0) + val notInKeyEqual = broadcastRelation.value.get(lookupKey) != null + !streamedRowIsNull && !notInKeyEqual + }) + } + } + } else { + streamedPlan.execute().mapPartitions { streamedIter => + val hashed = broadcastRelation.value.asReadOnlyCopy() + TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize) + join(streamedIter, hashed, numOutputRows) + } } } @@ -457,30 +477,30 @@ case class BroadcastHashJoinExec( val (matched, checkCondition, _) = getJoinCondition(ctx, input) val numOutput = metricTerm(ctx, "numOutputRows") - if (nullAwareJoin) { + if (isNullAwareAntiJoin) { require(leftKeys.length == 1, "leftKeys length should be 1") require(rightKeys.length == 1, "rightKeys length should be 1") require(right.output.length == 1, "not in subquery hash join optimize only single column.") require(joinType == LeftAnti, "joinType must be LeftAnti.") require(buildSide == BuildRight, "buildSide must be BuildRight.") - require(SQLConf.get.notInSubqueryHashJoinEnabled, - "notInSubqueryHashJoinEnabled must turn on for BroadcastNullAwareHashJoinExec.") + require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, + "nullAwareAntiJoinOptimizeEnabled must turn on for BroadcastNullAwareHashJoinExec.") require(checkCondition == "", "not in subquery hash join optimize empty condition.") - if (broadcastRelation.value.inputIsEmpty) { + if (broadcastRelation.value.inputEmpty) { s""" - |// singleColumnNullAware buildSideIsEmpty(true) accept all + |// singleColumn NAAJ inputEmpty(true) accept all |$numOutput.add(1); |${consume(ctx, input)} """.stripMargin } else if (broadcastRelation.value.anyNullKeyExists) { s""" - |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(true) reject all + |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(true) reject all """.stripMargin } else { val found = ctx.freshName("found") s""" - |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(false) + |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(false) |boolean $found = false; |// generate join key for stream side |${keyEv.code} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala index ab15bfef06a4c..8cb392a8e4dae 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.execution.joins -import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ @@ -30,13 +29,6 @@ import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.LongType -private case class NotInSubqueryHashJoinParams( - buildSideRelation: HashedRelation, - buildSideIsNullExists: Boolean, - buildSideIsEmpty: Boolean, - streamedSideKey: Attribute, - streamedSideKeyIndex: Int) - case class BroadcastNullAwareHashJoinExec( leftKeys: Seq[Expression], rightKeys: Seq[Expression], @@ -46,7 +38,7 @@ case class BroadcastNullAwareHashJoinExec( joinType: JoinType, condition: Option[Expression]) extends BaseJoinExec with CodegenSupport { - // TODO support multi column not in subquery in future. + // TODO support multi column NULL-aware anti join in future. // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6 // multi-column null aware anti join is much more complicated than single column ones. require(leftKeys.length == 1, "leftKeys length should be 1") @@ -54,8 +46,8 @@ case class BroadcastNullAwareHashJoinExec( require(right.output.length == 1, "not in subquery hash join optimize only single column.") require(joinType == LeftAnti, "joinType must be LeftAnti.") require(buildSide == BuildRight, "buildSide must be BuildRight.") - require(SQLConf.get.notInSubqueryHashJoinEnabled, - "notInSubqueryHashJoinEnabled must turn on for BroadcastNullAwareHashJoinExec.") + require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, + "nullAwareAntiJoinOptimizeEnabled must turn on for BroadcastNullAwareHashJoinExec.") override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) @@ -79,47 +71,31 @@ case class BroadcastNullAwareHashJoinExec( left.output } - private def buildParams(buildSideRows: Array[InternalRow]): NotInSubqueryHashJoinParams = { - val leftAttr = leftKeys.head.asInstanceOf[AttributeReference] - val rightAttr = rightKeys.head.asInstanceOf[AttributeReference] - - val leftNotInKeyFound = left.output.exists(_.semanticEquals(leftAttr)) - val (streamedKey, streamedKeyIndex) = if (leftNotInKeyFound) { - (leftAttr, AttributeSeq(left.output).indexOf(leftAttr.exprId)) - } else { - (rightAttr, AttributeSeq(left.output).indexOf(rightAttr.exprId)) - } - - NotInSubqueryHashJoinParams( - HashedRelation(buildSideRows.iterator, - BindReferences.bindReferences[Expression]( - Seq(right.output.head), AttributeSeq(right.output)), - buildSideRows.length), - buildSideRows.exists(row => row.isNullAt(0)), - buildSideRows.isEmpty, - streamedKey, - streamedKeyIndex - ) + private def prepareBroadcastHashedRelation = { + val buildSideRows = broadcast.executeBroadcast[Array[InternalRow]]().value + HashedRelation(buildSideRows.iterator, + BindReferences.bindReferences[Expression]( + Seq(right.output.head), AttributeSeq(right.output)), + buildSideRows.length) } - private def leftAntiNullAwareJoin( - relation: Broadcast[Array[InternalRow]]): RDD[InternalRow] = { - val params = buildParams(relation.value) + private def nullAwareLeftAntiJoin( + relation: HashedRelation): RDD[InternalRow] = { streamed.execute().mapPartitionsInternal { streamedIter => - if (params.buildSideIsEmpty) { + if (relation.inputEmpty) { streamedIter - } else if (params.buildSideIsNullExists) { + } else if (relation.anyNullKeyExists) { Iterator.empty } else { val keyGenerator = UnsafeProjection.create( BindReferences.bindReferences[Expression]( - Seq(params.streamedSideKey), + leftKeys, AttributeSeq(left.output)) ) streamedIter.filter(row => { - val streamedRowIsNull = row.isNullAt(params.streamedSideKeyIndex) val lookupKey: UnsafeRow = keyGenerator(row) - val notInKeyEqual = params.buildSideRelation.get(lookupKey) != null + val streamedRowIsNull = lookupKey.isNullAt(0) + val notInKeyEqual = relation.get(lookupKey) != null !streamedRowIsNull && !notInKeyEqual }) } @@ -127,11 +103,10 @@ case class BroadcastNullAwareHashJoinExec( } protected override def doExecute(): RDD[InternalRow] = { - val broadcastRelation = broadcast.executeBroadcast[Array[InternalRow]]() - + val relation: HashedRelation = prepareBroadcastHashedRelation val resultRdd = (joinType, buildSide) match { case (LeftAnti, BuildRight) => - leftAntiNullAwareJoin(broadcastRelation) + nullAwareLeftAntiJoin(relation) case _ => throw new IllegalArgumentException( s"BroadcastNullAwareHashJoinExec only supports (LeftAnti + BuildRight) for now.") @@ -177,36 +152,35 @@ case class BroadcastNullAwareHashJoinExec( } override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - val broadcastRelation = broadcast.executeBroadcast[Array[InternalRow]]() - val params = buildParams(broadcastRelation.value) - val broadcastRef = ctx.addReferenceObj("broadcast", params.buildSideRelation) - val clsName = params.buildSideRelation.getClass.getName + val relation: HashedRelation = prepareBroadcastHashedRelation + val broadcastRef = ctx.addReferenceObj("broadcast", relation) + val clsName = relation.getClass.getName // Inline mutable state since not many join operations in a task val relationTerm = ctx.addMutableState(clsName, "relation", v => s""" | $v = (($clsName) $broadcastRef).asReadOnlyCopy(); | incPeakExecutionMemory($v.estimatedSize()); - """.stripMargin, forceInline = true) + """.stripMargin, forceInline = true) val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) val matched = ctx.freshName("matched") val numOutput = metricTerm(ctx, "numOutputRows") val found = ctx.freshName("found") - if (params.buildSideIsEmpty) { + if (relation.inputEmpty) { s""" - |// singleColumnNullAware buildSideIsEmpty(true) accept all + |// singleColumn NAAJ inputEmpty(true) accept all |$numOutput.add(1); |${consume(ctx, input)} """.stripMargin - } else if (params.buildSideIsNullExists) { + } else if (relation.anyNullKeyExists) { s""" - |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(true) reject all + |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(true) reject all """.stripMargin } else { s""" - |// singleColumnNullAware buildSideIsEmpty(false) buildSideIsNullExists(false) + |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(false) |boolean $found = false; |// generate join key for stream side |${keyEv.code} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 08e5fb14f71fa..b3f498d4cf904 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -74,7 +74,7 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { /** * is input: Iterator[InternalRow] empty */ - def inputIsEmpty: Boolean + def inputEmpty: Boolean /** * anyNull key exists in input @@ -213,17 +213,20 @@ private[joins] class UnsafeHashedRelation( } override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException { - write(out.writeInt, out.writeLong, out.write) + write(out.writeBoolean, out.writeInt, out.writeLong, out.write) } override def write(kryo: Kryo, out: Output): Unit = Utils.tryOrIOException { - write(out.writeInt, out.writeLong, out.write) + write(out.writeBoolean, out.writeInt, out.writeLong, out.write) } private def write( + writeBoolean: (Boolean) => Unit, writeInt: (Int) => Unit, writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit) : Unit = { + writeBoolean(inputEmpty) + writeBoolean(anyNullKeyExists) writeInt(numFields) // TODO: move these into BytesToBytesMap writeLong(binaryMap.numKeys()) @@ -250,13 +253,16 @@ private[joins] class UnsafeHashedRelation( } override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException { - read(() => in.readInt(), () => in.readLong(), in.readFully) + read(() => in.readBoolean(), () => in.readInt(), () => in.readLong(), in.readFully) } private def read( + readBoolean: () => Boolean, readInt: () => Int, readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { + val inputEmpty = readBoolean() + val anyNullKeyExists = readBoolean() numFields = readInt() resultRow = new UnsafeRow(numFields) val nKeys = readLong() @@ -283,6 +289,9 @@ private[joins] class UnsafeHashedRelation( (nKeys * 1.5 + 1).toInt, // reduce hash collision pageSizeBytes) + binaryMap.setInputEmpty(inputEmpty) + binaryMap.setAnyNullKeyExists(anyNullKeyExists) + var i = 0 var keyBuffer = new Array[Byte](1024) var valuesBuffer = new Array[Byte](1024) @@ -310,10 +319,10 @@ private[joins] class UnsafeHashedRelation( } override def read(kryo: Kryo, in: Input): Unit = Utils.tryOrIOException { - read(() => in.readInt(), () => in.readLong(), in.readBytes) + read(() => in.readBoolean(), () => in.readInt(), () => in.readLong(), in.readBytes) } - override def inputIsEmpty: Boolean = binaryMap.isInputIsEmpty + override def inputEmpty: Boolean = binaryMap.isInputEmpty override def anyNullKeyExists: Boolean = binaryMap.isAnyNullKeyExists } @@ -337,7 +346,7 @@ private[joins] object UnsafeHashedRelation { // Create a mapping of buildKeys -> rows val keyGenerator = UnsafeProjection.create(key) var numFields = 0 - binaryMap.setInputIsEmpty(input.isEmpty) + binaryMap.setInputEmpty(input.isEmpty) while (input.hasNext) { val row = input.next().asInstanceOf[UnsafeRow] numFields = row.numFields() @@ -399,7 +408,7 @@ private[joins] object UnsafeHashedRelation { private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, capacity: Int) extends MemoryConsumer(mm) with Externalizable with KryoSerializable { - var inputIsEmpty = false + var inputEmpty = false var anyNullKeyExists = false // Whether the keys are stored in dense mode or not. @@ -780,6 +789,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap writeBoolean: (Boolean) => Unit, writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit): Unit = { + writeBoolean(inputEmpty) + writeBoolean(anyNullKeyExists) writeBoolean(isDense) writeLong(minKey) writeLong(maxKey) @@ -821,6 +832,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap readBoolean: () => Boolean, readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { + inputEmpty = readBoolean() + anyNullKeyExists = readBoolean() isDense = readBoolean() minKey = readLong() maxKey = readLong() @@ -900,7 +913,7 @@ class LongHashedRelation( */ override def keys(): Iterator[InternalRow] = map.keys() - override def inputIsEmpty: Boolean = map.inputIsEmpty + override def inputEmpty: Boolean = map.inputEmpty override def anyNullKeyExists: Boolean = map.anyNullKeyExists } @@ -920,7 +933,7 @@ private[joins] object LongHashedRelation { // Create a mapping of key -> rows var numFields = 0 - map.inputIsEmpty = input.isEmpty + map.inputEmpty = input.isEmpty while (input.hasNext) { val unsafeRow = input.next().asInstanceOf[UnsafeRow] numFields = unsafeRow.numFields() diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala index cdf9ea4b31ee7..0b754e9e3ec0b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala @@ -396,7 +396,7 @@ abstract class DynamicPartitionPruningSuiteBase """.stripMargin) val found = df.queryExecution.executedPlan.find { - case BroadcastHashJoinExec(_, _, p: ExistenceJoin, _, _, _, _) => true + case BroadcastHashJoinExec(_, _, p: ExistenceJoin, _, _, _, _, _) => true case _ => false } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index 3fcd8fa042341..93b3f0c6e8616 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -1149,12 +1149,16 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan } } - test("SPARK-32290: NotInSubquery SingleColumn Optimize positive") { - withSQLConf(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED.key -> "true", + test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize positive") { + withSQLConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> "true", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString) { assertJoin(( "select * from testData where key not in (select a from testData2)", classOf[BroadcastNullAwareHashJoinExec])) + // testData3.b nullable is true + assertJoin(( + "select * from testData left anti join testData3 ON key = b or isnull(key = b)", + classOf[BroadcastNullAwareHashJoinExec])) // multi column not supported assertJoin(( "select * from testData where (key, key + 1) not in (select * from testData2)", @@ -1162,12 +1166,16 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan } } - test("SPARK-32290: NotInSubquery SingleColumn Optimize negative") { - withSQLConf(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED.key -> "true", + test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize negative") { + withSQLConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> "true", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "0") { assertJoin(( "select * from testData where key not in (select a from testData2)", classOf[BroadcastNestedLoopJoinExec])) + // testData3.b nullable is true + assertJoin(( + "select * from testData left anti join testData3 ON key = b or isnull(key = b)", + classOf[BroadcastNestedLoopJoinExec])) // multi column not supported assertJoin(( "select * from testData where (key, key + 1) not in (select * from testData2)", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/NotInSubqueryHashJoinTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/NullAwareAntiJoinSQLQueryTestSuite.scala similarity index 85% rename from sql/core/src/test/scala/org/apache/spark/sql/NotInSubqueryHashJoinTestSuite.scala rename to sql/core/src/test/scala/org/apache/spark/sql/NullAwareAntiJoinSQLQueryTestSuite.scala index 94bee6ca653fa..916f9988aed88 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/NotInSubqueryHashJoinTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/NullAwareAntiJoinSQLQueryTestSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.internal.SQLConf /** * End-to-end test cases for subquery SQL queries coverage with - * NOT_IN_SUBQUERY_HASH_JOIN_ENABLED = true. + * NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED = true. * * Each case is loaded from a file in * "spark/sql/core/src/test/resources/sql-tests/inputs/subquery". @@ -33,24 +33,24 @@ import org.apache.spark.sql.internal.SQLConf * * To run the entire test suite: * {{{ - * build/sbt "sql/test-only *NotInSubqueryHashJoinTestSuite" + * build/sbt "sql/test-only *NullAwareAntiJoinSQLQueryTestSuite" * }}} * */ -class NotInSubqueryHashJoinTestSuite extends SQLQueryTestSuite { +class NullAwareAntiJoinSQLQueryTestSuite extends SQLQueryTestSuite { protected override def sparkConf: SparkConf = super.sparkConf // Fewer shuffle partitions to speed up testing. - // enable NOT_IN_SUBQUERY_HASH_JOIN_ENABLED for subquery case coverage. + // enable NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED for subquery case coverage. .set(SQLConf.SHUFFLE_PARTITIONS, 4) - .set(SQLConf.NOT_IN_SUBQUERY_HASH_JOIN_ENABLED, true) + .set(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED, true) override lazy val listTestCases: Seq[TestCase] = { listFilesRecursively(new File(inputFilePath)).flatMap { file => val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".out" val absPath = file.getAbsolutePath val testCaseName = - s"notInSubqueryHashJoin@" + + s"NAAJ@" + s"${absPath.stripPrefix(inputFilePath).stripPrefix(File.separator)}" if (file.getAbsolutePath.startsWith( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala index f7396ee2a89c8..03596d8654c66 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala @@ -342,7 +342,7 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession .join(baseTable, "idx") assert(distinctWithId.queryExecution.executedPlan.collectFirst { case WholeStageCodegenExec( - ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _))) => true + ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _, _))) => true }.isDefined) checkAnswer(distinctWithId, Seq(Row(1, 0), Row(1, 0))) @@ -353,7 +353,7 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession .join(baseTable, "idx") assert(groupByWithId.queryExecution.executedPlan.collectFirst { case WholeStageCodegenExec( - ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _))) => true + ProjectExec(_, BroadcastHashJoinExec(_, _, _, _, _, _: HashAggregateExec, _, _))) => true }.isDefined) checkAnswer(groupByWithId, Seq(Row(1, 2, 0), Row(1, 2, 0))) } From eebe303ba7a3b97425c41a0c98fed07e7052a193 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Tue, 21 Jul 2020 11:01:34 +0800 Subject: [PATCH 08/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. code refine 2. delete inputEmpty prop in BytesToBytesMap and LongToUnsafeRowMap, because it could be inlined as ((numKeys == 0) && !anyNullKeyExists) 3. update BroadcastHashJoinExec codegenAnti to remove seprately handle code. 4. rebase community master code because of conf conflict. Change-Id: I9c30f81ea9101dde1cf37a62ed4f5d9becaa9835 --- .../spark/unsafe/map/BytesToBytesMap.java | 10 +- .../sql/catalyst/planning/patterns.scala | 3 +- .../joins/BroadcastHashJoinExec.scala | 136 ++++++++---------- .../BroadcastNullAwareHashJoinExec.scala | 4 +- .../sql/execution/joins/HashedRelation.scala | 11 +- 5 files changed, 64 insertions(+), 100 deletions(-) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index d359c7bcc4578..1c392362b1b7f 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -171,17 +171,11 @@ public final class BytesToBytesMap extends MemoryConsumer { private volatile MapIterator destructiveIterator = null; private LinkedList spillWriters = new LinkedList<>(); - private boolean inputEmpty = false; private boolean anyNullKeyExists = false; - public boolean isInputEmpty() + public boolean inputEmpty() { - return inputEmpty; - } - - public void setInputEmpty(boolean inputEmpty) - { - this.inputEmpty = inputEmpty; + return ((numKeys == 0) && !anyNullKeyExists); } public boolean isAnyNullKeyExists() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index 6aa7da1c8ad80..4b955f3c002dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -396,8 +396,7 @@ object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper { // SingleColumn NullAwareAntiJoin // streamedSideKeys, buildSideKeys // currently these two return Seq[Expression] should have only one element - private type ReturnType = - (Seq[Expression], Seq[Expression]) + private type ReturnType = (Seq[Expression], Seq[Expression]) /** * See. [SPARK-32290] diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index d85b4eace1495..846effbfe3f50 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -47,7 +47,8 @@ case class BroadcastHashJoinExec( buildSide: BuildSide, condition: Option[Expression], left: SparkPlan, - right: SparkPlan, isNullAwareAntiJoin: Boolean = false) + right: SparkPlan, + isNullAwareAntiJoin: Boolean = false) extends HashJoin with CodegenSupport { override lazy val metrics = Map( @@ -480,98 +481,75 @@ case class BroadcastHashJoinExec( if (isNullAwareAntiJoin) { require(leftKeys.length == 1, "leftKeys length should be 1") require(rightKeys.length == 1, "rightKeys length should be 1") - require(right.output.length == 1, "not in subquery hash join optimize only single column.") + require(right.output.length == 1, "null aware anti join only supports single column.") require(joinType == LeftAnti, "joinType must be LeftAnti.") require(buildSide == BuildRight, "buildSide must be BuildRight.") require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, - "nullAwareAntiJoinOptimizeEnabled must turn on for BroadcastNullAwareHashJoinExec.") - require(checkCondition == "", "not in subquery hash join optimize empty condition.") + "nullAwareAntiJoinOptimizeEnabled must be on for null aware anti join optimize.") + require(checkCondition == "", "null aware anti join optimize condition should be empty.") if (broadcastRelation.value.inputEmpty) { - s""" + return s""" |// singleColumn NAAJ inputEmpty(true) accept all |$numOutput.add(1); |${consume(ctx, input)} """.stripMargin } else if (broadcastRelation.value.anyNullKeyExists) { - s""" + return s""" |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(true) reject all """.stripMargin - } else { - val found = ctx.freshName("found") - s""" - |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(false) - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | $found = true; - | } - |} else { - | $found = true; - |} - | - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin } + } + + if (uniqueKeyCodePath) { + val found = ctx.freshName("found") + s""" + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | // Evaluate the condition. + | $checkCondition { + | $found = true; + | } + | } + |} ${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin } else { - if (uniqueKeyCodePath) { - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); - | if ($matches != null) { - | // Evaluate the condition. - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } + val matches = ctx.freshName("matches") + val iteratorCls = classOf[Iterator[UnsafeRow]].getName + val found = ctx.freshName("found") + s""" + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |// Check if the key has nulls. + |if (!($anyNull)) { + | // Check if the HashedRelation exists. + | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); + | if ($matches != null) { + | // Evaluate the condition. + | while (!$found && $matches.hasNext()) { + | UnsafeRow $matched = (UnsafeRow) $matches.next(); + | $checkCondition { + | $found = true; + | } + | } + | } + |} ${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala index 8cb392a8e4dae..47f63e5e19d7f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala @@ -43,11 +43,11 @@ case class BroadcastNullAwareHashJoinExec( // multi-column null aware anti join is much more complicated than single column ones. require(leftKeys.length == 1, "leftKeys length should be 1") require(rightKeys.length == 1, "rightKeys length should be 1") - require(right.output.length == 1, "not in subquery hash join optimize only single column.") + require(right.output.length == 1, "null aware anti join only supports single column.") require(joinType == LeftAnti, "joinType must be LeftAnti.") require(buildSide == BuildRight, "buildSide must be BuildRight.") require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, - "nullAwareAntiJoinOptimizeEnabled must turn on for BroadcastNullAwareHashJoinExec.") + "nullAwareAntiJoinOptimizeEnabled must be on for null aware anti join optimize.") override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index b3f498d4cf904..5dc0cc088da9c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -225,7 +225,6 @@ private[joins] class UnsafeHashedRelation( writeInt: (Int) => Unit, writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit) : Unit = { - writeBoolean(inputEmpty) writeBoolean(anyNullKeyExists) writeInt(numFields) // TODO: move these into BytesToBytesMap @@ -261,7 +260,6 @@ private[joins] class UnsafeHashedRelation( readInt: () => Int, readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { - val inputEmpty = readBoolean() val anyNullKeyExists = readBoolean() numFields = readInt() resultRow = new UnsafeRow(numFields) @@ -289,7 +287,6 @@ private[joins] class UnsafeHashedRelation( (nKeys * 1.5 + 1).toInt, // reduce hash collision pageSizeBytes) - binaryMap.setInputEmpty(inputEmpty) binaryMap.setAnyNullKeyExists(anyNullKeyExists) var i = 0 @@ -322,7 +319,7 @@ private[joins] class UnsafeHashedRelation( read(() => in.readBoolean(), () => in.readInt(), () => in.readLong(), in.readBytes) } - override def inputEmpty: Boolean = binaryMap.isInputEmpty + override def inputEmpty: Boolean = binaryMap.inputEmpty override def anyNullKeyExists: Boolean = binaryMap.isAnyNullKeyExists } @@ -346,7 +343,6 @@ private[joins] object UnsafeHashedRelation { // Create a mapping of buildKeys -> rows val keyGenerator = UnsafeProjection.create(key) var numFields = 0 - binaryMap.setInputEmpty(input.isEmpty) while (input.hasNext) { val row = input.next().asInstanceOf[UnsafeRow] numFields = row.numFields() @@ -408,8 +404,8 @@ private[joins] object UnsafeHashedRelation { private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, capacity: Int) extends MemoryConsumer(mm) with Externalizable with KryoSerializable { - var inputEmpty = false var anyNullKeyExists = false + def inputEmpty: Boolean = (numKeys == 0) && !anyNullKeyExists // Whether the keys are stored in dense mode or not. private var isDense = false @@ -789,7 +785,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap writeBoolean: (Boolean) => Unit, writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit): Unit = { - writeBoolean(inputEmpty) writeBoolean(anyNullKeyExists) writeBoolean(isDense) writeLong(minKey) @@ -832,7 +827,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap readBoolean: () => Boolean, readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { - inputEmpty = readBoolean() anyNullKeyExists = readBoolean() isDense = readBoolean() minKey = readLong() @@ -933,7 +927,6 @@ private[joins] object LongHashedRelation { // Create a mapping of key -> rows var numFields = 0 - map.inputEmpty = input.isEmpty while (input.hasNext) { val unsafeRow = input.next().asInstanceOf[UnsafeRow] numFields = unsafeRow.numFields() From 39c2abee73709aab134ebef276ec775e4451d491 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Tue, 21 Jul 2020 11:08:57 +0800 Subject: [PATCH 09/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit add comment for isNullAwareAntiJoin special handled code. Change-Id: I8873744af8e9e2d7ced5978ecf8350dc859c846b --- .../spark/sql/execution/joins/BroadcastHashJoinExec.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 846effbfe3f50..c8696b8074cd7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -517,7 +517,9 @@ case class BroadcastHashJoinExec( | $found = true; | } | } - |} ${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} + |} + |// special case for NullAwareAntiJoin, if anyNull in streamedRow, row should be dropped. + |${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} |if (!$found) { | $numOutput.add(1); | ${consume(ctx, input)} @@ -544,7 +546,9 @@ case class BroadcastHashJoinExec( | } | } | } - |} ${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} + |} + |// special case for NullAwareAntiJoin, if anyNull in streamedRow, row should be dropped. + |${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} |if (!$found) { | $numOutput.add(1); | ${consume(ctx, input)} From 924ed2bb9bf5c0ff910a0f1743300394ec24cf27 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Tue, 21 Jul 2020 15:36:00 +0800 Subject: [PATCH 10/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. code refine and add more comments for new HashedRelation api. 2. update pattern match code simpler like ExtractEquiJoinKeys. Change-Id: I6864addbeebd47de539ccf8e88325fa96b041e87 --- .../apache/spark/unsafe/map/BytesToBytesMap.java | 2 +- .../spark/sql/catalyst/planning/patterns.scala | 14 +++++++------- .../spark/sql/execution/joins/HashedRelation.scala | 8 +++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index 1c392362b1b7f..f766f7520957d 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -175,7 +175,7 @@ public final class BytesToBytesMap extends MemoryConsumer { public boolean inputEmpty() { - return ((numKeys == 0) && !anyNullKeyExists); + return numKeys == 0 && !anyNullKeyExists; } public boolean isAnyNullKeyExists() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index 4b955f3c002dc..e245f5f51caad 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -391,7 +391,7 @@ object PhysicalWindow { } } -object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper { +object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper with PredicateHelper { // SingleColumn NullAwareAntiJoin // streamedSideKeys, buildSideKeys @@ -408,17 +408,17 @@ object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper { */ def unapply(join: Join): Option[ReturnType] = join match { case Join(left, right, LeftAnti, - Some(Or(EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), + Some(Or(equalTo @ EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), IsNull(EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference)))), _) if SQLConf.get.nullAwareAntiJoinOptimizeEnabled && leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) && canBroadcastBySize(right, SQLConf.get) && right.output.length == 1 => - val leftNotInKeyFound = left.output.exists(_.semanticEquals(leftAttr)) - if (leftNotInKeyFound) { - Some(Seq(leftAttr), Seq(rightAttr)) - } else { - Some(Seq(rightAttr), Seq(leftAttr)) + equalTo match { + case EqualTo(l, r) if canEvaluate(l, left) && canEvaluate(r, right) => + Some(Seq(l), Seq(r)) + case EqualTo(l, r) if canEvaluate(l, right) && canEvaluate(r, left) => + Some(Seq(r), Seq(l)) } case _ => None } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 5dc0cc088da9c..322ac101fe6f5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -72,12 +72,14 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { def keyIsUnique: Boolean /** - * is input: Iterator[InternalRow] empty + * Note that, the hashed relation can be empty despite the Iterator[InternalRow] being not empty, + * since the hashed relation skips over null keys. */ def inputEmpty: Boolean /** - * anyNull key exists in input + * It's only used in null aware anti join since the hashed relation skips over null keys. + * If buildSide include null key row, all streamedSide row will be dropped in NAAJ. */ def anyNullKeyExists: Boolean @@ -405,7 +407,7 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap extends MemoryConsumer(mm) with Externalizable with KryoSerializable { var anyNullKeyExists = false - def inputEmpty: Boolean = (numKeys == 0) && !anyNullKeyExists + def inputEmpty: Boolean = numKeys == 0 && !anyNullKeyExists // Whether the keys are stored in dense mode or not. private var isDense = false From b96453558348a2c60d414e7f814bd3c0bc2fb38f Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Tue, 21 Jul 2020 23:16:34 +0800 Subject: [PATCH 11/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. remove useless condition in ExtractSingleColumnNullAwareAntiJoin pattern match. 2. update JoinSuite cases. Change-Id: I3b282fc1abd6be6e544146b176e167adb1ed8e05 --- .../sql/catalyst/planning/patterns.scala | 17 ++++----- .../org/apache/spark/sql/JoinSuite.scala | 38 ++++++++++--------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index e245f5f51caad..eaa3cdfd39952 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -408,17 +408,16 @@ object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper with Pre */ def unapply(join: Join): Option[ReturnType] = join match { case Join(left, right, LeftAnti, - Some(Or(equalTo @ EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), + Some(Or(EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), IsNull(EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference)))), _) if SQLConf.get.nullAwareAntiJoinOptimizeEnabled && - leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) && - canBroadcastBySize(right, SQLConf.get) && - right.output.length == 1 => - equalTo match { - case EqualTo(l, r) if canEvaluate(l, left) && canEvaluate(r, right) => - Some(Seq(l), Seq(r)) - case EqualTo(l, r) if canEvaluate(l, right) && canEvaluate(r, left) => - Some(Seq(r), Seq(l)) + leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => + if (canEvaluate(leftAttr, left) && canEvaluate(rightAttr, right)) { + Some(Seq(leftAttr), Seq(rightAttr)) + } else if (canEvaluate(leftAttr, right) && canEvaluate(rightAttr, left)) { + Some(Seq(rightAttr), Seq(leftAttr)) + } else { + None } case _ => None } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index 93b3f0c6e8616..829618b44885b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -1149,36 +1149,38 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan } } - test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize positive") { + test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize") { withSQLConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> "true", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString) { + // positive not in subquery case assertJoin(( "select * from testData where key not in (select a from testData2)", classOf[BroadcastNullAwareHashJoinExec])) - // testData3.b nullable is true - assertJoin(( - "select * from testData left anti join testData3 ON key = b or isnull(key = b)", - classOf[BroadcastNullAwareHashJoinExec])) - // multi column not supported + + // negative not in subquery case since multi-column is not supported assertJoin(( "select * from testData where (key, key + 1) not in (select * from testData2)", classOf[BroadcastNestedLoopJoinExec])) - } - } - test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize negative") { - withSQLConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "0") { - assertJoin(( - "select * from testData where key not in (select a from testData2)", - classOf[BroadcastNestedLoopJoinExec])) - // testData3.b nullable is true + // positive hand-written left anti join + // testData.key nullable false + // testData3.b nullable true assertJoin(( "select * from testData left anti join testData3 ON key = b or isnull(key = b)", - classOf[BroadcastNestedLoopJoinExec])) - // multi column not supported + classOf[BroadcastNullAwareHashJoinExec])) + + // negative hand-written left anti join + // testData.key nullable false + // testData2.a nullable false assertJoin(( - "select * from testData where (key, key + 1) not in (select * from testData2)", + "select * from testData left anti join testData2 ON key = a or isnull(key = a)", + classOf[BroadcastHashJoinExec])) + + // negative hand-written left anti join + // not match pattern Or(EqualTo(a=b), IsNull(EqualTo(a=b)) + assertJoin(( + "select * from testData2 left anti join testData3 ON testData2.a = testData3.b or " + + "isnull(testData2.b = testData3.b)", classOf[BroadcastNestedLoopJoinExec])) } } From d541067ead6b31b1822c169b92b3a246b59958ac Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Wed, 22 Jul 2020 15:11:03 +0800 Subject: [PATCH 12/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. rename conf spark.sql.nullAware.antiJoin.optimize.enabled to spark.sql.nullAwareAntiJoin.optimize.enabled. 2. code style refine. 3. rename from BroadcastNullAwareHashJoinExec to BroadcastNullAwareLeftAntiHashJoinExec 4. add more cases in SubquerySuite to assert both result and whether BroadcastNullAwareLeftAntiHashJoinExec was used. 5. rebase community master code. Change-Id: Ie6487a860fd0318b279afe0d243a92b677311115 --- .../spark/unsafe/map/BytesToBytesMap.java | 6 +- .../apache/spark/sql/internal/SQLConf.scala | 6 +- .../spark/sql/execution/SparkStrategies.scala | 2 +- .../adaptive/LogicalQueryStageStrategy.scala | 2 +- .../joins/BroadcastHashJoinExec.scala | 1 - ...adcastNullAwareLeftAntiHashJoinExec.scala} | 22 ++--- .../org/apache/spark/sql/JoinSuite.scala | 6 +- .../org/apache/spark/sql/SubquerySuite.scala | 93 +++++++++++++++++++ 8 files changed, 111 insertions(+), 27 deletions(-) rename sql/core/src/main/scala/org/apache/spark/sql/execution/joins/{BroadcastNullAwareHashJoinExec.scala => BroadcastNullAwareLeftAntiHashJoinExec.scala} (90%) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index f766f7520957d..cf32e057d0916 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -173,8 +173,7 @@ public final class BytesToBytesMap extends MemoryConsumer { private boolean anyNullKeyExists = false; - public boolean inputEmpty() - { + public boolean inputEmpty() { return numKeys == 0 && !anyNullKeyExists; } @@ -183,8 +182,7 @@ public boolean isAnyNullKeyExists() return anyNullKeyExists; } - public void setAnyNullKeyExists(boolean anyNullKeyExists) - { + public void setAnyNullKeyExists(boolean anyNullKeyExists) { this.anyNullKeyExists = anyNullKeyExists; } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 9231634536093..164e23cc64a5a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2673,10 +2673,10 @@ object SQLConf { .createWithDefault(8) val NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED = - buildConf("spark.sql.nullAware.antiJoin.optimize.enabled") + buildConf("spark.sql.nullAwareAntiJoin.optimize.enabled") .internal() .doc("When true, NULL-aware anti join execution will be planed into " + - "BroadcastNullAwareHashJoinExec, " + + "BroadcastNullAwareLeftAntiHashJoinExec, " + "optimized from O(M*N) calculation into O(M) calculation " + "using Hash lookup instead of Looping lookup." + "Only support for singleColumn NAAJ for now.") @@ -2684,7 +2684,7 @@ object SQLConf { .createWithDefault(false) val NULL_AWARE_ANTI_JOIN_OPTIMIZE_USE_BHJ = - buildConf("spark.sql.nullAware.antiJoin.optimize.use.bhj") + buildConf("spark.sql.nullAwareAntiJoin.optimize.use.bhj") .internal() .doc("When true, NULL-aware anti join execution will be planed into " + "BroadcastHashJoinExec with special execution.") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala index 3dcc87bc3391a..652b5e93c9288 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala @@ -238,7 +238,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) } else { - Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, + Seq(joins.BroadcastNullAwareLeftAntiHashJoinExec(leftKeys, rightKeys, planLater(j.left), planLater(j.right), BuildRight, LeftAnti, None)) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala index 36f4cdd18c399..d0c1d1096260b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala @@ -57,7 +57,7 @@ object LogicalQueryStageStrategy extends Strategy with PredicateHelper { Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) } else { - Seq(joins.BroadcastNullAwareHashJoinExec(leftKeys, rightKeys, + Seq(joins.BroadcastNullAwareLeftAntiHashJoinExec(leftKeys, rightKeys, planLater(j.left), planLater(j.right), BuildRight, LeftAnti, None)) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index c8696b8074cd7..c406f0a383cb5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -481,7 +481,6 @@ case class BroadcastHashJoinExec( if (isNullAwareAntiJoin) { require(leftKeys.length == 1, "leftKeys length should be 1") require(rightKeys.length == 1, "rightKeys length should be 1") - require(right.output.length == 1, "null aware anti join only supports single column.") require(joinType == LeftAnti, "joinType must be LeftAnti.") require(buildSide == BuildRight, "buildSide must be BuildRight.") require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareLeftAntiHashJoinExec.scala similarity index 90% rename from sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala rename to sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareLeftAntiHashJoinExec.scala index 47f63e5e19d7f..55b6198887dbb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareLeftAntiHashJoinExec.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.LongType -case class BroadcastNullAwareHashJoinExec( +case class BroadcastNullAwareLeftAntiHashJoinExec( leftKeys: Seq[Expression], rightKeys: Seq[Expression], left: SparkPlan, @@ -43,7 +43,6 @@ case class BroadcastNullAwareHashJoinExec( // multi-column null aware anti join is much more complicated than single column ones. require(leftKeys.length == 1, "leftKeys length should be 1") require(rightKeys.length == 1, "rightKeys length should be 1") - require(right.output.length == 1, "null aware anti join only supports single column.") require(joinType == LeftAnti, "joinType must be LeftAnti.") require(buildSide == BuildRight, "buildSide must be BuildRight.") require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, @@ -60,7 +59,10 @@ case class BroadcastNullAwareHashJoinExec( } override def requiredChildDistribution: Seq[Distribution] = { - UnspecifiedDistribution :: BroadcastDistribution(IdentityBroadcastMode) :: Nil + val mode = HashedRelationBroadcastMode( + BindReferences.bindReferences(HashJoin.rewriteKeyExpr(rightKeys), right.output) + ) + UnspecifiedDistribution :: BroadcastDistribution(mode) :: Nil } private[this] def genResultProjection: UnsafeProjection = { @@ -71,14 +73,6 @@ case class BroadcastNullAwareHashJoinExec( left.output } - private def prepareBroadcastHashedRelation = { - val buildSideRows = broadcast.executeBroadcast[Array[InternalRow]]().value - HashedRelation(buildSideRows.iterator, - BindReferences.bindReferences[Expression]( - Seq(right.output.head), AttributeSeq(right.output)), - buildSideRows.length) - } - private def nullAwareLeftAntiJoin( relation: HashedRelation): RDD[InternalRow] = { streamed.execute().mapPartitionsInternal { streamedIter => @@ -103,13 +97,13 @@ case class BroadcastNullAwareHashJoinExec( } protected override def doExecute(): RDD[InternalRow] = { - val relation: HashedRelation = prepareBroadcastHashedRelation + val relation: HashedRelation = broadcast.executeBroadcast[HashedRelation]().value val resultRdd = (joinType, buildSide) match { case (LeftAnti, BuildRight) => nullAwareLeftAntiJoin(relation) case _ => throw new IllegalArgumentException( - s"BroadcastNullAwareHashJoinExec only supports (LeftAnti + BuildRight) for now.") + s"BroadcastNullAwareLeftAntiHashJoinExec only supports (LeftAnti + BuildRight) for now.") } val numOutputRows = longMetric("numOutputRows") @@ -152,7 +146,7 @@ case class BroadcastNullAwareHashJoinExec( } override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - val relation: HashedRelation = prepareBroadcastHashedRelation + val relation: HashedRelation = broadcast.executeBroadcast[HashedRelation]().value val broadcastRef = ctx.addReferenceObj("broadcast", relation) val clsName = relation.getClass.getName diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index 829618b44885b..70b31b7a4507d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -82,7 +82,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan case j: ShuffledHashJoinExec => j case j: CartesianProductExec => j case j: BroadcastNestedLoopJoinExec => j - case j: BroadcastNullAwareHashJoinExec => j + case j: BroadcastNullAwareLeftAntiHashJoinExec => j case j: SortMergeJoinExec => j } @@ -1155,7 +1155,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan // positive not in subquery case assertJoin(( "select * from testData where key not in (select a from testData2)", - classOf[BroadcastNullAwareHashJoinExec])) + classOf[BroadcastNullAwareLeftAntiHashJoinExec])) // negative not in subquery case since multi-column is not supported assertJoin(( @@ -1167,7 +1167,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan // testData3.b nullable true assertJoin(( "select * from testData left anti join testData3 ON key = b or isnull(key = b)", - classOf[BroadcastNullAwareHashJoinExec])) + classOf[BroadcastNullAwareLeftAntiHashJoinExec])) // negative hand-written left anti join // testData.key nullable false diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index 2bb9aa55e4579..684b7a10eed85 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, Sort} import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, FileSourceScanExec, InputAdapter, ReusedSubqueryExec, ScalarSubquery, SubqueryExec, WholeStageCodegenExec} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution} import org.apache.spark.sql.execution.datasources.FileScanRDD +import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, BroadcastNullAwareLeftAntiHashJoinExec} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession @@ -1646,4 +1647,96 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark checkAnswer(df, df2) checkAnswer(df, Nil) } + + test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize") { + Seq((true, true, true), (true, true, false), (true, false, true), + (true, false, false), (false, true, true), (false, true, false), + (false, false, true), (false, false, false)).foreach { config => + withSQLConf( + SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> config._1.toString, + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> config._2.toString, + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> config._3.toString) { + + def findJoinExec(df: DataFrame): BaseJoinExec = { + df.queryExecution.sparkPlan.collectFirst { + case j: BaseJoinExec => j + }.get + } + + var df: DataFrame = null + + // single column not in subquery -- empty sub-query + df = sql("select * from l where a not in (select c from r where c > 10)") + checkAnswer(df, + Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: + Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + if (config._1) { + assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- sub-query include null + df = sql("select * from l where a not in (select c from r where d < 6.0)") + checkAnswer(df, Seq.empty) + if (config._1) { + assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- streamedSide row is null + df = sql("select * from l where b = 5.0 and a not in (select c from r where c is not null)") + checkAnswer(df, Seq.empty) + if (config._1) { + assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- streamedSide row is not null, match found + df = sql("select * from l where a = 6 and a not in (select c from r where c is not null)") + checkAnswer(df, Seq.empty) + if (config._1) { + assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- streamedSide row is not null, match not found + df = sql("select * from l where a = 1 and a not in (select c from r where c is not null)") + checkAnswer(df, Row(1, 2.0) :: Row(1, 2.0) :: Nil) + if (config._1) { + assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- d = b + 10 joinKey found, match ExtractEquiJoinKeys + df = sql("select * from l where a not in (select c from r where d = b + 10)") + checkAnswer(df, + Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: + Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + + // single column not in subquery -- d = b + 10 and b = 5.0 => d = 15, joinKey not found + // match ExtractSingleColumnNullAwareAntiJoin + df = sql("select * from l where b = 5.0 and a not in (select c from r where d = b + 10)") + checkAnswer(df, + Row(null, 5.0) :: Nil) + if (config._1) { + assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // multi column not in subquery + df = sql("select * from l where (a, b) not in (select c, d from r where c > 10)") + checkAnswer(df, + Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: + Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + } + } } From b5831ad2d7c3790efe69fad55ce3a46048d780ab Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Thu, 23 Jul 2020 18:15:16 +0800 Subject: [PATCH 13/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. inputEmpty anyNullKeyExists should not be put in BytesToBytesMap and LongToUnsafeRowMap 2. inputEmpty -> isOriginInputEmpty 3. anyNullKeyExists -> allNullColumnKeyExistsInOriginInput 4. refer to not-in-unit-tests-single-column.sql and not-in-unit-tests-multi-column.sql extract common pattern is that, when row has null for all column exists, reject all rows. 5. Kick in BHJ as default implementation. 6. other code refine. 7. Suite update. 8. remove BroadcastNullAwareLeftAntiHashJoinExec. Change-Id: Icdf5f79c7c0ebe79b747093524ce27b2f8e1f375 --- .../spark/unsafe/map/BytesToBytesMap.java | 15 -- .../sql/catalyst/expressions/UnsafeRow.java | 9 + .../apache/spark/sql/internal/SQLConf.scala | 13 +- .../spark/sql/execution/SparkStrategies.scala | 10 +- .../adaptive/LogicalQueryStageStrategy.scala | 11 +- .../joins/BroadcastHashJoinExec.scala | 86 +++++--- ...oadcastNullAwareLeftAntiHashJoinExec.scala | 199 ------------------ .../spark/sql/execution/joins/HashJoin.scala | 7 + .../sql/execution/joins/HashedRelation.scala | 125 +++++++---- .../org/apache/spark/sql/JoinSuite.scala | 5 +- .../org/apache/spark/sql/SubquerySuite.scala | 14 +- 11 files changed, 178 insertions(+), 316 deletions(-) delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareLeftAntiHashJoinExec.scala diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index cf32e057d0916..6e028886f2318 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -171,21 +171,6 @@ public final class BytesToBytesMap extends MemoryConsumer { private volatile MapIterator destructiveIterator = null; private LinkedList spillWriters = new LinkedList<>(); - private boolean anyNullKeyExists = false; - - public boolean inputEmpty() { - return numKeys == 0 && !anyNullKeyExists; - } - - public boolean isAnyNullKeyExists() - { - return anyNullKeyExists; - } - - public void setAnyNullKeyExists(boolean anyNullKeyExists) { - this.anyNullKeyExists = anyNullKeyExists; - } - public BytesToBytesMap( TaskMemoryManager taskMemoryManager, BlockManager blockManager, diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java index 4dc5ce1de047b..0217b2fb9b0a4 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java @@ -591,6 +591,15 @@ public boolean anyNull() { return BitSetMethods.anySet(baseObject, baseOffset, bitSetWidthInBytes / 8); } + public boolean allNull() { + for (int i = 0; i < numFields; i++) { + if (!BitSetMethods.isSet(baseObject, baseOffset, i)) { + return false; + } + } + return true; + } + /** * Writes the content of this row into a memory address, identified by an object and an offset. * The target memory address must already been allocated, and have enough space to hold all the diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 164e23cc64a5a..de114a0fbddad 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2676,21 +2676,13 @@ object SQLConf { buildConf("spark.sql.nullAwareAntiJoin.optimize.enabled") .internal() .doc("When true, NULL-aware anti join execution will be planed into " + - "BroadcastNullAwareLeftAntiHashJoinExec, " + + "BroadcastJoinExec with flag isNullAwareAntiJoin enabled, " + "optimized from O(M*N) calculation into O(M) calculation " + "using Hash lookup instead of Looping lookup." + "Only support for singleColumn NAAJ for now.") .booleanConf .createWithDefault(false) - val NULL_AWARE_ANTI_JOIN_OPTIMIZE_USE_BHJ = - buildConf("spark.sql.nullAwareAntiJoin.optimize.use.bhj") - .internal() - .doc("When true, NULL-aware anti join execution will be planed into " + - "BroadcastHashJoinExec with special execution.") - .booleanConf - .createWithDefault(false) - /** * Holds information about keys that have been deprecated. * @@ -3299,9 +3291,6 @@ class SQLConf extends Serializable with Logging { def nullAwareAntiJoinOptimizeEnabled: Boolean = getConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED) - def nullAwareAntiJoinOptimizeUseBHJ: Boolean = - getConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_USE_BHJ) - /** ********************** SQLConf functionality methods ************ */ /** Set Spark SQL configuration properties. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala index 652b5e93c9288..4f9cb22ee0057 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala @@ -233,14 +233,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] { .getOrElse(createJoinWithoutHint()) case j @ ExtractSingleColumnNullAwareAntiJoin(leftKeys, rightKeys) => - if (SQLConf.get.nullAwareAntiJoinOptimizeUseBHJ) { - // for BHJ Prototype - Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, - None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) - } else { - Seq(joins.BroadcastNullAwareLeftAntiHashJoinExec(leftKeys, rightKeys, - planLater(j.left), planLater(j.right), BuildRight, LeftAnti, None)) - } + Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, + None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) // If it is not an equi-join, we first look at the join hints w.r.t. the following order: // 1. broadcast hint: pick broadcast nested loop join. If both sides have the broadcast diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala index d0c1d1096260b..bcf9dc1544ce3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LogicalQueryStageStrategy.scala @@ -25,7 +25,6 @@ import org.apache.spark.sql.catalyst.plans.LeftAnti import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan} import org.apache.spark.sql.execution.{joins, SparkPlan} import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec} -import org.apache.spark.sql.internal.SQLConf /** * Strategy for plans containing [[LogicalQueryStage]] nodes: @@ -52,14 +51,8 @@ object LogicalQueryStageStrategy extends Strategy with PredicateHelper { case j @ ExtractSingleColumnNullAwareAntiJoin(leftKeys, rightKeys) if isBroadcastStage(j.right) => - if (SQLConf.get.nullAwareAntiJoinOptimizeUseBHJ) { - // for BHJ Prototype - Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, - None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) - } else { - Seq(joins.BroadcastNullAwareLeftAntiHashJoinExec(leftKeys, rightKeys, - planLater(j.left), planLater(j.right), BuildRight, LeftAnti, None)) - } + Seq(joins.BroadcastHashJoinExec(leftKeys, rightKeys, LeftAnti, BuildRight, + None, planLater(j.left), planLater(j.right), isNullAwareAntiJoin = true)) case j @ Join(left, right, joinType, condition, _) if isBroadcastStage(left) || isBroadcastStage(right) => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index c406f0a383cb5..1da4a90d90457 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -31,7 +31,6 @@ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical.{BroadcastDistribution, Distribution, HashPartitioning, Partitioning, PartitioningCollection, UnspecifiedDistribution} import org.apache.spark.sql.execution.{CodegenSupport, SparkPlan} import org.apache.spark.sql.execution.metric.SQLMetrics -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{BooleanType, LongType} /** @@ -51,11 +50,22 @@ case class BroadcastHashJoinExec( isNullAwareAntiJoin: Boolean = false) extends HashJoin with CodegenSupport { + if (isNullAwareAntiJoin) { + // TODO support multi column NULL-aware anti join in future. + // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6 + // multi-column null aware anti join is much more complicated than single column ones. + require(leftKeys.length == 1, "leftKeys length should be 1") + require(rightKeys.length == 1, "rightKeys length should be 1") + require(joinType == LeftAnti, "joinType must be LeftAnti.") + require(buildSide == BuildRight, "buildSide must be BuildRight.") + require(condition.isEmpty, "null aware anti join optimize condition should be empty.") + } + override lazy val metrics = Map( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) override def requiredChildDistribution: Seq[Distribution] = { - val mode = HashedRelationBroadcastMode(buildBoundKeys) + val mode = HashedRelationBroadcastMode(buildBoundKeys, isNullAwareAntiJoin) buildSide match { case BuildLeft => BroadcastDistribution(mode) :: UnspecifiedDistribution :: Nil @@ -137,9 +147,11 @@ case class BroadcastHashJoinExec( val broadcastRelation = buildPlan.executeBroadcast[HashedRelation]() if (isNullAwareAntiJoin) { streamedPlan.execute().mapPartitionsInternal { streamedIter => - if (broadcastRelation.value.inputEmpty) { + val hashed = broadcastRelation.value.asReadOnlyCopy() + TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize) + if (hashed.isOriginInputEmpty) { streamedIter - } else if (broadcastRelation.value.anyNullKeyExists) { + } else if (hashed.allNullColumnKeyExistsInOriginInput) { Iterator.empty } else { val keyGenerator = UnsafeProjection.create( @@ -149,9 +161,14 @@ case class BroadcastHashJoinExec( ) streamedIter.filter(row => { val lookupKey: UnsafeRow = keyGenerator(row) - val streamedRowIsNull = lookupKey.isNullAt(0) - val notInKeyEqual = broadcastRelation.value.get(lookupKey) != null - !streamedRowIsNull && !notInKeyEqual + if (lookupKey.allNull()) { + false + } else { + // in isNullAware mode + // UnsafeHashedRelation include anyNull key, if match, dropped row + // Same as LongHashedRelation where lookupKey isNotNullAt(0) + hashed.get(lookupKey) == null + } }) } } @@ -477,26 +494,45 @@ case class BroadcastHashJoinExec( val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) val (matched, checkCondition, _) = getJoinCondition(ctx, input) val numOutput = metricTerm(ctx, "numOutputRows") + val isLongHashedRelation = broadcastRelation.value.isInstanceOf[LongHashedRelation] + + // fast stop if isOriginInputEmpty = true + // whether isNullAwareAntiJoin is true or false + // should accept all rows in streamedSide + if (broadcastRelation.value.isOriginInputEmpty) { + return s""" + |// Common Anti Join isOriginInputEmpty(true) accept all + |$numOutput.add(1); + |${consume(ctx, input)} + """.stripMargin + } if (isNullAwareAntiJoin) { - require(leftKeys.length == 1, "leftKeys length should be 1") - require(rightKeys.length == 1, "rightKeys length should be 1") - require(joinType == LeftAnti, "joinType must be LeftAnti.") - require(buildSide == BuildRight, "buildSide must be BuildRight.") - require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, - "nullAwareAntiJoinOptimizeEnabled must be on for null aware anti join optimize.") - require(checkCondition == "", "null aware anti join optimize condition should be empty.") - - if (broadcastRelation.value.inputEmpty) { + if (broadcastRelation.value.allNullColumnKeyExistsInOriginInput) { return s""" - |// singleColumn NAAJ inputEmpty(true) accept all - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } else if (broadcastRelation.value.anyNullKeyExists) { + |// NAAJ isOriginInputEmpty(false) anyNullKeyExists(true) reject all + """.stripMargin + } else { + val found = ctx.freshName("found") return s""" - |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(true) reject all - """.stripMargin + |// NAAJ isOriginInputEmpty(false) allNullColumnKeyExistsInOriginInput(false) + |boolean $found = false; + |// generate join key for stream side + |${keyEv.code} + |if (${ if (isLongHashedRelation) s"$anyNull" else s"${keyEv.value}.allNull()"}) { + | $found = true; + |} else { + | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); + | if ($matched != null) { + | $found = true; + | } + |} + | + |if (!$found) { + | $numOutput.add(1); + | ${consume(ctx, input)} + |} + """.stripMargin } } @@ -517,8 +553,6 @@ case class BroadcastHashJoinExec( | } | } |} - |// special case for NullAwareAntiJoin, if anyNull in streamedRow, row should be dropped. - |${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} |if (!$found) { | $numOutput.add(1); | ${consume(ctx, input)} @@ -546,8 +580,6 @@ case class BroadcastHashJoinExec( | } | } |} - |// special case for NullAwareAntiJoin, if anyNull in streamedRow, row should be dropped. - |${ if (isNullAwareAntiJoin) s"else { $found = true; }" else ""} |if (!$found) { | $numOutput.add(1); | ${consume(ctx, input)} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareLeftAntiHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareLeftAntiHashJoinExec.scala deleted file mode 100644 index 55b6198887dbb..0000000000000 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNullAwareLeftAntiHashJoinExec.scala +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.joins - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, GenerateUnsafeProjection} -import org.apache.spark.sql.catalyst.optimizer.{BuildRight, BuildSide} -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.physical._ -import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, SparkPlan} -import org.apache.spark.sql.execution.metric.SQLMetrics -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.LongType - -case class BroadcastNullAwareLeftAntiHashJoinExec( - leftKeys: Seq[Expression], - rightKeys: Seq[Expression], - left: SparkPlan, - right: SparkPlan, - buildSide: BuildSide, - joinType: JoinType, - condition: Option[Expression]) extends BaseJoinExec with CodegenSupport { - - // TODO support multi column NULL-aware anti join in future. - // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6 - // multi-column null aware anti join is much more complicated than single column ones. - require(leftKeys.length == 1, "leftKeys length should be 1") - require(rightKeys.length == 1, "rightKeys length should be 1") - require(joinType == LeftAnti, "joinType must be LeftAnti.") - require(buildSide == BuildRight, "buildSide must be BuildRight.") - require(SQLConf.get.nullAwareAntiJoinOptimizeEnabled, - "nullAwareAntiJoinOptimizeEnabled must be on for null aware anti join optimize.") - - override lazy val metrics = Map( - "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) - - private val (streamed, broadcast) = (left, right) - - override def simpleStringWithNodeId(): String = { - val opId = ExplainUtils.getOpId(this) - s"$nodeName $joinType ${buildSide} ($opId)".trim - } - - override def requiredChildDistribution: Seq[Distribution] = { - val mode = HashedRelationBroadcastMode( - BindReferences.bindReferences(HashJoin.rewriteKeyExpr(rightKeys), right.output) - ) - UnspecifiedDistribution :: BroadcastDistribution(mode) :: Nil - } - - private[this] def genResultProjection: UnsafeProjection = { - UnsafeProjection.create(output, output) - } - - override def output: Seq[Attribute] = { - left.output - } - - private def nullAwareLeftAntiJoin( - relation: HashedRelation): RDD[InternalRow] = { - streamed.execute().mapPartitionsInternal { streamedIter => - if (relation.inputEmpty) { - streamedIter - } else if (relation.anyNullKeyExists) { - Iterator.empty - } else { - val keyGenerator = UnsafeProjection.create( - BindReferences.bindReferences[Expression]( - leftKeys, - AttributeSeq(left.output)) - ) - streamedIter.filter(row => { - val lookupKey: UnsafeRow = keyGenerator(row) - val streamedRowIsNull = lookupKey.isNullAt(0) - val notInKeyEqual = relation.get(lookupKey) != null - !streamedRowIsNull && !notInKeyEqual - }) - } - } - } - - protected override def doExecute(): RDD[InternalRow] = { - val relation: HashedRelation = broadcast.executeBroadcast[HashedRelation]().value - val resultRdd = (joinType, buildSide) match { - case (LeftAnti, BuildRight) => - nullAwareLeftAntiJoin(relation) - case _ => - throw new IllegalArgumentException( - s"BroadcastNullAwareLeftAntiHashJoinExec only supports (LeftAnti + BuildRight) for now.") - } - - val numOutputRows = longMetric("numOutputRows") - resultRdd.mapPartitionsWithIndexInternal { (index, iter) => - val resultProj = genResultProjection - resultProj.initialize(index) - iter.map { r => - numOutputRows += 1 - resultProj(r) - } - } - } - - override def needCopyResult: Boolean = - streamed.asInstanceOf[CodegenSupport].needCopyResult - - override def inputRDDs(): Seq[RDD[InternalRow]] = { - streamed.asInstanceOf[CodegenSupport].inputRDDs() - } - - override def doProduce(ctx: CodegenContext): String = { - streamed.asInstanceOf[CodegenSupport].produce(ctx, this) - } - - private def genStreamSideJoinKey( - ctx: CodegenContext, - input: Seq[ExprCode]): (ExprCode, String) = { - ctx.currentVars = input - if (leftKeys.length == 1 && leftKeys.head.dataType == LongType) { - // generate the join key as Long - val ev = - BindReferences.bindReference[Expression](leftKeys.head, left.output).genCode(ctx) - (ev, ev.isNull) - } else { - // generate the join key as UnsafeRow - val ev = GenerateUnsafeProjection.createCode(ctx, - BindReferences.bindReferences[Expression](leftKeys, left.output)) - (ev, s"${ev.value}.anyNull()") - } - } - - override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - val relation: HashedRelation = broadcast.executeBroadcast[HashedRelation]().value - val broadcastRef = ctx.addReferenceObj("broadcast", relation) - val clsName = relation.getClass.getName - - // Inline mutable state since not many join operations in a task - val relationTerm = ctx.addMutableState(clsName, "relation", - v => s""" - | $v = (($clsName) $broadcastRef).asReadOnlyCopy(); - | incPeakExecutionMemory($v.estimatedSize()); - """.stripMargin, forceInline = true) - - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val matched = ctx.freshName("matched") - val numOutput = metricTerm(ctx, "numOutputRows") - val found = ctx.freshName("found") - - if (relation.inputEmpty) { - s""" - |// singleColumn NAAJ inputEmpty(true) accept all - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } else if (relation.anyNullKeyExists) { - s""" - |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(true) reject all - """.stripMargin - } else { - s""" - |// singleColumn NAAJ inputEmpty(false) anyNullKeyExists(false) - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | $found = true; - | } - |} else { - | $found = true; - |} - | - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } - } -} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala index 4f22007b65845..8f6edbf769a0f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala @@ -268,6 +268,13 @@ trait HashJoin extends BaseJoinExec { private def antiJoin( streamIter: Iterator[InternalRow], hashedRelation: HashedRelation): Iterator[InternalRow] = { + // fast stop if isOriginInputEmpty = true + // whether isNullAwareAntiJoin is true or false + // should accept all rows in streamedSide + if (hashedRelation.isOriginInputEmpty) { + return streamIter + } + val joinKeys = streamSideKeyGenerator() val joinedRow = new JoinedRow diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 322ac101fe6f5..7013db45c9faf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -72,26 +72,40 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { def keyIsUnique: Boolean /** - * Note that, the hashed relation can be empty despite the Iterator[InternalRow] being not empty, - * since the hashed relation skips over null keys. + * Returns an iterator for keys of InternalRow type. */ - def inputEmpty: Boolean + def keys(): Iterator[InternalRow] /** - * It's only used in null aware anti join since the hashed relation skips over null keys. - * If buildSide include null key row, all streamedSide row will be dropped in NAAJ. + * Returns a read-only copy of this, to be safely used in current thread. */ - def anyNullKeyExists: Boolean + def asReadOnlyCopy(): HashedRelation + /** - * Returns an iterator for keys of InternalRow type. + * Normally HashedRelation is built from an Source (input: Iterator[InternalRow]). + * This indicates the origin input is empty. + * Note that, the hashed relation can be empty despite the input being not empty, + * since the hashed relation skips over null keys. */ - def keys(): Iterator[InternalRow] + var isOriginInputEmpty: Boolean + def setOriginInputEmtpy(isOriginInputEmpty: Boolean): HashedRelation = { + this.isOriginInputEmpty = isOriginInputEmpty + this + } /** - * Returns a read-only copy of this, to be safely used in current thread. + * It's only used in null aware anti join. + * This will be set true if Source (input: Iterator[InternalRow]) contains a key, + * which is allNullColumn. */ - def asReadOnlyCopy(): HashedRelation + var allNullColumnKeyExistsInOriginInput: Boolean + def setAllNullColumnKeyExistsInOriginInput( + allNullColumnKeyExistsInOriginInput: Boolean): HashedRelation = { + this.allNullColumnKeyExistsInOriginInput = + allNullColumnKeyExistsInOriginInput + this + } /** * Release any used resources. @@ -108,7 +122,8 @@ private[execution] object HashedRelation { input: Iterator[InternalRow], key: Seq[Expression], sizeEstimate: Int = 64, - taskMemoryManager: TaskMemoryManager = null): HashedRelation = { + taskMemoryManager: TaskMemoryManager = null, + isNullAware: Boolean = false): HashedRelation = { val mm = Option(taskMemoryManager).getOrElse { new TaskMemoryManager( new UnifiedMemoryManager( @@ -120,9 +135,9 @@ private[execution] object HashedRelation { } if (key.length == 1 && key.head.dataType == LongType) { - LongHashedRelation(input, key, sizeEstimate, mm) + LongHashedRelation(input, key, sizeEstimate, mm, isNullAware) } else { - UnsafeHashedRelation(input, key, sizeEstimate, mm) + UnsafeHashedRelation(input, key, sizeEstimate, mm, isNullAware) } } } @@ -146,6 +161,9 @@ private[joins] class UnsafeHashedRelation( override def asReadOnlyCopy(): UnsafeHashedRelation = { new UnsafeHashedRelation(numKeys, numFields, binaryMap) + .setOriginInputEmtpy(this.isOriginInputEmpty) + .setAllNullColumnKeyExistsInOriginInput(this.allNullColumnKeyExistsInOriginInput) + .asInstanceOf[UnsafeHashedRelation] } override def estimatedSize: Long = binaryMap.getTotalMemoryConsumption @@ -227,8 +245,9 @@ private[joins] class UnsafeHashedRelation( writeInt: (Int) => Unit, writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit) : Unit = { - writeBoolean(anyNullKeyExists) writeInt(numFields) + writeBoolean(isOriginInputEmpty) + writeBoolean(allNullColumnKeyExistsInOriginInput) // TODO: move these into BytesToBytesMap writeLong(binaryMap.numKeys()) writeLong(binaryMap.numValues()) @@ -262,8 +281,9 @@ private[joins] class UnsafeHashedRelation( readInt: () => Int, readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { - val anyNullKeyExists = readBoolean() numFields = readInt() + isOriginInputEmpty = readBoolean() + allNullColumnKeyExistsInOriginInput = readBoolean() resultRow = new UnsafeRow(numFields) val nKeys = readLong() val nValues = readLong() @@ -289,8 +309,6 @@ private[joins] class UnsafeHashedRelation( (nKeys * 1.5 + 1).toInt, // reduce hash collision pageSizeBytes) - binaryMap.setAnyNullKeyExists(anyNullKeyExists) - var i = 0 var keyBuffer = new Array[Byte](1024) var valuesBuffer = new Array[Byte](1024) @@ -321,9 +339,8 @@ private[joins] class UnsafeHashedRelation( read(() => in.readBoolean(), () => in.readInt(), () => in.readLong(), in.readBytes) } - override def inputEmpty: Boolean = binaryMap.inputEmpty - - override def anyNullKeyExists: Boolean = binaryMap.isAnyNullKeyExists + override var isOriginInputEmpty: Boolean = _ + override var allNullColumnKeyExistsInOriginInput: Boolean = _ } private[joins] object UnsafeHashedRelation { @@ -333,6 +350,15 @@ private[joins] object UnsafeHashedRelation { key: Seq[Expression], sizeEstimate: Int, taskMemoryManager: TaskMemoryManager): HashedRelation = { + apply(input, key, sizeEstimate, taskMemoryManager, isNullAware = false) + } + + def apply( + input: Iterator[InternalRow], + key: Seq[Expression], + sizeEstimate: Int, + taskMemoryManager: TaskMemoryManager, + isNullAware: Boolean = false): HashedRelation = { val pageSizeBytes = Option(SparkEnv.get).map(_.memoryManager.pageSizeBytes) .getOrElse(new SparkConf().get(BUFFER_PAGESIZE).getOrElse(16L * 1024 * 1024)) @@ -345,11 +371,19 @@ private[joins] object UnsafeHashedRelation { // Create a mapping of buildKeys -> rows val keyGenerator = UnsafeProjection.create(key) var numFields = 0 + val numKeys = key.length + val isOriginInputEmpty = !input.hasNext + var allNullColumnKeyExistsInOriginInput: Boolean = false while (input.hasNext) { val row = input.next().asInstanceOf[UnsafeRow] numFields = row.numFields() val key = keyGenerator(row) - if (!key.anyNull) { + if ((0 until numKeys).forall(key.isNullAt)) { + allNullColumnKeyExistsInOriginInput = true + } + + // TODO keep anyNull key for multi column NAAJ support in future + if (isNullAware || (!isNullAware && !key.anyNull)) { val loc = binaryMap.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes) val success = loc.append( key.getBaseObject, key.getBaseOffset, key.getSizeInBytes, @@ -360,12 +394,12 @@ private[joins] object UnsafeHashedRelation { throw new SparkOutOfMemoryError("There is not enough memory to build hash map") // scalastyle:on throwerror } - } else { - binaryMap.setAnyNullKeyExists(true) } } new UnsafeHashedRelation(key.size, numFields, binaryMap) + .setOriginInputEmtpy(isOriginInputEmpty) + .setAllNullColumnKeyExistsInOriginInput(allNullColumnKeyExistsInOriginInput) } } @@ -406,9 +440,6 @@ private[joins] object UnsafeHashedRelation { private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, capacity: Int) extends MemoryConsumer(mm) with Externalizable with KryoSerializable { - var anyNullKeyExists = false - def inputEmpty: Boolean = numKeys == 0 && !anyNullKeyExists - // Whether the keys are stored in dense mode or not. private var isDense = false @@ -787,7 +818,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap writeBoolean: (Boolean) => Unit, writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit): Unit = { - writeBoolean(anyNullKeyExists) writeBoolean(isDense) writeLong(minKey) writeLong(maxKey) @@ -829,7 +859,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap readBoolean: () => Boolean, readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { - anyNullKeyExists = readBoolean() isDense = readBoolean() minKey = readLong() maxKey = readLong() @@ -863,7 +892,11 @@ class LongHashedRelation( // Needed for serialization (it is public to make Java serialization work) def this() = this(0, null) - override def asReadOnlyCopy(): LongHashedRelation = new LongHashedRelation(nFields, map) + override def asReadOnlyCopy(): LongHashedRelation = + new LongHashedRelation(nFields, map) + .setOriginInputEmtpy(this.isOriginInputEmpty) + .setAllNullColumnKeyExistsInOriginInput(this.allNullColumnKeyExistsInOriginInput) + .asInstanceOf[LongHashedRelation] override def estimatedSize: Long = map.getTotalMemoryConsumption @@ -895,12 +928,16 @@ class LongHashedRelation( override def writeExternal(out: ObjectOutput): Unit = { out.writeInt(nFields) + out.writeBoolean(isOriginInputEmpty) + out.writeBoolean(allNullColumnKeyExistsInOriginInput) out.writeObject(map) } override def readExternal(in: ObjectInput): Unit = { nFields = in.readInt() resultRow = new UnsafeRow(nFields) + isOriginInputEmpty = in.readBoolean() + allNullColumnKeyExistsInOriginInput = in.readBoolean() map = in.readObject().asInstanceOf[LongToUnsafeRowMap] } @@ -909,9 +946,8 @@ class LongHashedRelation( */ override def keys(): Iterator[InternalRow] = map.keys() - override def inputEmpty: Boolean = map.inputEmpty - - override def anyNullKeyExists: Boolean = map.anyNullKeyExists + override var isOriginInputEmpty: Boolean = _ + override var allNullColumnKeyExistsInOriginInput: Boolean = _ } /** @@ -923,30 +959,47 @@ private[joins] object LongHashedRelation { key: Seq[Expression], sizeEstimate: Int, taskMemoryManager: TaskMemoryManager): LongHashedRelation = { + apply(input, key, sizeEstimate, taskMemoryManager, isNullAware = false) + } + + def apply( + input: Iterator[InternalRow], + key: Seq[Expression], + sizeEstimate: Int, + taskMemoryManager: TaskMemoryManager, + isNullAware: Boolean): LongHashedRelation = { val map = new LongToUnsafeRowMap(taskMemoryManager, sizeEstimate) val keyGenerator = UnsafeProjection.create(key) // Create a mapping of key -> rows var numFields = 0 + val isOriginInputEmpty: Boolean = !input.hasNext + var allNullColumnKeyExistsInOriginInput: Boolean = false while (input.hasNext) { val unsafeRow = input.next().asInstanceOf[UnsafeRow] numFields = unsafeRow.numFields() val rowKey = keyGenerator(unsafeRow) if (!rowKey.isNullAt(0)) { + // LongToUnsafeRowMap can't insert null key val key = rowKey.getLong(0) map.append(key, unsafeRow) } else { - map.anyNullKeyExists = true + // LongHashedRelation is single-column key + // rowKey.isNullAt(0) equivalent to allNullColumnKeyExistsInOriginInput + allNullColumnKeyExistsInOriginInput = true } } map.optimize() new LongHashedRelation(numFields, map) + .setOriginInputEmtpy(isOriginInputEmpty) + .setAllNullColumnKeyExistsInOriginInput(allNullColumnKeyExistsInOriginInput) + .asInstanceOf[LongHashedRelation] } } /** The HashedRelationBroadcastMode requires that rows are broadcasted as a HashedRelation. */ -case class HashedRelationBroadcastMode(key: Seq[Expression]) +case class HashedRelationBroadcastMode(key: Seq[Expression], isNullAware: Boolean = false) extends BroadcastMode { override def transform(rows: Array[InternalRow]): HashedRelation = { @@ -958,9 +1011,9 @@ case class HashedRelationBroadcastMode(key: Seq[Expression]) sizeHint: Option[Long]): HashedRelation = { sizeHint match { case Some(numRows) => - HashedRelation(rows, canonicalized.key, numRows.toInt) + HashedRelation(rows, canonicalized.key, numRows.toInt, isNullAware = isNullAware) case None => - HashedRelation(rows, canonicalized.key) + HashedRelation(rows, canonicalized.key, isNullAware = isNullAware) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index 70b31b7a4507d..edfd6d4d8facc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -82,7 +82,6 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan case j: ShuffledHashJoinExec => j case j: CartesianProductExec => j case j: BroadcastNestedLoopJoinExec => j - case j: BroadcastNullAwareLeftAntiHashJoinExec => j case j: SortMergeJoinExec => j } @@ -1155,7 +1154,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan // positive not in subquery case assertJoin(( "select * from testData where key not in (select a from testData2)", - classOf[BroadcastNullAwareLeftAntiHashJoinExec])) + classOf[BroadcastHashJoinExec])) // negative not in subquery case since multi-column is not supported assertJoin(( @@ -1167,7 +1166,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan // testData3.b nullable true assertJoin(( "select * from testData left anti join testData3 ON key = b or isnull(key = b)", - classOf[BroadcastNullAwareLeftAntiHashJoinExec])) + classOf[BroadcastHashJoinExec])) // negative hand-written left anti join // testData.key nullable false diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index 684b7a10eed85..940c0bb26e456 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, Sort} import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, FileSourceScanExec, InputAdapter, ReusedSubqueryExec, ScalarSubquery, SubqueryExec, WholeStageCodegenExec} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution} import org.apache.spark.sql.execution.datasources.FileScanRDD -import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, BroadcastNullAwareLeftAntiHashJoinExec} +import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, BroadcastNestedLoopJoinExec} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession @@ -1671,7 +1671,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1680,7 +1680,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark df = sql("select * from l where a not in (select c from r where d < 6.0)") checkAnswer(df, Seq.empty) if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1689,7 +1689,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark df = sql("select * from l where b = 5.0 and a not in (select c from r where c is not null)") checkAnswer(df, Seq.empty) if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1698,7 +1698,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark df = sql("select * from l where a = 6 and a not in (select c from r where c is not null)") checkAnswer(df, Seq.empty) if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1707,7 +1707,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark df = sql("select * from l where a = 1 and a not in (select c from r where c is not null)") checkAnswer(df, Row(1, 2.0) :: Row(1, 2.0) :: Nil) if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1725,7 +1725,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark checkAnswer(df, Row(null, 5.0) :: Nil) if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastNullAwareLeftAntiHashJoinExec]) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } From 4d26695d2a6569d9531a63bfc923d31d11e911fb Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Thu, 23 Jul 2020 19:59:25 +0800 Subject: [PATCH 14/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit fix UT regression caused by added field isNullAware in HashedRelationBroadcastMode Change-Id: I7c906c879eb86a41fd089155ff03b004b32d7692 --- .../sql-tests/results/explain-aqe.sql.out | 8 ++++---- .../sql-tests/results/explain.sql.out | 8 ++++---- .../sql/execution/debug/DebuggingSuite.scala | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out index 36757863ffcb5..79f4d48101ca4 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out @@ -308,7 +308,7 @@ Input [2]: [key#x, val#x] (7) BroadcastExchange Input [2]: [key#x, val#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (8) BroadcastHashJoin Left keys [1]: [key#x] @@ -362,7 +362,7 @@ Input [2]: [key#x, val#x] (5) BroadcastExchange Input [2]: [key#x, val#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (6) BroadcastHashJoin Left keys [1]: [key#x] @@ -533,7 +533,7 @@ Input [2]: [key#x, val#x] (7) BroadcastExchange Input [2]: [key#x, val#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (8) BroadcastHashJoin Left keys [1]: [key#x] @@ -643,7 +643,7 @@ Results [2]: [key#x, max(val#x)#x AS max(val)#x] (13) BroadcastExchange Input [2]: [key#x, max(val)#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (14) BroadcastHashJoin Left keys [1]: [key#x] diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out index 2b07dac0e5d0a..614cb2a137d01 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out @@ -316,7 +316,7 @@ Input [2]: [key#x, val#x] (9) BroadcastExchange Input [2]: [key#x, val#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (10) BroadcastHashJoin [codegen id : 2] Left keys [1]: [key#x] @@ -373,7 +373,7 @@ Input [2]: [key#x, val#x] (7) BroadcastExchange Input [2]: [key#x, val#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (8) BroadcastHashJoin [codegen id : 2] Left keys [1]: [key#x] @@ -771,7 +771,7 @@ Input [2]: [key#x, val#x] (9) BroadcastExchange Input [2]: [key#x, val#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (10) BroadcastHashJoin [codegen id : 2] Left keys [1]: [key#x] @@ -853,7 +853,7 @@ Results [2]: [key#x, max(val#x)#x AS max(val)#x] (10) BroadcastExchange Input [2]: [key#x, max(val)#x] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#x] (11) BroadcastHashJoin [codegen id : 4] Left keys [1]: [key#x] diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala index e9ef7c1a0c540..61c4196660b06 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala @@ -70,15 +70,15 @@ class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSui val output = captured.toString() assert(output.replaceAll("\\[id=#\\d+\\]", "[id=#x]").contains( - """== BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false])), [id=#x] == - |Tuples output: 0 - | id LongType: {} - |== WholeStageCodegen (1) == - |Tuples output: 10 - | id LongType: {java.lang.Long} - |== Range (0, 10, step=1, splits=2) == - |Tuples output: 0 - | id LongType: {}""".stripMargin)) +"""== BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [id=#x] == + |Tuples output: 0 + | id LongType: {} + |== WholeStageCodegen (1) == + |Tuples output: 10 + | id LongType: {java.lang.Long} + |== Range (0, 10, step=1, splits=2) == + |Tuples output: 0 + | id LongType: {}""".stripMargin)) } test("SPARK-28537: DebugExec cannot debug columnar related queries") { From 1bca27303145c24acca37695735e4ff414285777 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Fri, 24 Jul 2020 08:13:40 +0800 Subject: [PATCH 15/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. remove allNull api and usage since this PR should focus on single-column. 2. refine comments Change-Id: I679738460e4b24859ed4a9e362ebc7eeee57bded --- .../sql/catalyst/expressions/UnsafeRow.java | 9 -- .../sql/catalyst/planning/patterns.scala | 6 +- .../joins/BroadcastHashJoinExec.scala | 35 ++++---- .../spark/sql/execution/joins/HashJoin.scala | 6 +- .../sql/execution/joins/HashedRelation.scala | 87 +++++++++---------- 5 files changed, 62 insertions(+), 81 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java index 0217b2fb9b0a4..4dc5ce1de047b 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java @@ -591,15 +591,6 @@ public boolean anyNull() { return BitSetMethods.anySet(baseObject, baseOffset, bitSetWidthInBytes / 8); } - public boolean allNull() { - for (int i = 0; i < numFields; i++) { - if (!BitSetMethods.isSet(baseObject, baseOffset, i)) { - return false; - } - } - return true; - } - /** * Writes the content of this row into a memory address, identified by an object and an offset. * The target memory address must already been allocated, and have enough space to hold all the diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index eaa3cdfd39952..df200add5ebbc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -393,9 +393,11 @@ object PhysicalWindow { object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper with PredicateHelper { - // SingleColumn NullAwareAntiJoin + // TODO support multi column NULL-aware anti join in future. + // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6 + // multi-column null aware anti join is much more complicated than single column ones. + // streamedSideKeys, buildSideKeys - // currently these two return Seq[Expression] should have only one element private type ReturnType = (Seq[Expression], Seq[Expression]) /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 1da4a90d90457..c06b3ea4a8a70 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -51,9 +51,6 @@ case class BroadcastHashJoinExec( extends HashJoin with CodegenSupport { if (isNullAwareAntiJoin) { - // TODO support multi column NULL-aware anti join in future. - // See. http://www.vldb.org/pvldb/vol2/vldb09-423.pdf Section 6 - // multi-column null aware anti join is much more complicated than single column ones. require(leftKeys.length == 1, "leftKeys length should be 1") require(rightKeys.length == 1, "rightKeys length should be 1") require(joinType == LeftAnti, "joinType must be LeftAnti.") @@ -149,9 +146,9 @@ case class BroadcastHashJoinExec( streamedPlan.execute().mapPartitionsInternal { streamedIter => val hashed = broadcastRelation.value.asReadOnlyCopy() TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize) - if (hashed.isOriginInputEmpty) { + if (hashed.isOriginalInputEmpty) { streamedIter - } else if (hashed.allNullColumnKeyExistsInOriginInput) { + } else if (hashed.allNullColumnKeyExistsInOriginalInput) { Iterator.empty } else { val keyGenerator = UnsafeProjection.create( @@ -161,12 +158,11 @@ case class BroadcastHashJoinExec( ) streamedIter.filter(row => { val lookupKey: UnsafeRow = keyGenerator(row) - if (lookupKey.allNull()) { + // anyNull is equivalent to allNull since it's a single-column key. + if (lookupKey.anyNull()) { false } else { - // in isNullAware mode - // UnsafeHashedRelation include anyNull key, if match, dropped row - // Same as LongHashedRelation where lookupKey isNotNullAt(0) + // Anti Join: Drop the row on the streamed side if it is a match on the build hashed.get(lookupKey) == null } }) @@ -494,32 +490,33 @@ case class BroadcastHashJoinExec( val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) val (matched, checkCondition, _) = getJoinCondition(ctx, input) val numOutput = metricTerm(ctx, "numOutputRows") - val isLongHashedRelation = broadcastRelation.value.isInstanceOf[LongHashedRelation] - // fast stop if isOriginInputEmpty = true - // whether isNullAwareAntiJoin is true or false - // should accept all rows in streamedSide - if (broadcastRelation.value.isOriginInputEmpty) { + // fast stop if isOriginalInputEmpty = true, should accept all rows in streamedSide + if (broadcastRelation.value.isOriginalInputEmpty) { return s""" - |// Common Anti Join isOriginInputEmpty(true) accept all + |// Anti Join isOriginalInputEmpty(true) accept all |$numOutput.add(1); |${consume(ctx, input)} """.stripMargin } if (isNullAwareAntiJoin) { - if (broadcastRelation.value.allNullColumnKeyExistsInOriginInput) { + if (broadcastRelation.value.allNullColumnKeyExistsInOriginalInput) { return s""" - |// NAAJ isOriginInputEmpty(false) anyNullKeyExists(true) reject all + |// NAAJ + |// isOriginalInputEmpty(false) allNullColumnKeyExistsInOriginalInput(true) + |// reject all """.stripMargin } else { val found = ctx.freshName("found") return s""" - |// NAAJ isOriginInputEmpty(false) allNullColumnKeyExistsInOriginInput(false) + |// NAAJ + |// isOriginalInputEmpty(false) allNullColumnKeyExistsInOriginalInput(false) |boolean $found = false; |// generate join key for stream side |${keyEv.code} - |if (${ if (isLongHashedRelation) s"$anyNull" else s"${keyEv.value}.allNull()"}) { + |// anyNull is equivalent to allNull since it's a single-column key. + |if ($anyNull) { | $found = true; |} else { | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala index 8f6edbf769a0f..a043d2744b444 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala @@ -268,10 +268,8 @@ trait HashJoin extends BaseJoinExec { private def antiJoin( streamIter: Iterator[InternalRow], hashedRelation: HashedRelation): Iterator[InternalRow] = { - // fast stop if isOriginInputEmpty = true - // whether isNullAwareAntiJoin is true or false - // should accept all rows in streamedSide - if (hashedRelation.isOriginInputEmpty) { + // fast stop if isOriginalInputEmpty = true, should accept all rows in streamedSide + if (hashedRelation.isOriginalInputEmpty) { return streamIter } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 7013db45c9faf..aac2d4fbe6f1b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -84,13 +84,14 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { /** * Normally HashedRelation is built from an Source (input: Iterator[InternalRow]). - * This indicates the origin input is empty. + * This indicates the original input is empty. * Note that, the hashed relation can be empty despite the input being not empty, * since the hashed relation skips over null keys. */ - var isOriginInputEmpty: Boolean - def setOriginInputEmtpy(isOriginInputEmpty: Boolean): HashedRelation = { - this.isOriginInputEmpty = isOriginInputEmpty + var isOriginalInputEmpty: Boolean + + def setOriginInputEmtpy(isOriginalInputEmpty: Boolean): HashedRelation = { + this.isOriginalInputEmpty = isOriginalInputEmpty this } @@ -99,11 +100,12 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { * This will be set true if Source (input: Iterator[InternalRow]) contains a key, * which is allNullColumn. */ - var allNullColumnKeyExistsInOriginInput: Boolean - def setAllNullColumnKeyExistsInOriginInput( - allNullColumnKeyExistsInOriginInput: Boolean): HashedRelation = { - this.allNullColumnKeyExistsInOriginInput = - allNullColumnKeyExistsInOriginInput + var allNullColumnKeyExistsInOriginalInput: Boolean + + def setAllNullColumnKeyExistsInOriginalInput( + allNullColumnKeyExistsInOriginalInput: Boolean): HashedRelation = { + this.allNullColumnKeyExistsInOriginalInput = + allNullColumnKeyExistsInOriginalInput this } @@ -117,6 +119,7 @@ private[execution] object HashedRelation { /** * Create a HashedRelation from an Iterator of InternalRow. + * If isNullAware is true, UnsafeHashedRelation will store rows with null keys. */ def apply( input: Iterator[InternalRow], @@ -135,7 +138,7 @@ private[execution] object HashedRelation { } if (key.length == 1 && key.head.dataType == LongType) { - LongHashedRelation(input, key, sizeEstimate, mm, isNullAware) + LongHashedRelation(input, key, sizeEstimate, mm) } else { UnsafeHashedRelation(input, key, sizeEstimate, mm, isNullAware) } @@ -161,8 +164,8 @@ private[joins] class UnsafeHashedRelation( override def asReadOnlyCopy(): UnsafeHashedRelation = { new UnsafeHashedRelation(numKeys, numFields, binaryMap) - .setOriginInputEmtpy(this.isOriginInputEmpty) - .setAllNullColumnKeyExistsInOriginInput(this.allNullColumnKeyExistsInOriginInput) + .setOriginInputEmtpy(this.isOriginalInputEmpty) + .setAllNullColumnKeyExistsInOriginalInput(this.allNullColumnKeyExistsInOriginalInput) .asInstanceOf[UnsafeHashedRelation] } @@ -246,8 +249,8 @@ private[joins] class UnsafeHashedRelation( writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit) : Unit = { writeInt(numFields) - writeBoolean(isOriginInputEmpty) - writeBoolean(allNullColumnKeyExistsInOriginInput) + writeBoolean(isOriginalInputEmpty) + writeBoolean(allNullColumnKeyExistsInOriginalInput) // TODO: move these into BytesToBytesMap writeLong(binaryMap.numKeys()) writeLong(binaryMap.numValues()) @@ -282,8 +285,8 @@ private[joins] class UnsafeHashedRelation( readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { numFields = readInt() - isOriginInputEmpty = readBoolean() - allNullColumnKeyExistsInOriginInput = readBoolean() + isOriginalInputEmpty = readBoolean() + allNullColumnKeyExistsInOriginalInput = readBoolean() resultRow = new UnsafeRow(numFields) val nKeys = readLong() val nValues = readLong() @@ -339,8 +342,8 @@ private[joins] class UnsafeHashedRelation( read(() => in.readBoolean(), () => in.readInt(), () => in.readLong(), in.readBytes) } - override var isOriginInputEmpty: Boolean = _ - override var allNullColumnKeyExistsInOriginInput: Boolean = _ + override var isOriginalInputEmpty: Boolean = _ + override var allNullColumnKeyExistsInOriginalInput: Boolean = _ } private[joins] object UnsafeHashedRelation { @@ -372,17 +375,16 @@ private[joins] object UnsafeHashedRelation { val keyGenerator = UnsafeProjection.create(key) var numFields = 0 val numKeys = key.length - val isOriginInputEmpty = !input.hasNext - var allNullColumnKeyExistsInOriginInput: Boolean = false + val isOriginalInputEmpty = !input.hasNext + var allNullColumnKeyExistsInOriginalInput: Boolean = false while (input.hasNext) { val row = input.next().asInstanceOf[UnsafeRow] numFields = row.numFields() val key = keyGenerator(row) if ((0 until numKeys).forall(key.isNullAt)) { - allNullColumnKeyExistsInOriginInput = true + allNullColumnKeyExistsInOriginalInput = true } - // TODO keep anyNull key for multi column NAAJ support in future if (isNullAware || (!isNullAware && !key.anyNull)) { val loc = binaryMap.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes) val success = loc.append( @@ -398,8 +400,8 @@ private[joins] object UnsafeHashedRelation { } new UnsafeHashedRelation(key.size, numFields, binaryMap) - .setOriginInputEmtpy(isOriginInputEmpty) - .setAllNullColumnKeyExistsInOriginInput(allNullColumnKeyExistsInOriginInput) + .setOriginInputEmtpy(isOriginalInputEmpty) + .setAllNullColumnKeyExistsInOriginalInput(allNullColumnKeyExistsInOriginalInput) } } @@ -894,8 +896,8 @@ class LongHashedRelation( override def asReadOnlyCopy(): LongHashedRelation = new LongHashedRelation(nFields, map) - .setOriginInputEmtpy(this.isOriginInputEmpty) - .setAllNullColumnKeyExistsInOriginInput(this.allNullColumnKeyExistsInOriginInput) + .setOriginInputEmtpy(this.isOriginalInputEmpty) + .setAllNullColumnKeyExistsInOriginalInput(this.allNullColumnKeyExistsInOriginalInput) .asInstanceOf[LongHashedRelation] override def estimatedSize: Long = map.getTotalMemoryConsumption @@ -928,16 +930,16 @@ class LongHashedRelation( override def writeExternal(out: ObjectOutput): Unit = { out.writeInt(nFields) - out.writeBoolean(isOriginInputEmpty) - out.writeBoolean(allNullColumnKeyExistsInOriginInput) + out.writeBoolean(isOriginalInputEmpty) + out.writeBoolean(allNullColumnKeyExistsInOriginalInput) out.writeObject(map) } override def readExternal(in: ObjectInput): Unit = { nFields = in.readInt() resultRow = new UnsafeRow(nFields) - isOriginInputEmpty = in.readBoolean() - allNullColumnKeyExistsInOriginInput = in.readBoolean() + isOriginalInputEmpty = in.readBoolean() + allNullColumnKeyExistsInOriginalInput = in.readBoolean() map = in.readObject().asInstanceOf[LongToUnsafeRowMap] } @@ -946,8 +948,8 @@ class LongHashedRelation( */ override def keys(): Iterator[InternalRow] = map.keys() - override var isOriginInputEmpty: Boolean = _ - override var allNullColumnKeyExistsInOriginInput: Boolean = _ + override var isOriginalInputEmpty: Boolean = _ + override var allNullColumnKeyExistsInOriginalInput: Boolean = _ } /** @@ -959,23 +961,14 @@ private[joins] object LongHashedRelation { key: Seq[Expression], sizeEstimate: Int, taskMemoryManager: TaskMemoryManager): LongHashedRelation = { - apply(input, key, sizeEstimate, taskMemoryManager, isNullAware = false) - } - - def apply( - input: Iterator[InternalRow], - key: Seq[Expression], - sizeEstimate: Int, - taskMemoryManager: TaskMemoryManager, - isNullAware: Boolean): LongHashedRelation = { val map = new LongToUnsafeRowMap(taskMemoryManager, sizeEstimate) val keyGenerator = UnsafeProjection.create(key) // Create a mapping of key -> rows var numFields = 0 - val isOriginInputEmpty: Boolean = !input.hasNext - var allNullColumnKeyExistsInOriginInput: Boolean = false + val isOriginalInputEmpty: Boolean = !input.hasNext + var allNullColumnKeyExistsInOriginalInput: Boolean = false while (input.hasNext) { val unsafeRow = input.next().asInstanceOf[UnsafeRow] numFields = unsafeRow.numFields() @@ -986,14 +979,14 @@ private[joins] object LongHashedRelation { map.append(key, unsafeRow) } else { // LongHashedRelation is single-column key - // rowKey.isNullAt(0) equivalent to allNullColumnKeyExistsInOriginInput - allNullColumnKeyExistsInOriginInput = true + // rowKey.isNullAt(0) equivalent to allNullColumnKeyExistsInOriginalInput + allNullColumnKeyExistsInOriginalInput = true } } map.optimize() new LongHashedRelation(numFields, map) - .setOriginInputEmtpy(isOriginInputEmpty) - .setAllNullColumnKeyExistsInOriginInput(allNullColumnKeyExistsInOriginInput) + .setOriginInputEmtpy(isOriginalInputEmpty) + .setAllNullColumnKeyExistsInOriginalInput(allNullColumnKeyExistsInOriginalInput) .asInstanceOf[LongHashedRelation] } } From f2cc24a2e1e66f66f2b82a7fa448ff989c30f900 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Fri, 24 Jul 2020 09:47:30 +0800 Subject: [PATCH 16/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. setOriginInputEmtpy -> setOriginalInputEmtpy. 2. remove unnecessary code while isNullWare is false. Change-Id: I2437b590d4d7f38bfb8e3d611ad7b18a431bfc2d --- .../sql/execution/joins/HashedRelation.scala | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index aac2d4fbe6f1b..50332c53e8347 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -90,7 +90,7 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { */ var isOriginalInputEmpty: Boolean - def setOriginInputEmtpy(isOriginalInputEmpty: Boolean): HashedRelation = { + def setOriginalInputEmtpy(isOriginalInputEmpty: Boolean): HashedRelation = { this.isOriginalInputEmpty = isOriginalInputEmpty this } @@ -164,7 +164,7 @@ private[joins] class UnsafeHashedRelation( override def asReadOnlyCopy(): UnsafeHashedRelation = { new UnsafeHashedRelation(numKeys, numFields, binaryMap) - .setOriginInputEmtpy(this.isOriginalInputEmpty) + .setOriginalInputEmtpy(this.isOriginalInputEmpty) .setAllNullColumnKeyExistsInOriginalInput(this.allNullColumnKeyExistsInOriginalInput) .asInstanceOf[UnsafeHashedRelation] } @@ -381,11 +381,13 @@ private[joins] object UnsafeHashedRelation { val row = input.next().asInstanceOf[UnsafeRow] numFields = row.numFields() val key = keyGenerator(row) - if ((0 until numKeys).forall(key.isNullAt)) { + if (isNullAware && + !allNullColumnKeyExistsInOriginalInput && + (0 until numKeys).forall(key.isNullAt)) { allNullColumnKeyExistsInOriginalInput = true } - if (isNullAware || (!isNullAware && !key.anyNull)) { + if (isNullAware || !key.anyNull) { val loc = binaryMap.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes) val success = loc.append( key.getBaseObject, key.getBaseOffset, key.getSizeInBytes, @@ -400,7 +402,7 @@ private[joins] object UnsafeHashedRelation { } new UnsafeHashedRelation(key.size, numFields, binaryMap) - .setOriginInputEmtpy(isOriginalInputEmpty) + .setOriginalInputEmtpy(isOriginalInputEmpty) .setAllNullColumnKeyExistsInOriginalInput(allNullColumnKeyExistsInOriginalInput) } } @@ -896,7 +898,7 @@ class LongHashedRelation( override def asReadOnlyCopy(): LongHashedRelation = new LongHashedRelation(nFields, map) - .setOriginInputEmtpy(this.isOriginalInputEmpty) + .setOriginalInputEmtpy(this.isOriginalInputEmpty) .setAllNullColumnKeyExistsInOriginalInput(this.allNullColumnKeyExistsInOriginalInput) .asInstanceOf[LongHashedRelation] @@ -977,15 +979,15 @@ private[joins] object LongHashedRelation { // LongToUnsafeRowMap can't insert null key val key = rowKey.getLong(0) map.append(key, unsafeRow) - } else { - // LongHashedRelation is single-column key + } else if (!allNullColumnKeyExistsInOriginalInput) { + // LongHashedRelation stores single-column key // rowKey.isNullAt(0) equivalent to allNullColumnKeyExistsInOriginalInput allNullColumnKeyExistsInOriginalInput = true } } map.optimize() new LongHashedRelation(numFields, map) - .setOriginInputEmtpy(isOriginalInputEmpty) + .setOriginalInputEmtpy(isOriginalInputEmpty) .setAllNullColumnKeyExistsInOriginalInput(allNullColumnKeyExistsInOriginalInput) .asInstanceOf[LongHashedRelation] } From cdc04b26ef0ef10546f63a8599e7e4f42767ba80 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Sat, 25 Jul 2020 04:34:56 +0800 Subject: [PATCH 17/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize some code comments refined. Change-Id: I669f1d9e468836ba71de67d6355051a3fdbdf072 --- .../sql/execution/joins/BroadcastHashJoinExec.scala | 2 -- .../spark/sql/execution/joins/HashedRelation.scala | 11 +++-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index c06b3ea4a8a70..d30161171c8f2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -158,7 +158,6 @@ case class BroadcastHashJoinExec( ) streamedIter.filter(row => { val lookupKey: UnsafeRow = keyGenerator(row) - // anyNull is equivalent to allNull since it's a single-column key. if (lookupKey.anyNull()) { false } else { @@ -515,7 +514,6 @@ case class BroadcastHashJoinExec( |boolean $found = false; |// generate join key for stream side |${keyEv.code} - |// anyNull is equivalent to allNull since it's a single-column key. |if ($anyNull) { | $found = true; |} else { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 50332c53e8347..660a130835385 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -98,14 +98,13 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { /** * It's only used in null aware anti join. * This will be set true if Source (input: Iterator[InternalRow]) contains a key, - * which is allNullColumn. + * which has all null columns. */ var allNullColumnKeyExistsInOriginalInput: Boolean def setAllNullColumnKeyExistsInOriginalInput( allNullColumnKeyExistsInOriginalInput: Boolean): HashedRelation = { - this.allNullColumnKeyExistsInOriginalInput = - allNullColumnKeyExistsInOriginalInput + this.allNullColumnKeyExistsInOriginalInput = allNullColumnKeyExistsInOriginalInput this } @@ -374,7 +373,6 @@ private[joins] object UnsafeHashedRelation { // Create a mapping of buildKeys -> rows val keyGenerator = UnsafeProjection.create(key) var numFields = 0 - val numKeys = key.length val isOriginalInputEmpty = !input.hasNext var allNullColumnKeyExistsInOriginalInput: Boolean = false while (input.hasNext) { @@ -383,7 +381,7 @@ private[joins] object UnsafeHashedRelation { val key = keyGenerator(row) if (isNullAware && !allNullColumnKeyExistsInOriginalInput && - (0 until numKeys).forall(key.isNullAt)) { + key.isNullAt(0)) { allNullColumnKeyExistsInOriginalInput = true } @@ -976,12 +974,9 @@ private[joins] object LongHashedRelation { numFields = unsafeRow.numFields() val rowKey = keyGenerator(unsafeRow) if (!rowKey.isNullAt(0)) { - // LongToUnsafeRowMap can't insert null key val key = rowKey.getLong(0) map.append(key, unsafeRow) } else if (!allNullColumnKeyExistsInOriginalInput) { - // LongHashedRelation stores single-column key - // rowKey.isNullAt(0) equivalent to allNullColumnKeyExistsInOriginalInput allNullColumnKeyExistsInOriginalInput = true } } From 19de983a8b2429cee607177b8eb4e52a0a767ba6 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 27 Jul 2020 13:35:35 +0800 Subject: [PATCH 18/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. comments refine. 2. make spark.sql.nullAwareAntiJoin.optimize.enabled default true. Change-Id: I5a4967b2a1f3269c7c6485a9de152acb31165891 --- .../apache/spark/sql/catalyst/planning/patterns.scala | 10 +++++----- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index df200add5ebbc..6584790069e60 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -405,15 +405,15 @@ object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper with Pre * LeftAnti(condition: Or(EqualTo(a=b), IsNull(EqualTo(a=b))) * will almost certainly be planned as a Broadcast Nested Loop join, * which is very time consuming because it's an O(M*N) calculation. - * But if it's a single column case, and buildSide data is small enough, - * O(M*N) calculation could be optimized into O(M) using hash lookup instead of loop lookup. + * But if it's a single column case O(M*N) calculation could be optimized into O(M) + * using hash lookup instead of loop lookup. */ def unapply(join: Join): Option[ReturnType] = join match { case Join(left, right, LeftAnti, - Some(Or(EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), - IsNull(EqualTo(tmpLeft: AttributeReference, tmpRight: AttributeReference)))), _) + Some(Or(e @ EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), + IsNull(e2 @ EqualTo(_, _)))), _) if SQLConf.get.nullAwareAntiJoinOptimizeEnabled && - leftAttr.semanticEquals(tmpLeft) && rightAttr.semanticEquals(tmpRight) => + e.semanticEquals(e2) => if (canEvaluate(leftAttr, left) && canEvaluate(rightAttr, right)) { Some(Seq(leftAttr), Seq(rightAttr)) } else if (canEvaluate(leftAttr, right) && canEvaluate(rightAttr, left)) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index de114a0fbddad..6047f1e52e3b1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2676,12 +2676,12 @@ object SQLConf { buildConf("spark.sql.nullAwareAntiJoin.optimize.enabled") .internal() .doc("When true, NULL-aware anti join execution will be planed into " + - "BroadcastJoinExec with flag isNullAwareAntiJoin enabled, " + + "BroadcastHashJoinExec with flag isNullAwareAntiJoin enabled, " + "optimized from O(M*N) calculation into O(M) calculation " + "using Hash lookup instead of Looping lookup." + "Only support for singleColumn NAAJ for now.") .booleanConf - .createWithDefault(false) + .createWithDefault(true) /** * Holds information about keys that have been deprecated. From 36b024c6faa6067f767575daf2f9d0caf2c581ec Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 27 Jul 2020 16:24:41 +0800 Subject: [PATCH 19/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. remove isOriginalInputEmpty and allNullColumnKeyExistsInOriginalInput. 2. added two more HashedRelation implementation EmptyHashedRelation and EmptyHashedRelationWithAllNullKeys. Change-Id: I91480b80a4106f5fce3c188d563a8f952a0ae36d --- .../joins/BroadcastHashJoinExec.scala | 24 ++- .../spark/sql/execution/joins/HashJoin.scala | 5 - .../sql/execution/joins/HashedRelation.scala | 149 +++++++++--------- .../execution/joins/HashedRelationSuite.scala | 1 + 4 files changed, 83 insertions(+), 96 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index d30161171c8f2..e8f77e5a912cb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -146,9 +146,9 @@ case class BroadcastHashJoinExec( streamedPlan.execute().mapPartitionsInternal { streamedIter => val hashed = broadcastRelation.value.asReadOnlyCopy() TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize) - if (hashed.isOriginalInputEmpty) { + if (hashed.isInstanceOf[EmptyHashedRelation]) { streamedIter - } else if (hashed.allNullColumnKeyExistsInOriginalInput) { + } else if (hashed.isInstanceOf[EmptyHashedRelationWithAllNullKeys]) { Iterator.empty } else { val keyGenerator = UnsafeProjection.create( @@ -490,27 +490,23 @@ case class BroadcastHashJoinExec( val (matched, checkCondition, _) = getJoinCondition(ctx, input) val numOutput = metricTerm(ctx, "numOutputRows") - // fast stop if isOriginalInputEmpty = true, should accept all rows in streamedSide - if (broadcastRelation.value.isOriginalInputEmpty) { - return s""" - |// Anti Join isOriginalInputEmpty(true) accept all - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } - if (isNullAwareAntiJoin) { - if (broadcastRelation.value.allNullColumnKeyExistsInOriginalInput) { + if (broadcastRelation.value.isInstanceOf[EmptyHashedRelation]) { + return s""" + |// NAAJ Join EmptyHashedRelation accept all + |$numOutput.add(1); + |${consume(ctx, input)} + """.stripMargin + } else if (broadcastRelation.value.isInstanceOf[EmptyHashedRelationWithAllNullKeys]) { return s""" |// NAAJ - |// isOriginalInputEmpty(false) allNullColumnKeyExistsInOriginalInput(true) + |// EmptyHashedRelationWithAllNullKeys |// reject all """.stripMargin } else { val found = ctx.freshName("found") return s""" |// NAAJ - |// isOriginalInputEmpty(false) allNullColumnKeyExistsInOriginalInput(false) |boolean $found = false; |// generate join key for stream side |${keyEv.code} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala index a043d2744b444..4f22007b65845 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala @@ -268,11 +268,6 @@ trait HashJoin extends BaseJoinExec { private def antiJoin( streamIter: Iterator[InternalRow], hashedRelation: HashedRelation): Iterator[InternalRow] = { - // fast stop if isOriginalInputEmpty = true, should accept all rows in streamedSide - if (hashedRelation.isOriginalInputEmpty) { - return streamIter - } - val joinKeys = streamSideKeyGenerator() val joinedRow = new JoinedRow diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 660a130835385..557413e248a93 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -81,33 +81,6 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation { */ def asReadOnlyCopy(): HashedRelation - - /** - * Normally HashedRelation is built from an Source (input: Iterator[InternalRow]). - * This indicates the original input is empty. - * Note that, the hashed relation can be empty despite the input being not empty, - * since the hashed relation skips over null keys. - */ - var isOriginalInputEmpty: Boolean - - def setOriginalInputEmtpy(isOriginalInputEmpty: Boolean): HashedRelation = { - this.isOriginalInputEmpty = isOriginalInputEmpty - this - } - - /** - * It's only used in null aware anti join. - * This will be set true if Source (input: Iterator[InternalRow]) contains a key, - * which has all null columns. - */ - var allNullColumnKeyExistsInOriginalInput: Boolean - - def setAllNullColumnKeyExistsInOriginalInput( - allNullColumnKeyExistsInOriginalInput: Boolean): HashedRelation = { - this.allNullColumnKeyExistsInOriginalInput = allNullColumnKeyExistsInOriginalInput - this - } - /** * Release any used resources. */ @@ -118,7 +91,6 @@ private[execution] object HashedRelation { /** * Create a HashedRelation from an Iterator of InternalRow. - * If isNullAware is true, UnsafeHashedRelation will store rows with null keys. */ def apply( input: Iterator[InternalRow], @@ -136,8 +108,10 @@ private[execution] object HashedRelation { 0) } - if (key.length == 1 && key.head.dataType == LongType) { - LongHashedRelation(input, key, sizeEstimate, mm) + if (isNullAware && !input.hasNext) { + new EmptyHashedRelation + } else if (key.length == 1 && key.head.dataType == LongType) { + LongHashedRelation(input, key, sizeEstimate, mm, isNullAware) } else { UnsafeHashedRelation(input, key, sizeEstimate, mm, isNullAware) } @@ -163,9 +137,6 @@ private[joins] class UnsafeHashedRelation( override def asReadOnlyCopy(): UnsafeHashedRelation = { new UnsafeHashedRelation(numKeys, numFields, binaryMap) - .setOriginalInputEmtpy(this.isOriginalInputEmpty) - .setAllNullColumnKeyExistsInOriginalInput(this.allNullColumnKeyExistsInOriginalInput) - .asInstanceOf[UnsafeHashedRelation] } override def estimatedSize: Long = binaryMap.getTotalMemoryConsumption @@ -235,21 +206,18 @@ private[joins] class UnsafeHashedRelation( } override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException { - write(out.writeBoolean, out.writeInt, out.writeLong, out.write) + write(out.writeInt, out.writeLong, out.write) } override def write(kryo: Kryo, out: Output): Unit = Utils.tryOrIOException { - write(out.writeBoolean, out.writeInt, out.writeLong, out.write) + write(out.writeInt, out.writeLong, out.write) } private def write( - writeBoolean: (Boolean) => Unit, writeInt: (Int) => Unit, writeLong: (Long) => Unit, writeBuffer: (Array[Byte], Int, Int) => Unit) : Unit = { writeInt(numFields) - writeBoolean(isOriginalInputEmpty) - writeBoolean(allNullColumnKeyExistsInOriginalInput) // TODO: move these into BytesToBytesMap writeLong(binaryMap.numKeys()) writeLong(binaryMap.numValues()) @@ -275,17 +243,14 @@ private[joins] class UnsafeHashedRelation( } override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException { - read(() => in.readBoolean(), () => in.readInt(), () => in.readLong(), in.readFully) + read(() => in.readInt(), () => in.readLong(), in.readFully) } private def read( - readBoolean: () => Boolean, readInt: () => Int, readLong: () => Long, readBuffer: (Array[Byte], Int, Int) => Unit): Unit = { numFields = readInt() - isOriginalInputEmpty = readBoolean() - allNullColumnKeyExistsInOriginalInput = readBoolean() resultRow = new UnsafeRow(numFields) val nKeys = readLong() val nValues = readLong() @@ -338,11 +303,8 @@ private[joins] class UnsafeHashedRelation( } override def read(kryo: Kryo, in: Input): Unit = Utils.tryOrIOException { - read(() => in.readBoolean(), () => in.readInt(), () => in.readLong(), in.readBytes) + read(() => in.readInt(), () => in.readLong(), in.readBytes) } - - override var isOriginalInputEmpty: Boolean = _ - override var allNullColumnKeyExistsInOriginalInput: Boolean = _ } private[joins] object UnsafeHashedRelation { @@ -373,19 +335,11 @@ private[joins] object UnsafeHashedRelation { // Create a mapping of buildKeys -> rows val keyGenerator = UnsafeProjection.create(key) var numFields = 0 - val isOriginalInputEmpty = !input.hasNext - var allNullColumnKeyExistsInOriginalInput: Boolean = false while (input.hasNext) { val row = input.next().asInstanceOf[UnsafeRow] numFields = row.numFields() val key = keyGenerator(row) - if (isNullAware && - !allNullColumnKeyExistsInOriginalInput && - key.isNullAt(0)) { - allNullColumnKeyExistsInOriginalInput = true - } - - if (isNullAware || !key.anyNull) { + if (!key.anyNull) { val loc = binaryMap.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes) val success = loc.append( key.getBaseObject, key.getBaseOffset, key.getSizeInBytes, @@ -396,12 +350,12 @@ private[joins] object UnsafeHashedRelation { throw new SparkOutOfMemoryError("There is not enough memory to build hash map") // scalastyle:on throwerror } + } else if (isNullAware) { + return new EmptyHashedRelationWithAllNullKeys } } new UnsafeHashedRelation(key.size, numFields, binaryMap) - .setOriginalInputEmtpy(isOriginalInputEmpty) - .setAllNullColumnKeyExistsInOriginalInput(allNullColumnKeyExistsInOriginalInput) } } @@ -894,11 +848,7 @@ class LongHashedRelation( // Needed for serialization (it is public to make Java serialization work) def this() = this(0, null) - override def asReadOnlyCopy(): LongHashedRelation = - new LongHashedRelation(nFields, map) - .setOriginalInputEmtpy(this.isOriginalInputEmpty) - .setAllNullColumnKeyExistsInOriginalInput(this.allNullColumnKeyExistsInOriginalInput) - .asInstanceOf[LongHashedRelation] + override def asReadOnlyCopy(): LongHashedRelation = new LongHashedRelation(nFields, map) override def estimatedSize: Long = map.getTotalMemoryConsumption @@ -930,16 +880,12 @@ class LongHashedRelation( override def writeExternal(out: ObjectOutput): Unit = { out.writeInt(nFields) - out.writeBoolean(isOriginalInputEmpty) - out.writeBoolean(allNullColumnKeyExistsInOriginalInput) out.writeObject(map) } override def readExternal(in: ObjectInput): Unit = { nFields = in.readInt() resultRow = new UnsafeRow(nFields) - isOriginalInputEmpty = in.readBoolean() - allNullColumnKeyExistsInOriginalInput = in.readBoolean() map = in.readObject().asInstanceOf[LongToUnsafeRowMap] } @@ -947,9 +893,6 @@ class LongHashedRelation( * Returns an iterator for keys of InternalRow type. */ override def keys(): Iterator[InternalRow] = map.keys() - - override var isOriginalInputEmpty: Boolean = _ - override var allNullColumnKeyExistsInOriginalInput: Boolean = _ } /** @@ -960,15 +903,22 @@ private[joins] object LongHashedRelation { input: Iterator[InternalRow], key: Seq[Expression], sizeEstimate: Int, - taskMemoryManager: TaskMemoryManager): LongHashedRelation = { + taskMemoryManager: TaskMemoryManager): HashedRelation = { + apply(input, key, sizeEstimate, taskMemoryManager, false) + } + + def apply( + input: Iterator[InternalRow], + key: Seq[Expression], + sizeEstimate: Int, + taskMemoryManager: TaskMemoryManager, + isNullAware: Boolean = false): HashedRelation = { val map = new LongToUnsafeRowMap(taskMemoryManager, sizeEstimate) val keyGenerator = UnsafeProjection.create(key) // Create a mapping of key -> rows var numFields = 0 - val isOriginalInputEmpty: Boolean = !input.hasNext - var allNullColumnKeyExistsInOriginalInput: Boolean = false while (input.hasNext) { val unsafeRow = input.next().asInstanceOf[UnsafeRow] numFields = unsafeRow.numFields() @@ -976,18 +926,63 @@ private[joins] object LongHashedRelation { if (!rowKey.isNullAt(0)) { val key = rowKey.getLong(0) map.append(key, unsafeRow) - } else if (!allNullColumnKeyExistsInOriginalInput) { - allNullColumnKeyExistsInOriginalInput = true + } else if (isNullAware) { + return new EmptyHashedRelationWithAllNullKeys } } map.optimize() new LongHashedRelation(numFields, map) - .setOriginalInputEmtpy(isOriginalInputEmpty) - .setAllNullColumnKeyExistsInOriginalInput(allNullColumnKeyExistsInOriginalInput) - .asInstanceOf[LongHashedRelation] } } +/** + * A special HashedRelation indicates it built from a empty input:Iterator[InternalRow]. + */ +class EmptyHashedRelation extends HashedRelation with Externalizable { + override def get(key: InternalRow): Iterator[InternalRow] = null + + override def getValue(key: InternalRow): InternalRow = null + + override def keyIsUnique: Boolean = true + + override def keys(): Iterator[InternalRow] = null + + override def asReadOnlyCopy(): EmptyHashedRelation = new EmptyHashedRelation + + override def close(): Unit = {} + + override def writeExternal(out: ObjectOutput): Unit = {} + + override def readExternal(in: ObjectInput): Unit = {} + + override def estimatedSize: Long = 0 +} + +/** + * A special HashedRelation indicates it build from a non-empty input:Iterator[InternalRow], + * which contains all null columns key. + */ +class EmptyHashedRelationWithAllNullKeys extends HashedRelation with Externalizable { + override def get(key: InternalRow): Iterator[InternalRow] = null + + override def getValue(key: InternalRow): InternalRow = null + + override def keyIsUnique: Boolean = true + + override def keys(): Iterator[InternalRow] = null + + override def asReadOnlyCopy(): EmptyHashedRelationWithAllNullKeys = + new EmptyHashedRelationWithAllNullKeys + + override def close(): Unit = {} + + override def writeExternal(out: ObjectOutput): Unit = {} + + override def readExternal(in: ObjectInput): Unit = {} + + override def estimatedSize: Long = 0 +} + /** The HashedRelationBroadcastMode requires that rows are broadcasted as a HashedRelation. */ case class HashedRelationBroadcastMode(key: Seq[Expression], isNullAware: Boolean = false) extends BroadcastMode { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala index 3526aa254c280..21ee88f0d7426 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala @@ -144,6 +144,7 @@ class HashedRelationSuite extends SharedSparkSession { } val longRelation2 = LongHashedRelation(rows.iterator ++ rows.iterator, key, 100, mm) + .asInstanceOf[LongHashedRelation] assert(!longRelation2.keyIsUnique) (0 until 100).foreach { i => val rows = longRelation2.get(i).toArray From 5bfec224e4b37abc99ec13d117427bf3211d075f Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 27 Jul 2020 20:33:58 +0800 Subject: [PATCH 20/24] 1. extract common trait NullAwareHashedRelation. 2. remove NullAwareAntiJoinSQLQueryTestSuite and using configDim. 3. Suite code refine. Change-Id: I5873a80004f0dfbae6ba3bbe06545bf672c987b9 --- .../joins/BroadcastHashJoinExec.scala | 7 +- .../sql/execution/joins/HashedRelation.scala | 51 ++--- .../sql-tests/inputs/group-by-filter.sql | 3 + .../inputs/subquery/in-subquery/in-basic.sql | 3 + .../inputs/subquery/in-subquery/in-having.sql | 3 + .../inputs/subquery/in-subquery/in-joins.sql | 3 + .../inputs/subquery/in-subquery/in-limit.sql | 3 + .../in-subquery/in-multiple-columns.sql | 3 + .../subquery/in-subquery/in-order-by.sql | 3 + .../subquery/in-subquery/in-with-cte.sql | 3 + .../subquery/in-subquery/nested-not-in.sql | 3 + .../subquery/in-subquery/not-in-joins.sql | 3 + ...not-in-unit-tests-multi-column-literal.sql | 3 + .../not-in-unit-tests-multi-column.sql | 3 + ...ot-in-unit-tests-single-column-literal.sql | 3 + .../not-in-unit-tests-single-column.sql | 3 + .../inputs/subquery/in-subquery/simple-in.sql | 3 + .../org/apache/spark/sql/JoinSuite.scala | 10 +- .../NullAwareAntiJoinSQLQueryTestSuite.scala | 64 ------- .../org/apache/spark/sql/SubquerySuite.scala | 178 +++++++++--------- .../sql/execution/debug/DebuggingSuite.scala | 19 +- 21 files changed, 169 insertions(+), 205 deletions(-) delete mode 100644 sql/core/src/test/scala/org/apache/spark/sql/NullAwareAntiJoinSQLQueryTestSuite.scala diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index e8f77e5a912cb..71046fc96a07f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -493,20 +493,17 @@ case class BroadcastHashJoinExec( if (isNullAwareAntiJoin) { if (broadcastRelation.value.isInstanceOf[EmptyHashedRelation]) { return s""" - |// NAAJ Join EmptyHashedRelation accept all + |// If the right side is empty, NAAJ simply returns the left side. |$numOutput.add(1); |${consume(ctx, input)} """.stripMargin } else if (broadcastRelation.value.isInstanceOf[EmptyHashedRelationWithAllNullKeys]) { return s""" - |// NAAJ - |// EmptyHashedRelationWithAllNullKeys - |// reject all + |// If the right side contains any all-null key, NAAJ simply returns Nil. """.stripMargin } else { val found = ctx.freshName("found") return s""" - |// NAAJ |boolean $found = false; |// generate join key for stream side |${keyEv.code} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 557413e248a93..bb059cd0d4e60 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -309,14 +309,6 @@ private[joins] class UnsafeHashedRelation( private[joins] object UnsafeHashedRelation { - def apply( - input: Iterator[InternalRow], - key: Seq[Expression], - sizeEstimate: Int, - taskMemoryManager: TaskMemoryManager): HashedRelation = { - apply(input, key, sizeEstimate, taskMemoryManager, isNullAware = false) - } - def apply( input: Iterator[InternalRow], key: Seq[Expression], @@ -899,14 +891,6 @@ class LongHashedRelation( * Create hashed relation with key that is long. */ private[joins] object LongHashedRelation { - def apply( - input: Iterator[InternalRow], - key: Seq[Expression], - sizeEstimate: Int, - taskMemoryManager: TaskMemoryManager): HashedRelation = { - apply(input, key, sizeEstimate, taskMemoryManager, false) - } - def apply( input: Iterator[InternalRow], key: Seq[Expression], @@ -936,9 +920,11 @@ private[joins] object LongHashedRelation { } /** - * A special HashedRelation indicates it built from a empty input:Iterator[InternalRow]. + * Common trait with dummy implementation for NAAJ special HashedRelation + * EmptyHashedRelation + * EmptyHashedRelationWithAllNullKeys */ -class EmptyHashedRelation extends HashedRelation with Externalizable { +trait NullAwareHashedRelation extends HashedRelation with Externalizable { override def get(key: InternalRow): Iterator[InternalRow] = null override def getValue(key: InternalRow): InternalRow = null @@ -947,8 +933,6 @@ class EmptyHashedRelation extends HashedRelation with Externalizable { override def keys(): Iterator[InternalRow] = null - override def asReadOnlyCopy(): EmptyHashedRelation = new EmptyHashedRelation - override def close(): Unit = {} override def writeExternal(out: ObjectOutput): Unit = {} @@ -959,28 +943,19 @@ class EmptyHashedRelation extends HashedRelation with Externalizable { } /** - * A special HashedRelation indicates it build from a non-empty input:Iterator[InternalRow], - * which contains all null columns key. + * A special HashedRelation indicates it built from a empty input:Iterator[InternalRow]. */ -class EmptyHashedRelationWithAllNullKeys extends HashedRelation with Externalizable { - override def get(key: InternalRow): Iterator[InternalRow] = null - - override def getValue(key: InternalRow): InternalRow = null - - override def keyIsUnique: Boolean = true - - override def keys(): Iterator[InternalRow] = null +class EmptyHashedRelation extends NullAwareHashedRelation { + override def asReadOnlyCopy(): EmptyHashedRelation = new EmptyHashedRelation +} +/** + * A special HashedRelation indicates it built from a non-empty input:Iterator[InternalRow], + * which contains all null columns key. + */ +class EmptyHashedRelationWithAllNullKeys extends NullAwareHashedRelation { override def asReadOnlyCopy(): EmptyHashedRelationWithAllNullKeys = new EmptyHashedRelationWithAllNullKeys - - override def close(): Unit = {} - - override def writeExternal(out: ObjectOutput): Unit = {} - - override def readExternal(in: ObjectInput): Unit = {} - - override def estimatedSize: Long = 0 } /** The HashedRelationBroadcastMode requires that rows are broadcasted as a HashedRelation. */ diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql index beb5b9e5fe516..97ab00103ca69 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql @@ -1,5 +1,8 @@ -- Test filter clause for aggregate expression. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + -- Test data. CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES (1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2), (null, 1), (3, null), (null, null) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql index f4ffc20086386..be1183d13e7bf 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql @@ -1,3 +1,6 @@ +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view tab_a as select * from values (1, 1) as tab_a(a1, b1); create temporary view tab_b as select * from values (1, 1) as tab_b(a2, b2); create temporary view struct_tab as select struct(col1 as a, col2 as b) as record from diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql index 8f98ae1155062..8edfe6f28112a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql @@ -1,6 +1,9 @@ -- A test suite for IN HAVING in parent side, subquery, and both predicate subquery -- It includes correlated cases. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql index 200a71ebbb622..e6e612b4397d1 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql @@ -13,6 +13,9 @@ --CONFIG_DIM2 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY --CONFIG_DIM2 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN +--CONFIG_DIM3 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM3 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql index 0a16f118f0455..d63c505193a17 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql @@ -1,6 +1,9 @@ -- A test suite for IN LIMIT in parent side, subquery, and both predicate subquery -- It includes correlated cases. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2BD, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql index 4643605148a0c..7857ee88de201 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql @@ -1,6 +1,9 @@ -- A test suite for multiple columns in predicate in parent side, subquery, and both predicate subquery -- It includes correlated cases. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql index 001c49c460b06..8a8cfb9942c3f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql @@ -6,6 +6,9 @@ --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN +--CONFIG_DIM2 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM2 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2BD, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql index e65cb9106c1d4..86f829b826b4c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql @@ -1,6 +1,9 @@ -- A test suite for in with cte in parent side, subquery, and both predicate subquery -- It includes correlated cases. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql index 2f6835b59fdd5..96ab5d43e3ba1 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql @@ -1,5 +1,8 @@ -- Tests NOT-IN subqueries nested inside OR expression(s). +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + CREATE TEMPORARY VIEW EMP AS SELECT * FROM VALUES (100, "emp 1", 10), (200, "emp 2", NULL), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql index fcdb667ad4523..9426c6a13ed53 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql @@ -1,6 +1,9 @@ -- A test suite for not-in-joins in parent side, subquery, and both predicate subquery -- It includes correlated cases. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql index 8eea84f4f5272..9dd90c00ca95b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql @@ -4,6 +4,9 @@ -- This file has the same test cases as not-in-unit-tests-multi-column.sql with literals instead of -- subqueries. Small changes have been made to the literals to make them typecheck. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, null), (null, 1.0), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql index 9f8dc7fca3b94..b4ffd65840803 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql @@ -15,6 +15,9 @@ -- This can be generalized to include more tests for more columns, but it covers the main cases -- when there is more than one column. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, null), (null, 1.0), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql index b261363d1dde7..8075c36c969c7 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql @@ -4,6 +4,9 @@ -- This file has the same test cases as not-in-unit-tests-single-column.sql with literals instead of -- subqueries. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, 1.0), (2, 3.0), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql index 2cc08e10acf67..628ef13c97f0c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql @@ -31,6 +31,9 @@ -- cause cases 2, 3, or 4 to be reduced to case 1 by limiting the number of rows returned by the -- subquery, so the row from the parent table should always be included in the output. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, 1.0), (2, 3.0), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql index 2748a959cbef8..fd767d096433f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql @@ -1,6 +1,9 @@ -- A test suite for simple IN predicate subquery -- It includes correlated cases. +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true +--CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false + create temporary view t1 as select * from values ("t1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), ("t1b", 8S, 16, 19L, float(17.0), 25D, 26E2BD, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index edfd6d4d8facc..f57dcaf2e2800 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -89,6 +89,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan if (operators.head.getClass != c) { fail(s"$sqlString expected operator: $c, but got ${operators.head}\n physical: \n$physical") } + operators.head } test("join operator selection") { @@ -1152,9 +1153,10 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan withSQLConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> "true", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString) { // positive not in subquery case - assertJoin(( + var joinExec = assertJoin(( "select * from testData where key not in (select a from testData2)", classOf[BroadcastHashJoinExec])) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) // negative not in subquery case since multi-column is not supported assertJoin(( @@ -1164,16 +1166,18 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan // positive hand-written left anti join // testData.key nullable false // testData3.b nullable true - assertJoin(( + joinExec = assertJoin(( "select * from testData left anti join testData3 ON key = b or isnull(key = b)", classOf[BroadcastHashJoinExec])) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) // negative hand-written left anti join // testData.key nullable false // testData2.a nullable false - assertJoin(( + joinExec = assertJoin(( "select * from testData left anti join testData2 ON key = a or isnull(key = a)", classOf[BroadcastHashJoinExec])) + assert(!joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) // negative hand-written left anti join // not match pattern Or(EqualTo(a=b), IsNull(EqualTo(a=b)) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/NullAwareAntiJoinSQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/NullAwareAntiJoinSQLQueryTestSuite.scala deleted file mode 100644 index 916f9988aed88..0000000000000 --- a/sql/core/src/test/scala/org/apache/spark/sql/NullAwareAntiJoinSQLQueryTestSuite.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql - -import java.io.File - -import org.apache.spark.SparkConf -import org.apache.spark.sql.internal.SQLConf - -/** - * End-to-end test cases for subquery SQL queries coverage with - * NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED = true. - * - * Each case is loaded from a file in - * "spark/sql/core/src/test/resources/sql-tests/inputs/subquery". - * Each case has a golden result file in - * "spark/sql/core/src/test/resources/sql-tests/results/subquery". - * - * To run the entire test suite: - * {{{ - * build/sbt "sql/test-only *NullAwareAntiJoinSQLQueryTestSuite" - * }}} - * - */ -class NullAwareAntiJoinSQLQueryTestSuite extends SQLQueryTestSuite { - - protected override def sparkConf: SparkConf = super.sparkConf - // Fewer shuffle partitions to speed up testing. - // enable NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED for subquery case coverage. - .set(SQLConf.SHUFFLE_PARTITIONS, 4) - .set(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED, true) - - override lazy val listTestCases: Seq[TestCase] = { - listFilesRecursively(new File(inputFilePath)).flatMap { file => - val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".out" - val absPath = file.getAbsolutePath - val testCaseName = - s"NAAJ@" + - s"${absPath.stripPrefix(inputFilePath).stripPrefix(File.separator)}" - - if (file.getAbsolutePath.startsWith( - s"$inputFilePath${File.separator}subquery")) { - RegularTestCase(testCaseName, absPath, resultFile) :: Nil - } else { - Seq.empty - } - } - } -} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index 940c0bb26e456..efaad4ed0c095 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -1649,93 +1649,99 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark } test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize") { - Seq((true, true, true), (true, true, false), (true, false, true), - (true, false, false), (false, true, true), (false, true, false), - (false, false, true), (false, false, false)).foreach { config => - withSQLConf( - SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> config._1.toString, - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> config._2.toString, - SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> config._3.toString) { - - def findJoinExec(df: DataFrame): BaseJoinExec = { - df.queryExecution.sparkPlan.collectFirst { - case j: BaseJoinExec => j - }.get + Seq(true, false).foreach { enableNAAJ => + Seq(true, false).foreach { enableAQE => + Seq(true, false).foreach { enableCodegen => + withSQLConf( + SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> enableNAAJ.toString, + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString, + SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> enableCodegen.toString) { + + def findJoinExec(df: DataFrame): BaseJoinExec = { + df.queryExecution.sparkPlan.collectFirst { + case j: BaseJoinExec => j + }.get + } + + var df: DataFrame = null + + // single column not in subquery -- empty sub-query + df = sql("select * from l where a not in (select c from r where c > 10)") + checkAnswer(df, + Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: + Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + if (enableNAAJ) { + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- sub-query include null + df = sql("select * from l where a not in (select c from r where d < 6.0)") + checkAnswer(df, Seq.empty) + if (enableNAAJ) { + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- streamedSide row is null + df = + sql("select * from l where b = 5.0 and a not in(select c from r where c is not null)") + checkAnswer(df, Seq.empty) + if (enableNAAJ) { + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- streamedSide row is not null, match found + df = + sql("select * from l where a = 6 and a not in (select c from r where c is not null)") + checkAnswer(df, Seq.empty) + if (enableNAAJ) { + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- streamedSide row is not null, match not found + df = + sql("select * from l where a = 1 and a not in (select c from r where c is not null)") + checkAnswer(df, Row(1, 2.0) :: Row(1, 2.0) :: Nil) + if (enableNAAJ) { + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // single column not in subquery -- d = b + 10 joinKey found, match ExtractEquiJoinKeys + df = sql("select * from l where a not in (select c from r where d = b + 10)") + checkAnswer(df, + Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: + Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + + // single column not in subquery -- d = b + 10 and b = 5.0 => d = 15, joinKey not found + // match ExtractSingleColumnNullAwareAntiJoin + df = + sql("select * from l where b = 5.0 and a not in (select c from r where d = b + 10)") + checkAnswer(df, + Row(null, 5.0) :: Nil) + if (enableNAAJ) { + assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + } else { + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } + + // multi column not in subquery + df = sql("select * from l where (a, b) not in (select c, d from r where c > 10)") + checkAnswer(df, + Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: + Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) + } } - - var df: DataFrame = null - - // single column not in subquery -- empty sub-query - df = sql("select * from l where a not in (select c from r where c > 10)") - checkAnswer(df, - Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: - Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) - if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) - } else { - assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) - } - - // single column not in subquery -- sub-query include null - df = sql("select * from l where a not in (select c from r where d < 6.0)") - checkAnswer(df, Seq.empty) - if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) - } else { - assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) - } - - // single column not in subquery -- streamedSide row is null - df = sql("select * from l where b = 5.0 and a not in (select c from r where c is not null)") - checkAnswer(df, Seq.empty) - if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) - } else { - assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) - } - - // single column not in subquery -- streamedSide row is not null, match found - df = sql("select * from l where a = 6 and a not in (select c from r where c is not null)") - checkAnswer(df, Seq.empty) - if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) - } else { - assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) - } - - // single column not in subquery -- streamedSide row is not null, match not found - df = sql("select * from l where a = 1 and a not in (select c from r where c is not null)") - checkAnswer(df, Row(1, 2.0) :: Row(1, 2.0) :: Nil) - if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) - } else { - assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) - } - - // single column not in subquery -- d = b + 10 joinKey found, match ExtractEquiJoinKeys - df = sql("select * from l where a not in (select c from r where d = b + 10)") - checkAnswer(df, - Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: - Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) - - // single column not in subquery -- d = b + 10 and b = 5.0 => d = 15, joinKey not found - // match ExtractSingleColumnNullAwareAntiJoin - df = sql("select * from l where b = 5.0 and a not in (select c from r where d = b + 10)") - checkAnswer(df, - Row(null, 5.0) :: Nil) - if (config._1) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) - } else { - assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) - } - - // multi column not in subquery - df = sql("select * from l where (a, b) not in (select c, d from r where c > 10)") - checkAnswer(df, - Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: - Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) - assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala index 61c4196660b06..6b22fe2f6615e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala @@ -69,16 +69,17 @@ class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSui } val output = captured.toString() + val hashedModeString = "HashedRelationBroadcastMode(List(input[0, bigint, false]),false)" assert(output.replaceAll("\\[id=#\\d+\\]", "[id=#x]").contains( -"""== BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [id=#x] == - |Tuples output: 0 - | id LongType: {} - |== WholeStageCodegen (1) == - |Tuples output: 10 - | id LongType: {java.lang.Long} - |== Range (0, 10, step=1, splits=2) == - |Tuples output: 0 - | id LongType: {}""".stripMargin)) + s"""== BroadcastExchange $hashedModeString, [id=#x] == + |Tuples output: 0 + | id LongType: {} + |== WholeStageCodegen (1) == + |Tuples output: 10 + | id LongType: {java.lang.Long} + |== Range (0, 10, step=1, splits=2) == + |Tuples output: 0 + | id LongType: {}""".stripMargin)) } test("SPARK-28537: DebugExec cannot debug columnar related queries") { From dcccdebb5333cc3d1dc933cbb77b8fd4b3533722 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 27 Jul 2020 22:38:05 +0800 Subject: [PATCH 21/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. change spark.sql.nullAwareAntiJoin.optimize.enabled => spark.sql.optimizeNullAwareAntiJoin 2. add assertion for isNullAware 3. update CONFIG_DIM with spark.sql.optimizeNullAwareAntiJoin 4. code style refined. Change-Id: I871fe95664e233908bb39b63444e73c4a24126c0 --- .../sql/catalyst/planning/patterns.scala | 2 +- .../apache/spark/sql/internal/SQLConf.scala | 6 +-- .../sql/execution/joins/HashedRelation.scala | 5 +- .../sql-tests/inputs/group-by-filter.sql | 4 +- .../inputs/subquery/in-subquery/in-basic.sql | 4 +- .../inputs/subquery/in-subquery/in-having.sql | 4 +- .../inputs/subquery/in-subquery/in-joins.sql | 4 +- .../inputs/subquery/in-subquery/in-limit.sql | 4 +- .../in-subquery/in-multiple-columns.sql | 4 +- .../subquery/in-subquery/in-order-by.sql | 4 +- .../subquery/in-subquery/in-with-cte.sql | 4 +- .../subquery/in-subquery/nested-not-in.sql | 4 +- .../subquery/in-subquery/not-in-joins.sql | 4 +- ...not-in-unit-tests-multi-column-literal.sql | 4 +- .../not-in-unit-tests-multi-column.sql | 4 +- ...ot-in-unit-tests-single-column-literal.sql | 4 +- .../not-in-unit-tests-single-column.sql | 4 +- .../inputs/subquery/in-subquery/simple-in.sql | 4 +- .../org/apache/spark/sql/JoinSuite.scala | 3 +- .../org/apache/spark/sql/SubquerySuite.scala | 46 +++++++++++-------- .../sql/execution/debug/DebuggingSuite.scala | 16 +++---- 21 files changed, 73 insertions(+), 65 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index 6584790069e60..5a994f1ad0a39 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -412,7 +412,7 @@ object ExtractSingleColumnNullAwareAntiJoin extends JoinSelectionHelper with Pre case Join(left, right, LeftAnti, Some(Or(e @ EqualTo(leftAttr: AttributeReference, rightAttr: AttributeReference), IsNull(e2 @ EqualTo(_, _)))), _) - if SQLConf.get.nullAwareAntiJoinOptimizeEnabled && + if SQLConf.get.optimizeNullAwareAntiJoin && e.semanticEquals(e2) => if (canEvaluate(leftAttr, left) && canEvaluate(rightAttr, right)) { Some(Seq(leftAttr), Seq(rightAttr)) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 6047f1e52e3b1..6b2f0b4cc7302 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2672,7 +2672,7 @@ object SQLConf { .checkValue(_ >= 0, "The value must be non-negative.") .createWithDefault(8) - val NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED = + val OPTIMIZE_NULL_AWARE_ANTI_JOIN = buildConf("spark.sql.nullAwareAntiJoin.optimize.enabled") .internal() .doc("When true, NULL-aware anti join execution will be planed into " + @@ -3288,8 +3288,8 @@ class SQLConf extends Serializable with Logging { def coalesceBucketsInJoinMaxBucketRatio: Int = getConf(SQLConf.COALESCE_BUCKETS_IN_JOIN_MAX_BUCKET_RATIO) - def nullAwareAntiJoinOptimizeEnabled: Boolean = - getConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED) + def optimizeNullAwareAntiJoin: Boolean = + getConf(SQLConf.OPTIMIZE_NULL_AWARE_ANTI_JOIN) /** ********************** SQLConf functionality methods ************ */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index bb059cd0d4e60..982df555362d4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -946,7 +946,7 @@ trait NullAwareHashedRelation extends HashedRelation with Externalizable { * A special HashedRelation indicates it built from a empty input:Iterator[InternalRow]. */ class EmptyHashedRelation extends NullAwareHashedRelation { - override def asReadOnlyCopy(): EmptyHashedRelation = new EmptyHashedRelation + override def asReadOnlyCopy(): EmptyHashedRelation = this } /** @@ -954,8 +954,7 @@ class EmptyHashedRelation extends NullAwareHashedRelation { * which contains all null columns key. */ class EmptyHashedRelationWithAllNullKeys extends NullAwareHashedRelation { - override def asReadOnlyCopy(): EmptyHashedRelationWithAllNullKeys = - new EmptyHashedRelationWithAllNullKeys + override def asReadOnlyCopy(): EmptyHashedRelationWithAllNullKeys = this } /** The HashedRelationBroadcastMode requires that rows are broadcasted as a HashedRelation. */ diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql index 97ab00103ca69..4c1816e93b083 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/group-by-filter.sql @@ -1,7 +1,7 @@ -- Test filter clause for aggregate expression. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false -- Test data. CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql index be1183d13e7bf..5669423148f80 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-basic.sql @@ -1,5 +1,5 @@ ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false create temporary view tab_a as select * from values (1, 1) as tab_a(a1, b1); create temporary view tab_b as select * from values (1, 1) as tab_b(a2, b2); diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql index 8edfe6f28112a..750cc42b8641c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-having.sql @@ -1,8 +1,8 @@ -- A test suite for IN HAVING in parent side, subquery, and both predicate subquery -- It includes correlated cases. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql index e6e612b4397d1..2353560137d21 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql @@ -13,8 +13,8 @@ --CONFIG_DIM2 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY --CONFIG_DIM2 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN ---CONFIG_DIM3 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM3 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM3 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM3 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql index d63c505193a17..53fc2b8be7501 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql @@ -1,8 +1,8 @@ -- A test suite for IN LIMIT in parent side, subquery, and both predicate subquery -- It includes correlated cases. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql index 7857ee88de201..1a6c06f9dad49 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-multiple-columns.sql @@ -1,8 +1,8 @@ -- A test suite for multiple columns in predicate in parent side, subquery, and both predicate subquery -- It includes correlated cases. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql index 8a8cfb9942c3f..568854ebe2d9b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-order-by.sql @@ -6,8 +6,8 @@ --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN ---CONFIG_DIM2 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM2 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM2 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM2 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql index 86f829b826b4c..fa4ae87f041cf 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-with-cte.sql @@ -1,8 +1,8 @@ -- A test suite for in with cte in parent side, subquery, and both predicate subquery -- It includes correlated cases. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql index 96ab5d43e3ba1..e2d4ad522d446 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/nested-not-in.sql @@ -1,7 +1,7 @@ -- Tests NOT-IN subqueries nested inside OR expression(s). ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false CREATE TEMPORARY VIEW EMP AS SELECT * FROM VALUES (100, "emp 1", 10), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql index 9426c6a13ed53..2d11c5da20633 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql @@ -1,8 +1,8 @@ -- A test suite for not-in-joins in parent side, subquery, and both predicate subquery -- It includes correlated cases. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql index 9dd90c00ca95b..a061e495f51b8 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql @@ -4,8 +4,8 @@ -- This file has the same test cases as not-in-unit-tests-multi-column.sql with literals instead of -- subqueries. Small changes have been made to the literals to make them typecheck. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, null), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql index b4ffd65840803..28ab75121573a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column.sql @@ -15,8 +15,8 @@ -- This can be generalized to include more tests for more columns, but it covers the main cases -- when there is more than one column. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, null), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql index 8075c36c969c7..79747022eb1e8 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql @@ -4,8 +4,8 @@ -- This file has the same test cases as not-in-unit-tests-single-column.sql with literals instead of -- subqueries. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, 1.0), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql index 628ef13c97f0c..8060246bf3a3f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column.sql @@ -31,8 +31,8 @@ -- cause cases 2, 3, or 4 to be reduced to case 1 by limiting the number of rows returned by the -- subquery, so the row from the parent table should always be included in the output. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES (null, 1.0), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql index fd767d096433f..d8a58afa344db 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/simple-in.sql @@ -1,8 +1,8 @@ -- A test suite for simple IN predicate subquery -- It includes correlated cases. ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=true ---CONFIG_DIM1 spark.sql.nullAwareAntiJoin.optimize.enabled=false +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=true +--CONFIG_DIM1 spark.sql.optimizeNullAwareAntiJoin=false create temporary view t1 as select * from values ("t1a", 6S, 8, 10L, float(15.0), 20D, 20E2BD, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala index f57dcaf2e2800..bedfbffc789ac 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala @@ -1150,7 +1150,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan } test("SPARK-32290: SingleColumn Null Aware Anti Join Optimize") { - withSQLConf(SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> "true", + withSQLConf(SQLConf.OPTIMIZE_NULL_AWARE_ANTI_JOIN.key -> "true", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString) { // positive not in subquery case var joinExec = assertJoin(( @@ -1174,6 +1174,7 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan // negative hand-written left anti join // testData.key nullable false // testData2.a nullable false + // isnull(key = a) will be optimized to true literal and removed joinExec = assertJoin(( "select * from testData left anti join testData2 ON key = a or isnull(key = a)", classOf[BroadcastHashJoinExec])) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index efaad4ed0c095..a21c461e84588 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -1653,7 +1653,7 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark Seq(true, false).foreach { enableAQE => Seq(true, false).foreach { enableCodegen => withSQLConf( - SQLConf.NULL_AWARE_ANTI_JOIN_OPTIMIZE_ENABLED.key -> enableNAAJ.toString, + SQLConf.OPTIMIZE_NULL_AWARE_ANTI_JOIN.key -> enableNAAJ.toString, SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString, SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> enableCodegen.toString) { @@ -1664,14 +1664,15 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark } var df: DataFrame = null + var joinExec: BaseJoinExec = null // single column not in subquery -- empty sub-query df = sql("select * from l where a not in (select c from r where c > 10)") - checkAnswer(df, - Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: - Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + checkAnswer(df, spark.table("l")) if (enableNAAJ) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + joinExec = findJoinExec(df) + assert(joinExec.isInstanceOf[BroadcastHashJoinExec]) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1680,7 +1681,9 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark df = sql("select * from l where a not in (select c from r where d < 6.0)") checkAnswer(df, Seq.empty) if (enableNAAJ) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + joinExec = findJoinExec(df) + assert(joinExec.isInstanceOf[BroadcastHashJoinExec]) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1690,7 +1693,9 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark sql("select * from l where b = 5.0 and a not in(select c from r where c is not null)") checkAnswer(df, Seq.empty) if (enableNAAJ) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + joinExec = findJoinExec(df) + assert(joinExec.isInstanceOf[BroadcastHashJoinExec]) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1700,7 +1705,9 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark sql("select * from l where a = 6 and a not in (select c from r where c is not null)") checkAnswer(df, Seq.empty) if (enableNAAJ) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + joinExec = findJoinExec(df) + assert(joinExec.isInstanceOf[BroadcastHashJoinExec]) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } @@ -1710,35 +1717,36 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark sql("select * from l where a = 1 and a not in (select c from r where c is not null)") checkAnswer(df, Row(1, 2.0) :: Row(1, 2.0) :: Nil) if (enableNAAJ) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + joinExec = findJoinExec(df) + assert(joinExec.isInstanceOf[BroadcastHashJoinExec]) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } // single column not in subquery -- d = b + 10 joinKey found, match ExtractEquiJoinKeys df = sql("select * from l where a not in (select c from r where d = b + 10)") - checkAnswer(df, - Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: - Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + checkAnswer(df, spark.table("l")) + joinExec = findJoinExec(df) + assert(joinExec.isInstanceOf[BroadcastHashJoinExec]) + assert(!joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) // single column not in subquery -- d = b + 10 and b = 5.0 => d = 15, joinKey not found // match ExtractSingleColumnNullAwareAntiJoin df = sql("select * from l where b = 5.0 and a not in (select c from r where d = b + 10)") - checkAnswer(df, - Row(null, 5.0) :: Nil) + checkAnswer(df, Row(null, 5.0) :: Nil) if (enableNAAJ) { - assert(findJoinExec(df).isInstanceOf[BroadcastHashJoinExec]) + joinExec = findJoinExec(df) + assert(joinExec.isInstanceOf[BroadcastHashJoinExec]) + assert(joinExec.asInstanceOf[BroadcastHashJoinExec].isNullAwareAntiJoin) } else { assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } // multi column not in subquery df = sql("select * from l where (a, b) not in (select c, d from r where c > 10)") - checkAnswer(df, - Row(1, 2.0) :: Row(1, 2.0) :: Row(2, 1.0) :: Row(2, 1.0) :: - Row(3, 3.0) :: Row(null, null) :: Row(null, 5.0) :: Row(6, null) :: Nil) + checkAnswer(df, spark.table("l")) assert(findJoinExec(df).isInstanceOf[BroadcastNestedLoopJoinExec]) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala index 6b22fe2f6615e..d2c9322685d94 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala @@ -72,14 +72,14 @@ class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSui val hashedModeString = "HashedRelationBroadcastMode(List(input[0, bigint, false]),false)" assert(output.replaceAll("\\[id=#\\d+\\]", "[id=#x]").contains( s"""== BroadcastExchange $hashedModeString, [id=#x] == - |Tuples output: 0 - | id LongType: {} - |== WholeStageCodegen (1) == - |Tuples output: 10 - | id LongType: {java.lang.Long} - |== Range (0, 10, step=1, splits=2) == - |Tuples output: 0 - | id LongType: {}""".stripMargin)) + |Tuples output: 0 + | id LongType: {} + |== WholeStageCodegen (1) == + |Tuples output: 10 + | id LongType: {java.lang.Long} + |== Range (0, 10, step=1, splits=2) == + |Tuples output: 0 + | id LongType: {}""".stripMargin)) } test("SPARK-28537: DebugExec cannot debug columnar related queries") { From 7824c45e1f8328a4450660290ccd94d759c9600b Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Mon, 27 Jul 2020 23:07:51 +0800 Subject: [PATCH 22/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize typo. Change-Id: Id5db52227468cb4b22bad1923eab85bd6ce6fb5d --- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 6b2f0b4cc7302..f4bc328b24676 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2673,7 +2673,7 @@ object SQLConf { .createWithDefault(8) val OPTIMIZE_NULL_AWARE_ANTI_JOIN = - buildConf("spark.sql.nullAwareAntiJoin.optimize.enabled") + buildConf("spark.sql.optimizeNullAwareAntiJoin") .internal() .doc("When true, NULL-aware anti join execution will be planed into " + "BroadcastHashJoinExec with flag isNullAwareAntiJoin enabled, " + From 5e050c47a9b63a633ab6ff0c7d5e7d2a6f2df795 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Tue, 28 Jul 2020 00:59:24 +0800 Subject: [PATCH 23/24] comment refine. Change-Id: Icbf28bdbee90de6b09172ab4c495383002f340a4 --- .../spark/sql/execution/joins/BroadcastHashJoinExec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 71046fc96a07f..20ff2cf4f350f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -499,7 +499,7 @@ case class BroadcastHashJoinExec( """.stripMargin } else if (broadcastRelation.value.isInstanceOf[EmptyHashedRelationWithAllNullKeys]) { return s""" - |// If the right side contains any all-null key, NAAJ simply returns Nil. + |// If the right side contains any all-null key, NAAJ simply returns Nothing. """.stripMargin } else { val found = ctx.freshName("found") From 233eff6549377d6c7c850fe8d1990fcd58fe0ea0 Mon Sep 17 00:00:00 2001 From: "xuewei.linxuewei" Date: Tue, 28 Jul 2020 06:03:28 +0800 Subject: [PATCH 24/24] [SPARK-32290][SQL] SingleColumn Null Aware Anti Join Optimize sub commit 1. change EmptyHashedRelation and EmptyHashedRelationWithAllNullKeys to singleton object 2. change default implementation of NullAwareHashedRelation to throw UnsupportedOperationException Change-Id: I173ce102bb704677699b89daa1c9906f748c94aa --- .../joins/BroadcastHashJoinExec.scala | 8 +++--- .../sql/execution/joins/HashedRelation.scala | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala index 20ff2cf4f350f..2a283013aceef 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala @@ -146,9 +146,9 @@ case class BroadcastHashJoinExec( streamedPlan.execute().mapPartitionsInternal { streamedIter => val hashed = broadcastRelation.value.asReadOnlyCopy() TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize) - if (hashed.isInstanceOf[EmptyHashedRelation]) { + if (hashed == EmptyHashedRelation) { streamedIter - } else if (hashed.isInstanceOf[EmptyHashedRelationWithAllNullKeys]) { + } else if (hashed == EmptyHashedRelationWithAllNullKeys) { Iterator.empty } else { val keyGenerator = UnsafeProjection.create( @@ -491,13 +491,13 @@ case class BroadcastHashJoinExec( val numOutput = metricTerm(ctx, "numOutputRows") if (isNullAwareAntiJoin) { - if (broadcastRelation.value.isInstanceOf[EmptyHashedRelation]) { + if (broadcastRelation.value == EmptyHashedRelation) { return s""" |// If the right side is empty, NAAJ simply returns the left side. |$numOutput.add(1); |${consume(ctx, input)} """.stripMargin - } else if (broadcastRelation.value.isInstanceOf[EmptyHashedRelationWithAllNullKeys]) { + } else if (broadcastRelation.value == EmptyHashedRelationWithAllNullKeys) { return s""" |// If the right side contains any all-null key, NAAJ simply returns Nothing. """.stripMargin diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala index 982df555362d4..f2835c2fa6626 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala @@ -109,7 +109,7 @@ private[execution] object HashedRelation { } if (isNullAware && !input.hasNext) { - new EmptyHashedRelation + EmptyHashedRelation } else if (key.length == 1 && key.head.dataType == LongType) { LongHashedRelation(input, key, sizeEstimate, mm, isNullAware) } else { @@ -343,7 +343,7 @@ private[joins] object UnsafeHashedRelation { // scalastyle:on throwerror } } else if (isNullAware) { - return new EmptyHashedRelationWithAllNullKeys + return EmptyHashedRelationWithAllNullKeys } } @@ -911,7 +911,7 @@ private[joins] object LongHashedRelation { val key = rowKey.getLong(0) map.append(key, unsafeRow) } else if (isNullAware) { - return new EmptyHashedRelationWithAllNullKeys + return EmptyHashedRelationWithAllNullKeys } } map.optimize() @@ -925,13 +925,19 @@ private[joins] object LongHashedRelation { * EmptyHashedRelationWithAllNullKeys */ trait NullAwareHashedRelation extends HashedRelation with Externalizable { - override def get(key: InternalRow): Iterator[InternalRow] = null + override def get(key: InternalRow): Iterator[InternalRow] = { + throw new UnsupportedOperationException + } - override def getValue(key: InternalRow): InternalRow = null + override def getValue(key: InternalRow): InternalRow = { + throw new UnsupportedOperationException + } override def keyIsUnique: Boolean = true - override def keys(): Iterator[InternalRow] = null + override def keys(): Iterator[InternalRow] = { + throw new UnsupportedOperationException + } override def close(): Unit = {} @@ -945,16 +951,16 @@ trait NullAwareHashedRelation extends HashedRelation with Externalizable { /** * A special HashedRelation indicates it built from a empty input:Iterator[InternalRow]. */ -class EmptyHashedRelation extends NullAwareHashedRelation { - override def asReadOnlyCopy(): EmptyHashedRelation = this +object EmptyHashedRelation extends NullAwareHashedRelation { + override def asReadOnlyCopy(): EmptyHashedRelation.type = this } /** * A special HashedRelation indicates it built from a non-empty input:Iterator[InternalRow], * which contains all null columns key. */ -class EmptyHashedRelationWithAllNullKeys extends NullAwareHashedRelation { - override def asReadOnlyCopy(): EmptyHashedRelationWithAllNullKeys = this +object EmptyHashedRelationWithAllNullKeys extends NullAwareHashedRelation { + override def asReadOnlyCopy(): EmptyHashedRelationWithAllNullKeys.type = this } /** The HashedRelationBroadcastMode requires that rows are broadcasted as a HashedRelation. */