Skip to content

Commit 77c9964

Browse files
wangyumgatorsmile
authored andcommitted
[SPARK-25368][SQL] Incorrect predicate pushdown returns wrong result
## What changes were proposed in this pull request? How to reproduce: ```scala val df1 = spark.createDataFrame(Seq( (1, 1) )).toDF("a", "b").withColumn("c", lit(null).cast("int")) val df2 = df1.union(df1).withColumn("d", spark_partition_id).filter($"c".isNotNull) df2.show +---+---+----+---+ | a| b| c| d| +---+---+----+---+ | 1| 1|null| 0| | 1| 1|null| 1| +---+---+----+---+ ``` `filter($"c".isNotNull)` was transformed to `(null <=> c#10)` before #19201, but it is transformed to `(c#10 = null)` since #20155. This pr revert it to `(null <=> c#10)` to fix this issue. ## How was this patch tested? unit tests Closes #22368 from wangyum/SPARK-25368. Authored-by: Yuming Wang <[email protected]> Signed-off-by: gatorsmile <[email protected]>
1 parent 88a930d commit 77c9964

File tree

3 files changed

+19
-2
lines changed

3 files changed

+19
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ abstract class UnaryNode extends LogicalPlan {
159159
var allConstraints = child.constraints.asInstanceOf[Set[Expression]]
160160
projectList.foreach {
161161
case a @ Alias(l: Literal, _) =>
162-
allConstraints += EqualTo(a.toAttribute, l)
162+
allConstraints += EqualNullSafe(a.toAttribute, l)
163163
case a @ Alias(e, _) =>
164164
// For every alias in `projectList`, replace the reference in constraints by its attribute.
165165
allConstraints ++= allConstraints.map(_ transform {

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromConstraintsSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ class InferFiltersFromConstraintsSuite extends PlanTest {
196196

197197
test("constraints should be inferred from aliased literals") {
198198
val originalLeft = testRelation.subquery('left).as("left")
199-
val optimizedLeft = testRelation.subquery('left).where(IsNotNull('a) && 'a === 2).as("left")
199+
val optimizedLeft = testRelation.subquery('left).where(IsNotNull('a) && 'a <=> 2).as("left")
200200

201201
val right = Project(Seq(Literal(2).as("two")), testRelation.subquery('right)).as("right")
202202
val condition = Some("left.a".attr === "right.two".attr)

sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2552,4 +2552,21 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
25522552
}
25532553
}
25542554

2555+
test("SPARK-25368 Incorrect predicate pushdown returns wrong result") {
2556+
def check(newCol: Column, filter: Column, result: Seq[Row]): Unit = {
2557+
val df1 = spark.createDataFrame(Seq(
2558+
(1, 1)
2559+
)).toDF("a", "b").withColumn("c", newCol)
2560+
2561+
val df2 = df1.union(df1).withColumn("d", spark_partition_id).filter(filter)
2562+
checkAnswer(df2, result)
2563+
}
2564+
2565+
check(lit(null).cast("int"), $"c".isNull, Seq(Row(1, 1, null, 0), Row(1, 1, null, 1)))
2566+
check(lit(null).cast("int"), $"c".isNotNull, Seq())
2567+
check(lit(2).cast("int"), $"c".isNull, Seq())
2568+
check(lit(2).cast("int"), $"c".isNotNull, Seq(Row(1, 1, 2, 0), Row(1, 1, 2, 1)))
2569+
check(lit(2).cast("int"), $"c" === 2, Seq(Row(1, 1, 2, 0), Row(1, 1, 2, 1)))
2570+
check(lit(2).cast("int"), $"c" =!= 2, Seq())
2571+
}
25552572
}

0 commit comments

Comments
 (0)