Skip to content

[SPARK-33847][SQL] Simplify CaseWhen if elseValue is None #30852

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 10 commits into from
Closed

[SPARK-33847][SQL] Simplify CaseWhen if elseValue is None #30852

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
09edff5
Replace None of elseValue inside CaseWhen if all branches are FalseLi…
wangyum Dec 19, 2020
ca2a224
Merge remote-tracking branch 'upstream/master' into SPARK-33847
wangyum Dec 21, 2020
3c5f3da
Fix
wangyum Dec 21, 2020
b837e37
fix
wangyum Dec 21, 2020
d07344f
Merge remote-tracking branch 'upstream/master' into SPARK-33847
wangyum Dec 22, 2020
81c38f8
fix
wangyum Dec 22, 2020
7f3529d
fix
wangyum Dec 23, 2020
1684d67
Merge remote-tracking branch 'upstream/master' into SPARK-33847
wangyum Dec 23, 2020
0933019
fix
wangyum Dec 23, 2020
d20fede
remove
wangyum Dec 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions .../main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,12 @@ object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
val newBranches = cw.branches.map { case (cond, value) =>
replaceNullWithFalse(cond) -> replaceNullWithFalse(value)
}
val newElseValue = cw.elseValue.map(replaceNullWithFalse)
CaseWhen(newBranches, newElseValue)
if (newBranches.forall(_._2 == FalseLiteral) && cw.elseValue.isEmpty) {
FalseLiteral
} else {
val newElseValue = cw.elseValue.map(replaceNullWithFalse)
CaseWhen(newBranches, newElseValue)
}
case i @ If(pred, trueVal, falseVal) if i.dataType == BooleanType =>
If(replaceNullWithFalse(pred), replaceNullWithFalse(trueVal), replaceNullWithFalse(falseVal))
case e if e.dataType == BooleanType =>
Expand Down
4 changes: 4 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,10 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {
} else {
e.copy(branches = branches.take(i).map(branch => (branch._1, elseValue)))
}

case e @ CaseWhen(branches, None)
if branches.forall(_._2.semanticEquals(Literal(null, e.dataType))) =>
Literal(null, e.dataType)
}
}
}
Expand Down
18 changes: 18 additions & 0 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/optimizer/PushFoldableIntoBranchesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,4 +258,22 @@ class PushFoldableIntoBranchesSuite
EqualTo(CaseWhen(Seq((a, Literal(1)), (c, Literal(2))), None).cast(StringType), Literal("4")),
CaseWhen(Seq((a, FalseLiteral), (c, FalseLiteral)), None))
}

test("SPARK-33847: Remove the CaseWhen if elseValue is empty and other outputs are null") {
assertEquivalent(
Copy link
Contributor

@cloud-fan cloud-fan Dec 23, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: to make the test more readable

Seq(a, LessThan(Rand(1), Literal(0.5)).foreach { condition =>
  assertEquivalent(EqualTo(CaseWhen(Seq((condition, ...
  assertEquivalent(// with cast ...
}

EqualTo(CaseWhen(Seq((a, Literal.create(null, IntegerType)))), Literal(2)),
Literal.create(null, BooleanType))
assertEquivalent(
EqualTo(CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), Literal.create(null, IntegerType)))),
Literal(2)),
Literal.create(null, BooleanType))

assertEquivalent(
EqualTo(CaseWhen(Seq((a, Literal("str")))).cast(IntegerType), Literal(2)),
Literal.create(null, BooleanType))
assertEquivalent(
EqualTo(CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), Literal("str")))).cast(IntegerType),
Literal(2)),
Literal.create(null, BooleanType))
}
}
70 changes: 59 additions & 11 deletions .../scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, ArrayTransform, CaseWhen, Expression, GreaterThan, If, LambdaFunction, LessThanOrEqual, Literal, MapFilter, NamedExpression, Or, UnresolvedNamedLambdaVariable}
import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, ArrayTransform, CaseWhen, EqualTo, Expression, GreaterThan, If, LambdaFunction, LessThanOrEqual, Literal, MapFilter, NamedExpression, Not, Or, UnresolvedNamedLambdaVariable}
import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, LocalRelation, LogicalPlan, UpdateTable}
Expand All @@ -38,6 +38,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
ConstantFolding,
BooleanSimplification,
SimplifyConditionals,
PushFoldableIntoBranches,
ReplaceNullWithFalseInPredicate) :: Nil
}

Expand Down Expand Up @@ -222,10 +223,14 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
Literal(null, IntegerType),
Literal(3)),
FalseLiteral)
testFilter(originalCond = condition, expectedCond = condition)
testJoin(originalCond = condition, expectedCond = condition)
testDelete(originalCond = condition, expectedCond = condition)
testUpdate(originalCond = condition, expectedCond = condition)
val expectedCond = If(
Copy link
Contributor

@cloud-fan cloud-fan Dec 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to update the test name. Now it's inability to replace null in non-boolean branches of If

Copy link
Contributor

@cloud-fan cloud-fan Dec 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or we change this test to preserve the original purpose.

Copy link
Member Author

@wangyum wangyum Dec 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed PushFoldableIntoBranches from this test suite, and add integration test to ReplaceNullWithFalseInPredicateEndToEndSuite.

UnresolvedAttribute("i") > Literal(10),
Not(UnresolvedAttribute("i") === Literal(15)),
FalseLiteral)
testFilter(originalCond = condition, expectedCond = expectedCond)
testJoin(originalCond = condition, expectedCond = expectedCond)
testDelete(originalCond = condition, expectedCond = expectedCond)
testUpdate(originalCond = condition, expectedCond = expectedCond)
}

test("inability to replace null in non-boolean values of CaseWhen") {
Expand All @@ -237,8 +242,8 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
TrueLiteral,
FalseLiteral)
val condition = CaseWhen(Seq((UnresolvedAttribute("i") > Literal(10)) -> branchValue))
val expectedCond =
CaseWhen(Seq((UnresolvedAttribute("i") > Literal(10)) -> (Literal(2) === nestedCaseWhen)))
val expectedCond = CaseWhen(Seq((UnresolvedAttribute("i") > Literal(10)) ->
CaseWhen(Seq((UnresolvedAttribute("i") > Literal(20)) -> TrueLiteral), FalseLiteral)))
testFilter(originalCond = condition, expectedCond = expectedCond)
testJoin(originalCond = condition, expectedCond = expectedCond)
testDelete(originalCond = condition, expectedCond = expectedCond)
Expand All @@ -253,10 +258,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
Literal(3)),
TrueLiteral,
FalseLiteral)
val expectedCond = Literal(5) > If(
UnresolvedAttribute("i") === Literal(15),
Literal(null, IntegerType),
Literal(3))
val expectedCond = Not(UnresolvedAttribute("i") === Literal(15))
testFilter(originalCond = condition, expectedCond = expectedCond)
testJoin(originalCond = condition, expectedCond = expectedCond)
testDelete(originalCond = condition, expectedCond = expectedCond)
Expand Down Expand Up @@ -380,6 +382,52 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
testProjection(originalExpr = column, expectedExpr = column)
}

test("replace None of elseValue inside CaseWhen if all branches are FalseLiteral") {
val allFalseBranches = Seq(
(UnresolvedAttribute("i") < Literal(10)) -> FalseLiteral,
(UnresolvedAttribute("i") > Literal(40)) -> FalseLiteral)
val allFalseCond = CaseWhen(allFalseBranches)

val nonAllFalseBranches = Seq(
(UnresolvedAttribute("i") < Literal(10)) -> FalseLiteral,
(UnresolvedAttribute("i") > Literal(40)) -> TrueLiteral)
val nonAllFalseCond = CaseWhen(nonAllFalseBranches, FalseLiteral)

testFilter(allFalseCond, FalseLiteral)
testJoin(allFalseCond, FalseLiteral)
testDelete(allFalseCond, FalseLiteral)
testUpdate(allFalseCond, FalseLiteral)

testFilter(nonAllFalseCond, nonAllFalseCond)
testJoin(nonAllFalseCond, nonAllFalseCond)
testDelete(nonAllFalseCond, nonAllFalseCond)
testUpdate(nonAllFalseCond, nonAllFalseCond)
}

test("replace None of elseValue inside CaseWhen if all branches are null") {
val allFalseBranches = Seq(
Copy link
Contributor

@cloud-fan cloud-fan Dec 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

allNullBranches

(UnresolvedAttribute("i") < Literal(10)) -> Literal.create(null, BooleanType),
(UnresolvedAttribute("i") > Literal(40)) -> Literal.create(null, BooleanType))
val allFalseCond = CaseWhen(allFalseBranches)

testFilter(allFalseCond, FalseLiteral)
testJoin(allFalseCond, FalseLiteral)
testDelete(allFalseCond, FalseLiteral)
testUpdate(allFalseCond, FalseLiteral)
}

test("replace None of elseValue inside CaseWhen with PushFoldableIntoBranches") {
val allFalseBranches = Seq(
(UnresolvedAttribute("i") < Literal(10)) -> Literal("a"),
(UnresolvedAttribute("i") > Literal(40)) -> Literal("b"))
val allFalseCond = EqualTo(CaseWhen(allFalseBranches), "c")

testFilter(allFalseCond, FalseLiteral)
testJoin(allFalseCond, FalseLiteral)
testDelete(allFalseCond, FalseLiteral)
testUpdate(allFalseCond, FalseLiteral)
}

private def testFilter(originalCond: Expression, expectedCond: Expression): Unit = {
test((rel, exp) => rel.where(exp), originalCond, expectedCond)
}
Expand Down
20 changes: 20 additions & 0 deletions ...yst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,4 +215,24 @@ class SimplifyConditionalSuite extends PlanTest with ExpressionEvalHelper with P
If(GreaterThan(Rand(0), UnresolvedAttribute("a")), FalseLiteral, TrueLiteral),
LessThanOrEqual(Rand(0), UnresolvedAttribute("a")))
}

test("SPARK-33847: Remove the CaseWhen if elseValue is empty and other outputs are null") {
assertEquivalent(
CaseWhen((GreaterThan('a, 1), Literal.create(null, IntegerType)) :: Nil,
Copy link
Contributor

@cloud-fan cloud-fan Dec 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto, Seq(GreaterThan('a, 1), GreaterThan(Rand(0), 1)).foreach...

None),
Literal.create(null, IntegerType))
assertEquivalent(
CaseWhen((GreaterThan(Rand(0), 1), Literal.create(null, IntegerType)) :: Nil,
None),
Literal.create(null, IntegerType))

assertEquivalent(
Copy link
Contributor

@cloud-fan cloud-fan Dec 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The 2 tests below are not related to the test name Remove the CaseWhen if elseValue is empty and other outputs are null

Copy link
Member Author

@wangyum wangyum Dec 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed it

CaseWhen((GreaterThan('a, 1), Literal.create(null, IntegerType)) :: Nil,
Some(Literal.create(null, IntegerType))),
Literal.create(null, IntegerType))
assertEquivalent(
CaseWhen((GreaterThan('a, 1), Literal(20)) :: (GreaterThan('b, 1), Literal(20)) :: Nil,
Some(Literal(20))),
Literal(20))
}
}