From 65833e7d38a98495ec898bae2ec33adef7054aa5 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 14 Sep 2020 12:27:45 -0700 Subject: [PATCH 1/5] [SPARK-32858][SQL] UnwrapCastInBinaryComparison: support other numeric types --- .../UnwrapCastInBinaryComparison.scala | 202 +++++++++++------- .../UnwrapCastInBinaryComparisonSuite.scala | 141 +++++++++--- 2 files changed, 237 insertions(+), 106 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala index d0acfe036d443..4e6edf54ba678 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala @@ -35,18 +35,34 @@ import org.apache.spark.sql.types._ * to be optimized away later and pushed down to data sources. * * Currently this only handles cases where: - * 1). `fromType` (of `fromExp`) and `toType` are of integral types (i.e., byte, short, int and - * long) + * 1). `fromType` (of `fromExp`) and `toType` are of numeric types (i.e., short, int, float, + * decimal, etc) * 2). `fromType` can be safely coerced to `toType` without precision loss (e.g., short to int, * int to long, but not long to int) * * If the above conditions are satisfied, the rule checks to see if the literal `value` is within * range `(min, max)`, where `min` and `max` are the minimum and maximum value of `fromType`, - * respectively. If this is true then it means we can safely cast `value` to `fromType` and thus + * respectively. If this is true then it means we may safely cast `value` to `fromType` and thus * able to move the cast to the literal side. That is: * * `cast(fromExp, toType) op value` ==> `fromExp op cast(value, fromType)` * + * Note there are some exceptions to the above: if casting from `value` to `fromType` causes + * rounding up or down, the above conversion will no longer be valid. Instead, the rule does the + * following: + * + * if casting `value` to `fromType` causes rounding up: + * - `cast(fromExp, toType) > value` ==> `fromExp >= cast(value, fromType)` + * - `cast(fromExp, toType) >= value` ==> `fromExp >= cast(value, fromType)` + * - `cast(fromExp, toType) === value` ==> if(isnull(fromExp), null, false) + * - `cast(fromExp, toType) <=> value` ==> false (if `fromExp` is deterministic) + * - `cast(fromExp, toType) <=> value` ==> `cast(fromExp, toType) <=> value` (if `fromExp` is + * non-deterministic) + * - `cast(fromExp, toType) <= value` ==> `fromExp < cast(value, fromType)` + * - `cast(fromExp, toType) < value` ==> `fromExp < cast(value, fromType)` + * + * Similarly for the case when casting `value` to `fromType` causes rounding down. + * * If the `value` is not within range `(min, max)`, the rule breaks the scenario into different * cases and try to replace each with simpler constructs. * @@ -100,12 +116,12 @@ object UnwrapCastInBinaryComparison extends Rule[LogicalPlan] { swap(unwrapCast(swap(exp))) - // In case both sides have integral type, optimize the comparison by removing casts or + // In case both sides have numeric type, optimize the comparison by removing casts or // moving cast to the literal side. case be @ BinaryComparison( - Cast(fromExp, toType: IntegralType, _), Literal(value, literalType)) + Cast(fromExp, toType: NumericType, _), Literal(value, literalType)) if canImplicitlyCast(fromExp, toType, literalType) => - simplifyIntegralComparison(be, fromExp, toType, value) + simplifyNumericComparison(be, fromExp, toType, value) case _ => exp } @@ -116,82 +132,118 @@ object UnwrapCastInBinaryComparison extends Rule[LogicalPlan] { * optimizes the expression by moving the cast to the literal side. Otherwise if result is not * true, this replaces the input binary comparison `exp` with simpler expressions. */ - private def simplifyIntegralComparison( + private def simplifyNumericComparison( exp: BinaryComparison, fromExp: Expression, - toType: IntegralType, + toType: NumericType, value: Any): Expression = { val fromType = fromExp.dataType - val (min, max) = getRange(fromType) - val (minInToType, maxInToType) = { - (Cast(Literal(min), toType).eval(), Cast(Literal(max), toType).eval()) - } val ordering = toType.ordering.asInstanceOf[Ordering[Any]] - val minCmp = ordering.compare(value, minInToType) - val maxCmp = ordering.compare(value, maxInToType) + val range = getRange(fromType) - if (maxCmp > 0) { - exp match { - case EqualTo(_, _) | GreaterThan(_, _) | GreaterThanOrEqual(_, _) => - falseIfNotNull(fromExp) - case LessThan(_, _) | LessThanOrEqual(_, _) => - trueIfNotNull(fromExp) - // make sure the expression is evaluated if it is non-deterministic - case EqualNullSafe(_, _) if exp.deterministic => - FalseLiteral - case _ => exp + if (range.isDefined) { + val (min, max) = range.get + val (minInToType, maxInToType) = { + (Cast(Literal(min), toType).eval(), Cast(Literal(max), toType).eval()) } - } else if (maxCmp == 0) { - exp match { - case GreaterThan(_, _) => - falseIfNotNull(fromExp) - case LessThanOrEqual(_, _) => - trueIfNotNull(fromExp) - case LessThan(_, _) => - Not(EqualTo(fromExp, Literal(max, fromType))) - case GreaterThanOrEqual(_, _) | EqualTo(_, _) => - EqualTo(fromExp, Literal(max, fromType)) - case EqualNullSafe(_, _) => - EqualNullSafe(fromExp, Literal(max, fromType)) - case _ => exp + val minCmp = ordering.compare(value, minInToType) + val maxCmp = ordering.compare(value, maxInToType) + + if (maxCmp >= 0 || minCmp <= 0) { + return if (maxCmp > 0) { + exp match { + case EqualTo(_, _) | GreaterThan(_, _) | GreaterThanOrEqual(_, _) => + falseIfNotNull(fromExp) + case LessThan(_, _) | LessThanOrEqual(_, _) => + trueIfNotNull(fromExp) + // make sure the expression is evaluated if it is non-deterministic + case EqualNullSafe(_, _) if exp.deterministic => + FalseLiteral + case _ => exp + } + } else if (maxCmp == 0) { + exp match { + case GreaterThan(_, _) => + falseIfNotNull(fromExp) + case LessThanOrEqual(_, _) => + trueIfNotNull(fromExp) + case LessThan(_, _) => + Not(EqualTo(fromExp, Literal(max, fromType))) + case GreaterThanOrEqual(_, _) | EqualTo(_, _) => + EqualTo(fromExp, Literal(max, fromType)) + case EqualNullSafe(_, _) => + EqualNullSafe(fromExp, Literal(max, fromType)) + case _ => exp + } + } else if (minCmp < 0) { + exp match { + case GreaterThan(_, _) | GreaterThanOrEqual(_, _) => + trueIfNotNull(fromExp) + case LessThan(_, _) | LessThanOrEqual(_, _) | EqualTo(_, _) => + falseIfNotNull(fromExp) + // make sure the expression is evaluated if it is non-deterministic + case EqualNullSafe(_, _) if exp.deterministic => + FalseLiteral + case _ => exp + } + } else { // minCmp == 0 + exp match { + case LessThan(_, _) => + falseIfNotNull(fromExp) + case GreaterThanOrEqual(_, _) => + trueIfNotNull(fromExp) + case GreaterThan(_, _) => + Not(EqualTo(fromExp, Literal(min, fromType))) + case LessThanOrEqual(_, _) | EqualTo(_, _) => + EqualTo(fromExp, Literal(min, fromType)) + case EqualNullSafe(_, _) => + EqualNullSafe(fromExp, Literal(min, fromType)) + case _ => exp + } + } } - } else if (minCmp < 0) { + } + + // When we reach to this point, it means either there is no min/max for the `fromType` (e.g., + // decimal type), or that the literal `value` is within range `(min, max)`. For these, we + // optimize by moving the cast to the literal side. + + val newValue = Cast(Literal(value), fromType).eval() + if (newValue == null) { + // This means the cast failed, for instance, due to the value is not representable in the + // narrower type. In this case we simply return the original expression. + return exp + } + val valueRoundTrip = Cast(Literal(newValue, fromType), toType).eval() + val lit = Literal(newValue, fromType) + val cmp = ordering.compare(value, valueRoundTrip) + if (cmp == 0) { exp match { - case GreaterThan(_, _) | GreaterThanOrEqual(_, _) => - trueIfNotNull(fromExp) - case LessThan(_, _) | LessThanOrEqual(_, _) | EqualTo(_, _) => - falseIfNotNull(fromExp) - // make sure the expression is evaluated if it is non-deterministic - case EqualNullSafe(_, _) if exp.deterministic => - FalseLiteral + case GreaterThan(_, _) => GreaterThan(fromExp, lit) + case GreaterThanOrEqual(_, _) => GreaterThanOrEqual(fromExp, lit) + case EqualTo(_, _) => EqualTo(fromExp, lit) + case EqualNullSafe(_, _) => EqualNullSafe(fromExp, lit) + case LessThan(_, _) => LessThan(fromExp, lit) + case LessThanOrEqual(_, _) => LessThanOrEqual(fromExp, lit) case _ => exp } - } else if (minCmp == 0) { + } else if (cmp < 0) { + // This means the literal value is rounded up after casting to `fromType` exp match { - case LessThan(_, _) => - falseIfNotNull(fromExp) - case GreaterThanOrEqual(_, _) => - trueIfNotNull(fromExp) - case GreaterThan(_, _) => - Not(EqualTo(fromExp, Literal(min, fromType))) - case LessThanOrEqual(_, _) | EqualTo(_, _) => - EqualTo(fromExp, Literal(min, fromType)) - case EqualNullSafe(_, _) => - EqualNullSafe(fromExp, Literal(min, fromType)) + case EqualTo(_, _) => falseIfNotNull(fromExp) + case EqualNullSafe(_, _) if fromExp.deterministic => FalseLiteral + case GreaterThan(_, _) | GreaterThanOrEqual(_, _) => GreaterThanOrEqual(fromExp, lit) + case LessThan(_, _) | LessThanOrEqual(_, _) => LessThan(fromExp, lit) case _ => exp } } else { - // This means `value` is within range `(min, max)`. Optimize this by moving the cast to the - // literal side. - val lit = Literal(Cast(Literal(value), fromType).eval(), fromType) + // This means the literal value is rounded down after casting to `fromType` exp match { - case GreaterThan(_, _) => GreaterThan(fromExp, lit) - case GreaterThanOrEqual(_, _) => GreaterThanOrEqual(fromExp, lit) - case EqualTo(_, _) => EqualTo(fromExp, lit) - case EqualNullSafe(_, _) => EqualNullSafe(fromExp, lit) - case LessThan(_, _) => LessThan(fromExp, lit) - case LessThanOrEqual(_, _) => LessThanOrEqual(fromExp, lit) + case EqualTo(_, _) => falseIfNotNull(fromExp) + case EqualNullSafe(_, _) => FalseLiteral + case GreaterThan(_, _) | GreaterThanOrEqual(_, _) => GreaterThan(fromExp, lit) + case LessThan(_, _) | LessThanOrEqual(_, _) => LessThanOrEqual(fromExp, lit) case _ => exp } } @@ -200,7 +252,7 @@ object UnwrapCastInBinaryComparison extends Rule[LogicalPlan] { /** * Check if the input `fromExp` can be safely cast to `toType` without any loss of precision, * i.e., the conversion is injective. Note this only handles the case when both sides are of - * integral type. + * numeric type. */ private def canImplicitlyCast( fromExp: Expression, @@ -208,17 +260,19 @@ object UnwrapCastInBinaryComparison extends Rule[LogicalPlan] { literalType: DataType): Boolean = { toType.sameType(literalType) && !fromExp.foldable && - fromExp.dataType.isInstanceOf[IntegralType] && - toType.isInstanceOf[IntegralType] && + fromExp.dataType.isInstanceOf[NumericType] && + toType.isInstanceOf[NumericType] && Cast.canUpCast(fromExp.dataType, toType) } - private def getRange(dt: DataType): (Any, Any) = dt match { - case ByteType => (Byte.MinValue, Byte.MaxValue) - case ShortType => (Short.MinValue, Short.MaxValue) - case IntegerType => (Int.MinValue, Int.MaxValue) - case LongType => (Long.MinValue, Long.MaxValue) - case other => throw new IllegalArgumentException(s"Unsupported type: ${other.catalogString}") + private def getRange(dt: DataType): Option[(Any, Any)] = dt match { + case ByteType => Some((Byte.MinValue, Byte.MaxValue)) + case ShortType => Some((Short.MinValue, Short.MaxValue)) + case IntegerType => Some((Int.MinValue, Int.MaxValue)) + case LongType => Some((Long.MinValue, Long.MaxValue)) + case FloatType => Some((Float.NegativeInfinity, Float.NaN)) + case DoubleType => Some((Double.NegativeInfinity, Double.NaN)) + case _ => None } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala index 373c1febd2488..b54ec167f61f2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala @@ -36,8 +36,10 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp NullPropagation, UnwrapCastInBinaryComparison) :: Nil } - val testRelation: LocalRelation = LocalRelation('a.short, 'b.float) + val testRelation: LocalRelation = LocalRelation('a.short, 'b.float, 'c.decimal(5, 2)) val f: BoundReference = 'a.short.canBeNull.at(0) + val f2: BoundReference = 'b.float.canBeNull.at(1) + val f3: BoundReference = 'c.decimal(5, 2).canBeNull.at(2) test("unwrap casts when literal == max") { val v = Short.MaxValue @@ -47,6 +49,14 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp assertEquivalent(castInt(f) <=> v.toInt, f <=> v) assertEquivalent(castInt(f) <= v.toInt, trueIfNotNull(f)) assertEquivalent(castInt(f) < v.toInt, f =!= v) + + val d = Float.NaN + assertEquivalent(castDouble(f2) > d.toDouble, falseIfNotNull(f2)) + assertEquivalent(castDouble(f2) >= d.toDouble, f2 === d) + assertEquivalent(castDouble(f2) === d.toDouble, f2 === d) + assertEquivalent(castDouble(f2) <=> d.toDouble, f2 <=> d) + assertEquivalent(castDouble(f2) <= d.toDouble, trueIfNotNull(f2)) + assertEquivalent(castDouble(f2) < d.toDouble, f2 =!= d) } test("unwrap casts when literal > max") { @@ -67,6 +77,23 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp assertEquivalent(castInt(f) <=> v.toInt, f <=> v) assertEquivalent(castInt(f) <= v.toInt, f === v) assertEquivalent(castInt(f) < v.toInt, falseIfNotNull(f)) + + val d = Float.NegativeInfinity + assertEquivalent(castDouble(f2) > d.toDouble, f2 =!= d) + assertEquivalent(castDouble(f2) >= d.toDouble, trueIfNotNull(f2)) + assertEquivalent(castDouble(f2) === d.toDouble, f2 === d) + assertEquivalent(castDouble(f2) <=> d.toDouble, f2 <=> d) + assertEquivalent(castDouble(f2) <= d.toDouble, f2 === d) + assertEquivalent(castDouble(f2) < d.toDouble, falseIfNotNull(f2)) + + // Double.NegativeInfinity == Float.NegativeInfinity + val d2 = Double.NegativeInfinity + assertEquivalent(castDouble(f2) > d2, f2 =!= d) + assertEquivalent(castDouble(f2) >= d2, trueIfNotNull(f2)) + assertEquivalent(castDouble(f2) === d2, f2 === d) + assertEquivalent(castDouble(f2) <=> d2, f2 <=> d) + assertEquivalent(castDouble(f2) <= d2, f2 === d) + assertEquivalent(castDouble(f2) < d2, falseIfNotNull(f2)) } test("unwrap casts when literal < min") { @@ -79,13 +106,65 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp assertEquivalent(castInt(f) < v, falseIfNotNull(f)) } - test("unwrap casts when literal is within range (min, max)") { - assertEquivalent(castInt(f) > 300, f > 300.toShort) - assertEquivalent(castInt(f) >= 500, f >= 500.toShort) - assertEquivalent(castInt(f) === 32766, f === 32766.toShort) - assertEquivalent(castInt(f) <=> 32766, f <=> 32766.toShort) - assertEquivalent(castInt(f) <= -6000, f <= -6000.toShort) - assertEquivalent(castInt(f) < -32767, f < -32767.toShort) + test("unwrap casts when literal is within range (min, max) or fromType has no range") { + Seq(300, 500, 32766, -6000, -32767).foreach(v => { + assertEquivalent(castInt(f) > v, f > v.toShort) + assertEquivalent(castInt(f) >= v, f >= v.toShort) + assertEquivalent(castInt(f) === v, f === v.toShort) + assertEquivalent(castInt(f) <=> v, f <=> v.toShort) + assertEquivalent(castInt(f) <= v, f <= v.toShort) + assertEquivalent(castInt(f) < v, f < v.toShort) + }) + + Seq(3.14.toFloat.toDouble, -1000.0.toFloat.toDouble, + 20.0.toFloat.toDouble, -2.414.toFloat.toDouble, + Float.MinValue.toDouble, Float.MaxValue.toDouble, Float.PositiveInfinity.toDouble + ).foreach(v => { + assertEquivalent(castDouble(f2) > v, f2 > v.toFloat) + assertEquivalent(castDouble(f2) >= v, f2 >= v.toFloat) + assertEquivalent(castDouble(f2) === v, f2 === v.toFloat) + assertEquivalent(castDouble(f2) <=> v, f2 <=> v.toFloat) + assertEquivalent(castDouble(f2) <= v, f2 <= v.toFloat) + assertEquivalent(castDouble(f2) < v, f2 < v.toFloat) + }) + + Seq(decimal2(100.20), decimal2(-200.50)).foreach(v => { + assertEquivalent(castDecimal2(f3) > v, f3 > decimal(v)) + assertEquivalent(castDecimal2(f3) >= v, f3 >= decimal(v)) + assertEquivalent(castDecimal2(f3) === v, f3 === decimal(v)) + assertEquivalent(castDecimal2(f3) <=> v, f3 <=> decimal(v)) + assertEquivalent(castDecimal2(f3) <= v, f3 <= decimal(v)) + assertEquivalent(castDecimal2(f3) < v, f3 < decimal(v)) + }) + } + + test("unwrap cast when literal is within range (min, max) AND has round up or down") { + // Cases for rounding down + var doubleValue = 100.6 + assertEquivalent(castDouble(f) > doubleValue, f > doubleValue.toShort) + assertEquivalent(castDouble(f) > doubleValue, f > doubleValue.toShort) + assertEquivalent(castDouble(f) === doubleValue, falseIfNotNull(f)) + assertEquivalent(castDouble(f) <=> doubleValue, false) + assertEquivalent(castDouble(f) <= doubleValue, f <= doubleValue.toShort) + assertEquivalent(castDouble(f) < doubleValue, f <= doubleValue.toShort) + + // Cases for rounding up: 3.14 will be rounded to 3.14000010... after casting to float + doubleValue = 3.14 + assertEquivalent(castDouble(f2) > doubleValue, f2 >= doubleValue.toFloat) + assertEquivalent(castDouble(f2) >= doubleValue, f2 >= doubleValue.toFloat) + assertEquivalent(castDouble(f2) === doubleValue, falseIfNotNull(f2)) + assertEquivalent(castDouble(f2) <=> doubleValue, false) + assertEquivalent(castDouble(f2) <= doubleValue, f2 < doubleValue.toFloat) + assertEquivalent(castDouble(f2) < doubleValue, f2 < doubleValue.toFloat) + + // Another case: 400.5678 is rounded up to 400.57 + val decimalValue = decimal2(400.5678) + assertEquivalent(castDecimal2(f3) > decimalValue, f3 >= decimal(decimalValue)) + assertEquivalent(castDecimal2(f3) >= decimalValue, f3 >= decimal(decimalValue)) + assertEquivalent(castDecimal2(f3) === decimalValue, falseIfNotNull(f3)) + assertEquivalent(castDecimal2(f3) <=> decimalValue, false) + assertEquivalent(castDecimal2(f3) <= decimalValue, f3 < decimal(decimalValue)) + assertEquivalent(castDecimal2(f3) < decimalValue, f3 < decimal(decimalValue)) } test("unwrap casts when cast is on rhs") { @@ -100,27 +179,8 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp assertEquivalent(Literal(30) <= castInt(f), Literal(30.toShort, ShortType) <= f) } - test("unwrap cast should have no effect when input is not integral type") { - Seq( - castDouble('b) > 42.0, - castDouble('b) >= 42.0, - castDouble('b) === 42.0, - castDouble('b) <=> 42.0, - castDouble('b) <= 42.0, - castDouble('b) < 42.0, - Literal(42.0) > castDouble('b), - Literal(42.0) >= castDouble('b), - Literal(42.0) === castDouble('b), - Literal(42.0) <=> castDouble('b), - Literal(42.0) <= castDouble('b), - Literal(42.0) < castDouble('b) - ).foreach(e => - assertEquivalent(e, e, evaluate = false) - ) - } - - test("unwrap cast should skip when expression is non-deterministic or foldable") { - Seq(positiveInt, negativeInt).foreach (v => { + test("unwrap cast should skip when expression is non-deterministic or foldable") { + Seq(positiveInt, negativeInt).foreach(v => { val e = Cast(First(f, ignoreNulls = true), IntegerType) <=> v assertEquivalent(e, e, evaluate = false) val e2 = Cast(Literal(30.toShort), IntegerType) >= v @@ -139,13 +199,21 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp assertEquivalent(castInt(f) < intLit, nullLit) } + test("unwrap casts should skip if downcast failed") { + val decimalValue = decimal2(123456.1234) + assertEquivalent(castDecimal2(f3) === decimalValue, castDecimal2(f3) === decimalValue) + } + test("unwrap cast should skip if cannot coerce type") { assertEquivalent(Cast(f, ByteType) > 100.toByte, Cast(f, ByteType) > 100.toByte) } private def castInt(e: Expression): Expression = Cast(e, IntegerType) - private def castDouble(e: Expression): Expression = Cast(e, DoubleType) + private def castDecimal2(e: Expression): Expression = Cast(e, DecimalType(10, 4)) + + private def decimal(v: Decimal): Decimal = Decimal(v.toJavaBigDecimal, 5, 2) + private def decimal2(v: BigDecimal): Decimal = Decimal(v, 10, 4) private def assertEquivalent(e1: Expression, e2: Expression, evaluate: Boolean = true): Unit = { val plan = testRelation.where(e1).analyze @@ -154,8 +222,17 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp comparePlans(actual, expected) if (evaluate) { - Seq(100.toShort, -300.toShort, null).foreach(v => { - val row = create_row(v) + Seq( + (100.toShort, 3.1415926.toFloat, decimal2(100)), + (-300.toShort, 3.1415927.toFloat, decimal2(-3000.50)), + (null, Float.NaN, decimal2(-3.14)), + (null, null, null), + (Short.MaxValue, Float.PositiveInfinity, decimal2(Short.MaxValue)), + (Short.MinValue, Float.NegativeInfinity, decimal2(Short.MinValue)), + (0.toShort, Float.MaxValue, decimal2(0)), + (0.toShort, Float.MinValue, decimal2(0.01)) + ).foreach(v => { + val row = create_row(v._1, v._2, v._3) checkEvaluation(e1, e2.eval(row), row) }) } From 3fcc396075a6b17d692eba1d3c4d719082f72eff Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Thu, 17 Sep 2020 11:39:36 -0700 Subject: [PATCH 2/5] More test coverage --- .../catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala index b54ec167f61f2..7ba03348724cd 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala @@ -225,7 +225,7 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp Seq( (100.toShort, 3.1415926.toFloat, decimal2(100)), (-300.toShort, 3.1415927.toFloat, decimal2(-3000.50)), - (null, Float.NaN, decimal2(-3.14)), + (null, Float.NaN, decimal2(12345.6789)), (null, null, null), (Short.MaxValue, Float.PositiveInfinity, decimal2(Short.MaxValue)), (Short.MinValue, Float.NegativeInfinity, decimal2(Short.MinValue)), From 3340de447c551ca2c7d0e9a470b99a02a87913b1 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Thu, 1 Oct 2020 09:41:17 -0700 Subject: [PATCH 3/5] Minor refactoring --- .../sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala | 4 ---- .../optimizer/UnwrapCastInBinaryComparisonSuite.scala | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala index 4e6edf54ba678..9da9423901e02 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala @@ -56,8 +56,6 @@ import org.apache.spark.sql.types._ * - `cast(fromExp, toType) >= value` ==> `fromExp >= cast(value, fromType)` * - `cast(fromExp, toType) === value` ==> if(isnull(fromExp), null, false) * - `cast(fromExp, toType) <=> value` ==> false (if `fromExp` is deterministic) - * - `cast(fromExp, toType) <=> value` ==> `cast(fromExp, toType) <=> value` (if `fromExp` is - * non-deterministic) * - `cast(fromExp, toType) <= value` ==> `fromExp < cast(value, fromType)` * - `cast(fromExp, toType) < value` ==> `fromExp < cast(value, fromType)` * @@ -71,8 +69,6 @@ import org.apache.spark.sql.types._ * - `cast(fromExp, toType) >= value` ==> if(isnull(fromExp), null, false) * - `cast(fromExp, toType) === value` ==> if(isnull(fromExp), null, false) * - `cast(fromExp, toType) <=> value` ==> false (if `fromExp` is deterministic) - * - `cast(fromExp, toType) <=> value` ==> cast(fromExp, toType) <=> value (if `fromExp` is - * non-deterministic) * - `cast(fromExp, toType) <= value` ==> if(isnull(fromExp), null, true) * - `cast(fromExp, toType) < value` ==> if(isnull(fromExp), null, true) * diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala index 7ba03348724cd..23c989511eae0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala @@ -142,7 +142,7 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp // Cases for rounding down var doubleValue = 100.6 assertEquivalent(castDouble(f) > doubleValue, f > doubleValue.toShort) - assertEquivalent(castDouble(f) > doubleValue, f > doubleValue.toShort) + assertEquivalent(castDouble(f) >= doubleValue, f > doubleValue.toShort) assertEquivalent(castDouble(f) === doubleValue, falseIfNotNull(f)) assertEquivalent(castDouble(f) <=> doubleValue, false) assertEquivalent(castDouble(f) <= doubleValue, f <= doubleValue.toShort) @@ -223,7 +223,7 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp if (evaluate) { Seq( - (100.toShort, 3.1415926.toFloat, decimal2(100)), + (100.toShort, 3.14.toFloat, decimal2(100)), (-300.toShort, 3.1415927.toFloat, decimal2(-3000.50)), (null, Float.NaN, decimal2(12345.6789)), (null, null, null), From 94942bf2042a5c179ce9eda7fb90493636905078 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Tue, 6 Oct 2020 15:39:44 -0700 Subject: [PATCH 4/5] Add end-to-end test suite --- .../UnwrapCastInComparisonEndToEndSuite.scala | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala new file mode 100644 index 0000000000000..4b9e3e1ffd228 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.IntegralLiteralTestUtils.{negativeInt, positiveInt} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.Decimal + +class UnwrapCastInComparisonEndToEndSuite extends QueryTest with SharedSparkSession { + import testImplicits._ + + test("cases when literal is max") { + val t = "test_table" + withTable(t) { + Seq[(Integer, java.lang.Short, java.lang.Float)]( + (1, 100.toShort, 3.14.toFloat), (2, Short.MaxValue, Float.NaN), (3, null, null)) + .toDF("c1", "c2", "c3").write.saveAsTable(t) + val df = spark.table(t) + + var lit = Short.MaxValue.toInt + checkAnswer(df.select("c1").where(s"c2 > $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 >= $lit"), Row(2)) + checkAnswer(df.select("c1").where(s"c2 == $lit"), Row(2)) + checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Row(2)) + checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1)) + checkAnswer(df.select("c1").where(s"c2 <= $lit"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c2 < $lit"), Row(1)) + + checkAnswer(df.select("c1").where(s"c3 > double('nan')"), Seq.empty) + checkAnswer(df.select("c1").where(s"c3 >= double('nan')"), Row(2)) + checkAnswer(df.select("c1").where(s"c3 == double('nan')"), Row(2)) + checkAnswer(df.select("c1").where(s"c3 <=> double('nan')"), Row(2)) + checkAnswer(df.select("c1").where(s"c3 != double('nan')"), Row(1)) + checkAnswer(df.select("c1").where(s"c3 <= double('nan')"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c3 < double('nan')"), Row(1)) + + lit = positiveInt + checkAnswer(df.select("c1").where(s"c2 > $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 >= $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 == $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c2 <= $lit"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c2 < $lit"), Row(1) :: Row(2) :: Nil) + } + } + + test("cases when literal is min") { + val t = "test_table" + withTable(t) { + Seq[(Integer, java.lang.Short, java.lang.Float)]( + (1, 100.toShort, 3.14.toFloat), (2, Short.MinValue, Float.NegativeInfinity), + (3, null, null)) + .toDF("c1", "c2", "c3").write.saveAsTable(t) + val df = spark.table(t) + + var lit = Short.MinValue.toInt + checkAnswer(df.select("c1").where(s"c2 > $lit"), Row(1)) + checkAnswer(df.select("c1").where(s"c2 >= $lit"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c2 == $lit"), Row(2)) + checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Row(2)) + checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1)) + checkAnswer(df.select("c1").where(s"c2 <= $lit"), Row(2)) + checkAnswer(df.select("c1").where(s"c2 < $lit"), Seq.empty) + + checkAnswer(df.select("c1").where(s"c3 > double('-inf')"), Row(1)) + checkAnswer(df.select("c1").where(s"c3 >= double('-inf')"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c3 == double('-inf')"), Row(2)) + checkAnswer(df.select("c1").where(s"c3 <=> double('-inf')"), Row(2)) + checkAnswer(df.select("c1").where(s"c3 != double('-inf')"), Row(1)) + checkAnswer(df.select("c1").where(s"c3 <= double('-inf')"), Row(2)) + checkAnswer(df.select("c1").where(s"c3 < double('-inf')"), Seq.empty) + + lit = negativeInt + checkAnswer(df.select("c1").where(s"c2 > $lit"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c2 >= $lit"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c2 == $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where(s"c2 <= $lit"), Seq.empty) + checkAnswer(df.select("c1").where(s"c2 < $lit"), Seq.empty) + } + } + + test("cases when literal is within range (min, max)") { + val t = "test_table" + withTable(t) { + Seq((1, 300.toShort), (2, 500.toShort)).toDF("c1", "c2").write.saveAsTable(t) + val df = spark.table(t) + + checkAnswer(df.select("c1").where("c2 < 200"), Seq.empty) + checkAnswer(df.select("c1").where("c2 < 400"), Row(1) :: Nil) + checkAnswer(df.select("c1").where("c2 < 600"), Row(1) :: Row(2) :: Nil) + + checkAnswer(df.select("c1").where("c2 <= 100"), Seq.empty) + checkAnswer(df.select("c1").where("c2 <= 300"), Row(1) :: Nil) + checkAnswer(df.select("c1").where("c2 <= 500"), Row(1) :: Row(2) :: Nil) + + checkAnswer(df.select("c1").where("c2 == 100"), Seq.empty) + checkAnswer(df.select("c1").where("c2 == 300"), Row(1) :: Nil) + checkAnswer(df.select("c1").where("c2 == 500"), Row(2) :: Nil) + + checkAnswer(df.select("c1").where("c2 <=> 100"), Seq.empty) + checkAnswer(df.select("c1").where("c2 <=> 300"), Row(1) :: Nil) + checkAnswer(df.select("c1").where("c2 <=> 500"), Row(2) :: Nil) + checkAnswer(df.select("c1").where("c2 <=> null"), Seq.empty) + + checkAnswer(df.select("c1").where("c2 >= 200"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where("c2 >= 400"), Row(2) :: Nil) + checkAnswer(df.select("c1").where("c2 >= 600"), Seq.empty) + + checkAnswer(df.select("c1").where("c2 > 100"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.select("c1").where("c2 > 300"), Row(2) :: Nil) + checkAnswer(df.select("c1").where("c2 > 500"), Seq.empty) + } + } + + test("cases when literal is within range (min, max) and rounding for ") { + val t = "test_table" + withTable(t) { + Seq((1, 100, 3.14.toFloat, decimal(200.12))) + .toDF("c1", "c2", "c3", "c4").write.saveAsTable(t) + val df = spark.table(t) + + checkAnswer(df.select("c1").where("c2 > 99.6"), Row(1)) + checkAnswer(df.select("c1").where("c2 > 100.4"), Seq.empty) + checkAnswer(df.select("c1").where("c2 == 100.4"), Seq.empty) + checkAnswer(df.select("c1").where("c2 <=> 100.4"), Seq.empty) + checkAnswer(df.select("c1").where("c2 < 99.6"), Seq.empty) + checkAnswer(df.select("c1").where("c2 < 100.4"), Row(1)) + + checkAnswer(df.select("c1").where("c3 >= 3.14"), Row(1)) + // float(3.14) is casted to double(3.140000104904175) + checkAnswer(df.select("c1").where("c3 >= 3.14000010"), Row(1)) + checkAnswer(df.select("c1").where("c3 == 3.14"), Seq.empty) + checkAnswer(df.select("c1").where("c3 <=> 3.14"), Seq.empty) + checkAnswer(df.select("c1").where("c3 < 3.14000010"), Seq.empty) + checkAnswer(df.select("c1").where("c3 <= 3.14"), Seq.empty) + + checkAnswer(df.select("c1").where("c4 > cast(200.1199 as decimal(10, 4))"), Row(1)) + checkAnswer(df.select("c1").where("c4 >= cast(200.1201 as decimal(10, 4))"), Seq.empty) + checkAnswer(df.select("c1").where("c4 == cast(200.1156 as decimal(10, 4))"), Seq.empty) + checkAnswer(df.select("c1").where("c4 <=> cast(200.1201 as decimal(10, 4))"), Seq.empty) + checkAnswer(df.select("c1").where("c4 <= cast(200.1201 as decimal(10, 4))"), Row(1)) + checkAnswer(df.select("c1").where("c4 < cast(200.1159 as decimal(10, 4))"), Seq.empty) + } + } + + private def decimal(v: BigDecimal): Decimal = Decimal(v, 5, 2) +} From 76b9f73862380dda40b332b56464278bfb0b88d3 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 12 Oct 2020 12:37:02 -0700 Subject: [PATCH 5/5] Address comments - Added test case for getRange() - Added test for Float.PositiveInfinity and Float.MinValue/Float/MaxValue - Move `select` after `where` - Separate test cases - Fix indentation --- .../UnwrapCastInBinaryComparison.scala | 4 +- .../UnwrapCastInBinaryComparisonSuite.scala | 25 ++ .../UnwrapCastInComparisonEndToEndSuite.scala | 221 ++++++++++-------- 3 files changed, 152 insertions(+), 98 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala index 9da9423901e02..fe325f00e0baf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala @@ -59,7 +59,7 @@ import org.apache.spark.sql.types._ * - `cast(fromExp, toType) <= value` ==> `fromExp < cast(value, fromType)` * - `cast(fromExp, toType) < value` ==> `fromExp < cast(value, fromType)` * - * Similarly for the case when casting `value` to `fromType` causes rounding down. + * Similarly for the case when casting `value` to `fromType` causes rounding down. * * If the `value` is not within range `(min, max)`, the rule breaks the scenario into different * cases and try to replace each with simpler constructs. @@ -261,7 +261,7 @@ object UnwrapCastInBinaryComparison extends Rule[LogicalPlan] { Cast.canUpCast(fromExp.dataType, toType) } - private def getRange(dt: DataType): Option[(Any, Any)] = dt match { + private[optimizer] def getRange(dt: DataType): Option[(Any, Any)] = dt match { case ByteType => Some((Byte.MinValue, Byte.MaxValue)) case ShortType => Some((Short.MinValue, Short.MaxValue)) case IntegerType => Some((Int.MinValue, Int.MaxValue)) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala index 23c989511eae0..0afb166b80ca5 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala @@ -208,6 +208,31 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp assertEquivalent(Cast(f, ByteType) > 100.toByte, Cast(f, ByteType) > 100.toByte) } + test("test getRange()") { + assert(Some((Byte.MinValue, Byte.MaxValue)) === getRange(ByteType)) + assert(Some((Short.MinValue, Short.MaxValue)) === getRange(ShortType)) + assert(Some((Int.MinValue, Int.MaxValue)) === getRange(IntegerType)) + assert(Some((Long.MinValue, Long.MaxValue)) === getRange(LongType)) + + val floatRange = getRange(FloatType) + assert(floatRange.isDefined) + val (floatMin, floatMax) = floatRange.get + assert(floatMin.isInstanceOf[Float]) + assert(floatMin.asInstanceOf[Float].isNegInfinity) + assert(floatMax.isInstanceOf[Float]) + assert(floatMax.asInstanceOf[Float].isNaN) + + val doubleRange = getRange(DoubleType) + assert(doubleRange.isDefined) + val (doubleMin, doubleMax) = doubleRange.get + assert(doubleMin.isInstanceOf[Double]) + assert(doubleMin.asInstanceOf[Double].isNegInfinity) + assert(doubleMax.isInstanceOf[Double]) + assert(doubleMax.asInstanceOf[Double].isNaN) + + assert(getRange(DecimalType(5, 2)).isEmpty) + } + private def castInt(e: Expression): Expression = Cast(e, IntegerType) private def castDouble(e: Expression): Expression = Cast(e, DoubleType) private def castDecimal2(e: Expression): Expression = Cast(e, DecimalType(10, 4)) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala index 4b9e3e1ffd228..e6f0426428bd4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala @@ -24,140 +24,169 @@ import org.apache.spark.sql.types.Decimal class UnwrapCastInComparisonEndToEndSuite extends QueryTest with SharedSparkSession { import testImplicits._ + val t = "test_table" + test("cases when literal is max") { - val t = "test_table" withTable(t) { Seq[(Integer, java.lang.Short, java.lang.Float)]( - (1, 100.toShort, 3.14.toFloat), (2, Short.MaxValue, Float.NaN), (3, null, null)) + (1, 100.toShort, 3.14.toFloat), + (2, Short.MaxValue, Float.NaN), + (3, Short.MinValue, Float.PositiveInfinity), + (4, 0.toShort, Float.MaxValue), + (5, null, null)) .toDF("c1", "c2", "c3").write.saveAsTable(t) val df = spark.table(t) - var lit = Short.MaxValue.toInt - checkAnswer(df.select("c1").where(s"c2 > $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 >= $lit"), Row(2)) - checkAnswer(df.select("c1").where(s"c2 == $lit"), Row(2)) - checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Row(2)) - checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1)) - checkAnswer(df.select("c1").where(s"c2 <= $lit"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c2 < $lit"), Row(1)) - - checkAnswer(df.select("c1").where(s"c3 > double('nan')"), Seq.empty) - checkAnswer(df.select("c1").where(s"c3 >= double('nan')"), Row(2)) - checkAnswer(df.select("c1").where(s"c3 == double('nan')"), Row(2)) - checkAnswer(df.select("c1").where(s"c3 <=> double('nan')"), Row(2)) - checkAnswer(df.select("c1").where(s"c3 != double('nan')"), Row(1)) - checkAnswer(df.select("c1").where(s"c3 <= double('nan')"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c3 < double('nan')"), Row(1)) - - lit = positiveInt - checkAnswer(df.select("c1").where(s"c2 > $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 >= $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 == $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c2 <= $lit"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c2 < $lit"), Row(1) :: Row(2) :: Nil) + val lit = Short.MaxValue.toInt + checkAnswer(df.where(s"c2 > $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 >= $lit").select("c1"), Row(2)) + checkAnswer(df.where(s"c2 == $lit").select("c1"), Row(2)) + checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Row(2)) + checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) + checkAnswer(df.where(s"c2 <= $lit").select("c1"), Row(1) :: Row(2) :: Row(3) :: Row(4) :: Nil) + checkAnswer(df.where(s"c2 < $lit").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) + + checkAnswer(df.where(s"c3 > double('nan')").select("c1"), Seq.empty) + checkAnswer(df.where(s"c3 >= double('nan')").select("c1"), Row(2)) + checkAnswer(df.where(s"c3 == double('nan')").select("c1"), Row(2)) + checkAnswer(df.where(s"c3 <=> double('nan')").select("c1"), Row(2)) + checkAnswer(df.where(s"c3 != double('nan')").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) + checkAnswer(df.where(s"c3 <= double('nan')").select("c1"), + Row(1) :: Row(2) :: Row(3) :: Row(4) :: Nil) + checkAnswer(df.where(s"c3 < double('nan')").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) + } + } + + test("cases when literal is > max") { + withTable(t) { + Seq[(Integer, java.lang.Short)]( + (1, 100.toShort), + (2, Short.MaxValue), + (3, null)) + .toDF("c1", "c2").write.saveAsTable(t) + val df = spark.table(t) + val lit = positiveInt + checkAnswer(df.where(s"c2 > $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 >= $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 == $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where(s"c2 <= $lit").select("c1"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where(s"c2 < $lit").select("c1"), Row(1) :: Row(2) :: Nil) + + // No test for float case since NaN is greater than any other numeric value } } test("cases when literal is min") { - val t = "test_table" withTable(t) { Seq[(Integer, java.lang.Short, java.lang.Float)]( - (1, 100.toShort, 3.14.toFloat), (2, Short.MinValue, Float.NegativeInfinity), - (3, null, null)) + (1, 100.toShort, 3.14.toFloat), + (2, Short.MinValue, Float.NegativeInfinity), + (3, Short.MaxValue, Float.MinValue), + (4, null, null)) .toDF("c1", "c2", "c3").write.saveAsTable(t) val df = spark.table(t) - var lit = Short.MinValue.toInt - checkAnswer(df.select("c1").where(s"c2 > $lit"), Row(1)) - checkAnswer(df.select("c1").where(s"c2 >= $lit"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c2 == $lit"), Row(2)) - checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Row(2)) - checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1)) - checkAnswer(df.select("c1").where(s"c2 <= $lit"), Row(2)) - checkAnswer(df.select("c1").where(s"c2 < $lit"), Seq.empty) - - checkAnswer(df.select("c1").where(s"c3 > double('-inf')"), Row(1)) - checkAnswer(df.select("c1").where(s"c3 >= double('-inf')"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c3 == double('-inf')"), Row(2)) - checkAnswer(df.select("c1").where(s"c3 <=> double('-inf')"), Row(2)) - checkAnswer(df.select("c1").where(s"c3 != double('-inf')"), Row(1)) - checkAnswer(df.select("c1").where(s"c3 <= double('-inf')"), Row(2)) - checkAnswer(df.select("c1").where(s"c3 < double('-inf')"), Seq.empty) - - lit = negativeInt - checkAnswer(df.select("c1").where(s"c2 > $lit"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c2 >= $lit"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c2 == $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 <=> $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 != $lit"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where(s"c2 <= $lit"), Seq.empty) - checkAnswer(df.select("c1").where(s"c2 < $lit"), Seq.empty) + val lit = Short.MinValue.toInt + checkAnswer(df.where(s"c2 > $lit").select("c1"), Row(1) :: Row(3) :: Nil) + checkAnswer(df.where(s"c2 >= $lit").select("c1"), Row(1) :: Row(2) :: Row(3) :: Nil) + checkAnswer(df.where(s"c2 == $lit").select("c1"), Row(2)) + checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Row(2)) + checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(3) :: Nil) + checkAnswer(df.where(s"c2 <= $lit").select("c1"), Row(2)) + checkAnswer(df.where(s"c2 < $lit").select("c1"), Seq.empty) + + checkAnswer(df.where(s"c3 > double('-inf')").select("c1"), Row(1) :: Row(3) :: Nil) + checkAnswer(df.where(s"c3 >= double('-inf')").select("c1"), Row(1) :: Row(2) :: Row(3) :: Nil) + checkAnswer(df.where(s"c3 == double('-inf')").select("c1"), Row(2)) + checkAnswer(df.where(s"c3 <=> double('-inf')").select("c1"), Row(2)) + checkAnswer(df.where(s"c3 != double('-inf')").select("c1"), Row(1) :: Row(3) :: Nil) + checkAnswer(df.where(s"c3 <= double('-inf')").select("c1"), Row(2) :: Nil) + checkAnswer(df.where(s"c3 < double('-inf')").select("c1"), Seq.empty) } } - test("cases when literal is within range (min, max)") { + test("cases when literal is < min") { val t = "test_table" + withTable(t) { + Seq[(Integer, java.lang.Short)]( + (1, 100.toShort), + (2, Short.MinValue), + (3, null)) + .toDF("c1", "c2").write.saveAsTable(t) + val df = spark.table(t) + + val lit = negativeInt + checkAnswer(df.where(s"c2 > $lit").select("c1"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where(s"c2 >= $lit").select("c1"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where(s"c2 == $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where(s"c2 <= $lit").select("c1"), Seq.empty) + checkAnswer(df.where(s"c2 < $lit").select("c1"), Seq.empty) + } + } + + test("cases when literal is within range (min, max)") { withTable(t) { Seq((1, 300.toShort), (2, 500.toShort)).toDF("c1", "c2").write.saveAsTable(t) val df = spark.table(t) - checkAnswer(df.select("c1").where("c2 < 200"), Seq.empty) - checkAnswer(df.select("c1").where("c2 < 400"), Row(1) :: Nil) - checkAnswer(df.select("c1").where("c2 < 600"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where("c2 < 200").select("c1"), Seq.empty) + checkAnswer(df.where("c2 < 400").select("c1"), Row(1) :: Nil) + checkAnswer(df.where("c2 < 600").select("c1"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 <= 100"), Seq.empty) - checkAnswer(df.select("c1").where("c2 <= 300"), Row(1) :: Nil) - checkAnswer(df.select("c1").where("c2 <= 500"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where("c2 <= 100").select("c1"), Seq.empty) + checkAnswer(df.where("c2 <= 300").select("c1"), Row(1) :: Nil) + checkAnswer(df.where("c2 <= 500").select("c1"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 == 100"), Seq.empty) - checkAnswer(df.select("c1").where("c2 == 300"), Row(1) :: Nil) - checkAnswer(df.select("c1").where("c2 == 500"), Row(2) :: Nil) + checkAnswer(df.where("c2 == 100").select("c1"), Seq.empty) + checkAnswer(df.where("c2 == 300").select("c1"), Row(1) :: Nil) + checkAnswer(df.where("c2 == 500").select("c1"), Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 <=> 100"), Seq.empty) - checkAnswer(df.select("c1").where("c2 <=> 300"), Row(1) :: Nil) - checkAnswer(df.select("c1").where("c2 <=> 500"), Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 <=> null"), Seq.empty) + checkAnswer(df.where("c2 <=> 100").select("c1"), Seq.empty) + checkAnswer(df.where("c2 <=> 300").select("c1"), Row(1) :: Nil) + checkAnswer(df.where("c2 <=> 500").select("c1"), Row(2) :: Nil) + checkAnswer(df.where("c2 <=> null").select("c1"), Seq.empty) - checkAnswer(df.select("c1").where("c2 >= 200"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 >= 400"), Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 >= 600"), Seq.empty) + checkAnswer(df.where("c2 >= 200").select("c1"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where("c2 >= 400").select("c1"), Row(2) :: Nil) + checkAnswer(df.where("c2 >= 600").select("c1"), Seq.empty) - checkAnswer(df.select("c1").where("c2 > 100"), Row(1) :: Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 > 300"), Row(2) :: Nil) - checkAnswer(df.select("c1").where("c2 > 500"), Seq.empty) + checkAnswer(df.where("c2 > 100").select("c1"), Row(1) :: Row(2) :: Nil) + checkAnswer(df.where("c2 > 300").select("c1"), Row(2) :: Nil) + checkAnswer(df.where("c2 > 500").select("c1"), Seq.empty) } } - test("cases when literal is within range (min, max) and rounding for ") { - val t = "test_table" + test("cases when literal is within range (min, max) and has rounding up or down") { withTable(t) { Seq((1, 100, 3.14.toFloat, decimal(200.12))) .toDF("c1", "c2", "c3", "c4").write.saveAsTable(t) val df = spark.table(t) - checkAnswer(df.select("c1").where("c2 > 99.6"), Row(1)) - checkAnswer(df.select("c1").where("c2 > 100.4"), Seq.empty) - checkAnswer(df.select("c1").where("c2 == 100.4"), Seq.empty) - checkAnswer(df.select("c1").where("c2 <=> 100.4"), Seq.empty) - checkAnswer(df.select("c1").where("c2 < 99.6"), Seq.empty) - checkAnswer(df.select("c1").where("c2 < 100.4"), Row(1)) + checkAnswer(df.where("c2 > 99.6").select("c1"), Row(1)) + checkAnswer(df.where("c2 > 100.4").select("c1"), Seq.empty) + checkAnswer(df.where("c2 == 100.4").select("c1"), Seq.empty) + checkAnswer(df.where("c2 <=> 100.4").select("c1"), Seq.empty) + checkAnswer(df.where("c2 < 99.6").select("c1"), Seq.empty) + checkAnswer(df.where("c2 < 100.4").select("c1"), Row(1)) - checkAnswer(df.select("c1").where("c3 >= 3.14"), Row(1)) + checkAnswer(df.where("c3 >= 3.14").select("c1"), Row(1)) // float(3.14) is casted to double(3.140000104904175) - checkAnswer(df.select("c1").where("c3 >= 3.14000010"), Row(1)) - checkAnswer(df.select("c1").where("c3 == 3.14"), Seq.empty) - checkAnswer(df.select("c1").where("c3 <=> 3.14"), Seq.empty) - checkAnswer(df.select("c1").where("c3 < 3.14000010"), Seq.empty) - checkAnswer(df.select("c1").where("c3 <= 3.14"), Seq.empty) - - checkAnswer(df.select("c1").where("c4 > cast(200.1199 as decimal(10, 4))"), Row(1)) - checkAnswer(df.select("c1").where("c4 >= cast(200.1201 as decimal(10, 4))"), Seq.empty) - checkAnswer(df.select("c1").where("c4 == cast(200.1156 as decimal(10, 4))"), Seq.empty) - checkAnswer(df.select("c1").where("c4 <=> cast(200.1201 as decimal(10, 4))"), Seq.empty) - checkAnswer(df.select("c1").where("c4 <= cast(200.1201 as decimal(10, 4))"), Row(1)) - checkAnswer(df.select("c1").where("c4 < cast(200.1159 as decimal(10, 4))"), Seq.empty) + checkAnswer(df.where("c3 >= 3.14000010").select("c1"), Row(1)) + checkAnswer(df.where("c3 == 3.14").select("c1"), Seq.empty) + checkAnswer(df.where("c3 <=> 3.14").select("c1"), Seq.empty) + checkAnswer(df.where("c3 < 3.14000010").select("c1"), Seq.empty) + checkAnswer(df.where("c3 <= 3.14").select("c1"), Seq.empty) + + checkAnswer(df.where("c4 > cast(200.1199 as decimal(10, 4))").select("c1"), Row(1)) + checkAnswer(df.where("c4 >= cast(200.1201 as decimal(10, 4))").select("c1"), Seq.empty) + checkAnswer(df.where("c4 == cast(200.1156 as decimal(10, 4))").select("c1"), Seq.empty) + checkAnswer(df.where("c4 <=> cast(200.1201 as decimal(10, 4))").select("c1"), Seq.empty) + checkAnswer(df.where("c4 <= cast(200.1201 as decimal(10, 4))").select("c1"), Row(1)) + checkAnswer(df.where("c4 < cast(200.1159 as decimal(10, 4))").select("c1"), Seq.empty) } }