Skip to content

[SPARK-40014][SQL] Support cast of decimals to ANSI intervals #37466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 6 commits into from
Closed

[SPARK-40014][SQL] Support cast of decimals to ANSI intervals #37466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
592567f
Support cast of decimal to day-time interval
MaxGekk Aug 10, 2022
379cd88
Add decimalToYearMonthInterval
MaxGekk Aug 11, 2022
4f34130
Merge remote-tracking branch 'origin/master' into cast-decimal-to-ans…
MaxGekk Aug 11, 2022
d142924
Pass the start field
MaxGekk Aug 11, 2022
ccd5999
Support cast of decimals to year-month intervals
MaxGekk Aug 11, 2022
800c3e2
Add tests for the Cast expression
MaxGekk Aug 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ object Cast {
case (StringType, _: AnsiIntervalType) => true

case (_: AnsiIntervalType, _: IntegralType | _: DecimalType) => true
case (_: IntegralType, _: AnsiIntervalType) => true
case (_: IntegralType | _: DecimalType, _: AnsiIntervalType) => true

case (_: DayTimeIntervalType, _: DayTimeIntervalType) => true
case (_: YearMonthIntervalType, _: YearMonthIntervalType) => true
Expand Down Expand Up @@ -197,7 +197,7 @@ object Cast {
case (_: DayTimeIntervalType, _: DayTimeIntervalType) => true
case (_: YearMonthIntervalType, _: YearMonthIntervalType) => true
case (_: AnsiIntervalType, _: IntegralType | _: DecimalType) => true
case (_: IntegralType, _: AnsiIntervalType) => true
case (_: IntegralType | _: DecimalType, _: AnsiIntervalType) => true

case (StringType, _: NumericType) => true
case (BooleanType, _: NumericType) => true
Expand Down Expand Up @@ -795,6 +795,9 @@ case class Cast(
b => IntervalUtils.intToDayTimeInterval(
x.integral.asInstanceOf[Integral[Any]].toInt(b), it.startField, it.endField)
}
case DecimalType.Fixed(p, s) =>
buildCast[Decimal](_, d =>
IntervalUtils.decimalToDayTimeInterval(d, p, s, it.startField, it.endField))
}

private[this] def castToYearMonthInterval(
Expand All @@ -812,6 +815,9 @@ case class Cast(
b => IntervalUtils.intToYearMonthInterval(
x.integral.asInstanceOf[Integral[Any]].toInt(b), it.startField, it.endField)
}
case DecimalType.Fixed(p, s) =>
buildCast[Decimal](_, d =>
IntervalUtils.decimalToYearMonthInterval(d, p, s, it.startField, it.endField))
}

// LongConverter
Expand Down Expand Up @@ -1815,6 +1821,13 @@ case class Cast(
$evPrim = $iu.intToDayTimeInterval($c, (byte)${it.startField}, (byte)${it.endField});
"""
}
case DecimalType.Fixed(p, s) =>
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
(c, evPrim, _) =>
code"""
$evPrim = $iu.decimalToDayTimeInterval(
$c, $p, $s, (byte)${it.startField}, (byte)${it.endField});
"""
}

private[this] def castToYearMonthIntervalCode(
Expand Down Expand Up @@ -1845,6 +1858,13 @@ case class Cast(
$evPrim = $iu.intToYearMonthInterval($c, (byte)${it.startField}, (byte)${it.endField});
"""
}
case DecimalType.Fixed(p, s) =>
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
(c, evPrim, _) =>
code"""
$evPrim = $iu.decimalToYearMonthInterval(
$c, $p, $s, (byte)${it.startField}, (byte)${it.endField});
"""
}

private[this] def decimalToTimestampCode(d: ExprValue): Block = {
Expand Down
31 changes: 31 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1283,6 +1283,20 @@ object IntervalUtils {
intToYearMonthInterval(vInt, startField, endField)
}

def decimalToYearMonthInterval(
d: Decimal, p: Int, s: Int, startField: Byte, endField: Byte): Int = {
try {
val months = if (endField == YEAR) d.toBigDecimal * MONTHS_PER_YEAR else d.toBigDecimal
months.setScale(0, BigDecimal.RoundingMode.HALF_UP).toIntExact
Copy link
Contributor

@cloud-fan cloud-fan Aug 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we have a sql reference doc for cast? we should mention the rounding behavior.

Copy link
Member Author

@MaxGekk MaxGekk Aug 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have the doc for cast: https://spark.apache.org/docs/latest/sql-ref-ansi-compliance.html#cast but it says that Spark doesn't support cast of intervals to numerics. Let me open a PR and document recent changes in casting of ANSI intervals.

Copy link
Member Author

@MaxGekk MaxGekk Aug 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cloud-fan I opened the PR #37495 which updates the doc.

} catch {
case _: ArithmeticException =>
throw QueryExecutionErrors.castingCauseOverflowError(
d,
DecimalType(p, s),
YearMonthIntervalType(startField, endField))
}
}

def yearMonthIntervalToInt(v: Int, startField: Byte, endField: Byte): Int = {
endField match {
case YEAR => v / MONTHS_PER_YEAR
Expand Down Expand Up @@ -1367,6 +1381,23 @@ object IntervalUtils {
}
}

def decimalToDayTimeInterval(
d: Decimal, p: Int, s: Int, startField: Byte, endField: Byte): Long = {
try {
val micros = endField match {
case DAY => d.toBigDecimal * MICROS_PER_DAY
case HOUR => d.toBigDecimal * MICROS_PER_HOUR
case MINUTE => d.toBigDecimal * MICROS_PER_MINUTE
case SECOND => d.toBigDecimal * MICROS_PER_SECOND
}
micros.setScale(0, BigDecimal.RoundingMode.HALF_UP).toLongExact
} catch {
case _: ArithmeticException =>
throw QueryExecutionErrors.castingCauseOverflowError(
d, DecimalType(p, s), DT(startField, endField))
}
}

def dayTimeIntervalToInt(v: Long, startField: Byte, endField: Byte): Int = {
val vLong = dayTimeIntervalToLong(v, startField, endField)
val vInt = vLong.toInt
Expand Down
9 changes: 8 additions & 1 deletion sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1273,7 +1273,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
}
}

test("cast ANSI intervals to decimals") {
test("cast ANSI intervals to/from decimals") {
Seq(
(Duration.ZERO, DayTimeIntervalType(DAY), DecimalType(10, 3)) -> Decimal(0, 10, 3),
(Duration.ofHours(-1), DayTimeIntervalType(HOUR), DecimalType(10, 1)) -> Decimal(-10, 10, 1),
Expand All @@ -1293,16 +1293,23 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(
Cast(Literal.create(duration, intervalType), targetType),
expected)
checkEvaluation(
Cast(Literal.create(expected, targetType), intervalType),
duration)
}

dayTimeIntervalTypes.foreach { it =>
checkConsistencyBetweenInterpretedAndCodegenAllowingException((child: Expression) =>
Cast(child, DecimalType.USER_DEFAULT), it)
checkConsistencyBetweenInterpretedAndCodegenAllowingException((child: Expression) =>
Cast(child, it), DecimalType.USER_DEFAULT)
}

yearMonthIntervalTypes.foreach { it =>
checkConsistencyBetweenInterpretedAndCodegenAllowingException((child: Expression) =>
Cast(child, DecimalType.USER_DEFAULT), it)
checkConsistencyBetweenInterpretedAndCodegenAllowingException((child: Expression) =>
Cast(child, it), DecimalType.USER_DEFAULT)
}
}

Expand Down
6 changes: 6 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/cast.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,3 +140,9 @@ select cast(interval '10.123' second as decimal(4, 2));
select cast(interval '10.005' second as decimal(4, 2));
select cast(interval '10.123' second as decimal(5, 2));
select cast(interval '10.123' second as decimal(1, 0));

-- cast decimals to ANSI intervals
select cast(10.123456BD as interval day to second);
select cast(80.654321BD as interval hour to minute);
select cast(-10.123456BD as interval year to month);
select cast(10.654321BD as interval month);
32 changes: 32 additions & 0 deletions sql/core/src/test/resources/sql-tests/results/ansi/cast.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -996,3 +996,35 @@ org.apache.spark.SparkArithmeticException
== SQL(line 1, position 8) ==
select cast(interval '10.123' second as decimal(1, 0))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


-- !query
select cast(10.123456BD as interval day to second)
-- !query schema
struct<CAST(10.123456 AS INTERVAL DAY TO SECOND):interval day to second>
-- !query output
0 00:00:10.123456000


-- !query
select cast(80.654321BD as interval hour to minute)
-- !query schema
struct<CAST(80.654321 AS INTERVAL HOUR TO MINUTE):interval hour to minute>
-- !query output
0 01:20:00.000000000


-- !query
select cast(-10.123456BD as interval year to month)
-- !query schema
struct<CAST(-10.123456 AS INTERVAL YEAR TO MONTH):interval year to month>
-- !query output
-0-10


-- !query
select cast(10.654321BD as interval month)
-- !query schema
struct<CAST(10.654321 AS INTERVAL MONTH):interval month>
-- !query output
0-11
Expand Down
32 changes: 32 additions & 0 deletions sql/core/src/test/resources/sql-tests/results/cast.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,3 +821,35 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
[CANNOT_CHANGE_DECIMAL_PRECISION] Decimal(compact, 10, 18, 6) cannot be represented as Decimal(1, 0). If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error.


-- !query
select cast(10.123456BD as interval day to second)
-- !query schema
struct<CAST(10.123456 AS INTERVAL DAY TO SECOND):interval day to second>
-- !query output
0 00:00:10.123456000


-- !query
select cast(80.654321BD as interval hour to minute)
-- !query schema
struct<CAST(80.654321 AS INTERVAL HOUR TO MINUTE):interval hour to minute>
-- !query output
0 01:20:00.000000000


-- !query
select cast(-10.123456BD as interval year to month)
-- !query schema
struct<CAST(-10.123456 AS INTERVAL YEAR TO MONTH):interval year to month>
-- !query output
-0-10


-- !query
select cast(10.654321BD as interval month)
-- !query schema
struct<CAST(10.654321 AS INTERVAL MONTH):interval month>
-- !query output
0-11
Expand Down
30 changes: 17 additions & 13 deletions sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import org.apache.spark.sql.functions.{lit, lower, struct, sum, udf}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy.EXCEPTION
import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
import org.apache.spark.sql.types.{DataType, DecimalType, MetadataBuilder, StructType}
import org.apache.spark.sql.types.{DataType, DecimalType, LongType, MetadataBuilder, StructType}
import org.apache.spark.util.Utils

class QueryExecutionErrorsSuite
Expand Down Expand Up @@ -655,18 +655,22 @@ class QueryExecutionErrorsSuite
}

test("CAST_OVERFLOW: from long to ANSI intervals") {
Seq("INTERVAL YEAR TO MONTH", "INTERVAL HOUR TO MINUTE").foreach { it =>
checkError(
exception = intercept[SparkArithmeticException] {
sql(s"select CAST(9223372036854775807L AS $it)").collect()
},
errorClass = "CAST_OVERFLOW",
parameters = Map(
"value" -> "9223372036854775807L",
"sourceType" -> "\"BIGINT\"",
"targetType" -> s""""$it"""",
"ansiConfig" -> s""""${SQLConf.ANSI_ENABLED.key}""""),
sqlState = "22005")
Seq(
LongType -> "9223372036854775807L",
DecimalType(19, 0) -> "9223372036854775807BD").foreach { case (sourceType, sourceValue) =>
Seq("INTERVAL YEAR TO MONTH", "INTERVAL HOUR TO MINUTE").foreach { it =>
checkError(
exception = intercept[SparkArithmeticException] {
sql(s"select CAST($sourceValue AS $it)").collect()
},
errorClass = "CAST_OVERFLOW",
parameters = Map(
"value" -> sourceValue,
"sourceType" -> s""""${sourceType.sql}"""",
"targetType" -> s""""$it"""",
"ansiConfig" -> s""""${SQLConf.ANSI_ENABLED.key}""""),
sqlState = "22005")
}
}
}
}
Expand Down