diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index b9ba32b8ee337..8545651438395 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1994,13 +1994,17 @@ case class MakeTimestamp( day: Int, hour: Int, min: Int, - secAndNanos: Decimal, + secAndMicros: Decimal, zoneId: ZoneId): Any = { try { - val secFloor = secAndNanos.floor - val nanosPerSec = Decimal(NANOS_PER_SECOND, 10, 0) - val nanos = ((secAndNanos - secFloor) * nanosPerSec).toInt - val seconds = secFloor.toInt + assert(secAndMicros.scale == 6, + s"Seconds fraction must have 6 digits for microseconds but got ${secAndMicros.scale}") + val unscaledSecFrac = secAndMicros.toUnscaledLong + assert(secAndMicros.precision <= 8, + s"Seconds and fraction cannot have more than 8 digits but got ${secAndMicros.precision}") + val totalMicros = unscaledSecFrac.toInt // 8 digits cannot overflow Int + val seconds = Math.floorDiv(totalMicros, MICROS_PER_SECOND.toInt) + val nanos = Math.floorMod(totalMicros, MICROS_PER_SECOND.toInt) * NANOS_PER_MICROS.toInt val ldt = if (seconds == 60) { if (nanos == 0) { // This case of sec = 60 and nanos = 0 is supported for compatibility with PostgreSQL diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 7e5a71e0cb3f7..fe7f6522826ff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -751,7 +751,8 @@ object IntervalUtils { secs: Decimal): CalendarInterval = { val totalMonths = Math.addExact(months, Math.multiplyExact(years, MONTHS_PER_YEAR)) val totalDays = Math.addExact(days, Math.multiplyExact(weeks, DAYS_PER_WEEK)) - var micros = (secs * Decimal(MICROS_PER_SECOND)).toLong + assert(secs.scale == 6, "Seconds fractional must have 6 digits for microseconds") + var micros = secs.toUnscaledLong micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR)) micros = Math.addExact(micros, Math.multiplyExact(mins, MICROS_PER_MINUTE)) diff --git a/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt index 65faa752b94cb..9690ebb02dd5d 100644 --- a/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt +++ b/sql/core/benchmarks/MakeDateTimeBenchmark-jdk11-results.txt @@ -1,22 +1,22 @@ -OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws +OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz make_date(): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -prepare make_date() 3204 3323 139 31.2 32.0 1.0X -make_date(2019, 9, 16) 2529 2604 126 39.5 25.3 1.3X -make_date(*, *, *) 5102 5113 10 19.6 51.0 0.6X +prepare make_date() 3170 3321 230 31.5 31.7 1.0X +make_date(2019, 9, 16) 2359 2566 343 42.4 23.6 1.3X +make_date(*, *, *) 4398 4455 53 22.7 44.0 0.7X -OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws +OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz make_timestamp(): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -prepare make_timestamp() 3484 3513 28 0.3 3484.3 1.0X -make_timestamp(2019, 1, 2, 3, 4, 50.123456) 112 131 17 9.0 111.5 31.2X -make_timestamp(2019, 1, 2, 3, 4, 60.000000) 93 102 10 10.8 92.8 37.6X -make_timestamp(2019, 12, 31, 23, 59, 60.00) 85 88 4 11.8 84.8 41.1X -make_timestamp(*, *, *, 3, 4, 50.123456) 303 308 8 3.3 302.8 11.5X -make_timestamp(*, *, *, *, *, 0) 303 307 3 3.3 302.8 11.5X -make_timestamp(*, *, *, *, *, 60.0) 289 297 8 3.5 289.1 12.1X -make_timestamp(2019, 1, 2, *, *, *) 3576 3585 11 0.3 3576.4 1.0X -make_timestamp(*, *, *, *, *, *) 3610 3618 12 0.3 3610.0 1.0X +prepare make_timestamp() 3492 3536 70 0.3 3491.7 1.0X +make_timestamp(2019, 1, 2, 3, 4, 50.123456) 94 110 16 10.7 93.6 37.3X +make_timestamp(2019, 1, 2, 3, 4, 60.000000) 78 83 8 12.8 78.0 44.7X +make_timestamp(2019, 12, 31, 23, 59, 60.00) 66 72 6 15.1 66.2 52.7X +make_timestamp(*, *, *, 3, 4, 50.123456) 306 313 12 3.3 305.9 11.4X +make_timestamp(*, *, *, *, *, 0) 282 284 2 3.5 281.9 12.4X +make_timestamp(*, *, *, *, *, 60.0) 281 288 6 3.6 281.0 12.4X +make_timestamp(2019, 1, 2, *, *, *) 3637 3641 3 0.3 3636.7 1.0X +make_timestamp(*, *, *, *, *, *) 3635 3648 12 0.3 3634.9 1.0X diff --git a/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt b/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt index 92bcc4444e60a..157bce3a38dbf 100644 --- a/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt +++ b/sql/core/benchmarks/MakeDateTimeBenchmark-results.txt @@ -1,22 +1,22 @@ -OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws +OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz make_date(): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -prepare make_date() 2920 3239 278 34.3 29.2 1.0X -make_date(2019, 9, 16) 2322 2371 61 43.1 23.2 1.3X -make_date(*, *, *) 4553 4560 6 22.0 45.5 0.6X +prepare make_date() 2951 3182 355 33.9 29.5 1.0X +make_date(2019, 9, 16) 2325 2415 101 43.0 23.2 1.3X +make_date(*, *, *) 4556 4573 17 21.9 45.6 0.6X -OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws +OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz make_timestamp(): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -prepare make_timestamp() 3636 3673 38 0.3 3635.7 1.0X -make_timestamp(2019, 1, 2, 3, 4, 50.123456) 94 99 4 10.7 93.8 38.8X -make_timestamp(2019, 1, 2, 3, 4, 60.000000) 68 80 13 14.6 68.3 53.2X -make_timestamp(2019, 12, 31, 23, 59, 60.00) 65 79 19 15.3 65.3 55.7X -make_timestamp(*, *, *, 3, 4, 50.123456) 271 280 14 3.7 270.7 13.4X -make_timestamp(*, *, *, *, *, 0) 255 263 11 3.9 255.5 14.2X -make_timestamp(*, *, *, *, *, 60.0) 254 258 4 3.9 254.2 14.3X -make_timestamp(2019, 1, 2, *, *, *) 3714 3722 8 0.3 3713.9 1.0X -make_timestamp(*, *, *, *, *, *) 3736 3741 6 0.3 3736.3 1.0X +prepare make_timestamp() 3677 3730 87 0.3 3677.0 1.0X +make_timestamp(2019, 1, 2, 3, 4, 50.123456) 76 92 15 13.1 76.5 48.1X +make_timestamp(2019, 1, 2, 3, 4, 60.000000) 70 72 3 14.4 69.7 52.8X +make_timestamp(2019, 12, 31, 23, 59, 60.00) 68 75 12 14.7 68.0 54.1X +make_timestamp(*, *, *, 3, 4, 50.123456) 266 277 12 3.8 265.8 13.8X +make_timestamp(*, *, *, *, *, 0) 258 286 46 3.9 258.3 14.2X +make_timestamp(*, *, *, *, *, 60.0) 261 269 11 3.8 260.7 14.1X +make_timestamp(2019, 1, 2, *, *, *) 3748 3762 22 0.3 3748.3 1.0X +make_timestamp(*, *, *, *, *, *) 3761 3762 2 0.3 3760.9 1.0X