diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala index cc75340cd8fcd..663f03e865a41 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala @@ -188,6 +188,27 @@ object RebaseDateTime { Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY)) } + private[sql] def POC_localRebaseGregorianToJulianDays(tz: TimeZone, days: Int, + hour: Int, div: Boolean): Int = { + var localDate = LocalDate.ofEpochDay(days) + if (localDate.isAfter(julianEndDate) && localDate.isBefore(gregorianStartDate)) { + localDate = gregorianStartDate + } + val utcCal = new Calendar.Builder() + // `gregory` is a hybrid calendar that supports both + // the Julian and Gregorian calendar systems + .setCalendarType("gregory") + .setTimeZone(tz) + .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth) + .setTimeOfDay(hour, 0, 0) + .build() + if (div) { + Math.toIntExact(utcCal.getTimeInMillis/MILLIS_PER_DAY) + } else { + Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY)) + } + } + /** * An optimized version of [[localRebaseGregorianToJulianDays(Int)]]. This method leverages the * pre-calculated rebasing array to save calculation. For dates of Before Common Era, the @@ -204,6 +225,14 @@ object RebaseDateTime { } } +def POC_rebaseGregorianToJulianDays(tz: TimeZone, days: Int, + hour: Int, div: Boolean): Int = { + if (days < gregorianCommonEraStartDay) { + POC_localRebaseGregorianToJulianDays(tz, days, hour, div) + } else { + rebaseDays(gregJulianDiffSwitchDay, gregJulianDiffs, days) + } + } /** * The class describes JSON records with microseconds rebasing info. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala index cb5f8e43d762f..277ceb8344f5b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala @@ -167,6 +167,19 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper { } } + test("SPARK-31579: Replace floorDiv by / in localRebaseGregorianToJulianDays()") { + val start = localDateToDays(LocalDate.of(1, 1, 1)) + val end = localDateToDays(LocalDate.of(2100, 1, 1)) + for (tz <- TimeZone.getAvailableIDs()) { + for (day <- start to end) { + for (hr <- List.range(0, 24)) { + assert(POC_rebaseGregorianToJulianDays(TimeZone.getTimeZone(tz), day, hr, false) === + POC_rebaseGregorianToJulianDays(TimeZone.getTimeZone(tz), day, hr, true)) + } + } + } + } + test("SPARK-31328: rebasing overlapped timestamps during daylight saving time") { Seq( LA.getId -> Seq("2019-11-03T08:00:00Z", "2019-11-03T08:30:00Z", "2019-11-03T09:00:00Z"),