Skip to content

[SPARK-40851][INFRA ][SQL][TESTS][3.3] Make GA run successfully with the latest Java 8/11/17 #38363

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
LuciferYang wants to merge 1 commit into from
Closed

[SPARK-40851][INFRA ][SQL][TESTS][3.3] Make GA run successfully with the latest Java 8/11/17 #38363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
338 changes: 173 additions & 165 deletions sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
View file
Open in desktop

Large diffs are not rendered by default.

290 changes: 149 additions & 141 deletions sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions ...talyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3009,9 +3009,9 @@ object SubtractDates {
""",
examples = """
Examples:
> SELECT _FUNC_('Europe/Amsterdam', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00');
> SELECT _FUNC_('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00');
2021-12-05 15:00:00
> SELECT _FUNC_('Europe/Amsterdam', timestamp_ntz'2021-12-05 15:00:00');
> SELECT _FUNC_('Europe/Brussels', timestamp_ntz'2021-12-05 15:00:00');
2021-12-06 00:00:00
""",
group = "datetime_funcs",
Expand Down
2 changes: 1 addition & 1 deletion ...alyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1888,7 +1888,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {

checkEvaluation(
ConvertTimezone(
Literal("Europe/Amsterdam"),
Literal("Europe/Brussels"),
Literal("Europe/Moscow"),
Literal(LocalDateTime.of(2022, 3, 27, 3, 0, 0))),
LocalDateTime.of(2022, 3, 27, 4, 0, 0))
Expand Down
4 changes: 2 additions & 2 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ object DateTimeTestUtils {
CET.getId,
"Africa/Dakar",
LA.getId,
"Antarctica/Vostok",
"Asia/Urumqi",
"Asia/Hong_Kong",
"Europe/Amsterdam")
"Europe/Brussels")
val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId)

def withDefaultTimeZone[T](newDefaultTimeZone: ZoneId)(block: => T): T = {
Expand Down
14 changes: 7 additions & 7 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -937,16 +937,16 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
("1970-01-01T00:00:00", "UTC") -> ("1969-12-31T16:00:00", "America/Los_Angeles"),
("2021-12-05T22:00:00", "Europe/Moscow") -> ("2021-12-06T00:00:00", "Asia/Yekaterinburg"),
("2021-12-06T00:01:02.123456", "Asia/Yekaterinburg") ->
("2021-12-05T20:01:02.123456", "Europe/Amsterdam"),
("2021-12-05T20:01:02.123456", "Europe/Brussels"),
// 7 Nov 2021 is the DST day in the America/Los_Angeles time zone
// Sunday, 7 November 2021, 02:00:00 clocks were turned backward 1 hour to
// Sunday, 7 November 2021, 01:00:00 local standard time instead.
("2021-11-07T09:00:00", "Europe/Amsterdam") -> ("2021-11-07T01:00:00", "America/Los_Angeles"),
("2021-11-07T10:00:00", "Europe/Amsterdam") -> ("2021-11-07T01:00:00", "America/Los_Angeles"),
("2021-11-07T11:00:00", "Europe/Amsterdam") -> ("2021-11-07T02:00:00", "America/Los_Angeles"),
("2021-11-07T00:30:00", "America/Los_Angeles") -> ("2021-11-07T08:30:00", "Europe/Amsterdam"),
("2021-11-07T01:30:00", "America/Los_Angeles") -> ("2021-11-07T09:30:00", "Europe/Amsterdam"),
("2021-11-07T02:30:00", "America/Los_Angeles") -> ("2021-11-07T11:30:00", "Europe/Amsterdam")
("2021-11-07T09:00:00", "Europe/Brussels") -> ("2021-11-07T01:00:00", "America/Los_Angeles"),
("2021-11-07T10:00:00", "Europe/Brussels") -> ("2021-11-07T01:00:00", "America/Los_Angeles"),
("2021-11-07T11:00:00", "Europe/Brussels") -> ("2021-11-07T02:00:00", "America/Los_Angeles"),
("2021-11-07T00:30:00", "America/Los_Angeles") -> ("2021-11-07T08:30:00", "Europe/Brussels"),
("2021-11-07T01:30:00", "America/Los_Angeles") -> ("2021-11-07T09:30:00", "Europe/Brussels"),
("2021-11-07T02:30:00", "America/Los_Angeles") -> ("2021-11-07T11:30:00", "Europe/Brussels")
).foreach { case ((inputTs, sourceTz), (expectedTs, targetTz)) =>
val micros = DateTimeUtils.localDateTimeToMicros(LocalDateTime.parse(inputTs))
val result = DateTimeUtils.convertTimestampNtzToAnotherTz(sourceTz, targetTz, micros)
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
test("SPARK-31328: rebasing overlapped timestamps during daylight saving time") {
Seq(
LA.getId -> Seq("2019-11-03T08:00:00Z", "2019-11-03T08:30:00Z", "2019-11-03T09:00:00Z"),
"Europe/Amsterdam" ->
"Europe/Brussels" ->
Seq("2019-10-27T00:00:00Z", "2019-10-27T00:30:00Z", "2019-10-27T01:00:00Z")
).foreach { case (tz, ts) =>
withDefaultTimeZone(getZoneId(tz)) {
Expand Down
8 changes: 4 additions & 4 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
CET.getId -> 1543741872001234L,
"Africa/Dakar" -> 1543745472001234L,
"America/Los_Angeles" -> 1543774272001234L,
"Antarctica/Vostok" -> 1543723872001234L,
"Asia/Urumqi" -> 1543723872001234L,
"Asia/Hong_Kong" -> 1543716672001234L,
"Europe/Amsterdam" -> 1543741872001234L)
"Europe/Brussels" -> 1543741872001234L)
outstandingTimezonesIds.foreach { zoneId =>
val formatter = TimestampFormatter(
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
Expand All @@ -65,9 +65,9 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
CET.getId -> "2018-12-02 11:11:12.001234",
"Africa/Dakar" -> "2018-12-02 10:11:12.001234",
"America/Los_Angeles" -> "2018-12-02 02:11:12.001234",
"Antarctica/Vostok" -> "2018-12-02 16:11:12.001234",
"Asia/Urumqi" -> "2018-12-02 16:11:12.001234",
"Asia/Hong_Kong" -> "2018-12-02 18:11:12.001234",
"Europe/Amsterdam" -> "2018-12-02 11:11:12.001234")
"Europe/Brussels" -> "2018-12-02 11:11:12.001234")
outstandingTimezonesIds.foreach { zoneId =>
Seq(
TimestampFormatter(
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-functions/sql-expression-schema.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
| org.apache.spark.sql.catalyst.expressions.ConcatWs | concat_ws | SELECT concat_ws(' ', 'Spark', 'SQL') | struct<concat_ws( , Spark, SQL):string> |
| org.apache.spark.sql.catalyst.expressions.ContainsExpressionBuilder | contains | SELECT contains('Spark SQL', 'Spark') | struct<contains(Spark SQL, Spark):boolean> |
| org.apache.spark.sql.catalyst.expressions.Conv | conv | SELECT conv('100', 2, 10) | struct<conv(100, 2, 10):string> |
| org.apache.spark.sql.catalyst.expressions.ConvertTimezone | convert_timezone | SELECT convert_timezone('Europe/Amsterdam', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00') | struct<convert_timezone(Europe/Amsterdam, America/Los_Angeles, TIMESTAMP_NTZ '2021-12-06 00:00:00'):timestamp_ntz> |
| org.apache.spark.sql.catalyst.expressions.ConvertTimezone | convert_timezone | SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00') | struct<convert_timezone(Europe/Brussels, America/Los_Angeles, TIMESTAMP_NTZ '2021-12-06 00:00:00'):timestamp_ntz> |
| org.apache.spark.sql.catalyst.expressions.Cos | cos | SELECT cos(0) | struct<COS(0):double> |
| org.apache.spark.sql.catalyst.expressions.Cosh | cosh | SELECT cosh(0) | struct<COSH(0):double> |
| org.apache.spark.sql.catalyst.expressions.Cot | cot | SELECT cot(1) | struct<COT(1):double> |
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 45.678);
SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 45.678, 'CET');
SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 60.007);

SELECT convert_timezone('Europe/Amsterdam', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles');
SELECT convert_timezone('Europe/Brussels', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles');
2 changes: 1 addition & 1 deletion sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ SELECT make_timestamp_ntz(2021, 07, 11, 6, 30, 45.678, 'CET');
SELECT make_timestamp_ntz(2021, 07, 11, 6, 30, 60.007);

SELECT convert_timezone('Europe/Moscow', 'America/Los_Angeles', timestamp_ntz'2022-01-01 00:00:00');
SELECT convert_timezone('Europe/Amsterdam', timestamp_ntz'2022-03-23 00:00:00');
SELECT convert_timezone('Europe/Brussels', timestamp_ntz'2022-03-23 00:00:00');

-- Get the difference between timestamps w/o time zone in the specified units
select timestampdiff(QUARTER, timestamp_ntz'2022-01-01 01:02:03', timestamp_ntz'2022-05-02 05:06:07');
Expand Down
4 changes: 2 additions & 2 deletions sql/core/src/test/resources/sql-tests/results/timestamp-ltz.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ NULL


-- !query
SELECT convert_timezone('Europe/Amsterdam', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles')
SELECT convert_timezone('Europe/Brussels', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles')
-- !query schema
struct<convert_timezone(current_timezone(), Europe/Amsterdam, TIMESTAMP '2022-03-23 00:00:00'):timestamp_ntz>
struct<convert_timezone(current_timezone(), Europe/Brussels, TIMESTAMP '2022-03-23 00:00:00'):timestamp_ntz>
-- !query output
2022-03-23 08:00:00
4 changes: 2 additions & 2 deletions sql/core/src/test/resources/sql-tests/results/timestamp-ntz.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ struct<convert_timezone(Europe/Moscow, America/Los_Angeles, TIMESTAMP_NTZ '2022-


-- !query
SELECT convert_timezone('Europe/Amsterdam', timestamp_ntz'2022-03-23 00:00:00')
SELECT convert_timezone('Europe/Brussels', timestamp_ntz'2022-03-23 00:00:00')
-- !query schema
struct<convert_timezone(current_timezone(), Europe/Amsterdam, TIMESTAMP_NTZ '2022-03-23 00:00:00'):timestamp_ntz>
struct<convert_timezone(current_timezone(), Europe/Brussels, TIMESTAMP_NTZ '2022-03-23 00:00:00'):timestamp_ntz>
-- !query output
2022-03-23 08:00:00

Expand Down