diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala index 8965a81feee1..a52c3450e83d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala @@ -259,7 +259,7 @@ class JacksonParser( // In Spark 1.5.0, we store the data as number of days since epoch in string. // So, we just convert it to Int. try { - parser.getText.toInt + RebaseDateTime.rebaseJulianToGregorianDays(parser.getText.toInt) } catch { case _: NumberFormatException => throw e } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 8d4d89f76bfc..dcea4835b5f2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -2653,13 +2653,17 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson } } - test("SPARK-30960: parse date/timestamp string with legacy format") { - val ds = Seq("{'t': '2020-1-12 3:23:34.12', 'd': '2020-1-12 T', 'd2': '12345'}").toDS() - val json = spark.read.schema("t timestamp, d date, d2 date").json(ds) + test("SPARK-30960, SPARK-31641: parse date/timestamp string with legacy format") { + val julianDay = -141704 // 1582-01-01 in Julian calendar + val ds = Seq( + s"{'t': '2020-1-12 3:23:34.12', 'd': '2020-1-12 T', 'd2': '12345', 'd3': '$julianDay'}" + ).toDS() + val json = spark.read.schema("t timestamp, d date, d2 date, d3 date").json(ds) checkAnswer(json, Row( Timestamp.valueOf("2020-1-12 3:23:34.12"), Date.valueOf("2020-1-12"), - Date.valueOf(LocalDate.ofEpochDay(12345)))) + Date.valueOf(LocalDate.ofEpochDay(12345)), + Date.valueOf("1582-01-01"))) } test("exception mode for parsing date/timestamp string") {