diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 2272c90384847..0130923e694b1 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -27,6 +27,8 @@ license: | - In Spark 3.1, grouping_id() returns long values. In Spark version 3.0 and earlier, this function returns int values. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.integerGroupingId` to `true`. - In Spark 3.1, SQL UI data adopts the `formatted` mode for the query plan explain results. To restore the behavior before Spark 3.0, you can set `spark.sql.ui.explainMode` to `extended`. + + - In Spark 3.1, `from_unixtime`, `unix_timestamp`,`to_unix_timestamp`, `to_timestamp` and `to_date` will fail if the specified datetime pattern is invalid. In Spark 3.0 or earlier, they result `NULL`. ## Upgrading from Spark SQL 2.4 to 3.0 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index c5ead9412a438..c5cf447c103b7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -17,15 +17,14 @@ package org.apache.spark.sql.catalyst.expressions +import java.text.ParseException import java.time.{DateTimeException, LocalDate, LocalDateTime, ZoneId} +import java.time.format.DateTimeParseException import java.time.temporal.IsoFields import java.util.Locale -import scala.util.control.NonFatal - import org.apache.commons.text.StringEscapeUtils -import org.apache.spark.SparkUpgradeException import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen._ @@ -34,7 +33,6 @@ import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, Tim import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeUtils._ import org.apache.spark.sql.catalyst.util.LegacyDateFormats.SIMPLE_DATE_FORMAT -import org.apache.spark.sql.catalyst.util.toPrettySQL import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} @@ -56,6 +54,26 @@ trait TimeZoneAwareExpression extends Expression { @transient lazy val zoneId: ZoneId = DateTimeUtils.getZoneId(timeZoneId.get) } +trait TimestampFormatterHelper extends TimeZoneAwareExpression { + + protected def formatString: Expression + + protected def isParsing: Boolean + + @transient final protected lazy val formatterOption: Option[TimestampFormatter] = + if (formatString.foldable) { + Option(formatString.eval()).map(fmt => getFormatter(fmt.toString)) + } else None + + final protected def getFormatter(fmt: String): TimestampFormatter = { + TimestampFormatter( + format = fmt, + zoneId = zoneId, + legacyFormat = SIMPLE_DATE_FORMAT, + isParsing = isParsing) + } +} + /** * Returns the current date at the start of query evaluation. * All calls of current_date within the same query return the same value. @@ -715,7 +733,7 @@ case class WeekOfYear(child: Expression) since = "1.5.0") // scalastyle:on line.size.limit case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Option[String] = None) - extends BinaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes + extends BinaryExpression with TimestampFormatterHelper with ImplicitCastInputTypes with NullIntolerant { def this(left: Expression, right: Expression) = this(left, right, None) @@ -727,33 +745,13 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression = copy(timeZoneId = Option(timeZoneId)) - @transient private lazy val formatter: Option[TimestampFormatter] = { - if (right.foldable) { - Option(right.eval()).map { format => - TimestampFormatter( - format.toString, - zoneId, - legacyFormat = SIMPLE_DATE_FORMAT, - isParsing = false) - } - } else None - } - override protected def nullSafeEval(timestamp: Any, format: Any): Any = { - val tf = if (formatter.isEmpty) { - TimestampFormatter( - format.toString, - zoneId, - legacyFormat = SIMPLE_DATE_FORMAT, - isParsing = false) - } else { - formatter.get - } - UTF8String.fromString(tf.format(timestamp.asInstanceOf[Long])) + val formatter = formatterOption.getOrElse(getFormatter(format.toString)) + UTF8String.fromString(formatter.format(timestamp.asInstanceOf[Long])) } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - formatter.map { tf => + formatterOption.map { tf => val timestampFormatter = ctx.addReferenceObj("timestampFormatter", tf) defineCodeGen(ctx, ev, (timestamp, _) => { s"""UTF8String.fromString($timestampFormatter.format($timestamp))""" @@ -774,6 +772,10 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti } override def prettyName: String = "date_format" + + override protected def formatString: Expression = right + + override protected def isParsing: Boolean = false } /** @@ -871,31 +873,21 @@ case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Op } abstract class ToTimestamp - extends BinaryExpression with TimeZoneAwareExpression with ExpectsInputTypes { + extends BinaryExpression with TimestampFormatterHelper with ExpectsInputTypes { // The result of the conversion to timestamp is microseconds divided by this factor. // For example if the factor is 1000000, the result of the expression is in seconds. protected def downScaleFactor: Long + override protected def formatString: Expression = right + override protected def isParsing = true + override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, DateType, TimestampType), StringType) override def dataType: DataType = LongType override def nullable: Boolean = true - private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String] - private lazy val formatter: TimestampFormatter = - try { - TimestampFormatter( - constFormat.toString, - zoneId, - legacyFormat = SIMPLE_DATE_FORMAT, - isParsing = true) - } catch { - case e: SparkUpgradeException => throw e - case NonFatal(_) => null - } - override def eval(input: InternalRow): Any = { val t = left.eval(input) if (t == null) { @@ -906,34 +898,18 @@ abstract class ToTimestamp epochDaysToMicros(t.asInstanceOf[Int], zoneId) / downScaleFactor case TimestampType => t.asInstanceOf[Long] / downScaleFactor - case StringType if right.foldable => - if (constFormat == null || formatter == null) { - null - } else { - try { - formatter.parse( - t.asInstanceOf[UTF8String].toString) / downScaleFactor - } catch { - case e: SparkUpgradeException => throw e - case NonFatal(_) => null - } - } case StringType => - val f = right.eval(input) - if (f == null) { + val fmt = right.eval(input) + if (fmt == null) { null } else { - val formatString = f.asInstanceOf[UTF8String].toString + val formatter = formatterOption.getOrElse(getFormatter(fmt.toString)) try { - TimestampFormatter( - formatString, - zoneId, - legacyFormat = SIMPLE_DATE_FORMAT, - isParsing = true) - .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor + formatter.parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor } catch { - case e: SparkUpgradeException => throw e - case NonFatal(_) => null + case _: DateTimeParseException | + _: DateTimeException | + _: ParseException => null } } } @@ -943,55 +919,44 @@ abstract class ToTimestamp override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val javaType = CodeGenerator.javaType(dataType) left.dataType match { - case StringType if right.foldable => + case StringType => formatterOption.map { fmt => val df = classOf[TimestampFormatter].getName - if (formatter == null) { - ExprCode.forNullValue(dataType) - } else { - val formatterName = ctx.addReferenceObj("formatter", formatter, df) - val eval1 = left.genCode(ctx) - ev.copy(code = code""" - ${eval1.code} - boolean ${ev.isNull} = ${eval1.isNull}; - $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)}; - if (!${ev.isNull}) { - try { - ${ev.value} = $formatterName.parse(${eval1.value}.toString()) / $downScaleFactor; - } catch (java.lang.IllegalArgumentException e) { - ${ev.isNull} = true; - } catch (java.text.ParseException e) { - ${ev.isNull} = true; - } catch (java.time.format.DateTimeParseException e) { - ${ev.isNull} = true; - } catch (java.time.DateTimeException e) { - ${ev.isNull} = true; - } - }""") - } - case StringType => + val formatterName = ctx.addReferenceObj("formatter", fmt, df) + nullSafeCodeGen(ctx, ev, (datetimeStr, _) => + s""" + |try { + | ${ev.value} = $formatterName.parse($datetimeStr.toString()) / $downScaleFactor; + |} catch (java.time.DateTimeException e) { + | ${ev.isNull} = true; + |} catch (java.time.format.DateTimeParseException e) { + | ${ev.isNull} = true; + |} catch (java.text.ParseException e) { + | ${ev.isNull} = true; + |} + |""".stripMargin) + }.getOrElse { val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName) val tf = TimestampFormatter.getClass.getName.stripSuffix("$") val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$") - nullSafeCodeGen(ctx, ev, (string, format) => { + val timestampFormatter = ctx.freshName("timestampFormatter") + nullSafeCodeGen(ctx, ev, (string, format) => s""" - try { - ${ev.value} = $tf$$.MODULE$$.apply( - $format.toString(), - $zid, - $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(), - true) - .parse($string.toString()) / $downScaleFactor; - } catch (java.lang.IllegalArgumentException e) { - ${ev.isNull} = true; - } catch (java.text.ParseException e) { - ${ev.isNull} = true; - } catch (java.time.format.DateTimeParseException e) { - ${ev.isNull} = true; - } catch (java.time.DateTimeException e) { - ${ev.isNull} = true; - } - """ - }) + |$tf $timestampFormatter = $tf$$.MODULE$$.apply( + | $format.toString(), + | $zid, + | $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(), + | true); + |try { + | ${ev.value} = $timestampFormatter.parse($string.toString()) / $downScaleFactor; + |} catch (java.time.format.DateTimeParseException e) { + | ${ev.isNull} = true; + |} catch (java.time.DateTimeException e) { + | ${ev.isNull} = true; + |} catch (java.text.ParseException e) { + | ${ev.isNull} = true; + |} + |""".stripMargin) + } case TimestampType => val eval1 = left.genCode(ctx) ev.copy(code = code""" @@ -1044,7 +1009,8 @@ abstract class UnixTime extends ToTimestamp { since = "1.5.0") // scalastyle:on line.size.limit case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[String] = None) - extends BinaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes { + extends BinaryExpression with TimestampFormatterHelper with ImplicitCastInputTypes + with NullIntolerant { def this(sec: Expression, format: Expression) = this(sec, format, None) @@ -1065,93 +1031,34 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[ override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression = copy(timeZoneId = Option(timeZoneId)) - private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String] - private lazy val formatter: TimestampFormatter = - try { - TimestampFormatter( - constFormat.toString, - zoneId, - legacyFormat = SIMPLE_DATE_FORMAT, - isParsing = false) - } catch { - case e: SparkUpgradeException => throw e - case NonFatal(_) => null - } - - override def eval(input: InternalRow): Any = { - val time = left.eval(input) - if (time == null) { - null - } else { - if (format.foldable) { - if (constFormat == null || formatter == null) { - null - } else { - try { - UTF8String.fromString(formatter.format(time.asInstanceOf[Long] * MICROS_PER_SECOND)) - } catch { - case e: SparkUpgradeException => throw e - case NonFatal(_) => null - } - } - } else { - val f = format.eval(input) - if (f == null) { - null - } else { - try { - UTF8String.fromString( - TimestampFormatter( - f.toString, - zoneId, - legacyFormat = SIMPLE_DATE_FORMAT, - isParsing = false) - .format(time.asInstanceOf[Long] * MICROS_PER_SECOND)) - } catch { - case e: SparkUpgradeException => throw e - case NonFatal(_) => null - } - } - } - } + override def nullSafeEval(seconds: Any, format: Any): Any = { + val fmt = formatterOption.getOrElse(getFormatter(format.toString)) + UTF8String.fromString(fmt.format(seconds.asInstanceOf[Long] * MICROS_PER_SECOND)) } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - val df = classOf[TimestampFormatter].getName - if (format.foldable) { - if (formatter == null) { - ExprCode.forNullValue(StringType) - } else { - val formatterName = ctx.addReferenceObj("formatter", formatter, df) - val t = left.genCode(ctx) - ev.copy(code = code""" - ${t.code} - boolean ${ev.isNull} = ${t.isNull}; - ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)}; - if (!${ev.isNull}) { - try { - ${ev.value} = UTF8String.fromString($formatterName.format(${t.value} * 1000000L)); - } catch (java.lang.IllegalArgumentException e) { - ${ev.isNull} = true; - } - }""") - } - } else { - val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName) + formatterOption.map { f => + val formatterName = ctx.addReferenceObj("formatter", f) + defineCodeGen(ctx, ev, (seconds, _) => + s"UTF8String.fromString($formatterName.format($seconds * 1000000L))") + }.getOrElse { val tf = TimestampFormatter.getClass.getName.stripSuffix("$") val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$") - nullSafeCodeGen(ctx, ev, (seconds, f) => { + val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName) + defineCodeGen(ctx, ev, (seconds, format) => s""" - try { - ${ev.value} = UTF8String.fromString( - $tf$$.MODULE$$.apply($f.toString(), $zid, $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(), false) - .format($seconds * 1000000L)); - } catch (java.lang.IllegalArgumentException e) { - ${ev.isNull} = true; - }""" - }) + |UTF8String.fromString( + | $tf$$.MODULE$$.apply($format.toString(), + | $zid, + | $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(), + | false).format($seconds * 1000000L)) + |""".stripMargin) } } + + override protected def formatString: Expression = format + + override protected def isParsing: Boolean = false } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala index f3b589657b254..f460404800264 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala @@ -292,14 +292,14 @@ object TimestampFormatter { legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT, isParsing: Boolean): TimestampFormatter = { val pattern = format.getOrElse(defaultPattern) - if (SQLConf.get.legacyTimeParserPolicy == LEGACY) { + val formatter = if (SQLConf.get.legacyTimeParserPolicy == LEGACY) { getLegacyFormatter(pattern, zoneId, locale, legacyFormat) } else { - val tf = new Iso8601TimestampFormatter( + new Iso8601TimestampFormatter( pattern, zoneId, locale, legacyFormat, isParsing) - tf.validatePatternString() - tf } + formatter.validatePatternString() + formatter } def getLegacyFormatter( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 2dc5990eb6103..f248a3454f39a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -23,6 +23,8 @@ import java.time.{Instant, LocalDate, ZoneId} import java.util.{Calendar, Locale, TimeZone} import java.util.concurrent.TimeUnit._ +import scala.reflect.ClassTag + import org.apache.spark.{SparkFunSuite, SparkUpgradeException} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection @@ -777,8 +779,6 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation( FromUnixTime(Literal(1000L), Literal.create(null, StringType), timeZoneId), null) - checkEvaluation( - FromUnixTime(Literal(0L), Literal("not a valid format"), timeZoneId), null) // SPARK-28072 The codegen path for non-literal input should also work checkEvaluation( @@ -792,7 +792,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } } // Test escaping of format - GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\"quote"), UTC_OPT) :: Nil) + GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\""), UTC_OPT) :: Nil) } test("unix_timestamp") { @@ -854,15 +854,13 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId), MICROSECONDS.toSeconds( DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), tz.toZoneId))) - checkEvaluation( - UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null) } } } } // Test escaping of format GenerateUnsafeProjection.generate( - UnixTimestamp(Literal("2015-07-24"), Literal("\"quote"), UTC_OPT) :: Nil) + UnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } test("to_unix_timestamp") { @@ -920,10 +918,6 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Literal(date1), Literal.create(null, StringType), timeZoneId), MICROSECONDS.toSeconds( DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), zid))) - checkEvaluation( - ToUnixTimestamp( - Literal("2015-07-24"), - Literal("not a valid format"), timeZoneId), null) // SPARK-28072 The codegen path for non-literal input should also work checkEvaluation( @@ -940,7 +934,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } // Test escaping of format GenerateUnsafeProjection.generate( - ToUnixTimestamp(Literal("2015-07-24"), Literal("\"quote"), UTC_OPT) :: Nil) + ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } test("datediff") { @@ -1169,36 +1163,28 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { MillisToTimestamp(Literal(-92233720368547758L)), "long overflow") } - test("Disable week-based date fields and quarter fields for parsing") { + test("Consistent error handling for datetime formatting and parsing functions") { - def checkSparkUpgrade(c: Char): Unit = { - checkExceptionInExpression[SparkUpgradeException]( - new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, "3.0") - checkExceptionInExpression[SparkUpgradeException]( - new ParseToDate(Literal("1"), Literal(c.toString)).child, "3.0") - checkExceptionInExpression[SparkUpgradeException]( - ToUnixTimestamp(Literal("1"), Literal(c.toString)), "3.0") - checkExceptionInExpression[SparkUpgradeException]( - UnixTimestamp(Literal("1"), Literal(c.toString)), "3.0") - } - - def checkNullify(c: Char): Unit = { - checkEvaluation(new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, null) - checkEvaluation(new ParseToDate(Literal("1"), Literal(c.toString)).child, null) - checkEvaluation(ToUnixTimestamp(Literal("1"), Literal(c.toString)), null) - checkEvaluation(UnixTimestamp(Literal("1"), Literal(c.toString)), null) + def checkException[T <: Exception : ClassTag](c: String): Unit = { + checkExceptionInExpression[T](new ParseToTimestamp(Literal("1"), Literal(c)).child, c) + checkExceptionInExpression[T](new ParseToDate(Literal("1"), Literal(c)).child, c) + checkExceptionInExpression[T](ToUnixTimestamp(Literal("1"), Literal(c)), c) + checkExceptionInExpression[T](UnixTimestamp(Literal("1"), Literal(c)), c) + if (!Set("E", "F", "q", "Q").contains(c)) { + checkExceptionInExpression[T](DateFormatClass(CurrentTimestamp(), Literal(c)), c) + checkExceptionInExpression[T](FromUnixTime(Literal(0L), Literal(c)), c) + } } Seq('Y', 'W', 'w', 'E', 'u', 'F').foreach { l => - checkSparkUpgrade(l) + checkException[SparkUpgradeException](l.toString) } - Seq('q', 'Q').foreach { l => - checkNullify(l) + Seq('q', 'Q', 'e', 'c', 'A', 'n', 'N', 'p').foreach { l => + checkException[IllegalArgumentException](l.toString) } } - test("SPARK-31896: Handle am-pm timestamp parsing when hour is missing") { checkEvaluation( new ParseToTimestamp(Literal("PM"), Literal("a")).child, diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index a63bb8526da44..06765627f5545 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -138,25 +138,11 @@ select to_timestamp("2019 40", "yyyy mm"); select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss"); -- Unsupported narrow text style -select date_format(date '2020-05-23', 'GGGGG'); -select date_format(date '2020-05-23', 'MMMMM'); -select date_format(date '2020-05-23', 'LLLLL'); -select date_format(timestamp '2020-05-23', 'EEEEE'); -select date_format(timestamp '2020-05-23', 'uuuuu'); -select date_format('2020-05-23', 'QQQQQ'); -select date_format('2020-05-23', 'qqqqq'); select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG'); select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE'); select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE'); select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE'); -select from_unixtime(12345, 'MMMMM'); -select from_unixtime(54321, 'QQQQQ'); -select from_unixtime(23456, 'aaaaa'); select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy')); select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy')); select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy')); select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy')); - -select from_unixtime(1, 'yyyyyyyyyyy-MM-dd'); -select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss'); -select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd'); diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out index a4e6e79b4573e..26adb40ce1b14 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 116 +-- Number of queries: 103 -- !query @@ -814,69 +814,6 @@ struct 2019-01-01 10:10:10 --- !query -select date_format(date '2020-05-23', 'GGGGG') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(date '2020-05-23', 'MMMMM') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(date '2020-05-23', 'LLLLL') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(timestamp '2020-05-23', 'EEEEE') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(timestamp '2020-05-23', 'uuuuu') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'uuuuu' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format('2020-05-23', 'QQQQQ') --- !query schema -struct<> --- !query output -java.lang.IllegalArgumentException -Too many pattern letters: Q - - --- !query -select date_format('2020-05-23', 'qqqqq') --- !query schema -struct<> --- !query output -java.lang.IllegalArgumentException -Too many pattern letters: q - - -- !query select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG') -- !query schema @@ -913,32 +850,6 @@ org.apache.spark.SparkUpgradeException You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html --- !query -select from_unixtime(12345, 'MMMMM') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select from_unixtime(54321, 'QQQQQ') --- !query schema -struct --- !query output -NULL - - --- !query -select from_unixtime(23456, 'aaaaa') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aaaaa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - -- !query select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy')) -- !query schema @@ -973,29 +884,3 @@ struct<> -- !query output org.apache.spark.SparkUpgradeException You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select from_unixtime(1, 'yyyyyyyyyyy-MM-dd') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss') --- !query schema -struct --- !query output -0000002018-11-17 13:33:33 - - --- !query -select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out index 38d078838ebee..15092f0a27c1f 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 116 +-- Number of queries: 103 -- !query @@ -786,64 +786,6 @@ struct 2019-01-01 10:10:10 --- !query -select date_format(date '2020-05-23', 'GGGGG') --- !query schema -struct --- !query output -AD - - --- !query -select date_format(date '2020-05-23', 'MMMMM') --- !query schema -struct --- !query output -May - - --- !query -select date_format(date '2020-05-23', 'LLLLL') --- !query schema -struct --- !query output -May - - --- !query -select date_format(timestamp '2020-05-23', 'EEEEE') --- !query schema -struct --- !query output -Saturday - - --- !query -select date_format(timestamp '2020-05-23', 'uuuuu') --- !query schema -struct --- !query output -00006 - - --- !query -select date_format('2020-05-23', 'QQQQQ') --- !query schema -struct<> --- !query output -java.lang.IllegalArgumentException -Illegal pattern character 'Q' - - --- !query -select date_format('2020-05-23', 'qqqqq') --- !query schema -struct<> --- !query output -java.lang.IllegalArgumentException -Illegal pattern character 'q' - - -- !query select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG') -- !query schema @@ -876,30 +818,6 @@ struct 1590130800 --- !query -select from_unixtime(12345, 'MMMMM') --- !query schema -struct --- !query output -December - - --- !query -select from_unixtime(54321, 'QQQQQ') --- !query schema -struct --- !query output -NULL - - --- !query -select from_unixtime(23456, 'aaaaa') --- !query schema -struct --- !query output -PM - - -- !query select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy')) -- !query schema @@ -930,27 +848,3 @@ select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy struct> -- !query output {"date":2015-10-26} - - --- !query -select from_unixtime(1, 'yyyyyyyyyyy-MM-dd') --- !query schema -struct --- !query output -00000001969-12-31 - - --- !query -select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss') --- !query schema -struct --- !query output -0000002018-11-17 13:33:33 - - --- !query -select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd') --- !query schema -struct --- !query output -00000002018-11-17 diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index dc4220ff62261..b80f36e9c2347 100755 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 116 +-- Number of queries: 103 -- !query @@ -786,69 +786,6 @@ struct 2019-01-01 10:10:10 --- !query -select date_format(date '2020-05-23', 'GGGGG') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(date '2020-05-23', 'MMMMM') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(date '2020-05-23', 'LLLLL') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(timestamp '2020-05-23', 'EEEEE') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(timestamp '2020-05-23', 'uuuuu') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'uuuuu' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format('2020-05-23', 'QQQQQ') --- !query schema -struct<> --- !query output -java.lang.IllegalArgumentException -Too many pattern letters: Q - - --- !query -select date_format('2020-05-23', 'qqqqq') --- !query schema -struct<> --- !query output -java.lang.IllegalArgumentException -Too many pattern letters: q - - -- !query select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG') -- !query schema @@ -885,32 +822,6 @@ org.apache.spark.SparkUpgradeException You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html --- !query -select from_unixtime(12345, 'MMMMM') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select from_unixtime(54321, 'QQQQQ') --- !query schema -struct --- !query output -NULL - - --- !query -select from_unixtime(23456, 'aaaaa') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aaaaa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - -- !query select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy')) -- !query schema @@ -945,29 +856,3 @@ struct<> -- !query output org.apache.spark.SparkUpgradeException You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select from_unixtime(1, 'yyyyyyyyyyy-MM-dd') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html - - --- !query -select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss') --- !query schema -struct --- !query output -0000002018-11-17 13:33:33 - - --- !query -select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd') --- !query schema -struct<> --- !query output -org.apache.spark.SparkUpgradeException -You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala index c12468a4e70f8..5cc9e156db1b5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala @@ -689,8 +689,9 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession { Row(secs(ts5.getTime)), Row(null))) // invalid format - checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')"), Seq( - Row(null), Row(null), Row(null), Row(null))) + val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") + val e = intercept[IllegalArgumentException](invalid.collect()) + assert(e.getMessage.contains('b')) } } }