[SPARK-31879][SQL][test-java11] Make week-based pattern invalid for formatting too

docs/sql-ref-datetime-pattern.md

@@ -36,11 +36,8 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 |**M/L**|month-of-year|month|7; 07; Jul; July|
 |**d**|day-of-month|number(3)|28|
 |**Q/q**|quarter-of-year|number/text|3; 03; Q3; 3rd quarter|
-|**Y**|week-based-year|year|1996; 96|
-|**w**|week-of-week-based-year|number(2)|27|
-|**W**|week-of-month|number(1)|4|
 |**E**|day-of-week|text|Tue; Tuesday|
-|**u**|localized day-of-week|number/text|2; 02; Tue; Tuesday|
+|**e**|localized day-of-week|number/text|2; 02; Tue; Tuesday|
 |**F**|week-of-month|number(1)|3|
 |**a**|am-pm-of-day|am-pm|PM|
 |**h**|clock-hour-of-am-pm (1-12)|number(2)|12|
@@ -63,7 +60,7 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 
 The count of pattern letters determines the format.
 
-- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short form. Exactly 4 pattern letters will use the full form. Exactly 5 pattern letters will use the narrow form. 5 or more letters will fail.
+- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short text form, typically an abbreviation, e.g. day-of-week Monday might output "Mon". Exactly 4 pattern letters will use the full text form, typically the full description, e.g, day-of-week Monday might output "Monday". 5 or more letters will fail.
 
 - Number(n): The n here represents the maximum count of letters this type of datetime pattern can be used. If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary.
 
@@ -137,10 +134,4 @@ The count of pattern letters determines the format.
   During parsing, the whole section may be missing from the parsed string.
   An optional section is started by `[` and ended using `]` (or at the end of the pattern).
 
-- Symbols of 'Y', 'W', 'w', 'E', 'u', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.
-
-More details for the text style:
-
-- Short Form: Short text, typically an abbreviation. For example, day-of-week Monday might output "Mon".
-
-- Full Form: Full text, typically the full description. For example, day-of-week Monday might output "Monday".
+- Symbols of 'E', 'e', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala

@@ -46,7 +46,7 @@ class Iso8601DateFormatter(
   extends DateFormatter with DateTimeFormatterHelper {
 
   @transient
-  private lazy val formatter = getOrCreateFormatter(pattern, locale)
+  private lazy val formatter = getOrCreateFormatter(pattern, locale, isParsing)
 
   @transient
   private lazy val legacyFormatter = DateFormatter.getLegacyFormatter(
@@ -159,8 +159,8 @@ object DateFormatter {
     getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
   }
 
-  def apply(format: String, zoneId: ZoneId): DateFormatter = {
-    getFormatter(Some(format), zoneId)
+  def apply(format: String, zoneId: ZoneId, isParsing: Boolean = false): DateFormatter = {
+    getFormatter(Some(format), zoneId, isParsing = isParsing)
   }
 
   def apply(zoneId: ZoneId): DateFormatter = {

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala

@@ -234,22 +234,22 @@ private object DateTimeFormatterHelper {
     val formatter = DateTimeFormatter.ofPattern("LLL qqq", Locale.US)
     formatter.format(LocalDate.of(2000, 1, 1)) == "1 1"
   }
-  final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
+  final val unsupportedLetters = Set('A', 'c', 'n', 'N', 'p', 'Y', 'W', 'w', 'u')
   // SPARK-31892: The week-based date fields are rarely used and really confusing for parsing values
   // to datetime, especially when they are mixed with other non-week-based ones
   // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can
   // be directly resolved then the `q` do check for whether the month is valid, but if the date
   // fields is incomplete, e.g. `yM`, the checking will be bypassed.
-  final val unsupportedLettersForParsing = Set('Y', 'W', 'w', 'E', 'u', 'F', 'q', 'Q')
+  final val unsupportedLettersForParsing = Set('E', 'e', 'F', 'q', 'Q')
   final val unsupportedPatternLengths = {
     // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in
     // 2.4
-    Seq("G", "M", "L", "E", "u", "Q", "q").map(_ * 5) ++
+    Seq("G", "M", "L", "E", "e", "Q", "q").map(_ * 5) ++
       // SPARK-31867: Disable year pattern longer than 10 which will cause Java time library throw
       // unchecked `ArrayIndexOutOfBoundsException` by the `NumberPrinterParser` for formatting. It
       // makes the call side difficult to handle exceptions and easily leads to silent data change
       // because of the exceptions being suppressed.
-      Seq("y", "Y").map(_ * 11)
+      Seq("y").map(_ * 11)
   }.toSet
 
   /**
@@ -282,20 +282,13 @@ private object DateTimeFormatterHelper {
               "or upgrade your Java version. For more details, please read " +
               "https://bugs.openjdk.java.net/browse/JDK-8114833")
           }
-          // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
-          // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
-          // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
-          // legacy SimpleDateFormat parser to parse. When it is successfully parsed, throw an
-          // exception and ask users to change the pattern strings or turn on the legacy mode;
-          // otherwise, return NULL as what Spark 2.4 does.
-          val res = patternPart.replace("u", "e")
           // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
           // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
           // We only do this substitution when there is no era designator found in the pattern.
           if (!eraDesignatorContained) {
-            res.replace("y", "u")
+            patternPart.replace("y", "u")
           } else {
-            res
+            patternPart
           }
         } else {
           patternPart

sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala → sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala

@@ -15,19 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.util
+package org.apache.spark.sql.catalyst.util
 
 import java.time.{DateTimeException, LocalDate}
 
-import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{DateFormatter, LegacyDateFormats}
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
-class DateFormatterSuite extends SparkFunSuite with SQLHelper {
+class DateFormatterSuite extends DatetimeFormatterSuite {
   test("parsing dates") {
     outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
@@ -199,4 +197,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
     // SparkUpgradeException here.
     intercept[SparkUpgradeException](formatter.parse("02-29"))
   }
+
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    DateFormatter(pattern, UTC, isParsing)
+  }
 }

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala

@@ -23,13 +23,10 @@ import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
 class DateTimeFormatterHelperSuite extends SparkFunSuite {
 
   test("check incompatible pattern") {
-    assert(convertIncompatiblePattern("MM-DD-u") === "MM-DD-e")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
       === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
     assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
       === "uuuu-MM'y contains in quoted text'HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM-dd-u'T'HH:mm:ss.SSSz")
-      === "uuuu-MM-dd-e'T'HH:mm:ss.SSSz")
     assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
       === "uuuu-MM'u contains in quoted text'HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
@@ -57,7 +54,6 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
       }
       assert(e2.getMessage === s"Too many pattern letters: ${style.head}")
     }
-    assert(convertIncompatiblePattern("yyyy-MM-dd uuuu") === "uuuu-MM-dd eeee")
     assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
     assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'") === "uuuu-MM-dd'T'")

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.scalatest.Matchers
+
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+
+trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
+  import DateTimeFormatterHelper._
+  def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit
+
+  test("explicitly forbidden datetime patterns") {
+
+    Seq(true, false).foreach { isParsing =>
+      // not support by the legacy one too
+      val unsupportedBoth = Seq("QQQQQ", "qqqqq", "eeeee", "A", "c", "n", "N", "p")
+      unsupportedBoth.foreach { pattern =>
+        intercept[IllegalArgumentException](checkFormatterCreation(pattern, isParsing))
+      }
+      // supported by the legacy one, then we will suggest users with SparkUpgradeException
+      (unsupportedLetters.map(_.toString) ++ unsupportedPatternLengths -- unsupportedBoth).foreach {
+        pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, isParsing))
+      }
+    }
+
+    // not support by the legacy one too
+    val unsupportedBoth = Seq("e", "q", "Q")
+    unsupportedBoth.foreach { pattern =>
+      intercept[IllegalArgumentException](checkFormatterCreation(pattern, true))
+    }
+    // supported by the legacy one, then we will suggest users with SparkUpgradeException
+    (unsupportedLettersForParsing.map(_.toString) -- unsupportedBoth).foreach {
+      pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, true))
+    }
+  }
+}

sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala → sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.util
+package org.apache.spark.sql.catalyst.util
 
 import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime}
 import java.util.concurrent.TimeUnit
@@ -24,14 +24,13 @@ import org.scalatest.Matchers
 
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{LegacyDateFormats, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.unsafe.types.UTF8String
 
-class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
+class TimestampFormatterSuite extends DatetimeFormatterSuite {
 
   test("parsing timestamps using time zones") {
     val localDate = "2018-12-02T10:11:12.001234"
@@ -418,15 +417,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     assert(t5 === date(1970))
   }
 
-  test("explicitly forbidden datetime patterns") {
-    // not support by the legacy one too
-    Seq("QQQQQ", "qqqqq", "A", "c", "e", "n", "N", "p").foreach { pattern =>
-      intercept[IllegalArgumentException](TimestampFormatter(pattern, UTC).format(0))
-    }
-    // supported by the legacy one, then we will suggest users with SparkUpgradeException
-    Seq("GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "aa", "aaa", "y" * 11, "y" * 11)
-      .foreach { pattern =>
-        intercept[SparkUpgradeException](TimestampFormatter(pattern, UTC).format(0))
-    }
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    TimestampFormatter(pattern, UTC, isParsing)
   }
 }

sql/core/src/test/resources/sql-tests/inputs/datetime.sql

@@ -125,8 +125,8 @@ select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'");
 select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd");
 
 select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee');
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc');
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu');
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eecc');
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eeee');
 
 select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS"); -- middle
 select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''"); -- tail

sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out

@@ -740,11 +740,11 @@ select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
 struct<>
 -- !query output
 java.lang.IllegalArgumentException
-Illegal pattern character: e
+Illegal pattern character: u
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eecc')
 -- !query schema
 struct<>
 -- !query output
@@ -753,9 +753,9 @@ Illegal pattern character: c
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eeee')
 -- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd eeee):string>
 -- !query output
 2019-10-06 Sunday
 

sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out

@@ -716,20 +716,21 @@ Illegal pattern character 'e'
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eecc')
 -- !query schema
 struct<>
 -- !query output
 java.lang.IllegalArgumentException
-Illegal pattern character 'c'
+Illegal pattern character 'e'
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eeee')
 -- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+struct<>
 -- !query output
-2019-10-06 0007
+java.lang.IllegalArgumentException
+Illegal pattern character 'e'
 
 
 -- !query

sql/core/src/test/resources/sql-tests/results/datetime.sql.out

@@ -712,11 +712,11 @@ select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
 struct<>
 -- !query output
 java.lang.IllegalArgumentException
-Illegal pattern character: e
+Illegal pattern character: u
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eecc')
 -- !query schema
 struct<>
 -- !query output
@@ -725,9 +725,9 @@ Illegal pattern character: c
 
 
 -- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eeee')
 -- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd eeee):string>
 -- !query output
 2019-10-06 Sunday