Support ISO 8601 Format in Date Format.

core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java

@@ -79,6 +79,8 @@ interface DateTimeFormatHandler {
       .build();
 
   private static final Pattern pattern = Pattern.compile("%.");
+  private static final Pattern CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
+          = Pattern.compile("(?<!%)[a-zA-Z&&[^aydmshiHIMYDSEL]]+");
   private static final String MOD_LITERAL = "%";
 
   private DateTimeFormatterUtil() {
@@ -92,7 +94,15 @@ private DateTimeFormatterUtil() {
    */
   static ExprValue getFormattedDate(ExprValue dateExpr, ExprValue formatExpr) {
     final LocalDateTime date = dateExpr.datetimeValue();
-    final Matcher matcher = pattern.matcher(formatExpr.stringValue());
+    final StringBuffer cleanFormat = new StringBuffer();
+    final Matcher m = CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
+            .matcher(formatExpr.stringValue());
+    while (m.find()) {
+      m.appendReplacement(cleanFormat,String.format("'%s'", m.group()));
+    }
+    m.appendTail(cleanFormat);
+
+    final Matcher matcher = pattern.matcher(cleanFormat.toString());
     final StringBuffer format = new StringBuffer();
     while (matcher.find()) {
       matcher.appendReplacement(format,

core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java

@@ -123,7 +123,18 @@ public void setup() {
       new DateFormatTester("2008-12-31",
           ImmutableList.of("%v","%V","%u","%U"),
           ImmutableList.of("53","52","53","52")
-      )
+      ),
+      new DateFormatTester("1998-01-31 13:14:15.012345",
+          ImmutableList.of("%Y-%m-%dT%TZ"),
+          ImmutableList.of("1998-01-31T13:14:15Z")
+      ),
+      new DateFormatTester("1998-01-31 13:14:15.012345",
+          ImmutableList.of("%Y-%m-%da %T a"),
+          ImmutableList.of("1998-01-31PM 13:14:15 PM")
+      ),
+      new DateFormatTester("1998-01-31 13:14:15.012345",
+          ImmutableList.of("%Y-%m-%db %T b"),
+          ImmutableList.of("1998-01-31b 13:14:15 b"))
   );
 
   @AllArgsConstructor

docs/user/dql/functions.rst

@@ -1004,6 +1004,8 @@ Usage: date_format(date, format) formats the date argument using the specifiers
      - A literal % character
    * - %x
      - x, for any “x” not listed above
+   * - x
+     - x, for any smallcase/uppercase alphabet except [aydmshiHIMYDSEL]
 
 Argument type: STRING/DATE/DATETIME/TIMESTAMP, STRING
 

docs/user/ppl/functions/datetime.rst

@@ -173,6 +173,8 @@ Usage: date_format(date, format) formats the date argument using the specifiers
      - A literal % character
    * - %x
      - x, for any “x” not listed above
+   * - x
+     - x, for any smallcase/uppercase alphabet except [aydmshiHIMYDSEL]
 
 Argument type: STRING/DATE/DATETIME/TIMESTAMP, STRING
 

integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java

@@ -448,4 +448,19 @@ public void testDateFormat() throws IOException {
     String dateFormatted = "4 4 4 4 Saturday 6 1998 1998 1998 98";
     verifyDateFormat(date, "date", dateFormat, dateFormatted);
   }
+
+
+  @Test
+  public void testDateFormatISO8601() throws IOException {
+    String timestamp = "1998-01-31 13:14:15.012345";
+    String timestampFormat = "%Y-%m-%dT%TZ";
+    String timestampFormatted = "1998-01-31T13:14:15Z";
+    verifyDateFormat(timestamp, "timestamp", timestampFormat, timestampFormatted);
+
+    String date = "1998-01-31";
+    String dateFormat = "%Y-%m-%dT%TZ";
+    String dateFormatted = "1998-01-31T00:00:00Z";
+    verifyDateFormat(date, "date", dateFormat, dateFormatted);
+  }
+
 }

opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java

@@ -94,6 +94,7 @@ void should_build_bucket_with_parse_expression() {
             + "      \"lang\" : \"opensearch_query_expression\"\n"
             + "    },\n"
             + "    \"missing_bucket\" : true,\n"
+            + "    \"missing_order\" : \"first\",\n"
             + "    \"order\" : \"asc\"\n"
             + "  }\n"
             + "}",