Support ISO8601-formatted string in PPL

core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java

@@ -8,6 +8,7 @@
 import com.google.common.base.Objects;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -32,7 +33,14 @@ public class ExprDateValue extends AbstractExprValue {
    */
   public ExprDateValue(String date) {
     try {
-      this.date = LocalDate.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
+      LocalDateTime ldt;
+      try {
+        ldt = LocalDateTime.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
+      } catch (DateTimeParseException ignored) {
+        ZonedDateTime zdt = ZonedDateTime.parse(date, DateTimeFormatter.ISO_DATE_TIME);
+        ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime();
+      }
+      this.date = ldt.toLocalDate();
     } catch (DateTimeParseException e) {
       throw new ExpressionEvaluationException(
           String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", date));

core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java

@@ -10,8 +10,10 @@
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalTime;
+import java.time.OffsetTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.util.Objects;
 import lombok.RequiredArgsConstructor;
@@ -34,7 +36,23 @@ public class ExprTimeValue extends AbstractExprValue {
    */
   public ExprTimeValue(String time) {
     try {
-      this.time = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER);
+      LocalTime lt;
+      try {
+        lt = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER);
+      } catch (DateTimeParseException ignore) {
+        try {
+          lt =
+              ZonedDateTime.parse(time, DateTimeFormatter.ISO_DATE_TIME)
+                  .withZoneSameInstant(ZoneOffset.UTC)
+                  .toLocalTime();
+        } catch (DateTimeParseException ignore2) {
+          lt =
+              OffsetTime.parse(time, DateTimeFormatter.ISO_TIME)
+                  .withOffsetSameInstant(ZoneOffset.UTC)
+                  .toLocalTime();
+        }
+      }
+      this.time = lt;
     } catch (DateTimeParseException e) {
       throw new ExpressionEvaluationException(
           String.format("time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", time));

core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java

@@ -13,6 +13,8 @@
 import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.time.temporal.ChronoUnit;
 import java.util.Objects;
@@ -31,17 +33,24 @@ public class ExprTimestampValue extends AbstractExprValue {
   /**
    * Constructor with timestamp string.
    *
-   * @param timestamp a date or timestamp string (does not accept time string)
+   * @param timestamp a date or timestamp string (does not accept time string). It accepts both ISO
+   *     8601 format and {@code yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]} format
    */
   public ExprTimestampValue(String timestamp) {
     try {
-      this.timestamp =
-          LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER)
-              .toInstant(ZoneOffset.UTC);
+      LocalDateTime ldt;
+      try {
+        ldt = LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
+      } catch (DateTimeParseException ignored) {
+        ZonedDateTime zdt = ZonedDateTime.parse(timestamp, DateTimeFormatter.ISO_DATE_TIME);
+        ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime();
+      }
+      this.timestamp = ldt.toInstant(ZoneOffset.UTC);
     } catch (DateTimeParseException e) {
       throw new ExpressionEvaluationException(
           String.format(
-              "timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
+              "timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or"
+                  + " ISO 8601 format",
               timestamp));
     }
   }

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -265,7 +265,11 @@ public RelNode optimize(RelNode plan, CalcitePlanContext context) {
 
   private boolean isCalciteFallbackAllowed() {
     if (settings != null) {
-      return settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
+      Boolean fallback_allowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
+      if (fallback_allowed == null) {
+        return false;
+      }
+      return fallback_allowed;
     } else {
       return true;
     }

core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java

@@ -16,6 +16,7 @@
 import java.time.LocalTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import org.junit.jupiter.api.Test;
 import org.opensearch.sql.exception.ExpressionEvaluationException;
 import org.opensearch.sql.expression.function.FunctionProperties;
@@ -112,15 +113,21 @@ public void timeInUnsupportedFormat() {
   }
 
   @Test
-  public void timestampInUnsupportedFormat() {
-    Throwable exception =
-        assertThrows(
-            ExpressionEvaluationException.class,
-            () -> new ExprTimestampValue("2020-07-07T01:01:01Z"));
+  public void timestampInISO8601Format() {
+    ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01Z");
     assertEquals(
-        "timestamp:2020-07-07T01:01:01Z in unsupported format, "
-            + "please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
-        exception.getMessage());
+        LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
+            .toInstant(ZoneOffset.UTC),
+        timestampValue.timestampValue());
+  }
+
+  @Test
+  public void timestampInISO8601FormatWithTimeZone() {
+    ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01-01:00");
+    assertEquals(
+        LocalDateTime.parse("2020-07-07T02:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
+            .toInstant(ZoneOffset.UTC),
+        timestampValue.timestampValue());
   }
 
   @Test
@@ -134,13 +141,11 @@ public void stringTimestampValue() {
     assertEquals(LocalTime.parse("19:44:00"), stringValue.timeValue());
     assertEquals("\"2020-08-17 19:44:00\"", stringValue.toString());
 
-    Throwable exception =
-        assertThrows(
-            ExpressionEvaluationException.class,
-            () -> new ExprStringValue("2020-07-07T01:01:01Z").timestampValue());
+    ExprValue stringValueWithIsoTimestamp = new ExprStringValue("2020-07-07T01:01:01Z");
     assertEquals(
-        "date:2020-07-07T01:01:01Z in unsupported format, " + "please use 'yyyy-MM-dd'",
-        exception.getMessage());
+        LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
+            .toInstant(ZoneOffset.UTC),
+        stringValueWithIsoTimestamp.timestampValue());
   }
 
   @Test
@@ -221,7 +226,7 @@ public void timestampOverMaxNanoPrecision() {
             () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567890"));
     assertEquals(
         "timestamp:2020-07-07 01:01:01.1234567890 in unsupported format, please use "
-            + "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
+            + "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format",
         exception.getMessage());
   }
 

core/src/test/java/org/opensearch/sql/expression/datetime/TimestampTest.java

@@ -64,7 +64,8 @@ public void timestamp_one_arg_string_invalid_format(String value, String testNam
             () -> DSL.timestamp(functionProperties, DSL.literal(value)).valueOf());
     assertEquals(
         String.format(
-            "timestamp:%s in unsupported format, please " + "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
+            "timestamp:%s in unsupported format, please "
+                + "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format",
             value),
         exception.getMessage());
   }

integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java

@@ -707,6 +707,27 @@ public void testComparisonBetweenDateAndTimestamp() throws IOException {
     verifyDataRows(actual, rows(2));
   }
 
+  @Test
+  public void testComparisonWithIso8601DateLiteral() {
+    Object[][] tests = {
+      {"date_optional_time = '1984-04-12T09:07:42.000Z'", 2},
+      {"date_time = '1984-04-12T07:07:42-02:00'", 2},
+      {"date = '1984-04-12'", 2},
+      {"date = '1984-04-12T09:07:42.000Z'", 2},
+      {"basic_t_time = '1984-04-12T09:07:42.000Z'", 2},
+      {"basic_t_time = '10:07:42.000+01:00'", 2}
+    };
+    for (Object[] pair : tests) {
+      String query = (String) pair[0];
+      int result = (int) pair[1];
+      JSONObject actual =
+          executeQuery(
+              String.format(
+                  "source=%s | where %s | stats COUNT() AS cnt", TEST_INDEX_DATE_FORMATS, query));
+      verifyDataRows(actual, rows(result));
+    }
+  }
+
   @Test
   public void testAddSubTime() throws IOException {
     JSONObject actual =