Support Alternate Datetime Formats

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -651,6 +651,10 @@ public enum Index {
         "calcs",
         getMappingFile("calcs_index_mappings.json"),
         "src/test/resources/calcs.json"),
+    DATE_FORMATS(TestsConstants.TEST_INDEX_DATE_FORMATS,
+        "date_formats",
+        getMappingFile("date_formats_index_mapping.json"),
+        "src/test/resources/date_formats.json"),
     WILDCARD(TestsConstants.TEST_INDEX_WILDCARD,
         "wildcard",
         getMappingFile("wildcard_index_mappings.json"),

integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java

@@ -56,6 +56,7 @@ public class TestsConstants {
   public final static String TEST_INDEX_BEER = TEST_INDEX + "_beer";
   public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing";
   public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs";
+  public final static String TEST_INDEX_DATE_FORMATS = TEST_INDEX + "_date_formats";
   public final static String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard";
   public final static String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested";
   public final static String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";

integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java

@@ -8,6 +8,7 @@
 
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS;
 import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
@@ -40,7 +41,7 @@ public void init() throws Exception {
     loadIndex(Index.BANK);
     loadIndex(Index.CALCS);
     loadIndex(Index.PEOPLE2);
-    loadIndex(Index.CALCS);
+    loadIndex(Index.DATE_FORMATS);
   }
 
   // Integration test framework sets for OpenSearch instance a random timezone.
@@ -1278,6 +1279,31 @@ public void testTimeFormat() throws IOException {
     verifyTimeFormat(timestamp, "timestamp", timestampFormat, timestampFormatted);
   }
 
+  @Test
+  public void testReadingDateFormats() throws IOException {
+    String query = String.format("SELECT weekyear_week_day, hour_minute_second_millis," +
+        " strict_ordinal_date_time FROM %s LIMIT 1", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("weekyear_week_day", null, "date"),
+        schema("hour_minute_second_millis", null, "time"),
+        schema("strict_ordinal_date_time", null, "timestamp"));
+    verifyDataRows(result,
+        rows("1984-04-12",
+            "09:07:42",
+            "1984-04-12 09:07:42.000123456"
+        ));
+  }
+
+  @Test
+  public void testDateFormatsWithOr() throws IOException {
+    String query = String.format("SELECT yyyy-MM-dd_OR_epoch_millis FROM %s", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifyDataRows(result,
+        rows("1984-04-12 00:00:00"),
+        rows("1984-04-12 09:07:42.000123456"));
+  }
+
   protected JSONObject executeQuery(String query) throws IOException {
     Request request = new Request("POST", QUERY_API_ENDPOINT);
     request.setJsonEntity(String.format(Locale.ROOT, "{\n" + "  \"query\": \"%s\"\n" + "}", query));

integ-test/src/test/resources/date_formats.json

@@ -0,0 +1,4 @@
+{"index": {}}
+{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42"}
+{"index": {}}
+{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z"}

integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json

@@ -0,0 +1,306 @@
+{
+  "mappings" : {
+    "properties" : {
+      "epoch_millis" : {
+        "type" : "date",
+        "format" : "epoch_millis"
+      },
+      "epoch_second" : {
+        "type" : "date",
+        "format" : "epoch_second"
+      },
+      "date_optional_time" : {
+        "type" : "date",
+        "format" : "date_optional_time"
+      },
+      "strict_date_optional_time" : {
+        "type" : "date",
+        "format" : "strict_date_optional_time"
+      },
+      "strict_date_optional_time_nanos" : {
+        "type" : "date",
+        "format" : "strict_date_optional_time_nanos"
+      },
+      "basic_date" : {
+        "type" : "date",
+        "format" : "basic_date"
+      },
+      "basic_date_time" : {
+        "type" : "date",
+        "format" : "basic_date_time"
+      },
+      "basic_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "basic_date_time_no_millis"
+      },
+      "basic_ordinal_date" : {
+        "type" : "date",
+        "format" : "basic_ordinal_date"
+      },
+      "basic_ordinal_date_time" : {
+        "type" : "date",
+        "format" : "basic_ordinal_date_time"
+      },
+      "basic_ordinal_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "basic_ordinal_date_time_no_millis"
+      },
+      "basic_time" : {
+        "type" : "date",
+        "format" : "basic_time"
+      },
+      "basic_time_no_millis" : {
+        "type" : "date",
+        "format" : "basic_time_no_millis"
+      },
+      "basic_t_time" : {
+        "type" : "date",
+        "format" : "basic_t_time"
+      },
+      "basic_t_time_no_millis" : {
+        "type" : "date",
+        "format" : "basic_t_time_no_millis"
+      },
+      "basic_week_date" : {
+        "type" : "date",
+        "format" : "basic_week_date"
+      },
+      "strict_basic_week_date" : {
+        "type" : "date",
+        "format" : "strict_basic_week_date"
+      },
+      "basic_week_date_time" : {
+        "type" : "date",
+        "format" : "basic_week_date_time"
+      },
+      "strict_basic_week_date_time" : {
+        "type" : "date",
+        "format" : "strict_basic_week_date_time"
+      },
+      "basic_week_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "basic_week_date_time_no_millis"
+      },
+      "strict_basic_week_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "strict_basic_week_date_time_no_millis"
+      },
+      "date" : {
+        "type" : "date",
+        "format" : "date"
+      },
+      "strict_date" : {
+        "type" : "date",
+        "format" : "strict_date"
+      },
+      "date_hour" : {
+        "type" : "date",
+        "format" : "date_hour"
+      },
+      "strict_date_hour" : {
+        "type" : "date",
+        "format" : "strict_date_hour"
+      },
+      "date_hour_minute" : {
+        "type" : "date",
+        "format" : "date_hour_minute"
+      },
+      "strict_date_hour_minute" : {
+        "type" : "date",
+        "format" : "strict_date_hour_minute"
+      },
+      "date_hour_minute_second" : {
+        "type" : "date",
+        "format" : "date_hour_minute_second"
+      },
+      "strict_date_hour_minute_second" : {
+        "type" : "date",
+        "format" : "strict_date_hour_minute_second"
+      },
+      "date_hour_minute_second_fraction" : {
+        "type" : "date",
+        "format" : "date_hour_minute_second_fraction"
+      },
+      "strict_date_hour_minute_second_fraction" : {
+        "type" : "date",
+        "format" : "strict_date_hour_minute_second_fraction"
+      },
+      "date_hour_minute_second_millis" : {
+        "type" : "date",
+        "format" : "date_hour_minute_second_millis"
+      },
+      "strict_date_hour_minute_second_millis" : {
+        "type" : "date",
+        "format" : "strict_date_hour_minute_second_millis"
+      },
+      "date_time" : {
+        "type" : "date",
+        "format" : "date_time"
+      },
+      "strict_date_time" : {
+        "type" : "date",
+        "format" : "strict_date_time"
+      },
+      "date_time_no_millis" : {
+        "type" : "date",
+        "format" : "date_time_no_millis"
+      },
+      "strict_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "strict_date_time_no_millis"
+      },
+      "hour" : {
+        "type" : "date",
+        "format" : "hour"
+      },
+      "strict_hour" : {
+        "type" : "date",
+        "format" : "strict_hour"
+      },
+      "hour_minute" : {
+        "type" : "date",
+        "format" : "hour_minute"
+      },
+      "strict_hour_minute" : {
+        "type" : "date",
+        "format" : "strict_hour_minute"
+      },
+      "hour_minute_second" : {
+        "type" : "date",
+        "format" : "hour_minute_second"
+      },
+      "strict_hour_minute_second" : {
+        "type" : "date",
+        "format" : "strict_hour_minute_second"
+      },
+      "hour_minute_second_fraction" : {
+        "type" : "date",
+        "format" : "hour_minute_second_fraction"
+      },
+      "strict_hour_minute_second_fraction" : {
+        "type" : "date",
+        "format" : "strict_hour_minute_second_fraction"
+      },
+      "hour_minute_second_millis" : {
+        "type" : "date",
+        "format" : "hour_minute_second_millis"
+      },
+      "strict_hour_minute_second_millis" : {
+        "type" : "date",
+        "format" : "strict_hour_minute_second_millis"
+      },
+      "ordinal_date" : {
+        "type" : "date",
+        "format" : "ordinal_date"
+      },
+      "strict_ordinal_date" : {
+        "type" : "date",
+        "format" : "strict_ordinal_date"
+      },
+      "ordinal_date_time" : {
+        "type" : "date",
+        "format" : "ordinal_date_time"
+      },
+      "strict_ordinal_date_time" : {
+        "type" : "date",
+        "format" : "strict_ordinal_date_time"
+      },
+      "ordinal_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "ordinal_date_time_no_millis"
+      },
+      "strict_ordinal_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "strict_ordinal_date_time_no_millis"
+      },
+      "time" : {
+        "type" : "date",
+        "format" : "time"
+      },
+      "strict_time" : {
+        "type" : "date",
+        "format" : "strict_time"
+      },
+      "time_no_millis" : {
+        "type" : "date",
+        "format" : "time_no_millis"
+      },
+      "strict_time_no_millis" : {
+        "type" : "date",
+        "format" : "strict_time_no_millis"
+      },
+      "t_time" : {
+        "type" : "date",
+        "format" : "t_time"
+      },
+      "strict_t_time" : {
+        "type" : "date",
+        "format" : "strict_t_time"
+      },
+      "t_time_no_millis" : {
+        "type" : "date",
+        "format" : "t_time_no_millis"
+      },
+      "strict_t_time_no_millis" : {
+        "type" : "date",
+        "format" : "strict_t_time_no_millis"
+      },
+      "week_date" : {
+        "type" : "date",
+        "format" : "week_date"
+      },
+      "strict_week_date" : {
+        "type" : "date",
+        "format" : "strict_week_date"
+      },
+      "week_date_time" : {
+        "type" : "date",
+        "format" : "week_date_time"
+      },
+      "strict_week_date_time" : {
+        "type" : "date",
+        "format" : "strict_week_date_time"
+      },
+      "week_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "week_date_time_no_millis"
+      },
+      "strict_week_date_time_no_millis" : {
+        "type" : "date",
+        "format" : "strict_week_date_time_no_millis"
+      },
+      "weekyear_week_day" : {
+        "type" : "date",
+        "format" : "weekyear_week_day"
+      },
+      "strict_weekyear_week_day" : {
+        "type" : "date",
+        "format" : "strict_weekyear_week_day"
+      },
+      "year_month_day" : {
+        "type" : "date",
+        "format" : "year_month_day"
+      },
+      "strict_year_month_day" : {
+        "type" : "date",
+        "format" : "strict_year_month_day"
+      },
+      "yyyy-MM-dd" : {
+        "type" : "date",
+        "format": "yyyy-MM-dd"
+      },
+      "HH:mm:ss" : {
+        "type" : "date",
+        "format": "HH:mm:ss"
+      },
+      "yyyy-MM-dd_OR_epoch_millis" : {
+        "type" : "date",
+        "format": "yyyy-MM-dd||epoch_millis"
+      },
+      "hour_minute_second_OR_t_time" : {
+        "type" : "date",
+        "format": "hour_minute_second||t_time"
+      }
+    }
+  }
+}