diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDatetimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDatetimeValue.java index 92d0e01f330..02a584fe5b6 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDatetimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDatetimeValue.java @@ -27,15 +27,15 @@ public class ExprDatetimeValue extends AbstractExprValue { private final LocalDateTime datetime; - private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; + private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS; private static final int MIN_FRACTION_SECONDS = 0; - private static final int MAX_FRACTION_SECONDS = 6; + private static final int MAX_FRACTION_SECONDS = 9; static { - FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() + FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd HH:mm:ss") .appendFraction( - ChronoField.MICRO_OF_SECOND, + ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) @@ -47,10 +47,10 @@ public class ExprDatetimeValue extends AbstractExprValue { */ public ExprDatetimeValue(String datetime) { try { - this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_MICROS); + this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_NANOS); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please " - + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime)); + + "use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", datetime)); } } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprStringValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprStringValue.java index 7e68cfc4b05..c41c23d6ac9 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprStringValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprStringValue.java @@ -48,7 +48,7 @@ public LocalDateTime datetimeValue() { .datetimeValue(); } catch (SemanticCheckException exception) { throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please " - + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", value)); + + "use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", value)); } } } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index c8f4e223636..df5b5498e9f 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -24,15 +24,15 @@ public class ExprTimeValue extends AbstractExprValue { private final LocalTime time; - private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; + private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS; private static final int MIN_FRACTION_SECONDS = 0; - private static final int MAX_FRACTION_SECONDS = 6; + private static final int MAX_FRACTION_SECONDS = 9; static { - FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() + FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder() .appendPattern("HH:mm:ss") .appendFraction( - ChronoField.MICRO_OF_SECOND, + ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) @@ -44,10 +44,10 @@ public class ExprTimeValue extends AbstractExprValue { */ public ExprTimeValue(String time) { try { - this.time = LocalTime.parse(time, FORMATTER_VARIABLE_MICROS); + this.time = LocalTime.parse(time, FORMATTER_VARIABLE_NANOS); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("time:%s in unsupported format, please use " - + "HH:mm:ss[.SSSSSS]", time)); + + "HH:mm:ss[.SSSSSSSSS]", time)); } } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java index 4235bfee8d6..46903d877e3 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java @@ -38,15 +38,15 @@ public class ExprTimestampValue extends AbstractExprValue { .ofPattern("yyyy-MM-dd HH:mm:ss"); private final Instant timestamp; - private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS; + private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS; private static final int MIN_FRACTION_SECONDS = 0; - private static final int MAX_FRACTION_SECONDS = 6; + private static final int MAX_FRACTION_SECONDS = 9; static { - FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder() + FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd HH:mm:ss") .appendFraction( - ChronoField.MICRO_OF_SECOND, + ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) @@ -58,12 +58,12 @@ public class ExprTimestampValue extends AbstractExprValue { */ public ExprTimestampValue(String timestamp) { try { - this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS) + this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_NANOS) .atZone(ZONE) .toInstant(); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("timestamp:%s in unsupported format, please " - + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", timestamp)); + + "use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", timestamp)); } } @@ -72,7 +72,7 @@ public ExprTimestampValue(String timestamp) { public String value() { return timestamp.getNano() == 0 ? FORMATTER_WITHOUT_NANO.withZone(ZONE) .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) - : FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp); + : FORMATTER_VARIABLE_NANOS.withZone(ZONE).format(timestamp); } @Override diff --git a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java index 8937098b1c6..3a7df17d905 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java @@ -23,7 +23,7 @@ public class DateTimeValueTest { - private static final int MICROS_PRECISION_MAX = 6; + private static final int NANOS_PRECISION_MAX = 9; @Test public void timeValueInterfaceTest() { @@ -94,7 +94,7 @@ public void dateInUnsupportedFormat() { public void timeInUnsupportedFormat() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, () -> new ExprTimeValue("01:01:0")); - assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSS]", + assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } @@ -105,7 +105,7 @@ public void timestampInUnsupportedFormat() { () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); assertEquals( "timestamp:2020-07-07T01:01:01Z in unsupported format, " - + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", + + "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } @@ -116,7 +116,7 @@ public void datetimeInUnsupportedFormat() { () -> new ExprDatetimeValue("2020-07-07T01:01:01Z")); assertEquals( "datetime:2020-07-07T01:01:01Z in unsupported format, " - + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", + + "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } @@ -134,7 +134,7 @@ public void stringDateTimeValue() { () -> new ExprStringValue("2020-07-07T01:01:01Z").datetimeValue()); assertEquals( "datetime:2020-07-07T01:01:01Z in unsupported format, " - + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", + + "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } @@ -163,96 +163,96 @@ public void stringTimeValue() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, () -> new ExprStringValue("01:01:0").timeValue()); - assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSS]", + assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } @Test - public void timeWithVariableMicroPrecision() { - String timeWithMicrosFormat = "10:11:12.%s"; + public void timeWithVariableNanoPrecision() { + String timeWithNanosFormat = "10:11:12.%s"; - // Check all lengths of microsecond precision, up to max precision accepted - StringBuilder micros = new StringBuilder(); - for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) { - micros.append(microPrecision); - String timeWithMicros = String.format(timeWithMicrosFormat, micros); + // Check all lengths of nanosecond precision, up to max precision accepted + StringBuilder nanos = new StringBuilder(); + for (int nanosPrecision = 1; nanosPrecision <= NANOS_PRECISION_MAX; nanosPrecision++) { + nanos.append(nanosPrecision); + String timeWithNanos = String.format(timeWithNanosFormat, nanos); - ExprValue timeValue = new ExprTimeValue(timeWithMicros); - assertEquals(LocalTime.parse(timeWithMicros), timeValue.timeValue()); + ExprValue timeValue = new ExprTimeValue(timeWithNanos); + assertEquals(LocalTime.parse(timeWithNanos), timeValue.timeValue()); } } @Test - public void timestampWithVariableMicroPrecision() { + public void timestampWithVariableNanoPrecision() { String dateValue = "2020-08-17"; - String timeWithMicrosFormat = "10:11:12.%s"; + String timeWithNanosFormat = "10:11:12.%s"; - // Check all lengths of microsecond precision, up to max precision accepted - StringBuilder micros = new StringBuilder(); - for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) { - micros.append(microPrecision); - String timeWithMicros = String.format(timeWithMicrosFormat, micros); + // Check all lengths of nanosecond precision, up to max precision accepted + StringBuilder nanos = new StringBuilder(); + for (int nanoPrecision = 1; nanoPrecision <= NANOS_PRECISION_MAX; nanoPrecision++) { + nanos.append(nanoPrecision); + String timeWithNanos = String.format(timeWithNanosFormat, nanos); - String timestampString = String.format("%s %s", dateValue, timeWithMicros); + String timestampString = String.format("%s %s", dateValue, timeWithNanos); ExprValue timestampValue = new ExprTimestampValue(timestampString); assertEquals(LocalDate.parse(dateValue), timestampValue.dateValue()); - assertEquals(LocalTime.parse(timeWithMicros), timestampValue.timeValue()); - String localDateTime = String.format("%sT%s", dateValue, timeWithMicros); + assertEquals(LocalTime.parse(timeWithNanos), timestampValue.timeValue()); + String localDateTime = String.format("%sT%s", dateValue, timeWithNanos); assertEquals(LocalDateTime.parse(localDateTime), timestampValue.datetimeValue()); } } @Test - public void datetimeWithVariableMicroPrecision() { + public void datetimeWithVariableNanoPrecision() { String dateValue = "2020-08-17"; - String timeWithMicrosFormat = "10:11:12.%s"; + String timeWithNanosFormat = "10:11:12.%s"; - // Check all lengths of microsecond precision, up to max precision accepted - StringBuilder micros = new StringBuilder(); - for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) { - micros.append(microPrecision); - String timeWithMicros = String.format(timeWithMicrosFormat, micros); + // Check all lengths of nanosecond precision, up to max precision accepted + StringBuilder nanos = new StringBuilder(); + for (int nanoPrecision = 1; nanoPrecision <= NANOS_PRECISION_MAX; nanoPrecision++) { + nanos.append(nanoPrecision); + String timeWithNanos = String.format(timeWithNanosFormat, nanos); - String datetimeString = String.format("%s %s", dateValue, timeWithMicros); + String datetimeString = String.format("%s %s", dateValue, timeWithNanos); ExprValue datetimeValue = new ExprDatetimeValue(datetimeString); assertEquals(LocalDate.parse(dateValue), datetimeValue.dateValue()); - assertEquals(LocalTime.parse(timeWithMicros), datetimeValue.timeValue()); - String localDateTime = String.format("%sT%s", dateValue, timeWithMicros); + assertEquals(LocalTime.parse(timeWithNanos), datetimeValue.timeValue()); + String localDateTime = String.format("%sT%s", dateValue, timeWithNanos); assertEquals(LocalDateTime.parse(localDateTime), datetimeValue.datetimeValue()); } } @Test - public void timestampOverMaxMicroPrecision() { + public void timestampOverMaxNanoPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567")); + () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567890")); assertEquals( - "timestamp:2020-07-07 01:01:01.1234567 in unsupported format, " - + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", + "timestamp:2020-07-07 01:01:01.1234567890 in unsupported format, " + + "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } @Test - public void datetimeOverMaxMicroPrecision() { + public void datetimeOverMaxNanoPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567")); + () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567890")); assertEquals( - "datetime:2020-07-07 01:01:01.1234567 in unsupported format, " - + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]", + "datetime:2020-07-07 01:01:01.1234567890 in unsupported format, " + + "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } @Test - public void timeOverMaxMicroPrecision() { + public void timeOverMaxNanoPrecision() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> new ExprTimeValue("01:01:01.1234567")); + () -> new ExprTimeValue("01:01:01.1234567890")); assertEquals( - "time:01:01:01.1234567 in unsupported format, please use HH:mm:ss[.SSSSSS]", + "time:01:01:01.1234567890 in unsupported format, please use HH:mm:ss[.SSSSSSSSS]", exception.getMessage()); } } diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index c0a3bf62aca..dd8cc68a442 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -44,6 +44,8 @@ The OpenSearch SQL Engine support the following data types. +---------------------+ | date | +---------------------+ +| date_nanos | ++---------------------+ | time | +---------------------+ | interval | @@ -91,9 +93,9 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da +-----------------+---------------------+-----------+ | date | timestamp | TIMESTAMP | +-----------------+---------------------+-----------+ -| ip | ip | VARCHAR | +| date_nanos | timestamp | TIMESTAMP | +-----------------+---------------------+-----------+ -| date | timestamp | TIMESTAMP | +| ip | ip | VARCHAR | +-----------------+---------------------+-----------+ | binary | binary | VARBINARY | +-----------------+---------------------+-----------+ @@ -250,11 +252,11 @@ Time Time represents the time on the clock or watch with no regard for which timezone it might be related with. Time type data does not have date information. -+------+-----------------------+----------------------------------------+ -| Type | Syntax | Range | -+======+=======================+========================================+ -| Time | 'hh:mm:ss[.fraction]' | '00:00:00.000000' to '23:59:59.999999' | -+------+-----------------------+----------------------------------------+ ++------+-----------------------+----------------------------------------------+ +| Type | Syntax | Range | ++======+=======================+==============================================+ +| Time | 'hh:mm:ss[.fraction]' | '00:00:00.000000000' to '23:59:59.999999999' | ++------+-----------------------+----------------------------------------------+ Datetime @@ -262,11 +264,11 @@ Datetime Datetime type is the combination of date and time. The conversion rule of date or time to datetime is described in `Conversion between date and time types`_. Datetime type does not contain timezone information. For an absolute time point that contains both date time and timezone information, see `Timestamp`_. -+----------+----------------------------------+--------------------------------------------------------------+ -| Type | Syntax | Range | -+==========+==================================+==============================================================+ -| Datetime | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999' | -+----------+----------------------------------+--------------------------------------------------------------+ ++----------+----------------------------------+--------------------------------------------------------------------+ +| Type | Syntax | Range | ++==========+==================================+====================================================================+ +| Datetime | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:00.000000000' to '9999-12-31 23:59:59.999999999' | ++----------+----------------------------------+--------------------------------------------------------------------+ @@ -275,11 +277,11 @@ Timestamp A timestamp instance is an absolute instant independent of timezone or convention. For example, for a given point of time, if we set the timestamp of this time point into another timezone, the value should also be different accordingly. Besides, the storage of timestamp type is also different from the other types. The timestamp is converted from the current timezone to UTC for storage, and is converted back to the set timezone from UTC when retrieving. -+-----------+----------------------------------+------------------------------------------------------------------+ -| Type | Syntax | Range | -+===========+==================================+==================================================================+ -| Timestamp | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:01.000000' UTC to '9999-12-31 23:59:59.999999' | -+-----------+----------------------------------+------------------------------------------------------------------+ ++-----------+----------------------------------+------------------------------------------------------------------------+ +| Type | Syntax | Range | ++===========+==================================+========================================================================+ +| Timestamp | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:01.000000000' UTC to '9999-12-31 23:59:59.999999999' | ++-----------+----------------------------------+------------------------------------------------------------------------+ Interval diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 24a14043c16..5c0d268f6dc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -178,8 +178,10 @@ private ExprType type(String field) { } /** - * Only default strict_date_optional_time||epoch_millis is supported. + * Only default strict_date_optional_time||epoch_millis is supported, + * strict_date_optional_time_nanos||epoch_millis if field is date_nanos. * https://www.elastic.co/guide/en/elasticsearch/reference/current/date.html + * https://www.elastic.co/guide/en/elasticsearch/reference/current/date_nanos.html * The customized date_format is not supported. */ private ExprValue constructTimestamp(String value) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java index 4695eb7c1a2..b1315078237 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java @@ -61,6 +61,7 @@ public class OpenSearchDescribeIndexRequest implements OpenSearchSystemRequest { .put("nested", ExprCoreType.ARRAY) .put("object", ExprCoreType.STRUCT) .put("date", ExprCoreType.TIMESTAMP) + .put("date_nanos", ExprCoreType.TIMESTAMP) .put("ip", OpenSearchDataType.OPENSEARCH_IP) .put("geo_point", OpenSearchDataType.OPENSEARCH_GEO_POINT) .put("binary", OpenSearchDataType.OPENSEARCH_BINARY)