diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 2ff0a8061b1..4e258088b5e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -650,6 +650,11 @@ public enum Index { "_doc", getDateTimeIndexMapping(), "src/test/resources/datetime.json"), + DATETIME_NESTED( + TestsConstants.TEST_INDEX_DATE_TIME_NESTED, + "_doc", + getDateTimeNestedIndexMapping(), + "src/test/resources/datetime_nested.json"), NESTED_SIMPLE( TestsConstants.TEST_INDEX_NESTED_SIMPLE, "_doc", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index aa3eb6ceeba..09a24269692 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -230,6 +230,11 @@ public static String getDateTimeIndexMapping() { return getMappingFile(mappingFile); } + public static String getDateTimeNestedIndexMapping() { + String mappingFile = "date_time_nested_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getNestedSimpleIndexMapping() { String mappingFile = "nested_simple_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index b139c02920a..17e30e5938c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -46,6 +46,7 @@ public class TestsConstants { public static final String TEST_INDEX_WEBLOGS = TEST_INDEX + "_weblogs"; public static final String TEST_INDEX_DATE = TEST_INDEX + "_date"; public static final String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime"; + public static final String TEST_INDEX_DATE_TIME_NESTED = TEST_INDEX + "_datetime_nested"; /** A test index with @timestamp field */ public static final String TEST_INDEX_TIME_DATA = TEST_INDEX + "_time_data"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java new file mode 100644 index 00000000000..c0eb800c103 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java @@ -0,0 +1,136 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME_NESTED; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class ComplexTimestampQueryIT extends SQLIntegTestCase { + @Override + protected void init() throws Exception { + loadIndex(SQLIntegTestCase.Index.DATETIME); + loadIndex(SQLIntegTestCase.Index.DATETIME_NESTED); + } + + /** See: 3159 */ + @Test + public void joinWithTimestampFieldsSchema() throws IOException { + String query = + String.format( + Locale.ROOT, + "SELECT one.login_time, two.login_time " + + "FROM %s AS one JOIN %s AS two " + + "ON one._id = two._id", + TEST_INDEX_DATE_TIME, + TEST_INDEX_DATE_TIME); + + JSONObject result = executeQuery(query); + JSONArray schema = result.getJSONArray("schema"); + + Assert.assertFalse(schema.isEmpty()); + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + Assert.assertEquals("timestamp", column.getString("type")); + } + } + + /** Control for joinWithTimestampFieldsSchema */ + @Test + public void nonJoinTimestampFieldsSchema() throws IOException { + String query = + String.format( + Locale.ROOT, "SELECT one.login_time " + "FROM %s AS one", TEST_INDEX_DATE_TIME); + + JSONObject result = executeQuery(query); + JSONArray schema = result.getJSONArray("schema"); + + Assert.assertFalse(schema.isEmpty()); + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + Assert.assertEquals("timestamp", column.getString("type")); + } + } + + /** See: 3204 */ + // TODO currently out of scope due to V1/V2 engine feature mismatch. Should be fixed with Calcite. + @Test + @Ignore + public void joinTimestampComparison() throws IOException { + String query = + String.format( + Locale.ROOT, + "SELECT one.login_time, two.login_time " + + "FROM %s AS one JOIN %s AS two " + + "ON one._id = two._id " + + "WHERE one.login_time > timestamp('2018-05-07 00:00:00')", + TEST_INDEX_DATE_TIME, + TEST_INDEX_DATE_TIME); + + JSONObject result = executeQuery(query); + Assert.assertEquals(1, result.getJSONArray("datarows").length()); + } + + /** Control for joinTimestampComparison */ + @Test + public void nonJoinTimestampComparison() throws IOException { + String query = + String.format( + Locale.ROOT, + "SELECT login_time " + + "FROM %s " + + "WHERE login_time > timestamp('2018-05-07 00:00:00')", + TEST_INDEX_DATE_TIME); + + JSONObject result = executeQuery(query); + System.err.println(result.getJSONArray("datarows").toString()); + Assert.assertEquals(1, result.getJSONArray("datarows").length()); + } + + /** See: 1545 */ + @Test + public void selectDatetimeWithNested() throws IOException { + String query = + String.format( + Locale.ROOT, + "SELECT tab.login_time " + "FROM %s AS tab, tab.projects AS pro", + TEST_INDEX_DATE_TIME_NESTED); + + JSONObject result = executeQuery(query); + JSONArray schema = result.getJSONArray("schema"); + + Assert.assertFalse(schema.isEmpty()); + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + Assert.assertEquals("timestamp", column.getString("type")); + } + } + + /** Control for selectDatetimeWithNested */ + @Test + public void selectDatetimeWithoutNested() throws IOException { + String query = + String.format( + Locale.ROOT, "SELECT tab.login_time " + "FROM %s AS tab", TEST_INDEX_DATE_TIME_NESTED); + + JSONObject result = executeQuery(query); + JSONArray schema = result.getJSONArray("schema"); + + Assert.assertFalse(schema.isEmpty()); + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + Assert.assertEquals("timestamp", column.getString("type")); + } + } +} diff --git a/integ-test/src/test/resources/datetime_nested.json b/integ-test/src/test/resources/datetime_nested.json new file mode 100644 index 00000000000..4da608a3d4d --- /dev/null +++ b/integ-test/src/test/resources/datetime_nested.json @@ -0,0 +1,8 @@ +{"index":{"_id":"1"}} +{"login_time":"2015-01-01", "projects": [{"name": "abc"}]} +{"index":{"_id":"2"}} +{"login_time":"2015-01-01T12:10:30Z", "projects": [{"name": "def"}, {"name": "ghi"}]} +{"index":{"_id":"3"}} +{"login_time":"1585882955000", "projects": []} +{"index":{"_id":"4"}} +{"login_time":"2020-04-08T11:10:30+05:00", "projects": [{"name": "jkl"}]} diff --git a/integ-test/src/test/resources/indexDefinitions/date_time_nested_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_time_nested_index_mapping.json new file mode 100644 index 00000000000..d113b95060b --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/date_time_nested_index_mapping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "login_time": { + "type": "date" + }, + "projects": { + "type": "nested" + } + } + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java index 3c3bbdc51be..0deb2fd9ccd 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java @@ -202,19 +202,40 @@ private String cursorOutputInJDBCFormat() { private String rawEntry(Row row, Schema schema) { // TODO String separator is being kept to "|" for the time being as using "\t" will require - // formatting since - // TODO tabs are occurring in multiple of 4 (one option is Guava's Strings.padEnd() method) + // formatting since tabs are occurring in multiple of 4 (one option is Guava's Strings.padEnd() + // method) return StreamSupport.stream(schema.spliterator(), false) .map(column -> row.getDataOrDefault(column.getName(), "NULL").toString()) .collect(Collectors.joining("|")); } + /** + * Apply the V2 type mapping described in docs/user/general/datatypes.rst#data-types-mapping. The + * legacy engine works directly on OpenSearch types internally (trying to map on reading the OS + * schema fails when other parts call Type.valueOf(...)), so we need to apply this mapping at the + * serialization stage. + * + * @param column The column to fetch the type for + * @return The type mapped to the appropriate OpenSearch SQL type + */ + private String v2MappedType(Column column) { + String type = column.getType(); + + return switch (type) { + case "date", "date_nanos" -> "timestamp"; + case "half_float", "scaled_float" -> "float"; + case "nested" -> "array"; + case "object" -> "struct"; + default -> type; + }; + } + private JSONArray getSchemaAsJson() { Schema schema = resultSet.getSchema(); JSONArray schemaJson = new JSONArray(); for (Column column : schema) { - schemaJson.put(schemaEntry(column.getName(), column.getAlias(), column.getType())); + schemaJson.put(schemaEntry(column.getName(), column.getAlias(), v2MappedType(column))); } return schemaJson;