Bugfix: SQL type mapping for legacy JDBC output

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -650,6 +650,11 @@ public enum Index {
         "_doc",
         getDateTimeIndexMapping(),
         "src/test/resources/datetime.json"),
+    DATETIME_NESTED(
+        TestsConstants.TEST_INDEX_DATE_TIME_NESTED,
+        "_doc",
+        getDateTimeNestedIndexMapping(),
+        "src/test/resources/datetime_nested.json"),
     NESTED_SIMPLE(
         TestsConstants.TEST_INDEX_NESTED_SIMPLE,
         "_doc",

integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java

@@ -230,6 +230,11 @@ public static String getDateTimeIndexMapping() {
     return getMappingFile(mappingFile);
   }
 
+  public static String getDateTimeNestedIndexMapping() {
+    String mappingFile = "date_time_nested_index_mapping.json";
+    return getMappingFile(mappingFile);
+  }
+
   public static String getNestedSimpleIndexMapping() {
     String mappingFile = "nested_simple_index_mapping.json";
     return getMappingFile(mappingFile);

integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java

@@ -46,6 +46,7 @@ public class TestsConstants {
   public static final String TEST_INDEX_WEBLOGS = TEST_INDEX + "_weblogs";
   public static final String TEST_INDEX_DATE = TEST_INDEX + "_date";
   public static final String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime";
+  public static final String TEST_INDEX_DATE_TIME_NESTED = TEST_INDEX + "_datetime_nested";
 
   /** A test index with @timestamp field */
   public static final String TEST_INDEX_TIME_DATA = TEST_INDEX + "_time_data";

integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.sql;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME_NESTED;
+
+import java.io.IOException;
+import java.util.Locale;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.opensearch.sql.legacy.SQLIntegTestCase;
+
+public class ComplexTimestampQueryIT extends SQLIntegTestCase {
+  @Override
+  protected void init() throws Exception {
+    loadIndex(SQLIntegTestCase.Index.DATETIME);
+    loadIndex(SQLIntegTestCase.Index.DATETIME_NESTED);
+  }
+
+  /** See: <a href="https://github.com/opensearch-project/sql/issues/3159">3159</a> */
+  @Test
+  public void joinWithTimestampFieldsSchema() throws IOException {
+    String query =
+        String.format(
+            Locale.ROOT,
+            "SELECT one.login_time, two.login_time "
+                + "FROM %s AS one JOIN %s AS two "
+                + "ON one._id = two._id",
+            TEST_INDEX_DATE_TIME,
+            TEST_INDEX_DATE_TIME);
+
+    JSONObject result = executeQuery(query);
+    JSONArray schema = result.getJSONArray("schema");
+
+    Assert.assertFalse(schema.isEmpty());
+    for (int i = 0; i < schema.length(); i++) {
+      JSONObject column = schema.getJSONObject(i);
+      Assert.assertEquals("timestamp", column.getString("type"));
+    }
+  }
+
+  /** Control for joinWithTimestampFieldsSchema */
+  @Test
+  public void nonJoinTimestampFieldsSchema() throws IOException {
+    String query =
+        String.format(
+            Locale.ROOT, "SELECT one.login_time " + "FROM %s AS one", TEST_INDEX_DATE_TIME);
+
+    JSONObject result = executeQuery(query);
+    JSONArray schema = result.getJSONArray("schema");
+
+    Assert.assertFalse(schema.isEmpty());
+    for (int i = 0; i < schema.length(); i++) {
+      JSONObject column = schema.getJSONObject(i);
+      Assert.assertEquals("timestamp", column.getString("type"));
+    }
+  }
+
+  /** See: <a href="https://github.com/opensearch-project/sql/issues/3204">3204</a> */
+  // TODO currently out of scope due to V1/V2 engine feature mismatch. Should be fixed with Calcite.
+  @Test
+  @Ignore
+  public void joinTimestampComparison() throws IOException {
+    String query =
+        String.format(
+            Locale.ROOT,
+            "SELECT one.login_time, two.login_time "
+                + "FROM %s AS one JOIN %s AS two "
+                + "ON one._id = two._id "
+                + "WHERE one.login_time > timestamp('2018-05-07 00:00:00')",
+            TEST_INDEX_DATE_TIME,
+            TEST_INDEX_DATE_TIME);
+
+    JSONObject result = executeQuery(query);
+    Assert.assertEquals(1, result.getJSONArray("datarows").length());
+  }
+
+  /** Control for joinTimestampComparison */
+  @Test
+  public void nonJoinTimestampComparison() throws IOException {
+    String query =
+        String.format(
+            Locale.ROOT,
+            "SELECT login_time "
+                + "FROM %s "
+                + "WHERE login_time > timestamp('2018-05-07 00:00:00')",
+            TEST_INDEX_DATE_TIME);
+
+    JSONObject result = executeQuery(query);
+    System.err.println(result.getJSONArray("datarows").toString());
+    Assert.assertEquals(1, result.getJSONArray("datarows").length());
+  }
+
+  /** See: <a href="https://github.com/opensearch-project/sql/issues/1545">1545</a> */
+  @Test
+  public void selectDatetimeWithNested() throws IOException {
+    String query =
+        String.format(
+            Locale.ROOT,
+            "SELECT tab.login_time " + "FROM %s AS tab, tab.projects AS pro",
+            TEST_INDEX_DATE_TIME_NESTED);
+
+    JSONObject result = executeQuery(query);
+    JSONArray schema = result.getJSONArray("schema");
+
+    Assert.assertFalse(schema.isEmpty());
+    for (int i = 0; i < schema.length(); i++) {
+      JSONObject column = schema.getJSONObject(i);
+      Assert.assertEquals("timestamp", column.getString("type"));
+    }
+  }
+
+  /** Control for selectDatetimeWithNested */
+  @Test
+  public void selectDatetimeWithoutNested() throws IOException {
+    String query =
+        String.format(
+            Locale.ROOT, "SELECT tab.login_time " + "FROM %s AS tab", TEST_INDEX_DATE_TIME_NESTED);
+
+    JSONObject result = executeQuery(query);
+    JSONArray schema = result.getJSONArray("schema");
+
+    Assert.assertFalse(schema.isEmpty());
+    for (int i = 0; i < schema.length(); i++) {
+      JSONObject column = schema.getJSONObject(i);
+      Assert.assertEquals("timestamp", column.getString("type"));
+    }
+  }
+}

integ-test/src/test/resources/datetime_nested.json

@@ -0,0 +1,8 @@
+{"index":{"_id":"1"}}
+{"login_time":"2015-01-01", "projects": [{"name":  "abc"}]}
+{"index":{"_id":"2"}}
+{"login_time":"2015-01-01T12:10:30Z", "projects": [{"name":  "def"}, {"name": "ghi"}]}
+{"index":{"_id":"3"}}
+{"login_time":"1585882955000", "projects": []}
+{"index":{"_id":"4"}}
+{"login_time":"2020-04-08T11:10:30+05:00", "projects": [{"name": "jkl"}]}

integ-test/src/test/resources/indexDefinitions/date_time_nested_index_mapping.json

@@ -0,0 +1,12 @@
+{
+  "mappings": {
+    "properties": {
+      "login_time": {
+        "type": "date"
+      },
+      "projects": {
+        "type": "nested"
+      }
+    }
+  }
+}

legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java

@@ -202,19 +202,40 @@ private String cursorOutputInJDBCFormat() {
 
   private String rawEntry(Row row, Schema schema) {
     // TODO String separator is being kept to "|" for the time being as using "\t" will require
-    // formatting since
-    // TODO tabs are occurring in multiple of 4 (one option is Guava's Strings.padEnd() method)
+    // formatting since tabs are occurring in multiple of 4 (one option is Guava's Strings.padEnd()
+    // method)
     return StreamSupport.stream(schema.spliterator(), false)
         .map(column -> row.getDataOrDefault(column.getName(), "NULL").toString())
         .collect(Collectors.joining("|"));
   }
 
+  /**
+   * Apply the V2 type mapping described in docs/user/general/datatypes.rst#data-types-mapping. The
+   * legacy engine works directly on OpenSearch types internally (trying to map on reading the OS
+   * schema fails when other parts call Type.valueOf(...)), so we need to apply this mapping at the
+   * serialization stage.
+   *
+   * @param column The column to fetch the type for
+   * @return The type mapped to the appropriate OpenSearch SQL type
+   */
+  private String v2MappedType(Column column) {
+    String type = column.getType();
+
+    return switch (type) {
+      case "date", "date_nanos" -> "timestamp";
+      case "half_float", "scaled_float" -> "float";
+      case "nested" -> "array";
+      case "object" -> "struct";
+      default -> type;
+    };
+  }
+
   private JSONArray getSchemaAsJson() {
     Schema schema = resultSet.getSchema();
     JSONArray schemaJson = new JSONArray();
 
     for (Column column : schema) {
-      schemaJson.put(schemaEntry(column.getName(), column.getAlias(), column.getType()));
+      schemaJson.put(schemaEntry(column.getName(), column.getAlias(), v2MappedType(column)));
     }
 
     return schemaJson;