Bugfix: SQL type mapping for legacy JDBC output

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -15,6 +15,7 @@
 import static org.opensearch.sql.legacy.TestUtils.getDataTypeNumericIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getDateIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getDateTimeIndexMapping;
+import static org.opensearch.sql.legacy.TestUtils.getDateTimeNestedIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getDeepNestedIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getDogIndexMapping;
 import static org.opensearch.sql.legacy.TestUtils.getDogs2IndexMapping;
@@ -716,6 +717,11 @@ public enum Index {
         "_doc",
         getDateTimeIndexMapping(),
         "src/test/resources/datetime.json"),
+    DATETIME_NESTED(
+            TestsConstants.TEST_INDEX_DATE_TIME_NESTED,
+            "_doc",
+            getDateTimeNestedIndexMapping(),
+            "src/test/resources/datetime_nested.json"),
     NESTED_SIMPLE(
         TestsConstants.TEST_INDEX_NESTED_SIMPLE,
         "_doc",

integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java

@@ -230,6 +230,11 @@ public static String getDateTimeIndexMapping() {
     return getMappingFile(mappingFile);
   }
 
+  public static String getDateTimeNestedIndexMapping() {
+    String mappingFile = "date_time_nested_index_mapping.json";
+    return getMappingFile(mappingFile);
+  }
+
   public static String getNestedSimpleIndexMapping() {
     String mappingFile = "nested_simple_index_mapping.json";
     return getMappingFile(mappingFile);

integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java

@@ -46,6 +46,7 @@ public class TestsConstants {
   public static final String TEST_INDEX_WEBLOGS = TEST_INDEX + "_weblogs";
   public static final String TEST_INDEX_DATE = TEST_INDEX + "_date";
   public static final String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime";
+  public static final String TEST_INDEX_DATE_TIME_NESTED = TEST_INDEX + "_datetime_nested";
   public static final String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested";
   public static final String TEST_INDEX_STRINGS = TEST_INDEX + "_strings";
   public static final String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric";

integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java

@@ -0,0 +1,150 @@
+package org.opensearch.sql.sql;
+
+import org.junit.Ignore;
+import org.opensearch.sql.legacy.SQLIntegTestCase;
+
+import java.io.IOException;
+import java.util.Locale;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME_NESTED;
+
+public class ComplexTimestampQueryIT extends SQLIntegTestCase {
+    @Override
+    protected void init() throws Exception {
+        loadIndex(SQLIntegTestCase.Index.DATETIME);
+        loadIndex(SQLIntegTestCase.Index.DATETIME_NESTED);
+    }
+
+    /**
+     * See: <a href="https://github.com/opensearch-project/sql/issues/3159">3159</a>
+     */
+    @Test
+    public void joinWithTimestampFieldsSchema() throws IOException {
+        String query = String.format(
+                Locale.ROOT,
+                "SELECT one.login_time, two.login_time " +
+                        "FROM %s AS one JOIN %s AS two " +
+                        "ON one._id = two._id",
+                TEST_INDEX_DATE_TIME,
+                TEST_INDEX_DATE_TIME
+        );
+
+        JSONObject result = executeQuery(query);
+        JSONArray schema = result.getJSONArray("schema");
+
+        Assert.assertFalse(schema.isEmpty());
+        for (int i = 0; i < schema.length(); i++) {
+            JSONObject column = schema.getJSONObject(i);
+            Assert.assertEquals("timestamp", column.getString("type"));
+        }
+    }
+
+    /**
+     * Control for joinWithTimestampFieldsSchema
+     */
+    @Test
+    public void nonJoinTimestampFieldsSchema() throws IOException {
+        String query = String.format(
+                Locale.ROOT,
+                "SELECT one.login_time " +
+                        "FROM %s AS one",
+                TEST_INDEX_DATE_TIME
+        );
+
+        JSONObject result = executeQuery(query);
+        JSONArray schema = result.getJSONArray("schema");
+
+        Assert.assertFalse(schema.isEmpty());
+        for (int i = 0; i < schema.length(); i++) {
+            JSONObject column = schema.getJSONObject(i);
+            Assert.assertEquals("timestamp", column.getString("type"));
+        }
+    }
+
+    /**
+     * See: <a href="https://github.com/opensearch-project/sql/issues/3204">3204</a>
+     */
+    // TODO currently out of scope due to V1/V2 engine feature mismatch. Should be fixed with Calcite.
+    @Test
+    @Ignore
+    public void joinTimestampComparison() throws IOException {
+        String query = String.format(
+                Locale.ROOT,
+                "SELECT one.login_time, two.login_time " +
+                        "FROM %s AS one JOIN %s AS two " +
+                        "ON one._id = two._id " +
+                        "WHERE one.login_time > timestamp('2018-05-07 00:00:00')",
+                TEST_INDEX_DATE_TIME, TEST_INDEX_DATE_TIME
+        );
+
+        JSONObject result = executeQuery(query);
+        Assert.assertEquals(2, result.getJSONArray("datarows").length());
+    }
+
+    /**
+     * Control for joinTimestampComparison
+     */
+    @Test
+    public void nonJoinTimestampComparison() throws IOException {
+        String query = String.format(
+                Locale.ROOT,
+                "SELECT login_time " +
+                        "FROM %s " +
+                        "WHERE login_time > timestamp('2018-05-07 00:00:00')",
+                TEST_INDEX_DATE_TIME
+        );
+
+        JSONObject result = executeQuery(query);
+        System.err.println(result.getJSONArray("datarows").toString());
+        Assert.assertEquals(2, result.getJSONArray("datarows").length());
+    }
+
+    /**
+     * See: <a href="https://github.com/opensearch-project/sql/issues/1545">1545</a>
+     */
+    @Test
+    public void selectDatetimeWithNested() throws IOException {
+        String query = String.format(
+                Locale.ROOT,
+                "SELECT tab.login_time " +
+                        "FROM %s AS tab, tab.projects AS pro",
+                TEST_INDEX_DATE_TIME_NESTED
+        );
+
+        JSONObject result = executeQuery(query);
+        JSONArray schema = result.getJSONArray("schema");
+
+        Assert.assertFalse(schema.isEmpty());
+        for (int i = 0; i < schema.length(); i++) {
+            JSONObject column = schema.getJSONObject(i);
+            Assert.assertEquals("timestamp", column.getString("type"));
+        }
+    }
+
+    /**
+     * Control for selectDatetimeWithNested
+     */
+    @Test
+    public void selectDatetimeWithoutNested() throws IOException {
+        String query = String.format(
+                Locale.ROOT,
+                "SELECT tab.login_time " +
+                        "FROM %s AS tab",
+                TEST_INDEX_DATE_TIME_NESTED
+        );
+
+        JSONObject result = executeQuery(query);
+        JSONArray schema = result.getJSONArray("schema");
+
+        Assert.assertFalse(schema.isEmpty());
+        for (int i = 0; i < schema.length(); i++) {
+            JSONObject column = schema.getJSONObject(i);
+            Assert.assertEquals("timestamp", column.getString("type"));
+        }
+    }
+}

integ-test/src/test/resources/datetime.json

@@ -3,6 +3,6 @@
 {"index":{"_id":"2"}}
 {"login_time":"2015-01-01T12:10:30Z"}
 {"index":{"_id":"3"}}
-{"login_time":"1585882955"}
+{"login_time":"1585882955000"}
 {"index":{"_id":"4"}}
 {"login_time":"2020-04-08T11:10:30+05:00"}

integ-test/src/test/resources/datetime_nested.json

@@ -0,0 +1,8 @@
+{"index":{"_id":"1"}}
+{"login_time":"2015-01-01", "projects": [{"name":  "abc"}]}
+{"index":{"_id":"2"}}
+{"login_time":"2015-01-01T12:10:30Z", "projects": [{"name":  "def"}, {"name": "ghi"}]}
+{"index":{"_id":"3"}}
+{"login_time":"1585882955000", "projects": []}
+{"index":{"_id":"4"}}
+{"login_time":"2020-04-08T11:10:30+05:00", "projects": [{"name": "jkl"}]}

integ-test/src/test/resources/indexDefinitions/date_time_nested_index_mapping.json

@@ -0,0 +1,12 @@
+{
+  "mappings": {
+    "properties": {
+      "login_time": {
+        "type": "date"
+      },
+      "projects": {
+        "type": "nested"
+      }
+    }
+  }
+}

legacy/src/main/java/org/opensearch/sql/legacy/executor/format/Protocol.java

@@ -214,7 +214,14 @@ private JSONArray getSchemaAsJson() {
     JSONArray schemaJson = new JSONArray();
 
     for (Column column : schema) {
-      schemaJson.put(schemaEntry(column.getName(), column.getAlias(), column.getType()));
+      // Hacky workaround for #3159, #1545: Legacy sometimes falsely converts `timestamp`s to `date`s, which causes
+      // breakage for consumers like JDBC. Until Calcite's done and we can delete legacy, just reset the type.
+      String t = column.getType();
+      if (t.equals("date")) {
+        schemaJson.put(schemaEntry(column.getName(), column.getAlias(), "timestamp"));
+      } else {
+        schemaJson.put(schemaEntry(column.getName(), column.getAlias(), t));
+      }
     }
 
     return schemaJson;