SQL: change the way unsupported data types fields are handled

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java

@@ -358,11 +358,12 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
 
         int dot = fieldName.lastIndexOf('.');
         String fullFieldName = fieldName;
+        EsField parent = null;
 
         if (dot >= 0) {
             String parentName = fieldName.substring(0, dot);
             fieldName = fieldName.substring(dot + 1);
-            EsField parent = flattedMapping.get(parentName);
+            parent = flattedMapping.get(parentName);
             if (parent == null) {
                 Map<String, FieldCapabilities> map = globalCaps.get(parentName);
                 Function<String, EsField> fieldFunction;
@@ -387,7 +388,22 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
         }
 
         EsField esField = field.apply(fieldName);
-
+
+        if (parent != null && parent instanceof UnsupportedEsField) {
+            UnsupportedEsField unsupportedParent = (UnsupportedEsField) parent;
+            String inherited = unsupportedParent.getInherited();
+            String type = unsupportedParent.getOriginalType();
+
+            if (inherited == null) {
+                // mark the sub-field as unsupported, just like its parent, setting the first unsupported parent as the current one
+                esField = new UnsupportedEsField(esField.getName(), type, unsupportedParent.getName(), esField.getProperties());
+            } else {
+                // mark the sub-field as unsupported, just like its parent, but setting the first unsupported parent
+                // as the parent's first unsupported grandparent
+                esField = new UnsupportedEsField(esField.getName(), type, inherited, esField.getProperties());
+            }
+        }
+
         parentProps.put(fieldName, esField);
         flattedMapping.put(fullFieldName, esField);
 
@@ -408,7 +424,7 @@ private static EsField createField(String fieldName, String typeName, Map<String
             case DATETIME:
                 return new DateEsField(fieldName, props, isAggregateable);
             case UNSUPPORTED:
-                return new UnsupportedEsField(fieldName, typeName);
+                return new UnsupportedEsField(fieldName, typeName, null, props);
             default:
                 return new EsField(fieldName, esType, props, isAggregateable, isAlias);
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java

@@ -73,7 +73,7 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
                     properties = Collections.emptyMap();
                 }
             } else {
-                properties = Collections.emptyMap();
+                properties = fromEs(content);
             }
             boolean docValues = boolSetting(content.get("doc_values"), esDataType.defaultDocValues);
             final EsField field;
@@ -91,7 +91,8 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
                     break;
                 case UNSUPPORTED:
                     String type = content.get("type").toString();
-                    field = new UnsupportedEsField(name, type);
+                    field = new UnsupportedEsField(name, type, null, properties);
+                    propagateUnsupportedType(name, type, properties);
                     break;
                 default:
                     field = new EsField(name, esDataType, properties, docValues);
@@ -113,4 +114,21 @@ private static boolean boolSetting(Object value, boolean defaultValue) {
     private static int intSetting(Object value, int defaultValue) {
         return value == null ? defaultValue : Integer.parseInt(value.toString());
     }
+
+    private static void propagateUnsupportedType(String inherited, String originalType, Map<String, EsField> properties) {
+        if (properties != null && properties.isEmpty() == false) {
+            for (Entry<String, EsField> entry : properties.entrySet()) {
+                EsField field = entry.getValue();
+                UnsupportedEsField u;
+                if (field instanceof UnsupportedEsField) {
+                    u = (UnsupportedEsField) field;
+                    u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties());
+                } else {
+                    u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties());
+                }
+                entry.setValue(u);
+                propagateUnsupportedType(inherited, originalType, u.getProperties());
+            }
+        }
+    }
 }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/UnsupportedEsField.java

@@ -5,24 +5,40 @@
  */
 package org.elasticsearch.xpack.ql.type;
 
-import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 
 /**
- * SQL-related information about an index field that cannot be supported by SQL
+ * SQL-related information about an index field that cannot be supported by SQL.
+ * All the subfields (properties) of an unsupported type should also be unsupported.
  */
 public class UnsupportedEsField extends EsField {
 
-    private String originalType;
-
+    private final String originalType;
+    private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type
+
     public UnsupportedEsField(String name, String originalType) {
-        super(name, DataType.UNSUPPORTED, Collections.emptyMap(), false);
+        this(name, originalType, null, new TreeMap<>());
+    }
+
+    public UnsupportedEsField(String name, String originalType, String inherited, Map<String, EsField> properties) {
+        super(name, DataType.UNSUPPORTED, properties, false);
         this.originalType = originalType;
+        this.inherited = inherited;
     }
 
     public String getOriginalType() {
         return originalType;
     }
+
+    public String getInherited() {
+        return inherited;
+    }
+
+    public boolean hasInherited() {
+        return inherited != null;
+    }
 
     @Override
     public boolean equals(Object o) {
@@ -36,11 +52,12 @@ public boolean equals(Object o) {
             return false;
         }
         UnsupportedEsField that = (UnsupportedEsField) o;
-        return Objects.equals(originalType, that.originalType);
+        return Objects.equals(originalType, that.originalType)
+                && Objects.equals(inherited, that.inherited);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), originalType);
+        return Objects.hash(super.hashCode(), originalType, inherited);
     }
 }

x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java

@@ -188,6 +188,10 @@ public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");
         assertThat(dt.getDataType().typeName(), is("unsupported"));
+        dt = mapping.get("time_frame");
+        assertThat(dt.getDataType().typeName(), is("unsupported"));
+        dt = mapping.get("flat");
+        assertThat(dt.getDataType().typeName(), is("unsupported"));
     }
 
     public static Map<String, EsField> loadMapping(String name) {

x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json

@@ -7,6 +7,26 @@
         "unsupported" : { "type" : "ip_range" },
         "date" : { "type" : "date"},
         "shape": { "type" : "geo_shape" },
+        "x" : {
+            "type" : "text",
+            "fields" : {
+                "y" : {
+                    "type" : "foobar",
+                    "fields" : {
+                        "z" : {
+                            "properties" : {
+                                "v" : {
+                                    "type" : "keyword"
+                                },
+                                "w" : {
+                                    "type" : "foo"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
         "some" : {
             "properties" : {
                 "dotted" : {

x-pack/plugin/ql/src/test/resources/mapping-unsupported.json

@@ -6,6 +6,9 @@
         "time_frame" : {
             "type" : "date_range",
             "format" : "yyyy-MM-dd"
+        },
+        "flat" : {
+            "type" : "flattened"
         }
     }
 }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java

@@ -217,8 +217,14 @@ private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute na
             // unsupported types
             else if (DataTypes.isUnsupported(fa.dataType())) {
                 UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
-                named = u.withUnresolvedMessage(
-                        "Cannot use field [" + fa.name() + "] type [" + unsupportedField.getOriginalType() + "] as is unsupported");
+                if (unsupportedField.hasInherited()) {
+                    named = u.withUnresolvedMessage(
+                            "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "] "
+                                    + "in hierarchy (field [" + unsupportedField.getInherited() + "])");
+                } else {
+                    named = u.withUnresolvedMessage(
+                            "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
+                }
             }
             // compound fields
             else if (allowCompound == false && fa.dataType().isPrimitive() == false) {

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java

@@ -482,19 +482,33 @@ public void testGroupByScalarFunctionWithAggOnTarget() {
     }
 
     public void testUnsupportedType() {
-        assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT unsupported FROM test"));
     }
 
     public void testUnsupportedStarExpansion() {
-        assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT unsupported.* FROM test"));
     }
 
     public void testUnsupportedTypeInFilter() {
-        assertEquals("1:26: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:26: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT * FROM test WHERE unsupported > 1"));
     }
+
+    public void testValidRootFieldWithUnsupportedChildren() {
+        accept("SELECT x FROM test");
+    }
+
+    public void testUnsupportedTypeInHierarchy() {
+        assertEquals("1:8: Cannot use field [x.y.z.w] with unsupported type [foobar] in hierarchy (field [y])",
+                error("SELECT x.y.z.w FROM test"));
+        assertEquals("1:8: Cannot use field [x.y.z.v] with unsupported type [foobar] in hierarchy (field [y])",
+                error("SELECT x.y.z.v FROM test"));
+        assertEquals("1:8: Cannot use field [x.y.z] with unsupported type [foobar] in hierarchy (field [y])",
+                error("SELECT x.y.z.* FROM test"));
+        assertEquals("1:8: Cannot use field [x.y] with unsupported type [foobar]", error("SELECT x.y FROM test"));
+    }
 
     public void testTermEqualitOnInexact() {
         assertEquals("1:26: [text = 'value'] cannot operate on first argument field of data type [text]: " +
@@ -509,12 +523,12 @@ public void testTermEqualityOnAmbiguous() {
     }
 
     public void testUnsupportedTypeInFunction() {
-        assertEquals("1:12: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:12: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT ABS(unsupported) FROM test"));
     }
 
     public void testUnsupportedTypeInOrder() {
-        assertEquals("1:29: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:29: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT * FROM test ORDER BY unsupported"));
     }
 

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java

@@ -128,6 +128,94 @@ public void testMetaFieldsAreIgnored() throws Exception {
         assertEquals(DataType.INTEGER, esIndex.mapping().get("_meta_field").getDataType());
         assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
     }
+
+    public void testFlattenedHiddenSubfield() throws Exception {
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "some_field", "flattened", false, false);
+        addFieldCaps(fieldCaps, "some_field._keyed", "flattened", false, false);
+        addFieldCaps(fieldCaps, "another_field", "object", true, false);
+        addFieldCaps(fieldCaps, "another_field._keyed", "keyword", true, false);
+        addFieldCaps(fieldCaps, "nested_field", "object", false, false);
+        addFieldCaps(fieldCaps, "nested_field.sub_field", "flattened", true, true);
+        addFieldCaps(fieldCaps, "nested_field.sub_field._keyed", "flattened", true, true);
+        addFieldCaps(fieldCaps, "text", "keyword", true, true);
+
+        String wildcard = "*";
+        IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getProperties().get("_keyed").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED,
+                esIndex.mapping().get("nested_field").getProperties().get("sub_field").getProperties().get("_keyed").getDataType());
+        assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
+        assertEquals(DataType.KEYWORD, esIndex.mapping().get("another_field").getProperties().get("_keyed").getDataType());
+    }
+
+    public void testPropagateUnsupportedTypeToSubFields() throws Exception {
+        // generate a field type having the name of the format "foobar43"
+        String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "a", "text", false, false);
+        addFieldCaps(fieldCaps, "a.b", esFieldType, false, false);
+        addFieldCaps(fieldCaps, "a.b.c", "object", true, false);
+        addFieldCaps(fieldCaps, "a.b.c.d", "keyword", true, false);
+        addFieldCaps(fieldCaps, "a.b.c.e", "foo", true, true);
+
+        String wildcard = "*";
+        IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        assertEquals(DataType.TEXT, esIndex.mapping().get("a").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
+                .getProperties().get("d").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
+                .getProperties().get("e").getDataType());
+    }
+
+    public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception {
+        // generate a field type having the name of the format "foobar43"
+        String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
+
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "some_field", esFieldType, false, false);
+        addFieldCaps(fieldCaps, "another_field", "object", true, false);
+        addFieldCaps(fieldCaps, "another_field._foo", esFieldType, true, false);
+        addFieldCaps(fieldCaps, "nested_field", "object", false, false);
+        addFieldCaps(fieldCaps, "nested_field.sub_field1", esFieldType, true, true);
+        addFieldCaps(fieldCaps, "nested_field.sub_field1.bar", esFieldType, true, true);
+        addFieldCaps(fieldCaps, "nested_field.sub_field2", esFieldType, true, true);
+        // even if this is of a supported type, because it belongs to an UNSUPPORTED type parent, it should also be UNSUPPORTED
+        addFieldCaps(fieldCaps, "nested_field.sub_field2.bar", "keyword", true, true);
+        addFieldCaps(fieldCaps, "text", "keyword", true, true);
+
+        String wildcard = "*";
+        IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getDataType());
+        assertEquals(DataType.UNSUPPORTED,
+                esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getProperties().get("bar").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getDataType());
+        assertEquals(DataType.UNSUPPORTED,
+                esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getProperties().get("bar").getDataType());
+        assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType());
+    }
 
     public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
         Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
@@ -330,7 +418,7 @@ private static <K, V> void assertEqualsMaps(Map<K, V> left, Map<K, V> right) {
     private void addFieldCaps(Map<String, Map<String, FieldCapabilities>> fieldCaps, String name, String type, boolean isSearchable,
             boolean isAggregatable) {
         Map<String, FieldCapabilities> cap = new HashMap<>();
-        cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
+        cap.put(type, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
         fieldCaps.put(name, cap);
     }
 }