QL: wildcard field type support

docs/reference/sql/language/data-types.asciidoc

@@ -25,6 +25,7 @@ s|SQL precision
 | <<number, `scaled_float`>>               | scaled_float    | DOUBLE      | 15
 | <<keyword, `keyword`>>                   | keyword         | VARCHAR     | 32,766
 | <<constant-keyword, `constant_keyword`>> | constant_keyword| VARCHAR     | 32,766
+| <<wildcard, `wildcard`>>                 | wildcard        | VARCHAR     | 32,766
 | <<text, `text`>>                         | text            | VARCHAR     | 2,147,483,647
 | <<binary, `binary`>>                     | binary          | VARBINARY   | 2,147,483,647
 | <<date, `date`>>                         | datetime        | TIMESTAMP   | 29

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java

@@ -32,6 +32,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT;
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
 /**
  * Extractor for ES fields. Works for both 'normal' fields but also nested ones (which require hitName to be set).
  * The latter is used as metadata in assembling the results in the tabular response.
@@ -214,6 +215,7 @@ protected Object unwrapMultiValue(Object values) {
     protected boolean isFromDocValuesOnly(DataType dataType) {
         return dataType == KEYWORD // because of ignore_above.
                     || dataType == DATETIME
+                    || dataType == WILDCARD // because of ignore_above.
                     || dataType == CONSTANT_KEYWORD // because a non-existent value is considered the constant value itself
                     || dataType == SCALED_FLOAT; // because of scaling_factor
     }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.ql.expression.function.scalar.whitelist;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
 import org.elasticsearch.xpack.ql.expression.function.scalar.string.StartsWithFunctionProcessor;
@@ -32,7 +33,9 @@ public static <T> Object docValue(Map<String, ScriptDocValues<T>> doc, String fi
         if (doc.containsKey(fieldName)) {
             ScriptDocValues<T> docValues = doc.get(fieldName);
             if (!docValues.isEmpty()) {
-                return docValues.get(0);
+                T value = docValues.get(0);
+                // FIXME temporary workaround until https://github.com/elastic/elasticsearch/issues/58044 gets fixed
+                return value instanceof BytesRef ? ((BytesRef) value).utf8ToString() : value;
             }
         }
         return null;

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.xpack.ql.type.KeywordEsField;
 import org.elasticsearch.xpack.ql.type.TextEsField;
 import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
+import org.elasticsearch.xpack.ql.type.WildcardEsField;
 import org.elasticsearch.xpack.ql.util.CollectionUtils;
 import org.elasticsearch.xpack.ql.util.Holder;
 
@@ -71,6 +72,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED;
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
 
 public class IndexResolver {
 
@@ -451,6 +453,9 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa
         if (esType == CONSTANT_KEYWORD) {
             return new ConstantKeywordEsField(fieldName);
         }
+        if (esType == WILDCARD) {
+            return new WildcardEsField(fieldName);
+        }
         if (esType == UNSUPPORTED) {
             return new UnsupportedEsField(fieldName, typeName, null, props);
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java

@@ -32,6 +32,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.NULL;
 import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive;
 import static org.elasticsearch.xpack.ql.type.DataTypes.isString;
 
@@ -63,9 +64,12 @@ public static DataType commonType(DataType left, DataType right) {
             if (left == TEXT || right == TEXT) {
                 return TEXT;
             }
-            if (left == KEYWORD) {
+            if (left == KEYWORD || right == KEYWORD) {
                 return KEYWORD;
             }
+            if (left == CONSTANT_KEYWORD) {
+                return CONSTANT_KEYWORD;
+            }
             return right;
         }
         if (left.isNumeric() && right.isNumeric()) {
@@ -124,7 +128,7 @@ public static Converter converterFor(DataType from, DataType to) {
             return DefaultConverter.TO_NULL;
         }
         // proper converters
-        if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
+        if (to == KEYWORD || to == TEXT || to == WILDCARD || to == CONSTANT_KEYWORD) {
             return conversionToString(from);
         }
         if (to == LONG) {

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java

@@ -37,6 +37,7 @@ public final class DataTypes {
     public static final DataType KEYWORD          = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
     public static final DataType TEXT             = new DataType("text",              Integer.MAX_VALUE, false, false, false);
     public static final DataType CONSTANT_KEYWORD = new DataType("constant_keyword",  Integer.MAX_VALUE, false, false, true);
+    public static final DataType WILDCARD         = new DataType("wildcard",          Integer.MAX_VALUE, false, false, true);
     // date
     public static final DataType DATETIME         = new DataType("DATETIME", "date",  Long.BYTES,        false, false, true);
     // ip
@@ -63,6 +64,7 @@ public final class DataTypes {
             KEYWORD,
             TEXT,
             CONSTANT_KEYWORD,
+            WILDCARD,
             DATETIME,
             IP,
             BINARY,
@@ -134,7 +136,7 @@ public static boolean isUnsupported(DataType from) {
     }
 
     public static boolean isString(DataType t) {
-        return t == KEYWORD || t == TEXT || t == CONSTANT_KEYWORD;
+        return t == KEYWORD || t == TEXT || t == WILDCARD || t == CONSTANT_KEYWORD;
     }
 
     public static boolean isPrimitive(DataType t) {

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java

@@ -14,6 +14,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
 
 /**
  * SQL-related information about an index field with text type
@@ -45,7 +46,8 @@ public Exact getExactInfo() {
     private Tuple<EsField, String> findExact() {
         EsField field = null;
         for (EsField property : getProperties().values()) {
-            if ((property.getDataType() == KEYWORD || property.getDataType() == CONSTANT_KEYWORD) && property.getExactInfo().hasExact()) {
+            if ((property.getDataType() == KEYWORD || property.getDataType() == CONSTANT_KEYWORD || property.getDataType() == WILDCARD)
+                && property.getExactInfo().hasExact()) {
                 if (field != null) {
                     return new Tuple<>(null, "Multiple exact keyword candidates available for [" + getName() +
                         "]; specify which one to use");

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java

@@ -21,6 +21,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED;
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
 
 public abstract class Types {
 
@@ -90,6 +91,8 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
                 int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
                 boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
                 field = new KeywordEsField(name, properties, docValues, length, normalized);
+            } else if (esDataType == WILDCARD) {
+                field = new WildcardEsField(name);
             } else if (esDataType == CONSTANT_KEYWORD) {
                 field = new ConstantKeywordEsField(name);
             } else if (esDataType == DATETIME) {

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/WildcardEsField.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ql.type;
+
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
+
+public class WildcardEsField extends KeywordEsField {
+
+    public WildcardEsField(String name) {
+        super(name, WILDCARD, Collections.emptyMap(), true, Short.MAX_VALUE, false, false);
+    }
+}

x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java

@@ -29,6 +29,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED;
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
 import static org.elasticsearch.xpack.ql.type.DateUtils.asDateTime;
 
 public class DataTypeConversionTests extends ESTestCase {
@@ -374,20 +375,30 @@ public void testCommonType() {
         assertEquals(NULL, commonType(NULL, NULL));
         assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
         assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
+        assertEquals(LONG, commonType(LONG, WILDCARD));
         assertEquals(LONG, commonType(TEXT, LONG));
         assertEquals(SHORT, commonType(SHORT, BYTE));
         assertEquals(FLOAT, commonType(BYTE, FLOAT));
         assertEquals(FLOAT, commonType(FLOAT, INTEGER));
         assertEquals(DOUBLE, commonType(DOUBLE, FLOAT));
 
         // strings
+        assertEquals(TEXT, commonType(TEXT, TEXT));
         assertEquals(TEXT, commonType(TEXT, KEYWORD));
         assertEquals(TEXT, commonType(KEYWORD, TEXT));
         assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
         assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
+        assertEquals(TEXT, commonType(TEXT, WILDCARD));
+        assertEquals(TEXT, commonType(WILDCARD, TEXT));
+        assertEquals(KEYWORD, commonType(KEYWORD, KEYWORD));
         assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
         assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
+        assertEquals(KEYWORD, commonType(KEYWORD, WILDCARD));
+        assertEquals(KEYWORD, commonType(WILDCARD, KEYWORD));
         assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
+        assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, WILDCARD));
+        assertEquals(CONSTANT_KEYWORD, commonType(WILDCARD, CONSTANT_KEYWORD));
+        assertEquals(WILDCARD, commonType(WILDCARD, WILDCARD));
     }
 
     public void testEsDataTypes() {

x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java

@@ -21,6 +21,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED;
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.ql.type.DataTypes.WILDCARD;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
@@ -136,10 +137,11 @@ public void testMultiField() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field.getDataType(), is(TEXT));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(3));
+        assertThat(fields.size(), is(4));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
         assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
+        assertThat(fields.get("wildcard").getDataType(), is(WILDCARD));
     }
 
     public void testMultiFieldTooManyOptions() {
@@ -150,10 +152,11 @@ public void testMultiFieldTooManyOptions() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field, instanceOf(TextEsField.class));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(3));
+        assertThat(fields.size(), is(4));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
         assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
+        assertThat(fields.get("wildcard").getDataType(), is(WILDCARD));
     }
 
     public void testNestedDoc() {
@@ -183,6 +186,13 @@ public void testConstantKeywordField() {
         assertThat(dt.getDataType().typeName(), is("constant_keyword"));
     }
 
+    public void testWildcardField() {
+        Map<String, EsField> mapping = loadMapping("mapping-wildcard.json");
+        assertThat(mapping.size(), is(1));
+        EsField dt = mapping.get("full_name");
+        assertThat(dt.getDataType().typeName(), is("wildcard"));
+    }
+
     public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");

x-pack/plugin/ql/src/test/resources/mapping-multi-field.json

@@ -13,6 +13,9 @@
                 "constant" : {
                     "type" : "constant_keyword",
                     "value" : "some constant value"
+                },
+                "wildcard" : {
+                    "type" : "wildcard"
                 }
             }
         }

x-pack/plugin/ql/src/test/resources/mapping-wildcard.json

@@ -0,0 +1,8 @@
+{
+    "properties" : {
+        "full_name" : {
+            "type" : "wildcard",
+            "ignore_above" : 256
+        }
+    }
+}

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java

@@ -26,6 +26,7 @@ public enum EsType implements SQLType {
     KEYWORD(                  Types.VARCHAR),
     TEXT(                     Types.VARCHAR),
     CONSTANT_KEYWORD(         Types.VARCHAR),
+    WILDCARD(                 Types.VARCHAR),
     OBJECT(                   Types.STRUCT),
     NESTED(                   Types.STRUCT),
     BINARY(                   Types.VARBINARY),

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java

@@ -215,6 +215,7 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
             case TEXT:
             case KEYWORD:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 return v; // These types are already represented correctly in JSON
             case BYTE:
                 return ((Number) v).byteValue(); // Parser might return it as integer or long - need to update to the correct type
@@ -329,6 +330,7 @@ private static Boolean asBoolean(Object val, EsType columnType, String typeStrin
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 return Boolean.valueOf((String) val);
             default:
                 return failConversion(val, columnType, typeString, Boolean.class);
@@ -352,6 +354,7 @@ private static Byte asByte(Object val, EsType columnType, String typeString) thr
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 try {
                     return Byte.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -380,6 +383,7 @@ private static Short asShort(Object val, EsType columnType, String typeString) t
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 try {
                     return Short.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -407,6 +411,7 @@ private static Integer asInteger(Object val, EsType columnType, String typeStrin
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 try {
                     return Integer.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -439,6 +444,7 @@ private static Long asLong(Object val, EsType columnType, String typeString) thr
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 try {
                     return Long.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -467,6 +473,7 @@ private static Float asFloat(Object val, EsType columnType, String typeString) t
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 try {
                     return Float.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -494,6 +501,7 @@ private static Double asDouble(Object val, EsType columnType, String typeString)
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 try {
                     return Double.valueOf((String) val);
                 } catch (NumberFormatException e) {
@@ -560,6 +568,7 @@ private static BigDecimal asBigDecimal(Object val, EsType columnType, String typ
             case KEYWORD:
             case TEXT:
             case CONSTANT_KEYWORD:
+            case WILDCARD:
                 try {
                     return new BigDecimal((String) val);
                 } catch (NumberFormatException nfe) {

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java

@@ -77,6 +77,7 @@ private TypeUtils() {}
         types.put(EsType.KEYWORD, String.class);
         types.put(EsType.TEXT, String.class);
         types.put(EsType.CONSTANT_KEYWORD, String.class);
+        types.put(EsType.WILDCARD, String.class);
         types.put(EsType.BINARY, byte[].class);
         types.put(EsType.DATETIME, Timestamp.class);
         types.put(EsType.IP, String.class);
@@ -158,7 +159,7 @@ static EsType of(String name) throws SQLException {
     }
 
     static boolean isString(EsType dataType) {
-        return dataType == EsType.KEYWORD || dataType == EsType.TEXT || dataType == EsType.CONSTANT_KEYWORD;
+        return dataType == EsType.KEYWORD || dataType == EsType.TEXT || dataType == EsType.CONSTANT_KEYWORD || dataType == EsType.WILDCARD;
     }
 
     static EsType of(Class<? extends Object> clazz) throws SQLException {