QL: wildcard field type support (#58062) (#61205)

astefan · web-flow · commit db8788e5a265 · 2020-08-17T18:24:32.000+03:00
(cherry picked from commit c874e6c)
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java
@@ -7,8 +7,8 @@
 package org.elasticsearch.xpack.ql.expression.function.scalar.whitelist;
 
 import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
 import org.elasticsearch.xpack.ql.expression.function.scalar.string.StartsWithFunctionProcessor;
+import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.NotProcessor;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation;
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
@@ -48,8 +48,12 @@ private static Map<String, EsField> startWalking(DataTypeRegistry typeRegistry,
 
     private static DataType getType(DataTypeRegistry typeRegistry, Map<String, Object> content) {
         if (content.containsKey("type")) {
+            String typeName = content.get("type").toString();
+            if ("wildcard".equals(typeName)) {
+                return KEYWORD;
+            }
             try {
-                return typeRegistry.fromEs(content.get("type").toString());
+                return typeRegistry.fromEs(typeName);
             } catch (IllegalArgumentException ex) {
                 return UNSUPPORTED;
             }
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java
@@ -136,10 +136,11 @@ public void testMultiField() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field.getDataType(), is(TEXT));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(3));
+        assertThat(fields.size(), is(4));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
         assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
+        assertThat(fields.get("wildcard").getDataType(), is(KEYWORD));
     }
 
     public void testMultiFieldTooManyOptions() {
@@ -150,10 +151,11 @@ public void testMultiFieldTooManyOptions() {
         assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
         assertThat(field, instanceOf(TextEsField.class));
         Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(3));
+        assertThat(fields.size(), is(4));
         assertThat(fields.get("raw").getDataType(), is(KEYWORD));
         assertThat(fields.get("english").getDataType(), is(TEXT));
         assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD));
+        assertThat(fields.get("wildcard").getDataType(), is(KEYWORD));
     }
 
     public void testNestedDoc() {
@@ -183,6 +185,13 @@ public void testConstantKeywordField() {
         assertThat(dt.getDataType().typeName(), is("constant_keyword"));
     }
 
+    public void testWildcardField() {
+        Map<String, EsField> mapping = loadMapping("mapping-wildcard.json");
+        assertThat(mapping.size(), is(1));
+        EsField dt = mapping.get("full_name");
+        assertThat(dt.getDataType().typeName(), is("keyword"));
+    }
+
     public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json
@@ -13,6 +13,9 @@
                 "constant" : {
                     "type" : "constant_keyword",
                     "value" : "some constant value"
+                },
+                "wildcard" : {
+                    "type" : "wildcard"
                 }
             }
         }
diff --git a/x-pack/plugin/ql/src/test/resources/mapping-wildcard.json b/x-pack/plugin/ql/src/test/resources/mapping-wildcard.json
@@ -0,0 +1,8 @@
+{
+    "properties" : {
+        "full_name" : {
+            "type" : "wildcard",
+            "ignore_above" : 256
+        }
+    }
+}
diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java
@@ -189,6 +189,65 @@ public void testOutOfRangeBigDecimal() throws SQLException {
         }
     }
 
+    public void testWildcardField() throws IOException, SQLException {
+        String mapping = "\"properties\":{\"id\":{\"type\":\"integer\"},\"text\":{\"type\":\"wildcard\"}}";
+        createIndex("test", Settings.EMPTY, mapping);
+        String text = randomAlphaOfLengthBetween(1, 10);
+
+        for (int i = 1; i <= 3; i++) {
+            int id = 1000 + i;
+            String valueToIndex = text + i;
+            index("test", "" + i, builder -> {
+                builder.field("id", id);
+                builder.field("text", valueToIndex);
+            });
+        }
+
+        try (Connection connection = esJdbc()) {
+            try (PreparedStatement statement = connection.prepareStatement("SELECT id, text FROM test WHERE text = ?")) {
+                int randomDocumentIndex = randomIntBetween(1, 3);
+                String randomDocumentText = text + randomDocumentIndex;
+
+                statement.setString(1, randomDocumentText);
+                try (ResultSet results = statement.executeQuery()) {
+                    assertTrue(results.next());
+                    assertEquals(1000 + randomDocumentIndex, results.getInt(1));
+                    assertEquals(randomDocumentText, results.getString(2));
+                    assertFalse(results.next());
+                }
+            }
+        }
+    }
+
+    public void testConstantKeywordField() throws IOException, SQLException {
+        String mapping = "\"properties\":{\"id\":{\"type\":\"integer\"},\"text\":{\"type\":\"constant_keyword\"}}";
+        createIndex("test", Settings.EMPTY, mapping);
+        String text = randomAlphaOfLengthBetween(1, 10);
+
+        for (int i = 1; i <= 3; i++) {
+            int id = 1000 + i;
+            index("test", "" + i, builder -> {
+                builder.field("id", id);
+                builder.field("text", text);
+            });
+        }
+
+        try (Connection connection = esJdbc()) {
+            try (PreparedStatement statement = connection.prepareStatement("SELECT id, text FROM test WHERE text = ?")) {
+                statement.setString(1, text);
+
+                try (ResultSet results = statement.executeQuery()) {
+                    for (int i = 1; i <= 3; i++) {
+                        assertTrue(results.next());
+                        assertEquals(1000 + i, results.getInt(1));
+                        assertEquals(text, results.getString(2));
+                    }
+                    assertFalse(results.next());
+                }
+            }
+        }
+    }
+
     public void testUnsupportedParameterUse() throws IOException, SQLException {
         index("library", builder -> {
             builder.field("name", "Don Quixote");
diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java
@@ -134,6 +134,39 @@ public void testConstantKeywordField() throws IOException {
         assertResponse(expected, runSql("SELECT constant_keyword_field FROM test"));
     }
 
+    /*
+     *    "wildcard_field": {
+     *       "type": "wildcard",
+     *       "ignore_above": 10
+     *    }
+     */
+    public void testWildcardField() throws IOException {
+        String wildcard = randomAlphaOfLength(20);
+        // _source for `wildcard` fields doesn't matter, as they should be taken from docvalue_fields
+        boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting
+        boolean enableSource = randomBoolean();          // enable _source at index level
+        boolean ignoreAbove = randomBoolean();
+
+        Map<String, Object> indexProps = new HashMap<>(1);
+        indexProps.put("_source", enableSource);
+
+        Map<String, Map<String, Object>> fieldProps = null;
+        if (ignoreAbove) {
+            fieldProps = new HashMap<>(1);
+            Map<String, Object> fieldProp = new HashMap<>(1);
+            fieldProp.put("ignore_above", 10);
+            fieldProps.put("wildcard_field", fieldProp);
+        }
+
+        createIndexWithFieldTypeAndProperties("wildcard", fieldProps, explicitSourceSetting ? indexProps : null);
+        index("{\"wildcard_field\":\"" + wildcard + "\"}");
+
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("columns", Arrays.asList(columnInfo("plain", "wildcard_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE)));
+        expected.put("rows", singletonList(singletonList(ignoreAbove ? null : wildcard)));
+        assertResponse(expected, runSql("SELECT wildcard_field FROM test"));
+    }
+
     /*
      *    "long/integer/short/byte_field": {
      *       "type": "long/integer/short/byte"
diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java
@@ -30,7 +30,7 @@ public abstract class CsvSpecTestCase extends SpecBaseIntegrationTestCase {
     @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
     public static List<Object[]> readScriptSpec() throws Exception {
         List<URL> urls = JdbcTestUtils.classpathResources("/*.csv-spec");
-        assertTrue("Not enough specs found " + urls.toString(), urls.size() > 15);
+        assertTrue("Not enough specs found (" + urls.size() + ") " + urls.toString(), urls.size() >= 23);
         return readScriptSpec(urls, specParser());
     }
 
diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
@@ -129,6 +129,7 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                 if (extraFields) {
                     createIndex.startObject("extra_gender").field("type", "constant_keyword").endObject();
                     createIndex.startObject("null_constant").field("type", "constant_keyword").endObject();
+                    createIndex.startObject("wildcard_name").field("type", "wildcard").endObject();
                     createIndex.startObject("extra.info.gender").field("type", "alias").field("path", "gender").endObject();
                 }
 
@@ -186,6 +187,8 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
 
             boolean hadLastItem = false;
 
+            String wildcard_name = null;
+            boolean setWildcardName = true;
             for (int f = 0; f < fields.size(); f++) {
                 // an empty value in the csv file is treated as 'null', thus skipping it in the bulk request
                 if (fields.get(f).trim().length() > 0) {
@@ -198,7 +201,19 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String
                         bulk.append(",\"extra_gender\":\"Female\"");
                     }
                 }
+                if ((titles.get(f).equals("first_name") || titles.get(f).equals("last_name")) && extraFields && setWildcardName) {
+                    if (fields.get(f).trim().length() == 0) {
+                        setWildcardName = false;
+                    } else {
+                        wildcard_name = wildcard_name == null ? fields.get(f) : wildcard_name + " " + fields.get(f);
+                    }
+                }
             }
+            // append the wildcard field
+            if (extraFields && setWildcardName) {
+                bulk.append(",\"wildcard_name\":\"" + wildcard_name + "\"");
+            }
+
             // append department
             List<List<String>> list = dep_emp.get(emp_no);
             if (!list.isEmpty()) {
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec
@@ -52,6 +52,7 @@ last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
 null_constant       |VARCHAR        |keyword
 salary              |INTEGER        |integer
+wildcard_name       |VARCHAR        |keyword
 ;
 
 describePattern
@@ -81,6 +82,7 @@ last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
 null_constant       |VARCHAR        |keyword
 salary              |INTEGER        |integer
+wildcard_name       |VARCHAR        |keyword
 ;
 
 showAlias
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec
@@ -295,6 +295,7 @@ last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
 null_constant       |VARCHAR        |keyword
 salary              |INTEGER        |integer
+wildcard_name       |VARCHAR        |keyword
 ;
 
 describeMultiLike
@@ -324,6 +325,7 @@ last_name           |VARCHAR        |text
 last_name.keyword   |VARCHAR        |keyword
 null_constant       |VARCHAR        |keyword
 salary              |INTEGER        |integer
+wildcard_name       |VARCHAR        |keyword
 ;
 
 describeSimpleIdentifier
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/wildcard.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/wildcard.csv-spec

Original file line number	Diff line number	Diff line change
`@@ -13,6 +13,9 @@`
`13`	`13`	`"constant" : {`
`14`	`14`	`"type" : "constant_keyword",`
`15`	`15`	`"value" : "some constant value"`
	`16`	`+ },`
	`17`	`+ "wildcard" : {`
	`18`	`+ "type" : "wildcard"`
`16`	`19`	`}`
`17`	`20`	`}`
`18`	`21`	`}`
Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ public abstract class CsvSpecTestCase extends SpecBaseIntegrationTestCase {`
`30`	`30`	`@ParametersFactory(argumentFormatting = PARAM_FORMATTING)`
`31`	`31`	`public static List<Object[]> readScriptSpec() throws Exception {`
`32`	`32`	`List<URL> urls = JdbcTestUtils.classpathResources("/*.csv-spec");`
`33`		`- assertTrue("Not enough specs found " + urls.toString(), urls.size() > 15);`
	`33`	`+ assertTrue("Not enough specs found (" + urls.size() + ") " + urls.toString(), urls.size() >= 23);`
`34`	`34`	`return readScriptSpec(urls, specParser());`
`35`	`35`	`}`
`36`	`36`
Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,7 @@ last_name \|VARCHAR \|text`
`52`	`52`	`last_name.keyword \|VARCHAR \|keyword`
`53`	`53`	`null_constant \|VARCHAR \|keyword`
`54`	`54`	`salary \|INTEGER \|integer`
	`55`	`+wildcard_name \|VARCHAR \|keyword`
`55`	`56`	`;`
`56`	`57`
`57`	`58`	`describePattern`
`@@ -81,6 +82,7 @@ last_name \|VARCHAR \|text`
`81`	`82`	`last_name.keyword \|VARCHAR \|keyword`
`82`	`83`	`null_constant \|VARCHAR \|keyword`
`83`	`84`	`salary \|INTEGER \|integer`
	`85`	`+wildcard_name \|VARCHAR \|keyword`
`84`	`86`	`;`
`85`	`87`
`86`	`88`	`showAlias`