Merge remote-tracking branch 'elastic/master' into 61034-fix

Author: original-brownbear

docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc

@@ -78,3 +78,103 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification
 `prediction_field_type`::
 (Optional, string)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type]
+
+
+[[inference-bucket-agg-example]]
+==== Example 
+
+The following snippet aggregates a web log by `client_ip` and extracts a number 
+of features via metric and bucket sub-aggregations as input to the {infer} 
+aggregation configured with a model trained to identify suspicious client IPs:
+
+[source,console]
+-------------------------------------------------
+GET kibana_sample_data_logs/_search
+{
+  "size": 0,
+  "aggs": {
+    "client_ip": { <1>
+      "composite": {
+        "sources": [
+          {
+            "client_ip": {
+              "terms": {
+                "field": "clientip"
+              }
+            }
+          }
+        ]
+      },
+      "aggs": { <2>
+        "url_dc": {
+          "cardinality": {
+            "field": "url.keyword"
+          }
+        },
+        "bytes_sum": {
+          "sum": {
+            "field": "bytes"
+          }
+        },
+        "geo_src_dc": {
+          "cardinality": {
+            "field": "geo.src"
+          }
+        },
+        "geo_dest_dc": {
+          "cardinality": {
+            "field": "geo.dest"
+          }
+        },
+        "responses_total": {
+          "value_count": {
+            "field": "timestamp"
+          }
+        },
+        "success": {
+          "filter": {
+            "term": {
+              "response": "200"
+            }
+          }
+        },
+        "error404": {
+          "filter": {
+            "term": {
+              "response": "404"
+            }
+          }
+        },
+        "error503": {
+          "filter": {
+            "term": {
+              "response": "503"
+            }
+          }
+        },
+        "malicious_client_ip": { <3>
+          "inference": {
+            "model_id": "malicious_clients_model",
+            "buckets_path": {
+              "response_count": "responses_total",
+              "url_dc": "url_dc",
+              "bytes_sum": "bytes_sum",
+              "geo_src_dc": "geo_src_dc",
+              "geo_dest_dc": "geo_dest_dc",
+              "success": "success._count",
+              "error404": "error404._count",
+              "error503": "error503._count"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+-------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+<1> A composite bucket aggregation that aggregates the data by `client_ip`.
+<2> A series of metrics and bucket sub-aggregations.
+<3> {infer-cap} bucket aggregation that contains the model ID and maps the 
+aggregation names to the model's input fields.

plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java

@@ -39,7 +39,6 @@ protected String repositoryType() {
 
     @Override
     protected Settings repositorySettings() {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/31498", HdfsRepositoryTests.isJava11());
         return Settings.builder()
             .put("uri", "hdfs:///")
             .put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName())

plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java

@@ -21,7 +21,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -46,16 +45,8 @@ protected SecureSettings credentials() {
         return new MockSecureSettings();
     }
 
-    @Override
-    public void tearDown() throws Exception {
-        if (isJava11() == false) {
-            super.tearDown();
-        }
-    }
-
     @Override
     protected void createRepository(String repoName) {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/31498", isJava11());
         AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository(repoName)
             .setType("hdfs")
             .setSettings(Settings.builder()
@@ -77,8 +68,4 @@ protected void assertCleanupResponse(CleanupRepositoryResponse response, long by
             assertThat(response.result().blobs(), equalTo(0L));
         }
     }
-
-    public static boolean isJava11() {
-        return JavaVersion.current().equals(JavaVersion.parse("11"));
-    }
 }

x-pack/plugin/data-streams/qa/rest/build.gradle

@@ -17,4 +17,6 @@ dependencies {
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.license.self_generated.type', 'basic'
+  // disable ILM history, since it disturbs tests using _all
+  setting 'indices.lifecycle.history_index_enabled', 'false'
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContains.java

@@ -58,6 +58,14 @@ protected TypeResolution resolveType() {
         return isStringAndExact(substring, sourceText(), Expressions.ParamOrdinal.SECOND);
     }
 
+    public Expression string() {
+        return string;
+    }
+
+    public Expression substring() {
+        return substring;
+    }
+
     @Override
     protected Pipe makePipe() {
         return new StringContainsFunctionPipe(source(), this,

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java

@@ -7,11 +7,13 @@
 package org.elasticsearch.xpack.eql.planner;
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.StringContains;
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.function.scalar.string.CaseSensitiveScalarFunction;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.ql.planner.ExpressionTranslator;
@@ -20,6 +22,7 @@
 import org.elasticsearch.xpack.ql.querydsl.query.Query;
 import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
 import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery;
+import org.elasticsearch.xpack.ql.querydsl.query.WildcardQuery;
 import org.elasticsearch.xpack.ql.util.CollectionUtils;
 
 import java.util.LinkedHashSet;
@@ -41,6 +44,7 @@ final class QueryTranslator {
             new ExpressionTranslators.StringQueries(),
             new ExpressionTranslators.Matches(),
             new ExpressionTranslators.MultiMatches(),
+            new CaseSensitiveScalarFunctions(),
             new Scalars()
     );
 
@@ -112,4 +116,34 @@ public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) {
             return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript()));
         }
     }
+
+    public static class CaseSensitiveScalarFunctions extends ExpressionTranslator<CaseSensitiveScalarFunction> {
+
+        @Override
+        protected Query asQuery(CaseSensitiveScalarFunction f, TranslatorHandler handler) {
+            return f.isCaseSensitive() ? doTranslate(f, handler) : null;
+        }
+
+        public static Query doTranslate(CaseSensitiveScalarFunction f, TranslatorHandler handler) {
+            Expression field = null;
+            Expression constant = null;
+
+            if (f instanceof StringContains) {
+                StringContains sc = (StringContains) f;
+                field = sc.string();
+                constant = sc.substring();
+            } else {
+                return null;
+            }
+
+            if (field instanceof FieldAttribute && constant.foldable()) {
+                String targetFieldName = handler.nameOf(((FieldAttribute) field).exactAttribute());
+                String substring = (String) constant.fold();
+
+                return new WildcardQuery(f.source(), targetFieldName, "*" + substring + "*");
+            }
+
+            return null;
+        }
+    }
 }

x-pack/plugin/eql/src/test/resources/mapping-default.json

@@ -56,7 +56,7 @@
             "type" : "text",
             "fields" : {
                 "keyword" : {
-                    "type" : "keyword",
+                    "type" : "wildcard",
                     "ignore_above" : 256
                 }
             }

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -189,12 +189,18 @@ process where startsWith(user_name, 'A') or startsWith(user_name, 'B')
 {"prefix":{"user_name":{"value":"B","boost":1.0}}}],"boost":1.0}}],"boost":1.0}}
 ;
 
-stringContains-caseSensitive
+stringContainsExactField-caseSensitive
 process where stringContains(process_name, "foo")
 ;
-"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.stringContains(
-InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))"
-"params":{"v0":"process_name","v1":"foo","v2":true}
+{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}},
+{"wildcard":{"process_name":{"wildcard":"*foo*","boost":1.0}}}],"boost":1.0}}
+;
+
+stringContainsExactSubField-caseSensitive
+process where stringContains(hostname, "foo")
+;
+{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}},
+{"wildcard":{"hostname.keyword":{"wildcard":"*foo*","boost":1.0}}}],"boost":1.0}}
 ;
 
 stringContains-caseInsensitive