[ML] Docs and HRLC for datafeed runtime mappings (#65810) (#66007)

For the changes in #65606

Author: davidkyle

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java

@@ -41,6 +41,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -95,6 +96,7 @@ public class DatafeedConfig implements ToXContentObject {
         PARSER.declareObject(Builder::setIndicesOptions,
             (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)),
             INDICES_OPTIONS);
+        PARSER.declareObject(Builder::setRuntimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD);
     }
 
     private static BytesReference parseBytes(XContentParser parser) throws IOException {
@@ -116,11 +118,12 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti
     private final DelayedDataCheckConfig delayedDataCheckConfig;
     private final Integer maxEmptySearches;
     private final IndicesOptions indicesOptions;
+    private final Map<String, Object> runtimeMappings;
 
     private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, BytesReference query,
                            BytesReference aggregations, List<SearchSourceBuilder.ScriptField> scriptFields, Integer scrollSize,
                            ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig,
-                           Integer maxEmptySearches, IndicesOptions indicesOptions) {
+                           Integer maxEmptySearches, IndicesOptions indicesOptions, Map<String, Object> runtimeMappings) {
         this.id = id;
         this.jobId = jobId;
         this.queryDelay = queryDelay;
@@ -134,6 +137,7 @@ private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue
         this.delayedDataCheckConfig = delayedDataCheckConfig;
         this.maxEmptySearches = maxEmptySearches;
         this.indicesOptions = indicesOptions;
+        this.runtimeMappings = Collections.unmodifiableMap(runtimeMappings);
     }
 
     public String getId() {
@@ -188,6 +192,10 @@ public IndicesOptions getIndicesOptions() {
         return indicesOptions;
     }
 
+    public Map<String, Object> getRuntimeMappings() {
+        return runtimeMappings;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -232,6 +240,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             indicesOptions.toXContent(builder, params);
             builder.endObject();
         }
+        if (runtimeMappings.isEmpty() == false) {
+            builder.field(SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings);
+        }
 
         builder.endObject();
         return builder;
@@ -274,7 +285,8 @@ public boolean equals(Object other) {
             && Objects.equals(this.chunkingConfig, that.chunkingConfig)
             && Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig)
             && Objects.equals(this.maxEmptySearches, that.maxEmptySearches)
-            && Objects.equals(this.indicesOptions, that.indicesOptions);
+            && Objects.equals(this.indicesOptions, that.indicesOptions)
+            && Objects.equals(this.runtimeMappings, that.runtimeMappings);
     }
 
     /**
@@ -285,7 +297,7 @@ public boolean equals(Object other) {
     @Override
     public int hashCode() {
         return Objects.hash(id, jobId, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields,
-            chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions);
+            chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions, runtimeMappings);
     }
 
     public static Builder builder(String id, String jobId) {
@@ -294,8 +306,8 @@ public static Builder builder(String id, String jobId) {
 
     public static class Builder {
 
-        private String id;
-        private String jobId;
+        private final String id;
+        private final String jobId;
         private TimeValue queryDelay;
         private TimeValue frequency;
         private List<String> indices;
@@ -307,6 +319,7 @@ public static class Builder {
         private DelayedDataCheckConfig delayedDataCheckConfig;
         private Integer maxEmptySearches;
         private IndicesOptions indicesOptions;
+        private Map<String, Object> runtimeMappings = Collections.emptyMap();
 
         public Builder(String id, String jobId) {
             this.id = Objects.requireNonNull(id, ID.getPreferredName());
@@ -327,6 +340,7 @@ public Builder(DatafeedConfig config) {
             this.delayedDataCheckConfig = config.getDelayedDataCheckConfig();
             this.maxEmptySearches = config.getMaxEmptySearches();
             this.indicesOptions = config.indicesOptions;
+            this.runtimeMappings = new HashMap<>(config.runtimeMappings);
         }
 
         public Builder setIndices(List<String> indices) {
@@ -419,9 +433,15 @@ public Builder setIndicesOptions(IndicesOptions indicesOptions) {
             return this;
         }
 
+        public Builder setRuntimeMappings(Map<String, Object> runtimeMappings) {
+            this.runtimeMappings = Objects.requireNonNull(runtimeMappings,
+                SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD.getPreferredName());
+            return this;
+        }
+
         public DatafeedConfig build() {
             return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize,
-                chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions);
+                chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions, runtimeMappings);
         }
 
         private static BytesReference xContentToBytes(ToXContentObject object) throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -711,6 +711,16 @@ public void testPutDatafeed() throws Exception {
             datafeedBuilder.setScrollSize(1000); // <1>
             // end::put-datafeed-config-set-scroll-size
 
+            // tag::put-datafeed-config-set-runtime-mappings
+            Map<String, Object> fieldProperties = new HashMap<>();
+            fieldProperties.put("type", "keyword");
+            fieldProperties.put("script", "emit(params._source.agent.toLowerCase())");
+            Map<String, Object> runtimeMappings = new HashMap<>();
+            runtimeMappings.put("agent_lowercase", fieldProperties);
+
+            datafeedBuilder.setRuntimeMappings(runtimeMappings); // <1>
+            // end::put-datafeed-config-set-runtime-mappings
+
             // tag::put-datafeed-request
             PutDatafeedRequest request = new PutDatafeedRequest(datafeedBuilder.build()); // <1>
             // end::put-datafeed-request

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java

@@ -36,7 +36,9 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class DatafeedConfigTests extends AbstractXContentTestCase<DatafeedConfig> {
 
@@ -117,6 +119,14 @@ public static DatafeedConfig.Builder createRandomBuilder() {
                 randomBoolean(),
                 randomBoolean()));
         }
+        if (randomBoolean()) {
+            Map<String, Object> settings = new HashMap<>();
+            settings.put("type", "keyword");
+            settings.put("script", "");
+            Map<String, Object> field = new HashMap<>();
+            field.put("runtime_field_foo", settings);
+            builder.setRuntimeMappings(field);
+        }
         return builder;
     }
 

docs/java-rest/high-level/ml/put-datafeed.asciidoc

@@ -89,6 +89,12 @@ include-tagged::{doc-tests-file}[{api}-config-set-scroll-size]
 --------------------------------------------------
 <1> The `size` parameter used in the searches.
 
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-config-set-runtime-mappings]
+--------------------------------------------------
+<1> Set the runtime mappings used in the searches.
+
 include::../execution.asciidoc[]
 
 [id="{upid}-{api}-response"]

docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc

@@ -18,7 +18,7 @@ Instantiates a {dfeed}.
 [[ml-put-datafeed-prereqs]]
 == {api-prereq-title}
 
-* You must create an {anomaly-job} before you create a {dfeed}.  
+* You must create an {anomaly-job} before you create a {dfeed}.
 * If {es} {security-features} are enabled, you must have `manage_ml` or `manage`
 cluster privileges to use this API. See <<security-privileges>> and
 {ml-docs-setup-privileges}.
@@ -105,6 +105,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size]
 (Optional, object)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options]
 
+`runtime_mappings`::
+(Optional, object)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings]
 
 [[ml-put-datafeed-example]]
 == {api-examples-title}

docs/reference/ml/ml-shared.asciidoc

@@ -765,6 +765,25 @@ For more information about these options, see <<multi-index>>.
 --
 end::indices-options[]
 
+tag::runtime-mappings[]
+Specifies runtime fields for the datafeed search.
++
+--
+For example:
+```
+{
+  "day_of_week": {
+    "type": "keyword",
+    "script": {
+      "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+    }
+  }
+}
+```
+
+--
+end::runtime-mappings[]
+
 tag::inference-config-classification-num-top-classes[]
 Specifies the number of top class predictions to return. Defaults to 0.
 end::inference-config-classification-num-top-classes[]
@@ -992,7 +1011,7 @@ There are three available modes:
 --
 * `auto`: The chunk size is dynamically calculated. This is the default and
 recommended value when the {dfeed} does not use aggregations.
-* `manual`: Chunking is applied according to the specified `time_span`. Use this 
+* `manual`: Chunking is applied according to the specified `time_span`. Use this
 mode when the {dfeed} uses aggregations.
 * `off`: No chunking is applied.
 --
@@ -1297,8 +1316,8 @@ and <<script-fields,Script fields>>.
 end::script-fields[]
 
 tag::scroll-size[]
-The `size` parameter that is used in {es} searches when the {dfeed} does not use 
-aggregations. The default value is `1000`. The maximum value is the value of 
+The `size` parameter that is used in {es} searches when the {dfeed} does not use
+aggregations. The default value is `1000`. The maximum value is the value of
 `index.max_result_window` which is 10,000 by default.
 end::scroll-size[]