Aggregate metric double use average

docs/changelog/142135.yaml

@@ -0,0 +1,5 @@
+area: Mapping
+issues: []
+pr: 142135
+summary: Aggregate metric double use average
+type: enhancement

server/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java

@@ -9,7 +9,11 @@
 
 package org.elasticsearch.index.fielddata;
 
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.util.NumericUtils;
 
 import java.io.IOException;
 
@@ -43,4 +47,85 @@ protected SortedNumericDoubleValues() {}
      */
     public abstract int docValueCount();
 
+    /**
+     * Converts a {@link SortedNumericDoubleValues} values to a singly valued {@link DoubleValues}
+     * if possible
+     */
+    public static DoubleValues unwrapSingleton(SortedNumericDoubleValues values) {
+        if (values instanceof SortedNumericDoubleValues.SingletonSortedNumericDoubleValues sv) {
+            return sv.values;
+        }
+        return null;
+    }
+
+    /**
+     * Converts a {@link DoubleValues} to a {@link SortedNumericDoubleValues}
+     */
+    public static SortedNumericDoubleValues singleton(DoubleValues values) {
+        return new SortedNumericDoubleValues.SingletonSortedNumericDoubleValues(values);
+    }
+
+    private static class SingletonSortedNumericDoubleValues extends SortedNumericDoubleValues {
+
+        private final DoubleValues values;
+
+        private SingletonSortedNumericDoubleValues(DoubleValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean advanceExact(int target) throws IOException {
+            return values.advanceExact(target);
+        }
+
+        @Override
+        public double nextValue() throws IOException {
+            return values.doubleValue();
+        }
+
+        @Override
+        public int docValueCount() {
+            return 1;
+        }
+    }
+
+    /**
+     * Converts a {@link SortedNumericDocValues} iterator to a {@link SortedNumericDoubleValues}
+     *
+     * Note that if the wrapped iterator can be unwrapped to a singleton {@link NumericDocValues}
+     * instance, then the returned {@link SortedNumericDoubleValues} can also be unwrapped to
+     * a {@link DoubleValues} instance via {@link #unwrapSingleton(SortedNumericDoubleValues)}
+     */
+    public static SortedNumericDoubleValues wrap(SortedNumericDocValues values) {
+        NumericDocValues singleton = DocValues.unwrapSingleton(values);
+        if (singleton != null) {
+            return new SortedNumericDoubleValues.SingletonSortedNumericDoubleValues(new DoubleValues() {
+                @Override
+                public double doubleValue() throws IOException {
+                    return NumericUtils.sortableLongToDouble(singleton.longValue());
+                }
+
+                @Override
+                public boolean advanceExact(int doc) throws IOException {
+                    return singleton.advanceExact(doc);
+                }
+            });
+        }
+        return new SortedNumericDoubleValues() {
+            @Override
+            public boolean advanceExact(int target) throws IOException {
+                return values.advanceExact(target);
+            }
+
+            @Override
+            public double nextValue() throws IOException {
+                return NumericUtils.sortableLongToDouble(values.nextValue());
+            }
+
+            @Override
+            public int docValueCount() {
+                return values.docValueCount();
+            }
+        };
+    }
 }

server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java

@@ -63,6 +63,7 @@ private SearchCapabilities() {}
     private static final String REJECT_INVALID_REVERSE_NESTING = "reject_invalid_reverse_nesting";
     private static final String DENSE_VECTOR_DOCVALUE_FIELDS_FORMAT = "dense_vector_docvalue_fields_format";
     private static final String KNN_QUERY_VECTOR_BASE64 = "knn_query_vector_base64";
+    private static final String AGGREGATE_METRIC_DOUBLE_DEFAULTS_TO_AVERAGE = "aggregate_metric_double_defaults_to_average";
 
     public static final Set<String> CAPABILITIES;
     static {
@@ -96,6 +97,7 @@ private SearchCapabilities() {}
         capabilities.add(REJECT_INVALID_REVERSE_NESTING);
         capabilities.add(DENSE_VECTOR_DOCVALUE_FIELDS_FORMAT);
         capabilities.add(KNN_QUERY_VECTOR_BASE64);
+        capabilities.add(AGGREGATE_METRIC_DOUBLE_DEFAULTS_TO_AVERAGE);
         CAPABILITIES = Set.copyOf(capabilities);
     }
 }

x-pack/plugin/build.gradle

@@ -175,6 +175,10 @@ tasks.named("yamlRestCompatTestTransform").configure(
     task.skipTest("esql/40_tsdb/PromQL irate on counter field", "We filter out null values now")
     // Expected deprecation warning to compat yaml tests:
     task.addAllowedWarningRegex("Parameter \\[default_metric\\] is deprecated and will be removed in a future version")
+    task.skipTest("aggregate-metrics/10_basic/Sort", "Average is used now instead of the default metric")
+    task.skipTest("aggregate-metrics/10_basic/Test term query on aggregate metric field", "Average is used now instead of the default metric")
+    task.skipTest("aggregate-metrics/10_basic/Test range query on aggregate metric field", "Average is used now instead of the default metric")
+    task.skipTest("aggregate-metrics/20_min_max_agg/Test min_max aggs with query", "Average is used now instead of the default metric")
     task.skipTest("aggregate-metrics/40_avg_agg/Test avg agg with query", "Default metric cannot be defined, so the query is empty")
     task.skipTest("aggregate-metrics/30_sum_agg/Test sum agg with query", "Default metric cannot be defined, so the query is empty")
     task.skipTest("aggregate-metrics/50_value_count_agg/Test value_count agg with query", "Default metric cannot be defined, so the query is empty")

x-pack/plugin/downsample/qa/rest/build.gradle

@@ -57,4 +57,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("downsample/10_basic/Downsample histogram as label", "Default metric has been removed")
   task.skipTest("downsample/10_basic/Downsample index", "Default metric has been removed")
   task.skipTest("downsample-with-security/10_basic/Downsample index", "Default metric has been removed")
+  task.skipTest("downsample/40_runtime_fields/Runtime fields accessing metric fields in downsample target index", "Using average as the value of an aggregate metric double")
+  task.skipTest("downsample-with-security/40_runtime_fields/Runtime fields accessing metric fields in downsample target index", "Using average as the value of an aggregate metric double")
 })

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml

@@ -1,9 +1,12 @@
 ---
 "Runtime fields accessing metric fields in downsample target index":
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: _tsid hashing introduced in 8.13
-      test_runner_features: [close_to, allowed_warnings]
+      reason: Using average as the value of a downsampled gauge was added in 8.14
+      test_runner_features: [close_to, allowed_warnings, capabilities]
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ aggregate_metric_double_defaults_to_average ]
 
   - do:
       indices.create:
@@ -144,23 +147,23 @@
 
   - close_to: { hits.hits.0.fields.received_kb.0: { value: 518142.3935, error: 0.0001 } }
   - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1400935.4472, error: 0.0001 } }
-  - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1400955.0009, error: 0.0001 } }
-  - close_to: { hits.hits.0.fields.rx_kb.0: { value: 518164.1484, error: 0.0001 } }
+  - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1400927.6132, error: 0.0001 } }
+  - close_to: { hits.hits.0.fields.rx_kb.0: { value: 518151.9951, error: 0.0001 } }
 
   - close_to: { hits.hits.1.fields.received_kb.0: { value: 518186.5039, error: 0.0001 } }
   - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1400988.2822, error: 0.0001 } }
-  - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1400971.9453, error: 0.0001 } }
-  - close_to: { hits.hits.1.fields.rx_kb.0: { value: 518169.4443, error: 0.0001 } }
+  - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1400968.2451, error: 0.0001 } }
+  - close_to: { hits.hits.1.fields.rx_kb.0: { value: 518169.0957, error: 0.0001 } }
 
   - close_to: { hits.hits.2.fields.received_kb.0: { value: 783343.5488, error: 0.0001 } }
   - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1954908.8779, error: 0.0001 } }
-  - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1958181.5957, error: 0.0001 } }
-  - close_to: { hits.hits.2.fields.rx_kb.0: { value: 783333.7832, error: 0.0001 } }
+  - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1956541.3305, error: 0.0001 } }
+  - close_to: { hits.hits.2.fields.rx_kb.0: { value: 783014.5332, error: 0.0001 } }
 
   - close_to: { hits.hits.3.fields.received_kb.0: { value: 783377.7343, error: 0.0001 } }
   - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1955339.7343, error: 0.0001 } }
-  - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1965738.4785, error: 0.0001 } }
-  - close_to: { hits.hits.3.fields.rx_kb.0: { value: 784849.3369, error: 0.0001 } }
+  - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1962470.67333, error: 0.0001 } }
+  - close_to: { hits.hits.3.fields.rx_kb.0: { value: 784190.9179, error: 0.0001 } }
 
 ---
 "Runtime field accessing dimension fields in downsample target index":

x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricAverageScript.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.aggregatemetric.mapper;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericLongValues;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.OnScriptError;
+import org.elasticsearch.script.DoubleFieldScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.field.DoubleDocValuesField;
+import org.elasticsearch.script.field.LongDocValuesField;
+import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.runtime.DoubleScriptFieldRangeQuery;
+import org.elasticsearch.search.runtime.DoubleScriptFieldTermQuery;
+import org.elasticsearch.search.runtime.DoubleScriptFieldTermsQuery;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * This script is using the aggregate metric double metrics sum and value count to calculate the average.
+ *
+ * All helper methods assume that the `aggregate_metric_double` has the metrics `sum` and `value_count` configured.
+ */
+class AggregateMetricAverageScript extends DoubleFieldScript {
+    private static final Script EMPTY_SCRIPT = new Script("");
+
+    private final DoubleDocValuesField sumDocValuesField;
+    private final LongDocValuesField countDocValuesField;
+
+    AggregateMetricAverageScript(String fieldName, SearchLookup lookup, LeafReaderContext ctx) {
+        super(fieldName, Map.of(), lookup, OnScriptError.FAIL, ctx);
+        try {
+            String sumFieldName = AggregateMetricDoubleFieldMapper.subfieldName(fieldName, AggregateMetricDoubleFieldMapper.Metric.sum);
+            sumDocValuesField = new DoubleDocValuesField(
+                SortedNumericDoubleValues.wrap(DocValues.getSortedNumeric(ctx.reader(), sumFieldName)),
+                sumFieldName
+            );
+            String countFieldName = AggregateMetricDoubleFieldMapper.subfieldName(
+                fieldName,
+                AggregateMetricDoubleFieldMapper.Metric.value_count
+            );
+            countDocValuesField = new LongDocValuesField(
+                SortedNumericLongValues.wrap(DocValues.getSortedNumeric(ctx.reader(), countFieldName)),
+                countFieldName
+            );
+        } catch (IOException e) {
+            throw new IllegalStateException("Cannot load doc values", e);
+        }
+    }
+
+    static LeafFactory newLeafFactory(String fieldName, SearchLookup lookup) {
+        return ctx -> new AggregateMetricAverageScript(fieldName, lookup, ctx);
+    }
+
+    static Query doubleRangeQuery(
+        String fieldName,
+        Object lowerTerm,
+        Object upperTerm,
+        boolean includeLower,
+        boolean includeUpper,
+        SearchLookup lookup
+    ) {
+        return NumberFieldMapper.NumberType.doubleRangeQuery(
+            lowerTerm,
+            upperTerm,
+            includeLower,
+            includeUpper,
+            (l, u) -> new DoubleScriptFieldRangeQuery(
+                EMPTY_SCRIPT,
+                AggregateMetricAverageScript.newLeafFactory(fieldName, lookup),
+                fieldName,
+                l,
+                u
+            )
+        );
+    }
+
+    static Query doubleTermsQuery(String fieldName, Collection<?> values, SearchLookup lookup) {
+        Set<Long> terms = Sets.newHashSetWithExpectedSize(values.size());
+        for (Object value : values) {
+            terms.add(Double.doubleToLongBits(NumberFieldMapper.NumberType.objectToDouble(value)));
+        }
+        return new DoubleScriptFieldTermsQuery(
+            EMPTY_SCRIPT,
+            AggregateMetricAverageScript.newLeafFactory(fieldName, lookup),
+            fieldName,
+            terms
+        );
+    }
+
+    static Query doubleTermQuery(String fieldName, Object value, SearchLookup lookup) {
+        return new DoubleScriptFieldTermQuery(
+            EMPTY_SCRIPT,
+            AggregateMetricAverageScript.newLeafFactory(fieldName, lookup),
+            fieldName,
+            NumberFieldMapper.NumberType.objectToDouble(value)
+        );
+    }
+
+    static SortField sortField(
+        String fieldName,
+        Object missingValue,
+        MultiValueMode sortMode,
+        IndexFieldData.XFieldComparatorSource.Nested nested,
+        boolean reverse,
+        Function<LeafReaderContext, SortedNumericDoubleValues> docValueLoader
+    ) {
+        return new SortField(fieldName, new DoubleValuesComparatorSource(null, missingValue, sortMode, nested) {
+            @Override
+            protected SortedNumericDoubleValues getValues(LeafReaderContext context) {
+                return docValueLoader.apply(context);
+            }
+        }, reverse);
+    }
+
+    @Override
+    protected void emitFromObject(Object v) {
+        // we only use doc-values, not _source, so no need to implement this method
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void setDocument(int docID) {
+        try {
+            sumDocValuesField.setNextDocId(docID);
+            countDocValuesField.setNextDocId(docID);
+        } catch (IOException e) {
+            throw new IllegalStateException("Cannot load doc values", e);
+        }
+    }
+
+    @Override
+    public void execute() {
+        Iterator<Double> sumIterator = sumDocValuesField.iterator();
+        Iterator<Long> countIterator = countDocValuesField.iterator();
+        while (sumIterator.hasNext() && countIterator.hasNext()) {
+            Double sum = sumIterator.next();
+            Long count = countIterator.next();
+            emit(sum / count);
+        }
+    }
+}