diff --git a/docs/changelog/142135.yaml b/docs/changelog/142135.yaml new file mode 100644 index 0000000000000..c4bc18f853f7d --- /dev/null +++ b/docs/changelog/142135.yaml @@ -0,0 +1,5 @@ +area: Mapping +issues: [] +pr: 142135 +summary: Aggregate metric double use average +type: enhancement diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java index 6ecd9293e574c..863441970def5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java @@ -9,7 +9,11 @@ package org.elasticsearch.index.fielddata; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DoubleValues; +import org.apache.lucene.util.NumericUtils; import java.io.IOException; @@ -43,4 +47,85 @@ protected SortedNumericDoubleValues() {} */ public abstract int docValueCount(); + /** + * Converts a {@link SortedNumericDoubleValues} values to a singly valued {@link DoubleValues} + * if possible + */ + public static DoubleValues unwrapSingleton(SortedNumericDoubleValues values) { + if (values instanceof SortedNumericDoubleValues.SingletonSortedNumericDoubleValues sv) { + return sv.values; + } + return null; + } + + /** + * Converts a {@link DoubleValues} to a {@link SortedNumericDoubleValues} + */ + public static SortedNumericDoubleValues singleton(DoubleValues values) { + return new SortedNumericDoubleValues.SingletonSortedNumericDoubleValues(values); + } + + private static class SingletonSortedNumericDoubleValues extends SortedNumericDoubleValues { + + private final DoubleValues values; + + private SingletonSortedNumericDoubleValues(DoubleValues values) { + this.values = values; + } + + @Override + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); + } + + @Override + public double nextValue() throws IOException { + return values.doubleValue(); + } + + @Override + public int docValueCount() { + return 1; + } + } + + /** + * Converts a {@link SortedNumericDocValues} iterator to a {@link SortedNumericDoubleValues} + * + * Note that if the wrapped iterator can be unwrapped to a singleton {@link NumericDocValues} + * instance, then the returned {@link SortedNumericDoubleValues} can also be unwrapped to + * a {@link DoubleValues} instance via {@link #unwrapSingleton(SortedNumericDoubleValues)} + */ + public static SortedNumericDoubleValues wrap(SortedNumericDocValues values) { + NumericDocValues singleton = DocValues.unwrapSingleton(values); + if (singleton != null) { + return new SortedNumericDoubleValues.SingletonSortedNumericDoubleValues(new DoubleValues() { + @Override + public double doubleValue() throws IOException { + return NumericUtils.sortableLongToDouble(singleton.longValue()); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return singleton.advanceExact(doc); + } + }); + } + return new SortedNumericDoubleValues() { + @Override + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); + } + + @Override + public double nextValue() throws IOException { + return NumericUtils.sortableLongToDouble(values.nextValue()); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index f95529fcc4dee..69596dcae1890 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -63,6 +63,7 @@ private SearchCapabilities() {} private static final String REJECT_INVALID_REVERSE_NESTING = "reject_invalid_reverse_nesting"; private static final String DENSE_VECTOR_DOCVALUE_FIELDS_FORMAT = "dense_vector_docvalue_fields_format"; private static final String KNN_QUERY_VECTOR_BASE64 = "knn_query_vector_base64"; + private static final String AGGREGATE_METRIC_DOUBLE_DEFAULTS_TO_AVERAGE = "aggregate_metric_double_defaults_to_average"; public static final Set CAPABILITIES; static { @@ -96,6 +97,7 @@ private SearchCapabilities() {} capabilities.add(REJECT_INVALID_REVERSE_NESTING); capabilities.add(DENSE_VECTOR_DOCVALUE_FIELDS_FORMAT); capabilities.add(KNN_QUERY_VECTOR_BASE64); + capabilities.add(AGGREGATE_METRIC_DOUBLE_DEFAULTS_TO_AVERAGE); CAPABILITIES = Set.copyOf(capabilities); } } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 18070c914e310..d8ad787d1e0b3 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -175,6 +175,10 @@ tasks.named("yamlRestCompatTestTransform").configure( task.skipTest("esql/40_tsdb/PromQL irate on counter field", "We filter out null values now") // Expected deprecation warning to compat yaml tests: task.addAllowedWarningRegex("Parameter \\[default_metric\\] is deprecated and will be removed in a future version") + task.skipTest("aggregate-metrics/10_basic/Sort", "Average is used now instead of the default metric") + task.skipTest("aggregate-metrics/10_basic/Test term query on aggregate metric field", "Average is used now instead of the default metric") + task.skipTest("aggregate-metrics/10_basic/Test range query on aggregate metric field", "Average is used now instead of the default metric") + task.skipTest("aggregate-metrics/20_min_max_agg/Test min_max aggs with query", "Average is used now instead of the default metric") task.skipTest("aggregate-metrics/40_avg_agg/Test avg agg with query", "Default metric cannot be defined, so the query is empty") task.skipTest("aggregate-metrics/30_sum_agg/Test sum agg with query", "Default metric cannot be defined, so the query is empty") task.skipTest("aggregate-metrics/50_value_count_agg/Test value_count agg with query", "Default metric cannot be defined, so the query is empty") diff --git a/x-pack/plugin/downsample/qa/rest/build.gradle b/x-pack/plugin/downsample/qa/rest/build.gradle index d3434c30924ea..d896ee0b0ed12 100644 --- a/x-pack/plugin/downsample/qa/rest/build.gradle +++ b/x-pack/plugin/downsample/qa/rest/build.gradle @@ -57,4 +57,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("downsample/10_basic/Downsample histogram as label", "Default metric has been removed") task.skipTest("downsample/10_basic/Downsample index", "Default metric has been removed") task.skipTest("downsample-with-security/10_basic/Downsample index", "Default metric has been removed") + task.skipTest("downsample/40_runtime_fields/Runtime fields accessing metric fields in downsample target index", "Using average as the value of an aggregate metric double") + task.skipTest("downsample-with-security/40_runtime_fields/Runtime fields accessing metric fields in downsample target index", "Using average as the value of an aggregate metric double") }) diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml index 71b8e5bc1c1d7..10ab2fcc5d128 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml @@ -1,9 +1,12 @@ --- "Runtime fields accessing metric fields in downsample target index": - requires: - cluster_features: ["gte_v8.13.0"] - reason: _tsid hashing introduced in 8.13 - test_runner_features: [close_to, allowed_warnings] + reason: Using average as the value of a downsampled gauge was added in 8.14 + test_runner_features: [close_to, allowed_warnings, capabilities] + capabilities: + - method: POST + path: /_search + capabilities: [ aggregate_metric_double_defaults_to_average ] - do: indices.create: @@ -144,23 +147,23 @@ - close_to: { hits.hits.0.fields.received_kb.0: { value: 518142.3935, error: 0.0001 } } - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1400935.4472, error: 0.0001 } } - - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1400955.0009, error: 0.0001 } } - - close_to: { hits.hits.0.fields.rx_kb.0: { value: 518164.1484, error: 0.0001 } } + - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1400927.6132, error: 0.0001 } } + - close_to: { hits.hits.0.fields.rx_kb.0: { value: 518151.9951, error: 0.0001 } } - close_to: { hits.hits.1.fields.received_kb.0: { value: 518186.5039, error: 0.0001 } } - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1400988.2822, error: 0.0001 } } - - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1400971.9453, error: 0.0001 } } - - close_to: { hits.hits.1.fields.rx_kb.0: { value: 518169.4443, error: 0.0001 } } + - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1400968.2451, error: 0.0001 } } + - close_to: { hits.hits.1.fields.rx_kb.0: { value: 518169.0957, error: 0.0001 } } - close_to: { hits.hits.2.fields.received_kb.0: { value: 783343.5488, error: 0.0001 } } - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1954908.8779, error: 0.0001 } } - - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1958181.5957, error: 0.0001 } } - - close_to: { hits.hits.2.fields.rx_kb.0: { value: 783333.7832, error: 0.0001 } } + - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1956541.3305, error: 0.0001 } } + - close_to: { hits.hits.2.fields.rx_kb.0: { value: 783014.5332, error: 0.0001 } } - close_to: { hits.hits.3.fields.received_kb.0: { value: 783377.7343, error: 0.0001 } } - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1955339.7343, error: 0.0001 } } - - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1965738.4785, error: 0.0001 } } - - close_to: { hits.hits.3.fields.rx_kb.0: { value: 784849.3369, error: 0.0001 } } + - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1962470.67333, error: 0.0001 } } + - close_to: { hits.hits.3.fields.rx_kb.0: { value: 784190.9179, error: 0.0001 } } --- "Runtime field accessing dimension fields in downsample target index": diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricAverageScript.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricAverageScript.java new file mode 100644 index 0000000000000..a898598260333 --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricAverageScript.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.aggregatemetric.mapper; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.fielddata.SortedNumericLongValues; +import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.OnScriptError; +import org.elasticsearch.script.DoubleFieldScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.field.DoubleDocValuesField; +import org.elasticsearch.script.field.LongDocValuesField; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.runtime.DoubleScriptFieldRangeQuery; +import org.elasticsearch.search.runtime.DoubleScriptFieldTermQuery; +import org.elasticsearch.search.runtime.DoubleScriptFieldTermsQuery; + +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** + * This script is using the aggregate metric double metrics sum and value count to calculate the average. + * + * All helper methods assume that the `aggregate_metric_double` has the metrics `sum` and `value_count` configured. + */ +class AggregateMetricAverageScript extends DoubleFieldScript { + private static final Script EMPTY_SCRIPT = new Script(""); + + private final DoubleDocValuesField sumDocValuesField; + private final LongDocValuesField countDocValuesField; + + AggregateMetricAverageScript(String fieldName, SearchLookup lookup, LeafReaderContext ctx) { + super(fieldName, Map.of(), lookup, OnScriptError.FAIL, ctx); + try { + String sumFieldName = AggregateMetricDoubleFieldMapper.subfieldName(fieldName, AggregateMetricDoubleFieldMapper.Metric.sum); + sumDocValuesField = new DoubleDocValuesField( + SortedNumericDoubleValues.wrap(DocValues.getSortedNumeric(ctx.reader(), sumFieldName)), + sumFieldName + ); + String countFieldName = AggregateMetricDoubleFieldMapper.subfieldName( + fieldName, + AggregateMetricDoubleFieldMapper.Metric.value_count + ); + countDocValuesField = new LongDocValuesField( + SortedNumericLongValues.wrap(DocValues.getSortedNumeric(ctx.reader(), countFieldName)), + countFieldName + ); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + static LeafFactory newLeafFactory(String fieldName, SearchLookup lookup) { + return ctx -> new AggregateMetricAverageScript(fieldName, lookup, ctx); + } + + static Query doubleRangeQuery( + String fieldName, + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + SearchLookup lookup + ) { + return NumberFieldMapper.NumberType.doubleRangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + (l, u) -> new DoubleScriptFieldRangeQuery( + EMPTY_SCRIPT, + AggregateMetricAverageScript.newLeafFactory(fieldName, lookup), + fieldName, + l, + u + ) + ); + } + + static Query doubleTermsQuery(String fieldName, Collection values, SearchLookup lookup) { + Set terms = Sets.newHashSetWithExpectedSize(values.size()); + for (Object value : values) { + terms.add(Double.doubleToLongBits(NumberFieldMapper.NumberType.objectToDouble(value))); + } + return new DoubleScriptFieldTermsQuery( + EMPTY_SCRIPT, + AggregateMetricAverageScript.newLeafFactory(fieldName, lookup), + fieldName, + terms + ); + } + + static Query doubleTermQuery(String fieldName, Object value, SearchLookup lookup) { + return new DoubleScriptFieldTermQuery( + EMPTY_SCRIPT, + AggregateMetricAverageScript.newLeafFactory(fieldName, lookup), + fieldName, + NumberFieldMapper.NumberType.objectToDouble(value) + ); + } + + static SortField sortField( + String fieldName, + Object missingValue, + MultiValueMode sortMode, + IndexFieldData.XFieldComparatorSource.Nested nested, + boolean reverse, + Function docValueLoader + ) { + return new SortField(fieldName, new DoubleValuesComparatorSource(null, missingValue, sortMode, nested) { + @Override + protected SortedNumericDoubleValues getValues(LeafReaderContext context) { + return docValueLoader.apply(context); + } + }, reverse); + } + + @Override + protected void emitFromObject(Object v) { + // we only use doc-values, not _source, so no need to implement this method + throw new UnsupportedOperationException(); + } + + @Override + public void setDocument(int docID) { + try { + sumDocValuesField.setNextDocId(docID); + countDocValuesField.setNextDocId(docID); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public void execute() { + Iterator sumIterator = sumDocValuesField.iterator(); + Iterator countIterator = countDocValuesField.iterator(); + while (sumIterator.hasNext() && countIterator.hasNext()) { + Double sum = sumIterator.next(); + Long count = countIterator.next(); + emit(sum / count); + } + } +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java index 99f6bf2b9bd89..74369a6d7ca3d 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java @@ -17,6 +17,7 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexSettings; @@ -32,6 +33,7 @@ import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; +import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -268,8 +270,9 @@ public AggregateMetricDoubleFieldMapper build(MapperBuilderContext context) { ); public static final class AggregateMetricDoubleFieldType extends SimpleMappedFieldType { - private final EnumMap metricFields; + // If there is only one metric configured for this field, we delegate to the field. + private final Metric singleMetric; private final MetricType metricType; public AggregateMetricDoubleFieldType( @@ -278,34 +281,16 @@ public AggregateMetricDoubleFieldType( EnumMap metricFields, Map meta ) { - super(name, metricFields.get(derivedDefaultMetric(name, metricFields)).indexType(), false, meta); + super( + name, + // If it contains a single metric, we use the index type of that metric, otherwise we default to doc values only + metricFields.size() == 1 ? metricFields.values().iterator().next().indexType() : IndexType.docValuesOnly(), + false, + meta + ); this.metricType = metricType; this.metricFields = metricFields; - } - - private Metric derivedDefaultMetric() { - return derivedDefaultMetric(name(), metricFields); - } - - private static Metric derivedDefaultMetric(String name, EnumMap metricFields) { - if (metricFields.containsKey(Metric.max)) { - return Metric.max; - } - // We iterate on the metrics to ensure that we have a deterministic order in metrics - for (Metric metric : Metric.values()) { - if (metricFields.containsKey(metric)) { - return metric; - } - } - throw new IllegalStateException("No metric found for aggregate metric field [" + name + "]"); - } - - /** - * Return a delegate field type which is max if it exists or any other metric. - * @return a field type - */ - private NumberFieldMapper.NumberFieldType delegateFieldType() { - return metricFields.get(derivedDefaultMetric()); + this.singleMetric = metricFields.size() == 1 ? metricFields.keySet().iterator().next() : null; } @Override @@ -324,12 +309,14 @@ public Map getMetricFields() { @Override public boolean mayExistInIndex(SearchExecutionContext context) { - return delegateFieldType().mayExistInIndex(context); // TODO how does searching actually work here? + assert metricFields.isEmpty() == false : "aggregate metric double should have at least one metric defined"; + return metricFields.values().iterator().next().mayExistInIndex(context); } @Override public Query existsQuery(SearchExecutionContext context) { - return delegateFieldType().existsQuery(context); + assert metricFields.isEmpty() == false : "aggregate metric double should have at least one metric defined"; + return metricFields.values().iterator().next().existsQuery(context); } @Override @@ -337,12 +324,27 @@ public Query termQuery(Object value, SearchExecutionContext context) { if (value == null) { throw new IllegalArgumentException("Cannot search for null."); } - return delegateFieldType().termQuery(value, context); + if (singleMetric != null) { + return metricFields.get(singleMetric).termQuery(value, context); + } + if (metricFields.containsKey(Metric.sum) && metricFields.containsKey(Metric.value_count)) { + return AggregateMetricAverageScript.doubleTermQuery(name(), value, context.lookup()); + } + return Queries.NO_DOCS_INSTANCE; } @Override public Query termsQuery(Collection values, SearchExecutionContext context) { - return delegateFieldType().termsQuery(values, context); + if (values.isEmpty()) { + return Queries.ALL_DOCS_INSTANCE; + } + if (singleMetric != null) { + return metricFields.get(singleMetric).termsQuery(values, context); + } + if (metricFields.containsKey(Metric.sum) && metricFields.containsKey(Metric.value_count)) { + return AggregateMetricAverageScript.doubleTermsQuery(name(), values, context.lookup()); + } + return Queries.NO_DOCS_INSTANCE; } @Override @@ -353,12 +355,20 @@ public Query rangeQuery( boolean includeUpper, SearchExecutionContext context ) { - return delegateFieldType().rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); - } - - @Override - public Object valueForDisplay(Object value) { - return delegateFieldType().valueForDisplay(value); + if (singleMetric != null) { + return metricFields.get(singleMetric).rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + } + if (metricFields.containsKey(Metric.sum) && metricFields.containsKey(Metric.value_count)) { + return AggregateMetricAverageScript.doubleRangeQuery( + name(), + lowerTerm, + upperTerm, + includeLower, + includeUpper, + context.lookup() + ); + } + return Queries.NO_DOCS_INSTANCE; } @Override @@ -381,7 +391,11 @@ public Relation isFieldWithinQuery( DateMathParser dateMathParser, QueryRewriteContext context ) throws IOException { - return delegateFieldType().isFieldWithinQuery(reader, from, to, includeLower, includeUpper, timeZone, dateMathParser, context); + if (singleMetric != null) { + return metricFields.get(singleMetric) + .isFieldWithinQuery(reader, from, to, includeLower, includeUpper, timeZone, dateMathParser, context); + } + return super.isFieldWithinQuery(reader, from, to, includeLower, includeUpper, timeZone, dateMathParser, context); } @Override @@ -436,7 +450,42 @@ public double nextValue() throws IOException { @Override public SortedNumericDoubleValues getAggregateMetricValues() { - return getAggregateMetricValues(derivedDefaultMetric()); + if (singleMetric != null) { + return getAggregateMetricValues(singleMetric); + } + try { + final SortedNumericDocValues sumValues = DocValues.getSortedNumeric( + context.reader(), + subfieldName(getFieldName(), Metric.sum) + ); + final SortedNumericDocValues countValues = DocValues.getSortedNumeric( + context.reader(), + subfieldName(getFieldName(), Metric.value_count) + ); + + return new SortedNumericDoubleValues() { + @Override + public int docValueCount() { + assert countValues.docValueCount() == sumValues.docValueCount() + : "docValueCount mismatch between sum and count values"; + return countValues.docValueCount(); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return sumValues.advanceExact(doc) && countValues.advanceExact(doc); + } + + @Override + public double nextValue() throws IOException { + double sum = NumericUtils.sortableLongToDouble(sumValues.nextValue()); + long count = countValues.nextValue(); + return count == 0 ? Double.NaN : sum / count; + } + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } } @Override @@ -475,7 +524,28 @@ public SortField sortField( XFieldComparatorSource.Nested nested, boolean reverse ) { - return new SortedNumericSortField(subfieldName(name(), derivedDefaultMetric()), SortField.Type.DOUBLE, reverse); + if (singleMetric != null) { + return new SortedNumericSortField(subfieldName(name(), singleMetric), SortField.Type.DOUBLE, reverse); + } + if (metricFields.containsKey(Metric.sum) && metricFields.containsKey(Metric.value_count)) { + return AggregateMetricAverageScript.sortField( + fieldName, + missingValue, + sortMode, + nested, + reverse, + ctx -> load(ctx).getAggregateMetricValues() + ); + } + throw new UnsupportedOperationException( + "[" + + CONTENT_TYPE + + "] supports sorting if it has configured a single metric or the metrics [" + + Metric.sum + + ", " + + Metric.value_count + + "]" + ); } @Override @@ -509,8 +579,19 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { case AMD_MAX -> Metric.max; case AMD_MIN -> Metric.min; case AMD_SUM -> Metric.sum; - case AMD_DEFAULT -> derivedDefaultMetric(); // TODO: temporary until we combine - // https://github.com/elastic/elasticsearch/pull/141331 + // TODO: temporary until we combine https://github.com/elastic/elasticsearch/pull/141331 + case AMD_DEFAULT -> { + String name = name(); + if (metricFields.containsKey(Metric.max)) { + yield Metric.max; + } + for (Metric m : Metric.values()) { + if (metricFields.containsKey(m)) { + yield m; + } + } + throw new IllegalStateException("No metric found for aggregate metric field [" + name + "]"); + } default -> null; }; if (metric == null) { @@ -555,7 +636,8 @@ public MetricType getMetricType() { /** A set of metrics supported */ private final EnumSet metrics; - /** The default metric to be when querying this field type */ + /** Deprecated, we default to average or the single metric, if there is only one configured */ + @Deprecated(since = "9.4.0", forRemoval = true) protected Metric defaultMetric; /** The metric type (gauge, counter, summary) if field is a time series metric */ diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java index db8703c749b7c..c18a77186a352 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java @@ -71,7 +71,7 @@ protected Collection getPlugins() { @Override protected void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", CONTENT_TYPE).field(METRICS_FIELD, new String[] { "min", "max", "value_count" }); + b.field("type", CONTENT_TYPE).field(METRICS_FIELD, new String[] { "sum", "value_count" }); } @Override @@ -91,7 +91,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { @Override protected Object getSampleValueForDocument() { - return Map.of("min", -10.1, "max", 50.0, "value_count", 14); + return Map.of("value_count", 14, "sum", 100.0); } @Override @@ -119,14 +119,12 @@ protected String[] getParseMaximalWarnings() { */ public void testParseValue() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument doc = mapper.parse( - source(b -> b.startObject("field").field("min", -10.1).field("max", 50.0).field("value_count", 14).endObject()) - ); + ParsedDocument doc = mapper.parse(source(b -> b.startObject("field").field("value_count", 14).field("sum", 100.0).endObject())); - IndexableField f = doc.rootDoc().getField("field.min"); + IndexableField f = doc.rootDoc().getField("field.sum"); assertThat(f, instanceOf(SortedNumericDocValuesField.class)); SortedNumericDocValuesField sdv = (SortedNumericDocValuesField) f; - assertEquals(NumericUtils.doubleToSortableLong(-10.1), sdv.numericValue()); + assertEquals(NumericUtils.doubleToSortableLong(100.0), sdv.numericValue()); assertThat(sdv.fieldType().docValuesSkipIndexType(), equalTo(DocValuesSkipIndexType.NONE)); Mapper fieldMapper = mapper.mappers().getMapper("field"); @@ -159,10 +157,7 @@ public void testParseEmptyValue() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.startObject("field").endObject()))); - assertThat( - e.getCause().getMessage(), - containsString("Aggregate metric field [field] must contain all metrics [min, max, value_count]") - ); + assertThat(e.getCause().getMessage(), containsString("Aggregate metric field [field] must contain all metrics [sum, value_count]")); } @Override @@ -172,8 +167,7 @@ protected boolean supportsIgnoreMalformed() { @Override protected List exampleMalformedValues() { - var min = randomDoubleBetween(-100, 100, false); - var max = randomDoubleBetween(min, 150, false); + var sum = randomDoubleBetween(-100, 100, false); var valueCount = randomIntBetween(1, Integer.MAX_VALUE); var randomString = randomAlphaOfLengthBetween(1, 10); @@ -189,31 +183,26 @@ protected List exampleMalformedValues() { exampleMalformedValue(b -> b.value(randomBoolean)).errorMatches("Failed to parse object"), // no metrics exampleMalformedValue(b -> b.startObject().endObject()).errorMatches( - "Aggregate metric field [field] must contain all metrics [min, max, value_count]" + "Aggregate metric field [field] must contain all metrics [sum, value_count]" ), // unmapped metric exampleMalformedValue( - b -> b.startObject() - .field("min", min) - .field("max", max) - .field("value_count", valueCount) - .field("sum", randomLong) - .endObject() - ).errorMatches("Aggregate metric [sum] does not exist in the mapping of field [field]"), + b -> b.startObject().field("min", randomLong).field("value_count", valueCount).field("sum", sum).endObject() + ).errorMatches("Aggregate metric [min] does not exist in the mapping of field [field]"), // missing metric - exampleMalformedValue(b -> b.startObject().field("min", min).field("max", max).endObject()).errorMatches( - "Aggregate metric field [field] must contain all metrics [min, max, value_count]" + exampleMalformedValue(b -> b.startObject().field("sum", sum).endObject()).errorMatches( + "Aggregate metric field [field] must contain all metrics [sum, value_count]" ), // invalid metric value - exampleMalformedValue(b -> b.startObject().field("min", "10.0").field("max", max).field("value_count", valueCount).endObject()) - .errorMatches("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [VALUE_STRING]"), + exampleMalformedValue(b -> b.startObject().field("sum", "100.0").field("value_count", valueCount).endObject()).errorMatches( + "Failed to parse object: expecting token of type [VALUE_NUMBER] but found [VALUE_STRING]" + ), // Invalid metric value with additional data. // `min` field triggers the error and all additional data should be preserved in synthetic source. exampleMalformedValue( b -> b.startObject() - .field("max", max) .field("value_count", valueCount) - .field("min", "10.0") + .field("sum", "100.0") .field("hello", randomString) .startObject("object") .field("hello", randomLong) @@ -223,25 +212,19 @@ protected List exampleMalformedValues() { ).errorMatches("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [VALUE_STRING]"), // metric is an object exampleMalformedValue( - b -> b.startObject() - .startObject("min") - .field("hello", "world") - .endObject() - .field("max", max) - .field("value_count", valueCount) - .endObject() + b -> b.startObject().startObject("sum").field("hello", "world").endObject().field("value_count", valueCount).endObject() ).errorMatches("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [START_OBJECT]"), // metric is an array - exampleMalformedValue( - b -> b.startObject().array("min", "hello", "world").field("max", max).field("value_count", valueCount).endObject() - ).errorMatches("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [START_ARRAY]"), + exampleMalformedValue(b -> b.startObject().array("sum", "hello", "world").field("value_count", valueCount).endObject()) + .errorMatches("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [START_ARRAY]"), // negative value count - exampleMalformedValue( - b -> b.startObject().field("min", min).field("max", max).field("value_count", -1 * valueCount).endObject() - ).errorMatches("Aggregate metric [value_count] of field [field] cannot be a negative number"), + exampleMalformedValue(b -> b.startObject().field("sum", sum).field("value_count", -1 * valueCount).endObject()).errorMatches( + "Aggregate metric [value_count] of field [field] cannot be a negative number" + ), // value count with decimal digits (whole numbers formatted as doubles are permitted, but non-whole numbers are not) - exampleMalformedValue(b -> b.startObject().field("min", min).field("max", max).field("value_count", 77.33).endObject()) - .errorMatches("failed to parse [value_count] sub field: 77.33 cannot be converted to Integer without data loss") + exampleMalformedValue(b -> b.startObject().field("sum", sum).field("value_count", 77.33).endObject()).errorMatches( + "failed to parse [value_count] sub field: 77.33 cannot be converted to Integer without data loss" + ) ); } @@ -263,9 +246,7 @@ public void testUnsupportedMetric() throws Exception { */ public void testValueCountDouble() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - ParsedDocument doc = mapper.parse( - source(b -> b.startObject("field").field("min", 10.0).field("max", 50.0).field("value_count", 77.0).endObject()) - ); + ParsedDocument doc = mapper.parse(source(b -> b.startObject("field").field("sum", 1250.0).field("value_count", 77.0).endObject())); assertEquals(77, doc.rootDoc().getField("field.value_count").numericValue().longValue()); } @@ -273,25 +254,25 @@ public void testValueCountDouble() throws Exception { * Test parsing a metric and check the min max value */ public void testCheckMinMaxValue() throws Exception { - DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + DocumentMapper mapper = createDocumentMapper( + fieldMapping(b -> b.field("type", CONTENT_TYPE).field(METRICS_FIELD, new String[] { "min", "max" })) + ); // min > max Exception e = expectThrows( DocumentParsingException.class, - () -> mapper.parse( - source(b -> b.startObject("field").field("min", 50.0).field("max", 10.0).field("value_count", 14).endObject()) - ) + () -> mapper.parse(source(b -> b.startObject("field").field("min", 50.0).field("max", 10.0).endObject())) ); assertThat(e.getCause().getMessage(), containsString("Aggregate metric field [field] max value cannot be smaller than min value")); // min == max - mapper.parse(source(b -> b.startObject("field").field("min", 50.0).field("max", 50.0).field("value_count", 14).endObject())); + mapper.parse(source(b -> b.startObject("field").field("min", 50.0).field("max", 50.0).endObject())); // min < max - mapper.parse(source(b -> b.startObject("field").field("min", 10.0).field("max", 50.0).field("value_count", 14).endObject())); + mapper.parse(source(b -> b.startObject("field").field("min", 10.0).field("max", 50.0).endObject())); } - private void randomMapping(XContentBuilder b, int randomNumber) throws IOException { + private void randomSingleMetricMapping(XContentBuilder b, int randomNumber) throws IOException { b.field("type", CONTENT_TYPE); switch (randomNumber) { case 0 -> b.field(METRICS_FIELD, new String[] { "min" }); @@ -306,7 +287,7 @@ private void randomMapping(XContentBuilder b, int randomNumber) throws IOExcepti */ public void testParseArrayValue() throws Exception { int randomNumber = randomIntBetween(0, 3); - DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> randomMapping(b, randomNumber))); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> randomSingleMetricMapping(b, randomNumber))); Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> { b.startArray("field").startObject(); switch (randomNumber) { @@ -571,7 +552,7 @@ protected List getSortShortcutSupport() { this::minimalMapping, b -> {}, this::writeField, - Assert::assertNotNull + Assert::assertNull ) ); } @@ -588,13 +569,24 @@ public void testIndexTypes() throws Exception { .build(); MapperService mapperService = createMapperService(indexSettings, fieldMapping(this::minimalMapping)); MappedFieldType ft = mapperService.fieldType("field"); - assertIndexType(ft, IndexType.skippers()); + assertIndexType(ft, IndexType.docValuesOnly(), IndexType.skippers()); + } + { + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field") + .build(); + MapperService mapperService = createMapperService( + indexSettings, + fieldMapping(b -> randomSingleMetricMapping(b, randomIntBetween(0, 3))) + ); + MappedFieldType ft = mapperService.fieldType("field"); + assertIndexType(ft, IndexType.skippers(), IndexType.skippers()); } { var oldVersion = IndexVersionUtils.randomPreviousCompatibleVersion(IndexVersions.AGG_METRIC_DOUBLE_REMOVE_POINTS); MapperService mapperService = createMapperService(oldVersion, fieldMapping(this::minimalMapping)); MappedFieldType ft = mapperService.fieldType("field"); - assertIndexType(ft, IndexType.points(true, true)); + assertIndexType(ft, IndexType.docValuesOnly(), IndexType.points(true, true)); } { MapperService mapperService = createMapperService( @@ -602,17 +594,17 @@ public void testIndexTypes() throws Exception { fieldMapping(this::minimalMapping) ); MappedFieldType ft = mapperService.fieldType("field"); - assertIndexType(ft, IndexType.docValuesOnly()); + assertIndexType(ft, IndexType.docValuesOnly(), IndexType.docValuesOnly()); } } - private void assertIndexType(MappedFieldType ft, IndexType indexType) { + private void assertIndexType(MappedFieldType ft, IndexType indexType, IndexType metricIndexType) { assertThat(ft, instanceOf(AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType.class)); AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aft = (AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType) ft; assertThat(aft.indexType(), equalTo(indexType)); for (MappedFieldType subField : aft.getMetricFields().values()) { - assertThat(subField.indexType(), equalTo(indexType)); + assertThat(subField.indexType(), equalTo(metricIndexType)); } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldTypeTests.java index 4e063bb546b2d..00a53c1e3dffa 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldTypeTests.java @@ -28,6 +28,9 @@ import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.runtime.DoubleScriptFieldRangeQuery; +import org.elasticsearch.search.runtime.DoubleScriptFieldTermQuery; +import org.elasticsearch.search.runtime.DoubleScriptFieldTermsQuery; import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; @@ -48,8 +51,12 @@ public class AggregateMetricDoubleFieldTypeTests extends FieldTypeTestCase { protected AggregateMetricDoubleFieldType createDefaultFieldType(String name, Map meta) { + return createFieldType(name, meta, Metric.values()); + } + + protected AggregateMetricDoubleFieldType createFieldType(String name, Map meta, Metric... metrics) { EnumMap metricFields = new EnumMap<>(Metric.class); - for (Metric m : List.of(Metric.min, Metric.max)) { + for (Metric m : metrics) { String subfieldName = subfieldName(name, m); NumberFieldMapper.NumberFieldType subfield = new NumberFieldMapper.NumberFieldType( subfieldName, @@ -61,21 +68,69 @@ protected AggregateMetricDoubleFieldType createDefaultFieldType(String name, Map } public void testTermQuery() { - final MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap()); - Query query = fieldType.termQuery(55.2, MOCK_CONTEXT); - assertThat(query, equalTo(DoubleField.newRangeQuery("foo.max", 55.2, 55.2))); + { + final MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap()); + Query query = fieldType.termQuery(55.2, MOCK_CONTEXT); + assertThat(query, instanceOf(DoubleScriptFieldTermQuery.class)); + assertThat(query.toString(), equalTo("foo:55.2")); + } + { + // Single metric + Metric singleMetric = randomFrom(Metric.values()); + final MappedFieldType fieldType = createFieldType("foo", Collections.emptyMap(), singleMetric); + Query query = fieldType.termQuery(55.2, MOCK_CONTEXT); + assertThat(query, equalTo(DoubleField.newRangeQuery("foo." + singleMetric.name(), 55.2, 55.2))); + } + { + // No default metric + final MappedFieldType fieldType = createFieldType("foo", Collections.emptyMap(), Metric.min, Metric.max); + Query query = fieldType.termQuery(55.2, MOCK_CONTEXT); + assertThat(query, equalTo(Queries.NO_DOCS_INSTANCE)); + } } public void testTermsQuery() { - final MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap()); - Query query = fieldType.termsQuery(asList(55.2, 500.3), MOCK_CONTEXT); - assertThat(query, equalTo(DoublePoint.newSetQuery("foo.max", 55.2, 500.3))); + { + final MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap()); + Query query = fieldType.termsQuery(asList(55.2, 500.3), MOCK_CONTEXT); + assertThat(query, instanceOf(DoubleScriptFieldTermsQuery.class)); + assertThat(query.toString(), equalTo("foo:[55.2, 500.3]")); + } + { + // Single metric + Metric singleMetric = randomFrom(Metric.values()); + final MappedFieldType fieldType = createFieldType("foo", Collections.emptyMap(), singleMetric); + Query query = fieldType.termsQuery(asList(55.2, 500.3), MOCK_CONTEXT); + assertThat(query, equalTo(DoublePoint.newSetQuery("foo." + singleMetric.name(), 55.2, 500.3))); + } + { + // No default metric + final MappedFieldType fieldType = createFieldType("foo", Collections.emptyMap(), Metric.min, Metric.max); + Query query = fieldType.termsQuery(asList(55.2, 500.3), MOCK_CONTEXT); + assertThat(query, equalTo(Queries.NO_DOCS_INSTANCE)); + } } public void testRangeQuery() { - final MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap()); - Query query = fieldType.rangeQuery(10.1, 100.1, true, true, null, null, null, MOCK_CONTEXT); - assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); + { + final MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap()); + Query query = fieldType.rangeQuery(10.1, 100.1, true, true, null, null, null, MOCK_CONTEXT); + assertThat(query, instanceOf(DoubleScriptFieldRangeQuery.class)); + assertThat(query.toString(), equalTo("foo:[10.1 TO 100.1]")); + } + { + // Single metric + Metric singleMetric = randomFrom(Metric.values()); + final MappedFieldType fieldType = createFieldType("foo", Collections.emptyMap(), singleMetric); + Query query = fieldType.rangeQuery(10.1, 100.1, true, true, null, null, null, MOCK_CONTEXT); + assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); + } + { + // No default metric + final MappedFieldType fieldType = createFieldType("foo", Collections.emptyMap(), Metric.min, Metric.max); + Query query = fieldType.rangeQuery(10.1, 100.1, true, true, null, null, null, MOCK_CONTEXT); + assertThat(query, equalTo(Queries.NO_DOCS_INSTANCE)); + } } public void testFetchSourceValueWithOneMetric() throws IOException { @@ -97,24 +152,9 @@ public void testFetchSourceValueWithMultipleMetrics() throws IOException { public void testUsedInScript() throws IOException { final MappedFieldType mappedFieldType = createDefaultFieldType("field", Collections.emptyMap()); try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { - iw.addDocument( - List.of( - new NumericDocValuesField(subfieldName("field", Metric.max), Double.doubleToLongBits(10)), - new NumericDocValuesField(subfieldName("field", Metric.min), Double.doubleToLongBits(2)) - ) - ); - iw.addDocument( - List.of( - new NumericDocValuesField(subfieldName("field", Metric.max), Double.doubleToLongBits(4)), - new NumericDocValuesField(subfieldName("field", Metric.min), Double.doubleToLongBits(1)) - ) - ); - iw.addDocument( - List.of( - new NumericDocValuesField(subfieldName("field", Metric.max), Double.doubleToLongBits(7)), - new NumericDocValuesField(subfieldName("field", Metric.min), Double.doubleToLongBits(4)) - ) - ); + iw.addDocument(createDocument("field", Map.of(Metric.max, 10, Metric.min, 2, Metric.sum, 20, Metric.value_count, 2))); + iw.addDocument(createDocument("field", Map.of(Metric.max, 4, Metric.min, 1, Metric.sum, 20, Metric.value_count, 5))); + iw.addDocument(createDocument("field", Map.of(Metric.max, 7, Metric.min, 4, Metric.sum, 21, Metric.value_count, 3))); try (DirectoryReader reader = iw.getReader()) { SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.getFieldType(anyString())).thenReturn(mappedFieldType); @@ -158,4 +198,68 @@ public double execute(ExplanationHolder explanation) { } } + /** Tests that aggregate_metric_double uses a single subfield's doc-values as values in scripts */ + public void testSingleFieldUsedInScript() throws IOException { + Metric singleMetric = randomFrom(Metric.values()); + final MappedFieldType mappedFieldType = createFieldType("field", Collections.emptyMap(), singleMetric); + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(createDocument("field", Map.of(singleMetric, 10))); + iw.addDocument(createDocument("field", Map.of(singleMetric, 4))); + iw.addDocument(createDocument("field", Map.of(singleMetric, 7))); + try (DirectoryReader reader = iw.getReader()) { + SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getFieldType(anyString())).thenReturn(mappedFieldType); + when(searchExecutionContext.allowExpensiveQueries()).thenReturn(true); + SearchLookup lookup = new SearchLookup( + searchExecutionContext::getFieldType, + (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( + new FieldDataContext("test", null, lookupSupplier, searchExecutionContext::sourcePath, fdo) + ).build(null, null), + (ctx, doc) -> null + ); + when(searchExecutionContext.lookup()).thenReturn(lookup); + IndexSearcher searcher = newSearcher(reader); + assertThat( + searcher.count(new ScriptScoreQuery(Queries.ALL_DOCS_INSTANCE, new Script("test"), new ScoreScript.LeafFactory() { + @Override + public boolean needs_score() { + return false; + } + + @Override + public boolean needs_termStats() { + return false; + } + + @Override + public ScoreScript newInstance(DocReader docReader) { + return new ScoreScript(Map.of(), searchExecutionContext.lookup(), docReader) { + @Override + public double execute(ExplanationHolder explanation) { + Map> doc = getDoc(); + ScriptDocValues.Doubles doubles = (ScriptDocValues.Doubles) doc.get("field"); + return doubles.get(0); + } + }; + } + }, searchExecutionContext.lookup(), 7f, "test", 0, IndexVersion.current())), + equalTo(2) + ); + } + } + } + + private static List createDocument(String fieldName, Map values) { + return values.entrySet() + .stream() + .map( + entry -> new NumericDocValuesField( + subfieldName(fieldName, entry.getKey()), + entry.getKey() == Metric.value_count + ? entry.getValue().longValue() + : Double.doubleToLongBits(entry.getValue().doubleValue()) + ) + ) + .toList(); + } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/10_basic.yml index 5f4243652bf9d..a7f1ed30a3adb 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/10_basic.yml @@ -68,9 +68,10 @@ setup: properties: metric: type: aggregate_metric_double - metrics: [min, max] + metrics: [min, max, sum, value_count] default_metric: max + # we keep max == avg so that the test will keep working bwc - do: index: index: test @@ -79,6 +80,8 @@ setup: metric: min: 18.2 max: 100 + sum: 10000 + value_count: 100 refresh: true - do: @@ -89,6 +92,8 @@ setup: metric: min: 50 max: 1000 + sum: 20000 + value_count: 20 refresh: true - do: @@ -142,9 +147,9 @@ setup: properties: metric: type: aggregate_metric_double - metrics: [min, max] + metrics: [min, max, sum, value_count] default_metric: max - + # we keep max == avg so that the test will keep working bwc - do: index: index: test @@ -153,6 +158,8 @@ setup: metric: min: 18.2 max: 100 + sum: 10000 + value_count: 100 refresh: true - do: @@ -163,6 +170,8 @@ setup: metric: min: 50 max: 1000 + sum: 20000 + value_count: 20 refresh: true - do: @@ -217,7 +226,7 @@ setup: properties: metric: type: aggregate_metric_double - metrics: [min, max] + metrics: [min, max, sum, value_count] default_metric: max - do: @@ -228,6 +237,8 @@ setup: metric: min: 18.2 max: 3000 + sum: 4000 + value_count: 5 refresh: true - do: @@ -238,6 +249,8 @@ setup: metric: min: 50 max: 1000 + sum: 4000 + value_count: 25 refresh: true - do: @@ -248,6 +261,8 @@ setup: metric: min: 150 max: 5000 + sum: 6000 + value_count: 5 refresh: true - do: @@ -286,6 +301,20 @@ setup: body: sort: [ { metric.min: desc } ] + - do: + catch: bad_request + search: + index: test + body: + sort: [ { metric.sum: desc } ] + + - do: + catch: bad_request + search: + index: test + body: + sort: [ { metric.value_count: desc } ] + --- "Test fields api": - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/20_min_max_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/20_min_max_agg.yml index 57e1314302e07..e6c4040aef8ec 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/20_min_max_agg.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/20_min_max_agg.yml @@ -15,7 +15,7 @@ setup: properties: metric: type: aggregate_metric_double - metrics: [min, max] + metrics: [min, max, sum, value_count] default_metric: max - do: @@ -23,16 +23,17 @@ setup: index: aggregate_metric_test refresh: true body: + - # we keep max == avg so that the test will keep working bwc - '{"index": {}}' - - '{"metric": {"min": 0, "max": 100} }' + - '{"metric": {"min": 0, "max": 100, "sum": 1000, "value_count": 10} }' - '{"index": {}}' - - '{"metric": {"min": 60, "max": 100} }' + - '{"metric": {"min": 60, "max": 100, "sum": 1000, "value_count": 10} }' - '{"index": {}}' - - '{"metric": {"min": -400.50, "max": 1000} }' + - '{"metric": {"min": -400.50, "max": 1000, "sum": 20000, "value_count": 20} }' - '{"index": {}}' - - '{"metric": {"min": 1, "max": 99.3} }' + - '{"metric": {"min": 1, "max": 99.3, "sum": 279.9, "value_count": 3} }' - '{"index": {}}' - - '{"metric": {"min": -100, "max": -40} }' + - '{"metric": {"min": -100, "max": -40, "sum": -80, "value_count": 2} }' --- "Test min_max aggs": - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/30_sum_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/30_sum_agg.yml index 577ff1beae0ed..4c898f304efc8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/30_sum_agg.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/30_sum_agg.yml @@ -1,6 +1,6 @@ setup: - requires: - test_runner_features: [ allowed_warnings ] + test_runner_features: [ allowed_warnings, capabilities ] - do: allowed_warnings: - "Parameter [default_metric] is deprecated and will be removed in a future version" @@ -11,7 +11,7 @@ setup: properties: metric: type: aggregate_metric_double - metrics: [sum, value_count, max] # temporarily add max so it can use it for filtering + metrics: [sum, value_count] default_metric: value_count - do: @@ -20,15 +20,15 @@ setup: refresh: true body: - '{"index": {}}' - - '{"metric": {"sum": 10000.45, "value_count": 100, "max": 100} }' + - '{"metric": {"sum": 10000.45, "value_count": 100} }' - '{"index": {}}' - - '{"metric": {"sum": 60, "value_count": 20, "max": 20} }' + - '{"metric": {"sum": 60, "value_count": 20} }' - '{"index": {}}' - - '{"metric": {"sum": -400.50, "value_count": 1000, "max": 1000} }' + - '{"metric": {"sum": -400.50, "value_count": 1000} }' - '{"index": {}}' - - '{"metric": {"sum": 1, "value_count": 20, "max": 20} }' + - '{"metric": {"sum": 1, "value_count": 20} }' - '{"index": {}}' - - '{"metric": {"sum": -100, "value_count": 40, "max": 40} }' + - '{"metric": {"sum": -100, "value_count": 40} }' --- "Test sum agg": - do: @@ -46,19 +46,26 @@ setup: --- "Test sum agg with query": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ aggregate_metric_double_defaults_to_average ] + reason: "Using average as default for aggregate metric double was added in 9.14" - do: search: index: aggregate_metric_test size: 0 body: query: - term: + range: metric: - value: 20 + gte: 2 + lte: 200 aggs: sum_agg: sum: field: metric - match: { hits.total.value: 2 } - - match: { aggregations.sum_agg.value: 61} + - match: { aggregations.sum_agg.value: 10060.45} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/40_avg_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/40_avg_agg.yml index c3c8cdd9c83c1..d8996fe40a10c 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/40_avg_agg.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/40_avg_agg.yml @@ -1,6 +1,6 @@ setup: - requires: - test_runner_features: [ allowed_warnings ] + test_runner_features: [ allowed_warnings, capabilities ] - do: allowed_warnings: - "Parameter [default_metric] is deprecated and will be removed in a future version" @@ -11,7 +11,7 @@ setup: properties: metric: type: aggregate_metric_double - metrics: [sum, value_count, max] + metrics: [sum, value_count] default_metric: value_count - do: @@ -20,15 +20,15 @@ setup: refresh: true body: - '{"index": {}}' - - '{"metric": {"sum": 10000, "value_count": 100, "max": 100} }' + - '{"metric": {"sum": 10000, "value_count": 100} }' - '{"index": {}}' - - '{"metric": {"sum": 60, "value_count": 20, "max": 20} }' + - '{"metric": {"sum": 60, "value_count": 20} }' - '{"index": {}}' - - '{"metric": {"sum": -400, "value_count": 780, "max": 780} }' + - '{"metric": {"sum": -400, "value_count": 780} }' - '{"index": {}}' - - '{"metric": {"sum": 40, "value_count": 20, "max": 20} }' + - '{"metric": {"sum": 40, "value_count": 20} }' - '{"index": {}}' - - '{"metric": {"sum": -100, "value_count": 40, "max": 40} }' + - '{"metric": {"sum": -100, "value_count": 40} }' --- "Test avg agg": - do: @@ -46,15 +46,22 @@ setup: --- "Test avg agg with query": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ aggregate_metric_double_defaults_to_average ] + reason: "Using average as default for aggregate metric double was added in 9.14" - do: search: index: aggregate_metric_test size: 0 body: query: - term: + range: metric: - value: 20 + gte: 2 + lte: 3 aggs: avg_agg: avg: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/50_value_count_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/50_value_count_agg.yml index 46629c78efdea..3c9fcab16b47b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/50_value_count_agg.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/50_value_count_agg.yml @@ -1,6 +1,6 @@ setup: - requires: - test_runner_features: [ allowed_warnings ] + test_runner_features: [ allowed_warnings, capabilities ] - do: allowed_warnings: - "Parameter [default_metric] is deprecated and will be removed in a future version" @@ -11,7 +11,7 @@ setup: properties: metric: type: aggregate_metric_double - metrics: [sum, value_count, max] + metrics: [sum, value_count] default_metric: value_count - do: @@ -20,15 +20,15 @@ setup: refresh: true body: - '{"index": {}}' - - '{"metric": {"sum": 10000, "value_count": 100, "max": 100} }' + - '{"metric": {"sum": 10000, "value_count": 100} }' - '{"index": {}}' - - '{"metric": {"sum": 60, "value_count": 20, "max": 20} }' + - '{"metric": {"sum": 60, "value_count": 20} }' - '{"index": {}}' - - '{"metric": {"sum": -400, "value_count": 780, "max": 780} }' + - '{"metric": {"sum": -400, "value_count": 780} }' - '{"index": {}}' - - '{"metric": {"sum": 40, "value_count": 20, "max": 20} }' + - '{"metric": {"sum": 40, "value_count": 20} }' - '{"index": {}}' - - '{"metric": {"sum": -100, "value_count": 40, "max": 40} }' + - '{"metric": {"sum": -100, "value_count": 40} }' --- "Test value_count agg": - do: @@ -46,15 +46,22 @@ setup: --- "Test value_count agg with query": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ aggregate_metric_double_defaults_to_average ] + reason: "Using average as default for aggregate metric double was added in 9.14" - do: search: index: aggregate_metric_test size: 0 body: query: - term: + range: metric: - value: 20 + gte: 2 + lte: 3 aggs: value_count_agg: value_count: