From 73614f35b84ab90698b7b0e5b47c20e23e7192c6 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 7 Jan 2026 15:30:19 +0200 Subject: [PATCH 01/11] Merge value fetchers and producers functionality a downsampler class --- .../downsample/AbstractFieldDownsampler.java | 154 ++++++++++ ...AggregateMetricDoubleFieldDownsampler.java | 269 ++++++++++++++++++ .../downsample/DimensionFieldDownsampler.java | 113 ++++++++ .../ExponentialHistogramFieldDownsampler.java | 155 ++++++++++ .../downsample/LastValueFieldDownsampler.java | 153 ++++++++++ .../NumericMetricFieldDownsampler.java | 151 ++++++++++ .../TDigestHistogramFieldDownsampler.java | 161 +++++++++++ 7 files changed, 1156 insertions(+) create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldDownsampler.java create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java new file mode 100644 index 0000000000000..c35b5a46cac98 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.TimeSeriesParams; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.mapper.TimeSeriesParams.MetricType.POSITION; + +/** + * Base class that reads fields from the source index and produces their downsampled values + */ +abstract class AbstractFieldDownsampler implements DownsampleFieldSerializer { + + private final String name; + protected boolean isEmpty; + private final FieldValueFetcher fieldValueFetcher; + + AbstractFieldDownsampler(String name, FieldValueFetcher fieldValueFetcher) { + this.name = name; + this.isEmpty = true; + this.fieldValueFetcher = fieldValueFetcher; + } + + /** + * @return the name of the field. + */ + public String name() { + return name; + } + + /** + * Resets the producer to an empty value. + */ + public abstract void reset(); + + /** + * @return true if the field has not collected any value. + */ + public boolean isEmpty() { + return isEmpty; + } + + /** + * @return the leaf reader that will retrieve the values per doc for this field. + */ + public T getLeaf(LeafReaderContext context) throws IOException { + return fieldValueFetcher.getLeaf(context); + } + + public abstract void collect(T docValues, IntArrayList docIdBuffer) throws IOException; + + abstract static class FieldValueFetcher { + protected final String name; + protected final MappedFieldType fieldType; + protected final IndexFieldData fieldData; + + FieldValueFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + this.name = name; + this.fieldType = fieldType; + this.fieldData = fieldData; + } + + String name() { + return name; + } + + abstract T getLeaf(LeafReaderContext context) throws IOException; + } + + /** + * Retrieve field value fetchers for a list of fields. + */ + static List> create( + SearchExecutionContext context, + String[] fields, + Map multiFieldSources, + DownsampleConfig.SamplingMethod samplingMethod + ) { + List> fetchers = new ArrayList<>(); + for (String field : fields) { + String sourceField = multiFieldSources.getOrDefault(field, field); + MappedFieldType fieldType = context.getFieldType(sourceField); + assert fieldType != null : "Unknown field type for field: [" + sourceField + "]"; + + if (fieldType instanceof AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aggMetricFieldType) { + fetchers.addAll(AggregateMetricDoubleFieldDownsampler.create(context, aggMetricFieldType, samplingMethod)); + } else { + if (context.fieldExistsInIndex(field)) { + final IndexFieldData fieldData; + if (fieldType instanceof FlattenedFieldMapper.RootFlattenedFieldType flattenedFieldType) { + var keyedFieldType = flattenedFieldType.getKeyedFieldType(); + fieldData = context.getForField(keyedFieldType, MappedFieldType.FielddataOperation.SEARCH); + } else { + fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); + } + fetchers.add(create(field, fieldType, fieldData, samplingMethod)); + } + } + } + return Collections.unmodifiableList(fetchers); + } + + private static AbstractFieldDownsampler create( + String fieldName, + MappedFieldType fieldType, + IndexFieldData fieldData, + DownsampleConfig.SamplingMethod samplingMethod + ) { + assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false + : "Aggregate metric double should be handled by a dedicated FieldValueFetcher"; + if (TDigestHistogramFieldProducer.TYPE.equals(fieldType.typeName())) { + return TDigestHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); + } + if (ExponentialHistogramFieldProducer.TYPE.equals(fieldType.typeName())) { + return ExponentialHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); + } + if (fieldType.getMetricType() != null) { + // TODO: Support POSITION in downsampling + if (fieldType.getMetricType() == POSITION) { + throw new IllegalArgumentException("Unsupported metric type [position] for down-sampling"); + } + assert fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE + || fieldType.getMetricType() == TimeSeriesParams.MetricType.COUNTER + : "only gauges and counters accepted, other metrics should have been handled earlier"; + if (samplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE + && fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE) { + return new NumericMetricFieldDownsampler.AggregateGauge(fieldName, fieldType, fieldData); + } + return new NumericMetricFieldDownsampler.LastValue(fieldName, fieldType, fieldData); + } else { + // If a field is not a metric, we downsample it as a label + return LastValueFieldDownsampler.create(fieldName, fieldType, fieldData); + } + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java new file mode 100644 index 0000000000000..0bb485aca9fa0 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java @@ -0,0 +1,269 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducer.MAX_NO_VALUE; +import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducer.MIN_NO_VALUE; + +/** + * A producer that can be used for downsampling aggregate metric double fields whether is a metric or a label. We need a separate producer + * for each a sub-metric of an aggregate metric double, this means to downsample an aggregate metric double we will need 4. + * This is mainly used when downsampling already downsampled indices. + */ +abstract sealed class AggregateMetricDoubleFieldDownsampler extends NumericMetricFieldDownsampler { + + protected final AggregateMetricDoubleFieldMapper.Metric metric; + + AggregateMetricDoubleFieldDownsampler( + String name, + AggregateMetricDoubleFieldMapper.Metric metric, + MappedFieldType fieldType, + IndexFieldData fieldData + ) { + super(name, fieldType, fieldData); + this.metric = metric; + } + + static final class Aggregate extends AggregateMetricDoubleFieldDownsampler { + + private double max = MAX_NO_VALUE; + private double min = MIN_NO_VALUE; + private final CompensatedSum sum = new CompensatedSum(); + private long count; + + Aggregate(String name, AggregateMetricDoubleFieldMapper.Metric metric, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, metric, fieldType, fieldData); + } + + @Override + public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + int docValuesCount = docValues.docValueCount(); + for (int j = 0; j < docValuesCount; j++) { + double value = docValues.nextValue(); + switch (metric) { + case min -> min = Math.min(value, min); + case max -> max = Math.max(value, max); + case sum -> sum.add(value); + // This is the reason why we can't use GaugeMetricFieldProducer + // For downsampled indices aggregate metric double's value count field needs to be summed. + // (Note: not using CompensatedSum here should be ok given that value_count is mapped as long) + case value_count -> count += Math.round(value); + } + } + } + } + + @Override + public void reset() { + isEmpty = true; + max = MAX_NO_VALUE; + min = MIN_NO_VALUE; + sum.reset(0, 0); + count = 0; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + switch (metric) { + case min -> builder.field("min", min); + case max -> builder.field("max", max); + case sum -> builder.field("sum", sum.value()); + case value_count -> builder.field("value_count", count); + } + } + } + } + + /** + * Important note: This class assumes that field values are collected and sorted by descending order by time + */ + static final class LastValue extends AggregateMetricDoubleFieldDownsampler { + + private final boolean supportsMultiValue; + private Object lastValue = null; + + LastValue( + String name, + AggregateMetricDoubleFieldMapper.Metric metric, + MappedFieldType fieldType, + IndexFieldData fieldData, + boolean supportsMultiValue + ) { + super(name, metric, fieldType, fieldData); + this.supportsMultiValue = supportsMultiValue; + } + + @Override + public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + return; + } + + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + int docValuesCount = docValues.docValueCount(); + assert docValuesCount > 0; + isEmpty = false; + if (docValuesCount == 1 || supportsMultiValue == false) { + lastValue = docValues.nextValue(); + } else { + var values = new Object[docValuesCount]; + for (int j = 0; j < docValuesCount; j++) { + values[j] = docValues.nextValue(); + } + lastValue = values; + } + return; + } + } + + @Override + public void reset() { + isEmpty = true; + lastValue = null; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(metric.name(), lastValue); + } + } + } + + /** + * We use a specialised serializer because we are combining all the available submetric producers. + */ + static class Serializer implements DownsampleFieldSerializer { + private final Collection> downsamplers; + private final String name; + + /** + * @param name the name of the aggregate_metric_double field as it will be serialized + * in the downsampled index + * @param downsamplers a collection of {@link AggregateMetricDoubleFieldDownsampler} instances with the subfields + * of the aggregate_metric_double field. + */ + Serializer(String name, Collection> downsamplers) { + this.name = name; + this.downsamplers = downsamplers; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty()) { + return; + } + + builder.startObject(name); + for (AbstractFieldDownsampler fieldDownsampler : downsamplers) { + assert name.equals(fieldDownsampler.name()) : "producer has a different name"; + if (fieldDownsampler.isEmpty()) { + continue; + } + if (fieldDownsampler instanceof AggregateMetricDoubleFieldDownsampler == false) { + throw new IllegalStateException( + "Unexpected field producer class: " + fieldDownsampler.getClass().getSimpleName() + " for " + name + " field" + ); + } + fieldDownsampler.write(builder); + } + builder.endObject(); + } + + private boolean isEmpty() { + for (AbstractFieldDownsampler d : downsamplers) { + if (d.isEmpty() == false) { + return false; + } + } + return true; + } + } + + /** + * For aggregate_metric_double fields we create separate fetchers for each sub-metric. This is usually a downsample-of-downsample case. + */ + static List create( + SearchExecutionContext context, + AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aggMetricFieldType, + DownsampleConfig.SamplingMethod samplingMethod + ) { + List downsamplers = new ArrayList<>(); + // If the field is an aggregate_metric_double field, we should load all its subfields + // This is usually a downsample-of-downsample case + for (var metricField : aggMetricFieldType.getMetricFields().entrySet()) { + var metric = metricField.getKey(); + var metricSubField = metricField.getValue(); + if (context.fieldExistsInIndex(metricSubField.name())) { + IndexFieldData fieldData = context.getForField(metricSubField, MappedFieldType.FielddataOperation.SEARCH); + if (aggMetricFieldType.getMetricType() != null) { + if (samplingMethod != DownsampleConfig.SamplingMethod.LAST_VALUE) { + // If the field is an aggregate_metric_double field, we should use the correct subfields + // for each aggregation. This is a downsample-of-downsample case + downsamplers.add( + new AggregateMetricDoubleFieldDownsampler.Aggregate( + aggMetricFieldType.name(), + metric, + metricSubField, + fieldData + ) + ); + } else { + downsamplers.add( + new AggregateMetricDoubleFieldDownsampler.LastValue( + aggMetricFieldType.name(), + metric, + metricSubField, + fieldData, + false + ) + ); + } + } else { + // If a field is not a metric, we downsample it as a label + downsamplers.add( + new AggregateMetricDoubleFieldDownsampler.LastValue( + aggMetricFieldType.name(), + metric, + metricSubField, + fieldData, + true + ) + ); + } + } + } + return downsamplers; + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldDownsampler.java new file mode 100644 index 0000000000000..b0e0240bfe169 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldDownsampler.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; +import org.elasticsearch.index.query.SearchExecutionContext; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * The dimension field producer is effectively a last value field producer that performs some extra validations when assertions are enabled. + * It checks: + * - that a tsid is only collected once, and + * - that all TSIDs that are being collected for a round have the same value. + * Important note: This class assumes that field values are collected and sorted by descending order by time + */ +public class DimensionFieldDownsampler extends LastValueFieldDownsampler { + + DimensionFieldDownsampler(final String name, final MappedFieldType fieldType, final IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + void collectOnce(final Object value) { + assert isEmpty; + Objects.requireNonNull(value); + this.lastValue = value; + this.isEmpty = false; + } + + /** + * This is an expensive check that slows down downsampling significantly. + * Given that index is sorted by tsid as a primary key, this shouldn't really happen. + */ + boolean validate(FormattedDocValues docValues, IntArrayList buffer) throws IOException { + for (int i = 0; i < buffer.size(); i++) { + int docId = buffer.get(i); + if (docValues.advanceExact(docId)) { + int docValueCount = docValues.docValueCount(); + for (int j = 0; j < docValueCount; j++) { + var value = docValues.nextValue(); + assert value.equals(this.lastValue) != false + : "Dimension value changed without tsid change [" + value + "] != [" + this.lastValue + "]"; + } + } + } + + return true; + } + + @Override + public void collect(FormattedDocValues docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + assert validate(docValues, docIdBuffer); + return; + } + + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + int docValueCount = docValues.docValueCount(); + for (int j = 0; j < docValueCount; j++) { + collectOnce(docValues.nextValue()); + } + // Only need to record one dimension value from one document, within in the same tsid-and-time-interval bucket values are the + // same. + return; + } + } + + /** + * Retrieve field value fetchers for a list of dimensions. + */ + static List create( + final SearchExecutionContext context, + final String[] dimensions, + final Map multiFieldSources + ) { + List downsamplers = new ArrayList<>(); + for (String dimension : dimensions) { + String sourceFieldName = multiFieldSources.getOrDefault(dimension, dimension); + MappedFieldType fieldType = context.getFieldType(sourceFieldName); + assert fieldType != null : "Unknown type for dimension field: [" + sourceFieldName + "]"; + + if (context.fieldExistsInIndex(fieldType.name())) { + final IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); + if (fieldType instanceof FlattenedFieldMapper.KeyedFlattenedFieldType flattenedFieldType) { + // Name of the field type and name of the dimension are different in this case. + var dimensionName = flattenedFieldType.rootName() + '.' + flattenedFieldType.key(); + downsamplers.add(new DimensionFieldDownsampler(dimensionName, fieldType, fieldData)); + } else { + downsamplers.add(new DimensionFieldDownsampler(dimension, fieldType, fieldData)); + } + } + } + return Collections.unmodifiableList(downsamplers); + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java new file mode 100644 index 0000000000000..62884f87db996 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.LeafExponentialHistogramFieldData; + +import java.io.IOException; + +/** + * A producer that can be used for downsampling ONLY an exponential histogram field whether it's a metric or a label. + */ +abstract class ExponentialHistogramFieldDownsampler extends AbstractFieldDownsampler { + static final String TYPE = "exponential_histogram"; + + ExponentialHistogramFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, new ExponentialHistogramFieldFetcher(name, fieldType, fieldData)); + } + + /** + * @return the requested producer based on the sampling method for an exponential histogram field + */ + static AbstractFieldDownsampler create( + String name, + MappedFieldType fieldType, + IndexFieldData fieldData, + DownsampleConfig.SamplingMethod samplingMethod + ) { + return switch (samplingMethod) { + case AGGREGATE -> new ExponentialHistogramFieldDownsampler.MergeProducer(name, fieldType, fieldData); + case LAST_VALUE -> new ExponentialHistogramFieldDownsampler.LastValueProducer(name, fieldType, fieldData); + }; + } + + protected abstract ExponentialHistogram downsampledValue(); + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(name()); + ExponentialHistogramXContent.serialize(builder, downsampledValue()); + } + } + + /** + * Downsamples an exponential histogram by merging all values. + */ + static class MergeProducer extends ExponentialHistogramFieldDownsampler { + private ExponentialHistogramMerger merger = null; + + MergeProducer(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + public void reset() { + isEmpty = true; + merger = null; + } + + @Override + protected ExponentialHistogram downsampledValue() { + if (isEmpty()) { + return null; + } + ExponentialHistogram exponentialHistogram = merger.get(); + merger.close(); + return exponentialHistogram; + } + + @Override + public void collect(ExponentialHistogramValuesReader docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + if (merger == null) { + merger = ExponentialHistogramMerger.create(ExponentialHistogramCircuitBreaker.noop()); + } + ExponentialHistogram value = docValues.histogramValue(); + merger.add(value); + } + } + } + + /** + * Downsamples an exponential histogram by preserving the last value. + * Important note: This class assumes that field values are collected and sorted by descending order by time + */ + static class LastValueProducer extends ExponentialHistogramFieldDownsampler { + private ExponentialHistogram lastValue = null; + + LastValueProducer(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + public void reset() { + isEmpty = true; + lastValue = null; + } + + @Override + public void collect(ExponentialHistogramValuesReader docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + return; + } + + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + lastValue = docValues.histogramValue(); + return; + } + } + + @Override + protected ExponentialHistogram downsampledValue() { + return lastValue; + } + } + + static class ExponentialHistogramFieldFetcher extends AbstractFieldDownsampler.FieldValueFetcher { + + ExponentialHistogramFieldFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + ExponentialHistogramValuesReader getLeaf(LeafReaderContext context) throws IOException { + LeafExponentialHistogramFieldData exponentialHistogramFieldData = (LeafExponentialHistogramFieldData) fieldData.load(context); + return exponentialHistogramFieldData.getHistogramValues(); + } + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java new file mode 100644 index 0000000000000..901b9680d1f94 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.internal.hppc.IntArrayList; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldSyntheticWriterHelper; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Class that downsamples a label field for downsampling by keeping the last value. + * Important note: This class assumes that field values are collected and sorted by descending order by time + */ +class LastValueFieldDownsampler extends AbstractFieldDownsampler { + Object lastValue = null; + + LastValueFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, new FormattedDocValueFetcher(name, fieldType, fieldData)); + } + + /** + * Creates a producer that can be used for downsampling labels. + */ + static LastValueFieldDownsampler create(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false + : "field type cannot be aggregate metric double: " + fieldType.typeName() + " for field " + name; + assert ExponentialHistogramFieldProducer.TYPE.equals(fieldType.typeName()) == false + : "field type cannot be exponential histogram: " + fieldType.typeName() + " for field " + name; + assert TDigestHistogramFieldProducer.TYPE.equals(fieldType.typeName()) == false + : "field type cannot be histogram: " + fieldType.typeName() + " for field " + name; + if ("flattened".equals(fieldType.typeName())) { + return new LastValueFieldDownsampler.FlattenedFieldProducer(name, fieldType, fieldData); + } + return new LastValueFieldDownsampler(name, fieldType, fieldData); + } + + @Override + public void reset() { + isEmpty = true; + lastValue = null; + } + + /** + * Collects the last value observed in these field values. This implementation assumes that field values are collected + * and sorted by descending order by time. In this case, it assumes that the last value of the time is the first value + * collected. Eventually, the implementation of this class ends up storing the first value it is empty and then + * ignoring everything else. + */ + @Override + public void collect(FormattedDocValues docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + return; + } + + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + int docValuesCount = docValues.docValueCount(); + assert docValuesCount > 0; + isEmpty = false; + if (docValuesCount == 1) { + lastValue = docValues.nextValue(); + } else { + var values = new Object[docValuesCount]; + for (int j = 0; j < docValuesCount; j++) { + values[j] = docValues.nextValue(); + } + lastValue = values; + } + // Only need to record one label value from one document, within in the same tsid-and-time-interval we only keep the first + // with downsampling. + return; + } + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(name(), lastValue); + } + } + + public Object lastValue() { + return lastValue; + } + + static final class FlattenedFieldProducer extends LastValueFieldDownsampler { + + private FlattenedFieldProducer(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.startObject(name()); + + var value = lastValue(); + List list; + if (value instanceof Object[] values) { + list = new ArrayList<>(values.length); + for (Object v : values) { + list.add(new BytesRef(v.toString())); + } + } else { + list = List.of(new BytesRef(value.toString())); + } + + var iterator = list.iterator(); + var helper = new FlattenedFieldSyntheticWriterHelper(() -> { + if (iterator.hasNext()) { + return iterator.next(); + } else { + return null; + } + }); + helper.write(builder); + builder.endObject(); + } + } + } + + static class FormattedDocValueFetcher extends AbstractFieldDownsampler.FieldValueFetcher { + + FormattedDocValueFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + FormattedDocValues getLeaf(LeafReaderContext context) { + DocValueFormat format = fieldType.docValueFormat(null, null); + return fieldData.load(context).getFormattedValues(format); + } + } + +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java new file mode 100644 index 0000000000000..3f416aa740a52 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Class that collects all raw values for a numeric metric field and computes its aggregate (downsampled) + * values. Based on the supported metric types, the subclasses of this class compute values for + * gauge and metric types. + */ +abstract sealed class NumericMetricFieldDownsampler extends AbstractFieldDownsampler permits AggregateMetricDoubleFieldDownsampler, NumericMetricFieldDownsampler.AggregateGauge, NumericMetricFieldDownsampler.LastValue { + + NumericMetricFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, new NumericFieldFetcher(name, fieldType, fieldData)); + } + + static final double MAX_NO_VALUE = -Double.MAX_VALUE; + static final double MIN_NO_VALUE = Double.MAX_VALUE; + + /** + * {@link NumericMetricFieldDownsampler} implementation for creating an aggregate gauge metric field + */ + static final class AggregateGauge extends NumericMetricFieldDownsampler { + + double max = MAX_NO_VALUE; + double min = MIN_NO_VALUE; + final CompensatedSum sum = new CompensatedSum(); + long count; + + AggregateGauge(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + int docValuesCount = docValues.docValueCount(); + for (int j = 0; j < docValuesCount; j++) { + double value = docValues.nextValue(); + this.max = Math.max(value, max); + this.min = Math.min(value, min); + sum.add(value); + count++; + } + } + } + + @Override + public void reset() { + isEmpty = true; + max = MAX_NO_VALUE; + min = MIN_NO_VALUE; + sum.reset(0, 0); + count = 0; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.startObject(name()); + builder.field("min", min); + builder.field("max", max); + builder.field("sum", sum.value()); + builder.field("value_count", count); + builder.endObject(); + } + } + } + + /** + * {@link NumericMetricFieldDownsampler} implementation for sampling the last value of a numeric metric field. + * Important note: This class assumes that field values are collected and sorted by descending order by time. + */ + static final class LastValue extends NumericMetricFieldDownsampler { + + double lastValue = Double.NaN; + + LastValue(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + return; + } + + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId)) { + isEmpty = false; + lastValue = docValues.nextValue(); + return; + } + } + } + + public Double lastValue() { + if (isEmpty()) { + return null; + } + return lastValue; + } + + @Override + public void reset() { + isEmpty = true; + lastValue = Double.NaN; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(name(), lastValue); + } + } + } + + static class NumericFieldFetcher extends AbstractFieldDownsampler.FieldValueFetcher { + + NumericFieldFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + SortedNumericDoubleValues getLeaf(LeafReaderContext context) { + LeafNumericFieldData numericFieldData = (LeafNumericFieldData) fieldData.load(context); + return numericFieldData.getDoubleValues(); + } + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java new file mode 100644 index 0000000000000..118ee949ae2be --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafHistogramFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.tdigest.Centroid; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Class that collects all raw values for an exponential histogram metric field and computes its aggregate (downsampled) + * values. + */ +abstract class TDigestHistogramFieldDownsampler extends AbstractFieldDownsampler { + + static final String TYPE = "histogram"; + public static final int COMPRESSION = 100; + + TDigestHistogramFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, new TDigestHistogramFieldFetcher(name, fieldType, fieldData)); + } + + /** + * @return the requested produces based on the sampling method for metric of type exponential histogram + */ + public static TDigestHistogramFieldDownsampler create( + String name, + MappedFieldType fieldType, + IndexFieldData fieldData, + DownsampleConfig.SamplingMethod samplingMethod + ) { + return switch (samplingMethod) { + case AGGREGATE -> new Aggregate(name, fieldType, fieldData); + case LAST_VALUE -> new LastValue(name, fieldType, fieldData); + }; + } + + private static class Aggregate extends TDigestHistogramFieldDownsampler { + + private TDigestState tDigestState = null; + + Aggregate(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + public void collect(HistogramValues docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + if (tDigestState == null) { + // TODO: figure out what circuit breaker to use here and in the other histogram + tDigestState = TDigestState.create(new NoopCircuitBreaker("downsampling-histograms"), COMPRESSION); + } + final HistogramValue sketch = docValues.histogram(); + while (sketch.next()) { + tDigestState.add(sketch.value(), sketch.count()); + } + } + } + + @Override + public void reset() { + isEmpty = true; + tDigestState = null; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + Iterator centroids = tDigestState.uniqueCentroids(); + final List values = new ArrayList<>(); + final List counts = new ArrayList<>(); + while (centroids.hasNext()) { + Centroid centroid = centroids.next(); + values.add(centroid.mean()); + counts.add(centroid.count()); + } + builder.startObject(name()).field("counts", counts).field("values", values).endObject(); + tDigestState.close(); + } + } + } + + private static class LastValue extends TDigestHistogramFieldDownsampler { + + private HistogramValue lastValue = null; + + LastValue(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + public void collect(HistogramValues docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + return; + } + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + lastValue = docValues.histogram(); + return; + } + } + + @Override + public void reset() { + isEmpty = true; + lastValue = null; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + final HistogramValue histogramValue = lastValue; + final List values = new ArrayList<>(); + final List counts = new ArrayList<>(); + while (histogramValue.next()) { + values.add(histogramValue.value()); + counts.add(histogramValue.count()); + } + builder.startObject(name()).field("counts", counts).field("values", values).endObject(); + } + } + } + + static class TDigestHistogramFieldFetcher extends AbstractFieldDownsampler.FieldValueFetcher { + + TDigestHistogramFieldFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { + super(name, fieldType, fieldData); + } + + @Override + HistogramValues getLeaf(LeafReaderContext context) throws IOException { + LeafHistogramFieldData histogramFieldData = (LeafHistogramFieldData) fieldData.load(context); + return histogramFieldData.getHistogramValues(); + } + } +} From b24ad231294797a88992115d9bbef8f3a8e9faa4 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 8 Jan 2026 13:03:29 +0200 Subject: [PATCH 02/11] Reduce object generation per segment. --- .../downsample/AbstractFieldDownsampler.java | 12 +- ...AggregateMetricDoubleFieldDownsampler.java | 14 +- .../AggregateMetricDoubleFieldProducer.java | 196 ------------------ ...ggregateMetricDoubleFieldValueFetcher.java | 91 -------- .../downsample/DimensionFieldProducer.java | 77 ------- .../DimensionFieldValueFetcher.java | 64 ------ .../downsample/DownsampleShardIndexer.java | 161 ++++++++------ .../xpack/downsample/FieldValueFetcher.java | 148 ------------- .../downsample/LastValueFieldDownsampler.java | 4 +- .../downsample/LastValueFieldProducer.java | 136 ------------ .../NumericMetricFieldDownsampler.java | 3 +- .../NumericMetricFieldProducer.java | 134 ------------ .../TDigestHistogramFieldProducer.java | 139 ------------- ...egateMetricDoubleFieldSerializerTests.java | 54 ++--- ...va => LastValueFieldDownsamplerTests.java} | 43 +++- ...> NumericMetricFieldDownsamplerTests.java} | 24 +-- ...DigestHistogramFieldDownsamplerTests.java} | 6 +- 17 files changed, 190 insertions(+), 1116 deletions(-) delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldProducer.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldValueFetcher.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducer.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducer.java rename x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/{LastValueFieldProducerTests.java => LastValueFieldDownsamplerTests.java} (78%) rename x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/{NumericMetricFieldProducerTests.java => NumericMetricFieldDownsamplerTests.java} (89%) rename x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/{TDigestHistogramFieldProducerTests.java => TDigestHistogramFieldDownsamplerTests.java} (91%) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java index c35b5a46cac98..43db8c1ccb63c 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java @@ -48,7 +48,7 @@ public String name() { } /** - * Resets the producer to an empty value. + * Resets the downsampler to an empty value. */ public abstract void reset(); @@ -68,6 +68,12 @@ public T getLeaf(LeafReaderContext context) throws IOException { public abstract void collect(T docValues, IntArrayList docIdBuffer) throws IOException; + /** + * Utility class used for fetching field values by reading field data. + * For fields whose type is multivalued the 'name' matches the parent field + * name (normally used for indexing data), while the actual multiField + * name is accessible by means of {@link MappedFieldType#name()}. + */ abstract static class FieldValueFetcher { protected final String name; protected final MappedFieldType fieldType; @@ -127,10 +133,10 @@ private static AbstractFieldDownsampler create( ) { assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false : "Aggregate metric double should be handled by a dedicated FieldValueFetcher"; - if (TDigestHistogramFieldProducer.TYPE.equals(fieldType.typeName())) { + if (TDigestHistogramFieldDownsampler.TYPE.equals(fieldType.typeName())) { return TDigestHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); } - if (ExponentialHistogramFieldProducer.TYPE.equals(fieldType.typeName())) { + if (ExponentialHistogramFieldDownsampler.TYPE.equals(fieldType.typeName())) { return ExponentialHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); } if (fieldType.getMetricType() != null) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java index 0bb485aca9fa0..f778c9c267f7d 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java @@ -22,12 +22,9 @@ import java.util.Collection; import java.util.List; -import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducer.MAX_NO_VALUE; -import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducer.MIN_NO_VALUE; - /** - * A producer that can be used for downsampling aggregate metric double fields whether is a metric or a label. We need a separate producer - * for each a sub-metric of an aggregate metric double, this means to downsample an aggregate metric double we will need 4. + * A downsampler that can be used for downsampling aggregate metric double fields whether is a metric or a label. We need a separate + * downsampler for each a sub-metric of an aggregate metric double, this means to downsample an aggregate metric double we will need 4. * This is mainly used when downsampling already downsampled indices. */ abstract sealed class AggregateMetricDoubleFieldDownsampler extends NumericMetricFieldDownsampler { @@ -70,7 +67,6 @@ public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffe case min -> min = Math.min(value, min); case max -> max = Math.max(value, max); case sum -> sum.add(value); - // This is the reason why we can't use GaugeMetricFieldProducer // For downsampled indices aggregate metric double's value count field needs to be summed. // (Note: not using CompensatedSum here should be ok given that value_count is mapped as long) case value_count -> count += Math.round(value); @@ -162,7 +158,7 @@ public void write(XContentBuilder builder) throws IOException { } /** - * We use a specialised serializer because we are combining all the available submetric producers. + * We use a specialised serializer because we are combining all the available submetric downsamplers. */ static class Serializer implements DownsampleFieldSerializer { private final Collection> downsamplers; @@ -187,13 +183,13 @@ public void write(XContentBuilder builder) throws IOException { builder.startObject(name); for (AbstractFieldDownsampler fieldDownsampler : downsamplers) { - assert name.equals(fieldDownsampler.name()) : "producer has a different name"; + assert name.equals(fieldDownsampler.name()) : "downsampler has a different name"; if (fieldDownsampler.isEmpty()) { continue; } if (fieldDownsampler instanceof AggregateMetricDoubleFieldDownsampler == false) { throw new IllegalStateException( - "Unexpected field producer class: " + fieldDownsampler.getClass().getSimpleName() + " for " + name + " field" + "Unexpected field downsampler class: " + fieldDownsampler.getClass().getSimpleName() + " for " + name + " field" ); } fieldDownsampler.write(builder); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldProducer.java deleted file mode 100644 index 32c9677b02b7e..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldProducer.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.internal.hppc.IntArrayList; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.search.aggregations.metrics.CompensatedSum; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; - -import java.io.IOException; -import java.util.Collection; - -import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducer.MAX_NO_VALUE; -import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducer.MIN_NO_VALUE; - -/** - * A producer that can be used for downsampling aggregate metric double fields whether is a metric or a label. We need a separate producer - * for each a sub-metric of an aggregate metric double, this means to downsample an aggregate metric double we will need 4. - * This is mainly used when downsampling already downsampled indices. - */ -abstract class AggregateMetricDoubleFieldProducer extends AbstractDownsampleFieldProducer { - - protected final AggregateMetricDoubleFieldMapper.Metric metric; - - AggregateMetricDoubleFieldProducer(String name, AggregateMetricDoubleFieldMapper.Metric metric) { - super(name); - this.metric = metric; - } - - static final class Aggregate extends AggregateMetricDoubleFieldProducer { - - private double max = MAX_NO_VALUE; - private double min = MIN_NO_VALUE; - private final CompensatedSum sum = new CompensatedSum(); - private long count; - - Aggregate(String name, AggregateMetricDoubleFieldMapper.Metric metric) { - super(name, metric); - } - - @Override - public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - isEmpty = false; - int docValuesCount = docValues.docValueCount(); - for (int j = 0; j < docValuesCount; j++) { - double value = docValues.nextValue(); - switch (metric) { - case min -> min = Math.min(value, min); - case max -> max = Math.max(value, max); - case sum -> sum.add(value); - // This is the reason why we can't use GaugeMetricFieldProducer - // For downsampled indices aggregate metric double's value count field needs to be summed. - // (Note: not using CompensatedSum here should be ok given that value_count is mapped as long) - case value_count -> count += Math.round(value); - } - } - } - } - - @Override - public void reset() { - isEmpty = true; - max = MAX_NO_VALUE; - min = MIN_NO_VALUE; - sum.reset(0, 0); - count = 0; - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - switch (metric) { - case min -> builder.field("min", min); - case max -> builder.field("max", max); - case sum -> builder.field("sum", sum.value()); - case value_count -> builder.field("value_count", count); - } - } - } - } - - /** - * Important note: This class assumes that field values are collected and sorted by descending order by time - */ - static final class LastValue extends AggregateMetricDoubleFieldProducer { - - private final boolean supportsMultiValue; - private Object lastValue = null; - - LastValue(String name, AggregateMetricDoubleFieldMapper.Metric metric, boolean supportsMultiValue) { - super(name, metric); - this.supportsMultiValue = supportsMultiValue; - } - - @Override - public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { - if (isEmpty() == false) { - return; - } - - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - int docValuesCount = docValues.docValueCount(); - assert docValuesCount > 0; - isEmpty = false; - if (docValuesCount == 1 || supportsMultiValue == false) { - lastValue = docValues.nextValue(); - } else { - var values = new Object[docValuesCount]; - for (int j = 0; j < docValuesCount; j++) { - values[j] = docValues.nextValue(); - } - lastValue = values; - } - return; - } - } - - @Override - public void reset() { - isEmpty = true; - lastValue = null; - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - builder.field(metric.name(), lastValue); - } - } - } - - /** - * We use a specialised serializer because we are combining all the available submetric producers. - */ - static class Serializer implements DownsampleFieldSerializer { - private final Collection> producers; - private final String name; - - /** - * @param name the name of the aggregate_metric_double field as it will be serialized - * in the downsampled index - * @param producers a collection of {@link AggregateMetricDoubleFieldProducer} instances with the subfields - * of the aggregate_metric_double field. - */ - Serializer(String name, Collection> producers) { - this.name = name; - this.producers = producers; - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty()) { - return; - } - - builder.startObject(name); - for (AbstractDownsampleFieldProducer fieldProducer : producers) { - assert name.equals(fieldProducer.name()) : "producer has a different name"; - if (fieldProducer.isEmpty()) { - continue; - } - if (fieldProducer instanceof AggregateMetricDoubleFieldProducer == false) { - throw new IllegalStateException( - "Unexpected field producer class: " + fieldProducer.getClass().getSimpleName() + " for " + name + " field" - ); - } - fieldProducer.write(builder); - } - builder.endObject(); - } - - private boolean isEmpty() { - for (AbstractDownsampleFieldProducer p : producers) { - if (p.isEmpty() == false) { - return false; - } - } - return true; - } - } -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldValueFetcher.java deleted file mode 100644 index 4b06507b9743f..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldValueFetcher.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.elasticsearch.action.downsample.DownsampleConfig; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; - -import java.util.ArrayList; -import java.util.List; - -/** - * This {@link FieldValueFetcher} is responsible for fetching the values of aggregate metric double fields. - * For performance reasons, we load an aggregate metric double by loading its sub-metrics separately. - */ -public final class AggregateMetricDoubleFieldValueFetcher extends FieldValueFetcher { - - private final AggregateMetricDoubleFieldType aggMetricFieldType; - - private final AggregateMetricDoubleFieldProducer fieldProducer; - - AggregateMetricDoubleFieldValueFetcher( - MappedFieldType fieldType, - AggregateMetricDoubleFieldType aggMetricFieldType, - IndexFieldData fieldData, - DownsampleConfig.SamplingMethod samplingMethod - ) { - super(fieldType.name(), fieldType, fieldData, samplingMethod); - this.aggMetricFieldType = aggMetricFieldType; - this.fieldProducer = createFieldProducer(samplingMethod); - } - - @Override - AbstractDownsampleFieldProducer fieldProducer() { - return fieldProducer; - } - - /** - * For aggregate_metric_double fields we create separate fetchers for each sub-metric. This is usually a downsample-of-downsample case. - */ - static List create( - SearchExecutionContext context, - AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aggMetricFieldType, - DownsampleConfig.SamplingMethod samplingMethod - ) { - List fetchers = new ArrayList<>(); - // If the field is an aggregate_metric_double field, we should load all its subfields - // This is usually a downsample-of-downsample case - for (NumberFieldMapper.NumberFieldType metricSubField : aggMetricFieldType.getMetricFields().values()) { - if (context.fieldExistsInIndex(metricSubField.name())) { - IndexFieldData fieldData = context.getForField(metricSubField, MappedFieldType.FielddataOperation.SEARCH); - fetchers.add(new AggregateMetricDoubleFieldValueFetcher(metricSubField, aggMetricFieldType, fieldData, samplingMethod)); - } - } - return fetchers; - } - - private AggregateMetricDoubleFieldProducer createFieldProducer(DownsampleConfig.SamplingMethod samplingMethod) { - AggregateMetricDoubleFieldMapper.Metric metric = null; - for (var e : aggMetricFieldType.getMetricFields().entrySet()) { - NumberFieldMapper.NumberFieldType metricSubField = e.getValue(); - if (metricSubField.name().equals(name())) { - metric = e.getKey(); - break; - } - } - assert metric != null : "Cannot resolve metric type for field " + name(); - - if (aggMetricFieldType.getMetricType() != null) { - if (samplingMethod != DownsampleConfig.SamplingMethod.LAST_VALUE) { - // If the field is an aggregate_metric_double field, we should use the correct subfields - // for each aggregation. This is a downsample-of-downsample case - return new AggregateMetricDoubleFieldProducer.Aggregate(aggMetricFieldType.name(), metric); - } else { - return new AggregateMetricDoubleFieldProducer.LastValue(aggMetricFieldType.name(), metric, false); - } - } else { - // If a field is not a metric, we downsample it as a label - return new AggregateMetricDoubleFieldProducer.LastValue(aggMetricFieldType.name(), metric, true); - } - } -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java deleted file mode 100644 index f54629f445687..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.internal.hppc.IntArrayList; -import org.elasticsearch.index.fielddata.FormattedDocValues; - -import java.io.IOException; -import java.util.Objects; - -/** - * The dimension field producer is effectively a last value field producer that performs some extra validations when assertions are enabled. - * It checks: - * - that a tsid is only collected once, and - * - that all TSIDs that are being collected for a round have the same value. - * Important note: This class assumes that field values are collected and sorted by descending order by time - */ -public class DimensionFieldProducer extends LastValueFieldProducer { - - DimensionFieldProducer(final String name) { - super(name); - } - - void collectOnce(final Object value) { - assert isEmpty; - Objects.requireNonNull(value); - this.lastValue = value; - this.isEmpty = false; - } - - /** - * This is an expensive check that slows down downsampling significantly. - * Given that index is sorted by tsid as a primary key, this shouldn't really happen. - */ - boolean validate(FormattedDocValues docValues, IntArrayList buffer) throws IOException { - for (int i = 0; i < buffer.size(); i++) { - int docId = buffer.get(i); - if (docValues.advanceExact(docId)) { - int docValueCount = docValues.docValueCount(); - for (int j = 0; j < docValueCount; j++) { - var value = docValues.nextValue(); - assert value.equals(this.lastValue) != false - : "Dimension value changed without tsid change [" + value + "] != [" + this.lastValue + "]"; - } - } - } - - return true; - } - - @Override - public void collect(FormattedDocValues docValues, IntArrayList docIdBuffer) throws IOException { - if (isEmpty() == false) { - assert validate(docValues, docIdBuffer); - return; - } - - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - int docValueCount = docValues.docValueCount(); - for (int j = 0; j < docValueCount; j++) { - collectOnce(docValues.nextValue()); - } - // Only need to record one dimension value from one document, within in the same tsid-and-time-interval bucket values are the - // same. - return; - } - } -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java deleted file mode 100644 index 8a99db70606bf..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; -import org.elasticsearch.index.query.SearchExecutionContext; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class DimensionFieldValueFetcher extends FieldValueFetcher { - - private final DimensionFieldProducer dimensionFieldProducer = createFieldProducer(); - - protected DimensionFieldValueFetcher(final String fieldName, final MappedFieldType fieldType, final IndexFieldData fieldData) { - super(fieldName, fieldType, fieldData, null); - } - - private DimensionFieldProducer createFieldProducer() { - return new DimensionFieldProducer(name); - } - - @Override - AbstractDownsampleFieldProducer fieldProducer() { - return this.dimensionFieldProducer; - } - - /** - * Retrieve field value fetchers for a list of dimensions. - */ - static List create( - final SearchExecutionContext context, - final String[] dimensions, - final Map multiFieldSources - ) { - List fetchers = new ArrayList<>(); - for (String dimension : dimensions) { - String sourceFieldName = multiFieldSources.getOrDefault(dimension, dimension); - MappedFieldType fieldType = context.getFieldType(sourceFieldName); - assert fieldType != null : "Unknown type for dimension field: [" + sourceFieldName + "]"; - - if (context.fieldExistsInIndex(fieldType.name())) { - final IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); - if (fieldType instanceof FlattenedFieldMapper.KeyedFlattenedFieldType flattenedFieldType) { - // Name of the field type and name of the dimension are different in this case. - var dimensionName = flattenedFieldType.rootName() + '.' + flattenedFieldType.key(); - fetchers.add(new DimensionFieldValueFetcher(dimensionName, fieldType, fieldData)); - } else { - fetchers.add(new DimensionFieldValueFetcher(dimension, fieldType, fieldData)); - } - } - } - return Collections.unmodifiableList(fetchers); - } -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 72ff2a070bf05..a1156440c35af 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -33,6 +33,9 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocCountFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; @@ -56,6 +59,7 @@ import org.elasticsearch.xpack.core.downsample.DownsampleShardIndexerStatus; import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState; import org.elasticsearch.xpack.core.downsample.DownsampleShardTask; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; import java.io.Closeable; import java.io.IOException; @@ -93,7 +97,7 @@ class DownsampleShardIndexer { private final DateFieldMapper.DateFieldType timestampField; private final DocValueFormat timestampFormat; private final Rounding.Prepared rounding; - private final List fieldValueFetchers; + private final List> fieldDownsamplers; private final DownsampleShardTask task; private final DownsampleShardPersistentTaskState state; private final String[] dimensions; @@ -138,14 +142,19 @@ class DownsampleShardIndexer { this.rounding = config.createRounding(); var samplingMethod = config.getSamplingMethodOrDefault(); - List fetchers = new ArrayList<>(metrics.length + labels.length + dimensions.length); - fetchers.addAll(FieldValueFetcher.create(searchExecutionContext, metrics, multiFieldSources, samplingMethod)); + List> downsamplers = new ArrayList<>(metrics.length + labels.length + dimensions.length); + downsamplers.addAll(AbstractFieldDownsampler.create(searchExecutionContext, metrics, multiFieldSources, samplingMethod)); // Labels are downsampled using the last value, they are not influenced by the requested sampling method - fetchers.addAll( - FieldValueFetcher.create(searchExecutionContext, labels, multiFieldSources, DownsampleConfig.SamplingMethod.LAST_VALUE) + downsamplers.addAll( + AbstractFieldDownsampler.create( + searchExecutionContext, + labels, + multiFieldSources, + DownsampleConfig.SamplingMethod.LAST_VALUE + ) ); - fetchers.addAll(DimensionFieldValueFetcher.create(searchExecutionContext, dimensions, multiFieldSources)); - this.fieldValueFetchers = Collections.unmodifiableList(fetchers); + downsamplers.addAll(DimensionFieldDownsampler.create(searchExecutionContext, dimensions, multiFieldSources)); + this.fieldDownsamplers = Collections.unmodifiableList(downsamplers); toClose = null; } finally { IOUtils.closeWhileHandlingException(toClose); @@ -345,6 +354,10 @@ private class TimeSeriesBucketCollector extends BucketCollector { private final BulkProcessor2 bulkProcessor; private final DownsampleBucketBuilder downsampleBucketBuilder; private final List leafBucketCollectors = new ArrayList<>(); + private final NumericMetricFieldDownsampler[] numericDownsamplers; + private final LastValueFieldDownsampler[] formattedDocValuesDownsamplers; + private final ExponentialHistogramFieldDownsampler[] exponentialHistogramDownsamplers; + private final TDigestHistogramFieldDownsampler[] tDigestHistogramDownsamplers; private long docsProcessed; private long bucketsCreated; long lastTimestamp = Long.MAX_VALUE; @@ -352,10 +365,19 @@ private class TimeSeriesBucketCollector extends BucketCollector { TimeSeriesBucketCollector(BulkProcessor2 bulkProcessor, String[] dimensions) { this.bulkProcessor = bulkProcessor; - AbstractDownsampleFieldProducer[] fieldProducers = fieldValueFetchers.stream() - .map(FieldValueFetcher::fieldProducer) - .toArray(AbstractDownsampleFieldProducer[]::new); - this.downsampleBucketBuilder = new DownsampleBucketBuilder(fieldProducers, dimensions); + this.numericDownsamplers = fieldDownsamplers.stream() + .filter(NumericMetricFieldDownsampler.class::isInstance) + .toArray(NumericMetricFieldDownsampler[]::new); + this.formattedDocValuesDownsamplers = fieldDownsamplers.stream() + .filter(LastValueFieldDownsampler.class::isInstance) + .toArray(LastValueFieldDownsampler[]::new); + this.exponentialHistogramDownsamplers = fieldDownsamplers.stream() + .filter(ExponentialHistogramFieldDownsampler.class::isInstance) + .toArray(ExponentialHistogramFieldDownsampler[]::new); + this.tDigestHistogramDownsamplers = fieldDownsamplers.stream() + .filter(TDigestHistogramFieldDownsampler.class::isInstance) + .toArray(TDigestHistogramFieldDownsampler[]::new); + this.downsampleBucketBuilder = new DownsampleBucketBuilder(fieldDownsamplers, dimensions); } @Override @@ -364,41 +386,32 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag final DocCountProvider docCountProvider = new DocCountProvider(); docCountProvider.setLeafReaderContext(ctx); - // For each field, return a tuple with the downsample field producer and the field value leaf - final LeafDownsampleCollector.FieldCollector[] fieldCollectors = new LeafDownsampleCollector.FieldCollector< - ?>[fieldValueFetchers.size()]; - for (int i = 0; i < fieldValueFetchers.size(); i++) { - var fieldValueFetcher = fieldValueFetchers.get(i); - var fieldProducer = fieldValueFetcher.fieldProducer(); - if (fieldProducer instanceof NumericMetricFieldProducer metricFieldProducer) { - fieldCollectors[i] = new LeafDownsampleCollector.FieldCollector<>( - metricFieldProducer, - fieldValueFetcher.getNumericLeaf(ctx) - ); - } else if (fieldProducer instanceof ExponentialHistogramFieldProducer exponentialHistogramProducer) { - fieldCollectors[i] = new LeafDownsampleCollector.FieldCollector<>( - exponentialHistogramProducer, - fieldValueFetcher.getExponentialHistogramLeaf(ctx) - ); - } else if (fieldProducer instanceof AggregateMetricDoubleFieldProducer numericFieldProducer) { - fieldCollectors[i] = new LeafDownsampleCollector.FieldCollector<>( - numericFieldProducer, - fieldValueFetcher.getNumericLeaf(ctx) - ); - } else if (fieldProducer instanceof LastValueFieldProducer lastValueFieldProducer) { - fieldCollectors[i] = new LeafDownsampleCollector.FieldCollector<>( - lastValueFieldProducer, - fieldValueFetcher.getLeaf(ctx) - ); - } else if (fieldProducer instanceof TDigestHistogramFieldProducer histogramFieldProducer) { - fieldCollectors[i] = new LeafDownsampleCollector.FieldCollector<>( - histogramFieldProducer, - fieldValueFetcher.getHistogramLeaf(ctx) - ); - } + // For each field, return a tuple with the downsample field downsampler and the field value leaf + var numericValues = new SortedNumericDoubleValues[numericDownsamplers.length]; + for (int i = 0; i < numericDownsamplers.length; i++) { + numericValues[i] = numericDownsamplers[i].getLeaf(ctx); + } + var formattedDocValues = new FormattedDocValues[formattedDocValuesDownsamplers.length]; + for (int i = 0; i < formattedDocValuesDownsamplers.length; i++) { + formattedDocValues[i] = formattedDocValuesDownsamplers[i].getLeaf(ctx); + } + var exponentialHistogramValues = new ExponentialHistogramValuesReader[exponentialHistogramDownsamplers.length]; + for (int i = 0; i < exponentialHistogramDownsamplers.length; i++) { + exponentialHistogramValues[i] = exponentialHistogramDownsamplers[i].getLeaf(ctx); + } + var tDigestHistogramValues = new HistogramValues[tDigestHistogramDownsamplers.length]; + for (int i = 0; i < tDigestHistogramDownsamplers.length; i++) { + tDigestHistogramValues[i] = tDigestHistogramDownsamplers[i].getLeaf(ctx); } - var leafBucketCollector = new LeafDownsampleCollector(aggCtx, docCountProvider, fieldCollectors); + var leafBucketCollector = new LeafDownsampleCollector( + aggCtx, + docCountProvider, + numericValues, + formattedDocValues, + exponentialHistogramValues, + tDigestHistogramValues + ); leafBucketCollectors.add(leafBucketCollector); return leafBucketCollector; } @@ -415,7 +428,10 @@ class LeafDownsampleCollector extends LeafBucketCollector { final AggregationExecutionContext aggCtx; final DocCountProvider docCountProvider; - final LeafDownsampleCollector.FieldCollector[] fieldCollectors; + final SortedNumericDoubleValues[] numericValues; + final FormattedDocValues[] formattedDocValues; + final ExponentialHistogramValuesReader[] exponentialHistogramValues; + final HistogramValues[] tDigestHistogramValues; // Capture the first timestamp in order to determine which leaf collector's leafBulkCollection() is invoked first. long firstTimeStampForBulkCollection; @@ -425,12 +441,17 @@ class LeafDownsampleCollector extends LeafBucketCollector { LeafDownsampleCollector( AggregationExecutionContext aggCtx, DocCountProvider docCountProvider, - LeafDownsampleCollector.FieldCollector[] fieldCollectors + SortedNumericDoubleValues[] numericValues, + FormattedDocValues[] formattedDocValues, + ExponentialHistogramValuesReader[] exponentialHistogramValues, + HistogramValues[] tDigestHistogramValues ) { - this.aggCtx = aggCtx; this.docCountProvider = docCountProvider; - this.fieldCollectors = fieldCollectors; + this.numericValues = numericValues; + this.formattedDocValues = formattedDocValues; + this.exponentialHistogramValues = exponentialHistogramValues; + this.tDigestHistogramValues = tDigestHistogramValues; } @Override @@ -498,10 +519,12 @@ void leafBulkCollection() throws IOException { } downsampleBucketBuilder.collectDocCount(docIdBuffer, docCountProvider); + // Iterate over all field values and collect the doc_values for this docId - for (int i = 0; i < fieldCollectors.length; i++) { - fieldCollectors[i].collect(docIdBuffer); - } + collect(numericDownsamplers, numericValues); + collect(formattedDocValuesDownsamplers, formattedDocValues); + collect(exponentialHistogramDownsamplers, exponentialHistogramValues); + collect(tDigestHistogramDownsamplers, tDigestHistogramValues); docsProcessed += docIdBuffer.size(); task.setDocsProcessed(docsProcessed); @@ -510,9 +533,11 @@ void leafBulkCollection() throws IOException { docIdBuffer.elementsCount = 0; } - record FieldCollector(AbstractDownsampleFieldProducer fieldProducer, T docValues) { - void collect(IntArrayList docIdBuffer) throws IOException { - fieldProducer.collect(docValues, docIdBuffer); + private void collect(AbstractFieldDownsampler[] downsamplers, T[] docValues) throws IOException { + assert downsamplers.length == docValues.length + : "Number of downsamplers [" + downsamplers.length + "] does not match number of doc values [" + docValues.length + "]"; + for (int i = 0; i < downsamplers.length; i++) { + downsamplers[i].collect(docValues[i], docIdBuffer); } } @@ -587,28 +612,28 @@ private class DownsampleBucketBuilder { private int tsidOrd = -1; private long timestamp; private int docCount; - private final AbstractDownsampleFieldProducer[] fieldProducers; - private final DownsampleFieldSerializer[] groupedProducers; + private final List> fieldDownsamplers; + private final DownsampleFieldSerializer[] groupedDownsamplers; private final String[] dimensions; - DownsampleBucketBuilder(AbstractDownsampleFieldProducer[] fieldProducers, String[] dimensions) { - this.fieldProducers = fieldProducers; + DownsampleBucketBuilder(List> fieldDownsamplers, String[] dimensions) { + this.fieldDownsamplers = fieldDownsamplers; this.dimensions = dimensions; /* - * The downsample field producers for aggregate_metric_double all share the same name (this is - * the name they will be serialized in the target index). We group all field producers by - * name. If grouping yields multiple downsample field producers, we delegate serialization to + * The field downsamplers for aggregate_metric_double all share the same name (this is + * the name they will be serialized in the target index). We group all field downsamplers by + * name. If grouping yields multiple field downsamplers, we delegate serialization to * the AggregateMetricFieldSerializer class. */ - groupedProducers = Arrays.stream(fieldProducers) - .collect(groupingBy(AbstractDownsampleFieldProducer::name)) + groupedDownsamplers = fieldDownsamplers.stream() + .collect(groupingBy(AbstractFieldDownsampler::name)) .entrySet() .stream() .map(e -> { if (e.getValue().size() == 1) { return e.getValue().get(0); } else { - return new AggregateMetricDoubleFieldProducer.Serializer(e.getKey(), e.getValue()); + return new AggregateMetricDoubleFieldDownsampler.Serializer(e.getKey(), e.getValue()); } }) .toArray(DownsampleFieldSerializer[]::new); @@ -629,8 +654,8 @@ public void resetTsid(BytesRef tsid, int tsidOrd, long timestamp) { public void resetTimestamp(long timestamp) { this.timestamp = timestamp; this.docCount = 0; - for (AbstractDownsampleFieldProducer producer : fieldProducers) { - producer.reset(); + for (AbstractFieldDownsampler downsampler : fieldDownsamplers) { + downsampler.reset(); } if (logger.isTraceEnabled()) { logger.trace( @@ -663,8 +688,8 @@ public XContentBuilder buildDownsampleDocument() throws IOException { builder.field(DocCountFieldMapper.NAME, docCount); // Serialize fields - for (DownsampleFieldSerializer fieldProducer : groupedProducers) { - fieldProducer.write(builder); + for (DownsampleFieldSerializer fieldDownsampler : groupedDownsamplers) { + fieldDownsampler.write(builder); } if (dimensions.length == 0) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java deleted file mode 100644 index f7105e47e903a..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.action.downsample.DownsampleConfig; -import org.elasticsearch.index.fielddata.FormattedDocValues; -import org.elasticsearch.index.fielddata.HistogramValues; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.LeafHistogramFieldData; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.TimeSeriesParams; -import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; -import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; -import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.LeafExponentialHistogramFieldData; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.index.mapper.TimeSeriesParams.MetricType.POSITION; - -/** - * Utility class used for fetching field values by reading field data. - * For fields whose type is multivalued the 'name' matches the parent field - * name (normally used for indexing data), while the actual multiField - * name is accessible by means of {@link MappedFieldType#name()}. - */ -class FieldValueFetcher { - - protected final String name; - protected final MappedFieldType fieldType; - protected final IndexFieldData fieldData; - protected final AbstractDownsampleFieldProducer fieldProducer; - - protected FieldValueFetcher( - String name, - MappedFieldType fieldType, - IndexFieldData fieldData, - DownsampleConfig.SamplingMethod samplingMethod - ) { - this.name = name; - this.fieldType = fieldType; - this.fieldData = fieldData; - this.fieldProducer = createFieldProducer(samplingMethod); - } - - public String name() { - return name; - } - - public FormattedDocValues getLeaf(LeafReaderContext context) { - DocValueFormat format = fieldType.docValueFormat(null, null); - return fieldData.load(context).getFormattedValues(format); - } - - public SortedNumericDoubleValues getNumericLeaf(LeafReaderContext context) { - LeafNumericFieldData numericFieldData = (LeafNumericFieldData) fieldData.load(context); - return numericFieldData.getDoubleValues(); - } - - public ExponentialHistogramValuesReader getExponentialHistogramLeaf(LeafReaderContext context) throws IOException { - LeafExponentialHistogramFieldData exponentialHistogramFieldData = (LeafExponentialHistogramFieldData) fieldData.load(context); - return exponentialHistogramFieldData.getHistogramValues(); - } - - AbstractDownsampleFieldProducer fieldProducer() { - return fieldProducer; - } - - public HistogramValues getHistogramLeaf(LeafReaderContext context) throws IOException { - LeafHistogramFieldData histogramFieldData = (LeafHistogramFieldData) fieldData.load(context); - return histogramFieldData.getHistogramValues(); - } - - private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.SamplingMethod samplingMethod) { - assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false - : "Aggregate metric double should be handled by a dedicated FieldValueFetcher"; - if (TDigestHistogramFieldProducer.TYPE.equals(fieldType.typeName())) { - return TDigestHistogramFieldProducer.create(name(), samplingMethod); - } - if (ExponentialHistogramFieldProducer.TYPE.equals(fieldType.typeName())) { - return ExponentialHistogramFieldProducer.create(name(), samplingMethod); - } - if (fieldType.getMetricType() != null) { - // TODO: Support POSITION in downsampling - if (fieldType.getMetricType() == POSITION) { - throw new IllegalArgumentException("Unsupported metric type [position] for down-sampling"); - } - assert fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE - || fieldType.getMetricType() == TimeSeriesParams.MetricType.COUNTER - : "only gauges and counters accepted, other metrics should have been handled earlier"; - if (samplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE - && fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE) { - return new NumericMetricFieldProducer.AggregateGauge(name()); - } - return new NumericMetricFieldProducer.LastValue(name()); - } else { - // If a field is not a metric, we downsample it as a label - return LastValueFieldProducer.create(name(), fieldType.typeName()); - } - } - - /** - * Retrieve field value fetchers for a list of fields. - */ - static List create( - SearchExecutionContext context, - String[] fields, - Map multiFieldSources, - DownsampleConfig.SamplingMethod samplingMethod - ) { - List fetchers = new ArrayList<>(); - for (String field : fields) { - String sourceField = multiFieldSources.getOrDefault(field, field); - MappedFieldType fieldType = context.getFieldType(sourceField); - assert fieldType != null : "Unknown field type for field: [" + sourceField + "]"; - - if (fieldType instanceof AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aggMetricFieldType) { - fetchers.addAll(AggregateMetricDoubleFieldValueFetcher.create(context, aggMetricFieldType, samplingMethod)); - } else { - if (context.fieldExistsInIndex(field)) { - final IndexFieldData fieldData; - if (fieldType instanceof FlattenedFieldMapper.RootFlattenedFieldType flattenedFieldType) { - var keyedFieldType = flattenedFieldType.getKeyedFieldType(); - fieldData = context.getForField(keyedFieldType, MappedFieldType.FielddataOperation.SEARCH); - } else { - fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); - } - fetchers.add(new FieldValueFetcher(field, fieldType, fieldData, samplingMethod)); - } - } - } - return Collections.unmodifiableList(fetchers); - } -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java index 901b9680d1f94..12ad9e0c9e9a6 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java @@ -39,9 +39,9 @@ class LastValueFieldDownsampler extends AbstractFieldDownsampler fieldData) { assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false : "field type cannot be aggregate metric double: " + fieldType.typeName() + " for field " + name; - assert ExponentialHistogramFieldProducer.TYPE.equals(fieldType.typeName()) == false + assert ExponentialHistogramFieldDownsampler.TYPE.equals(fieldType.typeName()) == false : "field type cannot be exponential histogram: " + fieldType.typeName() + " for field " + name; - assert TDigestHistogramFieldProducer.TYPE.equals(fieldType.typeName()) == false + assert TDigestHistogramFieldDownsampler.TYPE.equals(fieldType.typeName()) == false : "field type cannot be histogram: " + fieldType.typeName() + " for field " + name; if ("flattened".equals(fieldType.typeName())) { return new LastValueFieldDownsampler.FlattenedFieldProducer(name, fieldType, fieldData); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java deleted file mode 100644 index e27344f23bd1c..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.internal.hppc.IntArrayList; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fielddata.FormattedDocValues; -import org.elasticsearch.index.mapper.flattened.FlattenedFieldSyntheticWriterHelper; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Class that produces the last value of a label field for downsampling. - * Important note: This class assumes that field values are collected and sorted by descending order by time - */ -class LastValueFieldProducer extends AbstractDownsampleFieldProducer { - Object lastValue = null; - - LastValueFieldProducer(String name) { - super(name); - } - - /** - * Creates a producer that can be used for downsampling labels. - */ - static LastValueFieldProducer create(String name, String fieldType) { - assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType) == false - : "field type cannot be aggregate metric double: " + fieldType + " for field " + name; - assert ExponentialHistogramFieldProducer.TYPE.equals(fieldType) == false - : "field type cannot be exponential histogram: " + fieldType + " for field " + name; - assert TDigestHistogramFieldProducer.TYPE.equals(fieldType) == false - : "field type cannot be histogram: " + fieldType + " for field " + name; - if ("flattened".equals(fieldType)) { - return new LastValueFieldProducer.FlattenedFieldProducer(name); - } - return new LastValueFieldProducer(name); - } - - @Override - public void reset() { - isEmpty = true; - lastValue = null; - } - - /** - * Collects the last value observed in these field values. This implementation assumes that field values are collected - * and sorted by descending order by time. In this case, it assumes that the last value of the time is the first value - * collected. Eventually, the implementation of this class ends up storing the first value it is empty and then - * ignoring everything else. - */ - @Override - public void collect(FormattedDocValues docValues, IntArrayList docIdBuffer) throws IOException { - if (isEmpty() == false) { - return; - } - - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - int docValuesCount = docValues.docValueCount(); - assert docValuesCount > 0; - isEmpty = false; - if (docValuesCount == 1) { - lastValue = docValues.nextValue(); - } else { - var values = new Object[docValuesCount]; - for (int j = 0; j < docValuesCount; j++) { - values[j] = docValues.nextValue(); - } - lastValue = values; - } - // Only need to record one label value from one document, within in the same tsid-and-time-interval we only keep the first - // with downsampling. - return; - } - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - builder.field(name(), lastValue); - } - } - - public Object lastValue() { - return lastValue; - } - - static final class FlattenedFieldProducer extends LastValueFieldProducer { - - private FlattenedFieldProducer(String name) { - super(name); - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - builder.startObject(name()); - - var value = lastValue(); - List list; - if (value instanceof Object[] values) { - list = new ArrayList<>(values.length); - for (Object v : values) { - list.add(new BytesRef(v.toString())); - } - } else { - list = List.of(new BytesRef(value.toString())); - } - - var iterator = list.iterator(); - var helper = new FlattenedFieldSyntheticWriterHelper(() -> { - if (iterator.hasNext()) { - return iterator.next(); - } else { - return null; - } - }); - helper.write(builder); - builder.endObject(); - } - } - } - -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java index 3f416aa740a52..50de26896c6f8 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java @@ -23,7 +23,8 @@ * values. Based on the supported metric types, the subclasses of this class compute values for * gauge and metric types. */ -abstract sealed class NumericMetricFieldDownsampler extends AbstractFieldDownsampler permits AggregateMetricDoubleFieldDownsampler, NumericMetricFieldDownsampler.AggregateGauge, NumericMetricFieldDownsampler.LastValue { +abstract sealed class NumericMetricFieldDownsampler extends AbstractFieldDownsampler permits + AggregateMetricDoubleFieldDownsampler, NumericMetricFieldDownsampler.AggregateGauge, NumericMetricFieldDownsampler.LastValue { NumericMetricFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { super(name, new NumericFieldFetcher(name, fieldType, fieldData)); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducer.java deleted file mode 100644 index 4c29485638622..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducer.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.internal.hppc.IntArrayList; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.search.aggregations.metrics.CompensatedSum; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; - -/** - * Class that collects all raw values for a numeric metric field and computes its aggregate (downsampled) - * values. Based on the supported metric types, the subclasses of this class compute values for - * gauge and metric types. - */ -abstract sealed class NumericMetricFieldProducer extends AbstractDownsampleFieldProducer { - - NumericMetricFieldProducer(String name) { - super(name); - } - - static final double MAX_NO_VALUE = -Double.MAX_VALUE; - static final double MIN_NO_VALUE = Double.MAX_VALUE; - - /** - * {@link NumericMetricFieldProducer} implementation for creating an aggregate gauge metric field - */ - static final class AggregateGauge extends NumericMetricFieldProducer { - - double max = MAX_NO_VALUE; - double min = MIN_NO_VALUE; - final CompensatedSum sum = new CompensatedSum(); - long count; - - AggregateGauge(String name) { - super(name); - } - - @Override - public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - isEmpty = false; - int docValuesCount = docValues.docValueCount(); - for (int j = 0; j < docValuesCount; j++) { - double value = docValues.nextValue(); - this.max = Math.max(value, max); - this.min = Math.min(value, min); - sum.add(value); - count++; - } - } - } - - @Override - public void reset() { - isEmpty = true; - max = MAX_NO_VALUE; - min = MIN_NO_VALUE; - sum.reset(0, 0); - count = 0; - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - builder.startObject(name()); - builder.field("min", min); - builder.field("max", max); - builder.field("sum", sum.value()); - builder.field("value_count", count); - builder.endObject(); - } - } - } - - /** - * {@link NumericMetricFieldProducer} implementation for sampling the last value of a numeric metric field. - * Important note: This class assumes that field values are collected and sorted by descending order by time. - */ - static final class LastValue extends NumericMetricFieldProducer { - - double lastValue = Double.NaN; - - LastValue(String name) { - super(name); - } - - @Override - public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { - if (isEmpty() == false) { - return; - } - - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId)) { - isEmpty = false; - lastValue = docValues.nextValue(); - return; - } - } - } - - public Double lastValue() { - if (isEmpty()) { - return null; - } - return lastValue; - } - - @Override - public void reset() { - isEmpty = true; - lastValue = Double.NaN; - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - builder.field(name(), lastValue); - } - } - } -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducer.java deleted file mode 100644 index cbb747eeee387..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducer.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.internal.hppc.IntArrayList; -import org.elasticsearch.action.downsample.DownsampleConfig; -import org.elasticsearch.common.breaker.NoopCircuitBreaker; -import org.elasticsearch.index.fielddata.HistogramValue; -import org.elasticsearch.index.fielddata.HistogramValues; -import org.elasticsearch.search.aggregations.metrics.TDigestState; -import org.elasticsearch.tdigest.Centroid; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -/** - * Class that collects all raw values for an exponential histogram metric field and computes its aggregate (downsampled) - * values. - */ -abstract class TDigestHistogramFieldProducer extends AbstractDownsampleFieldProducer { - - static final String TYPE = "histogram"; - public static final int COMPRESSION = 100; - - TDigestHistogramFieldProducer(String name) { - super(name); - } - - /** - * @return the requested produces based on the sampling method for metric of type exponential histogram - */ - public static TDigestHistogramFieldProducer create(String name, DownsampleConfig.SamplingMethod samplingMethod) { - return switch (samplingMethod) { - case AGGREGATE -> new Aggregate(name); - case LAST_VALUE -> new LastValue(name); - }; - } - - private static class Aggregate extends TDigestHistogramFieldProducer { - - private TDigestState tDigestState = null; - - Aggregate(String name) { - super(name); - } - - public void collect(HistogramValues docValues, IntArrayList docIdBuffer) throws IOException { - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - isEmpty = false; - if (tDigestState == null) { - // TODO: figure out what circuit breaker to use here and in the other histogram - tDigestState = TDigestState.create(new NoopCircuitBreaker("downsampling-histograms"), COMPRESSION); - } - final HistogramValue sketch = docValues.histogram(); - while (sketch.next()) { - tDigestState.add(sketch.value(), sketch.count()); - } - } - } - - @Override - public void reset() { - isEmpty = true; - tDigestState = null; - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - Iterator centroids = tDigestState.uniqueCentroids(); - final List values = new ArrayList<>(); - final List counts = new ArrayList<>(); - while (centroids.hasNext()) { - Centroid centroid = centroids.next(); - values.add(centroid.mean()); - counts.add(centroid.count()); - } - builder.startObject(name()).field("counts", counts).field("values", values).endObject(); - tDigestState.close(); - } - } - } - - private static class LastValue extends TDigestHistogramFieldProducer { - - private HistogramValue lastValue = null; - - LastValue(String name) { - super(name); - } - - public void collect(HistogramValues docValues, IntArrayList docIdBuffer) throws IOException { - if (isEmpty() == false) { - return; - } - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - isEmpty = false; - lastValue = docValues.histogram(); - return; - } - } - - @Override - public void reset() { - isEmpty = true; - lastValue = null; - } - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - final HistogramValue histogramValue = lastValue; - final List values = new ArrayList<>(); - final List counts = new ArrayList<>(); - while (histogramValue.next()) { - values.add(histogramValue.value()); - counts.add(histogramValue.count()); - } - builder.startObject(name()).field("counts", counts).field("values", values).endObject(); - } - } - } -} diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java index f44247d554a52..1d8935656990e 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java @@ -17,13 +17,13 @@ import java.io.IOException; import java.util.List; -import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducerTests.createNumericValuesInstance; +import static org.elasticsearch.xpack.downsample.NumericMetricFieldDownsamplerTests.createNumericValuesInstance; import static org.hamcrest.Matchers.equalTo; public class AggregateMetricDoubleFieldSerializerTests extends ESTestCase { public void testAggregatedGaugeFieldSerialization() throws IOException { - NumericMetricFieldProducer producer = new NumericMetricFieldProducer.AggregateGauge("my-gauge"); + NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.AggregateGauge("my-gauge", null, null); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); producer.collect(valuesInstance, docIdBuffer); @@ -34,10 +34,8 @@ public void testAggregatedGaugeFieldSerialization() throws IOException { assertThat(Strings.toString(builder), equalTo("{\"my-gauge\":{\"min\":5.5,\"max\":55.0,\"sum\":72.7,\"value_count\":3}}")); } // Ensure that the AggregateMetricDouble producer serializer does not work on an aggregated gauge. - AggregateMetricDoubleFieldProducer.Serializer aggregateMetricProducerSerializer = new AggregateMetricDoubleFieldProducer.Serializer( - "my-gauge", - List.of(producer) - ); + AggregateMetricDoubleFieldDownsampler.Serializer aggregateMetricProducerSerializer = + new AggregateMetricDoubleFieldDownsampler.Serializer("my-gauge", List.of(producer)); XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); builder.humanReadable(true).startObject(); IllegalStateException error = expectThrows(IllegalStateException.class, () -> aggregateMetricProducerSerializer.write(builder)); @@ -45,11 +43,11 @@ public void testAggregatedGaugeFieldSerialization() throws IOException { } public void testInvalidCounterFieldSerialization() throws IOException { - NumericMetricFieldProducer producer = new NumericMetricFieldProducer.LastValue("my-counter"); + NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.LastValue("my-counter", null, null); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55, 12, 5); producer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer.Serializer gaugeFieldSerializer = new AggregateMetricDoubleFieldProducer.Serializer( + AggregateMetricDoubleFieldDownsampler.Serializer gaugeFieldSerializer = new AggregateMetricDoubleFieldDownsampler.Serializer( "my-counter", List.of(producer) ); @@ -60,35 +58,43 @@ public void testInvalidCounterFieldSerialization() throws IOException { } public void testAggregateMetricDoubleFieldSerialization() throws IOException { - AggregateMetricDoubleFieldProducer minProducer = new AggregateMetricDoubleFieldProducer.Aggregate( + AggregateMetricDoubleFieldDownsampler minProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", - AggregateMetricDoubleFieldMapper.Metric.min + AggregateMetricDoubleFieldMapper.Metric.min, + null, + null ); var docIdBuffer = IntArrayList.from(0, 1); var valuesInstance = createNumericValuesInstance(docIdBuffer, 10, 5.5); minProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer maxProducer = new AggregateMetricDoubleFieldProducer.Aggregate( + AggregateMetricDoubleFieldDownsampler maxProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", - AggregateMetricDoubleFieldMapper.Metric.max + AggregateMetricDoubleFieldMapper.Metric.max, + null, + null ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 30, 55.0); maxProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer sumProducer = new AggregateMetricDoubleFieldProducer.Aggregate( + AggregateMetricDoubleFieldDownsampler sumProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", - AggregateMetricDoubleFieldMapper.Metric.sum + AggregateMetricDoubleFieldMapper.Metric.sum, + null, + null ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 30, 72.7); sumProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer countProducer = new AggregateMetricDoubleFieldProducer.Aggregate( + AggregateMetricDoubleFieldDownsampler countProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", - AggregateMetricDoubleFieldMapper.Metric.value_count + AggregateMetricDoubleFieldMapper.Metric.value_count, + null, + null ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 2, 3); countProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer.Serializer gaugeFieldSerializer = new AggregateMetricDoubleFieldProducer.Serializer( + AggregateMetricDoubleFieldDownsampler.Serializer gaugeFieldSerializer = new AggregateMetricDoubleFieldDownsampler.Serializer( "my-gauge", List.of(maxProducer, minProducer, sumProducer, countProducer) ); @@ -101,35 +107,35 @@ public void testAggregateMetricDoubleFieldSerialization() throws IOException { } public void testLastValuePreAggregatedFieldSerialization() throws IOException { - AggregateMetricDoubleFieldProducer minProducer = randomAggregateSubMetricFieldProducer( + AggregateMetricDoubleFieldDownsampler minProducer = randomAggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.min ); var docIdBuffer = IntArrayList.from(0, 1); var valuesInstance = createNumericValuesInstance(docIdBuffer, 10D, 5.5); minProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer maxProducer = randomAggregateSubMetricFieldProducer( + AggregateMetricDoubleFieldDownsampler maxProducer = randomAggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.max ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 30D, 55.0); maxProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer sumProducer = randomAggregateSubMetricFieldProducer( + AggregateMetricDoubleFieldDownsampler sumProducer = randomAggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.sum ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 30D, 72.7); sumProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer countProducer = randomAggregateSubMetricFieldProducer( + AggregateMetricDoubleFieldDownsampler countProducer = randomAggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.value_count ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 2, 3); countProducer.collect(valuesInstance, docIdBuffer); - AggregateMetricDoubleFieldProducer.Serializer gaugeFieldSerializer = new AggregateMetricDoubleFieldProducer.Serializer( + AggregateMetricDoubleFieldDownsampler.Serializer gaugeFieldSerializer = new AggregateMetricDoubleFieldDownsampler.Serializer( "my-gauge", List.of(maxProducer, minProducer, sumProducer, countProducer) ); @@ -144,10 +150,10 @@ public void testLastValuePreAggregatedFieldSerialization() throws IOException { /** * Serializing for a metric or a label shouldn't make a difference. */ - private AggregateMetricDoubleFieldProducer randomAggregateSubMetricFieldProducer( + private AggregateMetricDoubleFieldDownsampler randomAggregateSubMetricFieldProducer( String name, AggregateMetricDoubleFieldMapper.Metric metric ) { - return new AggregateMetricDoubleFieldProducer.LastValue(name, metric, randomBoolean()); + return new AggregateMetricDoubleFieldDownsampler.LastValue(name, metric, null, null, randomBoolean()); } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsamplerTests.java similarity index 78% rename from x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldProducerTests.java rename to x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsamplerTests.java index b4f8b1b9c8d4a..d303dce6d884c 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldProducerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsamplerTests.java @@ -9,8 +9,13 @@ import org.apache.lucene.internal.hppc.IntArrayList; import org.apache.lucene.internal.hppc.IntObjectHashMap; +import org.apache.lucene.search.Query; import org.elasticsearch.common.Strings; import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.mapper.IndexType; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -20,14 +25,15 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -public class LastValueFieldProducerTests extends AggregatorTestCase { +public class LastValueFieldDownsamplerTests extends AggregatorTestCase { public void testLastValueKeyword() throws IOException { - LastValueFieldProducer lastValueFieldProducer = new LastValueFieldProducer(randomAlphanumericOfLength(10)); + LastValueFieldDownsampler lastValueFieldProducer = new LastValueFieldDownsampler(randomAlphanumericOfLength(10), null, null); assertThat(lastValueFieldProducer.lastValue(), nullValue()); var docIdBuffer = IntArrayList.from(0, 1, 2); var values = createValuesInstance(docIdBuffer, new String[] { "aaa", "bbb", "ccc" }); @@ -38,7 +44,7 @@ public void testLastValueKeyword() throws IOException { } public void testLastValueDouble() throws IOException { - LastValueFieldProducer lastValueFieldProducer = new LastValueFieldProducer(randomAlphanumericOfLength(10)); + LastValueFieldDownsampler lastValueFieldProducer = new LastValueFieldDownsampler(randomAlphanumericOfLength(10), null, null); assertThat(lastValueFieldProducer.lastValue(), nullValue()); var docIdBuffer = IntArrayList.from(0, 1, 2); var values = createValuesInstance(docIdBuffer, new Double[] { 10.20D, 17.30D, 12.60D }); @@ -49,7 +55,7 @@ public void testLastValueDouble() throws IOException { } public void testLastValueInteger() throws IOException { - LastValueFieldProducer lastValueFieldProducer = new LastValueFieldProducer(randomAlphanumericOfLength(10)); + LastValueFieldDownsampler lastValueFieldProducer = new LastValueFieldDownsampler(randomAlphanumericOfLength(10), null, null); assertThat(lastValueFieldProducer.lastValue(), nullValue()); var docIdBuffer = IntArrayList.from(0, 1, 2); var values = createValuesInstance(docIdBuffer, new Integer[] { 10, 17, 12 }); @@ -60,7 +66,7 @@ public void testLastValueInteger() throws IOException { } public void testLastValueLong() throws IOException { - LastValueFieldProducer lastValueFieldProducer = new LastValueFieldProducer(randomAlphanumericOfLength(10)); + LastValueFieldDownsampler lastValueFieldProducer = new LastValueFieldDownsampler(randomAlphanumericOfLength(10), null, null); assertThat(lastValueFieldProducer.lastValue(), nullValue()); var docIdBuffer = IntArrayList.from(0, 1, 2); var values = createValuesInstance(docIdBuffer, new Long[] { 10L, 17L, 12L }); @@ -71,7 +77,7 @@ public void testLastValueLong() throws IOException { } public void testLastValueBoolean() throws IOException { - LastValueFieldProducer lastValueFieldProducer = new LastValueFieldProducer(randomAlphanumericOfLength(10)); + LastValueFieldDownsampler lastValueFieldProducer = new LastValueFieldDownsampler(randomAlphanumericOfLength(10), null, null); assertThat(lastValueFieldProducer.lastValue(), nullValue()); var docIdBuffer = IntArrayList.from(0, 1, 2); var values = createValuesInstance(docIdBuffer, new Boolean[] { true, false, false }); @@ -105,7 +111,7 @@ public Object nextValue() { }; values.iterator = Arrays.stream(multiValue).iterator(); - LastValueFieldProducer multiLastValueProducer = new LastValueFieldProducer(randomAlphanumericOfLength(10)); + LastValueFieldDownsampler multiLastValueProducer = new LastValueFieldDownsampler(randomAlphanumericOfLength(10), null, null); assertThat(multiLastValueProducer.lastValue(), nullValue()); multiLastValueProducer.collect(values, docIdBuffer); assertThat(multiLastValueProducer.lastValue(), equalTo(multiValue)); @@ -115,8 +121,8 @@ public Object nextValue() { assertThat(multiLastValueProducer.lastValue(), nullValue()); } - public void testFlattenedLastValueFieldProducer() throws IOException { - var producer = LastValueFieldProducer.create("dummy", "flattened"); + public void testFlattenedLastValueFieldDownsampler() throws IOException { + var producer = new LastValueFieldDownsampler("dummy", createDummyFlattenedFieldType(), null); assertTrue(producer.isEmpty()); assertEquals("dummy", producer.name()); @@ -184,4 +190,23 @@ public int docValueCount() { } }; } + + private static MappedFieldType createDummyFlattenedFieldType() { + return new MappedFieldType("flattened", IndexType.NONE, false, Map.of()) { + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return null; + } + + @Override + public String typeName() { + return ""; + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + return null; + } + }; + } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsamplerTests.java similarity index 89% rename from x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducerTests.java rename to x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsamplerTests.java index 204dc50de73b5..87e5003bc7a96 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsamplerTests.java @@ -17,10 +17,10 @@ import java.io.IOException; -public class NumericMetricFieldProducerTests extends AggregatorTestCase { +public class NumericMetricFieldDownsamplerTests extends AggregatorTestCase { public void testMinCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldProducer.AggregateGauge(randomAlphaOfLength(10)); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); assertEquals(Double.MAX_VALUE, aggregateMetricFieldProducer.min, 0); var docIdBuffer = IntArrayList.from(0, 1, 2, 3); var numericValues = createNumericValuesInstance(docIdBuffer, 40, 5.5, 12.2, 55); @@ -29,7 +29,7 @@ public void testMinCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(Double.MAX_VALUE, aggregateMetricFieldProducer.min, 0); - var lastValueProducer = new NumericMetricFieldProducer.LastValue(randomAlphaOfLength(10)); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2, 3); var values = createNumericValuesInstance(docIdBuffer, 40D, 5.5, 12.2, 55D); @@ -41,7 +41,7 @@ public void testMinCountMetric() throws IOException { } public void testMaxCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldProducer.AggregateGauge(randomAlphaOfLength(10)); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); assertEquals(-Double.MAX_VALUE, aggregateMetricFieldProducer.max, 0); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55); @@ -50,7 +50,7 @@ public void testMaxCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(-Double.MAX_VALUE, aggregateMetricFieldProducer.max, 0); - var lastValueProducer = new NumericMetricFieldProducer.LastValue(randomAlphaOfLength(10)); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2); var values = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55D); @@ -62,7 +62,7 @@ public void testMaxCountMetric() throws IOException { } public void testSumCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldProducer.AggregateGauge(randomAlphaOfLength(10)); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); assertEquals(0, aggregateMetricFieldProducer.sum.value(), 0); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55); @@ -71,7 +71,7 @@ public void testSumCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(0, aggregateMetricFieldProducer.sum.value(), 0); - var lastValueProducer = new NumericMetricFieldProducer.LastValue(randomAlphaOfLength(10)); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2); var values = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55D); @@ -87,7 +87,7 @@ public void testSumCountMetric() throws IOException { * Tests stolen from SumAggregatorTests#testSummationAccuracy */ public void testSummationAccuracy() throws IOException { - var instance = new NumericMetricFieldProducer.AggregateGauge(randomAlphaOfLength(10)); + var instance = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); assertEquals(0, instance.sum.value(), 0); var docIdBuffer = IntArrayList.from(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // Summing up a normal array and expect an accurate value @@ -159,7 +159,7 @@ public void testSummationAccuracy() throws IOException { } public void testValueCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldProducer.AggregateGauge(randomAlphaOfLength(10)); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); assertEquals(0, aggregateMetricFieldProducer.count); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 40, 30, 20); @@ -168,7 +168,7 @@ public void testValueCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(0, aggregateMetricFieldProducer.count); - var lastValueProducer = new NumericMetricFieldProducer.LastValue(randomAlphaOfLength(10)); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); docIdBuffer = IntArrayList.from(0, 1, 2); var values = createNumericValuesInstance(docIdBuffer, 40, 30, 20); lastValueProducer.collect(values, docIdBuffer); @@ -180,7 +180,7 @@ public void testValueCountMetric() throws IOException { public void testCounterMetricFieldProducer() throws IOException { final String field = "field"; - var producer = new NumericMetricFieldProducer.LastValue(field); + var producer = new NumericMetricFieldDownsampler.LastValue(field, null, null); assertTrue(producer.isEmpty()); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); @@ -200,7 +200,7 @@ public void testCounterMetricFieldProducer() throws IOException { public void testGaugeMetricFieldProducer() throws IOException { final String field = "field"; - NumericMetricFieldProducer producer = new NumericMetricFieldProducer.AggregateGauge(field); + NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.AggregateGauge(field, null, null); assertTrue(producer.isEmpty()); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsamplerTests.java similarity index 91% rename from x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducerTests.java rename to x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsamplerTests.java index cf1e1a6f9a56f..d0c1f12709707 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsamplerTests.java @@ -22,10 +22,10 @@ import static org.hamcrest.Matchers.equalTo; -public class TDigestHistogramFieldProducerTests extends ESTestCase { +public class TDigestHistogramFieldDownsamplerTests extends ESTestCase { public void testLastValueProducer() throws IOException { - var producer = TDigestHistogramFieldProducer.create("my-histogram", DownsampleConfig.SamplingMethod.LAST_VALUE); + var producer = TDigestHistogramFieldDownsampler.create("my-histogram", null, null, DownsampleConfig.SamplingMethod.LAST_VALUE); assertTrue(producer.isEmpty()); assertEquals("my-histogram", producer.name()); @@ -47,7 +47,7 @@ public void testLastValueProducer() throws IOException { } public void testAggregateProducer() throws IOException { - var producer = TDigestHistogramFieldProducer.create("my-histogram", DownsampleConfig.SamplingMethod.AGGREGATE); + var producer = TDigestHistogramFieldDownsampler.create("my-histogram", null, null, DownsampleConfig.SamplingMethod.AGGREGATE); assertTrue(producer.isEmpty()); assertEquals("my-histogram", producer.name()); From 80a52c2bb0866e76bc45599bd28ab46406ceeb9b Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 8 Jan 2026 14:27:15 +0200 Subject: [PATCH 03/11] Fix tests --- ...egateMetricDoubleFieldSerializerTests.java | 10 +++---- .../LastValueFieldDownsamplerTests.java | 30 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java index 1d8935656990e..fd51852b960c7 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java @@ -39,22 +39,22 @@ public void testAggregatedGaugeFieldSerialization() throws IOException { XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); builder.humanReadable(true).startObject(); IllegalStateException error = expectThrows(IllegalStateException.class, () -> aggregateMetricProducerSerializer.write(builder)); - assertThat(error.getMessage(), equalTo("Unexpected field producer class: AggregateGauge for my-gauge field")); + assertThat(error.getMessage(), equalTo("Unexpected field downsampler class: AggregateGauge for my-gauge field")); } public void testInvalidCounterFieldSerialization() throws IOException { - NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.LastValue("my-counter", null, null); + NumericMetricFieldDownsampler downsampler = new NumericMetricFieldDownsampler.LastValue("my-counter", null, null); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55, 12, 5); - producer.collect(valuesInstance, docIdBuffer); + downsampler.collect(valuesInstance, docIdBuffer); AggregateMetricDoubleFieldDownsampler.Serializer gaugeFieldSerializer = new AggregateMetricDoubleFieldDownsampler.Serializer( "my-counter", - List.of(producer) + List.of(downsampler) ); XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); builder.humanReadable(true).startObject(); IllegalStateException error = expectThrows(IllegalStateException.class, () -> gaugeFieldSerializer.write(builder)); - assertThat(error.getMessage(), equalTo("Unexpected field producer class: LastValue for my-counter field")); + assertThat(error.getMessage(), equalTo("Unexpected field downsampler class: LastValue for my-counter field")); } public void testAggregateMetricDoubleFieldSerialization() throws IOException { diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsamplerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsamplerTests.java index d303dce6d884c..9eb6d6948021c 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsamplerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsamplerTests.java @@ -122,9 +122,9 @@ public Object nextValue() { } public void testFlattenedLastValueFieldDownsampler() throws IOException { - var producer = new LastValueFieldDownsampler("dummy", createDummyFlattenedFieldType(), null); - assertTrue(producer.isEmpty()); - assertEquals("dummy", producer.name()); + var downsampler = LastValueFieldDownsampler.create("dummy", createDummyFlattenedFieldType(), null); + assertTrue(downsampler.isEmpty()); + assertEquals("dummy", downsampler.name()); var bytes = List.of("a\0value_a", "b\0value_b", "c\0value_c", "d\0value_d"); var docValues = new FormattedDocValues() { @@ -147,23 +147,23 @@ public Object nextValue() { } }; - producer.collect(docValues, IntArrayList.from(1)); - assertFalse(producer.isEmpty()); - assertEquals("a\0value_a", (((Object[]) producer.lastValue())[0]).toString()); - assertEquals("b\0value_b", (((Object[]) producer.lastValue())[1]).toString()); - assertEquals("c\0value_c", (((Object[]) producer.lastValue())[2]).toString()); - assertEquals("d\0value_d", (((Object[]) producer.lastValue())[3]).toString()); + downsampler.collect(docValues, IntArrayList.from(1)); + assertFalse(downsampler.isEmpty()); + assertEquals("a\0value_a", (((Object[]) downsampler.lastValue())[0]).toString()); + assertEquals("b\0value_b", (((Object[]) downsampler.lastValue())[1]).toString()); + assertEquals("c\0value_c", (((Object[]) downsampler.lastValue())[2]).toString()); + assertEquals("d\0value_d", (((Object[]) downsampler.lastValue())[3]).toString()); var builder = new XContentBuilder(XContentType.JSON.xContent(), new ByteArrayOutputStream()); builder.startObject(); - producer.write(builder); + downsampler.write(builder); builder.endObject(); var content = Strings.toString(builder); assertThat(content, equalTo("{\"dummy\":{\"a\":\"value_a\",\"b\":\"value_b\",\"c\":\"value_c\",\"d\":\"value_d\"}}")); - producer.reset(); - assertTrue(producer.isEmpty()); - assertNull(producer.lastValue()); + downsampler.reset(); + assertTrue(downsampler.isEmpty()); + assertNull(downsampler.lastValue()); } static FormattedDocValues createValuesInstance(IntArrayList docIdBuffer, T[] values) { @@ -192,7 +192,7 @@ public int docValueCount() { } private static MappedFieldType createDummyFlattenedFieldType() { - return new MappedFieldType("flattened", IndexType.NONE, false, Map.of()) { + return new MappedFieldType("dummy", IndexType.NONE, false, Map.of()) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { return null; @@ -200,7 +200,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public String typeName() { - return ""; + return "flattened"; } @Override From 9922ba35076d515927904d136e5acabf44ca3cea Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 10 Feb 2026 16:54:06 +0200 Subject: [PATCH 04/11] Remove unnecessary constructor --- .../downsample/TDigestHistogramFieldDownsampler.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java index 66e2ba25181de..cd58c2f77e77b 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java @@ -74,14 +74,9 @@ private static class Aggregate extends TDigestHistogramFieldDownsampler { private TDigestState tDigestState = null; Aggregate(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - this(name, fieldType, fieldData, DEFAULT_TYPE, DEFAULT_COMPRESSION); - - } - - Aggregate(String name, MappedFieldType fieldType, IndexFieldData fieldData, TDigestState.Type type, double compression) { super(name, fieldType, fieldData); - this.type = type; - this.compression = compression; + this.type = DEFAULT_TYPE; + this.compression = DEFAULT_COMPRESSION; } public void collect(HistogramValues docValues, IntArrayList docIdBuffer) throws IOException { From 50a030091f5aba4efbc3340148fdf6e89c10faf7 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 11 Feb 2026 13:11:44 +0200 Subject: [PATCH 05/11] Remove value fetcher --- .../downsample/AbstractFieldDownsampler.java | 48 ++++++------------- ...AggregateMetricDoubleFieldDownsampler.java | 48 ++++--------------- .../ExponentialHistogramFieldDownsampler.java | 43 ++++++----------- .../downsample/LastValueFieldDownsampler.java | 25 ++++------ .../NumericMetricFieldDownsampler.java | 32 +++++-------- .../TDigestHistogramFieldDownsampler.java | 31 ++++-------- ...egateMetricDoubleFieldSerializerTests.java | 10 ++-- .../NumericMetricFieldDownsamplerTests.java | 22 ++++----- 8 files changed, 82 insertions(+), 177 deletions(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java index 3b0d959aad653..766261fff5038 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java @@ -34,12 +34,12 @@ abstract class AbstractFieldDownsampler implements DownsampleFieldSerializer private final String name; protected boolean isEmpty; - private final FieldValueFetcher fieldValueFetcher; + protected final IndexFieldData fieldData; - AbstractFieldDownsampler(String name, FieldValueFetcher fieldValueFetcher) { + AbstractFieldDownsampler(String name, IndexFieldData fieldData) { this.name = name; this.isEmpty = true; - this.fieldValueFetcher = fieldValueFetcher; + this.fieldData = fieldData; } /** @@ -50,7 +50,7 @@ public String name() { } /** - * Resets the downsampler to an empty value. + * Resets the downsampler to an empty value. The downsampler should be reset before downsampling a field for a new bucket. */ public abstract void reset(); @@ -62,37 +62,17 @@ public boolean isEmpty() { } /** - * @return the leaf reader that will retrieve the values per doc for this field. + * @return the leaf reader that will retrieve the doc values for this field. */ - public T getLeaf(LeafReaderContext context) throws IOException { - return fieldValueFetcher.getLeaf(context); - } - - public abstract void collect(T docValues, IntArrayList docIdBuffer) throws IOException; + public abstract T getLeaf(LeafReaderContext context) throws IOException; /** - * Utility class used for fetching field values by reading field data. - * For fields whose type is multivalued the 'name' matches the parent field - * name (normally used for indexing data), while the actual multiField - * name is accessible by means of {@link MappedFieldType#name()}. + * Collects the values for this field of the doc ids requested. + * @param docValues the doc values for this field + * @param docIdBuffer the doc ids for which we need to retrieve the field values + * @throws IOException */ - abstract static class FieldValueFetcher { - protected final String name; - protected final MappedFieldType fieldType; - protected final IndexFieldData fieldData; - - FieldValueFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - this.name = name; - this.fieldType = fieldType; - this.fieldData = fieldData; - } - - String name() { - return name; - } - - abstract T getLeaf(LeafReaderContext context) throws IOException; - } + public abstract void collect(T docValues, IntArrayList docIdBuffer) throws IOException; /** * Retrieve field value fetchers for a list of fields. @@ -140,7 +120,7 @@ private static AbstractFieldDownsampler create( return TDigestHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); } if (ExponentialHistogramFieldDownsampler.TYPE.equals(fieldType.typeName())) { - return ExponentialHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); + return ExponentialHistogramFieldDownsampler.create(fieldName, fieldData, samplingMethod); } if (fieldType.getMetricType() != null) { // TODO: Support POSITION in downsampling @@ -152,9 +132,9 @@ private static AbstractFieldDownsampler create( : "only gauges and counters accepted, other metrics should have been handled earlier"; if (samplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE && fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE) { - return new NumericMetricFieldDownsampler.AggregateGauge(fieldName, fieldType, fieldData); + return new NumericMetricFieldDownsampler.AggregateGauge(fieldName, fieldData); } - return new NumericMetricFieldDownsampler.LastValue(fieldName, fieldType, fieldData); + return new NumericMetricFieldDownsampler.LastValue(fieldName, fieldData); } else { // If a field is not a metric, we downsample it as a label return LastValueFieldDownsampler.create(fieldName, fieldType, fieldData); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java index f778c9c267f7d..2737cbcc3e4cb 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java @@ -31,13 +31,8 @@ abstract sealed class AggregateMetricDoubleFieldDownsampler extends NumericMetri protected final AggregateMetricDoubleFieldMapper.Metric metric; - AggregateMetricDoubleFieldDownsampler( - String name, - AggregateMetricDoubleFieldMapper.Metric metric, - MappedFieldType fieldType, - IndexFieldData fieldData - ) { - super(name, fieldType, fieldData); + AggregateMetricDoubleFieldDownsampler(String name, AggregateMetricDoubleFieldMapper.Metric metric, IndexFieldData fieldData) { + super(name, fieldData); this.metric = metric; } @@ -48,8 +43,8 @@ static final class Aggregate extends AggregateMetricDoubleFieldDownsampler { private final CompensatedSum sum = new CompensatedSum(); private long count; - Aggregate(String name, AggregateMetricDoubleFieldMapper.Metric metric, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, metric, fieldType, fieldData); + Aggregate(String name, AggregateMetricDoubleFieldMapper.Metric metric, IndexFieldData fieldData) { + super(name, metric, fieldData); } @Override @@ -105,14 +100,8 @@ static final class LastValue extends AggregateMetricDoubleFieldDownsampler { private final boolean supportsMultiValue; private Object lastValue = null; - LastValue( - String name, - AggregateMetricDoubleFieldMapper.Metric metric, - MappedFieldType fieldType, - IndexFieldData fieldData, - boolean supportsMultiValue - ) { - super(name, metric, fieldType, fieldData); + LastValue(String name, AggregateMetricDoubleFieldMapper.Metric metric, IndexFieldData fieldData, boolean supportsMultiValue) { + super(name, metric, fieldData); this.supportsMultiValue = supportsMultiValue; } @@ -227,35 +216,16 @@ static List create( if (samplingMethod != DownsampleConfig.SamplingMethod.LAST_VALUE) { // If the field is an aggregate_metric_double field, we should use the correct subfields // for each aggregation. This is a downsample-of-downsample case - downsamplers.add( - new AggregateMetricDoubleFieldDownsampler.Aggregate( - aggMetricFieldType.name(), - metric, - metricSubField, - fieldData - ) - ); + downsamplers.add(new AggregateMetricDoubleFieldDownsampler.Aggregate(aggMetricFieldType.name(), metric, fieldData)); } else { downsamplers.add( - new AggregateMetricDoubleFieldDownsampler.LastValue( - aggMetricFieldType.name(), - metric, - metricSubField, - fieldData, - false - ) + new AggregateMetricDoubleFieldDownsampler.LastValue(aggMetricFieldType.name(), metric, fieldData, false) ); } } else { // If a field is not a metric, we downsample it as a label downsamplers.add( - new AggregateMetricDoubleFieldDownsampler.LastValue( - aggMetricFieldType.name(), - metric, - metricSubField, - fieldData, - true - ) + new AggregateMetricDoubleFieldDownsampler.LastValue(aggMetricFieldType.name(), metric, fieldData, true) ); } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java index 62884f87db996..8c4ad7f4ea059 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java @@ -15,7 +15,6 @@ import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger; import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.LeafExponentialHistogramFieldData; @@ -28,22 +27,17 @@ abstract class ExponentialHistogramFieldDownsampler extends AbstractFieldDownsampler { static final String TYPE = "exponential_histogram"; - ExponentialHistogramFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, new ExponentialHistogramFieldFetcher(name, fieldType, fieldData)); + ExponentialHistogramFieldDownsampler(String name, IndexFieldData fieldData) { + super(name, fieldData); } /** * @return the requested producer based on the sampling method for an exponential histogram field */ - static AbstractFieldDownsampler create( - String name, - MappedFieldType fieldType, - IndexFieldData fieldData, - DownsampleConfig.SamplingMethod samplingMethod - ) { + static AbstractFieldDownsampler create(String name, IndexFieldData fieldData, DownsampleConfig.SamplingMethod samplingMethod) { return switch (samplingMethod) { - case AGGREGATE -> new ExponentialHistogramFieldDownsampler.MergeProducer(name, fieldType, fieldData); - case LAST_VALUE -> new ExponentialHistogramFieldDownsampler.LastValueProducer(name, fieldType, fieldData); + case AGGREGATE -> new ExponentialHistogramFieldDownsampler.MergeProducer(name, fieldData); + case LAST_VALUE -> new ExponentialHistogramFieldDownsampler.LastValueProducer(name, fieldData); }; } @@ -57,14 +51,20 @@ public void write(XContentBuilder builder) throws IOException { } } + @Override + public ExponentialHistogramValuesReader getLeaf(LeafReaderContext context) throws IOException { + LeafExponentialHistogramFieldData exponentialHistogramFieldData = (LeafExponentialHistogramFieldData) fieldData.load(context); + return exponentialHistogramFieldData.getHistogramValues(); + } + /** * Downsamples an exponential histogram by merging all values. */ static class MergeProducer extends ExponentialHistogramFieldDownsampler { private ExponentialHistogramMerger merger = null; - MergeProducer(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); + MergeProducer(String name, IndexFieldData fieldData) { + super(name, fieldData); } @Override @@ -107,8 +107,8 @@ public void collect(ExponentialHistogramValuesReader docValues, IntArrayList doc static class LastValueProducer extends ExponentialHistogramFieldDownsampler { private ExponentialHistogram lastValue = null; - LastValueProducer(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); + LastValueProducer(String name, IndexFieldData fieldData) { + super(name, fieldData); } @Override @@ -139,17 +139,4 @@ protected ExponentialHistogram downsampledValue() { return lastValue; } } - - static class ExponentialHistogramFieldFetcher extends AbstractFieldDownsampler.FieldValueFetcher { - - ExponentialHistogramFieldFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); - } - - @Override - ExponentialHistogramValuesReader getLeaf(LeafReaderContext context) throws IOException { - LeafExponentialHistogramFieldData exponentialHistogramFieldData = (LeafExponentialHistogramFieldData) fieldData.load(context); - return exponentialHistogramFieldData.getHistogramValues(); - } - } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java index 6a84f9e394f02..bea9f091cb02c 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java @@ -30,10 +30,13 @@ * Important note: This class assumes that field values are collected and sorted by descending order by time */ class LastValueFieldDownsampler extends AbstractFieldDownsampler { + + private final MappedFieldType fieldType; Object lastValue = null; LastValueFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, new FormattedDocValueFetcher(name, fieldType, fieldData)); + super(name, fieldData); + this.fieldType = fieldType; } /** @@ -60,6 +63,12 @@ public void reset() { lastValue = null; } + @Override + public FormattedDocValues getLeaf(LeafReaderContext context) { + DocValueFormat format = fieldType.docValueFormat(null, null); + return fieldData.load(context).getFormattedValues(format); + } + /** * Collects the last value observed in these field values. This implementation assumes that field values are collected * and sorted by descending order by time. In this case, it assumes that the last value of the time is the first value @@ -141,18 +150,4 @@ public void write(XContentBuilder builder) throws IOException { } } } - - static class FormattedDocValueFetcher extends AbstractFieldDownsampler.FieldValueFetcher { - - FormattedDocValueFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); - } - - @Override - FormattedDocValues getLeaf(LeafReaderContext context) { - DocValueFormat format = fieldType.docValueFormat(null, null); - return fieldData.load(context).getFormattedValues(format); - } - } - } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java index 50de26896c6f8..6e99b826ed492 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java @@ -12,7 +12,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.metrics.CompensatedSum; import org.elasticsearch.xcontent.XContentBuilder; @@ -26,8 +25,14 @@ abstract sealed class NumericMetricFieldDownsampler extends AbstractFieldDownsampler permits AggregateMetricDoubleFieldDownsampler, NumericMetricFieldDownsampler.AggregateGauge, NumericMetricFieldDownsampler.LastValue { - NumericMetricFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, new NumericFieldFetcher(name, fieldType, fieldData)); + NumericMetricFieldDownsampler(String name, IndexFieldData fieldData) { + super(name, fieldData); + } + + @Override + public SortedNumericDoubleValues getLeaf(LeafReaderContext context) { + LeafNumericFieldData numericFieldData = (LeafNumericFieldData) fieldData.load(context); + return numericFieldData.getDoubleValues(); } static final double MAX_NO_VALUE = -Double.MAX_VALUE; @@ -43,8 +48,8 @@ static final class AggregateGauge extends NumericMetricFieldDownsampler { final CompensatedSum sum = new CompensatedSum(); long count; - AggregateGauge(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); + AggregateGauge(String name, IndexFieldData fieldData) { + super(name, fieldData); } @Override @@ -96,8 +101,8 @@ static final class LastValue extends NumericMetricFieldDownsampler { double lastValue = Double.NaN; - LastValue(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); + LastValue(String name, IndexFieldData fieldData) { + super(name, fieldData); } @Override @@ -136,17 +141,4 @@ public void write(XContentBuilder builder) throws IOException { } } } - - static class NumericFieldFetcher extends AbstractFieldDownsampler.FieldValueFetcher { - - NumericFieldFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); - } - - @Override - SortedNumericDoubleValues getLeaf(LeafReaderContext context) { - LeafNumericFieldData numericFieldData = (LeafNumericFieldData) fieldData.load(context); - return numericFieldData.getDoubleValues(); - } - } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java index cd58c2f77e77b..c67ff8ac2469d 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java @@ -35,21 +35,19 @@ abstract class TDigestHistogramFieldDownsampler extends AbstractFieldDownsampler static final double DEFAULT_COMPRESSION = 100; // MergingDigest is the best fit because we have pre-constructed histograms static final TDigestState.Type DEFAULT_TYPE = TDigestState.Type.MERGING; - static final String VALUES_FIELD = "values"; - static final String CENTROIDS_FIELD = "centroids"; + private static final String VALUES_FIELD = "values"; + private static final String CENTROIDS_FIELD = "centroids"; protected final String valueLabel; TDigestHistogramFieldDownsampler(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, new TDigestHistogramFieldFetcher(name, fieldType, fieldData)); - valueLabel = getValueLabel(fieldType); + super(name, fieldData); + valueLabel = TDigestFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) ? CENTROIDS_FIELD : VALUES_FIELD; } - private static String getValueLabel(MappedFieldType fieldType) { - if (TDigestFieldMapper.CONTENT_TYPE.equals(fieldType.typeName())) { - return CENTROIDS_FIELD; - } else { - return VALUES_FIELD; - } + @Override + public HistogramValues getLeaf(LeafReaderContext context) throws IOException { + LeafHistogramFieldData histogramFieldData = (LeafHistogramFieldData) fieldData.load(context); + return histogramFieldData.getHistogramValues(); } /** @@ -163,17 +161,4 @@ public void write(XContentBuilder builder) throws IOException { } } } - - static class TDigestHistogramFieldFetcher extends AbstractFieldDownsampler.FieldValueFetcher { - - TDigestHistogramFieldFetcher(String name, MappedFieldType fieldType, IndexFieldData fieldData) { - super(name, fieldType, fieldData); - } - - @Override - HistogramValues getLeaf(LeafReaderContext context) throws IOException { - LeafHistogramFieldData histogramFieldData = (LeafHistogramFieldData) fieldData.load(context); - return histogramFieldData.getHistogramValues(); - } - } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java index fd51852b960c7..1d8ca8134a547 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldSerializerTests.java @@ -23,7 +23,7 @@ public class AggregateMetricDoubleFieldSerializerTests extends ESTestCase { public void testAggregatedGaugeFieldSerialization() throws IOException { - NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.AggregateGauge("my-gauge", null, null); + NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.AggregateGauge("my-gauge", null); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); producer.collect(valuesInstance, docIdBuffer); @@ -43,7 +43,7 @@ public void testAggregatedGaugeFieldSerialization() throws IOException { } public void testInvalidCounterFieldSerialization() throws IOException { - NumericMetricFieldDownsampler downsampler = new NumericMetricFieldDownsampler.LastValue("my-counter", null, null); + NumericMetricFieldDownsampler downsampler = new NumericMetricFieldDownsampler.LastValue("my-counter", null); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55, 12, 5); downsampler.collect(valuesInstance, docIdBuffer); @@ -61,7 +61,6 @@ public void testAggregateMetricDoubleFieldSerialization() throws IOException { AggregateMetricDoubleFieldDownsampler minProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.min, - null, null ); var docIdBuffer = IntArrayList.from(0, 1); @@ -70,7 +69,6 @@ public void testAggregateMetricDoubleFieldSerialization() throws IOException { AggregateMetricDoubleFieldDownsampler maxProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.max, - null, null ); docIdBuffer = IntArrayList.from(0, 1); @@ -79,7 +77,6 @@ public void testAggregateMetricDoubleFieldSerialization() throws IOException { AggregateMetricDoubleFieldDownsampler sumProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.sum, - null, null ); docIdBuffer = IntArrayList.from(0, 1); @@ -88,7 +85,6 @@ public void testAggregateMetricDoubleFieldSerialization() throws IOException { AggregateMetricDoubleFieldDownsampler countProducer = new AggregateMetricDoubleFieldDownsampler.Aggregate( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.value_count, - null, null ); docIdBuffer = IntArrayList.from(0, 1); @@ -154,6 +150,6 @@ private AggregateMetricDoubleFieldDownsampler randomAggregateSubMetricFieldProdu String name, AggregateMetricDoubleFieldMapper.Metric metric ) { - return new AggregateMetricDoubleFieldDownsampler.LastValue(name, metric, null, null, randomBoolean()); + return new AggregateMetricDoubleFieldDownsampler.LastValue(name, metric, null, randomBoolean()); } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsamplerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsamplerTests.java index 87e5003bc7a96..2b2d8f68d37f8 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsamplerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsamplerTests.java @@ -20,7 +20,7 @@ public class NumericMetricFieldDownsamplerTests extends AggregatorTestCase { public void testMinCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null); assertEquals(Double.MAX_VALUE, aggregateMetricFieldProducer.min, 0); var docIdBuffer = IntArrayList.from(0, 1, 2, 3); var numericValues = createNumericValuesInstance(docIdBuffer, 40, 5.5, 12.2, 55); @@ -29,7 +29,7 @@ public void testMinCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(Double.MAX_VALUE, aggregateMetricFieldProducer.min, 0); - var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null); assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2, 3); var values = createNumericValuesInstance(docIdBuffer, 40D, 5.5, 12.2, 55D); @@ -41,7 +41,7 @@ public void testMinCountMetric() throws IOException { } public void testMaxCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null); assertEquals(-Double.MAX_VALUE, aggregateMetricFieldProducer.max, 0); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55); @@ -50,7 +50,7 @@ public void testMaxCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(-Double.MAX_VALUE, aggregateMetricFieldProducer.max, 0); - var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null); assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2); var values = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55D); @@ -62,7 +62,7 @@ public void testMaxCountMetric() throws IOException { } public void testSumCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null); assertEquals(0, aggregateMetricFieldProducer.sum.value(), 0); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55); @@ -71,7 +71,7 @@ public void testSumCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(0, aggregateMetricFieldProducer.sum.value(), 0); - var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null); assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2); var values = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55D); @@ -87,7 +87,7 @@ public void testSumCountMetric() throws IOException { * Tests stolen from SumAggregatorTests#testSummationAccuracy */ public void testSummationAccuracy() throws IOException { - var instance = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); + var instance = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null); assertEquals(0, instance.sum.value(), 0); var docIdBuffer = IntArrayList.from(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // Summing up a normal array and expect an accurate value @@ -159,7 +159,7 @@ public void testSummationAccuracy() throws IOException { } public void testValueCountMetric() throws IOException { - var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null, null); + var aggregateMetricFieldProducer = new NumericMetricFieldDownsampler.AggregateGauge(randomAlphaOfLength(10), null); assertEquals(0, aggregateMetricFieldProducer.count); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 40, 30, 20); @@ -168,7 +168,7 @@ public void testValueCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(0, aggregateMetricFieldProducer.count); - var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null, null); + var lastValueProducer = new NumericMetricFieldDownsampler.LastValue(randomAlphaOfLength(10), null); docIdBuffer = IntArrayList.from(0, 1, 2); var values = createNumericValuesInstance(docIdBuffer, 40, 30, 20); lastValueProducer.collect(values, docIdBuffer); @@ -180,7 +180,7 @@ public void testValueCountMetric() throws IOException { public void testCounterMetricFieldProducer() throws IOException { final String field = "field"; - var producer = new NumericMetricFieldDownsampler.LastValue(field, null, null); + var producer = new NumericMetricFieldDownsampler.LastValue(field, null); assertTrue(producer.isEmpty()); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); @@ -200,7 +200,7 @@ public void testCounterMetricFieldProducer() throws IOException { public void testGaugeMetricFieldProducer() throws IOException { final String field = "field"; - NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.AggregateGauge(field, null, null); + NumericMetricFieldDownsampler producer = new NumericMetricFieldDownsampler.AggregateGauge(field, null); assertTrue(producer.isEmpty()); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); From 4491391e81367b78bbe6d29a3929662c6ce9065b Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 11 Feb 2026 15:27:41 +0200 Subject: [PATCH 06/11] Polish --- .../downsample/AbstractFieldDownsampler.java | 44 +++++++------------ ...AggregateMetricDoubleFieldDownsampler.java | 4 ++ .../ExponentialHistogramFieldDownsampler.java | 6 +++ .../downsample/LastValueFieldDownsampler.java | 16 ++----- .../NumericMetricFieldDownsampler.java | 22 ++++++++++ .../TDigestHistogramFieldDownsampler.java | 6 +++ 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java index 766261fff5038..af3936bbb3183 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java @@ -12,12 +12,9 @@ import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; -import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper; import java.io.IOException; import java.util.ArrayList; @@ -83,14 +80,14 @@ static List> create( Map multiFieldSources, DownsampleConfig.SamplingMethod samplingMethod ) { - List> fetchers = new ArrayList<>(); + List> downsamplers = new ArrayList<>(); for (String field : fields) { String sourceField = multiFieldSources.getOrDefault(field, field); MappedFieldType fieldType = context.getFieldType(sourceField); assert fieldType != null : "Unknown field type for field: [" + sourceField + "]"; if (fieldType instanceof AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aggMetricFieldType) { - fetchers.addAll(AggregateMetricDoubleFieldDownsampler.create(context, aggMetricFieldType, samplingMethod)); + downsamplers.addAll(AggregateMetricDoubleFieldDownsampler.create(context, aggMetricFieldType, samplingMethod)); } else { if (context.fieldExistsInIndex(field)) { final IndexFieldData fieldData; @@ -100,11 +97,11 @@ static List> create( } else { fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } - fetchers.add(create(field, fieldType, fieldData, samplingMethod)); + downsamplers.add(create(field, fieldType, fieldData, samplingMethod)); } } } - return Collections.unmodifiableList(fetchers); + return Collections.unmodifiableList(downsamplers); } private static AbstractFieldDownsampler create( @@ -113,31 +110,22 @@ private static AbstractFieldDownsampler create( IndexFieldData fieldData, DownsampleConfig.SamplingMethod samplingMethod ) { - assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false - : "Aggregate metric double should be handled by a dedicated FieldValueFetcher"; - if (HistogramFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) - || TDigestFieldMapper.CONTENT_TYPE.equals(fieldType.typeName())) { + assert AggregateMetricDoubleFieldDownsampler.supportsFieldType(fieldType) + : "Aggregate metric double should be handled by a dedicated downsampler"; + if (TDigestHistogramFieldDownsampler.supportsFieldType(fieldType)) { return TDigestHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); } - if (ExponentialHistogramFieldDownsampler.TYPE.equals(fieldType.typeName())) { + if (ExponentialHistogramFieldDownsampler.supportsFieldType(fieldType)) { return ExponentialHistogramFieldDownsampler.create(fieldName, fieldData, samplingMethod); } - if (fieldType.getMetricType() != null) { - // TODO: Support POSITION in downsampling - if (fieldType.getMetricType() == POSITION) { - throw new IllegalArgumentException("Unsupported metric type [position] for down-sampling"); - } - assert fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE - || fieldType.getMetricType() == TimeSeriesParams.MetricType.COUNTER - : "only gauges and counters accepted, other metrics should have been handled earlier"; - if (samplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE - && fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE) { - return new NumericMetricFieldDownsampler.AggregateGauge(fieldName, fieldData); - } - return new NumericMetricFieldDownsampler.LastValue(fieldName, fieldData); - } else { - // If a field is not a metric, we downsample it as a label - return LastValueFieldDownsampler.create(fieldName, fieldType, fieldData); + if (NumericMetricFieldDownsampler.supportsFieldType(fieldType)) { + return NumericMetricFieldDownsampler.create(fieldName, fieldType.getMetricType(), fieldData, samplingMethod); + } + // TODO: Support POSITION in downsampling + if (fieldType.getMetricType() == POSITION) { + throw new IllegalArgumentException("Unsupported metric type [position] for downsampling"); } + // If a field is not a metric, we downsample it as a label + return LastValueFieldDownsampler.create(fieldName, fieldType, fieldData); } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java index 2737cbcc3e4cb..eb2afa8ff3c2f 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java @@ -36,6 +36,10 @@ abstract sealed class AggregateMetricDoubleFieldDownsampler extends NumericMetri this.metric = metric; } + public static boolean supportsFieldType(MappedFieldType fieldType) { + return AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()); + } + static final class Aggregate extends AggregateMetricDoubleFieldDownsampler { private double max = MAX_NO_VALUE; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java index 8c4ad7f4ea059..c68b6a145c76a 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java @@ -15,9 +15,11 @@ import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger; import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.LeafExponentialHistogramFieldData; +import org.elasticsearch.xpack.exponentialhistogram.ExponentialHistogramFieldMapper; import java.io.IOException; @@ -43,6 +45,10 @@ static AbstractFieldDownsampler create(String name, IndexFieldData fieldDa protected abstract ExponentialHistogram downsampledValue(); + public static boolean supportsFieldType(MappedFieldType fieldType) { + return ExponentialHistogramFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()); + } + @Override public void write(XContentBuilder builder) throws IOException { if (isEmpty() == false) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java index bea9f091cb02c..ea9b593e41723 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldDownsampler.java @@ -16,10 +16,6 @@ import org.elasticsearch.index.mapper.flattened.FlattenedFieldSyntheticWriterHelper; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; -import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; -import org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper; -import org.elasticsearch.xpack.exponentialhistogram.ExponentialHistogramFieldMapper; import java.io.IOException; import java.util.ArrayList; @@ -43,14 +39,10 @@ class LastValueFieldDownsampler extends AbstractFieldDownsampler fieldData) { - assert AggregateMetricDoubleFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false - : "field type cannot be aggregate metric double: " + fieldType.typeName() + " for field " + name; - assert ExponentialHistogramFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false - : "field type cannot be exponential histogram: " + fieldType.typeName() + " for field " + name; - assert HistogramFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false - : "field type cannot be histogram: " + fieldType.typeName() + " for field " + name; - assert TDigestFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) == false - : "field type cannot be histogram: " + fieldType.typeName() + " for field " + name; + assert AggregateMetricDoubleFieldDownsampler.supportsFieldType(fieldType) == false + && ExponentialHistogramFieldDownsampler.supportsFieldType(fieldType) == false + && TDigestHistogramFieldDownsampler.supportsFieldType(fieldType) == false + : "field '" + name + "' of type '" + fieldType.typeName() + "' should be processed by a dedicated downsampler"; if ("flattened".equals(fieldType.typeName())) { return new LastValueFieldDownsampler.FlattenedFieldProducer(name, fieldType, fieldData); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java index 6e99b826ed492..4a0e32803f76f 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java @@ -9,9 +9,12 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.search.aggregations.metrics.CompensatedSum; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,6 +38,25 @@ public SortedNumericDoubleValues getLeaf(LeafReaderContext context) { return numericFieldData.getDoubleValues(); } + public static boolean supportsFieldType(MappedFieldType fieldType) { + TimeSeriesParams.MetricType metricType = fieldType.getMetricType(); + return metricType == TimeSeriesParams.MetricType.GAUGE || metricType == TimeSeriesParams.MetricType.COUNTER; + } + + static NumericMetricFieldDownsampler create( + String fieldName, + TimeSeriesParams.MetricType metricType, + IndexFieldData fieldData, + DownsampleConfig.SamplingMethod samplingMethod + ) { + assert metricType == TimeSeriesParams.MetricType.GAUGE || metricType == TimeSeriesParams.MetricType.COUNTER + : "only gauges and counters accepted, other metrics should have been handled by dedicated downsamplers"; + if (samplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE && metricType == TimeSeriesParams.MetricType.GAUGE) { + return new NumericMetricFieldDownsampler.AggregateGauge(fieldName, fieldData); + } + return new NumericMetricFieldDownsampler.LastValue(fieldName, fieldData); + } + static final double MAX_NO_VALUE = -Double.MAX_VALUE; static final double MIN_NO_VALUE = Double.MAX_VALUE; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java index c67ff8ac2469d..60c87be9c2fb6 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldDownsampler.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.aggregations.metrics.TDigestState; import org.elasticsearch.tdigest.Centroid; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; import org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper; import java.io.IOException; @@ -50,6 +51,11 @@ public HistogramValues getLeaf(LeafReaderContext context) throws IOException { return histogramFieldData.getHistogramValues(); } + public static boolean supportsFieldType(MappedFieldType fieldType) { + return TDigestFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()) + || HistogramFieldMapper.CONTENT_TYPE.equals(fieldType.typeName()); + } + /** * @return the requested produces based on the sampling method for metric of type tdigest histogram */ From 8493e0159026490c2e988f2c8d45df2cffee045b Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 11 Feb 2026 16:26:26 +0200 Subject: [PATCH 07/11] bug fix --- .../downsample/AbstractFieldDownsampler.java | 2 +- ...AggregateMetricDoubleFieldDownsampler.java | 32 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java index af3936bbb3183..1320c9943d52f 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java @@ -110,7 +110,7 @@ private static AbstractFieldDownsampler create( IndexFieldData fieldData, DownsampleConfig.SamplingMethod samplingMethod ) { - assert AggregateMetricDoubleFieldDownsampler.supportsFieldType(fieldType) + assert AggregateMetricDoubleFieldDownsampler.supportsFieldType(fieldType) == false : "Aggregate metric double should be handled by a dedicated downsampler"; if (TDigestHistogramFieldDownsampler.supportsFieldType(fieldType)) { return TDigestHistogramFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java index eb2afa8ff3c2f..1821013432427 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricDoubleFieldDownsampler.java @@ -216,24 +216,24 @@ static List create( var metricSubField = metricField.getValue(); if (context.fieldExistsInIndex(metricSubField.name())) { IndexFieldData fieldData = context.getForField(metricSubField, MappedFieldType.FielddataOperation.SEARCH); - if (aggMetricFieldType.getMetricType() != null) { - if (samplingMethod != DownsampleConfig.SamplingMethod.LAST_VALUE) { - // If the field is an aggregate_metric_double field, we should use the correct subfields - // for each aggregation. This is a downsample-of-downsample case - downsamplers.add(new AggregateMetricDoubleFieldDownsampler.Aggregate(aggMetricFieldType.name(), metric, fieldData)); - } else { - downsamplers.add( - new AggregateMetricDoubleFieldDownsampler.LastValue(aggMetricFieldType.name(), metric, fieldData, false) - ); - } - } else { - // If a field is not a metric, we downsample it as a label - downsamplers.add( - new AggregateMetricDoubleFieldDownsampler.LastValue(aggMetricFieldType.name(), metric, fieldData, true) - ); - } + downsamplers.add(create(aggMetricFieldType, metric, fieldData, samplingMethod)); } } return downsamplers; } + + private static AggregateMetricDoubleFieldDownsampler create( + AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType fieldType, + AggregateMetricDoubleFieldMapper.Metric metric, + IndexFieldData fieldData, + DownsampleConfig.SamplingMethod samplingMethod + ) { + if (fieldType.getMetricType() == null) { + return new AggregateMetricDoubleFieldDownsampler.LastValue(fieldType.name(), metric, fieldData, true); + } + return switch (samplingMethod) { + case AGGREGATE -> new AggregateMetricDoubleFieldDownsampler.Aggregate(fieldType.name(), metric, fieldData); + case LAST_VALUE -> new AggregateMetricDoubleFieldDownsampler.LastValue(fieldType.name(), metric, fieldData, false); + }; + } } From fd7328da4e43bf30afca4b2ec67c3340bae9c08e Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 11 Feb 2026 16:30:43 +0200 Subject: [PATCH 08/11] Remove unused classes --- .../AbstractDownsampleFieldProducer.java | 47 ------- .../ExponentialHistogramFieldProducer.java | 133 ------------------ 2 files changed, 180 deletions(-) delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java delete mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldProducer.java diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java deleted file mode 100644 index b0e6cb2efffa3..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.internal.hppc.IntArrayList; - -import java.io.IOException; - -/** - * Base class that reads fields from the source index and produces their downsampled values - */ -abstract class AbstractDownsampleFieldProducer implements DownsampleFieldSerializer { - - private final String name; - protected boolean isEmpty; - - AbstractDownsampleFieldProducer(String name) { - this.name = name; - this.isEmpty = true; - } - - /** - * @return the name of the field. - */ - public String name() { - return name; - } - - /** - * Resets the producer to an empty value. - */ - public abstract void reset(); - - /** - * @return true if the field has not collected any value. - */ - public boolean isEmpty() { - return isEmpty; - } - - public abstract void collect(T docValues, IntArrayList docIdBuffer) throws IOException; -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldProducer.java deleted file mode 100644 index 5ad1d4eb979d5..0000000000000 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldProducer.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.downsample; - -import org.apache.lucene.internal.hppc.IntArrayList; -import org.elasticsearch.action.downsample.DownsampleConfig; -import org.elasticsearch.exponentialhistogram.ExponentialHistogram; -import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker; -import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger; -import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; - -import java.io.IOException; - -/** - * A producer that can be used for downsampling ONLY an exponential histogram field whether it's a metric or a label. - */ -abstract class ExponentialHistogramFieldProducer extends AbstractDownsampleFieldProducer { - static final String TYPE = "exponential_histogram"; - - ExponentialHistogramFieldProducer(String name) { - super(name); - } - - /** - * @return the requested producer based on the sampling method for an exponential histogram field - */ - static AbstractDownsampleFieldProducer create(String name, DownsampleConfig.SamplingMethod samplingMethod) { - return switch (samplingMethod) { - case AGGREGATE -> new ExponentialHistogramFieldProducer.MergeProducer(name); - case LAST_VALUE -> new ExponentialHistogramFieldProducer.LastValueProducer(name); - }; - } - - protected abstract ExponentialHistogram downsampledValue(); - - @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - builder.field(name()); - ExponentialHistogramXContent.serialize(builder, downsampledValue()); - } - } - - /** - * Downsamples an exponential histogram by merging all values. - */ - static class MergeProducer extends ExponentialHistogramFieldProducer { - private ExponentialHistogramMerger merger = null; - - MergeProducer(String name) { - super(name); - } - - @Override - public void reset() { - isEmpty = true; - merger = null; - } - - @Override - protected ExponentialHistogram downsampledValue() { - if (isEmpty()) { - return null; - } - ExponentialHistogram exponentialHistogram = merger.get(); - merger.close(); - return exponentialHistogram; - } - - @Override - public void collect(ExponentialHistogramValuesReader docValues, IntArrayList docIdBuffer) throws IOException { - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - isEmpty = false; - if (merger == null) { - merger = ExponentialHistogramMerger.create(ExponentialHistogramCircuitBreaker.noop()); - } - ExponentialHistogram value = docValues.histogramValue(); - merger.add(value); - } - } - } - - /** - * Downsamples an exponential histogram by preserving the last value. - * Important note: This class assumes that field values are collected and sorted by descending order by time - */ - static class LastValueProducer extends ExponentialHistogramFieldProducer { - private ExponentialHistogram lastValue = null; - - LastValueProducer(String name) { - super(name); - } - - @Override - public void reset() { - isEmpty = true; - lastValue = null; - } - - @Override - public void collect(ExponentialHistogramValuesReader docValues, IntArrayList docIdBuffer) throws IOException { - if (isEmpty() == false) { - return; - } - - for (int i = 0; i < docIdBuffer.size(); i++) { - int docId = docIdBuffer.get(i); - if (docValues.advanceExact(docId) == false) { - continue; - } - isEmpty = false; - lastValue = docValues.histogramValue(); - return; - } - } - - @Override - protected ExponentialHistogram downsampledValue() { - return lastValue; - } - } -} From 742478ec5b33b31b3d9bc68b474306e1fe19bd0a Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Thu, 12 Feb 2026 13:34:02 +0200 Subject: [PATCH 09/11] Update comments in AbstractFieldDownsampler --- .../xpack/downsample/AbstractFieldDownsampler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java index 1320c9943d52f..2afeb09709f5b 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java @@ -72,7 +72,7 @@ public boolean isEmpty() { public abstract void collect(T docValues, IntArrayList docIdBuffer) throws IOException; /** - * Retrieve field value fetchers for a list of fields. + * Create field downsamplers for the provided list of fields. */ static List> create( SearchExecutionContext context, @@ -104,6 +104,9 @@ static List> create( return Collections.unmodifiableList(downsamplers); } + /** + * Create a single field downsamplers for the provided field. + */ private static AbstractFieldDownsampler create( String fieldName, MappedFieldType fieldType, From 59b310835e6d08ba966eb26048c12331c552c5d8 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 12 Feb 2026 13:46:32 +0200 Subject: [PATCH 10/11] Polishing --- .../xpack/downsample/AbstractFieldDownsampler.java | 2 +- .../xpack/downsample/DownsampleShardIndexer.java | 2 +- .../xpack/downsample/NumericMetricFieldDownsampler.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java index 2afeb09709f5b..97eb564d38342 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractFieldDownsampler.java @@ -122,7 +122,7 @@ private static AbstractFieldDownsampler create( return ExponentialHistogramFieldDownsampler.create(fieldName, fieldData, samplingMethod); } if (NumericMetricFieldDownsampler.supportsFieldType(fieldType)) { - return NumericMetricFieldDownsampler.create(fieldName, fieldType.getMetricType(), fieldData, samplingMethod); + return NumericMetricFieldDownsampler.create(fieldName, fieldType, fieldData, samplingMethod); } // TODO: Support POSITION in downsampling if (fieldType.getMetricType() == POSITION) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 3e6d90517447d..842dde6294182 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -388,7 +388,7 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag final DocCountProvider docCountProvider = new DocCountProvider(); docCountProvider.setLeafReaderContext(ctx); - // For each field, return a tuple with the downsample field downsampler and the field value leaf + // For each field retrieve the doc values for this segment var numericValues = new SortedNumericDoubleValues[numericDownsamplers.length]; for (int i = 0; i < numericDownsamplers.length; i++) { numericValues[i] = numericDownsamplers[i].getLeaf(ctx); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java index 4a0e32803f76f..85caa630148d1 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldDownsampler.java @@ -45,13 +45,13 @@ public static boolean supportsFieldType(MappedFieldType fieldType) { static NumericMetricFieldDownsampler create( String fieldName, - TimeSeriesParams.MetricType metricType, + MappedFieldType fieldType, IndexFieldData fieldData, DownsampleConfig.SamplingMethod samplingMethod ) { - assert metricType == TimeSeriesParams.MetricType.GAUGE || metricType == TimeSeriesParams.MetricType.COUNTER + assert supportsFieldType(fieldType) : "only gauges and counters accepted, other metrics should have been handled by dedicated downsamplers"; - if (samplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE && metricType == TimeSeriesParams.MetricType.GAUGE) { + if (samplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE && fieldType.getMetricType() == TimeSeriesParams.MetricType.GAUGE) { return new NumericMetricFieldDownsampler.AggregateGauge(fieldName, fieldData); } return new NumericMetricFieldDownsampler.LastValue(fieldName, fieldData); From 9d80e822e3a59480e2f4dee0ca1cc8ad5f109e5c Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 27 Feb 2026 13:03:29 +0200 Subject: [PATCH 11/11] Update ExponentialHistogramFieldMapper import --- .../xpack/downsample/ExponentialHistogramFieldDownsampler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java index c68b6a145c76a..a50dc6552fb9c 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramFieldDownsampler.java @@ -17,9 +17,9 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.analytics.mapper.ExponentialHistogramFieldMapper; import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.LeafExponentialHistogramFieldData; -import org.elasticsearch.xpack.exponentialhistogram.ExponentialHistogramFieldMapper; import java.io.IOException;