diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java index a4635d74537f3..a3aa07c3a2cf7 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java +++ b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java @@ -54,6 +54,7 @@ public Set getFeatures() { public static final NodeFeature EXPONENTIAL_HISTOGRAM_QUERYDSL_BOXPLOT = new NodeFeature( "search.exponential_histogram_querydsl_boxplot" ); + public static final NodeFeature EXPONENTIAL_HISTOGRAM_QUERYDSL_RANGE = new NodeFeature("search.exponential_histogram_querydsl_range"); @Override public Set getTestFeatures() { @@ -77,7 +78,8 @@ public Set getTestFeatures() { EXPONENTIAL_HISTOGRAM_QUERYDSL_PERCENTILE_RANKS, CLOSING_INVALID_PIT_ID, FUNCTION_SCORE_NAMED_QUERIES, - EXPONENTIAL_HISTOGRAM_QUERYDSL_BOXPLOT + EXPONENTIAL_HISTOGRAM_QUERYDSL_BOXPLOT, + EXPONENTIAL_HISTOGRAM_QUERYDSL_RANGE ); } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java index 056e45672b24d..93b7ecd6ddfc5 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java @@ -170,6 +170,7 @@ public List> getAggregationExtentions() { AnalyticsAggregatorFactory::registerExponentialHistogramHistogramAggregator, AnalyticsAggregatorFactory::registerExponentialHistogramMinAggregator, AnalyticsAggregatorFactory::registerExponentialHistogramMaxAggregator, + AnalyticsAggregatorFactory::registerExponentialHistogramRangeAggregator, AnalyticsAggregatorFactory::registerExponentialHistogramPercentilesAggregator, AnalyticsAggregatorFactory::registerExponentialHistogramPercentileRanksAggregator ); diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/AnalyticsAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/AnalyticsAggregatorFactory.java index b0f4ccd1df672..3b1b247075572 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/AnalyticsAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/AnalyticsAggregatorFactory.java @@ -21,6 +21,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.xpack.analytics.aggregations.bucket.histogram.ExponentialHistogramBackedHistogramAggregator; import org.elasticsearch.xpack.analytics.aggregations.bucket.histogram.HistoBackedHistogramAggregator; +import org.elasticsearch.xpack.analytics.aggregations.bucket.range.ExponentialHistogramBackedRangeAggregator; import org.elasticsearch.xpack.analytics.aggregations.bucket.range.HistoBackedRangeAggregator; import org.elasticsearch.xpack.analytics.aggregations.metrics.ExponentialHistogramAvgAggregator; import org.elasticsearch.xpack.analytics.aggregations.metrics.ExponentialHistogramMaxAggregator; @@ -227,6 +228,15 @@ public static void registerExponentialHistogramMaxAggregator(ValuesSourceRegistr ); } + public static void registerExponentialHistogramRangeAggregator(ValuesSourceRegistry.Builder builder) { + builder.register( + RangeAggregationBuilder.REGISTRY_KEY, + AnalyticsValuesSourceType.EXPONENTIAL_HISTOGRAM, + ExponentialHistogramBackedRangeAggregator::build, + true + ); + } + public static void registerExponentialHistogramPercentilesAggregator(ValuesSourceRegistry.Builder builder) { builder.register( PercentilesAggregationBuilder.REGISTRY_KEY, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/ExponentialHistogramBackedRangeAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/ExponentialHistogramBackedRangeAggregator.java new file mode 100644 index 0000000000000..ee599759de866 --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/ExponentialHistogramBackedRangeAggregator.java @@ -0,0 +1,336 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.analytics.aggregations.bucket.range; + +import org.elasticsearch.exponentialhistogram.BucketIterator; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.exponentialhistogram.ExponentialScaleUtils; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationExecutionContext; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.range.InternalRange; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.xpack.analytics.aggregations.support.ExponentialHistogramValuesSource; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; + +import java.io.IOException; +import java.util.Map; + +/** + * Class for supporting range aggregation on exponential histogram mapped fields. + */ +public abstract class ExponentialHistogramBackedRangeAggregator extends RangeAggregator { + + record SearchBounds(int lo, int hi) {} + + public static ExponentialHistogramBackedRangeAggregator build( + String name, + AggregatorFactories factories, + ValuesSourceConfig valuesSourceConfig, + InternalRange.Factory rangeFactory, + RangeAggregator.Range[] ranges, + boolean keyed, + AggregationContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + final double avgRange = ((double) context.searcher().getIndexReader().maxDoc()) / ranges.length; + if (hasOverlap(ranges)) { + return new Overlap( + name, + factories, + valuesSourceConfig.getValuesSource(), + valuesSourceConfig.format(), + rangeFactory, + ranges, + avgRange, + keyed, + context, + parent, + cardinality, + metadata + ); + } + return new NoOverlap( + name, + factories, + valuesSourceConfig.getValuesSource(), + valuesSourceConfig.format(), + rangeFactory, + ranges, + avgRange, + keyed, + context, + parent, + cardinality, + metadata + ); + } + + @SuppressWarnings("this-escape") + public ExponentialHistogramBackedRangeAggregator( + String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + InternalRange.Factory rangeFactory, + Range[] ranges, + double averageDocsPerRange, + boolean keyed, + AggregationContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super( + name, + factories, + valuesSource, + format, + rangeFactory, + ranges, + averageDocsPerRange, + keyed, + context, + parent, + cardinality, + metadata + ); + if (subAggregators().length > 0) { + throw new IllegalArgumentException("Range aggregation on exponential_histogram fields does not support sub-aggregations"); + } + } + + @Override + public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException { + if ((valuesSource instanceof ExponentialHistogramValuesSource.ExponentialHistogram) == false) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final ExponentialHistogramValuesSource.ExponentialHistogram expHistoSource = + (ExponentialHistogramValuesSource.ExponentialHistogram) this.valuesSource; + final ExponentialHistogramValuesReader values = expHistoSource.getHistogramValues(aggCtx.getLeafReaderContext()); + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + if (values.advanceExact(doc)) { + final ExponentialHistogram histo = values.histogramValue(); + + // Negative bucket centers are emitted in descending order (most negative first), + // so we only narrow hi from each result. + int hi = ranges.length - 1; + BucketIterator negIt = histo.negativeBuckets().iterator(); + while (negIt.hasNext()) { + double center = -ExponentialScaleUtils.getPointOfLeastRelativeError(negIt.peekIndex(), negIt.scale()); + center = Math.clamp(center, histo.min(), histo.max()); + final long count = negIt.peekCount() - docCountProvider.getDocCount(doc); + hi = ExponentialHistogramBackedRangeAggregator.this.collect( + sub, + doc, + center, + bucket, + new SearchBounds(0, hi), + count + ).hi; + negIt.advance(); + } + + // Positive bucket centers (including zero bucket) are emitted in ascending order (smallest first), + // so we only narrow lo from each result. + int lo = 0; + if (histo.zeroBucket().count() > 0) { + final long count = histo.zeroBucket().count() - docCountProvider.getDocCount(doc); + lo = ExponentialHistogramBackedRangeAggregator.this.collect( + sub, + doc, + 0.0, + bucket, + new SearchBounds(0, ranges.length - 1), + count + ).lo; + } + + BucketIterator posIt = histo.positiveBuckets().iterator(); + while (posIt.hasNext()) { + double center = ExponentialScaleUtils.getPointOfLeastRelativeError(posIt.peekIndex(), posIt.scale()); + center = Math.clamp(center, histo.min(), histo.max()); + final long count = posIt.peekCount() - docCountProvider.getDocCount(doc); + lo = ExponentialHistogramBackedRangeAggregator.this.collect( + sub, + doc, + center, + bucket, + new SearchBounds(lo, ranges.length - 1), + count + ).lo; + posIt.advance(); + } + } + } + }; + } + + /** + * Collect a value into matching range buckets. The search is bounded by the given {@link SearchBounds}, + * and returns updated bounds to narrow subsequent searches when values are processed in monotonic order. + */ + abstract SearchBounds collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, SearchBounds bounds, long count) + throws IOException; + + private static class NoOverlap extends ExponentialHistogramBackedRangeAggregator { + + private NoOverlap( + String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + InternalRange.Factory rangeFactory, + Range[] ranges, + double averageDocsPerRange, + boolean keyed, + AggregationContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super( + name, + factories, + valuesSource, + format, + rangeFactory, + ranges, + averageDocsPerRange, + keyed, + context, + parent, + cardinality, + metadata + ); + } + + @Override + SearchBounds collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, SearchBounds bounds, long count) + throws IOException { + int lo = bounds.lo, hi = bounds.hi; + while (lo <= hi) { + final int mid = (lo + hi) >>> 1; + if (value < ranges[mid].getFrom()) { + hi = mid - 1; + } else if (value >= ranges[mid].getTo()) { + lo = mid + 1; + } else { + long bucketOrd = subBucketOrdinal(owningBucketOrdinal, mid); + collectBucket(sub, doc, bucketOrd); + incrementBucketDocCount(bucketOrd, count); + // Multiple values may fall in the same range, so don't advance past mid + return new SearchBounds(mid, mid); + } + } + return new SearchBounds(lo, hi); + } + } + + private static class Overlap extends ExponentialHistogramBackedRangeAggregator { + + private final double[] maxTo; + + Overlap( + String name, + AggregatorFactories factories, + ValuesSource valuesSource, + DocValueFormat format, + InternalRange.Factory rangeFactory, + Range[] ranges, + double averageDocsPerRange, + boolean keyed, + AggregationContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super( + name, + factories, + valuesSource, + format, + rangeFactory, + ranges, + averageDocsPerRange, + keyed, + context, + parent, + cardinality, + metadata + ); + maxTo = new double[ranges.length]; + maxTo[0] = ranges[0].getTo(); + for (int i = 1; i < ranges.length; ++i) { + maxTo[i] = Math.max(ranges[i].getTo(), maxTo[i - 1]); + } + } + + @Override + SearchBounds collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, SearchBounds bounds, long count) + throws IOException { + int lo = bounds.lo, hi = bounds.hi; + int mid = (lo + hi) >>> 1; + while (lo <= hi) { + if (value < ranges[mid].getFrom()) { + hi = mid - 1; + } else if (value >= maxTo[mid]) { + lo = mid + 1; + } else { + break; + } + mid = (lo + hi) >>> 1; + } + if (lo > hi) { + return new SearchBounds(lo, hi); + } + + // binary search the lower bound + int startLo = lo, startHi = mid; + while (startLo <= startHi) { + final int startMid = (startLo + startHi) >>> 1; + if (value >= maxTo[startMid]) { + startLo = startMid + 1; + } else { + startHi = startMid - 1; + } + } + + // binary search the upper bound + int endLo = mid, endHi = hi; + while (endLo <= endHi) { + final int endMid = (endLo + endHi) >>> 1; + if (value < ranges[endMid].getFrom()) { + endHi = endMid - 1; + } else { + endLo = endMid + 1; + } + } + + for (int i = startLo; i <= endHi; ++i) { + if (ranges[i].matches(value)) { + long bucketOrd = subBucketOrdinal(owningBucketOrdinal, i); + collectBucket(sub, doc, bucketOrd); + incrementBucketDocCount(bucketOrd, count); + } + } + return new SearchBounds(startLo, endHi); + } + } +} diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/ExponentialHistogramBackedRangeAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/ExponentialHistogramBackedRangeAggregatorTests.java new file mode 100644 index 0000000000000..52dc6e656b2c6 --- /dev/null +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/ExponentialHistogramBackedRangeAggregatorTests.java @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.analytics.aggregations.bucket.range; + +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.exponentialhistogram.BucketIterator; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker; +import org.elasticsearch.exponentialhistogram.ExponentialScaleUtils; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.range.InternalRange; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; +import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.xpack.analytics.aggregations.ExponentialHistogramAggregatorTestCase; +import org.elasticsearch.xpack.analytics.mapper.ExponentialHistogramFieldMapper; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + +public class ExponentialHistogramBackedRangeAggregatorTests extends ExponentialHistogramAggregatorTestCase { + + private static final String FIELD_NAME = "histo_field"; + + @SuppressWarnings({ "unchecked" }) + public void testNonOverlapping() throws Exception { + ExponentialHistogramCircuitBreaker noopBreaker = ExponentialHistogramCircuitBreaker.noop(); + List histograms = List.of( + ExponentialHistogram.create(100, noopBreaker, -99, -79, -49, -30, -9, -4, -1, 2, 15, 40, 80), + ExponentialHistogram.create(100, noopBreaker, -90, -70, -40, -19, -8, -3, -0.5, 0.1, 7, 25, 60, 95), + ExponentialHistogram.create(100, noopBreaker, -200, -150, -75, -25, -12, -2, 0, 5, 11, 30, 51, 110) + ); + + testCase(iw -> histograms.forEach(histo -> addHistogramDoc(iw, FIELD_NAME, histo)), aggBuilder -> { + aggBuilder.addUnboundedTo(-100) + .addRange(-100, -50) + .addRange(-50, -20) + .addRange(-20, -5) + .addRange(-5, 0) + .addRange(0, 10) + .addRange(10, 50) + .addRange(50, 100) + .addUnboundedFrom(100); + }, range -> { + assertTrue(AggregationInspectionHelper.hasValue(range)); + List buckets = range.getBuckets(); + assertEquals(9, buckets.size()); + + assertEquals("*--100.0", buckets.get(0).getKeyAsString()); + assertEquals(2, buckets.get(0).getDocCount()); + assertEquals("-100.0--50.0", buckets.get(1).getKeyAsString()); + assertEquals(5, buckets.get(1).getDocCount()); + assertEquals("-50.0--20.0", buckets.get(2).getKeyAsString()); + assertEquals(4, buckets.get(2).getDocCount()); + assertEquals("-20.0--5.0", buckets.get(3).getKeyAsString()); + assertEquals(4, buckets.get(3).getDocCount()); + assertEquals("-5.0-0.0", buckets.get(4).getKeyAsString()); + assertEquals(5, buckets.get(4).getDocCount()); + assertEquals("0.0-10.0", buckets.get(5).getKeyAsString()); + assertEquals(5, buckets.get(5).getDocCount()); + assertEquals("10.0-50.0", buckets.get(6).getKeyAsString()); + assertEquals(5, buckets.get(6).getDocCount()); + assertEquals("50.0-100.0", buckets.get(7).getKeyAsString()); + assertEquals(4, buckets.get(7).getDocCount()); + assertEquals("100.0-*", buckets.get(8).getKeyAsString()); + assertEquals(1, buckets.get(8).getDocCount()); + }); + } + + @SuppressWarnings({ "unchecked" }) + public void testOverlapping() throws Exception { + ExponentialHistogramCircuitBreaker noopBreaker = ExponentialHistogramCircuitBreaker.noop(); + List histograms = List.of( + ExponentialHistogram.create(100, noopBreaker, -99, -79, -49, -30, -9, -4, -1, 2, 15, 40, 80), + ExponentialHistogram.create(100, noopBreaker, -90, -70, -40, -19, -8, -3, -0.5, 0.1, 7, 25, 60, 95), + ExponentialHistogram.create(100, noopBreaker, -200, -150, -75, -25, -12, -2, 0, 5, 11, 30, 51, 110) + ); + + testCase(iw -> histograms.forEach(histo -> addHistogramDoc(iw, FIELD_NAME, histo)), aggBuilder -> { + aggBuilder.addUnboundedTo(0) + .addRange(-100, -20) + .addRange(-80, -10) + .addRange(-50, 50) + .addRange(0, 20) + .addRange(0, 100) + .addRange(10, 50) + .addUnboundedFrom(50); + }, range -> { + assertTrue(AggregationInspectionHelper.hasValue(range)); + List buckets = range.getBuckets(); + assertEquals(8, buckets.size()); + + assertEquals("*-0.0", buckets.get(0).getKeyAsString()); + assertEquals(20, buckets.get(0).getDocCount()); + assertEquals("-100.0--20.0", buckets.get(1).getKeyAsString()); + assertEquals(9, buckets.get(1).getDocCount()); + assertEquals("-80.0--10.0", buckets.get(2).getKeyAsString()); + assertEquals(9, buckets.get(2).getDocCount()); + assertEquals("-50.0-50.0", buckets.get(3).getKeyAsString()); + assertEquals(23, buckets.get(3).getDocCount()); + assertEquals("0.0-20.0", buckets.get(4).getKeyAsString()); + assertEquals(7, buckets.get(4).getDocCount()); + assertEquals("0.0-100.0", buckets.get(5).getKeyAsString()); + assertEquals(14, buckets.get(5).getDocCount()); + assertEquals("10.0-50.0", buckets.get(6).getKeyAsString()); + assertEquals(5, buckets.get(6).getDocCount()); + assertEquals("50.0-*", buckets.get(7).getKeyAsString()); + assertEquals(5, buckets.get(7).getDocCount()); + }); + } + + @SuppressWarnings({ "unchecked" }) + public void testRandomRanges() throws Exception { + List histograms = createRandomHistograms(randomIntBetween(1, 100)); + int numRanges = randomIntBetween(1, 10); + double[] bounds = new double[numRanges + 1]; + for (int i = 0; i < bounds.length; i++) { + bounds[i] = randomDoubleBetween(-100, 100, true); + } + Arrays.sort(bounds); + + RangeAggregator.Range[] ranges = new RangeAggregator.Range[numRanges]; + for (int i = 0; i < numRanges; i++) { + ranges[i] = new RangeAggregator.Range(null, bounds[i], bounds[i + 1]); + } + + testCase(iw -> histograms.forEach(histo -> addHistogramDoc(iw, FIELD_NAME, histo)), aggBuilder -> { + for (RangeAggregator.Range r : ranges) { + aggBuilder.addRange(r); + } + }, range -> { + long[] expectedCounts = computeExpectedRangeCounts(histograms, ranges); + List buckets = range.getBuckets(); + assertEquals(ranges.length, buckets.size()); + for (int i = 0; i < buckets.size(); i++) { + assertEquals("bucket " + buckets.get(i).getKey(), expectedCounts[i], buckets.get(i).getDocCount()); + } + }); + } + + @SuppressWarnings({ "unchecked" }) + public void testNoDocs() throws Exception { + testCase(iw -> { + // Intentionally not writing any docs + }, aggBuilder -> aggBuilder.addRange(0, 10), range -> { + List buckets = range.getBuckets(); + assertEquals(1, buckets.size()); + assertEquals(0, buckets.get(0).getDocCount()); + assertFalse(AggregationInspectionHelper.hasValue(range)); + }); + } + + public void testSubAggs() throws Exception { + ExponentialHistogramCircuitBreaker noopBreaker = ExponentialHistogramCircuitBreaker.noop(); + List histograms = List.of(ExponentialHistogram.create(100, noopBreaker, -4.5, 4.3)); + + RangeAggregationBuilder aggBuilder = new RangeAggregationBuilder("my_agg").field(FIELD_NAME) + .addRange(-1.0, 3.0) + .subAggregation(new TopHitsAggregationBuilder("top_hits")); + + var fieldType = defaultFieldType(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + testCase( + iw -> histograms.forEach(histo -> addHistogramDoc(iw, FIELD_NAME, histo)), + aggBuilder, + fieldType, + range -> fail("should have thrown") + ); + }); + assertEquals("Range aggregation on exponential_histogram fields does not support sub-aggregations", e.getMessage()); + } + + @SuppressWarnings("rawtypes") + private void testCase( + CheckedConsumer buildIndex, + Consumer aggCustomizer, + Consumer verify + ) throws IOException { + var fieldType = defaultFieldType(); + RangeAggregationBuilder aggBuilder = new RangeAggregationBuilder("my_agg").field(FIELD_NAME); + aggCustomizer.accept(aggBuilder); + testCase(buildIndex, aggBuilder, fieldType, verify); + } + + @SuppressWarnings("rawtypes") + private void testCase( + CheckedConsumer buildIndex, + AggregationBuilder aggBuilder, + MappedFieldType fieldType, + Consumer verify + ) throws IOException { + testCase(buildIndex, verify, new AggTestConfig(aggBuilder, fieldType)); + } + + private MappedFieldType defaultFieldType() { + return new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(FIELD_NAME, Collections.emptyMap(), null); + } + + /** + * Compute expected range counts by iterating over all histogram bucket centers and assigning them to ranges, + * mirroring the logic of ExponentialHistogramBackedRangeAggregator, but without the fancy binary search optimizations. + */ + private long[] computeExpectedRangeCounts(List histograms, RangeAggregator.Range[] ranges) { + long[] counts = new long[ranges.length]; + for (ExponentialHistogram histogram : histograms) { + BucketIterator negIt = histogram.negativeBuckets().iterator(); + while (negIt.hasNext()) { + double center = -ExponentialScaleUtils.getPointOfLeastRelativeError(negIt.peekIndex(), negIt.scale()); + center = Math.clamp(center, histogram.min(), histogram.max()); + addToRanges(counts, ranges, center, negIt.peekCount()); + negIt.advance(); + } + if (histogram.zeroBucket().count() > 0) { + addToRanges(counts, ranges, 0.0, histogram.zeroBucket().count()); + } + BucketIterator posIt = histogram.positiveBuckets().iterator(); + while (posIt.hasNext()) { + double center = ExponentialScaleUtils.getPointOfLeastRelativeError(posIt.peekIndex(), posIt.scale()); + center = Math.clamp(center, histogram.min(), histogram.max()); + addToRanges(counts, ranges, center, posIt.peekCount()); + posIt.advance(); + } + } + return counts; + } + + private void addToRanges(long[] counts, RangeAggregator.Range[] ranges, double value, long count) { + for (int i = 0; i < ranges.length; i++) { + if (ranges[i].matches(value)) { + counts[i] += count; + } + } + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new RangeAggregationBuilder("_name").field(fieldName).addRange(0, 10); + } +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/20_aggregations.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/20_aggregations.yml index d371a33573dda..43f821b507dbd 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/20_aggregations.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/20_aggregations.yml @@ -243,3 +243,37 @@ setup: - match: { hits.total.value: 5 } +--- +"Range aggregation": + - requires: + cluster_features: [ "search.exponential_histogram_querydsl_range" ] + reason: exponential_histogram range aggregation was introduced + - do: + search: + index: test_exponential_histogram + size: 0 + body: + aggs: + range_agg: + range: + field: histo + ranges: + - {to: -500} + - {from: -500, to: 0} + - {from: 0, to: 100} + - {from: 100, to: 10000} + - {from: 10000} + + - match: { hits.total.value: 5 } + - length: { aggregations.range_agg.buckets: 5 } + - match: { aggregations.range_agg.buckets.0.key: "*--500.0" } + - match: { aggregations.range_agg.buckets.0.doc_count: 5 } + - match: { aggregations.range_agg.buckets.1.key: "-500.0-0.0" } + - match: { aggregations.range_agg.buckets.1.doc_count: 2 } + - match: { aggregations.range_agg.buckets.2.key: "0.0-100.0" } + - match: { aggregations.range_agg.buckets.2.doc_count: 20 } + - match: { aggregations.range_agg.buckets.3.key: "100.0-10000.0" } + - match: { aggregations.range_agg.buckets.3.doc_count: 9 } + - match: { aggregations.range_agg.buckets.4.key: "10000.0-*" } + - match: { aggregations.range_agg.buckets.4.doc_count: 5 } + diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/40_tdigest_histogram_aggregations_compatibility.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/40_tdigest_histogram_aggregations_compatibility.yml index c4ee86a36ef8b..f2123aa531af5 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/40_tdigest_histogram_aggregations_compatibility.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/exponential_histogram/40_tdigest_histogram_aggregations_compatibility.yml @@ -259,3 +259,34 @@ setup: - match: { hits.total.value: 2 } +--- +"Range aggregation over tdigest and exponential_histogram": + - requires: + cluster_features: [ "search.exponential_histogram_querydsl_range" ] + reason: exponential_histogram range aggregation was introduced + - do: + search: + index: test-data-stream + size: 0 + body: + aggs: + range_agg: + range: + field: tdigest_or_exp_histo + ranges: + - {to: 1} + - {from: 1, to: 3} + - {from: 3, to: 6} + - {from: 6} + + - match: { hits.total.value: 2 } + - length: { aggregations.range_agg.buckets: 4 } + - match: { aggregations.range_agg.buckets.0.key: "*-1.0" } + - match: { aggregations.range_agg.buckets.0.doc_count: 3 } + - match: { aggregations.range_agg.buckets.1.key: "1.0-3.0" } + - match: { aggregations.range_agg.buckets.1.doc_count: 5 } + - match: { aggregations.range_agg.buckets.2.key: "3.0-6.0" } + - match: { aggregations.range_agg.buckets.2.doc_count: 2 } + - match: { aggregations.range_agg.buckets.3.key: "6.0-*" } + - match: { aggregations.range_agg.buckets.3.doc_count: 0 } +