diff --git a/docs/changelog/139247.yaml b/docs/changelog/139247.yaml new file mode 100644 index 0000000000000..53a865673bdb6 --- /dev/null +++ b/docs/changelog/139247.yaml @@ -0,0 +1,5 @@ +pr: 139247 +summary: Add TDigest histogram as metric to time series data streams +area: "TSDB" +type: enhancement +issues: [] diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 02e1a4fa4be21..f447bac11b7b2 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -78,6 +79,11 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final Parameter> ignoreMalformed; private final Parameter> coerce; + /** + * Parameter that marks this field as a time series metric defining its time series metric type. + * Only the metric type histogram is supported. + */ + private final Parameter metric; public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault) { super(name); @@ -88,18 +94,24 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe ignoreMalformedByDefault ); this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault); + this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, TimeSeriesParams.MetricType.HISTOGRAM); + } + + public Builder metric(TimeSeriesParams.MetricType metric) { + this.metric.setValue(metric); + return this; } @Override protected Parameter[] getParameters() { - return new Parameter[] { ignoreMalformed, coerce, meta }; + return new Parameter[] { ignoreMalformed, coerce, meta, metric }; } @Override public HistogramFieldMapper build(MapperBuilderContext context) { return new HistogramFieldMapper( leafName(), - new HistogramFieldType(context.buildFullName(leafName()), meta.getValue()), + new HistogramFieldType(context.buildFullName(leafName()), meta.getValue(), this.metric.getValue()), builderParams(this, context), this ); @@ -116,6 +128,7 @@ public HistogramFieldMapper build(MapperBuilderContext context) { private final Explicit coerce; private final boolean coerceByDefault; + private final TimeSeriesParams.MetricType metricType; public HistogramFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, Builder builder) { super(simpleName, mappedFieldType, builderParams); @@ -123,6 +136,7 @@ public HistogramFieldMapper(String simpleName, MappedFieldType mappedFieldType, this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value(); this.coerce = builder.coerce.getValue(); this.coerceByDefault = builder.coerce.getDefaultValue().value(); + this.metricType = builder.metric.get(); } @Override @@ -141,7 +155,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault).init(this); + return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault).metric(metricType).init(this); } @Override @@ -150,9 +164,11 @@ protected void parseCreateField(DocumentParserContext context) { } public static class HistogramFieldType extends MappedFieldType { + private final TimeSeriesParams.MetricType metricType; - public HistogramFieldType(String name, Map meta) { + public HistogramFieldType(String name, Map meta, TimeSeriesParams.MetricType metricType) { super(name, IndexType.docValuesOnly(), false, meta); + this.metricType = metricType; } @Override @@ -170,6 +186,11 @@ public boolean isSearchable() { return false; } + @Override + public TimeSeriesParams.MetricType getMetricType() { + return metricType; + } + @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { failIfNoDocValues(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java index 82dd240bfc5f6..f06ecd1215a12 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java @@ -224,7 +224,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy } private MappedFieldType defaultFieldType(String fieldName) { - return new HistogramFieldMapper.HistogramFieldType(fieldName, Collections.emptyMap()); + return new HistogramFieldMapper.HistogramFieldType(fieldName, Collections.emptyMap(), null); } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/HistoBackedRangeAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/HistoBackedRangeAggregatorTests.java index b6a198a8206e8..83a5c2ff68d13 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/HistoBackedRangeAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/range/HistoBackedRangeAggregatorTests.java @@ -353,7 +353,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy private MappedFieldType defaultFieldType(String fieldName) { if (fieldName.equals(HISTO_FIELD_NAME)) { - return new HistogramFieldMapper.HistogramFieldType(fieldName, Collections.emptyMap()); + return new HistogramFieldMapper.HistogramFieldType(fieldName, Collections.emptyMap(), null); } else { return new NumberFieldMapper.NumberFieldType(fieldName, NumberFieldMapper.NumberType.DOUBLE); } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentileRanksAggregatorTests.java index 8fce6c145bc22..c8f0d0e3b2b44 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentileRanksAggregatorTests.java @@ -88,7 +88,7 @@ public void testSimple() throws IOException { PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg", new double[] { 0.1, 0.5, 12 }) .field("field") .method(PercentilesMethod.HDR); - MappedFieldType fieldType = new HistogramFieldMapper.HistogramFieldType("field", Collections.emptyMap()); + MappedFieldType fieldType = new HistogramFieldMapper.HistogramFieldType("field", Collections.emptyMap(), null); try (IndexReader reader = w.getReader()) { PercentileRanks ranks = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); Iterator rankIterator = ranks.iterator(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java index 6f1370e7fd4f5..864f0ae7c630c 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java @@ -133,7 +133,7 @@ private void testCase(Query query, CheckedConsumer rankIterator = ranks.iterator(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java index d078c737aeda3..1516da4d65976 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java @@ -120,7 +120,7 @@ private void testCase( ) throws IOException { PercentilesAggregationBuilder builder = new PercentilesAggregationBuilder("test").field("number").method(PercentilesMethod.TDIGEST); - MappedFieldType fieldType = new HistogramFieldMapper.HistogramFieldType("number", Collections.emptyMap()); + MappedFieldType fieldType = new HistogramFieldMapper.HistogramFieldType("number", Collections.emptyMap(), null); testCase(buildIndex, verify, new AggTestConfig(builder, fieldType).withQuery(query)); } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 9432074bfed07..7b289915161cd 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -463,6 +464,41 @@ public void testArrayValueSyntheticSource() throws Exception { assertEquals(Strings.toString(expected), syntheticSource); } + public void testMetricType() throws IOException { + // Test default setting + MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping)); + HistogramFieldMapper.HistogramFieldType ft = (HistogramFieldMapper.HistogramFieldType) mapperService.fieldType("field"); + assertNull(ft.getMetricType()); + + assertMetricType("histogram", HistogramFieldMapper.HistogramFieldType::getMetricType); + + { + String unsupportedMetricTypes = randomFrom("counter", "gauge", "position"); + // Test invalid metric type for this field type + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_metric", unsupportedMetricTypes); + }))); + assertThat( + e.getCause().getMessage(), + containsString( + "Unknown value [" + unsupportedMetricTypes + "] for field [time_series_metric] - accepted values are [histogram]" + ) + ); + } + { + // Test invalid metric type + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_metric", "unknown"); + }))); + assertThat( + e.getCause().getMessage(), + containsString("Unknown value [unknown] for field [time_series_metric] - accepted values are [histogram]") + ); + } + } + @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { return new HistogramFieldSyntheticSourceSupport(ignoreMalformed); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java index 518d770e0e937..8d49e026989ef 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java @@ -727,7 +727,7 @@ public void testFormatter() throws IOException { } public void testHistogramFieldMonthToMonth() throws IOException { - MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap()); + MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap(), null); MappedFieldType dateType = dateFieldType(DATE_FIELD); RateAggregationBuilder rateAggregationBuilder = new RateAggregationBuilder("my_rate").rateUnit("month").field("val"); if (randomBoolean()) { @@ -750,7 +750,7 @@ public void testHistogramFieldMonthToMonth() throws IOException { } public void testHistogramFieldMonthToYear() throws IOException { - MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap()); + MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap(), null); MappedFieldType dateType = dateFieldType(DATE_FIELD); RateAggregationBuilder rateAggregationBuilder = new RateAggregationBuilder("my_rate").rateUnit("month").field("val"); if (randomBoolean()) { @@ -770,7 +770,7 @@ public void testHistogramFieldMonthToYear() throws IOException { } public void testHistogramFieldMonthToMonthValueCount() throws IOException { - MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap()); + MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap(), null); MappedFieldType dateType = dateFieldType(DATE_FIELD); RateAggregationBuilder rateAggregationBuilder = new RateAggregationBuilder("my_rate").rateUnit("month") .rateMode("value_count") @@ -792,7 +792,7 @@ public void testHistogramFieldMonthToMonthValueCount() throws IOException { } public void testHistogramFieldMonthToYearValueCount() throws IOException { - MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap()); + MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap(), null); MappedFieldType dateType = dateFieldType(DATE_FIELD); RateAggregationBuilder rateAggregationBuilder = new RateAggregationBuilder("my_rate").rateUnit("month") .rateMode("value_count") @@ -813,7 +813,7 @@ public void testHistogramFieldMonthToYearValueCount() throws IOException { } public void testFilterWithHistogramField() throws IOException { - MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap()); + MappedFieldType histType = new HistogramFieldMapper.HistogramFieldType("val", Collections.emptyMap(), null); MappedFieldType dateType = dateFieldType(DATE_FIELD); MappedFieldType keywordType = new KeywordFieldMapper.KeywordFieldType("term"); RateAggregationBuilder rateAggregationBuilder = new RateAggregationBuilder("my_rate").rateUnit("month").field("val"); diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/100_downsample_histograms.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/100_downsample_histograms.yml new file mode 100644 index 0000000000000..5a5dd83e9b228 --- /dev/null +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/100_downsample_histograms.yml @@ -0,0 +1,234 @@ +setup: + - requires: + capabilities: + - method: POST + path: /{index}/_downsample/{target_index} + capabilities: [ "downsample.sampling_mode.last_value", "downsampling.exponential_histograms", "downsampling.tdigest_histograms" ] + test_runner_features: [ "capabilities" ] + reason: Downsampling histograms as metrics was added in 9.3 + + - do: + indices.create: + index: test-exponential-histogram + body: + settings: + number_of_shards: 1 + index: + mode: time_series + routing_path: [ metricset ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + metric: + type: exponential_histogram + time_series_metric: histogram + non-metric: + type: exponential_histogram + + - do: + bulk: + refresh: true + index: test-exponential-histogram + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "set_1", "metric": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "non-metric": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "set_1", "metric": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "non-metric": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:57:04.467Z", "metricset": "set_2", "metric": {"scale":3,"sum":-2.612341035444145,"min":-0.596,"max":0.9,"positive":{"indices":[-100,-90,-88],"counts":[1,1,1]},"negative":{"indices":[-142,-92,-87],"counts":[1,1,1]}}, "non-metric": {"scale":3,"sum":-2.612341035444145,"min":-0.596,"max":0.9,"positive":{"indices":[-100,-90,-88],"counts":[1,1,1]},"negative":{"indices":[-142,-92,-87],"counts":[1,1,1]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:57:04.467Z", "metricset": "set_1", "metric": {"scale":3,"sum":-2.612341035444145,"min":-0.596,"max":0.9,"positive":{"indices":[-100,-90,-88],"counts":[1,1,1]},"negative":{"indices":[-142,-92,-87],"counts":[1,1,1]}}, "non-metric": {"scale":3,"sum":-2.612341035444145,"min":-0.596,"max":0.9,"positive":{"indices":[-100,-90,-88],"counts":[1,1,1]},"negative":{"indices":[-142,-92,-87],"counts":[1,1,1]}}}' + + - do: + indices.put_settings: + index: test-exponential-histogram + body: + index.blocks.write: true + + - do: + indices.create: + index: test-histogram + body: + settings: + number_of_shards: 1 + index: + mode: time_series + routing_path: [ metricset ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + metric: + type: histogram + time_series_metric: histogram + non-metric: + type: histogram + + - do: + bulk: + refresh: true + index: test-histogram + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "set_1", "metric": {"values": [-3, -2, 0], "counts": [2, 11, 36]}, "non-metric": {"values": [-3, -2, 0], "counts": [2, 11, 36]}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "set_1", "metric": {"values": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "non-metric": {"values": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:57:04.467Z", "metricset": "set_2", "metric": {"values": [3, 5, 7, 8, 9, 22, 41], "counts": [3, 4, 1, 5, 32, 21, 6]}, "non-metric": {"values": [3, 5, 7, 8, 9, 22, 41], "counts": [3, 4, 1, 5, 32, 21, 6]}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:57:04.467Z", "metricset": "set_1", "metric": {"values": [3, 5, 7, 8, 9, 22, 41], "counts": [3, 4, 1, 5, 32, 21, 6]}, "non-metric": {"values": [3, 5, 7, 8, 9, 22, 41], "counts": [3, 4, 1, 5, 32, 21, 6]}}' + + - do: + indices.put_settings: + index: test-histogram + body: + index.blocks.write: true + +--- +"Downsample exponential histogram with last value": + - do: + indices.downsample: + index: test-exponential-histogram + target_index: last-value-exponential-histogram + body: > + { + "fixed_interval": "1h", + "sampling_method": "last_value" + } + - is_true: acknowledged + + - do: + search: + index: last-value-exponential-histogram + body: > + { + "query": { "term": { "metricset": "set_1" } }, + "sort": [ "@timestamp" ] + } + + + - length: { hits.hits: 2 } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.metricset: set_1 } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.metric.max: 00.9412120998532649 } + - match: { hits.hits.0._source.metric.min: -0.8596842630789088 } + - match: { hits.hits.0._source.metric.scale: 38 } + - match: { hits.hits.0._source.metric.sum: -2.612341035444145 } + - match: { hits.hits.0._source.non-metric.max: 0.9412120998532649 } + - match: { hits.hits.0._source.non-metric.min: -0.8596842630789088 } + - match: { hits.hits.0._source.non-metric.scale: 38 } + - match: { hits.hits.0._source.non-metric.sum: -2.612341035444145 } + +--- +"Downsample exponential histogram with aggregate": + - do: + indices.downsample: + index: test-exponential-histogram + target_index: aggregate-exponential-histogram + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: aggregate-exponential-histogram + body: > + { + "query": { "term": { "metricset": "set_1" } }, + "sort": [ "@timestamp" ] + } + + - length: { hits.hits: 2 } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.metricset: set_1 } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.metric.max: 0.9991606633362218 } + - match: { hits.hits.0._source.metric.min: -0.9985699620066124 } + - match: { hits.hits.0._source.metric.scale: 38 } + - match: { hits.hits.0._source.metric.sum: -5.271780866133354 } + - match: { hits.hits.0._source.non-metric.max: 0.9412120998532649 } + - match: { hits.hits.0._source.non-metric.min: -0.8596842630789088 } + - match: { hits.hits.0._source.non-metric.scale: 38 } + - match: { hits.hits.0._source.non-metric.sum: -2.612341035444145 } + +--- +"Downsample histogram with last value": + + - do: + indices.downsample: + index: test-histogram + target_index: last-value-histogram + body: > + { + "fixed_interval": "1h", + "sampling_method": "last_value" + } + - is_true: acknowledged + + - do: + search: + index: last-value-histogram + body: > + { + "query": { "term": { "metricset": "set_1" } }, + "sort": [ "@timestamp" ] + } + + - length: { hits.hits: 2 } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.metricset: set_1 } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.metric.values: [-9.0, -8.0, -7.0, -4.0, -3.0, -2.0, -1.0] } + - match: { hits.hits.0._source.metric.counts: [1, 2, 1, 3, 3, 2, 6] } + - match: { hits.hits.0._source.non-metric.values: [-9.0, -8.0, -7.0, -4.0, -3.0, -2.0, -1.0] } + - match: { hits.hits.0._source.non-metric.counts: [1, 2, 1, 3, 3, 2, 6] } + +--- +"Downsample histogram with aggregate": + - do: + indices.downsample: + index: test-histogram + target_index: aggregate-histogram + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: aggregate-histogram + body: > + { + "query": { "term": { "metricset": "set_1" } }, + "sort": [ "@timestamp" ] + } + + - length: { hits.hits: 2 } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.metricset: set_1 } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.metric.values: [-9.0, -8.0, -7.0, -4.0, -3.0, -2.0, -1.0, 0.0] } + - match: { hits.hits.0._source.metric.counts: [1, 2, 1, 3, 5, 13, 6, 36] } + - match: { hits.hits.0._source.non-metric.values: [-9.0, -8.0, -7.0, -4.0, -3.0, -2.0, -1.0] } + - match: { hits.hits.0._source.non-metric.counts: [1, 2, 1, 3, 3, 2, 6] } diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 4e76a71011fa9..74e572708cbab 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -1846,168 +1846,3 @@ setup: - match: { hits.hits.1._source._doc_count: 1 } - match: { hits.hits.1._source.k8s\.pod\.name: cat } - match: { hits.hits.1._source.k8s\.pod\.empty: "" } - ---- -"Downsample exponential histogram with last value": - - requires: - capabilities: - - method: POST - path: /{index}/_downsample/{target_index} - capabilities: [ "downsample.sampling_mode.last_value", "downsampling.exponential_histograms" ] - test_runner_features: [ "capabilities" ] - reason: Last value sampling method was added in 9.3 - - do: - indices.create: - index: test-exponential-histogram - body: - settings: - number_of_shards: 1 - index: - mode: time_series - routing_path: [ metricset ] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - mappings: - properties: - "@timestamp": - type: date - metricset: - type: keyword - time_series_dimension: true - metric: - type: exponential_histogram - time_series_metric: histogram - label: - type: exponential_histogram - - is_true: shards_acknowledged - - - do: - bulk: - refresh: true - index: test-exponential-histogram - body: - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}, "label": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}, "label": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}}' - - match: { items.0.index.result: "created"} - - match: { items.1.index.result: "created"} - - - do: - indices.put_settings: - index: test-exponential-histogram - body: - index.blocks.write: true - - is_true: acknowledged - - - do: - indices.downsample: - index: test-exponential-histogram - target_index: downsampled-exponential-histogram - body: > - { - "fixed_interval": "1h", - "sampling_method": "last_value" - } - - is_true: acknowledged - - - do: - search: - index: downsampled-exponential-histogram - - - length: { hits.hits: 1 } - - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.metricset: lala } - - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.metric.max: 0.9821606962370899 } - - match: { hits.hits.0._source.metric.min: -0.9410989907817573 } - - match: { hits.hits.0._source.metric.scale: 1 } - - match: { hits.hits.0._source.metric.sum: 5.89290987609197 } - - match: { hits.hits.0._source.label.max: 0.9821606962370899 } - - match: { hits.hits.0._source.label.min: -0.9410989907817573 } - - match: { hits.hits.0._source.label.scale: 1 } - - match: { hits.hits.0._source.label.sum: 5.89290987609197 } - ---- -"Downsample exponential histogram with aggregate": - - requires: - capabilities: - - method: POST - path: /{index}/_downsample/{target_index} - capabilities: [ "downsampling.exponential_histograms" ] - test_runner_features: [ "capabilities" ] - reason: Last value sampling method was added in 9.3 - - do: - indices.create: - index: test-exponential-histogram - body: - settings: - number_of_shards: 1 - index: - mode: time_series - routing_path: [ metricset ] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - mappings: - properties: - "@timestamp": - type: date - metricset: - type: keyword - time_series_dimension: true - metric: - type: exponential_histogram - time_series_metric: histogram - label: - type: exponential_histogram - - is_true: shards_acknowledged - - - do: - bulk: - refresh: true - index: test-exponential-histogram - body: - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}' - - match: { items.0.index.result: "created"} - - match: { items.1.index.result: "created"} - - - do: - indices.put_settings: - index: test-exponential-histogram - body: - index.blocks.write: true - - is_true: acknowledged - - - do: - indices.downsample: - index: test-exponential-histogram - target_index: downsampled-exponential-histogram-aggregate - body: > - { - "fixed_interval": "1h" - } - - is_true: acknowledged - - - do: - search: - index: downsampled-exponential-histogram-aggregate - - - length: { hits.hits: 1 } - - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.metricset: lala } - - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.metric.max: 0.9991606633362218 } - - match: { hits.hits.0._source.metric.min: -0.9985699620066124 } - - match: { hits.hits.0._source.metric.scale: 38 } - - match: { hits.hits.0._source.metric.sum: -5.271780866133354 } - - match: { hits.hits.0._source.label.max: 0.9412120998532649 } - - match: { hits.hits.0._source.label.min: -0.8596842630789088 } - - match: { hits.hits.0._source.label.scale: 38 } - - match: { hits.hits.0._source.label.sum: -2.612341035444145 } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java index fe6a5d2f84a11..d57ca8735cd9a 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java @@ -177,6 +177,10 @@ private void downsampleWithSamplingMethod(DownsampleConfig.SamplingMethod method "type": "exponential_histogram", "time_series_metric": "histogram" }, + "metrics.tdigest": { + "type": "histogram", + "time_series_metric": "histogram" + }, "my_labels": { "properties": { "my_histogram": { @@ -228,6 +232,11 @@ private void downsampleWithSamplingMethod(DownsampleConfig.SamplingMethod method .endObject() .endObject() + .startObject("metrics.tdigest") + .array("values", randomHistogramValues(maxHistogramSize)) + .array("counts", randomHistogramValueCounts(maxHistogramSize)) + .endObject() + .startObject("my_labels.my_histogram") .array("values", randomHistogramValues(maxHistogramSize)) .array("counts", randomHistogramValueCounts(maxHistogramSize)) diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java index 4ed9ab5a58164..c1f001ecc8439 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java @@ -66,6 +66,7 @@ import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM; import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_METRIC_PARAM; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.equalTo; @@ -304,7 +305,7 @@ void assertDownsampleIndexFieldsAndDimensions(String sourceIndex, String downsam assertThat(fieldMapping.get("type"), equalTo("aggregate_metric_double")); } } - case HISTOGRAM -> assertThat(fieldMapping.get("type"), equalTo("exponential_histogram")); + case HISTOGRAM -> assertThat(fieldMapping.get("type"), anyOf(equalTo("exponential_histogram"), equalTo("histogram"))); default -> fail("Unsupported field type"); } assertThat(fieldMapping.get("time_series_metric"), equalTo(metricType.toString())); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 8413c488fcb73..5a1622ac95fba 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -385,6 +385,11 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag lastValueFieldProducer, fieldValueFetcher.getLeaf(ctx) ); + } else if (fieldProducer instanceof TDigestHistogramFieldProducer histogramFieldProducer) { + fieldCollectors[i] = new LeafDownsampleCollector.FieldCollector<>( + histogramFieldProducer, + fieldValueFetcher.getHistogramLeaf(ctx) + ); } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java index 6af0fea74edbc..7d4cb64379774 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java @@ -23,7 +23,7 @@ * values. */ final class ExponentialHistogramMetricFieldProducer extends AbstractDownsampleFieldProducer { - + static final String TYPE = "exponential_histogram"; private ExponentialHistogramMerger merger = null; ExponentialHistogramMetricFieldProducer(String name) { @@ -33,10 +33,7 @@ final class ExponentialHistogramMetricFieldProducer extends AbstractDownsampleFi /** * @return the requested produces based on the sampling method for metric of type exponential histogram */ - static AbstractDownsampleFieldProducer createMetricProducerForExponentialHistogram( - String name, - DownsampleConfig.SamplingMethod samplingMethod - ) { + static AbstractDownsampleFieldProducer create(String name, DownsampleConfig.SamplingMethod samplingMethod) { return switch (samplingMethod) { case AGGREGATE -> new ExponentialHistogramMetricFieldProducer(name); case LAST_VALUE -> new LastValueFieldProducer.ExponentialHistogramFieldProducer(name); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java index 9df6f031aaf8b..f22ab478acf87 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java @@ -10,7 +10,9 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.HistogramValues; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafHistogramFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; @@ -75,16 +77,24 @@ AbstractDownsampleFieldProducer fieldProducer() { return fieldProducer; } + public HistogramValues getHistogramLeaf(LeafReaderContext context) throws IOException { + LeafHistogramFieldData histogramFieldData = (LeafHistogramFieldData) fieldData.load(context); + return histogramFieldData.getHistogramValues(); + } + private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.SamplingMethod samplingMethod) { assert "aggregate_metric_double".equals(fieldType.typeName()) == false : "Aggregate metric double should be handled by a dedicated FieldValueFetcher"; + if (TDigestHistogramFieldProducer.TYPE.equals(fieldType.typeName())) { + return TDigestHistogramFieldProducer.create(name(), samplingMethod); + } if (fieldType.getMetricType() != null) { return switch (fieldType.getMetricType()) { case GAUGE -> NumericMetricFieldProducer.createFieldProducerForGauge(name(), samplingMethod); case COUNTER -> LastValueFieldProducer.createForMetric(name()); case HISTOGRAM -> { - if ("exponential_histogram".equals(fieldType.typeName())) { - yield ExponentialHistogramMetricFieldProducer.createMetricProducerForExponentialHistogram(name(), samplingMethod); + if (ExponentialHistogramMetricFieldProducer.TYPE.equals(fieldType.typeName())) { + yield ExponentialHistogramMetricFieldProducer.create(name(), samplingMethod); } throw new IllegalArgumentException("Time series metrics supports only exponential histogram"); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java index 5baabb2f0e9bc..0b022228b94d0 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java @@ -12,7 +12,6 @@ import org.elasticsearch.exponentialhistogram.ExponentialHistogram; import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent; import org.elasticsearch.index.fielddata.FormattedDocValues; -import org.elasticsearch.index.fielddata.HistogramValue; import org.elasticsearch.index.mapper.flattened.FlattenedFieldSyntheticWriterHelper; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; @@ -44,9 +43,9 @@ class LastValueFieldProducer extends AbstractDownsampleFieldProducer values = new ArrayList<>(); - final List counts = new ArrayList<>(); - while (histogramValue.next()) { - values.add(histogramValue.value()); - counts.add(histogramValue.count()); - } - builder.startObject(name()).field("counts", counts).field("values", values).endObject(); - } - } - } - static final class FlattenedFieldProducer extends LastValueFieldProducer { private FlattenedFieldProducer(String name, boolean producesMultiValue) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java index 060ca351702ce..668f09d78637c 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java @@ -30,7 +30,8 @@ public class RestDownsampleAction extends BaseRestHandler { private final Set CAPABILITIES = Set.of( "downsample.sampling_mode.last_value", "downsample.multi_field_fix", - "downsampling.exponential_histograms" + "downsampling.exponential_histograms", + "downsampling.tdigest_histograms" ); @Override diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducer.java new file mode 100644 index 0000000000000..cbb747eeee387 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducer.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.tdigest.Centroid; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Class that collects all raw values for an exponential histogram metric field and computes its aggregate (downsampled) + * values. + */ +abstract class TDigestHistogramFieldProducer extends AbstractDownsampleFieldProducer { + + static final String TYPE = "histogram"; + public static final int COMPRESSION = 100; + + TDigestHistogramFieldProducer(String name) { + super(name); + } + + /** + * @return the requested produces based on the sampling method for metric of type exponential histogram + */ + public static TDigestHistogramFieldProducer create(String name, DownsampleConfig.SamplingMethod samplingMethod) { + return switch (samplingMethod) { + case AGGREGATE -> new Aggregate(name); + case LAST_VALUE -> new LastValue(name); + }; + } + + private static class Aggregate extends TDigestHistogramFieldProducer { + + private TDigestState tDigestState = null; + + Aggregate(String name) { + super(name); + } + + public void collect(HistogramValues docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + if (tDigestState == null) { + // TODO: figure out what circuit breaker to use here and in the other histogram + tDigestState = TDigestState.create(new NoopCircuitBreaker("downsampling-histograms"), COMPRESSION); + } + final HistogramValue sketch = docValues.histogram(); + while (sketch.next()) { + tDigestState.add(sketch.value(), sketch.count()); + } + } + } + + @Override + public void reset() { + isEmpty = true; + tDigestState = null; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + Iterator centroids = tDigestState.uniqueCentroids(); + final List values = new ArrayList<>(); + final List counts = new ArrayList<>(); + while (centroids.hasNext()) { + Centroid centroid = centroids.next(); + values.add(centroid.mean()); + counts.add(centroid.count()); + } + builder.startObject(name()).field("counts", counts).field("values", values).endObject(); + tDigestState.close(); + } + } + } + + private static class LastValue extends TDigestHistogramFieldProducer { + + private HistogramValue lastValue = null; + + LastValue(String name) { + super(name); + } + + public void collect(HistogramValues docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + return; + } + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + lastValue = docValues.histogram(); + return; + } + } + + @Override + public void reset() { + isEmpty = true; + lastValue = null; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + final HistogramValue histogramValue = lastValue; + final List values = new ArrayList<>(); + final List counts = new ArrayList<>(); + while (histogramValue.next()) { + values.add(histogramValue.value()); + counts.add(histogramValue.count()); + } + builder.startObject(name()).field("counts", counts).field("values", values).endObject(); + } + } + } +} diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducerTests.java new file mode 100644 index 0000000000000..cf1e1a6f9a56f --- /dev/null +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TDigestHistogramFieldProducerTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.internal.hppc.IntArrayList; +import org.apache.lucene.internal.hppc.IntObjectHashMap; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class TDigestHistogramFieldProducerTests extends ESTestCase { + + public void testLastValueProducer() throws IOException { + var producer = TDigestHistogramFieldProducer.create("my-histogram", DownsampleConfig.SamplingMethod.LAST_VALUE); + assertTrue(producer.isEmpty()); + assertEquals("my-histogram", producer.name()); + + var docValues = createValuesInstance( + IntArrayList.from(1, 2), + new HistogramValue[] { + histogram(new double[] { 1, 2 }, new long[] { 1, 2 }), + histogram(new double[] { 1, 4 }, new long[] { 3, 4 }) } + ); + producer.collect(docValues, IntArrayList.from(1, 2)); + assertFalse(producer.isEmpty()); + + var builder = new XContentBuilder(XContentType.JSON.xContent(), new ByteArrayOutputStream()); + builder.startObject(); + producer.write(builder); + builder.endObject(); + var content = Strings.toString(builder); + assertThat(content, equalTo("{\"my-histogram\":{\"counts\":[1,2],\"values\":[1.0,2.0]}}")); + } + + public void testAggregateProducer() throws IOException { + var producer = TDigestHistogramFieldProducer.create("my-histogram", DownsampleConfig.SamplingMethod.AGGREGATE); + assertTrue(producer.isEmpty()); + assertEquals("my-histogram", producer.name()); + + var docValues = createValuesInstance( + IntArrayList.from(1, 2), + new HistogramValue[] { + histogram(new double[] { 1, 2 }, new long[] { 1, 2 }), + histogram(new double[] { 1, 4 }, new long[] { 3, 4 }) } + ); + producer.collect(docValues, IntArrayList.from(1, 2)); + assertFalse(producer.isEmpty()); + + var builder = new XContentBuilder(XContentType.JSON.xContent(), new ByteArrayOutputStream()); + builder.startObject(); + producer.write(builder); + builder.endObject(); + var content = Strings.toString(builder); + assertThat(content, equalTo("{\"my-histogram\":{\"counts\":[4,2,4],\"values\":[1.0,2.0,4.0]}}")); + } + + HistogramValues createValuesInstance(IntArrayList docIdBuffer, HistogramValue[] values) { + return new HistogramValues() { + final IntObjectHashMap docIdToValue = IntObjectHashMap.from(docIdBuffer.toArray(), values); + + int currentDocId = -1; + + @Override + public boolean advanceExact(int target) throws IOException { + currentDocId = target; + return docIdToValue.containsKey(target); + } + + @Override + public HistogramValue histogram() { + return docIdToValue.get(currentDocId); + } + }; + } + + private HistogramValue histogram(double[] value, long[] count) { + return new HistogramValue() { + int i = -1; + + @Override + public boolean next() { + i++; + return i < value.length; + } + + @Override + public double value() { + return value[i]; + } + + @Override + public long count() { + return count[i]; + } + }; + } + +}