From 77ae4c33b27893a9b419521d021db61f55899bfe Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 12 Sep 2022 11:49:47 -0400 Subject: [PATCH 1/2] REST tests for `moving_fn` agg This expands on the REST layer tests for the `moving_fn` agg asserting the results of the various moving functions, some failure cases, and some access edge cases. --- .../test/painless/70_mov_fn_agg.yml | 308 ------- .../test/painless/70_moving_fn_agg.yml | 767 ++++++++++++++++++ .../pipeline/MovFnPipelineAggregator.java | 4 +- 3 files changed, 769 insertions(+), 310 deletions(-) delete mode 100644 modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml create mode 100644 modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_moving_fn_agg.yml diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml deleted file mode 100644 index d699b58607da3..0000000000000 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml +++ /dev/null @@ -1,308 +0,0 @@ -# Sanity integration test to make sure the custom context and whitelist work for moving_fn pipeline agg -# -setup: - - do: - indices.create: - index: test - body: - mappings: - properties: - value_field: - type: integer - date: - type: date - - - do: - bulk: - refresh: true - body: - - index: - _index: test - _id: "1" - - date: "2017-01-01T00:00:00" - value_field: 1 - - index: - _index: test - _id: "2" - - date: "2017-01-02T00:00:00" - value_field: 2 - - index: - _index: test - _id: "3" - - date: "2017-01-03T00:00:00" - value_field: 3 - - index: - _index: test - _id: "4" - - date: "2017-01-04T00:00:00" - value_field: 4 - - index: - _index: test - _id: "5" - - date: "2017-01-05T00:00:00" - value_field: 5 - - index: - _index: test - _id: "6" - - date: "2017-01-06T00:00:00" - value_field: 6 - - - do: - indices.refresh: - index: [test] - ---- -"max": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.max(values)" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - - is_false: aggregations.the_histo.buckets.0.the_mov_fn.value - - match: { aggregations.the_histo.buckets.1.the_mov_fn.value: 1.0 } - - match: { aggregations.the_histo.buckets.2.the_mov_fn.value: 2.0 } - - match: { aggregations.the_histo.buckets.3.the_mov_fn.value: 3.0 } - - match: { aggregations.the_histo.buckets.4.the_mov_fn.value: 4.0 } - - match: { aggregations.the_histo.buckets.5.the_mov_fn.value: 5.0 } - ---- -"min": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.min(values)" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - - is_false: aggregations.the_histo.buckets.0.the_mov_fn.value - - match: { aggregations.the_histo.buckets.1.the_mov_fn.value: 1.0 } - - match: { aggregations.the_histo.buckets.2.the_mov_fn.value: 1.0 } - - match: { aggregations.the_histo.buckets.3.the_mov_fn.value: 1.0 } - - match: { aggregations.the_histo.buckets.4.the_mov_fn.value: 2.0 } - - match: { aggregations.the_histo.buckets.5.the_mov_fn.value: 3.0 } - ---- -"sum": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.sum(values)" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - - match: { aggregations.the_histo.buckets.0.the_mov_fn.value: 0.0 } - - match: { aggregations.the_histo.buckets.1.the_mov_fn.value: 1.0 } - - match: { aggregations.the_histo.buckets.2.the_mov_fn.value: 3.0 } - - match: { aggregations.the_histo.buckets.3.the_mov_fn.value: 6.0 } - - match: { aggregations.the_histo.buckets.4.the_mov_fn.value: 9.0 } - - match: { aggregations.the_histo.buckets.5.the_mov_fn.value: 12.0 } - ---- -"unweightedAvg": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.unweightedAvg(values)" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - - ---- -"linearWeightedAvg": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.linearWeightedAvg(values)" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - - ---- -"ewma": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.ewma(values, 0.1)" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - - ---- -"holt": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.holt(values, 0.1, 0.1)" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - - ---- -"holtWinters": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 1 - script: "if (values.length > 1) { MovingFunctions.holtWinters(values, 0.1, 0.1, 0.1, 1, true)}" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } - ---- -"stdDev": - - - do: - search: - rest_total_hits_as_int: true - body: - size: 0 - aggs: - the_histo: - date_histogram: - field: "date" - calendar_interval: "1d" - aggs: - the_avg: - avg: - field: "value_field" - the_mov_fn: - moving_fn: - buckets_path: "the_avg" - window: 3 - script: "MovingFunctions.stdDev(values, MovingFunctions.unweightedAvg(values))" - - - match: { hits.total: 6 } - - length: { hits.hits: 0 } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_moving_fn_agg.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_moving_fn_agg.yml new file mode 100644 index 0000000000000..d2ff73ab16b1c --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_moving_fn_agg.yml @@ -0,0 +1,767 @@ +setup: + - do: + bulk: + index: no_gaps + refresh: true + body: + - { "index": { } } + - { "n": 10, "@timestamp": "2022-01-01T00:00:00", "v": 1 } + - { "index": { } } + - { "n": 20, "@timestamp": "2022-01-01T01:00:00", "v": 2 } + - { "index": { } } + - { "n": 30, "@timestamp": "2022-01-01T02:00:00", "v": 1 } + - { "index": { } } + - { "n": 40, "@timestamp": "2022-01-01T03:00:00", "v": 4 } + - { "index": { } } + - { "n": 50, "@timestamp": "2022-01-01T04:00:00", "v": 5 } + - { "index": { } } + - { "n": 60, "@timestamp": "2022-01-01T05:00:00", "v": 9 } + + - do: + bulk: + index: gaps + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2022-01-01T00:00:00", "v": 1 } + - { "index": { } } + - { "@timestamp": "2022-01-01T01:00:00", "v": 2 } + - { "index": { } } + - { "@timestamp": "2022-01-01T02:00:00", "v": 1 } + - { "index": { } } + - { "@timestamp": "2022-01-01T04:00:00", "v": 5 } + - { "index": { } } + - { "@timestamp": "2022-01-01T05:00:00", "v": 9 } + +--- +in date_histogram: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.min(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +in histogram: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + n: + histogram: + field: n + interval: 10 + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.min(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.n.buckets: 6 } + - is_false: aggregations.n.buckets.0.d.value + - close_to: { aggregations.n.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +min: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.min(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +max: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.max(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 4.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 5.000, error: 0.0005 } } + +--- +sum: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.sum(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - close_to: { aggregations.@timestamp.buckets.0.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 5.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 9.000, error: 0.0005 } } + +--- +stdDev: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.stdDev(values, MovingFunctions.unweightedAvg(values))" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 0.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 0.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 0.500, error: 0.0005 } } + +--- +unweightedAvg: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.unweightedAvg(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.500, error: 0.0005 } } + +--- +linearWeightedAvg: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.linearWeightedAvg(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.250, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.500, error: 0.0005 } } + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 3 + script: "MovingFunctions.linearWeightedAvg(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.143, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.286, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.429, error: 0.0005 } } + +--- +ewma: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.ewma(values, 0.3)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.300, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.700, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.900, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.300, error: 0.0005 } } + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 3 + script: "MovingFunctions.ewma(values, 0.3)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.300, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.210, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.390, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 2.830, error: 0.0005 } } + +--- +holt: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.holt(values, 0.3, 0.1)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.300, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.700, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.900, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.300, error: 0.0005 } } + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 3 + script: "MovingFunctions.holt(values, 0.3, 0.1)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.300, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.231, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.369, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 2.893, error: 0.0005 } } + +--- +holtWinters: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 5 + script: "if (values.length > 4) {MovingFunctions.holtWinters(values, 0.3, 0.1, 0.1, 2, false)}" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - close_to: { aggregations.@timestamp.buckets.0.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.028, error: 0.0005 } } + +--- +math on results: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.max(values) - MovingFunctions.min(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 1.000, error: 0.0005 } } + +--- +shift: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + shift: 1 + script: "MovingFunctions.min(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - close_to: { aggregations.@timestamp.buckets.0.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 4.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 5.000, error: 0.0005 } } + +--- +format: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.min(values)" + format: "0.00" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + - is_false: aggregations.@timestamp.buckets.0.d.value_as_string + - match: { aggregations.@timestamp.buckets.1.d.value_as_string: "1.00" } + - match: { aggregations.@timestamp.buckets.2.d.value_as_string: "1.00" } + - match: { aggregations.@timestamp.buckets.3.d.value_as_string: "1.00" } + - match: { aggregations.@timestamp.buckets.4.d.value_as_string: "1.00" } + - match: { aggregations.@timestamp.buckets.5.d.value_as_string: "4.00" } + +--- +gap_policy=skip: + - skip: + features: close_to + + - do: + search: + index: gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.max(values)" + gap_policy: skip + - match: { hits.total.value: 5 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 2.000, error: 0.0005 } } + - is_false: aggregations.@timestamp.buckets.3.d.value + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 5.000, error: 0.0005 } } + +--- +gap_policy=insert_zeros: + - skip: + features: close_to + + - do: + search: + index: gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.max(values)" + gap_policy: insert_zeros + - match: { hits.total.value: 5 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 5.000, error: 0.0005 } } + +--- +gap_policy=keep_value: + - skip: + features: close_to + + - do: + search: + index: gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.max(values)" + gap_policy: keep_values + - match: { hits.total.value: 5 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 5.000, error: 0.0005 } } + +--- +dotted name: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + "v.v": { avg: { field: v } } + d: + moving_fn: + buckets_path: "v.v.value" + window: 2 + script: "MovingFunctions.min(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +dotted value: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: + percentiles: + field: v + percents: [ 50, 99.9 ] + d: + moving_fn: + buckets_path: "v[99.9]" + window: 2 + script: "MovingFunctions.min(values)" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +not results: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + query: + match: + missing_field: not found + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "v" + window: 2 + script: "MovingFunctions.min(values)" + - match: { hits.total.value: 0 } + - length: { aggregations.@timestamp.buckets: 0 } + +--- +bad path: + - do: + catch: '/Validation Failed: 1: No aggregation found for path \[missing\];/' + search: + index: no_gaps + body: + size: 0 + query: + match: + missing_field: not found + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + moving_fn: + buckets_path: "missing" + window: 2 + script: "MovingFunctions.min(values)" diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregator.java index 9b2d938630417..f4995762cc789 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregator.java @@ -111,7 +111,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, AggregationRe // don't need null checks, etc. int fromIndex = clamp(index - window + shift, values); int toIndex = clamp(index + shift, values); - double movavg = executableScript.execute( + double result = executableScript.execute( vars, values.subList(fromIndex, toIndex).stream().mapToDouble(Double::doubleValue).toArray() ); @@ -119,7 +119,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, AggregationRe List aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false) .map(InternalAggregation.class::cast) .collect(Collectors.toCollection(ArrayList::new)); - aggs.add(new InternalSimpleValue(name(), movavg, formatter, metadata())); + aggs.add(new InternalSimpleValue(name(), result, formatter, metadata())); newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs)); index++; } From 38cd37b5ebe41f5210e0acc0e8422e7869f27acb Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 13 Sep 2022 14:21:14 -0400 Subject: [PATCH 2/2] Point tests inside of rest-api-tests to the right spot --- .../test/search.aggregation/250_moving_fn.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/250_moving_fn.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/250_moving_fn.yml index 3742fb976a058..4e977468927f9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/250_moving_fn.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/250_moving_fn.yml @@ -1,3 +1,4 @@ +# There are many more tests under modules/lang-painless/...moving_fn.yml so they can use painless --- "Bad window": @@ -24,13 +25,13 @@ moving_fn: buckets_path: "the_avg" window: -1 - script: "MovingFunctions.windowMax(values)" + script: "MovingFunctions.max(values)" --- "Not under date_histo": - do: - catch: /\[window\] must be a positive, non-zero integer\./ + catch: /moving_fn aggregation \[the_mov_fn\] must have a histogram, date_histogram or auto_date_histogram as parent but doesn't have a parent/ search: rest_total_hits_as_int: true body: @@ -42,6 +43,5 @@ the_mov_fn: moving_fn: buckets_path: "the_avg" - window: -1 - script: "MovingFunctions.windowMax(values)" - + window: 1 + script: "MovingFunctions.max(values)"