diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java index 007c22920b15e..b9c2723d4ad78 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java @@ -12,6 +12,7 @@ import org.elasticsearch.aggregations.bucket.adjacency.InternalAdjacencyMatrix; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.elasticsearch.aggregations.bucket.histogram.InternalAutoDateHistogram; +import org.elasticsearch.aggregations.pipeline.BucketSortPipelineAggregationBuilder; import org.elasticsearch.aggregations.pipeline.Derivative; import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.aggregations.pipeline.MovFnPipelineAggregationBuilder; @@ -44,6 +45,11 @@ public List getAggregations() { @Override public List getPipelineAggregations() { return List.of( + new PipelineAggregationSpec( + BucketSortPipelineAggregationBuilder.NAME, + BucketSortPipelineAggregationBuilder::new, + BucketSortPipelineAggregationBuilder::parse + ), new PipelineAggregationSpec( DerivativePipelineAggregationBuilder.NAME, DerivativePipelineAggregationBuilder::new, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java similarity index 97% rename from server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java index 15fa731e154a3..2bd65c6b0a99c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java @@ -5,12 +5,14 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregator.java similarity index 96% rename from server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregator.java rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregator.java index 4e3cef67acd01..709744450b361 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregator.java @@ -5,13 +5,15 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.pipeline.BucketHelpers; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilderTests.java similarity index 88% rename from server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortTests.java rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilderTests.java index 289898416744f..bb67c8da7eca4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilderTests.java @@ -5,9 +5,12 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; +import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase; +import org.elasticsearch.search.aggregations.pipeline.BucketHelpers; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; @@ -19,7 +22,11 @@ import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; -public class BucketSortTests extends BasePipelineAggregationTestCase { +public class BucketSortPipelineAggregationBuilderTests extends BasePipelineAggregationTestCase { + @Override + protected List plugins() { + return List.of(new AggregationsPlugin()); + } @Override protected BucketSortPipelineAggregationBuilder createTestAggregatorFactory() { diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/bucket_sort.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/bucket_sort.yml new file mode 100644 index 0000000000000..771ca0dab34e2 --- /dev/null +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/bucket_sort.yml @@ -0,0 +1,554 @@ +setup: + - do: + bulk: + index: test + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2022-01-01T00:00:00", "v": 1, "a": 2, "g": 12 } + - { "index": { } } + - { "@timestamp": "2022-01-01T01:00:00", "v": 2, "a": 9 } + - { "index": { } } + - { "@timestamp": "2022-01-01T02:00:00", "v": 1, "a": 1, "g": 11 } + +--- +single sort: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: + max: + field: v + sort: + bucket_sort: + sort: + - v: desc + + - match: { aggregations.timestamp.buckets.0.v.value: 2.0 } + - match: { aggregations.timestamp.buckets.1.v.value: 1.0 } + - match: { aggregations.timestamp.buckets.2.v.value: 1.0 } + +--- +two sorts: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: + max: + field: v + a: + max: + field: a + sort: + bucket_sort: + sort: + - v: desc + - a: asc + + - match: { aggregations.timestamp.buckets.0.v.value: 2.0 } + - match: { aggregations.timestamp.buckets.1.v.value: 1.0 } + - match: { aggregations.timestamp.buckets.2.v.value: 1.0 } + - match: { aggregations.timestamp.buckets.0.a.value: 9.0 } + - match: { aggregations.timestamp.buckets.1.a.value: 1.0 } + - match: { aggregations.timestamp.buckets.2.a.value: 2.0 } + +--- +sort terms on _key: + - do: + search: + body: + aggs: + a: + terms: + field: a + order: + - v: desc + size: 2 + aggs: + v: + max: + field: v + sort: + bucket_sort: + sort: + - _key: asc + + - length: { aggregations.a.buckets: 2 } + - match: { aggregations.a.buckets.0.v.value: 1.0 } + - match: { aggregations.a.buckets.1.v.value: 2.0 } + +--- +sort terms on _count: + - do: + search: + body: + aggs: + a: + terms: + field: v + order: + - _key: asc + aggs: + sort: + bucket_sort: + sort: + - _count: desc + + - match: { aggregations.a.buckets.0.key: 1 } + - match: { aggregations.a.buckets.1.key: 2 } + - match: { aggregations.a.buckets.0.doc_count: 2 } + - match: { aggregations.a.buckets.1.doc_count: 1 } + +--- +sort terms on _count with secondary sort: + - do: + bulk: + index: test + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2022-01-01T00:00:00", "v": 3, "a": 8 } + - { "index": { } } + - { "@timestamp": "2022-01-01T02:00:00", "v": 3, "a": 1 } + + - do: + search: + body: + aggs: + a: + terms: + field: v + order: + - _key: asc + aggs: + a: + max: + field: a + sort1: + bucket_sort: + sort: + - _count: desc + - a: desc + + - match: { aggregations.a.buckets.0.key: 3 } + - match: { aggregations.a.buckets.1.key: 1 } + - match: { aggregations.a.buckets.2.key: 2 } + - match: { aggregations.a.buckets.0.doc_count: 2 } + - match: { aggregations.a.buckets.1.doc_count: 2 } + - match: { aggregations.a.buckets.2.doc_count: 1 } + - match: { aggregations.a.buckets.0.a.value: 8 } + - match: { aggregations.a.buckets.1.a.value: 2 } + - match: { aggregations.a.buckets.2.a.value: 9 } + +--- +sort date_histogram on _key: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + sort: + bucket_sort: + sort: + - _key: desc + + - length: { aggregations.timestamp.buckets: 3 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T01:00:00.000Z" } + - match: { aggregations.timestamp.buckets.2.key_as_string: "2022-01-01T00:00:00.000Z" } + +--- +default gap_policy is skip: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + sort: + bucket_sort: + sort: + - g: asc + + - length: { aggregations.timestamp.buckets: 2 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T00:00:00.000Z" } + - match: { aggregations.timestamp.buckets.0.g.value: 11 } + - match: { aggregations.timestamp.buckets.1.g.value: 12 } + +--- +gap_policy=skip: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + sort: + bucket_sort: + gap_policy: skip + sort: + - g: asc + + - length: { aggregations.timestamp.buckets: 2 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T00:00:00.000Z" } + - match: { aggregations.timestamp.buckets.0.g.value: 11 } + - match: { aggregations.timestamp.buckets.1.g.value: 12 } + +--- +gap_policy=skip and size: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + sort: + bucket_sort: + gap_policy: skip + sort: + - g: asc + size: 1 + + - length: { aggregations.timestamp.buckets: 1 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.0.g.value: 11 } + +--- +gap_policy=skip and primary has gaps but secondary doesn't: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + sort: + bucket_sort: + gap_policy: skip + sort: + - g: asc + - _key: asc + + - length: { aggregations.timestamp.buckets: 3 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T00:00:00.000Z" } + - match: { aggregations.timestamp.buckets.2.key_as_string: "2022-01-01T01:00:00.000Z" } + - match: { aggregations.timestamp.buckets.0.g.value: 11 } + - match: { aggregations.timestamp.buckets.1.g.value: 12 } + - is_false: aggregations.timestamp.buckets.2.g.value + +--- +gap_policy=skip and secondary has gaps but primary doesn't: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + sort: + bucket_sort: + gap_policy: skip + sort: + - _key: asc + - g: asc + + - length: { aggregations.timestamp.buckets: 3 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T00:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T01:00:00.000Z" } + - match: { aggregations.timestamp.buckets.2.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.0.g.value: 12 } + - is_false: aggregations.timestamp.buckets.1.g.value + - match: { aggregations.timestamp.buckets.2.g.value: 11 } + +--- +gap_policy=skip and primary and secondary have gaps: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + g2: + max: + field: g + sort: + bucket_sort: + gap_policy: skip + sort: + - g: asc + - g2: asc + + - length: { aggregations.timestamp.buckets: 2 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T00:00:00.000Z" } + - match: { aggregations.timestamp.buckets.0.g.value: 11 } + - match: { aggregations.timestamp.buckets.1.g.value: 12 } + - match: { aggregations.timestamp.buckets.0.g2.value: 11 } + - match: { aggregations.timestamp.buckets.1.g2.value: 12 } + +--- +gap_policy=insert_zeros: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + sort: + bucket_sort: + gap_policy: insert_zeros + sort: + - g: asc + + - length: { aggregations.timestamp.buckets: 3 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T01:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.2.key_as_string: "2022-01-01T00:00:00.000Z" } + - is_false: aggregations.timestamp.buckets.0.g.value + - match: { aggregations.timestamp.buckets.1.g.value: 11 } + - match: { aggregations.timestamp.buckets.2.g.value: 12 } + +--- +gap_policy=keep_values: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + g: + max: + field: g + sort: + bucket_sort: + gap_policy: keep_values + sort: + - g: asc + + - length: { aggregations.timestamp.buckets: 2 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T02:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T00:00:00.000Z" } + - match: { aggregations.timestamp.buckets.0.g.value: 11 } + - match: { aggregations.timestamp.buckets.1.g.value: 12 } + +--- +just limit size: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + sort: + bucket_sort: + size: 1 + + - length: { aggregations.timestamp.buckets: 1 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T00:00:00.000Z" } + +--- +just offset: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + sort: + bucket_sort: + from: 1 + + - length: { aggregations.timestamp.buckets: 2 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T01:00:00.000Z" } + - match: { aggregations.timestamp.buckets.1.key_as_string: "2022-01-01T02:00:00.000Z" } + +--- +limit size and offset: + - do: + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + sort: + bucket_sort: + size: 1 + from: 1 + + - length: { aggregations.timestamp.buckets: 1 } + - match: { aggregations.timestamp.buckets.0.key_as_string: "2022-01-01T01:00:00.000Z" } + +--- +no matches: + - do: + search: + body: + query: + match: + missing: missing + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + sort: + bucket_sort: + size: 1 + from: 1 + + - length: { aggregations.timestamp.buckets: 0 } + +--- +sort is stable: + - do: + search: + body: + aggs: + a: + terms: + field: a + aggs: + v: + max: + field: v + sort: + bucket_sort: + sort: + - v: desc + + - match: { aggregations.a.buckets.0.key: 9.0 } + - match: { aggregations.a.buckets.1.key: 1.0 } + - match: { aggregations.a.buckets.2.key: 2.0 } + - match: { aggregations.a.buckets.0.v.value: 2.0 } + - match: { aggregations.a.buckets.1.v.value: 1.0 } + - match: { aggregations.a.buckets.2.v.value: 1.0 } + +--- +no action fails: + - do: + catch: /\[bad\] is configured to perform nothing. Please set either of \[sort, size, from\] to use bucket_sort/ + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: + max: + field: v + bad: + bucket_sort: {} + +--- +invalid path fails: + - do: + catch: /No aggregation found for path \[garbage\]/ + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: + max: + field: v + bad: + bucket_sort: + sort: + - garbage + +--- +top level fails: + - do: + catch: /bucket_sort aggregation \[bad\] must be declared inside of another aggregation/ + search: + body: + aggs: + timestamp: + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: + max: + field: v + bad: + bucket_sort: + sort: + - timestamp>v diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/pipeline.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/pipeline.yml index 40139178d0347..d78b5890ee43a 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/pipeline.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/pipeline.yml @@ -354,28 +354,6 @@ setup: - match: { aggregations.the_max.keys.0: "4" } --- -"Top level bucket_sort": - - skip: - version: " - 7.7.99" - reason: This validation was changed in 7.8.0 - - - do: - catch: /bucket_sort aggregation \[the_bad_bucket_sort\] must be declared inside of another aggregation/ - search: - body: - aggs: - the_terms: - terms: - field: "int_field" - aggs: - the_max: - max: - field: "int_field" - the_bad_bucket_sort: - bucket_sort: - sort: - - the_terms>the_max ---- "Max pipeline on percentiles with incorrect name": - skip: version: " - 8.3.99" diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java deleted file mode 100644 index b0e596d6e7ab0..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java +++ /dev/null @@ -1,535 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.Avg; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; -import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.max; -import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; -import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.bucketSort; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; - -@ESIntegTestCase.SuiteScopeTestCase -public class BucketSortIT extends ESIntegTestCase { - - private static final String INDEX = "bucket-sort-it-data-index"; - private static final String INDEX_WITH_GAPS = "bucket-sort-it-data-index-with-gaps"; - - private static final String TIME_FIELD = "time"; - private static final String TERM_FIELD = "foo"; - private static final String VALUE_1_FIELD = "value_1"; - private static final String VALUE_2_FIELD = "value_2"; - - @Override - public void setupSuiteScopeCluster() throws Exception { - createIndex(INDEX, INDEX_WITH_GAPS); - client().admin() - .indices() - .preparePutMapping(INDEX) - .setSource("time", "type=date", "foo", "type=keyword", "value_1", "type=float", "value_2", "type=float") - .get(); - - int numTerms = 10; - List terms = new ArrayList<>(numTerms); - for (int i = 0; i < numTerms; ++i) { - terms.add(randomAlphaOfLengthBetween(3, 8)); - } - - long now = System.currentTimeMillis(); - long time = now - TimeValue.timeValueHours(24).millis(); - List builders = new ArrayList<>(); - while (time < now) { - for (String term : terms) { - int termCount = randomIntBetween(3, 6); - for (int i = 0; i < termCount; ++i) { - builders.add( - client().prepareIndex(INDEX).setSource(newDocBuilder(time, term, randomIntBetween(1, 10) * randomDouble())) - ); - } - } - time += TimeValue.timeValueHours(1).millis(); - } - - builders.add(client().prepareIndex(INDEX_WITH_GAPS).setSource(newDocBuilder(1, "foo", 1.0, 42.0))); - builders.add(client().prepareIndex(INDEX_WITH_GAPS).setSource(newDocBuilder(2, "foo", null, 42.0))); - builders.add(client().prepareIndex(INDEX_WITH_GAPS).setSource(newDocBuilder(3, "foo", 3.0, 42.0))); - - indexRandom(true, builders); - ensureSearchable(); - } - - private XContentBuilder newDocBuilder(long timeMillis, String fooValue, Double value1) throws IOException { - return newDocBuilder(timeMillis, fooValue, value1, null); - } - - private XContentBuilder newDocBuilder(long timeMillis, String fooValue, Double value1, Double value2) throws IOException { - XContentBuilder jsonBuilder = jsonBuilder(); - jsonBuilder.startObject(); - jsonBuilder.field(TIME_FIELD, timeMillis); - jsonBuilder.field(TERM_FIELD, fooValue); - if (value1 != null) { - jsonBuilder.field(VALUE_1_FIELD, value1); - } - if (value2 != null) { - jsonBuilder.field(VALUE_2_FIELD, value2); - } - jsonBuilder.endObject(); - return jsonBuilder; - } - - public void testEmptyBucketSort() { - SearchResponse response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation(dateHistogram("time_buckets").field(TIME_FIELD).fixedInterval(DateHistogramInterval.HOUR)) - .get(); - - assertSearchResponse(response); - - Histogram histogram = response.getAggregations().get("time_buckets"); - assertThat(histogram, notNullValue()); - // These become our baseline - List timeBuckets = histogram.getBuckets(); - ZonedDateTime previousKey = (ZonedDateTime) timeBuckets.get(0).getKey(); - for (Histogram.Bucket timeBucket : timeBuckets) { - assertThat(previousKey, lessThanOrEqualTo((ZonedDateTime) timeBucket.getKey())); - previousKey = (ZonedDateTime) timeBucket.getKey(); - } - - // Now let's test using size - response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - dateHistogram("time_buckets").field(TIME_FIELD) - .fixedInterval(DateHistogramInterval.HOUR) - .subAggregation(bucketSort("bucketSort", Collections.emptyList()).size(3)) - ) - .get(); - - assertSearchResponse(response); - - Histogram size3Histogram = response.getAggregations().get("time_buckets"); - assertThat(size3Histogram, notNullValue()); - List size3TimeBuckets = size3Histogram.getBuckets(); - - for (int i = 0; i < size3TimeBuckets.size(); ++i) { - assertThat(size3TimeBuckets.get(i).getKey(), equalTo(timeBuckets.get(i).getKey())); - } - - // Finally, let's test using size + from - response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - dateHistogram("time_buckets").field(TIME_FIELD) - .fixedInterval(DateHistogramInterval.HOUR) - .subAggregation(bucketSort("bucketSort", Collections.emptyList()).size(3).from(2)) - ) - .get(); - - assertSearchResponse(response); - - Histogram size3From2Histogram = response.getAggregations().get("time_buckets"); - assertThat(size3From2Histogram, notNullValue()); - List size3From2TimeBuckets = size3From2Histogram.getBuckets(); - - for (int i = 0; i < size3From2TimeBuckets.size(); ++i) { - assertThat(size3From2TimeBuckets.get(i).getKey(), equalTo(timeBuckets.get(i + 2).getKey())); - } - } - - public void testSortTermsOnKey() { - SearchResponse response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - terms("foos").field(TERM_FIELD).subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("_key")))) - ) - .get(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("foos"); - assertThat(terms, notNullValue()); - List termsBuckets = terms.getBuckets(); - String previousKey = (String) termsBuckets.get(0).getKey(); - for (Terms.Bucket termBucket : termsBuckets) { - assertThat(previousKey, lessThanOrEqualTo((String) termBucket.getKey())); - previousKey = (String) termBucket.getKey(); - } - } - - public void testSortTermsOnKeyWithSize() { - SearchResponse response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - terms("foos").field(TERM_FIELD) - .subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("_key"))).size(3)) - ) - .get(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("foos"); - assertThat(terms, notNullValue()); - List termsBuckets = terms.getBuckets(); - assertEquals(3, termsBuckets.size()); - String previousKey = (String) termsBuckets.get(0).getKey(); - for (Terms.Bucket termBucket : termsBuckets) { - assertThat(previousKey, lessThanOrEqualTo((String) termBucket.getKey())); - previousKey = (String) termBucket.getKey(); - } - } - - public void testSortTermsOnSubAggregation() { - SearchResponse response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - terms("foos").field(TERM_FIELD) - .subAggregation(avg("avg_value").field(VALUE_1_FIELD)) - .subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("avg_value").order(SortOrder.DESC)))) - ) - .get(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("foos"); - assertThat(terms, notNullValue()); - List termsBuckets = terms.getBuckets(); - double previousAvgValue = ((Avg) termsBuckets.get(0).getAggregations().get("avg_value")).getValue(); - for (Terms.Bucket termBucket : termsBuckets) { - Avg avg = termBucket.getAggregations().get("avg_value"); - assertThat(avg, notNullValue()); - assertThat(previousAvgValue, greaterThanOrEqualTo(avg.getValue())); - previousAvgValue = avg.getValue(); - } - - response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - terms("foos").field(TERM_FIELD) - .subAggregation(avg("avg_value").field(VALUE_1_FIELD)) - .subAggregation( - bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("avg_value").order(SortOrder.DESC))).size(2).from(3) - ) - ) - .get(); - - assertSearchResponse(response); - - Terms size2From3Terms = response.getAggregations().get("foos"); - assertThat(size2From3Terms, notNullValue()); - List size2From3TermsBuckets = size2From3Terms.getBuckets(); - for (int i = 0; i < size2From3TermsBuckets.size(); ++i) { - assertThat(size2From3TermsBuckets.get(i).getKey(), equalTo(termsBuckets.get(i + 3).getKey())); - } - } - - public void testSortTermsOnSubAggregationPreservesOrderOnEquals() { - SearchResponse response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - terms("foos").field(TERM_FIELD) - .subAggregation(bucketSort("keyBucketSort", Arrays.asList(new FieldSortBuilder("_key")))) - .subAggregation(max("max").field("missingValue").missing(1)) - .subAggregation(bucketSort("maxBucketSort", Arrays.asList(new FieldSortBuilder("max")))) - ) - .get(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("foos"); - assertThat(terms, notNullValue()); - List termsBuckets = terms.getBuckets(); - - // Since all max values are equal, we expect the order of keyBucketSort to have been preserved - String previousKey = (String) termsBuckets.get(0).getKey(); - for (Terms.Bucket termBucket : termsBuckets) { - assertThat(previousKey, lessThanOrEqualTo((String) termBucket.getKey())); - previousKey = (String) termBucket.getKey(); - } - } - - public void testSortTermsOnCountWithSecondarySort() { - SearchResponse response = client().prepareSearch(INDEX) - .setSize(0) - .addAggregation( - terms("foos").field(TERM_FIELD) - .subAggregation(avg("avg_value").field(VALUE_1_FIELD)) - .subAggregation( - bucketSort( - "bucketSort", - Arrays.asList( - new FieldSortBuilder("_count").order(SortOrder.ASC), - new FieldSortBuilder("avg_value").order(SortOrder.DESC) - ) - ) - ) - ) - .get(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("foos"); - assertThat(terms, notNullValue()); - List termsBuckets = terms.getBuckets(); - long previousCount = termsBuckets.get(0).getDocCount(); - double previousAvgValue = ((Avg) termsBuckets.get(0).getAggregations().get("avg_value")).getValue(); - for (Terms.Bucket termBucket : termsBuckets) { - Avg avg = termBucket.getAggregations().get("avg_value"); - assertThat(avg, notNullValue()); - assertThat(previousCount, lessThanOrEqualTo(termBucket.getDocCount())); - if (previousCount == termBucket.getDocCount()) { - assertThat(previousAvgValue, greaterThanOrEqualTo(avg.getValue())); - } - previousCount = termBucket.getDocCount(); - previousAvgValue = avg.getValue(); - } - } - - public void testSortDateHistogramDescending() { - SearchResponse response = client().prepareSearch(INDEX) - .addAggregation(dateHistogram("time_buckets").field(TIME_FIELD).fixedInterval(DateHistogramInterval.HOUR)) - .get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("time_buckets"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("time_buckets")); - List ascendingTimeBuckets = histo.getBuckets(); - - response = client().prepareSearch(INDEX) - .addAggregation( - dateHistogram("time_buckets").field(TIME_FIELD) - .fixedInterval(DateHistogramInterval.HOUR) - .subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("_key").order(SortOrder.DESC)))) - ) - .get(); - - assertSearchResponse(response); - - histo = response.getAggregations().get("time_buckets"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("time_buckets")); - List descendingTimeBuckets = histo.getBuckets(); - - assertThat(ascendingTimeBuckets.size(), equalTo(descendingTimeBuckets.size())); - int bucketCount = ascendingTimeBuckets.size(); - for (int i = 0; i < bucketCount; ++i) { - assertThat(ascendingTimeBuckets.get(i).getKey(), equalTo(descendingTimeBuckets.get(bucketCount - i - 1).getKey())); - } - } - - public void testSortHistogram_GivenGapsAndGapPolicyIsSkip() { - SearchResponse response = client().prepareSearch(INDEX_WITH_GAPS) - .addAggregation( - histogram("time_buckets").field(TIME_FIELD) - .interval(1) - .subAggregation(avg("avg_value").field(VALUE_1_FIELD)) - .subAggregation( - bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("avg_value").order(SortOrder.DESC))).gapPolicy( - BucketHelpers.GapPolicy.SKIP - ) - ) - ) - .get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("time_buckets"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("time_buckets")); - List timeBuckets = histo.getBuckets(); - assertThat(timeBuckets.size(), equalTo(2)); - assertThat(timeBuckets.get(0).getKey(), equalTo(3.0)); - assertThat(timeBuckets.get(1).getKey(), equalTo(1.0)); - } - - public void testSortHistogram_GivenGapsAndGapPolicyIsSkipAndSizeIsLessThanAvailableBuckets() { - SearchResponse response = client().prepareSearch(INDEX_WITH_GAPS) - .addAggregation( - histogram("time_buckets").field(TIME_FIELD) - .interval(1) - .subAggregation(avg("avg_value").field(VALUE_1_FIELD)) - .subAggregation( - bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("avg_value").order(SortOrder.DESC))).gapPolicy( - BucketHelpers.GapPolicy.SKIP - ).size(2) - ) - ) - .get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("time_buckets"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("time_buckets")); - List timeBuckets = histo.getBuckets(); - assertThat(timeBuckets.size(), equalTo(2)); - assertThat(timeBuckets.get(0).getKey(), equalTo(3.0)); - assertThat(timeBuckets.get(1).getKey(), equalTo(1.0)); - } - - public void testSortHistogram_GivenGapsAndGapPolicyIsSkipAndPrimarySortHasGaps() { - SearchResponse response = client().prepareSearch(INDEX_WITH_GAPS) - .addAggregation( - histogram("time_buckets").field(TIME_FIELD) - .interval(1) - .subAggregation(avg("avg_value_1").field(VALUE_1_FIELD)) - .subAggregation(avg("avg_value_2").field(VALUE_2_FIELD)) - .subAggregation( - bucketSort( - "bucketSort", - Arrays.asList( - new FieldSortBuilder("avg_value_1").order(SortOrder.DESC), - new FieldSortBuilder("avg_value_2").order(SortOrder.DESC) - ) - ).gapPolicy(BucketHelpers.GapPolicy.SKIP) - ) - ) - .get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("time_buckets"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("time_buckets")); - List timeBuckets = histo.getBuckets(); - assertThat(timeBuckets.size(), equalTo(3)); - assertThat(timeBuckets.get(0).getKey(), equalTo(3.0)); - assertThat(timeBuckets.get(1).getKey(), equalTo(1.0)); - assertThat(timeBuckets.get(2).getKey(), equalTo(2.0)); - } - - public void testSortHistogram_GivenGapsAndGapPolicyIsSkipAndSecondarySortHasGaps() { - SearchResponse response = client().prepareSearch(INDEX_WITH_GAPS) - .addAggregation( - histogram("time_buckets").field(TIME_FIELD) - .interval(1) - .subAggregation(avg("avg_value_1").field(VALUE_1_FIELD)) - .subAggregation(avg("avg_value_2").field(VALUE_2_FIELD)) - .subAggregation( - bucketSort( - "bucketSort", - Arrays.asList( - new FieldSortBuilder("avg_value_2").order(SortOrder.DESC), - new FieldSortBuilder("avg_value_1").order(SortOrder.ASC) - ) - ).gapPolicy(BucketHelpers.GapPolicy.SKIP) - ) - ) - .get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("time_buckets"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("time_buckets")); - List timeBuckets = histo.getBuckets(); - assertThat(timeBuckets.size(), equalTo(3)); - assertThat(timeBuckets.get(0).getKey(), equalTo(1.0)); - assertThat(timeBuckets.get(1).getKey(), equalTo(3.0)); - assertThat(timeBuckets.get(2).getKey(), equalTo(2.0)); - } - - public void testSortHistogram_GivenGapsAndGapPolicyIsInsertZeros() { - SearchResponse response = client().prepareSearch(INDEX_WITH_GAPS) - .addAggregation( - histogram("time_buckets").field(TIME_FIELD) - .interval(1) - .subAggregation(avg("avg_value").field(VALUE_1_FIELD)) - .subAggregation( - bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("avg_value").order(SortOrder.DESC))).gapPolicy( - BucketHelpers.GapPolicy.INSERT_ZEROS - ) - ) - ) - .get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("time_buckets"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("time_buckets")); - List timeBuckets = histo.getBuckets(); - assertThat(timeBuckets.size(), equalTo(3)); - assertThat(timeBuckets.get(0).getKey(), equalTo(3.0)); - assertThat(timeBuckets.get(1).getKey(), equalTo(1.0)); - assertThat(timeBuckets.get(2).getKey(), equalTo(2.0)); - } - - public void testEmptyBuckets() { - SearchResponse response = client().prepareSearch(INDEX) - .setSize(0) - .setQuery(QueryBuilders.existsQuery("non-field")) - .addAggregation( - terms("foos").field(TERM_FIELD).subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("_key")))) - ) - .get(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("foos"); - assertThat(terms, notNullValue()); - List termsBuckets = terms.getBuckets(); - assertThat(termsBuckets.isEmpty(), is(true)); - } - - public void testInvalidPath() { - Exception e = expectThrows( - ActionRequestValidationException.class, - () -> client().prepareSearch(INDEX) - .addAggregation( - terms("foos").field(TERM_FIELD).subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("invalid")))) - ) - .get() - ); - assertThat(e.getMessage(), containsString("No aggregation found for path [invalid]")); - } - - public void testNeitherSortsNorSizeSpecifiedAndFromIsDefault_ShouldThrowValidation() { - Exception e = expectThrows( - ActionRequestValidationException.class, - () -> client().prepareSearch(INDEX) - .addAggregation(terms("foos").field(TERM_FIELD).subAggregation(bucketSort("bucketSort", Collections.emptyList()))) - .get() - ); - assertThat( - e.getMessage(), - containsString("[bucketSort] is configured to perform nothing. Please set either of [sort, size, from] to use bucket_sort") - ); - } -} diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index fe29b9bf9f14e..42c20def183d8 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -190,7 +190,6 @@ import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.BucketSortPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.CumulativeSumPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketParser; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; @@ -767,13 +766,6 @@ private void registerPipelineAggregations(List plugins) { BucketSelectorPipelineAggregationBuilder::parse ) ); - registerPipelineAggregation( - new PipelineAggregationSpec( - BucketSortPipelineAggregationBuilder.NAME, - BucketSortPipelineAggregationBuilder::new, - BucketSortPipelineAggregationBuilder::parse - ) - ); registerPipelineAggregation( new PipelineAggregationSpec( SerialDiffPipelineAggregationBuilder.NAME, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java index 3c5178d16117d..3108739c8b071 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java @@ -12,7 +12,6 @@ import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.BucketSortPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.CumulativeSumPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; @@ -21,9 +20,7 @@ import org.elasticsearch.search.aggregations.pipeline.SerialDiffPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.SumBucketPipelineAggregationBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import java.util.List; import java.util.Map; public final class PipelineAggregatorBuilders { @@ -74,10 +71,6 @@ public static BucketSelectorPipelineAggregationBuilder bucketSelector(String nam return new BucketSelectorPipelineAggregationBuilder(name, script, bucketsPaths); } - public static BucketSortPipelineAggregationBuilder bucketSort(String name, List sorts) { - return new BucketSortPipelineAggregationBuilder(name, sorts); - } - public static CumulativeSumPipelineAggregationBuilder cumulativeSum(String name, String bucketsPath) { return new CumulativeSumPipelineAggregationBuilder(name, bucketsPath); }