From b75938902605d0a193afb4c61da21c5603cad67f Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 12 Apr 2021 10:52:01 -0400 Subject: [PATCH 1/2] Tests for running composite under nested aggregation (#68243) Conflicts: server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java --- .../test/search.aggregation/230_composite.yml | 135 +++++++++++- .../test/search.aggregation/380_nested.yml | 162 +++++++++++++++ .../index/cache/bitset/BitsetFilterCache.java | 4 + .../composite/CompositeAggregatorTests.java | 193 ++++++++++++++++++ .../bucket/nested/NestedAggregatorTests.java | 16 ++ 5 files changed, 509 insertions(+), 1 deletion(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/380_nested.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/230_composite.yml index bfb067b8c2084..303eff623527b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -36,6 +36,41 @@ setup: nested_long: type: long + - do: + indices.create: + index: verynested + body: + mappings: + properties: + department: + type: keyword + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + sessions: + type: nested + properties: + semester: + type: keyword + + - do: + index: + index: verynested + id: 1 + body: { "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3, "sessions": [ { "semester": "spr2021", "students": 37 }, { "semester": "fall2020", "students": 45} ] }, { "name": "Theory of Computation", "credits": 4, "sessions": [ { "semester": "spr2021", "students": 19 }, { "semester": "fall2020", "students": 14 } ] } ] } + + - do: + index: + index: verynested + id: 2 + body: { "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1, "sessions": [ { "semester": "spr2021", "students": 100 }, { "semester": "fall2020", "students": 134 } ] }, { "name": "Linear Algebra", "credits": 3, "sessions": [ { "semester": "spr2021", "students": 29 }, { "semester": "fall2020", "students": 23 } ] } ] } + - do: index: index: test @@ -80,7 +115,7 @@ setup: - do: indices.refresh: - index: [test, other] + index: [test, other, verynested] --- "Simple Composite aggregation": @@ -1030,3 +1065,101 @@ setup: - length: { aggregations.test.buckets: 1 } - match: { aggregations.test.buckets.0.key.keyword: "foo" } - match: { aggregations.test.buckets.0.doc_count: 1 } + +--- +"Nested as parent": + - do: + search: + rest_total_hits_as_int: true + index: verynested + body: + "aggregations": { + "courses": { + "nested": { "path": "courses" }, + "aggregations": { + "names": { + "composite": { + "sources": [ + "kw": {"terms": {"field": "courses.name"}} + ] + } + } + } + } + } + + - match: {hits.total: 2} + - match: {aggregations.courses.doc_count: 4} + - length: { aggregations.courses.names.buckets: 4 } + - match: { aggregations.courses.names.buckets.0.key.kw: "Linear Algebra" } + - match: { aggregations.courses.names.buckets.0.doc_count: 1} + - match: { aggregations.courses.names.buckets.1.key.kw: "Object Oriented Programming" } + - match: { aggregations.courses.names.buckets.1.doc_count: 1} + - match: { aggregations.courses.names.buckets.2.key.kw: "Precalculus" } + - match: { aggregations.courses.names.buckets.2.doc_count: 1} + - match: { aggregations.courses.names.buckets.3.key.kw: "Theory of Computation" } + - match: { aggregations.courses.names.buckets.3.doc_count: 1} + +--- +"Nested parent with compound key": + - do: + search: + rest_total_hits_as_int: true + index: verynested + body: + "aggregations": { + "sessions": { + "nested": { "path": "courses.sessions" }, + "aggregations": { + "names": { + "composite": { + "sources": [ + "kw": { "terms": { "field": "courses.sessions.semester" } } + ] + } + } + } + } + } + - match: {hits.total: 2} + - match: {aggregations.sessions.doc_count: 8} + - length: { aggregations.sessions.names.buckets: 2 } + - match: { aggregations.sessions.names.buckets.0.key.kw: "fall2020" } + - match: { aggregations.sessions.names.buckets.0.doc_count: 4} + - match: { aggregations.sessions.names.buckets.1.key.kw: "spr2021" } + - match: { aggregations.sessions.names.buckets.1.doc_count: 4} + +--- +"Nested with a nested sub aggregation": + - do: + search: + rest_total_hits_as_int: true + index: verynested + body: + "aggregations": { + "courses": { + "nested": { "path": "courses" }, + "aggregations": { + "sessions": { + "nested": { "path": "courses.sessions" }, + "aggregations": { + "names": { + "composite": { + "sources": [ + "kw": {"terms": { "field": "courses.sessions.semester" }} + ] + } + } + } + } + } + } + } + - match: {hits.total: 2} + - match: {aggregations.courses.doc_count: 4} + - match: {aggregations.courses.sessions.doc_count: 8} + - length: { aggregations.courses.sessions.names.buckets: 2 } + - match: { aggregations.courses.sessions.names.buckets.0.key.kw: "fall2020" } + - match: { aggregations.courses.sessions.names.buckets.0.doc_count: 4} + - match: { aggregations.courses.sessions.names.buckets.1.key.kw: "spr2021" } + - match: { aggregations.courses.sessions.names.buckets.1.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/380_nested.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/380_nested.yml new file mode 100644 index 0000000000000..43623846c87c3 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/380_nested.yml @@ -0,0 +1,162 @@ +--- +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + department: + type: keyword + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + sessions: + type: nested + properties: + semester: + type: keyword + + - do: + index: + index: test + id: 1 + body: { "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3, "sessions": [ { "semester": "spr2021", "students": 37 }, { "semester": "fall2020", "students": 45} ] }, { "name": "Theory of Computation", "credits": 4, "sessions": [ { "semester": "spr2021", "students": 19 }, { "semester": "fall2020", "students": 14 } ] } ] } + + - do: + index: + index: test + id: 2 + body: { "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1, "sessions": [ { "semester": "spr2021", "students": 100 }, { "semester": "fall2020", "students": 134 } ] }, { "name": "Linear Algebra", "credits": 3, "sessions": [ { "semester": "spr2021", "students": 29 }, { "semester": "fall2020", "students": 23 } ] } ] } + + - do: + indices.refresh: + index: [test] + +--- +"Single Level Nested with Terms": + - do: + search: + rest_total_hits_as_int: true + index: test + body: + "aggregations": { + "courses": { + "nested": { "path": "courses" }, + "aggregations": { + "names": { + "terms": { "field": "courses.name" } + } + } + } + } + + - match: {hits.total: 2} + - match: {aggregations.courses.doc_count: 4} + - length: { aggregations.courses.names.buckets: 4 } + - match: { aggregations.courses.names.buckets.0.key: "Linear Algebra" } + - match: { aggregations.courses.names.buckets.0.doc_count: 1} + - match: { aggregations.courses.names.buckets.1.key: "Object Oriented Programming" } + - match: { aggregations.courses.names.buckets.1.doc_count: 1} + - match: { aggregations.courses.names.buckets.2.key: "Precalculus" } + - match: { aggregations.courses.names.buckets.2.doc_count: 1} + - match: { aggregations.courses.names.buckets.3.key: "Theory of Computation" } + - match: { aggregations.courses.names.buckets.3.doc_count: 1} + +--- +"Compound nested key": + - do: + search: + rest_total_hits_as_int: true + index: test + body: + "aggregations": { + "sessions": { + "nested": { "path": "courses.sessions" }, + "aggregations": { + "names": { + "terms": { "field": "courses.sessions.semester" } + } + } + } + } + - match: {hits.total: 2} + - match: {aggregations.sessions.doc_count: 8} + - length: { aggregations.sessions.names.buckets: 2 } + - match: { aggregations.sessions.names.buckets.0.key: "fall2020" } + - match: { aggregations.sessions.names.buckets.0.doc_count: 4} + - match: { aggregations.sessions.names.buckets.1.key: "spr2021" } + - match: { aggregations.sessions.names.buckets.1.doc_count: 4} + +--- +"Nested with a nested sub aggregation": + - do: + search: + rest_total_hits_as_int: true + index: test + body: + "aggregations": { + "courses": { + "nested": { "path": "courses" }, + "aggregations": { + "sessions": { + "nested": { "path": "courses.sessions" }, + "aggregations": { + "names": { + "terms": { "field": "courses.sessions.semester" } + } + } + } + } + } + } + - match: {hits.total: 2} + - match: {aggregations.courses.doc_count: 4} + - match: {aggregations.courses.sessions.doc_count: 8} + - length: { aggregations.courses.sessions.names.buckets: 2 } + - match: { aggregations.courses.sessions.names.buckets.0.key: "fall2020" } + - match: { aggregations.courses.sessions.names.buckets.0.doc_count: 4} + - match: { aggregations.courses.sessions.names.buckets.1.key: "spr2021" } + - match: { aggregations.courses.sessions.names.buckets.1.doc_count: 4} +--- +"Nested then filter then nested then terms": + - do: + search: + rest_total_hits_as_int: true + index: test + body: + "aggregations": { + "courses": { + "nested": { "path": "courses" }, + "aggregations": { + "highpass_filter": { + "filter": { "range": {"courses.credits": { "gt": 1 }}}, + "aggregations": { + "sessions": { + "nested": { "path": "courses.sessions" }, + "aggregations": { + "names": { + "terms": { "field": "courses.sessions.semester" } + } + } + } + } + } + } + } + } + - match: {hits.total: 2} + - match: {aggregations.courses.doc_count: 4} + - match: {aggregations.courses.highpass_filter.doc_count: 3} + - match: {aggregations.courses.highpass_filter.sessions.doc_count: 6} + - length: { aggregations.courses.highpass_filter.sessions.names.buckets: 2 } + - match: { aggregations.courses.highpass_filter.sessions.names.buckets.0.key: "fall2020" } + - match: { aggregations.courses.highpass_filter.sessions.names.buckets.0.doc_count: 3} + - match: { aggregations.courses.highpass_filter.sessions.names.buckets.1.key: "spr2021" } + - match: { aggregations.courses.highpass_filter.sessions.names.buckets.1.doc_count: 3} diff --git a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index 8b666e9d64fcd..10509859bc548 100644 --- a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -124,6 +124,10 @@ private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext } final IndexReader.CacheKey coreCacheReader = cacheHelper.getKey(); final ShardId shardId = ShardUtils.extractShardId(context.reader()); + if (shardId == null) { + throw new IllegalStateException("Null shardId. If you got here from a test, you need to wrap the directory reader. " + + "see for example AggregatorTestCase#wrapInMockESDirectoryReader. If you got here in production, please file a bug."); + } if (indexSettings.getIndex().equals(shardId.getIndex()) == false) { // insanity throw new IllegalStateException("Trying to load bit set for index " + shardId.getIndex() diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index d74e200196880..ce58564fa91e3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -46,15 +46,22 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalMax; @@ -86,12 +93,14 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.search.aggregations.bucket.nested.NestedAggregatorTests.nestedObject; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class CompositeAggregatorTests extends AggregatorTestCase { private static MappedFieldType[] FIELD_TYPES; + private List objectMappers; @Override @Before @@ -106,6 +115,8 @@ public void setUp() throws Exception { FIELD_TYPES[5] = new KeywordFieldMapper.KeywordFieldType("terms"); FIELD_TYPES[6] = new IpFieldMapper.IpFieldType("ip"); FIELD_TYPES[7] = new GeoPointFieldMapper.GeoPointFieldType("geo_point"); + + objectMappers = new ArrayList<>(); } @Override @@ -113,6 +124,20 @@ public void setUp() throws Exception { public void tearDown() throws Exception { super.tearDown(); FIELD_TYPES = null; + objectMappers = null; + } + + @Override + protected List objectMappers() { + return objectMappers; + } + + @Override + protected IndexReader wrapDirectoryReader(DirectoryReader reader) throws IOException { + if (false == objectMappers().isEmpty()) { + return wrapInMockESDirectoryReader(reader); + } + return reader; } public void testUnmappedFieldWithTerms() throws Exception { @@ -573,6 +598,155 @@ public void testWithKeyword() throws Exception { ); } + /** + * This is just a template for migrating to the test case execution in {@link AggregatorTestCase}, it doesn't test anything new. + */ + public void testUsingTestCase() throws Exception { + TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("keyword").field("keyword"); + final List>> dataset = new ArrayList<>(); + dataset.addAll( + Arrays.asList( + createDocument("keyword", "a"), + createDocument("keyword", "c"), + createDocument("keyword", "a"), + createDocument("keyword", "d"), + createDocument("keyword", "c") + ) + ); + testCase( + new CompositeAggregationBuilder("name", Collections.singletonList(terms)), + new MatchAllDocsQuery(), + iw -> { + Document document = new Document(); + int id = 0; + for (Map> fields : dataset) { + document.clear(); + addToDocument(id, document, fields); + iw.addDocument(document); + id++; + } + }, + (InternalComposite result) -> { + assertEquals(3, result.getBuckets().size()); + assertEquals("{keyword=d}", result.afterKey().toString()); + assertEquals("{keyword=a}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{keyword=c}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(1).getDocCount()); + assertEquals("{keyword=d}", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + }, + FIELD_TYPES + ); + } + + /** + * Test using Nested aggregation as a parent of composite + */ + public void testSubAggregationOfNested() throws Exception { + final String nestedPath = "sellers"; + objectMappers.add(nestedObject(nestedPath)); + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + final String leafNameField = "name"; + final String rootNameField = "name"; + TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("keyword").field(nestedPath + "." + leafNameField); + NestedAggregationBuilder builder = new NestedAggregationBuilder("nestedAggName", nestedPath); + builder.subAggregation(new CompositeAggregationBuilder("compositeAggName", Collections.singletonList(terms))); + // Without after + testCase( + builder, + new MatchAllDocsQuery(), + iw -> { + // Sub-Docs + List documents = new ArrayList<>(); + documents.add(createNestedDocument("1", nestedPath, leafNameField, "Pens and Stuff", "price" , 10L)); + documents.add(createNestedDocument("1", nestedPath, leafNameField, "Pen World", "price" , 9L)); + documents.add(createNestedDocument("2", nestedPath, leafNameField, "Pens and Stuff", "price" , 5L)); + documents.add(createNestedDocument("2", nestedPath, leafNameField, "Stationary", "price" , 7L)); + // Root docs + Document root; + root = new Document(); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(sequenceIDFields.primaryTerm); + root.add(new StringField(rootNameField, new BytesRef("Ballpoint"), Field.Store.NO)); + documents.add(root); + + root = new Document(); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(new StringField(rootNameField, new BytesRef("Notebook"), Field.Store.NO)); + root.add(sequenceIDFields.primaryTerm); + documents.add(root); + iw.addDocuments(documents); + }, + (InternalSingleBucketAggregation parent) -> { + assertEquals(1, parent.getAggregations().asList().size()); + InternalComposite result = (InternalComposite) parent.getProperty("compositeAggName"); + assertEquals(3, result.getBuckets().size()); + assertEquals("{keyword=Stationary}", result.afterKey().toString()); + assertEquals("{keyword=Pen World}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals("{keyword=Pens and Stuff}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(1).getDocCount()); + assertEquals("{keyword=Stationary}", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + }, + new KeywordFieldMapper.KeywordFieldType(nestedPath + "." + leafNameField), + new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.LONG) + ); + } + + /** + * Test aggregate after with top level nested aggregation + */ + public void testSubAggregationOfNestedAggregateAfter() throws Exception { + final String nestedPath = "sellers"; + objectMappers.add(nestedObject(nestedPath)); + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + final String leafNameField = "name"; + final String rootNameField = "name"; + TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("keyword").field(nestedPath + "." + leafNameField); + NestedAggregationBuilder builder = new NestedAggregationBuilder("nestedAggName", nestedPath); + builder.subAggregation( + new CompositeAggregationBuilder("compositeAggName", Collections.singletonList(terms)) + .aggregateAfter(createAfterKey("keyword", "Pens and Stuff"))); + testCase( + builder, + new MatchAllDocsQuery(), + iw -> { + // Sub-Docs + List documents = new ArrayList<>(); + documents.add(createNestedDocument("1", nestedPath, leafNameField, "Pens and Stuff", "price" , 10L)); + documents.add(createNestedDocument("1", nestedPath, leafNameField, "Pen World", "price" , 9L)); + documents.add(createNestedDocument("2", nestedPath, leafNameField, "Pens and Stuff", "price" , 5L)); + documents.add(createNestedDocument("2", nestedPath, leafNameField, "Stationary", "price" , 7L)); + // Root docs + Document root; + root = new Document(); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(sequenceIDFields.primaryTerm); + root.add(new StringField(rootNameField, new BytesRef("Ballpoint"), Field.Store.NO)); + documents.add(root); + + root = new Document(); + root.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.FIELD_TYPE)); + root.add(new StringField(rootNameField, new BytesRef("Notebook"), Field.Store.NO)); + root.add(sequenceIDFields.primaryTerm); + documents.add(root); + iw.addDocuments(documents); + }, + (InternalSingleBucketAggregation parent) -> { + assertEquals(1, parent.getAggregations().asList().size()); + InternalComposite result = (InternalComposite) parent.getProperty("compositeAggName"); + assertEquals(1, result.getBuckets().size()); + assertEquals("{keyword=Stationary}", result.afterKey().toString()); + assertEquals("{keyword=Stationary}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + }, + new KeywordFieldMapper.KeywordFieldType(nestedPath + "." + leafNameField), + new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.LONG) + ); + } + public void testWithKeywordAndMissingBucket() throws Exception { final List>> dataset = new ArrayList<>(); dataset.addAll( @@ -2401,6 +2575,10 @@ private static IndexSettings createIndexSettings(Sort sort) { private void addToDocument(int id, Document doc, Map> keys) { doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); + addToDocument(doc, keys); + } + + private void addToDocument(Document doc, Map> keys) { for (Map.Entry> entry : keys.entrySet()) { final String name = entry.getKey(); for (Object value : entry.getValue()) { @@ -2456,6 +2634,21 @@ private static Map> createDocument(Object... fields) { return map; } + private Document createNestedDocument(String id, String nestedPath, Object... rawFields) { + assert rawFields.length % 2 == 0; + Document doc = new Document(); + doc.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); + doc.add(new Field(NestedPathFieldMapper.NAME, nestedPath, NestedPathFieldMapper.Defaults.FIELD_TYPE)); + Object[] fields = new Object[rawFields.length]; + for (int i = 0; i < fields.length; i+=2) { + assert rawFields[i] instanceof String; + fields[i] = nestedPath + "." + rawFields[i]; + fields[i+1] = rawFields[i+1]; + } + addToDocument(doc, createDocument(fields)); + return doc; + } + private static long asLong(String dateTime) { return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index 2c048123a8154..11990e1ce440b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -88,6 +88,22 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.nested; import static org.hamcrest.Matchers.equalTo; +/** + * Tests for the Nested aggregator. + * + *

+ * Notes to people wanting to add nested aggregation tests to other test classes: + *

    + *
  • Nested aggregations require a different {@link DirectoryReader} implementation than we usually use in aggregation tests. You'll + * need to override {@link AggregatorTestCase#wrapDirectoryReader} as is done in this class
    • + *
  • Nested aggregations also require object mappers to be configured. You can mock this by overriding + * {@link AggregatorTestCase#objectMappers()} as seen below
    • + *
  • In a production nested field setup, we'll automatically prefix the nested path to the leaf document field names. This helps + * prevent name collisions between "levels" of nested docs. This mechanism isn't invoked during unit tests, so preventing field name + * collisions should be done by hand. For the closest approximation of how it looks in prod, leaf docs should have field names + * prefixed with the nested path: nestedPath + "." + fieldName
    • + *
+ */ public class NestedAggregatorTests extends AggregatorTestCase { private static final String VALUE_FIELD_NAME = "number"; From b81e3ef910bb9343930b465980ad8368b48262ab Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Mon, 12 Apr 2021 12:33:15 -0400 Subject: [PATCH 2/2] NestedPathFieldMapper isn't in 7.x --- .../bucket/composite/CompositeAggregatorTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index ce58564fa91e3..76ae2326397b1 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -50,10 +50,10 @@ import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; +import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -2638,7 +2638,7 @@ private Document createNestedDocument(String id, String nestedPath, Object... ra assert rawFields.length % 2 == 0; Document doc = new Document(); doc.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - doc.add(new Field(NestedPathFieldMapper.NAME, nestedPath, NestedPathFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(TypeFieldMapper.NAME, "__" + nestedPath, TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); Object[] fields = new Object[rawFields.length]; for (int i = 0; i < fields.length; i+=2) { assert rawFields[i] instanceof String;