From 474f4e58e7b73773790509a1c797d2c290a08445 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 5 Mar 2020 20:18:15 -0800 Subject: [PATCH 1/3] add tests for missing aggregation Test with unmapped fields and using the missing parameter, which isn't very useful with this aggregation but does work as expected. Also includes yaml tests For #42949 --- .../test/search.aggregation/320_missing.yml | 98 +++++++++++ .../missing/MissingAggregatorTests.java | 157 +++++++++++++----- 2 files changed, 211 insertions(+), 44 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml new file mode 100644 index 0000000000000..bd7bb925a7adb --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/320_missing.yml @@ -0,0 +1,98 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_replicas: 0 + mappings: + properties: + field1: + type: long + field2: + type: long + field3: + type: long + - do: + bulk: + refresh: true + body: + - index: + _index: test + - field1: 100 + - index: + _index: test + - field1: 200 + - index: + _index: test + - field1: 300 + field2: 300 + +--- +"match all": + + - do: + search: + rest_total_hits_as_int: true + body: + aggs: + missing_agg: + missing: + field: field3 + + - match: { hits.total: 3 } + - length: { hits.hits: 3 } + + - match: { aggregations.missing_agg.doc_count: 3 } + +--- +"match some": + + - do: + search: + rest_total_hits_as_int: true + body: + aggs: + missing_agg: + missing: + field: field2 + + - match: { hits.total: 3 } + - length: { hits.hits: 3 } + + - match: { aggregations.missing_agg.doc_count: 2 } + +--- +"match none": + + - do: + search: + rest_total_hits_as_int: true + body: + aggs: + missing_agg: + missing: + field: field1 + + - match: { hits.total: 3 } + - length: { hits.hits: 3 } + + - match: { aggregations.missing_agg.doc_count: 0 } + +--- +"missing param": + + - do: + search: + rest_total_hits_as_int: true + body: + aggs: + missing_agg: + missing: + field: field3 + missing: 1 + + - match: { hits.total: 3 } + - length: { hits.hits: 3 } + + - match: { aggregations.missing_agg.doc_count: 0 } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index dbc3ac3b490c2..934d6e934bc73 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -28,81 +28,115 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; -import java.util.Collections; +import java.util.Collection; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; import java.util.function.Consumer; +import static java.util.Collections.singleton; + public class MissingAggregatorTests extends AggregatorTestCase { public void testMatchNoDocs() throws IOException { int numDocs = randomIntBetween(10, 200); + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + fieldType.setName("field"); testBothCases(numDocs, - "field", + fieldType.name(), Queries.newMatchAllQuery(), - doc -> doc.add(new SortedNumericDocValuesField("field", randomLong())), + builder -> {}, + (index, doc) -> doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())), internalMissing -> { assertEquals(internalMissing.getDocCount(), 0); assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); - }); + }, + singleton(fieldType)); } public void testMatchAllDocs() throws IOException { int numDocs = randomIntBetween(10, 200); + + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + fieldType.setName("field"); + final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + anotherFieldType.setName("another_field"); + testBothCases(numDocs, - "field", + fieldType.name(), Queries.newMatchAllQuery(), - doc -> doc.add(new SortedNumericDocValuesField("another_field", randomLong())), + builder -> {}, + (index, doc) -> doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())), internalMissing -> { assertEquals(internalMissing.getDocCount(), numDocs); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); - }); + }, + List.of(fieldType, anotherFieldType)); } public void testMatchSparse() throws IOException { int numDocs = randomIntBetween(100, 200); final AtomicInteger count = new AtomicInteger(); + + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + fieldType.setName("field"); + final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + anotherFieldType.setName("another_field"); + testBothCases(numDocs, - "field", + fieldType.name(), Queries.newMatchAllQuery(), - doc -> { + builder -> {}, + (index, doc) -> { if (randomBoolean()) { - doc.add(new SortedNumericDocValuesField("another_field", randomLong())); + doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())); count.incrementAndGet(); } else { - doc.add(new SortedNumericDocValuesField("field", randomLong())); + doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())); } }, internalMissing -> { assertEquals(internalMissing.getDocCount(), count.get()); count.set(0); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); - }); + }, + List.of(fieldType, anotherFieldType)); } public void testMatchSparseRangeField() throws IOException { int numDocs = randomIntBetween(100, 200); final AtomicInteger count = new AtomicInteger(); + final String fieldName = "field"; - RangeType rangeType = RangeType.DOUBLE; - final BinaryDocValuesField field = new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton( - new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true)))); - MappedFieldType fieldType = new RangeFieldMapper.Builder(fieldName, rangeType).fieldType(); + final RangeType rangeType = RangeType.DOUBLE; + MappedFieldType fieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType(); fieldType.setName(fieldName); + + final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true); + final BytesRef encodedRange = rangeType.encodeRanges(singleton(range)); + final BinaryDocValuesField field = new BinaryDocValuesField(fieldName, encodedRange); + + final MappedFieldType anotherFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType(); + anotherFieldType.setName("another_field"); + testBothCases(numDocs, fieldName, Queries.newMatchAllQuery(), - doc -> { + builder -> {}, + (index, doc) -> { if (randomBoolean()) { - doc.add(new SortedNumericDocValuesField("another_field", randomLong())); + doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())); count.incrementAndGet(); } else { doc.add(field); @@ -112,55 +146,88 @@ public void testMatchSparseRangeField() throws IOException { assertEquals(internalMissing.getDocCount(), count.get()); count.set(0); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); - }, fieldType); + }, + List.of(fieldType, anotherFieldType)); } - public void testMissingField() throws IOException { + public void testUnmappedWithoutMissingParam() throws IOException { int numDocs = randomIntBetween(10, 20); + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + fieldType.setName("field"); testBothCases(numDocs, "unknown_field", Queries.newMatchAllQuery(), - doc -> { - doc.add(new SortedNumericDocValuesField("field", randomLong())); - }, + builder -> {}, + (index, doc) -> doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())), internalMissing -> { assertEquals(internalMissing.getDocCount(), numDocs); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); - }); + }, + singleton(fieldType)); } - private void testBothCases(int numDocs, - String fieldName, - Query query, - Consumer consumer, - Consumer verify) throws IOException { - NumberFieldMapper.Builder mapperBuilder = new NumberFieldMapper.Builder("_name", - NumberFieldMapper.NumberType.LONG); - final MappedFieldType fieldType = mapperBuilder.fieldType(); - fieldType.setHasDocValues(true); - fieldType.setName(fieldName); - testBothCases(numDocs, fieldName, query, consumer, verify, fieldType); + public void testUnmappedWithMissingParam() throws IOException { + final int numDocs = randomIntBetween(10, 20); + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + fieldType.setName("field"); + testBothCases(numDocs, + "unknown_field", + Queries.newMatchAllQuery(), + builder -> builder.missing(randomLong()), + (index, doc) -> doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())), + internalMissing -> { + assertEquals(internalMissing.getDocCount(), 0); + assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); + }, + singleton(fieldType)); + } + + public void testMissingParam() throws IOException { + final int numDocs = randomIntBetween(100, 200); + + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + fieldType.setName("field"); + final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + anotherFieldType.setName("another_field"); + + testBothCases(numDocs, + anotherFieldType.name(), + Queries.newMatchAllQuery(), + builder -> builder.missing(randomLong()), + (index, doc) -> { + doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())); + if (index % 10 == 0) { + doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())); + } + }, + internalMissing -> { + assertEquals(internalMissing.getDocCount(), 0); + assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); + }, + List.of(fieldType, anotherFieldType)); } private void testBothCases(int numDocs, String fieldName, Query query, - Consumer consumer, + Consumer builderConsumer, + BiConsumer documentConsumer, Consumer verify, - MappedFieldType fieldType) throws IOException { - executeTestCase(numDocs, fieldName, query, consumer, verify, false, fieldType); - executeTestCase(numDocs, fieldName, query, consumer, verify, true, fieldType); + Collection fieldTypes) throws IOException { + executeTestCase(numDocs, fieldName, query, builderConsumer, documentConsumer, verify, false, fieldTypes); + executeTestCase(numDocs, fieldName, query, builderConsumer, documentConsumer, verify, true, fieldTypes); } private void executeTestCase(int numDocs, String fieldName, Query query, - Consumer consumer, + Consumer builderConsumer, + BiConsumer documentConsumer, Consumer verify, boolean reduced, - MappedFieldType fieldType) throws IOException { + Collection fieldTypes) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); @@ -168,7 +235,7 @@ private void executeTestCase(int numDocs, if (frequently()) { indexWriter.commit(); } - consumer.accept(document); + documentConsumer.accept(i, document); indexWriter.addDocument(document); document.clear(); } @@ -179,12 +246,14 @@ private void executeTestCase(int numDocs, newSearcher(indexReader, true, true); MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null); builder.field(fieldName); + builderConsumer.accept(builder); + final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); InternalMissing missing; if (reduced) { - missing = searchAndReduce(indexSearcher, query, builder, fieldType); + missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); } else { - missing = search(indexSearcher, query, builder, fieldType); + missing = search(indexSearcher, query, builder, fieldTypesArray); } verify.accept(missing); } From a964952b9097a678c20d3e83db66092541c5b495 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Fri, 6 Mar 2020 15:52:43 -0800 Subject: [PATCH 2/3] add supported valuessourcetype tests --- .../missing/MissingAggregatorTests.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 934d6e934bc73..4edff5bdeafcb 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -35,8 +35,11 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Collection; @@ -259,4 +262,21 @@ private void executeTestCase(int numDocs, } } } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new MissingAggregationBuilder("_name", null) + .field(fieldName); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.BYTES, + CoreValuesSourceType.GEOPOINT, + CoreValuesSourceType.RANGE, + CoreValuesSourceType.HISTOGRAM + ); + } } From 8e222d93dc11675533050abafa1829a22651b6dd Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Mon, 9 Mar 2020 13:01:55 -0700 Subject: [PATCH 3/3] add script tests Also change the test case pattern to be more flexible and more similar to how the other agg tests are written --- .../missing/MissingAggregatorTests.java | 473 +++++++++++++----- 1 file changed, 341 insertions(+), 132 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 4edff5bdeafcb..04b1bbe10c4a7 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -20,239 +20,414 @@ package org.elasticsearch.search.aggregations.bucket.missing; import org.apache.lucene.document.BinaryDocValuesField; -import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; +import org.elasticsearch.script.MockScriptEngine; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.LeafDocLookup; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; +import java.util.Map; +import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; +import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; - +import static java.util.Collections.singletonMap; +import static java.util.stream.Collectors.toList; +import static org.elasticsearch.common.lucene.search.Queries.newMatchAllQuery; public class MissingAggregatorTests extends AggregatorTestCase { + + private static final String VALUE_SCRIPT_PARAMS = "value_script_params"; + private static final String VALUE_SCRIPT = "value_script"; + private static final String FIELD_SCRIPT_PARAMS = "field_script_params"; + private static final String FIELD_SCRIPT = "field_script"; + private static final long DEFAULT_INC_PARAM = 1; + private static final long DEFAULT_THRESHOLD_PARAM = 50; + public void testMatchNoDocs() throws IOException { - int numDocs = randomIntBetween(10, 200); + final int numDocs = randomIntBetween(10, 200); + final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); fieldType.setName("field"); - testBothCases(numDocs, - fieldType.name(), - Queries.newMatchAllQuery(), - builder -> {}, - (index, doc) -> doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())), + + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(fieldType.name()); + + testCase( + newMatchAllQuery(), + builder, + writer -> { + for (int i = 0; i < numDocs; i++) { + writer.addDocument(singleton(new SortedNumericDocValuesField(fieldType.name(), randomLong()))); + } + }, internalMissing -> { - assertEquals(internalMissing.getDocCount(), 0); + assertEquals(0, internalMissing.getDocCount()); assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); }, - singleton(fieldType)); + singleton(fieldType) + ); } public void testMatchAllDocs() throws IOException { int numDocs = randomIntBetween(10, 200); - final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - fieldType.setName("field"); + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - testBothCases(numDocs, - fieldType.name(), - Queries.newMatchAllQuery(), - builder -> {}, - (index, doc) -> doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())), + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(aggFieldType.name()); + + testCase( + newMatchAllQuery(), + builder, + writer -> { + for (int i = 0; i < numDocs; i++) { + writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); + } + }, internalMissing -> { - assertEquals(internalMissing.getDocCount(), numDocs); + assertEquals(numDocs, internalMissing.getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, - List.of(fieldType, anotherFieldType)); + List.of(aggFieldType, anotherFieldType) + ); } public void testMatchSparse() throws IOException { - int numDocs = randomIntBetween(100, 200); - final AtomicInteger count = new AtomicInteger(); - - final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - fieldType.setName("field"); + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - testBothCases(numDocs, - fieldType.name(), - Queries.newMatchAllQuery(), - builder -> {}, - (index, doc) -> { - if (randomBoolean()) { - doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())); - count.incrementAndGet(); - } else { - doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())); - } - }, + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(aggFieldType.name()); + + final int numDocs = randomIntBetween(100, 200); + int docsMissingAggField = 0; + final List> docs = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + if (randomBoolean()) { + docs.add(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); + } else { + docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); + docsMissingAggField++; + } + } + final int finalDocsMissingAggField = docsMissingAggField; + + testCase( + newMatchAllQuery(), + builder, + writer -> writer.addDocuments(docs), internalMissing -> { - assertEquals(internalMissing.getDocCount(), count.get()); - count.set(0); + assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, - List.of(fieldType, anotherFieldType)); + List.of(aggFieldType, anotherFieldType) + ); } public void testMatchSparseRangeField() throws IOException { - int numDocs = randomIntBetween(100, 200); - final AtomicInteger count = new AtomicInteger(); - - final String fieldName = "field"; final RangeType rangeType = RangeType.DOUBLE; - MappedFieldType fieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType(); - fieldType.setName(fieldName); + MappedFieldType aggFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType(); + aggFieldType.setName("agg_field"); + final MappedFieldType anotherFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType(); + anotherFieldType.setName("another_field"); final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true); final BytesRef encodedRange = rangeType.encodeRanges(singleton(range)); - final BinaryDocValuesField field = new BinaryDocValuesField(fieldName, encodedRange); + final BinaryDocValuesField encodedRangeField = new BinaryDocValuesField(aggFieldType.name(), encodedRange); - final MappedFieldType anotherFieldType = new RangeFieldMapper.Builder("_name", rangeType).fieldType(); - anotherFieldType.setName("another_field"); + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(aggFieldType.name()); - testBothCases(numDocs, - fieldName, - Queries.newMatchAllQuery(), - builder -> {}, - (index, doc) -> { - if (randomBoolean()) { - doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())); - count.incrementAndGet(); - } else { - doc.add(field); - } - }, + final int numDocs = randomIntBetween(100, 200); + int docsMissingAggField = 0; + final List> docs = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + if (randomBoolean()) { + docs.add(singleton(encodedRangeField)); + } else { + docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); + docsMissingAggField++; + } + } + final int finalDocsMissingAggField = docsMissingAggField; + + testCase( + newMatchAllQuery(), + builder, + writer -> writer.addDocuments(docs), internalMissing -> { - assertEquals(internalMissing.getDocCount(), count.get()); - count.set(0); + assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, - List.of(fieldType, anotherFieldType)); + List.of(aggFieldType, anotherFieldType) + ); } - public void testUnmappedWithoutMissingParam() throws IOException { - int numDocs = randomIntBetween(10, 20); - final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - fieldType.setName("field"); - testBothCases(numDocs, - "unknown_field", - Queries.newMatchAllQuery(), - builder -> {}, - (index, doc) -> doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())), + final int numDocs = randomIntBetween(10, 20); + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); + + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field("unknown_field"); + + testCase( + newMatchAllQuery(), + builder, + writer -> { + for (int i = 0; i < numDocs; i++) { + writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); + } + }, internalMissing -> { - assertEquals(internalMissing.getDocCount(), numDocs); + assertEquals(numDocs, internalMissing.getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); }, - singleton(fieldType)); + singleton(aggFieldType) + ); } public void testUnmappedWithMissingParam() throws IOException { final int numDocs = randomIntBetween(10, 20); - final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - fieldType.setName("field"); - testBothCases(numDocs, - "unknown_field", - Queries.newMatchAllQuery(), - builder -> builder.missing(randomLong()), - (index, doc) -> doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())), + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); + + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field("unknown_field") + .missing(randomLong()); + + testCase( + newMatchAllQuery(), + builder, + writer -> { + for (int i = 0; i < numDocs; i++) { + writer.addDocument(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); + } + }, internalMissing -> { - assertEquals(internalMissing.getDocCount(), 0); + assertEquals(0, internalMissing.getDocCount()); assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); }, - singleton(fieldType)); + singleton(aggFieldType) + ); } public void testMissingParam() throws IOException { - final int numDocs = randomIntBetween(100, 200); + final int numDocs = randomIntBetween(10, 20); - final MappedFieldType fieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); - fieldType.setName("field"); + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); anotherFieldType.setName("another_field"); - testBothCases(numDocs, - anotherFieldType.name(), - Queries.newMatchAllQuery(), - builder -> builder.missing(randomLong()), - (index, doc) -> { - doc.add(new SortedNumericDocValuesField(fieldType.name(), randomLong())); - if (index % 10 == 0) { - doc.add(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong())); + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(aggFieldType.name()) + .missing(randomLong()); + + testCase( + newMatchAllQuery(), + builder, + writer -> { + for (int i = 0; i < numDocs; i++) { + writer.addDocument(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); } }, internalMissing -> { - assertEquals(internalMissing.getDocCount(), 0); + assertEquals(0, internalMissing.getDocCount()); assertFalse(AggregationInspectionHelper.hasValue(internalMissing)); }, - List.of(fieldType, anotherFieldType)); + List.of(aggFieldType, anotherFieldType) + ); + } + + public void testMultiValuedField() throws IOException { + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); + final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + anotherFieldType.setName("another_field"); + + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(aggFieldType.name()); + + final int numDocs = randomIntBetween(100, 200); + int docsMissingAggField = 0; + final List> docs = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + if (randomBoolean()) { + final long randomLong = randomLong(); + docs.add(Set.of( + new SortedNumericDocValuesField(aggFieldType.name(), randomLong), + new SortedNumericDocValuesField(aggFieldType.name(), randomLong + 1) + )); + } else { + docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); + docsMissingAggField++; + } + } + final int finalDocsMissingAggField = docsMissingAggField; + + testCase( + newMatchAllQuery(), + builder, + writer -> writer.addDocuments(docs), + internalMissing -> { + assertEquals(finalDocsMissingAggField, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + }, + List.of(aggFieldType, anotherFieldType) + ); + } + + public void testSingleValuedFieldWithValueScript() throws IOException { + valueScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, emptyMap())); + } + + public void testSingleValuedFieldWithValueScriptWithParams() throws IOException { + valueScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_PARAMS, singletonMap("inc", 10))); + } + + private void valueScriptTestCase(Script script) throws IOException { + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); + final MappedFieldType anotherFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + anotherFieldType.setName("another_field"); + + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .field(aggFieldType.name()) + .script(script); + + final int numDocs = randomIntBetween(100, 200); + int docsMissingAggField = 0; + final List> docs = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + if (randomBoolean()) { + docs.add(singleton(new SortedNumericDocValuesField(aggFieldType.name(), randomLong()))); + } else { + docs.add(singleton(new SortedNumericDocValuesField(anotherFieldType.name(), randomLong()))); + docsMissingAggField++; + } + } + final int finalDocsMissingField = docsMissingAggField; + + testCase( + newMatchAllQuery(), + builder, + writer -> writer.addDocuments(docs), + internalMissing -> { + assertEquals(finalDocsMissingField, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + }, + List.of(aggFieldType, anotherFieldType) + ); } - private void testBothCases(int numDocs, - String fieldName, - Query query, - Consumer builderConsumer, - BiConsumer documentConsumer, - Consumer verify, - Collection fieldTypes) throws IOException { - executeTestCase(numDocs, fieldName, query, builderConsumer, documentConsumer, verify, false, fieldTypes); - executeTestCase(numDocs, fieldName, query, builderConsumer, documentConsumer, verify, true, fieldTypes); + public void testMultiValuedFieldWithFieldScriptWithParams() throws IOException { + final long threshold = 10; + final Map params = Map.of("field", "agg_field", "threshold", threshold); + fieldScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_PARAMS, params), threshold); + } + public void testMultiValuedFieldWithFieldScript() throws IOException { + fieldScriptTestCase(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT, singletonMap("field", "agg_field")), + DEFAULT_THRESHOLD_PARAM); } - private void executeTestCase(int numDocs, - String fieldName, - Query query, - Consumer builderConsumer, - BiConsumer documentConsumer, - Consumer verify, - boolean reduced, - Collection fieldTypes) throws IOException { + private void fieldScriptTestCase(Script script, long threshold) throws IOException { + final MappedFieldType aggFieldType = new NumberFieldMapper.Builder("_name", NumberType.LONG).fieldType(); + aggFieldType.setName("agg_field"); + + final MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null) + .script(script); + + final int numDocs = randomIntBetween(100, 200); + int docsBelowThreshold = 0; + final List> docs = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + final long firstValue = randomLongBetween(0, 100); + final long secondValue = firstValue + 1; + if (firstValue < threshold && secondValue < threshold) { + docsBelowThreshold++; + } + docs.add(Set.of( + new SortedNumericDocValuesField(aggFieldType.name(), firstValue), + new SortedNumericDocValuesField(aggFieldType.name(), secondValue) + )); + } + final int finalDocsBelowThreshold = docsBelowThreshold; + + testCase( + newMatchAllQuery(), + builder, + writer -> writer.addDocuments(docs), + internalMissing -> { + assertEquals(finalDocsBelowThreshold, internalMissing.getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(internalMissing)); + }, + singleton(aggFieldType) + ); + } + + private void testCase(Query query, + MissingAggregationBuilder builder, + CheckedConsumer writeIndex, + Consumer verify, + Collection fieldTypes) throws IOException { + testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false); + testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true); + } + + private void testCaseWithReduce(Query query, + MissingAggregationBuilder builder, + CheckedConsumer writeIndex, + Consumer verify, + Collection fieldTypes, + boolean reduced) throws IOException { + try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { - Document document = new Document(); - for (int i = 0; i < numDocs; i++) { - if (frequently()) { - indexWriter.commit(); - } - documentConsumer.accept(i, document); - indexWriter.addDocument(document); - document.clear(); - } + writeIndex.accept(indexWriter); } try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = - newSearcher(indexReader, true, true); - MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null); - builder.field(fieldName); - builderConsumer.accept(builder); - + final IndexSearcher indexSearcher = newSearcher(indexReader, true, true); final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); - InternalMissing missing; + final InternalMissing missing; if (reduced) { missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); } else { @@ -279,4 +454,38 @@ protected List getSupportedValuesSourceTypes() { CoreValuesSourceType.HISTOGRAM ); } + + @Override + protected ScriptService getMockScriptService() { + final Map, Object>> deterministicScripts = new HashMap<>(); + deterministicScripts.put(VALUE_SCRIPT_PARAMS, vars -> { + final double value = ((Number) vars.get("_value")).doubleValue(); + final long inc = ((Number) vars.get("inc")).longValue(); + return value + inc; + }); + deterministicScripts.put(VALUE_SCRIPT, vars -> { + final double value = ((Number) vars.get("_value")).doubleValue(); + return value + DEFAULT_INC_PARAM; + }); + deterministicScripts.put(FIELD_SCRIPT_PARAMS, vars -> { + final String fieldName = (String) vars.get("field"); + final long threshold = ((Number) vars.get("threshold")).longValue(); + return threshold(fieldName, threshold, vars); + }); + deterministicScripts.put(FIELD_SCRIPT, vars -> { + final String fieldName = (String) vars.get("field"); + return threshold(fieldName, DEFAULT_THRESHOLD_PARAM, vars); + }); + final MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, deterministicScripts, emptyMap(), emptyMap()); + final Map engines = singletonMap(scriptEngine.getType(), scriptEngine); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + } + + private static List threshold(String fieldName, long threshold, Map vars) { + final LeafDocLookup lookup = (LeafDocLookup) vars.get("doc"); + return lookup.get(fieldName).stream() + .map(value -> ((Number) value).longValue()) + .filter(value -> value >= threshold) + .collect(toList()); + } }