From 5ad9a9641f7c24a0476e27afe1d28c6dbd382ff7 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 28 Jan 2020 12:26:47 -0500 Subject: [PATCH 1/6] Put values source types on fields (#51503) --- .../index/mapper/ScaledFloatFieldMapper.java | 7 +++++++ .../join/mapper/MetaJoinFieldMapper.java | 7 +++++++ .../join/mapper/ParentIdFieldMapper.java | 7 +++++++ .../join/mapper/ParentJoinFieldMapper.java | 7 +++++++ .../index/mapper/ICUCollationKeywordFieldMapper.java | 7 +++++++ .../index/mapper/murmur3/Murmur3FieldMapper.java | 7 +++++++ .../elasticsearch/index/mapper/BinaryFieldMapper.java | 7 +++++++ .../index/mapper/BooleanFieldMapper.java | 7 +++++++ .../elasticsearch/index/mapper/DateFieldMapper.java | 7 +++++++ .../index/mapper/GeoPointFieldMapper.java | 7 +++++++ .../org/elasticsearch/index/mapper/IdFieldMapper.java | 8 ++++++++ .../elasticsearch/index/mapper/IndexFieldMapper.java | 8 ++++++++ .../org/elasticsearch/index/mapper/IpFieldMapper.java | 7 +++++++ .../index/mapper/KeywordFieldMapper.java | 7 +++++++ .../elasticsearch/index/mapper/MappedFieldType.java | 11 +++++++++++ .../elasticsearch/index/mapper/NumberFieldMapper.java | 7 +++++++ .../elasticsearch/index/mapper/RangeFieldMapper.java | 7 +++++++ .../elasticsearch/index/mapper/SeqNoFieldMapper.java | 6 ++++++ .../elasticsearch/index/mapper/TextFieldMapper.java | 7 +++++++ .../elasticsearch/index/mapper/TypeFieldMapper.java | 7 +++++++ .../xpack/analytics/mapper/HistogramFieldMapper.java | 8 ++++++++ .../xpack/flattened/mapper/FlatObjectFieldMapper.java | 7 +++++++ .../xpack/vectors/mapper/DenseVectorFieldMapper.java | 7 +++++++ 23 files changed, 167 insertions(+) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index d71992b8ed117..83a60834b7f1d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -62,6 +62,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.math.BigDecimal; @@ -295,6 +297,11 @@ public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fiel }; } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 388d4ca833ff4..f1b75672ccc4b 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -31,6 +31,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -95,6 +97,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 8130acac1af72..7dc43962ad9c4 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -41,6 +41,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Collection; @@ -118,6 +120,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 331746b4276de..4c7c14893af12 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -43,6 +43,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -227,6 +229,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 883468941a5d8..57a0dba31f41f 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -46,6 +46,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -139,6 +141,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override protected BytesRef indexedValueForSearch(Object value) { if (value == null) { diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index cf2b05fe54d90..b6c49a246f433 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -39,6 +39,8 @@ import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -124,6 +126,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public Query existsQuery(QueryShardContext context) { return new DocValuesFieldExistsQuery(name()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 7a5bd97770297..ea8d407f731f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -40,6 +40,8 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -136,6 +138,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new BytesBinaryDVIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index caf8baac24da1..3def7265d4c9c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -40,6 +40,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -189,6 +191,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(NumericType.BOOLEAN); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BOOLEAN; + } + @Override public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { if (format != null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 2feb5213c4568..5bf749ee11ef7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -52,6 +52,8 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.DateTimeException; @@ -505,6 +507,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(resolution.numericType()); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.DATE; + } + @Override public Object valueForDisplay(Object value) { Long val = (Long) value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index f319db284b2d2..93d8215f21cac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -40,6 +40,8 @@ import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointDVIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -235,6 +237,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new AbstractLatLonPointDVIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 74d69be93d363..25e5523c008dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -50,6 +50,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Arrays; @@ -157,6 +159,12 @@ public Query termsQuery(List values, QueryShardContext context) { return new TermInSetQuery(name(), bytesRefs); } + @Override + public ValuesSourceType getValuesSourceType() { + // TODO: should this even exist? Is aggregating on the ID field valid? + return CoreValuesSourceType.BYTES; + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { if (indexOptions() == IndexOptions.NONE) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 2500358a3e5d6..88282fa685728 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -33,6 +33,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -189,6 +191,12 @@ public Query wildcardQuery(String value, public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new ConstantIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName); } + + @Override + public ValuesSourceType getValuesSourceType() { + // TODO: Should Index fields be aggregatable? What even is an IndexField? + return CoreValuesSourceType.BYTES; + } } private IndexFieldMapper(Settings indexSettings, MappedFieldType existing) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 2b52e42ffe558..a3abf963104ae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -44,6 +44,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.net.InetAddress; @@ -294,6 +296,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().scriptFunction(IpScriptDocValues::new); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.IP; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 099ae9b2aa7a2..7fde4359742e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -43,6 +43,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -271,6 +273,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index e6e31e3379fb2..4fb38a7602f19 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -50,6 +50,7 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -117,6 +118,16 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } + /** + * Returns the {@link ValuesSourceType} which supports this field type. This is tightly coupled to field data and aggregations support, + * so any implementation that returns a value from {@link MappedFieldType#fielddataBuilder} should also return a value from here. + * + * @return The appropriate {@link ValuesSourceType} for this field type. + */ + public ValuesSourceType getValuesSourceType() { + throw new IllegalArgumentException("Aggregations are not supported on field [" + name() + "] of type [" + typeName() + "]"); + } + @Override public boolean equals(Object o) { if (!super.equals(o)) return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 0a473bd189e3b..fa7055cadd2a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -55,6 +55,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.time.ZoneId; @@ -958,6 +960,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(type.numericType()); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 289eae555514f..6caed5da00324 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -45,6 +45,8 @@ import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.net.InetAddress; @@ -242,6 +244,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().setRangeType(rangeType); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.RANGE; + } + @Override public String typeName() { return rangeType.name; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 5bb0007415866..5bbff9e90a21c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -37,6 +37,8 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.List; @@ -218,6 +220,10 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } } public SeqNoFieldMapper(Settings indexSettings) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 46ef8a732a697..b237d7cbd82ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -70,6 +70,8 @@ import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.IntervalBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -756,6 +758,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new PagedBytesIndexFieldData.Builder(fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public void checkCompatibility(MappedFieldType other, List conflicts) { super.checkCompatibility(other, conflicts); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 5d9c9e20a1c14..e5c33b83f24e5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -42,6 +42,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Arrays; @@ -113,6 +115,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new ConstantIndexFieldData.Builder(typeFunction); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public boolean isSearchable() { return true; diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 20326e82cadc9..f656f9bdfc83d 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -53,6 +53,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -276,6 +278,12 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu }; } + @Override + public ValuesSourceType getValuesSourceType() { + // TODO: Histogram ValuesSourceType should move into this plugin. + return CoreValuesSourceType.HISTOGRAM; + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 2cad828711701..09ef54db97a67 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -55,6 +55,8 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Iterator; @@ -349,6 +351,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); return new KeyedFlatObjectFieldData.Builder(key); } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } } /** diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index a33a9794acb43..1fd7543411efc 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -27,6 +27,8 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData; import java.io.IOException; @@ -150,6 +152,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new VectorDVIndexFieldData.Builder(); } + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new UnsupportedOperationException( From e5340d5bbf7f7f509c1e40a0d1a4198117d46852 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 18 Feb 2020 16:25:53 -0500 Subject: [PATCH 2/6] Comprehensively test supported/unsupported field type:aggs combinations This adds a test to AggregatorTestCase that allows us to programmatically verify that an aggregator supports or does not support a particular field type. It fetches the list of registered field type parsers, creates a MappedFieldType from the parser and then attempts to run a basic agg against the field. A supplied list of supported VSTypes are then compared against the output (success or exception) and suceeds or fails the test accordingly. --- .../index/mapper/RangeFieldMapper.java | 8 - .../support/CoreValuesSourceType.java | 66 ++++++ .../bucket/terms/TermsAggregatorTests.java | 16 ++ .../aggregations/AggregatorTestCase.java | 193 ++++++++++++++++++ .../org/elasticsearch/test/ESTestCase.java | 29 +++ 5 files changed, 304 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 6caed5da00324..44de6d88fd362 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -94,14 +94,6 @@ public RangeFieldType fieldType() { return (RangeFieldType)fieldType; } - @Override - public Builder docValues(boolean docValues) { - if (docValues) { - throw new IllegalArgumentException("field [" + name + "] does not currently support " + TypeParsers.DOC_VALUES); - } - return super.docValues(docValues); - } - public Builder coerce(boolean coerce) { this.coerce = coerce; return builder; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index 6df32b4deefa0..ec5a9d749ec41 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -224,6 +224,72 @@ public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFa public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); } + }, + // TODO: Ordinal Numbering sync with types from master + IP{ + @Override + public ValuesSource getEmpty() { + return BYTES.getEmpty(); + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + return BYTES.getScript(script, scriptValueType); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + return BYTES.getField(fieldContext, script); + } + + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + return BYTES.replaceMissing(valuesSource, rawMissing, docValueFormat, now); + } + }, + DATE { + @Override + public ValuesSource getEmpty() { + return NUMERIC.getEmpty(); + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + return NUMERIC.getScript(script, scriptValueType); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + return NUMERIC.getField(fieldContext, script); + } + + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + return NUMERIC.replaceMissing(valuesSource, rawMissing, docValueFormat, now); + } + + }, + BOOLEAN { + @Override + public ValuesSource getEmpty() { + return NUMERIC.getEmpty(); + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + return NUMERIC.getScript(script, scriptValueType); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + return NUMERIC.getField(fieldContext, script); + } + + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + return NUMERIC.replaceMissing(valuesSource, rawMissing, docValueFormat, now); + } + }; public static ValuesSourceType fromString(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 3c510f0bf7826..697311467b494 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -80,7 +80,9 @@ import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.test.geo.RandomGeoGenerator; @@ -122,6 +124,20 @@ protected A createAggregator(AggregationBuilder aggregati } } + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new TermsAggregationBuilder("foo", ValueType.STRING).field(fieldName); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.BOOLEAN, + CoreValuesSourceType.BYTES, + CoreValuesSourceType.IP, + CoreValuesSourceType.DATE); + } + public void testGlobalOrdinalsExecutionHint() throws Exception { randomizeAggregatorImpl = false; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index eca219a868bf6..5ce1f0b53da8e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -18,20 +18,29 @@ */ package org.elasticsearch.search.aggregations; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.AssertingDirectoryReader; import org.apache.lucene.index.CompositeReaderContext; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -51,23 +60,36 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.BinaryFieldMapper; +import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldAliasMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.ObjectMapper.Nested; +import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.index.mapper.RangeType; +import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.subphase.FetchDocValuesPhase; import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; @@ -79,6 +101,8 @@ import org.junit.After; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -90,6 +114,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.singleton; import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; @@ -107,6 +132,17 @@ public abstract class AggregatorTestCase extends ESTestCase { private List releasables = new ArrayList<>(); private static final String TYPE_NAME = "type"; + private static List TYPE_TEST_BLACKLIST = List.of( + ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects + GeoShapeFieldMapper.CONTENT_TYPE, // Cannot aggregate geoshapes (yet) + + TextFieldMapper.CONTENT_TYPE, // TODO Does not support doc values, but does support FD, needs a lot of mocking + ObjectMapper.NESTED_CONTENT_TYPE, // TODO support for nested + CompletionFieldMapper.CONTENT_TYPE, // TODO support completion + FieldAliasMapper.CONTENT_TYPE // TODO support alias + ); + + /** * Allows subclasses to provide alternate names for the provided field type, which * can be useful when testing aggregations on field aliases. @@ -524,6 +560,163 @@ protected static IndexReader maybeWrapReaderEs(DirectoryReader reader) throws IO } } + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return null; + } + + protected List getSupportedValuesSourceTypes() { + return Collections.emptyList(); + } + + public void testSupportedFieldTypes() throws IOException { + + MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); + Settings settings = Settings.builder().put("index.version.created", Version.CURRENT.id).build(); + String fieldName = "typeTestFieldName"; + List supportedVSTypes = getSupportedValuesSourceTypes(); + + if (supportedVSTypes.isEmpty()) { + // If the test says it doesn't support any VStypes, it has not been converted yet so skip + return; + } else if (supportedVSTypes.contains(CoreValuesSourceType.ANY)) { + throw new IllegalArgumentException("Tests should not specify CoreValuesSourceType.ANY as a supported ValuesSourceType, " + + "but should instead list the concrete ValuesSourceTypes that are supported"); + } + + for (Map.Entry mappedType : mapperRegistry.getMapperParsers().entrySet()) { + if (TYPE_TEST_BLACKLIST.contains(mappedType.getKey())) { + continue; + } + + if (mappedType.getKey().equals(ObjectMapper.CONTENT_TYPE)) { + // Cannot aggregate objects + continue; + } + + Map source = new HashMap<>(); + source.put("type", mappedType.getKey()); + source.put("doc_values", "true"); + + FieldMapper mapper = null; + try { + Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext()); + mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath())); + } catch (Exception e) { + fail(); + } + MappedFieldType fieldType = mapper.fieldType(); + + + try (Directory directory = newDirectory()) { + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + writeTestDoc(fieldType, fieldName, indexWriter); + indexWriter.close(); + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, fieldName); + + try { + searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); + if (supportedVSTypes.contains(fieldType.getValuesSourceType()) == false) { + fail("Aggregator [" + aggregationBuilder.getType() + "] should not support field type [" + + fieldType.typeName() + "] but executing against the field did not throw an excetion"); + } + } catch (Exception e) { + if (supportedVSTypes.contains(fieldType.getValuesSourceType())) { + fail("Aggregator [" + aggregationBuilder.getType() + "] supports field type [" + + fieldType.typeName() + "] but executing against the field threw an exception: [" + e.getMessage() + "]"); + } + } + } + } + } + } + + private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomIndexWriter iw) throws IOException { + + if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.NUMERIC)) { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomLong()))); + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.DATE)) { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong()))); + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.BYTES)) { + if (fieldType.typeName().equals(BinaryFieldMapper.CONTENT_TYPE)) { + iw.addDocument(singleton(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes))); + } else { + iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef("a")))); + } + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.RANGE)) { + Object start; + Object end; + RangeType rangeType; + + if (fieldType.typeName().equals(RangeType.DOUBLE.typeName())) { + start = randomDouble(); + end = randomDoubleBetween(Math.nextUp((Double)start), Double.MAX_VALUE, false); + rangeType = RangeType.DOUBLE; + } else if (fieldType.typeName().equals(RangeType.FLOAT.typeName())) { + start = randomFloat(); + end = randomFloatBetween(Math.nextUp((Float) start), Float.MAX_VALUE, false); + rangeType = RangeType.DOUBLE; + } else if (fieldType.typeName().equals(RangeType.IP.typeName())) { + boolean v4 = randomBoolean(); + start = randomIp(v4); + end = randomIp(v4); + rangeType = RangeType.IP; + } else if (fieldType.typeName().equals(RangeType.LONG.typeName())) { + start = randomLong(); + end = randomLongBetween((Long)start + 1, Long.MAX_VALUE); + rangeType = RangeType.LONG; + } else if (fieldType.typeName().equals(RangeType.INTEGER.typeName())) { + start = randomInt(); + end = randomIntBetween((Integer)start, Integer.MAX_VALUE); + rangeType = RangeType.INTEGER; + } else if (fieldType.typeName().equals(RangeType.DATE.typeName())) { + start = randomNonNegativeLong(); + end = randomLongBetween((Long)start + 1, Long.MAX_VALUE); + rangeType = RangeType.DATE; + } else { + throw new IllegalStateException("Unknown type of range [" + fieldType.typeName() + "]"); + } + + final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, start, end, true, true); + iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range))))); + + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.BOOLEAN)) { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomBoolean() ? 0 : 1))); + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.IP)) { + boolean v4 = randomBoolean(); + iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4)))))); + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.GEOPOINT)) { + iw.addDocument(singleton(new LatLonDocValuesField(fieldName, randomDouble(), randomDouble()))); + } else { + throw new IllegalStateException("Unknown field type [" + fieldType.typeName() + "]"); + } + } + + private class MockParserContext extends Mapper.TypeParser.ParserContext { + MockParserContext() { + super(null, null, null, null, null); + } + } + + private static InetAddress randomIp(boolean v4) { + try { + if (v4) { + byte[] ipv4 = new byte[4]; + random().nextBytes(ipv4); + return InetAddress.getByAddress(ipv4); + } else { + byte[] ipv6 = new byte[16]; + random().nextBytes(ipv6); + return InetAddress.getByAddress(ipv6); + } + } catch (UnknownHostException e) { + throw new AssertionError(); + } + } + + @After private void cleanupReleasables() { Releasables.close(releasables); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index a5005c7583a86..18dd6140ca335 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -620,6 +620,35 @@ public static float randomFloat() { return random().nextFloat(); } + /** + * Returns a float value in the interval [start, end) if lowerInclusive is + * set to true, (start, end) otherwise. + * + * @param start lower bound of interval to draw uniformly distributed random numbers from + * @param end upper bound + * @param lowerInclusive whether or not to include lower end of the interval + */ + public static float randomFloatBetween(float start, float end, boolean lowerInclusive) { + float result = 0.0f; + + if (start == -Float.MAX_VALUE || end == Float.MAX_VALUE) { + // formula below does not work with very large floats + result = Float.intBitsToFloat(randomInt()); + while (result < start || result > end || Float.isNaN(result)) { + result = Float.intBitsToFloat(randomInt()); + } + } else { + result = randomFloat(); + if (lowerInclusive == false) { + while (result <= 0.0) { + result = randomFloat(); + } + } + result = result * end + (1.0f - result) * start; + } + return result; + } + public static double randomDouble() { return random().nextDouble(); } From 197b1b0ff8f6da763195109aeb4c8ab2936e493e Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Wed, 19 Feb 2020 12:32:05 -0500 Subject: [PATCH 3/6] Cleanup, javadocs, review comments --- .../index/mapper/BooleanFieldMapper.java | 2 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/IpFieldMapper.java | 2 +- .../support/CoreValuesSourceType.java | 66 --------- .../aggregations/bucket/IpRangeTests.java | 30 +---- .../bucket/range/IpRangeAggregatorTests.java | 18 --- .../bucket/terms/TermsAggregatorTests.java | 5 +- .../aggregations/AggregatorTestCase.java | 126 ++++++++++-------- .../org/elasticsearch/test/ESTestCase.java | 48 +++---- 9 files changed, 102 insertions(+), 197 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 3def7265d4c9c..009848df1d8a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -193,7 +193,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.BOOLEAN; + return CoreValuesSourceType.NUMERIC; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 5bf749ee11ef7..e905ea2d5676b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -509,7 +509,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.DATE; + return CoreValuesSourceType.NUMERIC; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index a3abf963104ae..c1cf50a56cde5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -298,7 +298,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.IP; + return CoreValuesSourceType.BYTES; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index ec5a9d749ec41..6df32b4deefa0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -224,72 +224,6 @@ public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFa public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); } - }, - // TODO: Ordinal Numbering sync with types from master - IP{ - @Override - public ValuesSource getEmpty() { - return BYTES.getEmpty(); - } - - @Override - public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { - return BYTES.getScript(script, scriptValueType); - } - - @Override - public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - return BYTES.getField(fieldContext, script); - } - - @Override - public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { - return BYTES.replaceMissing(valuesSource, rawMissing, docValueFormat, now); - } - }, - DATE { - @Override - public ValuesSource getEmpty() { - return NUMERIC.getEmpty(); - } - - @Override - public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { - return NUMERIC.getScript(script, scriptValueType); - } - - @Override - public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - return NUMERIC.getField(fieldContext, script); - } - - @Override - public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { - return NUMERIC.replaceMissing(valuesSource, rawMissing, docValueFormat, now); - } - - }, - BOOLEAN { - @Override - public ValuesSource getEmpty() { - return NUMERIC.getEmpty(); - } - - @Override - public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { - return NUMERIC.getScript(script, scriptValueType); - } - - @Override - public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - return NUMERIC.getField(fieldContext, script); - } - - @Override - public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { - return NUMERIC.replaceMissing(valuesSource, rawMissing, docValueFormat, now); - } - }; public static ValuesSourceType fromString(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java index 56587c78708c1..0bf34f6e4a3e3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java @@ -23,27 +23,8 @@ import org.elasticsearch.search.aggregations.BaseAggregationTestCase; import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder; -import java.net.InetAddress; -import java.net.UnknownHostException; - public class IpRangeTests extends BaseAggregationTestCase { - private static String randomIp(boolean v4) { - try { - if (v4) { - byte[] ipv4 = new byte[4]; - random().nextBytes(ipv4); - return NetworkAddress.format(InetAddress.getByAddress(ipv4)); - } else { - byte[] ipv6 = new byte[16]; - random().nextBytes(ipv6); - return NetworkAddress.format(InetAddress.getByAddress(ipv6)); - } - } catch (UnknownHostException e) { - throw new AssertionError(); - } - } - @Override protected IpRangeAggregationBuilder createTestAggregatorBuilder() { int numRanges = randomIntBetween(1, 10); @@ -62,16 +43,17 @@ protected IpRangeAggregationBuilder createTestAggregatorBuilder() { } else { prefixLength = randomInt(128); } - factory.addMaskRange(key, randomIp(v4) + "/" + prefixLength); + factory.addMaskRange(key, NetworkAddress.format(randomIp(v4)) + "/" + prefixLength); break; case 1: - factory.addUnboundedFrom(key, randomIp(randomBoolean())); + factory.addUnboundedFrom(key, NetworkAddress.format(randomIp(randomBoolean()))); break; case 2: - factory.addUnboundedTo(key, randomIp(randomBoolean())); + factory.addUnboundedTo(key, NetworkAddress.format(randomIp(randomBoolean()))); break; case 3: - factory.addRange(key, randomIp(randomBoolean()), randomIp(randomBoolean())); + v4 = randomBoolean(); + factory.addRange(key, NetworkAddress.format(randomIp(v4)), NetworkAddress.format(randomIp(v4))); break; default: fail(); @@ -82,7 +64,7 @@ protected IpRangeAggregationBuilder createTestAggregatorBuilder() { factory.keyed(randomBoolean()); } if (randomBoolean()) { - factory.missing(randomIp(randomBoolean())); + factory.missing(NetworkAddress.format(randomIp(randomBoolean()))); } return factory; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java index f759d30cbbde5..f96e7b01f25ca 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java @@ -35,29 +35,11 @@ import org.elasticsearch.search.aggregations.AggregatorTestCase; import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.Arrays; import java.util.Comparator; public class IpRangeAggregatorTests extends AggregatorTestCase { - private static InetAddress randomIp(boolean v4) { - try { - if (v4) { - byte[] ipv4 = new byte[4]; - random().nextBytes(ipv4); - return InetAddress.getByAddress(ipv4); - } else { - byte[] ipv6 = new byte[16]; - random().nextBytes(ipv6); - return InetAddress.getByAddress(ipv6); - } - } catch (UnknownHostException e) { - throw new AssertionError(); - } - } - - private static boolean isInRange(BytesRef value, BytesRef from, BytesRef to) { if ((to == null || to.compareTo(value) > 0) && (from == null || from.compareTo(value) <= 0)) { return true; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 697311467b494..858d3207e0a77 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -132,10 +132,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { return List.of(CoreValuesSourceType.NUMERIC, - CoreValuesSourceType.BOOLEAN, - CoreValuesSourceType.BYTES, - CoreValuesSourceType.IP, - CoreValuesSourceType.DATE); + CoreValuesSourceType.BYTES); } public void testGlobalOrdinalsExecutionHint() throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 5ce1f0b53da8e..40934efbad630 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -61,12 +61,15 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.BinaryFieldMapper; +import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldAliasMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.Mapper.BuilderContext; @@ -101,8 +104,6 @@ import org.junit.After; import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -132,6 +133,7 @@ public abstract class AggregatorTestCase extends ESTestCase { private List releasables = new ArrayList<>(); private static final String TYPE_NAME = "type"; + // A list of field types that should not be tested, or are not currently supported private static List TYPE_TEST_BLACKLIST = List.of( ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects GeoShapeFieldMapper.CONTENT_TYPE, // Cannot aggregate geoshapes (yet) @@ -560,16 +562,50 @@ protected static IndexReader maybeWrapReaderEs(DirectoryReader reader) throws IO } } - protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return null; - } - + /** + * Implementors should return a list of {@link ValuesSourceType} that the aggregator supports. + * This is used to test the matrix of supported/unsupported field types against the aggregator + * and verify it works (or doesn't) as expected. + * + * If this method is implemented, {@link AggregatorTestCase#createAggBuilderForTypeTest(MappedFieldType, String)} + * should be implemented as well. + * + * @return list of supported ValuesSourceTypes + */ protected List getSupportedValuesSourceTypes() { + // If aggs don't override this method, an empty list allows the test to be skipped. + // Once all aggs implement this method we should make it abstract and not allow skipping. return Collections.emptyList(); } - public void testSupportedFieldTypes() throws IOException { + /** + * This method is invoked each time a field type is tested in {@link AggregatorTestCase#testSupportedFieldTypes()}. + * The field type and name are provided, and the implementor is expected to return an AggBuilder accordingly. + * The AggBuilder should be returned even if the aggregation does not support the field type, because + * the test will check if an exception is thrown in that case. + * + * The list of supported types are provided by {@link AggregatorTestCase#getSupportedValuesSourceTypes()}, + * which must also be implemented. + * + * @param fieldType the type of the field that will be tested + * @param fieldName the name of the field that will be test + * @return an aggregation builder to test against the field + */ + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + throw new UnsupportedOperationException("If getSupportedValuesSourceTypes() is implemented, " + + "createAggBuilderForTypeTest() must be implemented as well."); + } + /** + * This test will validate that an aggregator succeeds or fails to run against all the field types + * that are registered in {@link IndicesModule} (e.g. all the core field types). An aggregator + * is provided by the implementor class, and it is executed against each field type in turn. If + * an exception is thrown when the field is supported, that will fail the test. Similarly, if + * an exception _is not_ thrown when a field is unsupported, that will also fail the test. + * + * Exception types/messages are not currently checked, just presence/absence of an exception. + */ + public void testSupportedFieldTypes() throws IOException { MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); Settings settings = Settings.builder().put("index.version.created", Version.CURRENT.id).build(); String fieldName = "typeTestFieldName"; @@ -584,12 +620,9 @@ public void testSupportedFieldTypes() throws IOException { } for (Map.Entry mappedType : mapperRegistry.getMapperParsers().entrySet()) { - if (TYPE_TEST_BLACKLIST.contains(mappedType.getKey())) { - continue; - } - if (mappedType.getKey().equals(ObjectMapper.CONTENT_TYPE)) { - // Cannot aggregate objects + // Some field types should not be tested, or require more work and are not ready yet + if (TYPE_TEST_BLACKLIST.contains(mappedType.getKey())) { continue; } @@ -597,15 +630,10 @@ public void testSupportedFieldTypes() throws IOException { source.put("type", mappedType.getKey()); source.put("doc_values", "true"); - FieldMapper mapper = null; - try { - Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext()); - mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath())); - } catch (Exception e) { - fail(); - } - MappedFieldType fieldType = mapper.fieldType(); + Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext()); + FieldMapper mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath())); + MappedFieldType fieldType = mapper.fieldType(); try (Directory directory = newDirectory()) { RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); @@ -616,6 +644,7 @@ public void testSupportedFieldTypes() throws IOException { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, fieldName); + // TODO in the future we can make this more explicit with expectThrows(), when the exceptions are standardized try { searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); if (supportedVSTypes.contains(fieldType.getValuesSourceType()) == false) { @@ -633,15 +662,30 @@ public void testSupportedFieldTypes() throws IOException { } } + /** + * Helper method to write a single document with a single value specific to the requested fieldType. + * + * Throws an exception if it encounters an unknown field type, to prevent new ones from sneaking in without + * being tested. + */ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomIndexWriter iw) throws IOException { if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.NUMERIC)) { - iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomLong()))); - } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.DATE)) { - iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong()))); + // TODO note: once VS refactor adds DATE/BOOLEAN, this conditional will go away + if (fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE)) { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong()))); + } else if (fieldType.typeName().equals(BooleanFieldMapper.CONTENT_TYPE)) { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomBoolean() ? 0 : 1))); + } else { + iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomLong()))); + } } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.BYTES)) { if (fieldType.typeName().equals(BinaryFieldMapper.CONTENT_TYPE)) { iw.addDocument(singleton(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes))); + } else if (fieldType.typeName().equals(IpFieldMapper.CONTENT_TYPE)) { + // TODO note: once VS refactor adds IP, this conditional will go away + boolean v4 = randomBoolean(); + iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4)))))); } else { iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef("a")))); } @@ -652,28 +696,28 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd if (fieldType.typeName().equals(RangeType.DOUBLE.typeName())) { start = randomDouble(); - end = randomDoubleBetween(Math.nextUp((Double)start), Double.MAX_VALUE, false); + end = RangeType.DOUBLE.nextUp(start); rangeType = RangeType.DOUBLE; } else if (fieldType.typeName().equals(RangeType.FLOAT.typeName())) { start = randomFloat(); - end = randomFloatBetween(Math.nextUp((Float) start), Float.MAX_VALUE, false); + end = RangeType.FLOAT.nextUp(randomFloat()); rangeType = RangeType.DOUBLE; } else if (fieldType.typeName().equals(RangeType.IP.typeName())) { boolean v4 = randomBoolean(); start = randomIp(v4); - end = randomIp(v4); + end = RangeType.IP.nextUp(start); rangeType = RangeType.IP; } else if (fieldType.typeName().equals(RangeType.LONG.typeName())) { start = randomLong(); - end = randomLongBetween((Long)start + 1, Long.MAX_VALUE); + end = RangeType.LONG.nextUp(start); rangeType = RangeType.LONG; } else if (fieldType.typeName().equals(RangeType.INTEGER.typeName())) { start = randomInt(); - end = randomIntBetween((Integer)start, Integer.MAX_VALUE); + end = RangeType.INTEGER.nextUp(start); rangeType = RangeType.INTEGER; } else if (fieldType.typeName().equals(RangeType.DATE.typeName())) { start = randomNonNegativeLong(); - end = randomLongBetween((Long)start + 1, Long.MAX_VALUE); + end = RangeType.DATE.nextUp(start); rangeType = RangeType.DATE; } else { throw new IllegalStateException("Unknown type of range [" + fieldType.typeName() + "]"); @@ -682,12 +726,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, start, end, true, true); iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range))))); - } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.BOOLEAN)) { - iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomBoolean() ? 0 : 1))); - } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.IP)) { - boolean v4 = randomBoolean(); - iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4)))))); - } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.GEOPOINT)) { + } else if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.GEOPOINT)) { iw.addDocument(singleton(new LatLonDocValuesField(fieldName, randomDouble(), randomDouble()))); } else { throw new IllegalStateException("Unknown field type [" + fieldType.typeName() + "]"); @@ -700,23 +739,6 @@ private class MockParserContext extends Mapper.TypeParser.ParserContext { } } - private static InetAddress randomIp(boolean v4) { - try { - if (v4) { - byte[] ipv4 = new byte[4]; - random().nextBytes(ipv4); - return InetAddress.getByAddress(ipv4); - } else { - byte[] ipv6 = new byte[16]; - random().nextBytes(ipv6); - return InetAddress.getByAddress(ipv6); - } - } catch (UnknownHostException e) { - throw new AssertionError(); - } - } - - @After private void cleanupReleasables() { Releasables.close(releasables); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 18dd6140ca335..21078a085b017 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -29,7 +29,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; - import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -118,6 +117,8 @@ import java.io.IOException; import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.file.Path; import java.time.ZoneId; import java.util.ArrayList; @@ -620,35 +621,6 @@ public static float randomFloat() { return random().nextFloat(); } - /** - * Returns a float value in the interval [start, end) if lowerInclusive is - * set to true, (start, end) otherwise. - * - * @param start lower bound of interval to draw uniformly distributed random numbers from - * @param end upper bound - * @param lowerInclusive whether or not to include lower end of the interval - */ - public static float randomFloatBetween(float start, float end, boolean lowerInclusive) { - float result = 0.0f; - - if (start == -Float.MAX_VALUE || end == Float.MAX_VALUE) { - // formula below does not work with very large floats - result = Float.intBitsToFloat(randomInt()); - while (result < start || result > end || Float.isNaN(result)) { - result = Float.intBitsToFloat(randomInt()); - } - } else { - result = randomFloat(); - if (lowerInclusive == false) { - while (result <= 0.0) { - result = randomFloat(); - } - } - result = result * end + (1.0f - result) * start; - } - return result; - } - public static double randomDouble() { return random().nextDouble(); } @@ -1423,4 +1395,20 @@ protected static int getBasePort() { assert startAt >= 0 : "Unexpected test worker Id, resulting port range would be negative"; return 10300 + (startAt * 100); } + + protected static InetAddress randomIp(boolean v4) { + try { + if (v4) { + byte[] ipv4 = new byte[4]; + random().nextBytes(ipv4); + return InetAddress.getByAddress(ipv4); + } else { + byte[] ipv6 = new byte[16]; + random().nextBytes(ipv6); + return InetAddress.getByAddress(ipv6); + } + } catch (UnknownHostException e) { + throw new AssertionError(); + } + } } From f0973f97610cb8344fab357cedc98494fd0d2073 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Wed, 19 Feb 2020 14:01:16 -0500 Subject: [PATCH 4/6] Tweak for date_nanos --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 3 ++- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index e905ea2d5676b..a42a07a3d9e0b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -73,6 +73,7 @@ public final class DateFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "date"; + public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); public static class Defaults { @@ -96,7 +97,7 @@ public Instant clampToValidRange(Instant instant) { return instant; } }, - NANOSECONDS("date_nanos", NumericType.DATE_NANOSECONDS) { + NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS) { @Override public long convert(Instant instant) { return toLong(instant); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 40934efbad630..cc22e5d4f691b 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -672,7 +672,8 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd if (fieldType.getValuesSourceType().equals(CoreValuesSourceType.NUMERIC)) { // TODO note: once VS refactor adds DATE/BOOLEAN, this conditional will go away - if (fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE)) { + if (fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE) + || fieldType.typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong()))); } else if (fieldType.typeName().equals(BooleanFieldMapper.CONTENT_TYPE)) { iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomBoolean() ? 0 : 1))); From 24a62aa4479e5e31a869e030c754b6ad37b5bbbc Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 20 Feb 2020 11:10:50 -0500 Subject: [PATCH 5/6] Update test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java Co-Authored-By: Mark Tozzi --- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index cc22e5d4f691b..f70396e240651 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -701,7 +701,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd rangeType = RangeType.DOUBLE; } else if (fieldType.typeName().equals(RangeType.FLOAT.typeName())) { start = randomFloat(); - end = RangeType.FLOAT.nextUp(randomFloat()); + end = RangeType.FLOAT.nextUp(start); rangeType = RangeType.DOUBLE; } else if (fieldType.typeName().equals(RangeType.IP.typeName())) { boolean v4 = randomBoolean(); From 5b587e4817055a849e8a40a0fc5a1f17a88bc012 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 20 Feb 2020 11:15:25 -0500 Subject: [PATCH 6/6] Skip fields that are not aggregatable --- .../search/aggregations/AggregatorTestCase.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index f70396e240651..ee08622fef072 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -635,6 +635,11 @@ public void testSupportedFieldTypes() throws IOException { MappedFieldType fieldType = mapper.fieldType(); + // Non-aggregatable fields are not testable (they will throw an error on all aggs anyway), so skip + if (fieldType.isAggregatable() == false) { + continue; + } + try (Directory directory = newDirectory()) { RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); writeTestDoc(fieldType, fieldName, indexWriter);