diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index a9c6901d9820a..03368636555ac 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -95,6 +95,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridAggregationBuilder2; +import org.elasticsearch.search.aggregations.bucket.geogrid2.ParsedGeoGrid; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -1759,6 +1761,7 @@ static List getDefaultNamedXContents() { map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); map.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); + map.put(GeoGridAggregationBuilder2.NAME, (p, c) -> ParsedGeoGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index 795cc235ce759..7079bcc5b4606 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.GeoPointValues; import org.elasticsearch.index.fielddata.MultiGeoPointValues; @@ -555,23 +554,26 @@ private static GeoPoint parseGeoHash(GeoPoint point, String geohash, EffectivePo * @return int representing precision */ public static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException { - XContentParser.Token token = parser.currentToken(); - if (token.equals(XContentParser.Token.VALUE_NUMBER)) { - return XContentMapValues.nodeIntegerValue(parser.intValue()); - } else { - String precision = parser.text(); + return parser.currentToken() == Token.VALUE_NUMBER ? parser.intValue() : parsePrecisionString(parser.text()); + } + + /** + * Attempt to parse geohash precision string into an integer value + */ + public static int parsePrecisionString(String precision) { + try { + // we want to treat simple integer strings as precision levels, not distances + return checkPrecisionRange(Integer.parseInt(precision)); + // checkPrecisionRange could also throw IllegalArgumentException, but let it through + // to keep errors somewhat consistent with how they were shown before this change + } catch (NumberFormatException e) { + // try to parse as a distance value + final int parsedPrecision = GeoUtils.geoHashLevelsForPrecision(precision); try { - // we want to treat simple integer strings as precision levels, not distances - return XContentMapValues.nodeIntegerValue(precision); - } catch (NumberFormatException e) { - // try to parse as a distance value - final int parsedPrecision = GeoUtils.geoHashLevelsForPrecision(precision); - try { - return checkPrecisionRange(parsedPrecision); - } catch (IllegalArgumentException e2) { - // this happens when distance too small, so precision > 12. We'd like to see the original string - throw new IllegalArgumentException("precision too high [" + precision + "]", e2); - } + return checkPrecisionRange(parsedPrecision); + } catch (IllegalArgumentException e2) { + // this happens when distance too small, so precision > 12. We'd like to see the original string + throw new IllegalArgumentException("precision too high [" + precision + "]", e2); } } } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 2531685b94557..0d7b673d70815 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -110,6 +110,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridAggregationBuilder2; +import org.elasticsearch.search.aggregations.bucket.geogrid2.InternalGeoGrid; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -421,6 +423,8 @@ private void registerAggregations(List plugins) { GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new)); registerAggregation(new AggregationSpec(GeoGridAggregationBuilder.NAME, GeoGridAggregationBuilder::new, GeoGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new)); + registerAggregation(new AggregationSpec(GeoGridAggregationBuilder2.NAME, GeoGridAggregationBuilder2::new, + GeoGridAggregationBuilder2::parse).addResultReader(InternalGeoGrid::new)); registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new, NestedAggregationBuilder::parse).addResultReader(InternalNested::new)); registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index 6d8c8a94f3e6f..aded69584db87 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -30,6 +30,9 @@ import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridAggregationBuilder2; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridType; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -250,6 +253,13 @@ public static GeoGridAggregationBuilder geohashGrid(String name) { return new GeoGridAggregationBuilder(name); } + /** + * Create a new {@link GeoGrid} aggregation with the given name. + */ + public static GeoGridAggregationBuilder2 geoGrid(String name, GeoGridType type) { + return new GeoGridAggregationBuilder2(name, type); + } + /** * Create a new {@link SignificantTerms} aggregation with the given name. */ diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGrid.java new file mode 100644 index 0000000000000..d06efcef2fb88 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGrid.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; + +import java.util.List; + +/** + * A {@code geo_grid} aggregation. Defines multiple buckets, each representing a cell in a geo-grid of a specific + * precision. + */ +public interface GeoGrid extends MultiBucketsAggregation { + + /** + * A bucket that is associated with a {@code geohash_grid} cell. The key of the bucket is the {@code geohash} of the cell + */ + interface Bucket extends MultiBucketsAggregation.Bucket { + } + + /** + * @return The buckets of this aggregation (each bucket representing a geohash grid cell) + */ + @Override + List getBuckets(); +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregationBuilder2.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregationBuilder2.java new file mode 100644 index 0000000000000..88386bd508b3f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregationBuilder2.java @@ -0,0 +1,341 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.BucketUtils; +import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class GeoGridAggregationBuilder2 extends ValuesSourceAggregationBuilder + implements MultiBucketAggregationBuilder { + public static final String NAME = "geo_grid"; + public static final int DEFAULT_MAX_NUM_CELLS = 10000; + + /* recognized field names in JSON */ + static final ParseField FIELD_TYPE = new ParseField("hash_type"); + static final ParseField FIELD_PRECISION = new ParseField("precision"); + static final ParseField FIELD_SIZE = new ParseField("size"); + static final ParseField FIELD_SHARD_SIZE = new ParseField("shard_size"); + + public static GeoGridTypes types; + + private static final ConstructingObjectParser PARSER; + + static { + types = GeoGridTypes.DEFAULT; + + PARSER = new ConstructingObjectParser<>(GeoGridAggregationBuilder2.NAME, false, + (a, name) -> new GeoGridAggregationBuilder2(name, (GeoGridType) a[0])); + + PARSER.declareField( + constructorArg(), + GeoGridAggregationBuilder2::parseType, + FIELD_TYPE, + ObjectParser.ValueType.STRING); + PARSER.declareField( + GeoGridAggregationBuilder2::precisionRaw, + GeoGridAggregationBuilder2::parsePrecision, + FIELD_PRECISION, + ObjectParser.ValueType.VALUE); + PARSER.declareInt( + GeoGridAggregationBuilder2::size, + FIELD_SIZE); + PARSER.declareInt( + GeoGridAggregationBuilder2::shardSize, + FIELD_SHARD_SIZE); + + ValuesSourceParserHelper.declareGeoFields(PARSER, false, false); + } + + private static Object parsePrecision(XContentParser parser, String name) + throws IOException { + // Delay actual parsing until builder.precision() + // In some cases, this value cannot be fully parsed until after we know the type + final XContentParser.Token token = parser.currentToken(); + switch (token) { + case VALUE_NUMBER: + return parser.intValue(); + case VALUE_STRING: + return parser.text(); + default: + throw new XContentParseException(parser.getTokenLocation(), + "[geo_grid] failed to parse field [precision] in [" + name + + "]. It must be either an integer or a string"); + } + } + + public static GeoGridAggregationBuilder2 parse(String aggregationName, XContentParser parser) { + return PARSER.apply(parser, aggregationName); + } + + private final GeoGridType type; + private int precision; + private int requiredSize = DEFAULT_MAX_NUM_CELLS; + private int shardSize = -1; + + public GeoGridAggregationBuilder2(String name, GeoGridType type) { + super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + this.type = type; + this.precision = this.type.getDefaultPrecision(); + } + + protected GeoGridAggregationBuilder2(GeoGridAggregationBuilder2 clone, Builder factoriesBuilder, Map metaData) { + super(clone, factoriesBuilder, metaData); + this.type = clone.type; + this.precision = clone.precision; + this.requiredSize = clone.requiredSize; + this.shardSize = clone.shardSize; + } + + @Override + protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { + return new GeoGridAggregationBuilder2(this, factoriesBuilder, metaData); + } + + /** + * Read from a stream. + */ + public GeoGridAggregationBuilder2(StreamInput in) throws IOException { + super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + + // FIXME: better debug name than a class name? + type = types.get(in.readString(), this.getClass().getName()); + precision = in.readVInt(); + requiredSize = in.readVInt(); + shardSize = in.readVInt(); + } + + @Override + protected void innerWriteTo(StreamOutput out) throws IOException { + out.writeString(type.getName()); + out.writeVInt(precision); + out.writeVInt(requiredSize); + out.writeVInt(shardSize); + } + + private static GeoGridType parseType(XContentParser parser, String name) throws IOException { + return types.get(parser.text(), name); + } + + public GeoGridType type() { + return type; + } + + private GeoGridAggregationBuilder2 precisionRaw(Object precision) { + if (precision == null) { + this.precision(type.getDefaultPrecision()); + } else if (precision instanceof String) { + this.precision(type.parsePrecisionString((String) precision)); + } else { + this.precision((int) precision); + } + return this; + } + + public GeoGridAggregationBuilder2 precision(int precision) { + this.precision = type.validatePrecision(precision); + return this; + } + + public int precision() { + return precision; + } + + public GeoGridAggregationBuilder2 size(int size) { + if (size <= 0) { + throw new IllegalArgumentException( + "[size] must be greater than 0. Found [" + size + "] in [" + name + "]"); + } + this.requiredSize = size; + return this; + } + + public int size() { + return requiredSize; + } + + public GeoGridAggregationBuilder2 shardSize(int shardSize) { + if (shardSize <= 0) { + throw new IllegalArgumentException( + "[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]"); + } + this.shardSize = shardSize; + return this; + } + + public int shardSize() { + return shardSize; + } + + @Override + protected ValuesSourceAggregatorFactory innerBuild(SearchContext context, + ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) + throws IOException { + int shardSize = this.shardSize; + int requiredSize = this.requiredSize; + + if (shardSize < 0) { + // Use default heuristic to avoid any wrong-ranking caused by + // distributed counting + shardSize = BucketUtils.suggestShardSideQueueSize(requiredSize); + } + if (requiredSize <= 0 || shardSize <= 0) { + throw new ElasticsearchException( + "parameters [required_size] and [shard_size] must be >0 in geo_grid aggregation [" + name + "]."); + } + if (shardSize < requiredSize) { + shardSize = requiredSize; + } + return new GeoGridAggregatorFactory(name, config, type, precision, requiredSize, shardSize, context, parent, + subFactoriesBuilder, metaData); + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(FIELD_TYPE.getPreferredName(), type.getName()); + builder.field(FIELD_PRECISION.getPreferredName(), precision); + builder.field(FIELD_SIZE.getPreferredName(), requiredSize); + if (shardSize > -1) { + builder.field(FIELD_SHARD_SIZE.getPreferredName(), shardSize); + } + return builder; + } + + @Override + protected boolean innerEquals(Object obj) { + GeoGridAggregationBuilder2 other = (GeoGridAggregationBuilder2) obj; + return Objects.equals(type, other.type) && + Objects.equals(precision, other.precision) && + Objects.equals(requiredSize, other.requiredSize) && + Objects.equals(shardSize, other.shardSize); + } + + @Override + protected int innerHashCode() { + return Objects.hash(type, precision, requiredSize, shardSize); + } + + @Override + public String getType() { + return NAME; + } + + private static class CellValues extends AbstractSortingNumericDocValues { + private MultiGeoPointValues geoValues; + private GeoGridType type; + private int precision; + + protected CellValues(MultiGeoPointValues geoValues, GeoGridType type, int precision) { + this.geoValues = geoValues; + this.type = type; + this.precision = precision; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (geoValues.advanceExact(docId)) { + resize(geoValues.docValueCount()); + + for (int i = 0; i < docValueCount(); ++i) { + GeoPoint target = geoValues.nextValue(); + values[i] = type.calculateHash(target.getLon(), target.getLat(), precision); + } + sort(); + return true; + } else { + return false; + } + } + } + + static class CellIdSource extends ValuesSource.Numeric { + private final ValuesSource.GeoPoint valuesSource; + private final GeoGridType type; + private final int precision; + + CellIdSource(ValuesSource.GeoPoint valuesSource, GeoGridType type, int precision) { + this.valuesSource = valuesSource; + //different GeoPoints could map to the same or different geogrid cells. + this.type = type; + this.precision = precision; + } + + public GeoGridType type() { + return type; + } + + public int precision() { + return precision; + } + + @Override + public boolean isFloatingPoint() { + return false; + } + + @Override + public SortedNumericDocValues longValues(LeafReaderContext ctx) { + return new CellValues(valuesSource.geoPointValues(ctx), type, precision); + } + + @Override + public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) { + throw new UnsupportedOperationException(); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext ctx) { + throw new UnsupportedOperationException(); + } + + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregator.java new file mode 100644 index 0000000000000..9d410944aabf6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregator.java @@ -0,0 +1,139 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.LongHash; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Aggregates data expressed as Geogrid hash longs (for efficiency's sake) but formats results as geo grid strings. + */ +public class GeoGridAggregator extends BucketsAggregator { + + private final int requiredSize; + private final int shardSize; + private final GeoGridAggregationBuilder2.CellIdSource valuesSource; + private final LongHash bucketOrds; + private final GeoGridType type; + + GeoGridAggregator(String name, AggregatorFactories factories, GeoGridAggregationBuilder2.CellIdSource valuesSource, + int requiredSize, int shardSize, SearchContext aggregationContext, Aggregator parent, + List pipelineAggregators, Map metaData, GeoGridType type + ) throws IOException { + super(name, factories, aggregationContext, parent, pipelineAggregators, metaData); + this.valuesSource = valuesSource; + this.requiredSize = requiredSize; + this.shardSize = shardSize; + this.type = type; + bucketOrds = new LongHash(1, aggregationContext.bigArrays()); + } + + @Override + public ScoreMode scoreMode() { + if (valuesSource != null && valuesSource.needsScores()) { + return ScoreMode.COMPLETE; + } + return super.scoreMode(); + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, + final LeafBucketCollector sub) throws IOException { + final SortedNumericDocValues values = valuesSource.longValues(ctx); + return new LeafBucketCollectorBase(sub, null) { + @Override + public void collect(int doc, long bucket) throws IOException { + assert bucket == 0; + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + long previous = Long.MAX_VALUE; + for (int i = 0; i < valuesCount; ++i) { + final long val = values.nextValue(); + if (previous != val || i == 0) { + long bucketOrdinal = bucketOrds.add(val); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } + previous = val; + } + } + } + } + }; + } + + @Override + public InternalGeoGrid buildAggregation(long owningBucketOrdinal) throws IOException { + assert owningBucketOrdinal == 0; + final int size = (int) Math.min(bucketOrds.size(), shardSize); + consumeBucketsAndMaybeBreak(size); + + InternalGeoGrid.BucketPriorityQueue ordered = new InternalGeoGrid.BucketPriorityQueue(size); + GeoGridBucket spare = null; + for (long i = 0; i < bucketOrds.size(); i++) { + if (spare == null) { + spare = type.createBucket(0, 0, null); + } + + spare.hashAsLong = bucketOrds.get(i); + spare.docCount = bucketDocCount(i); + spare.bucketOrd = i; + spare = ordered.insertWithOverflow(spare); + } + + final GeoGridBucket[] list = new GeoGridBucket[ordered.size()]; + for (int i = ordered.size() - 1; i >= 0; --i) { + final GeoGridBucket bucket = ordered.pop(); + bucket.aggregations = bucketAggregations(bucket.bucketOrd); + list[i] = bucket; + } + return new InternalGeoGrid(name, type, requiredSize, Arrays.asList(list), pipelineAggregators(), metaData()); + } + + @Override + public InternalGeoGrid buildEmptyAggregation() { + return new InternalGeoGrid(name, type, requiredSize, Collections.emptyList(), pipelineAggregators(), metaData()); + } + + @Override + public void doClose() { + Releasables.close(bucketOrds); + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregatorFactory.java new file mode 100644 index 0000000000000..bea23d17df6d8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregatorFactory.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.NonCollectingAggregator; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridAggregationBuilder2.CellIdSource; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSource.GeoPoint; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class GeoGridAggregatorFactory extends ValuesSourceAggregatorFactory { + + private final GeoGridType type; + private final int precision; + private final int requiredSize; + private final int shardSize; + + GeoGridAggregatorFactory(String name, ValuesSourceConfig config, GeoGridType type, int precision, + int requiredSize, int shardSize, SearchContext context, AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, Map metaData + ) throws IOException { + super(name, config, context, parent, subFactoriesBuilder, metaData); + this.type = type; + this.precision = precision; + this.requiredSize = requiredSize; + this.shardSize = shardSize; + } + + @Override + protected Aggregator createUnmapped(Aggregator parent, List pipelineAggregators, Map metaData) + throws IOException { + final InternalAggregation aggregation = new InternalGeoGrid(name, type, requiredSize, + Collections. emptyList(), pipelineAggregators, metaData); + return new NonCollectingAggregator(name, context, parent, pipelineAggregators, metaData) { + @Override + public InternalAggregation buildEmptyAggregation() { + return aggregation; + } + }; + } + + @Override + protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, Aggregator parent, boolean collectsFromSingleBucket, + List pipelineAggregators, Map metaData) throws IOException { + if (collectsFromSingleBucket == false) { + return asMultiBucketAggregator(this, context, parent); + } + CellIdSource cellIdSource = new CellIdSource(valuesSource, type, precision); + return new GeoGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, context, parent, + pipelineAggregators, metaData, type); + + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridBucket.java new file mode 100644 index 0000000000000..75ffcac5d342e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridBucket.java @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.aggregations.Aggregation.CommonFields; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + + +abstract class GeoGridBucket extends InternalMultiBucketAggregation.InternalBucket implements GeoGrid.Bucket, Comparable { + + // This is only used by the private impl that stores a bucket ord. This allows for computing the aggregations lazily. + // This field is used temporarily, and will not be used in serialization or comparison. + long bucketOrd; + + protected long hashAsLong; + protected long docCount; + protected InternalAggregations aggregations; + + /** + * Factory method to instantiate a new bucket from inside the bucket. + * Derived buckets should do return new Bucket(hashAsLong, docCount, aggregations); + */ + protected abstract GeoGridBucket newBucket(long hashAsLong, long docCount, InternalAggregations aggregations); + + protected GeoGridBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { + this.docCount = docCount; + this.aggregations = aggregations; + this.hashAsLong = hashAsLong; + } + + /** + * Read from a stream. + */ + protected GeoGridBucket(StreamInput in) throws IOException { + hashAsLong = in.readLong(); + docCount = in.readVLong(); + aggregations = InternalAggregations.readAggregations(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(hashAsLong); + out.writeVLong(docCount); + aggregations.writeTo(out); + } + + @Override + public String getKey() { + return getKeyAsString(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString()); + builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); + aggregations.toXContentInternal(builder, params); + builder.endObject(); + return builder; + } + + @Override + public long getDocCount() { + return docCount; + } + + @Override + public Aggregations getAggregations() { + return aggregations; + } + + @Override + public int compareTo(GeoGridBucket other) { + if (this.hashAsLong > other.hashAsLong) { + return 1; + } + if (this.hashAsLong < other.hashAsLong) { + return -1; + } + return 0; + } + + public GeoGridBucket reduce(List buckets, InternalAggregation.ReduceContext context) { + List aggregationsList = new ArrayList<>(buckets.size()); + long docCount = 0; + for (GeoGridBucket bucket : buckets) { + docCount += bucket.docCount; + aggregationsList.add(bucket.aggregations); + } + final InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context); + return newBucket(hashAsLong, docCount, aggs); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GeoGridBucket bucket = (GeoGridBucket) o; + return hashAsLong == bucket.hashAsLong && + docCount == bucket.docCount && + Objects.equals(aggregations, bucket.aggregations); + } + + @Override + public int hashCode() { + return Objects.hash(hashAsLong, docCount, aggregations); + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridType.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridType.java new file mode 100644 index 0000000000000..cdbe52f640e83 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridType.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.InternalAggregations; + +import java.io.IOException; + +/** + * Instances implement different hashing algorithms for geo-grid aggregations + */ +public interface GeoGridType { + /** + * Returns the name of the grid aggregation, e.g. "geohash" + */ + String getName(); + + /** + * Returns default precision for the type, e.g. 5 for geohash + */ + int getDefaultPrecision(); + + /** + * Parses precision string into an integer, e.g. "100km" into 4 + */ + int parsePrecisionString(String precision); + + /** + * Validates precision for the given geo type, and throws an exception on error + * @param precision value to validate + * @return the original value if everything is ok + */ + int validatePrecision(int precision); + + /** + * Converts longitude/latitude into a bucket identifying hash value with the given precision + * @return hash value + */ + long calculateHash(double longitude, double latitude, int precision); + + /** + * Decodes hash value into a string returned to the user + * @param hash as generated by the {@link #calculateHash} + * @return bucket ID as a string + */ + String hashAsString(long hash); + + /** + * Factory method to create a new bucket. + */ + GeoGridBucket createBucket(long hashAsLong, long docCount, InternalAggregations aggregations); + + /** + * Factory method to create a new bucket from a stream. + */ + GeoGridBucket createBucket(StreamInput reader) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridTypes.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridTypes.java new file mode 100644 index 0000000000000..34b381741fad0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridTypes.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import java.util.Map; +import java.util.HashMap; + +/** + * Store for types - needed in multiple classes + */ +public class GeoGridTypes { + + public static GeoGridTypes DEFAULT = new GeoGridTypes(); + + private Map types; + + private GeoGridTypes() { + // TODO: we need to decide how types map is instantiated/stored + // TODO: especially this is important to allow type plugins + types = new HashMap<>(); + types.put(GeoHashType.SINGLETON.getName(), GeoHashType.SINGLETON); + } + + public GeoGridType get(String typeStr, String name) { + final GeoGridType type = types.get(typeStr); + if (type != null) { + return type; + } + throw new IllegalArgumentException( + "[type] is not valid. Allowed values: " + + String.join(", ", types.keySet()) + + ". Found [" + typeStr + "] in [" + name + "]"); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoHashBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoHashBucket.java new file mode 100644 index 0000000000000..718366586bb82 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoHashBucket.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.InternalAggregations; + +import java.io.IOException; + +/** + * A bucket to store GeoHash geogrid aggregation + */ +public class GeoHashBucket extends GeoGridBucket { + + protected GeoHashBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { + super(hashAsLong, docCount, aggregations); + } + + protected GeoHashBucket(StreamInput in) throws IOException { + super(in); + } + + @Override + protected GeoGridBucket newBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { + return new GeoHashBucket(hashAsLong, docCount, aggregations); + } + + @Override + public String getKeyAsString() { + return GeoHashType.SINGLETON.hashAsString(this.hashAsLong); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoHashType.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoHashType.java new file mode 100644 index 0000000000000..bc5747f0f7f7f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoHashType.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.common.geo.GeoHashUtils; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.InternalAggregations; + +import java.io.IOException; + +/** + * A simple wrapper for GeoUtils handling of the geohash hashing algorithm + */ +public class GeoHashType implements GeoGridType { + + /** + * GeoGridType must be a singleton because bucket does a reference compare for equality + */ + private GeoHashType() {} + + public static final String NAME = "geohash"; + + public static final GeoGridType SINGLETON = new GeoHashType(); + + @Override + public String getName() { + return NAME; + } + + @Override + public int getDefaultPrecision() { + return 5; + } + + @Override + public int parsePrecisionString(String precision) { + return GeoUtils.parsePrecisionString(precision); + } + + @Override + public int validatePrecision(int precision) { + return GeoUtils.checkPrecisionRange(precision); + } + + @Override + public long calculateHash(double longitude, double latitude, int precision) { + return GeoHashUtils.longEncode(longitude, latitude, precision); + } + + @Override + public String hashAsString(long hash) { + return GeoHashUtils.stringEncode(hash); + } + + @Override + public GeoGridBucket createBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { + return new GeoHashBucket(hashAsLong, docCount, aggregations); + } + + @Override + public GeoGridBucket createBucket(StreamInput reader) throws IOException { + return new GeoHashBucket(reader); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/InternalGeoGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/InternalGeoGrid.java new file mode 100644 index 0000000000000..7c2dd9154bf9e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/InternalGeoGrid.java @@ -0,0 +1,179 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.apache.lucene.util.PriorityQueue; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.LongObjectPagedHashMap; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Collections.unmodifiableList; + +/** + * Represents a grid of cells where each cell's location is determined by a hash. + * All hashes in a grid are of the same precision and held internally as a single long + * for efficiency's sake. + */ +public class InternalGeoGrid extends InternalMultiBucketAggregation implements GeoGrid { + + private final GeoGridType type; + private final int requiredSize; + private final List buckets; + + InternalGeoGrid(String name, GeoGridType type, int requiredSize, List buckets, + List pipelineAggregators, Map metaData) { + super(name, pipelineAggregators, metaData); + this.type = type; + this.requiredSize = requiredSize; + this.buckets = buckets; + } + + /** + * Read from a stream. + */ + public InternalGeoGrid(StreamInput in) throws IOException { + super(in); + type = GeoGridTypes.DEFAULT.get(in.readString(), "internal-worker"); + requiredSize = readSize(in); + buckets = in.readList(type::createBucket); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(type.getName()); + writeSize(requiredSize, out); + out.writeList(buckets); + } + + @Override + public String getWriteableName() { + return GeoGridAggregationBuilder2.NAME; + } + + @Override + public InternalGeoGrid create(List buckets) { + return new InternalGeoGrid(this.name, this.type, this.requiredSize, buckets, this.pipelineAggregators(), this.metaData); + } + + @Override + public GeoGridBucket createBucket(InternalAggregations aggregations, GeoGridBucket prototype) { + return type.createBucket(prototype.hashAsLong, prototype.docCount, aggregations); + } + + @Override + public List getBuckets() { + return unmodifiableList(buckets); + } + + @Override + public InternalGeoGrid doReduce(List aggregations, ReduceContext reduceContext) { + LongObjectPagedHashMap> buckets = null; + for (InternalAggregation aggregation : aggregations) { + InternalGeoGrid grid = (InternalGeoGrid) aggregation; + if (buckets == null) { + buckets = new LongObjectPagedHashMap<>(grid.buckets.size(), reduceContext.bigArrays()); + } + for (GeoGridBucket bucket : grid.buckets) { + List existingBuckets = buckets.get(bucket.hashAsLong); + if (existingBuckets == null) { + existingBuckets = new ArrayList<>(aggregations.size()); + buckets.put(bucket.hashAsLong, existingBuckets); + } + existingBuckets.add(bucket); + } + } + + final int size = Math.toIntExact(reduceContext.isFinalReduce() == false ? buckets.size() : Math.min(requiredSize, buckets.size())); + BucketPriorityQueue ordered = new BucketPriorityQueue(size); + for (LongObjectPagedHashMap.Cursor> cursor : buckets) { + List sameCellBuckets = cursor.value; + GeoGridBucket removed = ordered.insertWithOverflow(sameCellBuckets.get(0).reduce(sameCellBuckets, reduceContext)); + if (removed != null) { + reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed)); + } else { + reduceContext.consumeBucketsAndMaybeBreak(1); + } + } + buckets.close(); + GeoGridBucket[] list = new GeoGridBucket[ordered.size()]; + for (int i = ordered.size() - 1; i >= 0; i--) { + list[i] = ordered.pop(); + } + return new InternalGeoGrid(getName(), type, requiredSize, Arrays.asList(list), pipelineAggregators(), getMetaData()); + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.startArray(CommonFields.BUCKETS.getPreferredName()); + for (GeoGridBucket bucket : buckets) { + bucket.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + // package protected for testing + int getRequiredSize() { + return requiredSize; + } + + @Override + protected int doHashCode() { + return Objects.hash(requiredSize, buckets); + } + + @Override + protected boolean doEquals(Object obj) { + InternalGeoGrid other = (InternalGeoGrid) obj; + return Objects.equals(requiredSize, other.requiredSize) && + Objects.equals(buckets, other.buckets); + } + + static class BucketPriorityQueue extends PriorityQueue { + + BucketPriorityQueue(int size) { + super(size); + } + + @Override + protected boolean lessThan(GeoGridBucket o1, GeoGridBucket o2) { + int cmp = Long.compare(o2.getDocCount(), o1.getDocCount()); + if (cmp == 0) { + cmp = o2.compareTo(o1); + if (cmp == 0) { + cmp = System.identityHashCode(o2) - System.identityHashCode(o1); + } + } + return cmp > 0; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/ParsedGeoGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/ParsedGeoGrid.java new file mode 100644 index 0000000000000..04c3c057c3c80 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid2/ParsedGeoGrid.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; + +import java.io.IOException; +import java.util.List; + +public class ParsedGeoGrid extends ParsedMultiBucketAggregation implements GeoGrid { + + @Override + public String getType() { + return GeoGridAggregationBuilder2.NAME; + } + + @Override + public List getBuckets() { + return buckets; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedGeoGrid.class.getSimpleName(), true, ParsedGeoGrid::new); + static { + declareMultiBucketAggregationFields(PARSER, ParsedBucket::fromXContent, ParsedBucket::fromXContent); + } + + public static ParsedGeoGrid fromXContent(XContentParser parser, String name) throws IOException { + ParsedGeoGrid aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements GeoGrid.Bucket { + + private String geohashAsString; + + @Override + public String getKey() { + return getKeyAsString(); + } + + @Override + public String getKeyAsString() { + return geohashAsString; + } + + @Override + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), geohashAsString); + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseXContent(parser, false, ParsedBucket::new, (p, bucket) -> bucket.geohashAsString = p.textOrNull()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index ef001b35feffb..b7306ae48b816 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests; import org.elasticsearch.search.aggregations.bucket.filter.InternalFiltersTests; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridTests; +import org.elasticsearch.search.aggregations.bucket.geogrid2.InternalGeoGridGeoHashTests; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobalTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalAutoDateHistogramTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; @@ -140,6 +141,7 @@ private static List> getAggsTests() { aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); aggsTests.add(new InternalGeoHashGridTests()); + aggsTests.add(new InternalGeoGridGeoHashTests()); aggsTests.add(new InternalRangeTests()); aggsTests.add(new InternalDateRangeTests()); aggsTests.add(new InternalGeoDistanceTests()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoGridIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoGridIT.java new file mode 100644 index 0000000000000..ea818d929ff8c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoGridIT.java @@ -0,0 +1,330 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket; + +import com.carrotsearch.hppc.ObjectIntHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import com.carrotsearch.hppc.cursors.ObjectIntCursor; +import org.elasticsearch.Version; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.bucket.filter.Filter; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGrid.Bucket; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoHashType; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import static org.elasticsearch.common.geo.GeoHashUtils.PRECISION; +import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.search.aggregations.AggregationBuilders.geoGrid; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.SuiteScopeTestCase +public class GeoGridIT extends ESIntegTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + private Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, + Version.CURRENT); + + static ObjectIntMap expectedDocCountsForGeoHash = null; + static ObjectIntMap multiValuedExpectedDocCountsForGeoHash = null; + static int numDocs = 100; + + static String smallestGeoHash = null; + + private static IndexRequestBuilder indexCity(String index, String name, List latLon) throws Exception { + XContentBuilder source = jsonBuilder().startObject().field("city", name); + if (latLon != null) { + source = source.field("location", latLon); + } + source = source.endObject(); + return client().prepareIndex(index, "type").setSource(source); + } + + private static IndexRequestBuilder indexCity(String index, String name, String latLon) throws Exception { + return indexCity(index, name, Arrays.asList(latLon)); + } + + @Override + public void setupSuiteScopeCluster() throws Exception { + createIndex("idx_unmapped"); + + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + + assertAcked(prepareCreate("idx").setSettings(settings) + .addMapping("type", "location", "type=geo_point", "city", "type=keyword")); + + List cities = new ArrayList<>(); + Random random = random(); + expectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); + for (int i = 0; i < numDocs; i++) { + //generate random point + double lat = (180d * random.nextDouble()) - 90d; + double lng = (360d * random.nextDouble()) - 180d; + String randomGeoHash = stringEncode(lng, lat, PRECISION); + //Index at the highest resolution + cities.add(indexCity("idx", randomGeoHash, lat + ", " + lng)); + expectedDocCountsForGeoHash.put(randomGeoHash, expectedDocCountsForGeoHash.getOrDefault(randomGeoHash, 0) + 1); + //Update expected doc counts for all resolutions.. + for (int precision = PRECISION - 1; precision > 0; precision--) { + String hash = stringEncode(lng, lat, precision); + if ((smallestGeoHash == null) || (hash.length() < smallestGeoHash.length())) { + smallestGeoHash = hash; + } + expectedDocCountsForGeoHash.put(hash, expectedDocCountsForGeoHash.getOrDefault(hash, 0) + 1); + } + } + indexRandom(true, cities); + + assertAcked(prepareCreate("multi_valued_idx").setSettings(settings) + .addMapping("type", "location", "type=geo_point", "city", "type=keyword")); + + cities = new ArrayList<>(); + multiValuedExpectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); + for (int i = 0; i < numDocs; i++) { + final int numPoints = random.nextInt(4); + List points = new ArrayList<>(); + Set geoHashes = new HashSet<>(); + for (int j = 0; j < numPoints; ++j) { + double lat = (180d * random.nextDouble()) - 90d; + double lng = (360d * random.nextDouble()) - 180d; + points.add(lat + "," + lng); + // Update expected doc counts for all resolutions.. + for (int precision = PRECISION; precision > 0; precision--) { + final String geoHash = stringEncode(lng, lat, precision); + geoHashes.add(geoHash); + } + } + cities.add(indexCity("multi_valued_idx", Integer.toString(i), points)); + for (String hash : geoHashes) { + multiValuedExpectedDocCountsForGeoHash.put(hash, multiValuedExpectedDocCountsForGeoHash.getOrDefault(hash, 0) + 1); + } + } + indexRandom(true, cities); + + ensureSearchable(); + } + + public void testSimple() throws Exception { + for (int precision = 1; precision <= PRECISION; precision++) { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(geoGrid("geogrid", GeoHashType.SINGLETON) + .field("location") + .precision(precision) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid geoGrid = response.getAggregations().get("geogrid"); + List buckets = geoGrid.getBuckets(); + Object[] propertiesKeys = (Object[]) ((InternalAggregation)geoGrid).getProperty("_key"); + Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)geoGrid).getProperty("_count"); + for (int i = 0; i < buckets.size(); i++) { + Bucket cell = buckets.get(i); + String geohash = cell.getKeyAsString(); + + long bucketCount = cell.getDocCount(); + int expectedBucketCount = expectedDocCountsForGeoHash.get(geohash); + assertNotSame(bucketCount, 0); + assertEquals("Geohash " + geohash + " has wrong doc count ", + expectedBucketCount, bucketCount); + GeoPoint geoPoint = (GeoPoint) propertiesKeys[i]; + assertThat(stringEncode(geoPoint.lon(), geoPoint.lat(), precision), equalTo(geohash)); + assertThat((long) propertiesDocCounts[i], equalTo(bucketCount)); + } + } + } + + public void testMultivalued() throws Exception { + for (int precision = 1; precision <= PRECISION; precision++) { + SearchResponse response = client().prepareSearch("multi_valued_idx") + .addAggregation(geoGrid("geogrid", GeoHashType.SINGLETON) + .field("location") + .precision(precision) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid geoGrid = response.getAggregations().get("geogrid"); + for (Bucket cell : geoGrid.getBuckets()) { + String geohash = cell.getKeyAsString(); + + long bucketCount = cell.getDocCount(); + int expectedBucketCount = multiValuedExpectedDocCountsForGeoHash.get(geohash); + assertNotSame(bucketCount, 0); + assertEquals("Geogrid " + geohash + " has wrong doc count ", + expectedBucketCount, bucketCount); + } + } + } + + public void testFiltered() throws Exception { + GeoBoundingBoxQueryBuilder bbox = new GeoBoundingBoxQueryBuilder("location"); + bbox.setCorners(smallestGeoHash).queryName("bbox"); + for (int precision = 1; precision <= PRECISION; precision++) { + SearchResponse response = client().prepareSearch("idx") + .addAggregation( + AggregationBuilders.filter("filtered", bbox) + .subAggregation( + geoGrid("geogrid", GeoHashType.SINGLETON) + .field("location") + .precision(precision) + ) + ) + .get(); + + assertSearchResponse(response); + + Filter filter = response.getAggregations().get("filtered"); + + GeoGrid geoGrid = filter.getAggregations().get("geogrid"); + for (Bucket cell : geoGrid.getBuckets()) { + String geohash = cell.getKeyAsString(); + long bucketCount = cell.getDocCount(); + int expectedBucketCount = expectedDocCountsForGeoHash.get(geohash); + assertNotSame(bucketCount, 0); + assertTrue("Buckets must be filtered", geohash.startsWith(smallestGeoHash)); + assertEquals("Geohash " + geohash + " has wrong doc count ", + expectedBucketCount, bucketCount); + + } + } + } + + public void testUnmapped() throws Exception { + for (int precision = 1; precision <= PRECISION; precision++) { + SearchResponse response = client().prepareSearch("idx_unmapped") + .addAggregation(geoGrid("geogrid", GeoHashType.SINGLETON) + .field("location") + .precision(precision) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid geoGrid = response.getAggregations().get("geogrid"); + assertThat(geoGrid.getBuckets().size(), equalTo(0)); + } + + } + + public void testPartiallyUnmapped() throws Exception { + for (int precision = 1; precision <= PRECISION; precision++) { + SearchResponse response = client().prepareSearch("idx", "idx_unmapped") + .addAggregation(geoGrid("geogrid", GeoHashType.SINGLETON) + .field("location") + .precision(precision) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid geoGrid = response.getAggregations().get("geogrid"); + for (Bucket cell : geoGrid.getBuckets()) { + String geohash = cell.getKeyAsString(); + + long bucketCount = cell.getDocCount(); + int expectedBucketCount = expectedDocCountsForGeoHash.get(geohash); + assertNotSame(bucketCount, 0); + assertEquals("Geohash " + geohash + " has wrong doc count ", + expectedBucketCount, bucketCount); + } + } + } + + public void testTopMatch() throws Exception { + for (int precision = 1; precision <= PRECISION; precision++) { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(geoGrid("geogrid", GeoHashType.SINGLETON) + .field("location") + .size(1) + .shardSize(100) + .precision(precision) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid geoGrid = response.getAggregations().get("geogrid"); + //Check we only have one bucket with the best match for that resolution + assertThat(geoGrid.getBuckets().size(), equalTo(1)); + for (Bucket cell : geoGrid.getBuckets()) { + String geohash = cell.getKeyAsString(); + long bucketCount = cell.getDocCount(); + int expectedBucketCount = 0; + for (ObjectIntCursor cursor : expectedDocCountsForGeoHash) { + if (cursor.key.length() == precision) { + expectedBucketCount = Math.max(expectedBucketCount, cursor.value); + } + } + assertNotSame(bucketCount, 0); + assertEquals("Geohash " + geohash + " has wrong doc count ", + expectedBucketCount, bucketCount); + } + } + } + + public void testSizeIsZero() { + final int size = 0; + final int shardSize = 10000; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> client() + .prepareSearch("idx") + .addAggregation( + geoGrid("geogrid", GeoHashType.SINGLETON) + .field("location").size(size).shardSize(shardSize)).get()); + assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [geogrid]")); + } + + public void testShardSizeIsZero() { + final int size = 100; + final int shardSize = 0; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> client().prepareSearch("idx") + .addAggregation(geoGrid("geogrid", GeoHashType.SINGLETON).field("location").size(size).shardSize(shardSize)) + .get()); + assertThat(exception.getMessage(), containsString("[shardSize] must be greater than 0. Found [0] in [geogrid]")); + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoGridTests.java new file mode 100644 index 0000000000000..065510d3aa8f6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoGridTests.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket; + +import org.elasticsearch.search.aggregations.BaseAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridAggregationBuilder2; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridType; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoHashType; + +public class GeoGridTests extends BaseAggregationTestCase { + + /** + * Pick a random hash type + */ + public static GeoGridType randomType() { + // With more types, will use randomIntBetween() to pick one + return GeoHashType.SINGLETON; + } + + /** + * Pick a random precision for the given hash type. + */ + public static int randomPrecision(final GeoGridType type) { + if (type.getClass() == GeoHashType.class) { + return randomIntBetween(1, 12); + } + throw new IllegalArgumentException(type.getClass() + " was not added to the test"); + } + + public static int maxPrecision(GeoGridType type) { + if (type.getClass() == GeoHashType.class) { + return 12; + } + throw new IllegalArgumentException(type.getClass() + " was not added to the test"); + } + + @Override + protected GeoGridAggregationBuilder2 createTestAggregatorBuilder() { + String name = randomAlphaOfLengthBetween(3, 20); + GeoGridType type = randomType(); + GeoGridAggregationBuilder2 factory = new GeoGridAggregationBuilder2(name, type); + if (randomBoolean()) { + factory.precision(randomPrecision(factory.type())); + } + if (randomBoolean()) { + factory.size(randomIntBetween(1, Integer.MAX_VALUE)); + } + if (randomBoolean()) { + factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE)); + } + return factory; + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java index 18ab80305dd15..73aadb17c3905 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java @@ -25,6 +25,8 @@ import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoHashType; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -39,6 +41,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.dateRange; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; import static org.elasticsearch.search.aggregations.AggregationBuilders.geohashGrid; +import static org.elasticsearch.search.aggregations.AggregationBuilders.geoGrid; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.ipRange; @@ -307,4 +310,19 @@ public void testGeoHashGrid() throws Exception { } + public void testGeoGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation(geoGrid("grid", GeoHashType.SINGLETON).field("location") + .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY) + .minDocCount(0))) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } + } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregatorTests.java new file mode 100644 index 0000000000000..9b7f5e67721af --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridAggregatorTests.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.GeoGridTests; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +public class GeoGridAggregatorTests extends AggregatorTestCase { + + private static final String FIELD_NAME = "location"; + + public void testNoDocs() throws IOException { + final GeoGridType type = GeoGridTests.randomType(); + testCase(new MatchAllDocsQuery(), FIELD_NAME, type, GeoGridTests.randomPrecision(type), iw -> { + // Intentionally not writing any docs + }, geoHashGrid -> assertEquals(0, geoHashGrid.getBuckets().size())); + } + + public void testFieldMissing() throws IOException { + final GeoGridType type = GeoGridTests.randomType(); + testCase(new MatchAllDocsQuery(), "wrong_field", type, GeoGridTests.randomPrecision(type), iw -> { + iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))); + }, geoHashGrid -> assertEquals(0, geoHashGrid.getBuckets().size())); + } + + public void testHashcodeWithSeveralDocs() throws IOException { + final GeoGridType type = GeoGridTests.randomType(); + testWithSeveralDocs(type, GeoGridTests.randomPrecision(type)); + } + + private void testWithSeveralDocs(GeoGridType type, int precision) + throws IOException { + int numPoints = randomIntBetween(8, 128); + Map expectedCountPerGeoHash = new HashMap<>(); + testCase(new MatchAllDocsQuery(), FIELD_NAME, type, precision, iw -> { + List points = new ArrayList<>(); + Set distinctHashesPerDoc = new HashSet<>(); + for (int pointId = 0; pointId < numPoints; pointId++) { + double lat = (180d * randomDouble()) - 90d; + double lng = (360d * randomDouble()) - 180d; + + // Precision-adjust longitude/latitude to avoid wrong bucket placement + lng = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lng)); + lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + + points.add(new LatLonDocValuesField(FIELD_NAME, lat, lng)); + String hash = type.hashAsString(type.calculateHash(lng, lat, precision)); + if (distinctHashesPerDoc.contains(hash) == false) { + expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); + } + distinctHashesPerDoc.add(hash); + if (usually()) { + iw.addDocument(points); + points.clear(); + distinctHashesPerDoc.clear(); + } + } + if (points.size() != 0) { + iw.addDocument(points); + } + }, geoHashGrid -> { + assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); + for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { + assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); + } + }); + } + + private void testCase(Query query, String field, GeoGridType type, int precision, + CheckedConsumer buildIndex, + Consumer verify) throws IOException { + Directory directory = newDirectory(); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + buildIndex.accept(indexWriter); + indexWriter.close(); + + IndexReader indexReader = DirectoryReader.open(directory); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + + GeoGridAggregationBuilder2 aggregationBuilder = new GeoGridAggregationBuilder2("_name", type).field(field); + aggregationBuilder.precision(precision); + MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName(FIELD_NAME); + + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(query, aggregator); + aggregator.postCollection(); + verify.accept((InternalGeoGrid) aggregator.buildAggregation(0L)); + + indexReader.close(); + directory.close(); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridParserTests.java new file mode 100644 index 0000000000000..df09b207c0cbf --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/GeoGridParserTests.java @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.search.aggregations.bucket.GeoGridTests; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class GeoGridParserTests extends ESTestCase { + public void testParseValidFromInts() throws Exception { + GeoGridType type = GeoGridTests.randomType(); + int precision = GeoGridTests.randomPrecision(type); + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"hash_type\":\"" + type.getName() + "\", \"field\":\"my_loc\", \"precision\":" + precision + + ", \"size\": 500, \"shard_size\": 550}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + // can create a factory + assertNotNull(GeoGridAggregationBuilder2.parse("geo_grid", stParser)); + } + + public void testParseValidFromStrings() throws Exception { + GeoGridType type = GeoGridTests.randomType(); + int precision = GeoGridTests.randomPrecision(type); + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"hash_type\":\"" + type.getName() + "\", \"field\":\"my_loc\", \"precision\":\"" + precision + + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + // can create a factory + assertNotNull(GeoGridAggregationBuilder2.parse("geo_grid", stParser)); + } + + public void testParseDistanceUnitPrecision() throws Exception { + double distance = randomDoubleBetween(10.0, 100.00, true); + DistanceUnit unit = randomFrom(DistanceUnit.values()); + if (unit.equals(DistanceUnit.MILLIMETERS)) { + distance = 5600 + randomDouble(); // 5.6cm is approx. smallest distance represented by precision 12 + } + String distanceString = distance + unit.toString(); + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"hash_type\":\"geohash\", \"field\":\"my_loc\", \"precision\": \"" + distanceString + + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + // can create a factory + GeoGridAggregationBuilder2 builder = GeoGridAggregationBuilder2.parse("geo_grid", stParser); + assertNotNull(builder); + assertThat(builder.precision(), greaterThanOrEqualTo(0)); + assertThat(builder.precision(), lessThanOrEqualTo(12)); + } + + public void testParseInvalidUnitPrecision() throws Exception { + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"hash_type\":\"geohash\", \"field\":\"my_loc\", \"precision\": \"10kg\", " + + "\"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + XContentParseException ex = expectThrows(XContentParseException.class, + () -> GeoGridAggregationBuilder2.parse("geo_grid", stParser)); + assertThat(ex.getMessage(), containsString("[geo_grid] failed to parse field [precision]")); + + Throwable cause = ex.getCause(); + assertThat(cause, instanceOf(NumberFormatException.class)); + assertThat(cause.getMessage(), containsString("For input string: \"10kg\"")); + } + + public void testParseDistanceUnitPrecisionTooSmall() throws Exception { + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"hash_type\":\"geohash\", \"field\":\"my_loc\", " + + "\"precision\": \"1cm\", \"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + XContentParseException ex = expectThrows(XContentParseException.class, + () -> GeoGridAggregationBuilder2.parse("geo_grid", stParser)); + assertThat(ex.getMessage(), containsString("[geo_grid] failed to parse field [precision]")); + + Throwable cause = ex.getCause(); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertEquals("precision too high [1cm]", cause.getMessage()); + } + + public void testParseErrorOnBooleanPrecision() throws Exception { + GeoGridType type = GeoGridTests.randomType(); + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"hash_type\":\"" + type.getName() + "\", \"field\":\"my_loc\", \"precision\":false}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + XContentParseException ex = expectThrows(XContentParseException.class, + () -> GeoGridAggregationBuilder2.parse("geo_grid", stParser)); + assertThat(ex.getMessage(), containsString("[geo_grid] failed to parse field [precision]")); + + Throwable cause = ex.getCause(); + assertThat(cause, instanceOf(XContentParseException.class)); + assertThat(cause.getMessage(), containsString("[geo_grid] failed to parse field [precision]" + + " in [geo_grid]. It must be either an integer or a string")); + } + + public void testParseErrorOnPrecisionOutOfRange() throws Exception { + final GeoGridType type = GeoGridTests.randomType(); + final int precision = GeoGridTests.maxPrecision(type) + 1; + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"hash_type\":\"" + type.getName() + "\", \"field\":\"my_loc\", \"precision\":\""+ precision +"\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + try { + GeoGridAggregationBuilder2.parse("geo_grid", stParser); + fail(); + } catch (XContentParseException ex) { + assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); + String expectedMsg; + if (type.getClass() == GeoHashType.class) { + expectedMsg = "Invalid geohash aggregation precision of 13. Must be between 1 and 12."; + } else { + throw new IllegalArgumentException(type.getClass() + " was not added to the test"); + } + assertEquals(expectedMsg, ex.getCause().getMessage()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/InternalGeoGridGeoHashTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/InternalGeoGridGeoHashTests.java new file mode 100644 index 0000000000000..6324d3133664d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid2/InternalGeoGridGeoHashTests.java @@ -0,0 +1,142 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid2; + +import org.apache.lucene.index.IndexWriter; +import org.elasticsearch.common.geo.GeoHashUtils; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.test.InternalMultiBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class InternalGeoGridGeoHashTests extends InternalMultiBucketAggregationTestCase { + + @Override + protected int minNumberOfBuckets() { + return 1; + } + + @Override + protected int maxNumberOfBuckets() { + return 3; + } + + @Override + protected InternalGeoGrid createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations) { + int size = randomNumberOfBuckets(); + List buckets = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + double latitude = randomDoubleBetween(-90.0, 90.0, false); + double longitude = randomDoubleBetween(-180.0, 180.0, false); + + long geoHashAsLong = GeoHashUtils.longEncode(longitude, latitude, 4); + buckets.add(new GeoHashBucket(geoHashAsLong, randomInt(IndexWriter.MAX_DOCS), aggregations)); + } + return new InternalGeoGrid(name, GeoHashType.SINGLETON, size, buckets, pipelineAggregators, metaData); + } + + @Override + protected Writeable.Reader instanceReader() { + return InternalGeoGrid::new; + } + + @Override + protected void assertReduced(InternalGeoGrid reduced, List inputs) { + Map> map = new HashMap<>(); + for (InternalGeoGrid input : inputs) { + for (GeoGrid.Bucket bucket : input.getBuckets()) { + GeoHashBucket internalBucket = (GeoHashBucket) bucket; + List buckets = map.computeIfAbsent(internalBucket.hashAsLong, k -> new ArrayList<>()); + buckets.add(internalBucket); + } + } + List expectedBuckets = new ArrayList<>(); + for (Map.Entry> entry : map.entrySet()) { + long docCount = 0; + for (GeoHashBucket bucket : entry.getValue()) { + docCount += bucket.docCount; + } + expectedBuckets.add(new GeoHashBucket(entry.getKey(), docCount, InternalAggregations.EMPTY)); + } + expectedBuckets.sort((first, second) -> { + int cmp = Long.compare(second.docCount, first.docCount); + if (cmp == 0) { + return second.compareTo(first); + } + return cmp; + }); + int requestedSize = inputs.get(0).getRequiredSize(); + expectedBuckets = expectedBuckets.subList(0, Math.min(requestedSize, expectedBuckets.size())); + assertEquals(expectedBuckets.size(), reduced.getBuckets().size()); + for (int i = 0; i < reduced.getBuckets().size(); i++) { + GeoGrid.Bucket expected = expectedBuckets.get(i); + GeoGrid.Bucket actual = reduced.getBuckets().get(i); + assertEquals(expected.getDocCount(), actual.getDocCount()); + assertEquals(expected.getKey(), actual.getKey()); + } + } + + @Override + protected Class implementationClass() { + return ParsedGeoGrid.class; + } + + @Override + protected InternalGeoGrid mutateInstance(InternalGeoGrid instance) { + String name = instance.getName(); + int size = instance.getRequiredSize(); + List buckets = instance.getBuckets(); + List pipelineAggregators = instance.pipelineAggregators(); + Map metaData = instance.getMetaData(); + switch (between(0, 3)) { + case 0: + name += randomAlphaOfLength(5); + break; + case 1: + buckets = new ArrayList<>(buckets); + buckets.add( + new GeoHashBucket(randomNonNegativeLong(), randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); + break; + case 2: + size = size + between(1, 10); + break; + case 3: + if (metaData == null) { + metaData = new HashMap<>(1); + } else { + metaData = new HashMap<>(instance.getMetaData()); + } + metaData.put(randomAlphaOfLength(15), randomInt()); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + return new InternalGeoGrid(name, GeoHashType.SINGLETON, size, buckets, pipelineAggregators, metaData); + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java index dfc503219ed74..aa117bd1f1279 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoHashType; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.test.ESIntegTestCase; @@ -31,6 +33,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.geoCentroid; import static org.elasticsearch.search.aggregations.AggregationBuilders.geohashGrid; +import static org.elasticsearch.search.aggregations.AggregationBuilders.geoGrid; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; @@ -173,4 +176,26 @@ public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE)); } } + + public void testSingleValueFieldAsSubAggToGeoGrid() throws Exception { + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addAggregation(geoGrid("geoGrid", GeoHashType.SINGLETON).field(SINGLE_VALUED_FIELD_NAME) + .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME))) + .get(); + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("geoGrid"); + assertThat(grid, notNullValue()); + assertThat(grid.getName(), equalTo("geoGrid")); + List buckets = grid.getBuckets(); + for (GeoGrid.Bucket cell : buckets) { + String geohash = cell.getKeyAsString(); + GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); + GeoCentroid centroidAgg = cell.getAggregations().get(aggName); + assertThat("Geohash " + geohash + " has wrong centroid latitude ", expectedCentroid.lat(), + closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE)); + assertThat("Geohash " + geohash + " has wrong centroid longitude", expectedCentroid.lon(), + closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE)); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 551110ca2520a..f9f197b7d5c3f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -51,6 +51,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid2.GeoGridAggregationBuilder2; +import org.elasticsearch.search.aggregations.bucket.geogrid2.ParsedGeoGrid; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -212,6 +214,7 @@ public abstract class InternalAggregationTestCase map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); map.put(GeoGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); + map.put(GeoGridAggregationBuilder2.NAME, (p, c) -> ParsedGeoGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));