diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index c605b8d093644..cd08cbc8e01c7 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -21,7 +21,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; -import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; @@ -54,6 +53,8 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; + /** * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. *

@@ -96,6 +97,7 @@ public static class Defaults { public static final Orientation ORIENTATION = Orientation.RIGHT; public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d; public static final Explicit COERCE = new Explicit<>(false, false); + public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType(); @@ -115,6 +117,7 @@ public static class Defaults { public static class Builder extends FieldMapper.Builder { private Boolean coerce; + private Boolean ignoreMalformed; public Builder(String name) { super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); @@ -145,6 +148,21 @@ protected Explicit coerce(BuilderContext context) { return Defaults.COERCE; } + public Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return builder; + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return Defaults.IGNORE_MALFORMED; + } + @Override public GeoShapeFieldMapper build(BuilderContext context) { GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType; @@ -154,8 +172,8 @@ public GeoShapeFieldMapper build(BuilderContext context) { } setupFieldType(context); - return new GeoShapeFieldMapper(name, fieldType, coerce(context), context.indexSettings(), multiFieldsBuilder.build(this, - context), copyTo); + return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), context.indexSettings(), + multiFieldsBuilder.build(this, context), copyTo); } } @@ -186,6 +204,9 @@ public Mapper.Builder parse(String name, Map node, ParserContext } else if (Names.STRATEGY.equals(fieldName)) { builder.fieldType().setStrategyName(fieldNode.toString()); iterator.remove(); + } else if (IGNORE_MALFORMED.equals(fieldName)) { + builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed", fieldNode, parserContext)); + iterator.remove(); } else if (Names.COERCE.equals(fieldName)) { builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext)); iterator.remove(); @@ -428,11 +449,13 @@ public Query termQuery(Object value, QueryShardContext context) { } protected Explicit coerce; + protected Explicit ignoreMalformed; - public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit coerce, Settings indexSettings, - MultiFields multiFields, CopyTo copyTo) { + public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit ignoreMalformed, + Explicit coerce, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo); this.coerce = coerce; + this.ignoreMalformed = ignoreMalformed; } @Override @@ -461,7 +484,9 @@ public Mapper parse(ParseContext context) throws IOException { context.doc().add(field); } } catch (Exception e) { - throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e); + if (ignoreMalformed.value() == false) { + throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e); + } } return null; } @@ -478,6 +503,9 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) { if (gsfm.coerce.explicit()) { this.coerce = gsfm.coerce; } + if (gsfm.ignoreMalformed.explicit()) { + this.ignoreMalformed = gsfm.ignoreMalformed; + } } @Override @@ -506,7 +534,10 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); } if (includeDefaults || coerce.explicit()) { - builder.field("coerce", coerce.value()); + builder.field(Names.COERCE, coerce.value()); + } + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field(IGNORE_MALFORMED, ignoreMalformed.value()); } } @@ -514,6 +545,10 @@ public Explicit coerce() { return coerce; } + public Explicit ignoreMalformed() { + return ignoreMalformed; + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 5972a8ecee8c9..e43cfbe1fd1c1 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.elasticsearch.common.Explicit; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; @@ -103,7 +104,7 @@ public void testOrientationParsing() throws IOException { } /** - * Test that orientation parameter correctly parses + * Test that coerce parameter correctly parses */ public void testCoerceParsing() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -136,6 +137,41 @@ public void testCoerceParsing() throws IOException { assertThat(coerce, equalTo(false)); } + /** + * Test that ignore_malformed parameter correctly parses + */ + public void testIgnoreMalformedParsing() throws IOException { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("ignore_malformed", "true") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping)); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + Explicit ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed(); + assertThat(ignoreMalformed.value(), equalTo(true)); + + // explicit false ignore_malformed test + mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("ignore_malformed", "false") + .endObject().endObject() + .endObject().endObject().string(); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed(); + assertThat(ignoreMalformed.explicit(), equalTo(true)); + assertThat(ignoreMalformed.value(), equalTo(false)); + } + public void testGeohashConfiguration() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java index 13889cec7e15b..d56a98c2ea9ef 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.geo; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.common.geo.builders.ShapeBuilder; @@ -29,6 +30,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -88,6 +90,36 @@ public void testOrientationPersistence() throws Exception { assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); } + /** + * Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document + */ + public void testIgnoreMalformed() throws Exception { + // create index + assertAcked(client().admin().indices().prepareCreate("test") + .addMapping("geometry", "shape", "type=geo_shape,ignore_malformed=true").get()); + ensureGreen(); + + // test self crossing ccw poly not crossing dateline + String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon") + .startArray("coordinates") + .startArray() + .startArray().value(176.0).value(15.0).endArray() + .startArray().value(-177.0).value(10.0).endArray() + .startArray().value(-177.0).value(-10.0).endArray() + .startArray().value(176.0).value(-15.0).endArray() + .startArray().value(-177.0).value(15.0).endArray() + .startArray().value(172.0).value(0.0).endArray() + .startArray().value(176.0).value(15.0).endArray() + .endArray() + .endArray() + .endObject().string(); + + indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape", + polygonGeoJson)); + SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get(); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + } + private String findNodeName(String index) { ClusterState state = client().admin().cluster().prepareState().get().getState(); IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0); diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 18ffdbcbc6363..b3420dbb58a98 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -86,6 +86,10 @@ by improving point performance on a `geo_shape` field so that `geo_shape` querie optimal on a point only field. | `false` +|`ignore_malformed` |If true, malformed geojson shapes are ignored. If false (default), +malformed geojson shapes throw an exception and reject the whole document. +| `false` + |=======================================================================