Skip to content

Add ignore_malformed to geo_shape fields #24654

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nknize merged 1 commit into from
Nov 10, 2017
Merged

Add ignore_malformed to geo_shape fields #24654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 42 additions & 7 deletions core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
Expand Down Expand Up @@ -54,6 +53,8 @@
import java.util.Map;
import java.util.Objects;

import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED;

/**
* FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s.
* <p>
Expand Down Expand Up @@ -96,6 +97,7 @@ public static class Defaults {
public static final Orientation ORIENTATION = Orientation.RIGHT;
public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d;
public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);

public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType();

Expand All @@ -115,6 +117,7 @@ public static class Defaults {
public static class Builder extends FieldMapper.Builder<Builder, GeoShapeFieldMapper> {

private Boolean coerce;
private Boolean ignoreMalformed;

public Builder(String name) {
super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
Expand Down Expand Up @@ -145,6 +148,21 @@ protected Explicit<Boolean> coerce(BuilderContext context) {
return Defaults.COERCE;
}

public Builder ignoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed;
return builder;
}

protected Explicit<Boolean> ignoreMalformed(BuilderContext context) {
if (ignoreMalformed != null) {
return new Explicit<>(ignoreMalformed, true);
}
if (context.indexSettings() != null) {
return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false);
}
return Defaults.IGNORE_MALFORMED;
}

@Override
public GeoShapeFieldMapper build(BuilderContext context) {
GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType;
Expand All @@ -154,8 +172,8 @@ public GeoShapeFieldMapper build(BuilderContext context) {
}
setupFieldType(context);

return new GeoShapeFieldMapper(name, fieldType, coerce(context), context.indexSettings(), multiFieldsBuilder.build(this,
context), copyTo);
return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}
}

Expand Down Expand Up @@ -186,6 +204,9 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
} else if (Names.STRATEGY.equals(fieldName)) {
builder.fieldType().setStrategyName(fieldNode.toString());
iterator.remove();
} else if (IGNORE_MALFORMED.equals(fieldName)) {
builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed", fieldNode, parserContext));
iterator.remove();
} else if (Names.COERCE.equals(fieldName)) {
builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext));
iterator.remove();
Expand Down Expand Up @@ -428,11 +449,13 @@ public Query termQuery(Object value, QueryShardContext context) {
}

protected Explicit<Boolean> coerce;
protected Explicit<Boolean> ignoreMalformed;

public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> coerce, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> ignoreMalformed,
Explicit<Boolean> coerce, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
}

@Override
Expand Down Expand Up @@ -461,7 +484,9 @@ public Mapper parse(ParseContext context) throws IOException {
context.doc().add(field);
}
} catch (Exception e) {
throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
if (ignoreMalformed.value() == false) {
throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
}
}
return null;
}
Expand All @@ -478,6 +503,9 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
if (gsfm.coerce.explicit()) {
this.coerce = gsfm.coerce;
}
if (gsfm.ignoreMalformed.explicit()) {
this.ignoreMalformed = gsfm.ignoreMalformed;
}
}

@Override
Expand Down Expand Up @@ -506,14 +534,21 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly());
}
if (includeDefaults || coerce.explicit()) {
builder.field("coerce", coerce.value());
builder.field(Names.COERCE, coerce.value());
}
if (includeDefaults || ignoreMalformed.explicit()) {
builder.field(IGNORE_MALFORMED, ignoreMalformed.value());
}
}

public Explicit<Boolean> coerce() {
return coerce;
}

public Explicit<Boolean> ignoreMalformed() {
return ignoreMalformed;
}

@Override
protected String contentType() {
return CONTENT_TYPE;
Expand Down
38 changes: 37 additions & 1 deletion core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
Expand Down Expand Up @@ -103,7 +104,7 @@ public void testOrientationParsing() throws IOException {
}

/**
* Test that orientation parameter correctly parses
* Test that coerce parameter correctly parses
*/
public void testCoerceParsing() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
Expand Down Expand Up @@ -136,6 +137,41 @@ public void testCoerceParsing() throws IOException {
assertThat(coerce, equalTo(false));
}

/**
* Test that ignore_malformed parameter correctly parses
*/
public void testIgnoreMalformedParsing() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("ignore_malformed", "true")
.endObject().endObject()
.endObject().endObject().string();

DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));

Explicit<Boolean> ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
assertThat(ignoreMalformed.value(), equalTo(true));

// explicit false ignore_malformed test
mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("ignore_malformed", "false")
.endObject().endObject()
.endObject().endObject().string();

defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));

ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
assertThat(ignoreMalformed.explicit(), equalTo(true));
assertThat(ignoreMalformed.value(), equalTo(false));
}

public void testGeohashConfiguration() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
Expand Down
32 changes: 32 additions & 0 deletions core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.elasticsearch.search.geo;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
Expand All @@ -29,6 +30,7 @@
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.test.ESIntegTestCase;

import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
Expand Down Expand Up @@ -88,6 +90,36 @@ public void testOrientationPersistence() throws Exception {
assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW));
}

/**
* Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document
*/
public void testIgnoreMalformed() throws Exception {
// create index
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("geometry", "shape", "type=geo_shape,ignore_malformed=true").get());
ensureGreen();

// test self crossing ccw poly not crossing dateline
String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
.startArray("coordinates")
.startArray()
.startArray().value(176.0).value(15.0).endArray()
.startArray().value(-177.0).value(10.0).endArray()
.startArray().value(-177.0).value(-10.0).endArray()
.startArray().value(176.0).value(-15.0).endArray()
.startArray().value(-177.0).value(15.0).endArray()
.startArray().value(172.0).value(0.0).endArray()
.startArray().value(176.0).value(15.0).endArray()
.endArray()
.endArray()
.endObject().string();

indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape",
polygonGeoJson));
SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
}

private String findNodeName(String index) {
ClusterState state = client().admin().cluster().prepareState().get().getState();
IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0);
Expand Down
4 changes: 4 additions & 0 deletions docs/reference/mapping/types/geo-shape.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ by improving point performance on a `geo_shape` field so that `geo_shape` querie
optimal on a point only field.
| `false`

|`ignore_malformed` |If true, malformed geojson shapes are ignored. If false (default),
malformed geojson shapes throw an exception and reject the whole document.
| `false`


|=======================================================================

Expand Down