diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4874eafc971..e5520c7529cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Expand fetch phase profiling to support inner hits and top hits aggregation phases ([##18936](https://github.com/opensearch-project/OpenSearch/pull/18936)) - Add temporal routing processors for time-based document routing ([#18920](https://github.com/opensearch-project/OpenSearch/issues/18920)) -- The dynamic mapping parameter supports false_allow_templates ([#18825](https://github.com/opensearch-project/OpenSearch/pull/18825)) ### Changed diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml deleted file mode 100644 index adad2e321b68c..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml +++ /dev/null @@ -1,165 +0,0 @@ ---- -"Index documents with setting dynamic parameter to false_allow_templates in the mapping of the index": - - skip: - version: " - 3.2.99" - reason: "introduced in 3.3.0" - - - do: - indices.create: - index: test_1 - body: - mappings: - dynamic: false_allow_templates - dynamic_templates: [ - { - dates: { - "match": "date_*", - "match_mapping_type": "date", - "mapping": { - "type": "date" - } - } - }, - { - strings: { - "match": "stringField*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - }, - { - object: { - "match": "objectField*", - "match_mapping_type": "object", - "mapping": { - "type": "object", - "properties": { - "bar1": { - "type": "keyword" - }, - "bar2": { - "type": "text" - } - } - } - } - }, - { - boolean: { - "match": "booleanField*", - "match_mapping_type": "boolean", - "mapping": { - "type": "boolean" - } - } - }, - { - long: { - "match": "longField*", - "match_mapping_type": "long", - "mapping": { - "type": "long" - } - } - }, - { - double: { - "match": "doubleField*", - "match_mapping_type": "double", - "mapping": { - "type": "double" - } - } - }, - { - float: { - "match": "floatField*", - "match_mapping_type": "float", - "mapping": { - "type": "float" - } - } - }, - { - integer: { - "match": "integerField*", - "match_mapping_type": "integer", - "mapping": { - "type": "integer" - } - } - }, - { - array: { - "match": "arrayField*", - "mapping": { - "type": "keyword" - } - } - } - ] - properties: - url: - type: keyword - - - do: - index: - index: test_1 - id: 1 - body: { - url: "https://example.com", - date_timestamp: "2024-06-25T05:11:51.243Z", - stringField: "bar", - objectField: { - bar1: "bar1", - bar2: "bar2" - }, - booleanField: true, - longField: 123456789, - doubleField: 123.456, - floatField: 123.45, - integerField: 12345, - arrayField: ["item1", "item2", "item3"], - author: "John Doe" - } - - - do: - get: - index: test_1 - id: 1 - - match: - _source: - url: "https://example.com" - date_timestamp: "2024-06-25T05:11:51.243Z" - stringField: "bar" - objectField: - bar1: "bar1" - bar2: "bar2" - booleanField: true - longField: 123456789 - doubleField: 123.456 - floatField: 123.45 - integerField: 12345 - arrayField: ["item1", "item2", "item3"] - author: "John Doe" - - - do: - indices.get_mapping: - index: test_1 - - - match: {test_1.mappings.dynamic: false_allow_templates} - - match: {test_1.mappings.properties.url.type: keyword} - - match: {test_1.mappings.properties.date_timestamp.type: date} - - match: {test_1.mappings.properties.stringField.type: keyword} - - match: {test_1.mappings.properties.objectField.properties.bar1.type: keyword} - - match: {test_1.mappings.properties.objectField.properties.bar2.type: text} - - match: {test_1.mappings.properties.booleanField.type: boolean} - - match: {test_1.mappings.properties.longField.type: long} - - match: {test_1.mappings.properties.doubleField.type: double} - - match: {test_1.mappings.properties.floatField.type: float} - - match: {test_1.mappings.properties.integerField.type: integer} - - match: {test_1.mappings.properties.arrayField.type: keyword} - - match: {test_1.mappings.properties.author: null} - - length: {test_1.mappings.properties: 11} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml index 78b474f8f40b9..89b47fde2a72c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml @@ -190,34 +190,3 @@ setup: - match: {test_index1.mappings.dynamic: strict_allow_templates} - match: {test_index1.mappings.properties.test1.type: text} - ---- -"post a mapping with setting dynamic to false_allow_templates": - - skip: - version: " - 3.2.99" - reason: "introduced in 3.3.0" - - do: - indices.put_mapping: - index: test_index1 - body: - dynamic: false_allow_templates - dynamic_templates: [ - { - strings: { - "match": "foo*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - } - ] - properties: - test1: - type: text - - - do: - indices.get_mapping: {} - - - match: {test_index1.mappings.dynamic: false_allow_templates} - - match: {test_index1.mappings.properties.test1.type: text} diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index e8d8a63d9d0e0..134baa70f80c2 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -554,7 +554,6 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName); case TRUE: case STRICT_ALLOW_TEMPLATES: - case FALSE_ALLOW_TEMPLATES: Mapper.Builder builder = findTemplateBuilder( context, currentFieldName, @@ -564,10 +563,6 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, ); if (builder == null) { - if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { - context.parser().skipChildren(); - break; - } builder = new ObjectMapper.Builder(currentFieldName).enabled(true); } Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); @@ -619,7 +614,6 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, ); case TRUE: case STRICT_ALLOW_TEMPLATES: - case FALSE_ALLOW_TEMPLATES: Mapper.Builder builder = findTemplateBuilder( context, arrayFieldName, @@ -628,10 +622,6 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, parentMapper.fullPath() ); if (builder == null) { - if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { - context.parser().skipChildren(); - break; - } parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); } else { Mapper.BuilderContext builderContext = new Mapper.BuilderContext( @@ -796,13 +786,13 @@ private static Mapper.Builder createBuilderFromDynamicValue( if (parseableAsLong && context.root().numericDetection()) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { - return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings())); + builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); } return builder; } else if (parseableAsDouble && context.root().numericDetection()) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); if (builder == null) { - return handleNoTemplateFound(dynamic, () -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings())); + builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings()); } return builder; } else if (parseableAsLong == false && parseableAsDouble == false && context.root().dateDetection()) { @@ -818,16 +808,14 @@ private static Mapper.Builder createBuilderFromDynamicValue( } Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath); if (builder == null) { - return handleNoTemplateFound(dynamic, () -> { - boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); - return new DateFieldMapper.Builder( - currentFieldName, - DateFieldMapper.Resolution.MILLISECONDS, - dateTimeFormatter, - ignoreMalformed, - IndexMetadata.indexCreated(context.indexSettings().getSettings()) - ); - }); + boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); + builder = new DateFieldMapper.Builder( + currentFieldName, + DateFieldMapper.Resolution.MILLISECONDS, + dateTimeFormatter, + ignoreMalformed, + IndexMetadata.indexCreated(context.indexSettings().getSettings()) + ); } return builder; @@ -836,11 +824,8 @@ private static Mapper.Builder createBuilderFromDynamicValue( Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); if (builder == null) { - return handleNoTemplateFound( - dynamic, - () -> new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField( - new KeywordFieldMapper.Builder("keyword").ignoreAbove(256) - ) + builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField( + new KeywordFieldMapper.Builder("keyword").ignoreAbove(256) ); } return builder; @@ -851,7 +836,7 @@ private static Mapper.Builder createBuilderFromDynamicValue( || numberType == XContentParser.NumberType.BIG_INTEGER) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { - return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings())); + builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); } return builder; } else if (numberType == XContentParser.NumberType.FLOAT @@ -862,23 +847,20 @@ private static Mapper.Builder createBuilderFromDynamicValue( // no templates are defined, we use float by default instead of double // since this is much more space-efficient and should be enough most of // the time - return handleNoTemplateFound( - dynamic, - () -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings()) - ); + builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings()); } return builder; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath); if (builder == null) { - return handleNoTemplateFound(dynamic, () -> new BooleanFieldMapper.Builder(currentFieldName)); + builder = new BooleanFieldMapper.Builder(currentFieldName); } return builder; } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath); if (builder == null) { - return handleNoTemplateFound(dynamic, () -> new BinaryFieldMapper.Builder(currentFieldName)); + builder = new BinaryFieldMapper.Builder(currentFieldName); } return builder; } else { @@ -886,7 +868,6 @@ private static Mapper.Builder createBuilderFromDynamicValue( if (builder != null) { return builder; } - return handleNoTemplateFound(dynamic, () -> null); } // TODO how do we identify dynamically that its a binary value? throw new IllegalStateException( @@ -894,16 +875,6 @@ private static Mapper.Builder createBuilderFromDynamicValue( ); } - private static Mapper.Builder handleNoTemplateFound( - ObjectMapper.Dynamic dynamic, - java.util.function.Supplier> builderSupplier - ) { - if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { - return null; - } - return builderSupplier.get(); - } - private static void parseDynamicValue( final ParseContext context, ObjectMapper parentMapper, @@ -917,16 +888,8 @@ private static void parseDynamicValue( if (dynamic == ObjectMapper.Dynamic.FALSE) { return; } - final Mapper.Builder builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath()); - if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES && builder == null) { - // For FALSE_ALLOW_TEMPLATES, if no template matches, we still need to consume the token - // to maintain proper JSON parsing state - if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) { - context.parser().skipChildren(); - } - return; - } final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); + final Mapper.Builder builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath()); Mapper mapper = builder.build(builderContext); context.addDynamicMapper(mapper); @@ -1015,9 +978,8 @@ private static Tuple getDynamicParentMapper( switch (dynamic) { case STRICT: throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]); - case TRUE: case STRICT_ALLOW_TEMPLATES: - case FALSE_ALLOW_TEMPLATES: + case TRUE: Mapper.Builder builder = findTemplateBuilder( context, paths[i], @@ -1026,9 +988,6 @@ private static Tuple getDynamicParentMapper( parent.fullPath() ); if (builder == null) { - if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { - return new Tuple<>(pathsAdded, parent); - } builder = new ObjectMapper.Builder(paths[i]).enabled(true); } Mapper.BuilderContext builderContext = new Mapper.BuilderContext( diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 42373d5686543..dd66fb08cecba 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -93,8 +93,7 @@ public enum Dynamic { TRUE, FALSE, STRICT, - STRICT_ALLOW_TEMPLATES, - FALSE_ALLOW_TEMPLATES + STRICT_ALLOW_TEMPLATES } /** @@ -314,8 +313,6 @@ protected static boolean parseObjectOrDocumentTypeProperties( builder.dynamic(Dynamic.STRICT); } else if (value.equalsIgnoreCase("strict_allow_templates")) { builder.dynamic(Dynamic.STRICT_ALLOW_TEMPLATES); - } else if (value.equalsIgnoreCase("false_allow_templates")) { - builder.dynamic(Dynamic.FALSE_ALLOW_TEMPLATES); } else { boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic"); builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE); diff --git a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java index e8ce3341c062c..7a8c4ffe35021 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java @@ -49,7 +49,6 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class CopyToMapperTests extends MapperServiceTestCase { @@ -288,40 +287,6 @@ public void testCopyToStrictAllowTemplatesDynamicInnerObjectParsing() throws Exc ); } - public void testCopyToFalseAllowTemplatesDynamicInnerObjectParsing() throws Exception { - DocumentMapper docMapper = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "test"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - b.startObject("properties"); - { - b.startObject("copy_test"); - { - b.field("type", "text"); - b.field("copy_to", "very.inner.field"); - } - b.endObject(); - } - b.endObject(); - })); - - ParsedDocument doc = docMapper.parse(source(b -> b.field("copy_test", "foo"))); - assertThat(doc.rootDoc().get("copy_test"), equalTo("foo")); - assertThat(doc.rootDoc().get("very.inner.field"), nullValue()); - } - public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception { DocumentMapper docMapper = createDocumentMapper(mapping(b -> { diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index e9be8b671d7ba..c9763d634979b 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -2002,584 +2002,4 @@ public void testDocumentDeepNestedObjectAndArrayCombination() throws Exception { } - public void testDynamicFalseAllowTemplatesLongArray() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())); - assertEquals(0, doc.rootDoc().getFields("foo").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "foo"); - b.startObject("mapping").field("type", "long").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())); - assertEquals(4, docWithTemplate.rootDoc().getFields("foo").length); - } - - public void testDynamicFalseAllowTemplatesObject() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "test"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("test1"); - { - b.field("match", "test1"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - } - - )); - - ParsedDocument doc = mapper.parse(source(b -> b.startObject("test").field("test1", "foo").endObject())); - assertEquals(2, doc.rootDoc().getFields("test.test1").length); - } - - public void testDynamicFalseAllowTemplatesValue() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.field("bar", "baz"))); - assertEquals(0, doc.rootDoc().getFields("bar").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "bar"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bar", "baz"))); - assertEquals(2, docWithTemplate.rootDoc().getFields("bar").length); - } - - public void testDynamicFalseAllowTemplatesNull() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.nullField("bar"))); - assertEquals(0, doc.rootDoc().getFields("bar").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "bar"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.nullField("bar"))); - assertEquals(0, docWithTemplate.rootDoc().getFields("bar").length); // null fields don't create mappings - } - - public void testDynamicFalseAllowTemplatesDottedFieldNameLongArray() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray())); - assertEquals(0, doc.rootDoc().getFields("foo.bar.baz").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "foo"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("test1"); - { - b.field("match", "bar"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("test2"); - { - b.field("path_match", "foo.bar.baz"); - b.startObject("mapping").field("type", "long").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray())); - assertEquals(4, docWithTemplate.rootDoc().getFields("foo.bar.baz").length); - } - - public void testDynamicFalseAllowTemplatesDottedFieldNameLong() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.field("foo.bar.baz", 0))); - assertEquals(0, doc.rootDoc().getFields("foo.bar.baz").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "foo"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("test1"); - { - b.field("match", "bar"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("test2"); - { - b.field("path_match", "foo.bar.baz"); - b.startObject("mapping").field("type", "long").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("foo.bar.baz", 0))); - assertEquals(2, docWithTemplate.rootDoc().getFields("foo.bar.baz").length); - } - - public void testDynamicFalseAllowTemplatesDottedFieldNameObject() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())); - assertEquals(0, doc.rootDoc().getFields("foo.bar.baz.a").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("test"); - { - b.field("match", "foo"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("test1"); - { - b.field("match", "bar"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("test2"); - { - b.field("path_match", "foo.bar.baz"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())); - assertEquals(2, docWithTemplate.rootDoc().getFields("foo.bar.baz.a").length); - } - - public void testDynamicFalseAllowTemplatesWithEmbeddedObject() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.field("embedded_field", "value"))); - assertEquals(0, doc.rootDoc().getFields("embedded_field").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("embedded_template"); - { - b.field("match", "embedded_*"); - b.field("match_mapping_type", "string"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("embedded_field", "value"))); - assertEquals(2, docWithTemplate.rootDoc().getFields("embedded_field").length); - } - - public void testDynamicFalseAllowTemplatesWithDateDetection() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_date_formats").value("yyyy-MM-dd").endArray(); - })); - ParsedDocument doc = mapper.parse(source(b -> b.field("date_field", "2023-12-25"))); - assertEquals(0, doc.rootDoc().getFields("date_field").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_date_formats").value("yyyy-MM-dd").endArray(); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("date_template"); - { - b.field("match", "date_*"); - b.field("match_mapping_type", "date"); - b.startObject("mapping").field("type", "date").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("date_field", "2023-12-25"))); - assertEquals(2, docWithTemplate.rootDoc().getFields("date_field").length); - } - - public void testDynamicFalseAllowTemplatesWithNumericDetection() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.field("numeric_field", "123"))); - assertEquals(0, doc.rootDoc().getFields("numeric_field").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("numeric_template"); - { - b.field("match", "numeric_*"); - b.field("match_mapping_type", "long"); - b.startObject("mapping").field("type", "long").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("numeric_field", "123"))); - assertEquals(0, docWithTemplate.rootDoc().getFields("numeric_field").length); - } - - public void testDynamicFalseAllowTemplatesWithFloatDetection() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.field("float_field", "123.45"))); - assertEquals(0, doc.rootDoc().getFields("float_field").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("float_template"); - { - b.field("match", "float_*"); - b.field("match_mapping_type", "double"); - b.startObject("mapping").field("type", "float").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("float_field", "123.45"))); - assertEquals(0, docWithTemplate.rootDoc().getFields("float_field").length); - } - - public void testDynamicFalseAllowTemplatesWithBooleanDetection() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - ParsedDocument doc = mapper.parse(source(b -> b.field("bool_field", true))); - assertEquals(0, doc.rootDoc().getFields("bool_field").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("boolean_template"); - { - b.field("match", "bool_*"); - b.field("match_mapping_type", "boolean"); - b.startObject("mapping").field("type", "boolean").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bool_field", true))); - assertEquals(2, docWithTemplate.rootDoc().getFields("bool_field").length); - } - - public void testDynamicFalseAllowTemplatesWithBigInteger() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - BigInteger bigInt = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE); - ParsedDocument doc = mapper.parse(source(b -> b.field("bigint_field", bigInt))); - assertEquals(0, doc.rootDoc().getFields("bigint_field").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("bigint_template"); - { - b.field("match", "bigint_*"); - b.field("match_mapping_type", "long"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bigint_field", bigInt))); - assertEquals(2, docWithTemplate.rootDoc().getFields("bigint_field").length); - } - - public void testDynamicFalseAllowTemplatesWithBigDecimal() throws Exception { - DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); - BigDecimal bigDecimal = BigDecimal.valueOf(Double.MAX_VALUE).add(BigDecimal.valueOf(10.1)); - ParsedDocument doc = mapper.parse(source(b -> b.field("bigdecimal_field", bigDecimal))); - assertEquals(0, doc.rootDoc().getFields("bigdecimal_field").length); - - DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("bigdecimal_template"); - { - b.field("match", "bigdecimal_*"); - b.field("match_mapping_type", "double"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bigdecimal_field", bigDecimal))); - assertEquals(2, docWithTemplate.rootDoc().getFields("bigdecimal_field").length); - } - - public void testDynamicFalseAllowTemplatesWithCopyTo() throws Exception { - DocumentMapper mapperWithDynamic = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("target_template"); - { - b.field("match", "target_*"); - b.field("match_mapping_type", "string"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - - MapperService mapperService = createMapperService(mapping(b -> { - b.startObject("source_field"); - { - b.field("type", "text"); - b.field("copy_to", "target_field"); - } - b.endObject(); - })); - - ParsedDocument parsedDoc = mapperWithDynamic.parse(source(b -> b.field("source_field", "test value"))); - if (parsedDoc.dynamicMappingsUpdate() != null) { - merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); - } - - ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("source_field", "test value"))); - assertEquals(1, doc.rootDoc().getFields("source_field").length); - assertEquals(1, doc.rootDoc().getFields("target_field").length); - } - - public void testDynamicFalseAllowTemplatesWithNestedCopyTo() throws Exception { - DocumentMapper mapperWithDynamic = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("nested_template"); - { - b.field("match", "nested"); - b.field("match_mapping_type", "object"); - b.startObject("mapping").field("type", "object").endObject(); - } - b.endObject(); - } - b.endObject(); - } - { - b.startObject(); - { - b.startObject("target_template"); - { - b.field("match", "target_*"); - b.field("match_mapping_type", "string"); - b.startObject("mapping").field("type", "keyword").endObject(); - } - b.endObject(); - } - b.endObject(); - } - b.endArray(); - })); - - MapperService mapperService = createMapperService(mapping(b -> { - b.startObject("nested"); - { - b.field("type", "object"); - b.startObject("properties"); - { - b.startObject("source_field"); - { - b.field("type", "text"); - b.field("copy_to", "nested.target_field"); - } - b.endObject(); - } - b.endObject(); - } - b.endObject(); - })); - - ParsedDocument parsedDoc = mapperWithDynamic.parse(source(b -> { - b.startObject("nested"); - { - b.field("source_field", "test value"); - } - b.endObject(); - })); - if (parsedDoc.dynamicMappingsUpdate() != null) { - merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); - } - - ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { - b.startObject("nested"); - { - b.field("source_field", "test value"); - } - b.endObject(); - })); - assertEquals(1, doc.rootDoc().getFields("nested.source_field").length); - // Copying to a field that is not in the mapping should not create a field in the document - assertEquals(0, doc.rootDoc().getFields("nested.target_field").length); - } - } diff --git a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java index ea94f0be5d3f2..6cf9600f74341 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java @@ -33,7 +33,6 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -41,7 +40,6 @@ import java.io.IOException; import java.time.Instant; -import java.util.Map; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -88,67 +86,6 @@ public void testDynamicFalse() throws IOException { assertThat(doc.rootDoc().get("field1"), equalTo("value1")); assertThat(doc.rootDoc().get("field2"), nullValue()); - - // Verify that field2 is still present in _source even though it's not indexed - Map sourceMap = XContentHelper.convertToMap(doc.source(), false, doc.getMediaType()).v2(); - assertThat(sourceMap.get("field1"), equalTo("value1")); - assertThat(sourceMap.get("field2"), equalTo("value2")); - } - - public void testDynamicFalseAllowTemplates() throws IOException { - DocumentMapper defaultMapper = createDocumentMapper(topMapping(b -> { - b.field("dynamic", "false_allow_templates"); - b.startArray("dynamic_templates"); - { - b.startObject(); - { - b.startObject("dates"); - b.field("match", "date_*"); - b.startObject("mapping"); - b.field("type", "date"); - b.endObject(); - b.endObject(); - } - b.endObject(); - } - b.endArray(); - b.startObject("properties"); - { - b.startObject("url"); - b.field("type", "text"); - b.endObject(); - } - b.endObject(); - })); - - ParsedDocument doc = defaultMapper.parse(source(b -> { - b.field("url", "https://example.com/"); - b.field("date_timestamp", "2024-01-01T00:00:00Z"); - b.field("date_timezone", "2024-01-02T00:00:00Z"); - })); - - assertThat(doc.rootDoc().get("url"), equalTo("https://example.com/")); - assertThat(doc.rootDoc().get("date_timestamp"), equalTo("1704067200000")); - assertThat(doc.rootDoc().get("date_timezone"), equalTo("1704153600000")); - - ParsedDocument doc2 = defaultMapper.parse(source(b -> { - b.field("url", "https://example.com/"); - b.field("date_timestamp", "2024-01-01T00:00:00Z"); - b.field("date_timezone", "2024-01-02T00:00:00Z"); - b.field("author", "John Doe"); - })); - - assertThat(doc2.rootDoc().get("url"), equalTo("https://example.com/")); - assertThat(doc2.rootDoc().get("date_timestamp"), equalTo("1704067200000")); - assertThat(doc2.rootDoc().get("date_timezone"), equalTo("1704153600000")); - assertThat(doc2.rootDoc().get("author"), nullValue()); - - // Verify that author is still present in _source even though it's not indexed - Map sourceMap = XContentHelper.convertToMap(doc2.source(), false, doc2.getMediaType()).v2(); - assertThat(sourceMap.get("author"), equalTo("John Doe")); - assertThat(sourceMap.get("url"), equalTo("https://example.com/")); - assertThat(sourceMap.get("date_timestamp"), equalTo("2024-01-01T00:00:00Z")); - assertThat(sourceMap.get("date_timezone"), equalTo("2024-01-02T00:00:00Z")); } public void testDynamicStrict() throws IOException {