From a18dd07e5af8d0e3bafaf4dae39c5063ac6b10e2 Mon Sep 17 00:00:00 2001 From: Bruce Hong Date: Wed, 13 Aug 2025 13:29:30 -0700 Subject: [PATCH] Introduce false_allow_templates option Signed-off-by: Bruce Hong --- CHANGELOG.md | 1 + .../test/index/111_false_allow_templates.yml | 140 +++++ .../indices.put_mapping/all_path_options.yml | 31 + .../index/mapper/DocumentParser.java | 77 ++- .../opensearch/index/mapper/ObjectMapper.java | 5 +- .../index/mapper/CopyToMapperTests.java | 35 ++ .../index/mapper/DocumentParserTests.java | 580 ++++++++++++++++++ .../index/mapper/DynamicMappingTests.java | 63 ++ 8 files changed, 913 insertions(+), 19 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index e5520c7529cc1..328b66bc10fcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Expand fetch phase profiling to support inner hits and top hits aggregation phases ([##18936](https://github.com/opensearch-project/OpenSearch/pull/18936)) - Add temporal routing processors for time-based document routing ([#18920](https://github.com/opensearch-project/OpenSearch/issues/18920)) +- The dynamic mapping parameter supports false_allow_templates ([#19065](https://github.com/opensearch-project/OpenSearch/pull/19065)) ### Changed diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml new file mode 100644 index 0000000000000..216f0e16dcaa9 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml @@ -0,0 +1,140 @@ +--- +"Index documents with setting dynamic parameter to false_allow_templates in the mapping of the index": + - skip: + version: " - 3.2.99" + reason: "introduced in 3.3.0" + + - do: + indices.create: + index: test_1 + body: + mappings: + dynamic: false_allow_templates + dynamic_templates: [ + { + dates: { + "match": "date_*", + "match_mapping_type": "date", + "mapping": { + "type": "date" + } + } + }, + { + strings: { + "match": "stringField*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + }, + { + object: { + "match": "objectField*", + "match_mapping_type": "object", + "mapping": { + "type": "object", + "properties": { + "bar1": { + "type": "keyword" + }, + "bar2": { + "type": "text" + } + } + } + } + }, + { + boolean: { + "match": "booleanField*", + "match_mapping_type": "boolean", + "mapping": { + "type": "boolean" + } + } + }, + { + long: { + "match": "longField*", + "match_mapping_type": "long", + "mapping": { + "type": "long" + } + } + }, + { + double: { + "match": "doubleField*", + "match_mapping_type": "double", + "mapping": { + "type": "double" + } + } + }, + { + array: { + "match": "arrayField*", + "mapping": { + "type": "keyword" + } + } + } + ] + properties: + url: + type: keyword + + - do: + index: + index: test_1 + id: 1 + body: { + url: "https://example.com", + date_timestamp: "2024-06-25T05:11:51.243Z", + stringField: "bar", + objectField: { + bar1: "bar1", + bar2: "bar2" + }, + booleanField: true, + longField: 123456789, + doubleField: 123.456, + arrayField: ["item1", "item2", "item3"], + author: "John Doe" + } + + - do: + get: + index: test_1 + id: 1 + - match: + _source: + url: "https://example.com" + date_timestamp: "2024-06-25T05:11:51.243Z" + stringField: "bar" + objectField: + bar1: "bar1" + bar2: "bar2" + booleanField: true + longField: 123456789 + doubleField: 123.456 + arrayField: ["item1", "item2", "item3"] + author: "John Doe" + + - do: + indices.get_mapping: + index: test_1 + + - match: {test_1.mappings.dynamic: false_allow_templates} + - match: {test_1.mappings.properties.url.type: keyword} + - match: {test_1.mappings.properties.date_timestamp.type: date} + - match: {test_1.mappings.properties.stringField.type: keyword} + - match: {test_1.mappings.properties.objectField.properties.bar1.type: keyword} + - match: {test_1.mappings.properties.objectField.properties.bar2.type: text} + - match: {test_1.mappings.properties.booleanField.type: boolean} + - match: {test_1.mappings.properties.longField.type: long} + - match: {test_1.mappings.properties.doubleField.type: double} + - match: {test_1.mappings.properties.arrayField.type: keyword} + - match: {test_1.mappings.properties.author: null} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml index 89b47fde2a72c..78b474f8f40b9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml @@ -190,3 +190,34 @@ setup: - match: {test_index1.mappings.dynamic: strict_allow_templates} - match: {test_index1.mappings.properties.test1.type: text} + +--- +"post a mapping with setting dynamic to false_allow_templates": + - skip: + version: " - 3.2.99" + reason: "introduced in 3.3.0" + - do: + indices.put_mapping: + index: test_index1 + body: + dynamic: false_allow_templates + dynamic_templates: [ + { + strings: { + "match": "foo*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + properties: + test1: + type: text + + - do: + indices.get_mapping: {} + + - match: {test_index1.mappings.dynamic: false_allow_templates} + - match: {test_index1.mappings.properties.test1.type: text} diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 134baa70f80c2..e8d8a63d9d0e0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -554,6 +554,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName); case TRUE: case STRICT_ALLOW_TEMPLATES: + case FALSE_ALLOW_TEMPLATES: Mapper.Builder builder = findTemplateBuilder( context, currentFieldName, @@ -563,6 +564,10 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, ); if (builder == null) { + if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { + context.parser().skipChildren(); + break; + } builder = new ObjectMapper.Builder(currentFieldName).enabled(true); } Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); @@ -614,6 +619,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, ); case TRUE: case STRICT_ALLOW_TEMPLATES: + case FALSE_ALLOW_TEMPLATES: Mapper.Builder builder = findTemplateBuilder( context, arrayFieldName, @@ -622,6 +628,10 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, parentMapper.fullPath() ); if (builder == null) { + if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { + context.parser().skipChildren(); + break; + } parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); } else { Mapper.BuilderContext builderContext = new Mapper.BuilderContext( @@ -786,13 +796,13 @@ private static Mapper.Builder createBuilderFromDynamicValue( if (parseableAsLong && context.root().numericDetection()) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { - builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); + return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings())); } return builder; } else if (parseableAsDouble && context.root().numericDetection()) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); if (builder == null) { - builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings()); + return handleNoTemplateFound(dynamic, () -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings())); } return builder; } else if (parseableAsLong == false && parseableAsDouble == false && context.root().dateDetection()) { @@ -808,14 +818,16 @@ private static Mapper.Builder createBuilderFromDynamicValue( } Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath); if (builder == null) { - boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); - builder = new DateFieldMapper.Builder( - currentFieldName, - DateFieldMapper.Resolution.MILLISECONDS, - dateTimeFormatter, - ignoreMalformed, - IndexMetadata.indexCreated(context.indexSettings().getSettings()) - ); + return handleNoTemplateFound(dynamic, () -> { + boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); + return new DateFieldMapper.Builder( + currentFieldName, + DateFieldMapper.Resolution.MILLISECONDS, + dateTimeFormatter, + ignoreMalformed, + IndexMetadata.indexCreated(context.indexSettings().getSettings()) + ); + }); } return builder; @@ -824,8 +836,11 @@ private static Mapper.Builder createBuilderFromDynamicValue( Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); if (builder == null) { - builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField( - new KeywordFieldMapper.Builder("keyword").ignoreAbove(256) + return handleNoTemplateFound( + dynamic, + () -> new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField( + new KeywordFieldMapper.Builder("keyword").ignoreAbove(256) + ) ); } return builder; @@ -836,7 +851,7 @@ private static Mapper.Builder createBuilderFromDynamicValue( || numberType == XContentParser.NumberType.BIG_INTEGER) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { - builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); + return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings())); } return builder; } else if (numberType == XContentParser.NumberType.FLOAT @@ -847,20 +862,23 @@ private static Mapper.Builder createBuilderFromDynamicValue( // no templates are defined, we use float by default instead of double // since this is much more space-efficient and should be enough most of // the time - builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings()); + return handleNoTemplateFound( + dynamic, + () -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings()) + ); } return builder; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath); if (builder == null) { - builder = new BooleanFieldMapper.Builder(currentFieldName); + return handleNoTemplateFound(dynamic, () -> new BooleanFieldMapper.Builder(currentFieldName)); } return builder; } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath); if (builder == null) { - builder = new BinaryFieldMapper.Builder(currentFieldName); + return handleNoTemplateFound(dynamic, () -> new BinaryFieldMapper.Builder(currentFieldName)); } return builder; } else { @@ -868,6 +886,7 @@ private static Mapper.Builder createBuilderFromDynamicValue( if (builder != null) { return builder; } + return handleNoTemplateFound(dynamic, () -> null); } // TODO how do we identify dynamically that its a binary value? throw new IllegalStateException( @@ -875,6 +894,16 @@ private static Mapper.Builder createBuilderFromDynamicValue( ); } + private static Mapper.Builder handleNoTemplateFound( + ObjectMapper.Dynamic dynamic, + java.util.function.Supplier> builderSupplier + ) { + if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { + return null; + } + return builderSupplier.get(); + } + private static void parseDynamicValue( final ParseContext context, ObjectMapper parentMapper, @@ -888,8 +917,16 @@ private static void parseDynamicValue( if (dynamic == ObjectMapper.Dynamic.FALSE) { return; } - final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); final Mapper.Builder builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath()); + if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES && builder == null) { + // For FALSE_ALLOW_TEMPLATES, if no template matches, we still need to consume the token + // to maintain proper JSON parsing state + if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) { + context.parser().skipChildren(); + } + return; + } + final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); Mapper mapper = builder.build(builderContext); context.addDynamicMapper(mapper); @@ -978,8 +1015,9 @@ private static Tuple getDynamicParentMapper( switch (dynamic) { case STRICT: throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]); - case STRICT_ALLOW_TEMPLATES: case TRUE: + case STRICT_ALLOW_TEMPLATES: + case FALSE_ALLOW_TEMPLATES: Mapper.Builder builder = findTemplateBuilder( context, paths[i], @@ -988,6 +1026,9 @@ private static Tuple getDynamicParentMapper( parent.fullPath() ); if (builder == null) { + if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { + return new Tuple<>(pathsAdded, parent); + } builder = new ObjectMapper.Builder(paths[i]).enabled(true); } Mapper.BuilderContext builderContext = new Mapper.BuilderContext( diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index dd66fb08cecba..42373d5686543 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -93,7 +93,8 @@ public enum Dynamic { TRUE, FALSE, STRICT, - STRICT_ALLOW_TEMPLATES + STRICT_ALLOW_TEMPLATES, + FALSE_ALLOW_TEMPLATES } /** @@ -313,6 +314,8 @@ protected static boolean parseObjectOrDocumentTypeProperties( builder.dynamic(Dynamic.STRICT); } else if (value.equalsIgnoreCase("strict_allow_templates")) { builder.dynamic(Dynamic.STRICT_ALLOW_TEMPLATES); + } else if (value.equalsIgnoreCase("false_allow_templates")) { + builder.dynamic(Dynamic.FALSE_ALLOW_TEMPLATES); } else { boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic"); builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE); diff --git a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java index 7a8c4ffe35021..e8ce3341c062c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java @@ -49,6 +49,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class CopyToMapperTests extends MapperServiceTestCase { @@ -287,6 +288,40 @@ public void testCopyToStrictAllowTemplatesDynamicInnerObjectParsing() throws Exc ); } + public void testCopyToFalseAllowTemplatesDynamicInnerObjectParsing() throws Exception { + DocumentMapper docMapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + b.startObject("properties"); + { + b.startObject("copy_test"); + { + b.field("type", "text"); + b.field("copy_to", "very.inner.field"); + } + b.endObject(); + } + b.endObject(); + })); + + ParsedDocument doc = docMapper.parse(source(b -> b.field("copy_test", "foo"))); + assertThat(doc.rootDoc().get("copy_test"), equalTo("foo")); + assertThat(doc.rootDoc().get("very.inner.field"), nullValue()); + } + public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception { DocumentMapper docMapper = createDocumentMapper(mapping(b -> { diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index c9763d634979b..e9be8b671d7ba 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -2002,4 +2002,584 @@ public void testDocumentDeepNestedObjectAndArrayCombination() throws Exception { } + public void testDynamicFalseAllowTemplatesLongArray() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())); + assertEquals(0, doc.rootDoc().getFields("foo").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "foo"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())); + assertEquals(4, docWithTemplate.rootDoc().getFields("foo").length); + } + + public void testDynamicFalseAllowTemplatesObject() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test1"); + { + b.field("match", "test1"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + } + + )); + + ParsedDocument doc = mapper.parse(source(b -> b.startObject("test").field("test1", "foo").endObject())); + assertEquals(2, doc.rootDoc().getFields("test.test1").length); + } + + public void testDynamicFalseAllowTemplatesValue() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("bar", "baz"))); + assertEquals(0, doc.rootDoc().getFields("bar").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "bar"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bar", "baz"))); + assertEquals(2, docWithTemplate.rootDoc().getFields("bar").length); + } + + public void testDynamicFalseAllowTemplatesNull() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.nullField("bar"))); + assertEquals(0, doc.rootDoc().getFields("bar").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "bar"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.nullField("bar"))); + assertEquals(0, docWithTemplate.rootDoc().getFields("bar").length); // null fields don't create mappings + } + + public void testDynamicFalseAllowTemplatesDottedFieldNameLongArray() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray())); + assertEquals(0, doc.rootDoc().getFields("foo.bar.baz").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "foo"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test1"); + { + b.field("match", "bar"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test2"); + { + b.field("path_match", "foo.bar.baz"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray())); + assertEquals(4, docWithTemplate.rootDoc().getFields("foo.bar.baz").length); + } + + public void testDynamicFalseAllowTemplatesDottedFieldNameLong() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("foo.bar.baz", 0))); + assertEquals(0, doc.rootDoc().getFields("foo.bar.baz").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "foo"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test1"); + { + b.field("match", "bar"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test2"); + { + b.field("path_match", "foo.bar.baz"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("foo.bar.baz", 0))); + assertEquals(2, docWithTemplate.rootDoc().getFields("foo.bar.baz").length); + } + + public void testDynamicFalseAllowTemplatesDottedFieldNameObject() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())); + assertEquals(0, doc.rootDoc().getFields("foo.bar.baz.a").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "foo"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test1"); + { + b.field("match", "bar"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test2"); + { + b.field("path_match", "foo.bar.baz"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())); + assertEquals(2, docWithTemplate.rootDoc().getFields("foo.bar.baz.a").length); + } + + public void testDynamicFalseAllowTemplatesWithEmbeddedObject() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("embedded_field", "value"))); + assertEquals(0, doc.rootDoc().getFields("embedded_field").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("embedded_template"); + { + b.field("match", "embedded_*"); + b.field("match_mapping_type", "string"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("embedded_field", "value"))); + assertEquals(2, docWithTemplate.rootDoc().getFields("embedded_field").length); + } + + public void testDynamicFalseAllowTemplatesWithDateDetection() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_date_formats").value("yyyy-MM-dd").endArray(); + })); + ParsedDocument doc = mapper.parse(source(b -> b.field("date_field", "2023-12-25"))); + assertEquals(0, doc.rootDoc().getFields("date_field").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_date_formats").value("yyyy-MM-dd").endArray(); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("date_template"); + { + b.field("match", "date_*"); + b.field("match_mapping_type", "date"); + b.startObject("mapping").field("type", "date").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("date_field", "2023-12-25"))); + assertEquals(2, docWithTemplate.rootDoc().getFields("date_field").length); + } + + public void testDynamicFalseAllowTemplatesWithNumericDetection() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("numeric_field", "123"))); + assertEquals(0, doc.rootDoc().getFields("numeric_field").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("numeric_template"); + { + b.field("match", "numeric_*"); + b.field("match_mapping_type", "long"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("numeric_field", "123"))); + assertEquals(0, docWithTemplate.rootDoc().getFields("numeric_field").length); + } + + public void testDynamicFalseAllowTemplatesWithFloatDetection() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("float_field", "123.45"))); + assertEquals(0, doc.rootDoc().getFields("float_field").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("float_template"); + { + b.field("match", "float_*"); + b.field("match_mapping_type", "double"); + b.startObject("mapping").field("type", "float").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("float_field", "123.45"))); + assertEquals(0, docWithTemplate.rootDoc().getFields("float_field").length); + } + + public void testDynamicFalseAllowTemplatesWithBooleanDetection() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("bool_field", true))); + assertEquals(0, doc.rootDoc().getFields("bool_field").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("boolean_template"); + { + b.field("match", "bool_*"); + b.field("match_mapping_type", "boolean"); + b.startObject("mapping").field("type", "boolean").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bool_field", true))); + assertEquals(2, docWithTemplate.rootDoc().getFields("bool_field").length); + } + + public void testDynamicFalseAllowTemplatesWithBigInteger() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + BigInteger bigInt = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE); + ParsedDocument doc = mapper.parse(source(b -> b.field("bigint_field", bigInt))); + assertEquals(0, doc.rootDoc().getFields("bigint_field").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("bigint_template"); + { + b.field("match", "bigint_*"); + b.field("match_mapping_type", "long"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bigint_field", bigInt))); + assertEquals(2, docWithTemplate.rootDoc().getFields("bigint_field").length); + } + + public void testDynamicFalseAllowTemplatesWithBigDecimal() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "false_allow_templates"))); + BigDecimal bigDecimal = BigDecimal.valueOf(Double.MAX_VALUE).add(BigDecimal.valueOf(10.1)); + ParsedDocument doc = mapper.parse(source(b -> b.field("bigdecimal_field", bigDecimal))); + assertEquals(0, doc.rootDoc().getFields("bigdecimal_field").length); + + DocumentMapper mapperWithTemplate = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("bigdecimal_template"); + { + b.field("match", "bigdecimal_*"); + b.field("match_mapping_type", "double"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument docWithTemplate = mapperWithTemplate.parse(source(b -> b.field("bigdecimal_field", bigDecimal))); + assertEquals(2, docWithTemplate.rootDoc().getFields("bigdecimal_field").length); + } + + public void testDynamicFalseAllowTemplatesWithCopyTo() throws Exception { + DocumentMapper mapperWithDynamic = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("target_template"); + { + b.field("match", "target_*"); + b.field("match_mapping_type", "string"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("source_field"); + { + b.field("type", "text"); + b.field("copy_to", "target_field"); + } + b.endObject(); + })); + + ParsedDocument parsedDoc = mapperWithDynamic.parse(source(b -> b.field("source_field", "test value"))); + if (parsedDoc.dynamicMappingsUpdate() != null) { + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + } + + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("source_field", "test value"))); + assertEquals(1, doc.rootDoc().getFields("source_field").length); + assertEquals(1, doc.rootDoc().getFields("target_field").length); + } + + public void testDynamicFalseAllowTemplatesWithNestedCopyTo() throws Exception { + DocumentMapper mapperWithDynamic = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("nested_template"); + { + b.field("match", "nested"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("target_template"); + { + b.field("match", "target_*"); + b.field("match_mapping_type", "string"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("nested"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("source_field"); + { + b.field("type", "text"); + b.field("copy_to", "nested.target_field"); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + + ParsedDocument parsedDoc = mapperWithDynamic.parse(source(b -> { + b.startObject("nested"); + { + b.field("source_field", "test value"); + } + b.endObject(); + })); + if (parsedDoc.dynamicMappingsUpdate() != null) { + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + } + + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { + b.startObject("nested"); + { + b.field("source_field", "test value"); + } + b.endObject(); + })); + assertEquals(1, doc.rootDoc().getFields("nested.source_field").length); + // Copying to a field that is not in the mapping should not create a field in the document + assertEquals(0, doc.rootDoc().getFields("nested.target_field").length); + } + } diff --git a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java index 6cf9600f74341..ea94f0be5d3f2 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java @@ -33,6 +33,7 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -40,6 +41,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.Map; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -86,6 +88,67 @@ public void testDynamicFalse() throws IOException { assertThat(doc.rootDoc().get("field1"), equalTo("value1")); assertThat(doc.rootDoc().get("field2"), nullValue()); + + // Verify that field2 is still present in _source even though it's not indexed + Map sourceMap = XContentHelper.convertToMap(doc.source(), false, doc.getMediaType()).v2(); + assertThat(sourceMap.get("field1"), equalTo("value1")); + assertThat(sourceMap.get("field2"), equalTo("value2")); + } + + public void testDynamicFalseAllowTemplates() throws IOException { + DocumentMapper defaultMapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "false_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("dates"); + b.field("match", "date_*"); + b.startObject("mapping"); + b.field("type", "date"); + b.endObject(); + b.endObject(); + } + b.endObject(); + } + b.endArray(); + b.startObject("properties"); + { + b.startObject("url"); + b.field("type", "text"); + b.endObject(); + } + b.endObject(); + })); + + ParsedDocument doc = defaultMapper.parse(source(b -> { + b.field("url", "https://example.com/"); + b.field("date_timestamp", "2024-01-01T00:00:00Z"); + b.field("date_timezone", "2024-01-02T00:00:00Z"); + })); + + assertThat(doc.rootDoc().get("url"), equalTo("https://example.com/")); + assertThat(doc.rootDoc().get("date_timestamp"), equalTo("1704067200000")); + assertThat(doc.rootDoc().get("date_timezone"), equalTo("1704153600000")); + + ParsedDocument doc2 = defaultMapper.parse(source(b -> { + b.field("url", "https://example.com/"); + b.field("date_timestamp", "2024-01-01T00:00:00Z"); + b.field("date_timezone", "2024-01-02T00:00:00Z"); + b.field("author", "John Doe"); + })); + + assertThat(doc2.rootDoc().get("url"), equalTo("https://example.com/")); + assertThat(doc2.rootDoc().get("date_timestamp"), equalTo("1704067200000")); + assertThat(doc2.rootDoc().get("date_timezone"), equalTo("1704153600000")); + assertThat(doc2.rootDoc().get("author"), nullValue()); + + // Verify that author is still present in _source even though it's not indexed + Map sourceMap = XContentHelper.convertToMap(doc2.source(), false, doc2.getMediaType()).v2(); + assertThat(sourceMap.get("author"), equalTo("John Doe")); + assertThat(sourceMap.get("url"), equalTo("https://example.com/")); + assertThat(sourceMap.get("date_timestamp"), equalTo("2024-01-01T00:00:00Z")); + assertThat(sourceMap.get("date_timezone"), equalTo("2024-01-02T00:00:00Z")); } public void testDynamicStrict() throws IOException {