Skip to content

Add false_allow_templates as a dynamic mapping option #19065

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 13, 2025
Merged

Add false_allow_templates as a dynamic mapping option #19065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Added
- Expand fetch phase profiling to support inner hits and top hits aggregation phases ([##18936](https://github.com/opensearch-project/OpenSearch/pull/18936))
- Add temporal routing processors for time-based document routing ([#18920](https://github.com/opensearch-project/OpenSearch/issues/18920))
- The dynamic mapping parameter supports false_allow_templates ([#19065](https://github.com/opensearch-project/OpenSearch/pull/19065))


### Changed
Expand Down
140 changes: 140 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
---
"Index documents with setting dynamic parameter to false_allow_templates in the mapping of the index":
- skip:
version: " - 3.2.99"
reason: "introduced in 3.3.0"

- do:
indices.create:
index: test_1
body:
mappings:
dynamic: false_allow_templates
dynamic_templates: [
{
dates: {
"match": "date_*",
"match_mapping_type": "date",
"mapping": {
"type": "date"
}
}
},
{
strings: {
"match": "stringField*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
object: {
"match": "objectField*",
"match_mapping_type": "object",
"mapping": {
"type": "object",
"properties": {
"bar1": {
"type": "keyword"
},
"bar2": {
"type": "text"
}
}
}
}
},
{
boolean: {
"match": "booleanField*",
"match_mapping_type": "boolean",
"mapping": {
"type": "boolean"
}
}
},
{
long: {
"match": "longField*",
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
double: {
"match": "doubleField*",
"match_mapping_type": "double",
"mapping": {
"type": "double"
}
}
},
{
array: {
"match": "arrayField*",
"mapping": {
"type": "keyword"
}
}
}
]
properties:
url:
type: keyword

- do:
index:
index: test_1
id: 1
body: {
url: "https://example.com",
date_timestamp: "2024-06-25T05:11:51.243Z",
stringField: "bar",
objectField: {
bar1: "bar1",
bar2: "bar2"
},
booleanField: true,
longField: 123456789,
doubleField: 123.456,
arrayField: ["item1", "item2", "item3"],
author: "John Doe"
}

- do:
get:
index: test_1
id: 1
- match:
_source:
url: "https://example.com"
date_timestamp: "2024-06-25T05:11:51.243Z"
stringField: "bar"
objectField:
bar1: "bar1"
bar2: "bar2"
booleanField: true
longField: 123456789
doubleField: 123.456
arrayField: ["item1", "item2", "item3"]
author: "John Doe"

- do:
indices.get_mapping:
index: test_1

- match: {test_1.mappings.dynamic: false_allow_templates}
- match: {test_1.mappings.properties.url.type: keyword}
- match: {test_1.mappings.properties.date_timestamp.type: date}
- match: {test_1.mappings.properties.stringField.type: keyword}
- match: {test_1.mappings.properties.objectField.properties.bar1.type: keyword}
- match: {test_1.mappings.properties.objectField.properties.bar2.type: text}
- match: {test_1.mappings.properties.booleanField.type: boolean}
- match: {test_1.mappings.properties.longField.type: long}
- match: {test_1.mappings.properties.doubleField.type: double}
- match: {test_1.mappings.properties.arrayField.type: keyword}
- match: {test_1.mappings.properties.author: null}
31 changes: 31 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,3 +190,34 @@ setup:

- match: {test_index1.mappings.dynamic: strict_allow_templates}
- match: {test_index1.mappings.properties.test1.type: text}

---
"post a mapping with setting dynamic to false_allow_templates":
- skip:
version: " - 3.2.99"
reason: "introduced in 3.3.0"
- do:
indices.put_mapping:
index: test_index1
body:
dynamic: false_allow_templates
dynamic_templates: [
{
strings: {
"match": "foo*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
]
properties:
test1:
type: text

- do:
indices.get_mapping: {}

- match: {test_index1.mappings.dynamic: false_allow_templates}
- match: {test_index1.mappings.properties.test1.type: text}
77 changes: 59 additions & 18 deletions server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName);
case TRUE:
case STRICT_ALLOW_TEMPLATES:
case FALSE_ALLOW_TEMPLATES:
Mapper.Builder builder = findTemplateBuilder(
context,
currentFieldName,
Expand All @@ -563,6 +564,10 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
);

if (builder == null) {
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
context.parser().skipChildren();
break;
}
builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
}
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
Expand Down Expand Up @@ -614,6 +619,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
);
case TRUE:
case STRICT_ALLOW_TEMPLATES:
case FALSE_ALLOW_TEMPLATES:
Mapper.Builder builder = findTemplateBuilder(
context,
arrayFieldName,
Expand All @@ -622,6 +628,10 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
parentMapper.fullPath()
);
if (builder == null) {
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
context.parser().skipChildren();
break;
}
parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
} else {
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(
Expand Down Expand Up @@ -786,13 +796,13 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
if (parseableAsLong && context.root().numericDetection()) {
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
if (builder == null) {
builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings()));
}
return builder;
} else if (parseableAsDouble && context.root().numericDetection()) {
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
if (builder == null) {
builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings());
return handleNoTemplateFound(dynamic, () -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings()));
}
return builder;
} else if (parseableAsLong == false && parseableAsDouble == false && context.root().dateDetection()) {
Expand All @@ -808,14 +818,16 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
}
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath);
if (builder == null) {
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
builder = new DateFieldMapper.Builder(
currentFieldName,
DateFieldMapper.Resolution.MILLISECONDS,
dateTimeFormatter,
ignoreMalformed,
IndexMetadata.indexCreated(context.indexSettings().getSettings())
);
return handleNoTemplateFound(dynamic, () -> {
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
return new DateFieldMapper.Builder(
currentFieldName,
DateFieldMapper.Resolution.MILLISECONDS,
dateTimeFormatter,
ignoreMalformed,
IndexMetadata.indexCreated(context.indexSettings().getSettings())
);
});
}
return builder;

Expand All @@ -824,8 +836,11 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(

Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
if (builder == null) {
builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField(
new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
return handleNoTemplateFound(
dynamic,
() -> new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField(
new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
)
);
}
return builder;
Expand All @@ -836,7 +851,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
|| numberType == XContentParser.NumberType.BIG_INTEGER) {
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
if (builder == null) {
builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings()));
}
return builder;
} else if (numberType == XContentParser.NumberType.FLOAT
Expand All @@ -847,34 +862,48 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
// no templates are defined, we use float by default instead of double
// since this is much more space-efficient and should be enough most of
// the time
builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings());
return handleNoTemplateFound(
dynamic,
() -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings())
);
}
return builder;
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath);
if (builder == null) {
builder = new BooleanFieldMapper.Builder(currentFieldName);
return handleNoTemplateFound(dynamic, () -> new BooleanFieldMapper.Builder(currentFieldName));
}
return builder;
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath);
if (builder == null) {
builder = new BinaryFieldMapper.Builder(currentFieldName);
return handleNoTemplateFound(dynamic, () -> new BinaryFieldMapper.Builder(currentFieldName));
}
return builder;
} else {
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
if (builder != null) {
return builder;
}
return handleNoTemplateFound(dynamic, () -> null);
}
// TODO how do we identify dynamically that its a binary value?
throw new IllegalStateException(
"Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]"
);
}

private static Mapper.Builder<?> handleNoTemplateFound(
ObjectMapper.Dynamic dynamic,
java.util.function.Supplier<Mapper.Builder<?>> builderSupplier
) {
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
return null;
}
return builderSupplier.get();
}

private static void parseDynamicValue(
final ParseContext context,
ObjectMapper parentMapper,
Expand All @@ -888,8 +917,16 @@ private static void parseDynamicValue(
if (dynamic == ObjectMapper.Dynamic.FALSE) {
return;
}
final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES && builder == null) {
// For FALSE_ALLOW_TEMPLATES, if no template matches, we still need to consume the token
// to maintain proper JSON parsing state
if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) {
context.parser().skipChildren();
}
return;
}
final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
Mapper mapper = builder.build(builderContext);
context.addDynamicMapper(mapper);

Expand Down Expand Up @@ -978,8 +1015,9 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
switch (dynamic) {
case STRICT:
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]);
case STRICT_ALLOW_TEMPLATES:
case TRUE:
case STRICT_ALLOW_TEMPLATES:
case FALSE_ALLOW_TEMPLATES:
Mapper.Builder builder = findTemplateBuilder(
context,
paths[i],
Expand All @@ -988,6 +1026,9 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
parent.fullPath()
);
if (builder == null) {
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
return new Tuple<>(pathsAdded, parent);
}
builder = new ObjectMapper.Builder(paths[i]).enabled(true);
}
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(
Expand Down
5 changes: 4 additions & 1 deletion server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ public enum Dynamic {
TRUE,
FALSE,
STRICT,
STRICT_ALLOW_TEMPLATES
STRICT_ALLOW_TEMPLATES,
FALSE_ALLOW_TEMPLATES
}

/**
Expand Down Expand Up @@ -313,6 +314,8 @@ protected static boolean parseObjectOrDocumentTypeProperties(
builder.dynamic(Dynamic.STRICT);
} else if (value.equalsIgnoreCase("strict_allow_templates")) {
builder.dynamic(Dynamic.STRICT_ALLOW_TEMPLATES);
} else if (value.equalsIgnoreCase("false_allow_templates")) {
builder.dynamic(Dynamic.FALSE_ALLOW_TEMPLATES);
} else {
boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic");
builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE);
Expand Down
Loading
Loading