From b21c6dd5b598d059f4f8cb1c68cf245bb0ca8936 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 30 Aug 2024 17:34:14 +0300 Subject: [PATCH 01/18] Control storing array source with index setting --- .../indices.create/20_synthetic_source.yml | 398 ---------- .../21_synthetic_source_stored.yml | 732 ++++++++++++++++++ .../common/settings/IndexScopedSettings.java | 1 + .../elasticsearch/index/IndexSettings.java | 14 + .../index/mapper/DocumentParser.java | 29 +- .../index/mapper/DocumentParserContext.java | 4 + .../elasticsearch/index/mapper/Mapper.java | 42 + .../mapper/IgnoredSourceFieldMapperTests.java | 107 +++ 8 files changed, 925 insertions(+), 402 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index fa08efe402b43..265aec75dc9c2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -446,260 +446,6 @@ mixed disabled and enabled objects: - match: { hits.hits.0._source.path.to.bad.value: false } ---- -object array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - id: - type: integer - regular: - properties: - span: - properties: - id: - type: keyword - trace: - properties: - id: - type: keyword - stored: - store_array_source: true - properties: - span: - properties: - id: - type: keyword - trace: - properties: - id: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "id": 1, "regular": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - - '{ "create": { } }' - - '{ "id": 2, "stored": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - - - do: - search: - index: test - sort: id - - - length: { hits.hits.0._source.regular: 2 } - - match: { hits.hits.0._source.regular.span.id: "1" } - - match: { hits.hits.0._source.regular.trace.id: [ "a", "b" ] } - - - length: { hits.hits.1._source.stored: 2 } - - match: { hits.hits.1._source.stored.0.trace.id: a } - - match: { hits.hits.1._source.stored.0.span.id: "1" } - - match: { hits.hits.1._source.stored.1.trace.id: b } - - match: { hits.hits.1._source.stored.1.span.id: "1" } - - ---- -object array within array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - stored: - store_array_source: true - properties: - path: - store_array_source: true - properties: - to: - properties: - trace: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "stored": [ { "path": [{ "to": { "trace": "A" } }, { "to": { "trace": "B" } } ] }, { "path": { "to": { "trace": "C" } } } ] }' - - - do: - search: - index: test - - - length: { hits.hits.0._source.stored: 2 } - - match: { hits.hits.0._source.stored.0.path.0.to.trace: A } - - match: { hits.hits.0._source.stored.0.path.1.to.trace: B } - - match: { hits.hits.0._source.stored.1.path.to.trace: C } - - ---- -no object array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - stored: - store_array_source: true - properties: - span: - properties: - id: - type: keyword - trace: - properties: - id: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "stored": { "trace": { "id": "a" }, "span": { "id": "b" } } }' - - - do: - search: - index: test - - - match: { hits.hits.0._source.stored.trace.id: a } - - match: { hits.hits.0._source.stored.span.id: b } - - ---- -field ordering in object array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - a: - type: keyword - b: - store_array_source: true - properties: - aa: - type: keyword - bb: - type: keyword - c: - type: keyword - d: - store_array_source: true - properties: - aa: - type: keyword - bb: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "c": 1, "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "a": 2, "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ] }' - - - do: - search: - index: test - - - length: { hits.hits.0._source: 4 } - - match: { hits.hits.0._source: { "a": "2", "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "c": "1", "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ] } } - - ---- -nested object array next to other fields: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - a: - type: keyword - b: - properties: - c: - store_array_source: true - properties: - aa: - type: keyword - bb: - type: keyword - d: - properties: - aa: - type: keyword - bb: - type: keyword - e: - type: keyword - f: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "e": 1000 }, "f": 2000 }' - - - do: - search: - index: test - - - match: { hits.hits.0._source.a: "1" } - - match: { hits.hits.0._source.b.c: [{ "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 }] } - - match: { hits.hits.0._source.b.d.aa: [ "200", "300" ] } - - match: { hits.hits.0._source.b.d.bb: [ "100", "400" ] } - - match: { hits.hits.0._source.b.e: "1000" } - - match: { hits.hits.0._source.f: "2000" } - - --- object with dynamic override: - requires: @@ -1157,99 +903,6 @@ doubly nested object: - match: { hits.hits.3._source.id: 3 } ---- -nested object with stored array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - name: - type: keyword - nested_array_regular: - type: nested - nested_array_stored: - type: nested - store_array_source: true - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "name": "A", "nested_array_regular": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' - - '{ "create": { } }' - - '{ "name": "B", "nested_array_stored": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' - - - match: { errors: false } - - - do: - search: - index: test - sort: name - - match: { hits.total.value: 2 } - - match: { hits.hits.0._source.name: A } - - match: { hits.hits.0._source.nested_array_regular.0.b.c: [ 10, 100] } - - match: { hits.hits.0._source.nested_array_regular.1.b.c: [ 20, 200] } - - match: { hits.hits.1._source.name: B } - - match: { hits.hits.1._source.nested_array_stored.0.b.0.c: 10 } - - match: { hits.hits.1._source.nested_array_stored.0.b.1.c: 100 } - - match: { hits.hits.1._source.nested_array_stored.1.b.0.c: 20 } - - match: { hits.hits.1._source.nested_array_stored.1.b.1.c: 200 } - ---- -empty nested object sorted as a first document: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - settings: - index: - sort.field: "name" - sort.order: "asc" - mappings: - _source: - mode: synthetic - properties: - name: - type: keyword - nested: - type: nested - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "name": "B", "nested": { "a": "b" } }' - - '{ "create": { } }' - - '{ "name": "A" }' - - - match: { errors: false } - - - do: - search: - index: test - sort: name - - - match: { hits.total.value: 2 } - - match: { hits.hits.0._source.name: A } - - match: { hits.hits.1._source.name: B } - - match: { hits.hits.1._source.nested.a: "b" } - --- subobjects auto: - requires: @@ -1337,54 +990,3 @@ subobjects auto: - match: { hits.hits.3._source.id: 4 } - match: { hits.hits.3._source.auto_obj.foo: 40 } - match: { hits.hits.3._source.auto_obj.foo\.bar: 400 } - ---- -# 112156 -stored field under object with store_array_source: - - requires: - cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] - reason: requires bug fix to be implemented - - - do: - indices.create: - index: test - body: - settings: - index: - sort.field: "name" - sort.order: "asc" - mappings: - _source: - mode: synthetic - properties: - name: - type: keyword - obj: - store_array_source: true - properties: - foo: - type: keyword - store: true - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "name": "B", "obj": null }' - - '{ "create": { } }' - - '{ "name": "A", "obj": [ { "foo": "hello_from_the_other_side" } ] }' - - - match: { errors: false } - - - do: - search: - index: test - sort: name - - - match: { hits.total.value: 2 } - - match: { hits.hits.0._source.name: A } - - match: { hits.hits.0._source.obj: [ { "foo": "hello_from_the_other_side" } ] } - - match: { hits.hits.1._source.name: B } - - match: { hits.hits.1._source.obj: null } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml new file mode 100644 index 0000000000000..52a8f1b4cf70f --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -0,0 +1,732 @@ +--- +object param - object array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + regular: + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + stored: + store_array_source: true + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "regular": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "create": { } }' + - '{ "id": 2, "stored": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + + - do: + search: + index: test + sort: id + + - length: { hits.hits.0._source.regular: 2 } + - match: { hits.hits.0._source.regular.span.id: "1" } + - match: { hits.hits.0._source.regular.trace.id: [ "a", "b" ] } + + - length: { hits.hits.1._source.stored: 2 } + - match: { hits.hits.1._source.stored.0.trace.id: a } + - match: { hits.hits.1._source.stored.0.span.id: "1" } + - match: { hits.hits.1._source.stored.1.trace.id: b } + - match: { hits.hits.1._source.stored.1.span.id: "1" } + + +--- +object param - object array within array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + stored: + store_array_source: true + properties: + path: + store_array_source: true + properties: + to: + properties: + trace: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": [ { "path": [{ "to": { "trace": "A" } }, { "to": { "trace": "B" } } ] }, { "path": { "to": { "trace": "C" } } } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source.stored: 2 } + - match: { hits.hits.0._source.stored.0.path.0.to.trace: A } + - match: { hits.hits.0._source.stored.0.path.1.to.trace: B } + - match: { hits.hits.0._source.stored.1.path.to.trace: C } + + +--- +object param - no object array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + stored: + store_array_source: true + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": { "trace": { "id": "a" }, "span": { "id": "b" } } }' + + - do: + search: + index: test + + - match: { hits.hits.0._source.stored.trace.id: a } + - match: { hits.hits.0._source.stored.span.id: b } + + +--- +object param - field ordering in object array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + store_array_source: true + properties: + aa: + type: keyword + bb: + type: keyword + c: + type: keyword + d: + store_array_source: true + properties: + aa: + type: keyword + bb: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "c": 1, "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "a": 2, "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source: 4 } + - match: { hits.hits.0._source: { "a": "2", "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "c": "1", "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ] } } + + +--- +object param - nested object array next to other fields: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + properties: + c: + store_array_source: true + properties: + aa: + type: keyword + bb: + type: keyword + d: + properties: + aa: + type: keyword + bb: + type: keyword + e: + type: keyword + f: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "e": 1000 }, "f": 2000 }' + + - do: + search: + index: test + + - match: { hits.hits.0._source.a: "1" } + - match: { hits.hits.0._source.b.c: [{ "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 }] } + - match: { hits.hits.0._source.b.d.aa: [ "200", "300" ] } + - match: { hits.hits.0._source.b.d.bb: [ "100", "400" ] } + - match: { hits.hits.0._source.b.e: "1000" } + - match: { hits.hits.0._source.f: "2000" } + + +--- +object param - nested object with stored array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + nested_array_regular: + type: nested + nested_array_stored: + type: nested + store_array_source: true + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "nested_array_regular": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' + - '{ "create": { } }' + - '{ "name": "B", "nested_array_stored": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' + + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.nested_array_regular.0.b.c: [ 10, 100] } + - match: { hits.hits.0._source.nested_array_regular.1.b.c: [ 20, 200] } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.nested_array_stored.0.b.0.c: 10 } + - match: { hits.hits.1._source.nested_array_stored.0.b.1.c: 100 } + - match: { hits.hits.1._source.nested_array_stored.1.b.0.c: 20 } + - match: { hits.hits.1._source.nested_array_stored.1.b.1.c: 200 } + + +--- +# 112156 +stored field under object with store_array_source: + - requires: + cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] + reason: requires bug fix to be implemented + + - do: + indices.create: + index: test + body: + settings: + index: + sort.field: "name" + sort.order: "asc" + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + obj: + store_array_source: true + properties: + foo: + type: keyword + store: true + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "B", "obj": null }' + - '{ "create": { } }' + - '{ "name": "A", "obj": [ { "foo": "hello_from_the_other_side" } ] }' + + - match: { errors: false } + + - do: + search: + index: test + sort: name + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.obj: [ { "foo": "hello_from_the_other_side" } ] } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.obj: null } + + +--- +index param - root arrays: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + store_array_source: true + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + leaf: + type: integer + obj: + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "leaf": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "create": { } }' + - '{ "id": 2, "leaf": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.leaf: [30, 20, 10] } + - length: { hits.hits.0._source.obj: 2 } + - match: { hits.hits.0._source.obj.0.trace.id: a } + - match: { hits.hits.0._source.obj.0.span.id: "1" } + - match: { hits.hits.0._source.obj.1.trace.id: b } + - match: { hits.hits.0._source.obj.1.span.id: "1" } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.leaf: [ 130, 120, 110 ] } + - length: { hits.hits.1._source.obj: 2 } + - match: { hits.hits.1._source.obj.0.trace.id: aa } + - match: { hits.hits.1._source.obj.0.span.id: "2" } + - match: { hits.hits.1._source.obj.1.trace.id: bb } + - match: { hits.hits.1._source.obj.1.span.id: "2" } + + +--- +index param - dynamic root arrays: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + store_array_source: true + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "leaf": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "create": { } }' + - '{ "id": 2, "leaf": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.leaf: [30, 20, 10] } + - length: { hits.hits.0._source.obj: 2 } + - match: { hits.hits.0._source.obj.0.trace.id: a } + - match: { hits.hits.0._source.obj.0.span.id: "1" } + - match: { hits.hits.0._source.obj.1.trace.id: b } + - match: { hits.hits.0._source.obj.1.span.id: "1" } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.leaf: [ 130, 120, 110 ] } + - length: { hits.hits.1._source.obj: 2 } + - match: { hits.hits.1._source.obj.0.trace.id: aa } + - match: { hits.hits.1._source.obj.0.span.id: "2" } + - match: { hits.hits.1._source.obj.1.trace.id: bb } + - match: { hits.hits.1._source.obj.1.span.id: "2" } + + +--- +index param - object array within array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + store_array_source: true + mappings: + _source: + mode: synthetic + properties: + stored: + properties: + path: + properties: + to: + properties: + trace: + type: keyword + values: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": [ { "path": [{ "to": { "trace": "A", "values": [2, 1] } }, { "to": { "trace": "B", "values": [2, 1] } } ] }, { "path": { "to": { "trace": "C", "values": 3 } } } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source.stored: 2 } + - match: { hits.hits.0._source.stored.0.path.0.to.trace: A } + - match: { hits.hits.0._source.stored.0.path.0.to.values: [2, 1] } + - match: { hits.hits.0._source.stored.0.path.1.to.trace: B } + - match: { hits.hits.0._source.stored.0.path.1.to.values: [2, 1] } + - match: { hits.hits.0._source.stored.1.path.to.trace: C } + - match: { hits.hits.0._source.stored.1.path.to.values: 3 } + + +--- +index param - no object array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + store_array_source: true + mappings: + _source: + mode: synthetic + properties: + stored: + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": { "trace": { "id": "a" }, "span": { "id": "b" } } }' + + - do: + search: + index: test + + - match: { hits.hits.0._source.stored.trace.id: a } + - match: { hits.hits.0._source.stored.span.id: b } + + +--- +index param - field ordering: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + store_array_source: true + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + properties: + aa: + type: keyword + bb: + type: keyword + c: + type: keyword + d: + properties: + aa: + type: keyword + bb: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "c": [30, 20, 10], "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "a": 2, "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source: 4 } + - match: { hits.hits.0._source: { "a": "2", "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "c": [30, 20, 10], "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ] } } + + +--- +index param - nested arrays: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + store_array_source: true + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + properties: + c: + properties: + aa: + type: keyword + bb: + type: keyword + d: + type: integer + e: + type: keyword + f: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ 300, 200, 100 ], "e": 1000 }, "f": 2000 }' + - '{ "create": { } }' + - '{ "a": 11, "b": { "c": [ { "bb": 110, "aa": 120 }, { "aa": 130, "bb": 140 } ], "d": [ 1300, 1200, 1100 ], "e": 11000 }, "f": 12000 }' + + + - do: + search: + index: test + sort: a + + - match: { hits.hits.0._source.a: "1" } + - match: { hits.hits.0._source.b.c: [{ "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 }] } + - match: { hits.hits.0._source.b.d: [ 300, 200, 100 ] } + - match: { hits.hits.0._source.b.e: "1000" } + - match: { hits.hits.0._source.f: "2000" } + + - match: { hits.hits.1._source.a: "11" } + - match: { hits.hits.1._source.b.c: [ { "bb": 110, "aa": 120 }, { "aa": 130, "bb": 140 } ] } + - match: { hits.hits.1._source.b.d: [ 1300, 1200, 1100 ] } + - match: { hits.hits.1._source.b.e: "11000" } + - match: { hits.hits.1._source.f: "12000" } + +--- +index param - nested object with stored array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + store_array_source: true + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + nested: + type: nested + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "nested": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' + - '{ "create": { } }' + - '{ "name": "B", "nested": [ { "b": [ { "c": 30 }, { "c": 300 } ] }, { "b": [ { "c": 40 }, { "c": 400 } ] } ] }' + + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.nested.0.b.0.c: 10 } + - match: { hits.hits.0._source.nested.0.b.1.c: 100 } + - match: { hits.hits.0._source.nested.1.b.0.c: 20 } + - match: { hits.hits.0._source.nested.1.b.1.c: 200 } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.nested.0.b.0.c: 30 } + - match: { hits.hits.1._source.nested.0.b.1.c: 300 } + - match: { hits.hits.1._source.nested.1.b.0.c: 40 } + - match: { hits.hits.1._source.nested.1.b.1.c: 400 } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index fe6616cb4fb8e..40f57553f924d 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -181,6 +181,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, + FieldMapper.STORE_ARRAY_SOURCE_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 944d50f7ea06c..66c020f2567bd 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.Node; @@ -793,6 +794,16 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private final IndexRouting indexRouting; + /** + * The default mode for storing source, for all mappers not overriding setting. + * This is only relevant for indexes configured with synthetic-source code. + */ + public Mapper.StoreSourceMode storeSourceMode() { + return storeSourceMode; + } + + private final Mapper.StoreSourceMode storeSourceMode; + /** * Returns the default search fields for this index. */ @@ -922,6 +933,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); + storeSourceMode = scopedSettings.get(Mapper.STORE_ARRAY_SOURCE_SETTING) + ? Mapper.StoreSourceMode.ARRAYS + : Mapper.StoreSourceMode.DISABLED; es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); scopedSettings.addSettingsUpdateConsumer( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 35f0130c58706..3692bda3f9e59 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -295,9 +295,24 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio throwOnConcreteValue(context.parent(), currentFieldName, context); } + if (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ENABLED && context.canAddIgnoredField()) { + Tuple tuple = XContentDataHelper.cloneSubContext(context); + context.addIgnoredField( + new IgnoredSourceFieldMapper.NameValue( + context.parent().fullPath(), + context.parent().fullPath().lastIndexOf(context.parent().leafName()), + XContentDataHelper.encodeXContentBuilder(tuple.v2()), + context.doc() + ) + ); + context = tuple.v1(); + token = context.parser().currentToken(); + parser = context.parser(); + } + if (context.parent().isNested()) { // Handle a nested object that doesn't contain an array. Arrays are handled in #parseNonDynamicArray. - if (context.parent().storeArraySource() && context.mappingLookup().isSourceSynthetic() && context.getClonedSource() == false) { + if (context.parent().storeArraySource() && context.canAddIgnoredField()) { Tuple tuple = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( new IgnoredSourceFieldMapper.NameValue( @@ -441,7 +456,9 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr parseObjectOrNested(context.createFlattenContext(currentFieldName)); context.path().add(currentFieldName); } else { - if (context.canAddIgnoredField() && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK) { + if (context.canAddIgnoredField() + && (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK + || context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ENABLED)) { Tuple contextWithSourceToStore = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( @@ -686,11 +703,15 @@ private static void parseNonDynamicArray( // Check if we need to record the array source. This only applies to synthetic source. if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper - && (objectMapper.storeArraySource() || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); + && (objectMapper.storeArraySource() + || context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.DISABLED + || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; + boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper + && context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.DISABLED; boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; - if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext) { + if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext || fieldWithStoredArraySource) { Tuple tuple = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( IgnoredSourceFieldMapper.NameValue.fromContext( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 248369b249007..16ddc6ab067e1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -329,6 +329,10 @@ public final boolean canAddIgnoredField() { return mappingLookup.isSourceSynthetic() && clonedSource == false; } + Mapper.StoreSourceMode storeSourceModeFromIndexSettings() { + return indexSettings().storeSourceMode(); + } + /** * Description on the document being parsed used in error messages. Not * called unless there is an error. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 2c1e01c3cd196..ce44060b3e21f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -10,6 +10,7 @@ import org.apache.lucene.document.FieldType; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.StringLiteralDeduplicator; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -22,6 +23,47 @@ public abstract class Mapper implements ToXContentFragment, Iterable { + static final String STORE_SOURCE_PARAM = "store_source"; + + public enum StoreSourceMode { + DISABLED("disabled"), // No source recording + ARRAYS("arrays"), // Store source for arrays of mapped fields + ENABLED("enabled"); // Store source for both singletons and arrays of mapped fields + + StoreSourceMode(String name) { + this.name = name; + } + + static StoreSourceMode fromString(String input) { + if (input.equals(DISABLED.name)) { + return DISABLED; + } + if (input.equals(ENABLED.name)) { + return ENABLED; + } + if (input.equals(ARRAYS.name)) { + return ARRAYS; + } + throw new IllegalArgumentException("Unknown " + STORE_SOURCE_PARAM + " value [" + input + "]"); + } + + @Override + public String toString() { + return name; + } + + private final String name; + } + + // Setting StoreSourceMode to ENABLED at the index level is equivalent to disabling synthetic source, which is not desired. + // Since the only valid option is to track array source by default, we use a boolean index setting for it. + public static final Setting STORE_ARRAY_SOURCE_SETTING = Setting.boolSetting( + "index.mapping.store_array_source", + false, + Setting.Property.IndexScope, + Setting.Property.ServerlessPublic + ); + public abstract static class Builder { private String leafName; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index dcb5cd1711c8c..82b4c72f6cc39 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -48,6 +48,11 @@ private String getSyntheticSourceWithFieldLimit(CheckedConsumer b.field("my_value", value))); @@ -485,6 +490,108 @@ public void testMixedDisabledEnabledObjects() throws IOException { ); } + public void testIndexStoredArraySourceRootValueArray() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("int_value").field("type", "integer").endObject(); + b.startObject("bool_value").field("type", "boolean").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.array("int_value", new int[] { 30, 20, 10 }); + b.field("bool_value", true); + }); + assertEquals(""" + {"bool_value":true,"int_value":[30,20,10]}""", syntheticSource); + } + + public void testIndexStoredArraySourceRootObjectArray() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("path"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("bool_value").field("type", "boolean").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + b.startObject().field("int_value", 10).endObject(); + b.startObject().field("int_value", 20).endObject(); + b.endArray(); + b.field("bool_value", true); + }); + assertEquals(""" + {"bool_value":true,"path":[{"int_value":10},{"int_value":20}]}""", syntheticSource); + } + + public void testIndexStoredArraySourceNestedValueArray() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("path"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + b.startObject("bool_value").field("type", "boolean").endObject(); + } + b.endObject(); + } + b.endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startObject("path"); + { + b.array("int_value", new int[] { 30, 20, 10 }); + b.field("bool_value", true); + } + b.endObject(); + }); + assertEquals(""" + {"path":{"bool_value":true,"int_value":[30,20,10]}}""", syntheticSource); + } + + public void testIndexStoredArraySourceNestedObjectArray() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("path"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("to"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("bool_value").field("type", "boolean").endObject(); + } + b.endObject(); + } + b.endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startObject("path"); + { + b.startArray("to"); + b.startObject().field("int_value", 10).endObject(); + b.startObject().field("int_value", 20).endObject(); + b.endArray(); + b.field("bool_value", true); + } + b.endObject(); + }); + assertEquals(""" + {"path":{"bool_value":true,"to":[{"int_value":10},{"int_value":20}]}}""", syntheticSource); + } + public void testRootArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); From 717aacb1fff10191720b1e6a8b89b9b0f0344fac Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 30 Aug 2024 17:40:19 +0300 Subject: [PATCH 02/18] bypass for nested --- .../index/mapper/DocumentParser.java | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 3692bda3f9e59..583592e4ab08d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -295,21 +295,6 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio throwOnConcreteValue(context.parent(), currentFieldName, context); } - if (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ENABLED && context.canAddIgnoredField()) { - Tuple tuple = XContentDataHelper.cloneSubContext(context); - context.addIgnoredField( - new IgnoredSourceFieldMapper.NameValue( - context.parent().fullPath(), - context.parent().fullPath().lastIndexOf(context.parent().leafName()), - XContentDataHelper.encodeXContentBuilder(tuple.v2()), - context.doc() - ) - ); - context = tuple.v1(); - token = context.parser().currentToken(); - parser = context.parser(); - } - if (context.parent().isNested()) { // Handle a nested object that doesn't contain an array. Arrays are handled in #parseNonDynamicArray. if (context.parent().storeArraySource() && context.canAddIgnoredField()) { @@ -327,6 +312,19 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio parser = context.parser(); } context = context.createNestedContext((NestedObjectMapper) context.parent()); + } else if (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ENABLED && context.canAddIgnoredField()) { + Tuple tuple = XContentDataHelper.cloneSubContext(context); + context.addIgnoredField( + new IgnoredSourceFieldMapper.NameValue( + context.parent().fullPath(), + context.parent().fullPath().lastIndexOf(context.parent().leafName()), + XContentDataHelper.encodeXContentBuilder(tuple.v2()), + context.doc() + ) + ); + context = tuple.v1(); + token = context.parser().currentToken(); + parser = context.parser(); } // if we are at the end of the previous object, advance From 4d27b0c9e2c0593d0e9be485983e2eae9c473444 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:49:11 +0300 Subject: [PATCH 03/18] Update docs/changelog/112397.yaml --- docs/changelog/112397.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/112397.yaml diff --git a/docs/changelog/112397.yaml b/docs/changelog/112397.yaml new file mode 100644 index 0000000000000..e67478ec69b1c --- /dev/null +++ b/docs/changelog/112397.yaml @@ -0,0 +1,5 @@ +pr: 112397 +summary: Control storing array source with index setting +area: Mapping +type: enhancement +issues: [] From 0d1ec40d8bd589a25c5a43c4d56e5d5667f62401 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 30 Aug 2024 21:57:09 +0300 Subject: [PATCH 04/18] add node feature --- .../indices.create/21_synthetic_source_stored.yml | 14 +++++++------- .../org/elasticsearch/index/mapper/Mapper.java | 3 +++ .../elasticsearch/index/mapper/MapperFeatures.java | 3 ++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 52a8f1b4cf70f..2dcf8c986552b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -356,7 +356,7 @@ stored field under object with store_array_source: --- index param - root arrays: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.source.store_param"] reason: requires tracking ignored source - do: @@ -421,7 +421,7 @@ index param - root arrays: --- index param - dynamic root arrays: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.source.store_param"] reason: requires tracking ignored source - do: @@ -474,7 +474,7 @@ index param - dynamic root arrays: --- index param - object array within array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.source.store_param"] reason: requires tracking ignored source - do: @@ -524,7 +524,7 @@ index param - object array within array: --- index param - no object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.source.store_param"] reason: requires tracking ignored source - do: @@ -569,7 +569,7 @@ index param - no object array: --- index param - field ordering: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.source.store_param"] reason: requires tracking ignored source - do: @@ -620,7 +620,7 @@ index param - field ordering: --- index param - nested arrays: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.source.store_param"] reason: requires tracking ignored source - do: @@ -683,7 +683,7 @@ index param - nested arrays: --- index param - nested object with stored array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.source.store_param"] reason: requires tracking ignored source - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index ce44060b3e21f..86d2229140e6d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.StringLiteralDeduplicator; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.xcontent.ToXContentFragment; @@ -23,6 +24,8 @@ public abstract class Mapper implements ToXContentFragment, Iterable { + public static final NodeFeature STORE_SOURCE_MAPPER_PARAM = new NodeFeature("mapper.source.store_param"); + static final String STORE_SOURCE_PARAM = "store_source"; public enum StoreSourceMode { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 6dce9d6c7b86e..c2d804957ca91 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -33,7 +33,8 @@ public Set getFeatures() { NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS, BooleanFieldMapper.BOOLEAN_DIMENSION, ObjectMapper.SUBOBJECTS_AUTO, - SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX + SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX, + Mapper.STORE_SOURCE_MAPPER_PARAM ); } } From d87257afeeea35bd42a553627d0cd1a1abdea903 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 3 Sep 2024 15:03:52 +0300 Subject: [PATCH 05/18] update names --- .../org/elasticsearch/index/IndexSettings.java | 2 +- .../elasticsearch/index/mapper/DocumentParser.java | 8 ++++---- .../org/elasticsearch/index/mapper/Mapper.java | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 66c020f2567bd..a70fe50b71566 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -935,7 +935,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); storeSourceMode = scopedSettings.get(Mapper.STORE_ARRAY_SOURCE_SETTING) ? Mapper.StoreSourceMode.ARRAYS - : Mapper.StoreSourceMode.DISABLED; + : Mapper.StoreSourceMode.NONE; es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); scopedSettings.addSettingsUpdateConsumer( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 583592e4ab08d..526fc1369972f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -312,7 +312,7 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio parser = context.parser(); } context = context.createNestedContext((NestedObjectMapper) context.parent()); - } else if (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ENABLED && context.canAddIgnoredField()) { + } else if (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ALL && context.canAddIgnoredField()) { Tuple tuple = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( new IgnoredSourceFieldMapper.NameValue( @@ -456,7 +456,7 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr } else { if (context.canAddIgnoredField() && (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK - || context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ENABLED)) { + || context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ALL)) { Tuple contextWithSourceToStore = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( @@ -702,12 +702,12 @@ private static void parseNonDynamicArray( if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper && (objectMapper.storeArraySource() - || context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.DISABLED + || context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.NONE || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper - && context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.DISABLED; + && context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.NONE; boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext || fieldWithStoredArraySource) { Tuple tuple = XContentDataHelper.cloneSubContext(context); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 86d2229140e6d..1458d806ca568 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -29,20 +29,20 @@ public abstract class Mapper implements ToXContentFragment, Iterable { static final String STORE_SOURCE_PARAM = "store_source"; public enum StoreSourceMode { - DISABLED("disabled"), // No source recording - ARRAYS("arrays"), // Store source for arrays of mapped fields - ENABLED("enabled"); // Store source for both singletons and arrays of mapped fields + NONE("none"), // No source recording + ARRAYS("arrays"), // Store source for arrays of mapped fields + ALL("all"); // Store source for both singletons and arrays of mapped fields StoreSourceMode(String name) { this.name = name; } static StoreSourceMode fromString(String input) { - if (input.equals(DISABLED.name)) { - return DISABLED; + if (input.equals(NONE.name)) { + return NONE; } - if (input.equals(ENABLED.name)) { - return ENABLED; + if (input.equals(ALL.name)) { + return ALL; } if (input.equals(ARRAYS.name)) { return ARRAYS; From 524e82a2a98f7f4d1af226666ff888e0db0d1a24 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 3 Sep 2024 15:05:54 +0300 Subject: [PATCH 06/18] add comment --- server/src/main/java/org/elasticsearch/index/mapper/Mapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 1458d806ca568..0da27fb4ce376 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -28,6 +28,7 @@ public abstract class Mapper implements ToXContentFragment, Iterable { static final String STORE_SOURCE_PARAM = "store_source"; + // Only relevant for synthetic source mode. public enum StoreSourceMode { NONE("none"), // No source recording ARRAYS("arrays"), // Store source for arrays of mapped fields From 40e78287c5a0bd50566d6e6727b6e5d3b91c3f0e Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 3 Sep 2024 16:58:36 +0300 Subject: [PATCH 07/18] add randomized test coverage --- ...dVersusLogsIndexModeRandomDataChallengeRestIT.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index 8bd62480f333d..4846657ea0f6e 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -8,9 +8,11 @@ package org.elasticsearch.datastreams.logsdb.qa; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGenerator; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; @@ -36,12 +38,14 @@ */ public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends StandardVersusLogsIndexModeChallengeRestIT { private final ObjectMapper.Subobjects subobjects; + private final boolean storeArraySource; private final DataGenerator dataGenerator; public StandardVersusLogsIndexModeRandomDataChallengeRestIT() { super(); this.subobjects = randomFrom(ObjectMapper.Subobjects.values()); + this.storeArraySource = usually(); var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(randomBoolean()); if (subobjects != ObjectMapper.Subobjects.ENABLED) { @@ -120,6 +124,13 @@ public void contenderMappings(XContentBuilder builder) throws IOException { } } + @Override + public void contenderSettings(Settings.Builder builder) { + if (storeArraySource) { + builder.put(Mapper.STORE_ARRAY_SOURCE_SETTING.getKey(), true); + } + } + @Override protected XContentBuilder generateDocument(final Instant timestamp) throws IOException { var document = XContentFactory.jsonBuilder(); From 3bb31ebd8fde979ba8d4e6be39e01aee5a071301 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 11:04:36 +0300 Subject: [PATCH 08/18] use enum in index setting --- ...sLogsIndexModeRandomDataChallengeRestIT.java | 2 +- .../org/elasticsearch/index/IndexSettings.java | 4 +--- .../org/elasticsearch/index/mapper/Mapper.java | 17 ++++++++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index 4846657ea0f6e..c8f83e972b8c3 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -45,7 +45,7 @@ public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends Standa public StandardVersusLogsIndexModeRandomDataChallengeRestIT() { super(); this.subobjects = randomFrom(ObjectMapper.Subobjects.values()); - this.storeArraySource = usually(); + this.storeArraySource = randomBoolean(); var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(randomBoolean()); if (subobjects != ObjectMapper.Subobjects.ENABLED) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a70fe50b71566..26eea9b1fa88a 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -933,9 +933,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); - storeSourceMode = scopedSettings.get(Mapper.STORE_ARRAY_SOURCE_SETTING) - ? Mapper.StoreSourceMode.ARRAYS - : Mapper.StoreSourceMode.NONE; + storeSourceMode = scopedSettings.get(Mapper.STORE_ARRAY_SOURCE_SETTING); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); scopedSettings.addSettingsUpdateConsumer( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 0da27fb4ce376..c2129e7117426 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -59,11 +59,18 @@ public String toString() { private final String name; } - // Setting StoreSourceMode to ENABLED at the index level is equivalent to disabling synthetic source, which is not desired. - // Since the only valid option is to track array source by default, we use a boolean index setting for it. - public static final Setting STORE_ARRAY_SOURCE_SETTING = Setting.boolSetting( - "index.mapping.store_array_source", - false, + // Only relevant for indexes configured with synthetic source mode. Otherwise, it has no effect. + // Controls the default behavior for storing the source of leaf fields and objects, in singleton or array form. + // Setting to StoreSourceMode.ALL is equivalent to disabling synthetic source, so this is not allowed. + public static final Setting STORE_ARRAY_SOURCE_SETTING = Setting.enumSetting( + StoreSourceMode.class, + "index.mapping.store_source", + StoreSourceMode.NONE, + value -> { + if (value == StoreSourceMode.ALL) { + throw new IllegalArgumentException("index.mapping.store_source can't be set to [" + value.toString() + "]"); + } + }, Setting.Property.IndexScope, Setting.Property.ServerlessPublic ); From 50d5dc54cd395e3dd30556b8efb3781fc83a9655 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 11:32:05 +0300 Subject: [PATCH 09/18] revert change for using index.ALL --- .../21_synthetic_source_stored.yml | 14 ++++++------- .../index/mapper/DocumentParser.java | 21 +++---------------- .../elasticsearch/index/mapper/Mapper.java | 2 +- .../mapper/IgnoredSourceFieldMapperTests.java | 2 +- 4 files changed, 12 insertions(+), 27 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 2dcf8c986552b..1cd5628752e74 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -366,7 +366,7 @@ index param - root arrays: settings: index: mapping: - store_array_source: true + store_source: arrays mappings: _source: mode: synthetic @@ -431,7 +431,7 @@ index param - dynamic root arrays: settings: index: mapping: - store_array_source: true + store_source: arrays mappings: _source: mode: synthetic @@ -484,7 +484,7 @@ index param - object array within array: settings: index: mapping: - store_array_source: true + store_source: arrays mappings: _source: mode: synthetic @@ -534,7 +534,7 @@ index param - no object array: settings: index: mapping: - store_array_source: true + store_source: arrays mappings: _source: mode: synthetic @@ -579,7 +579,7 @@ index param - field ordering: settings: index: mapping: - store_array_source: true + store_source: arrays mappings: _source: mode: synthetic @@ -630,7 +630,7 @@ index param - nested arrays: settings: index: mapping: - store_array_source: true + store_source: arrays mappings: _source: mode: synthetic @@ -693,7 +693,7 @@ index param - nested object with stored array: settings: index: mapping: - store_array_source: true + store_source: arrays mappings: _source: mode: synthetic diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 526fc1369972f..d0a66dc611f0d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -312,19 +312,6 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio parser = context.parser(); } context = context.createNestedContext((NestedObjectMapper) context.parent()); - } else if (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ALL && context.canAddIgnoredField()) { - Tuple tuple = XContentDataHelper.cloneSubContext(context); - context.addIgnoredField( - new IgnoredSourceFieldMapper.NameValue( - context.parent().fullPath(), - context.parent().fullPath().lastIndexOf(context.parent().leafName()), - XContentDataHelper.encodeXContentBuilder(tuple.v2()), - context.doc() - ) - ); - context = tuple.v1(); - token = context.parser().currentToken(); - parser = context.parser(); } // if we are at the end of the previous object, advance @@ -454,9 +441,7 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr parseObjectOrNested(context.createFlattenContext(currentFieldName)); context.path().add(currentFieldName); } else { - if (context.canAddIgnoredField() - && (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK - || context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ALL)) { + if (context.canAddIgnoredField() && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK) { Tuple contextWithSourceToStore = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( @@ -702,12 +687,12 @@ private static void parseNonDynamicArray( if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper && (objectMapper.storeArraySource() - || context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.NONE + || context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ARRAYS || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper - && context.storeSourceModeFromIndexSettings() != Mapper.StoreSourceMode.NONE; + && context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ARRAYS; boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext || fieldWithStoredArraySource) { Tuple tuple = XContentDataHelper.cloneSubContext(context); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index c2129e7117426..1985c8a3e2749 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -38,7 +38,7 @@ public enum StoreSourceMode { this.name = name; } - static StoreSourceMode fromString(String input) { + static StoreSourceMode from(String input) { if (input.equals(NONE.name)) { return NONE; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 82b4c72f6cc39..387798f8f3a8e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -49,7 +49,7 @@ private String getSyntheticSourceWithFieldLimit(CheckedConsumer Date: Wed, 4 Sep 2024 11:34:27 +0300 Subject: [PATCH 10/18] update comment --- server/src/main/java/org/elasticsearch/index/IndexSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 26eea9b1fa88a..917d4f26dd3f0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -795,7 +795,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private final IndexRouting indexRouting; /** - * The default mode for storing source, for all mappers not overriding setting. + * The default mode for storing source, for all mappers not overriding this setting. * This is only relevant for indexes configured with synthetic-source code. */ public Mapper.StoreSourceMode storeSourceMode() { From e65959dfdadcf7a836b8a31605665ee056649c91 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 12:08:05 +0300 Subject: [PATCH 11/18] update randomized test --- .../StandardVersusLogsIndexModeRandomDataChallengeRestIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index c8f83e972b8c3..b0911f26e0c5d 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -127,7 +127,7 @@ public void contenderMappings(XContentBuilder builder) throws IOException { @Override public void contenderSettings(Settings.Builder builder) { if (storeArraySource) { - builder.put(Mapper.STORE_ARRAY_SOURCE_SETTING.getKey(), true); + builder.put(Mapper.STORE_ARRAY_SOURCE_SETTING.getKey(), "arrays"); } } From 2d4ea8e839f27a0da14c0a8e665526cd5676fa73 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 13:22:51 +0300 Subject: [PATCH 12/18] skip nested objects --- .../java/org/elasticsearch/index/mapper/DocumentParser.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index d0a66dc611f0d..47b39a3355cea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -687,7 +687,9 @@ private static void parseNonDynamicArray( if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper && (objectMapper.storeArraySource() - || context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ARRAYS + || (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ARRAYS + && objectMapper instanceof NestedObjectMapper == false) + || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; From 6ce3d8b5d37d0d7832a474a0d1ca95d59b1278cf Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 13:24:25 +0300 Subject: [PATCH 13/18] remove empty line --- .../main/java/org/elasticsearch/index/mapper/DocumentParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 47b39a3355cea..dba6183fa6ad8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -689,7 +689,6 @@ private static void parseNonDynamicArray( && (objectMapper.storeArraySource() || (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false) - || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; From 4ce6096b73d61228f80871ea23721157984a512c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 16:58:57 +0300 Subject: [PATCH 14/18] rename index setting --- ...ogsIndexModeRandomDataChallengeRestIT.java | 2 +- .../21_synthetic_source_stored.yml | 28 +++++++++---------- .../common/settings/IndexScopedSettings.java | 2 +- .../elasticsearch/index/IndexSettings.java | 2 +- .../elasticsearch/index/mapper/Mapper.java | 12 ++++---- .../index/mapper/MapperFeatures.java | 2 +- .../mapper/IgnoredSourceFieldMapperTests.java | 5 +++- 7 files changed, 28 insertions(+), 25 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index b0911f26e0c5d..e1fa1ecdc8097 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -127,7 +127,7 @@ public void contenderMappings(XContentBuilder builder) throws IOException { @Override public void contenderSettings(Settings.Builder builder) { if (storeArraySource) { - builder.put(Mapper.STORE_ARRAY_SOURCE_SETTING.getKey(), "arrays"); + builder.put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), "arrays"); } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 1cd5628752e74..917f0540c4dd4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -356,7 +356,7 @@ stored field under object with store_array_source: --- index param - root arrays: - requires: - cluster_features: ["mapper.source.store_param"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -366,7 +366,7 @@ index param - root arrays: settings: index: mapping: - store_source: arrays + synthetic_source_keep: arrays mappings: _source: mode: synthetic @@ -421,7 +421,7 @@ index param - root arrays: --- index param - dynamic root arrays: - requires: - cluster_features: ["mapper.source.store_param"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -431,7 +431,7 @@ index param - dynamic root arrays: settings: index: mapping: - store_source: arrays + synthetic_source_keep: arrays mappings: _source: mode: synthetic @@ -474,7 +474,7 @@ index param - dynamic root arrays: --- index param - object array within array: - requires: - cluster_features: ["mapper.source.store_param"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -484,7 +484,7 @@ index param - object array within array: settings: index: mapping: - store_source: arrays + synthetic_source_keep: arrays mappings: _source: mode: synthetic @@ -524,7 +524,7 @@ index param - object array within array: --- index param - no object array: - requires: - cluster_features: ["mapper.source.store_param"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -534,7 +534,7 @@ index param - no object array: settings: index: mapping: - store_source: arrays + synthetic_source_keep: arrays mappings: _source: mode: synthetic @@ -569,7 +569,7 @@ index param - no object array: --- index param - field ordering: - requires: - cluster_features: ["mapper.source.store_param"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -579,7 +579,7 @@ index param - field ordering: settings: index: mapping: - store_source: arrays + synthetic_source_keep: arrays mappings: _source: mode: synthetic @@ -620,7 +620,7 @@ index param - field ordering: --- index param - nested arrays: - requires: - cluster_features: ["mapper.source.store_param"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -630,7 +630,7 @@ index param - nested arrays: settings: index: mapping: - store_source: arrays + synthetic_source_keep: arrays mappings: _source: mode: synthetic @@ -683,7 +683,7 @@ index param - nested arrays: --- index param - nested object with stored array: - requires: - cluster_features: ["mapper.source.store_param"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: @@ -693,7 +693,7 @@ index param - nested object with stored array: settings: index: mapping: - store_source: arrays + synthetic_source_keep: arrays mappings: _source: mode: synthetic diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 40f57553f924d..6adf181014023 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -181,7 +181,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, - FieldMapper.STORE_ARRAY_SOURCE_SETTING, + FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 917d4f26dd3f0..1ce7c839aa155 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -933,7 +933,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); - storeSourceMode = scopedSettings.get(Mapper.STORE_ARRAY_SOURCE_SETTING); + storeSourceMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); scopedSettings.addSettingsUpdateConsumer( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 1985c8a3e2749..53efcc27e429e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -24,9 +24,9 @@ public abstract class Mapper implements ToXContentFragment, Iterable { - public static final NodeFeature STORE_SOURCE_MAPPER_PARAM = new NodeFeature("mapper.source.store_param"); + public static final NodeFeature SYNTHETIC_SOURCE_KEEP_FEATURE = new NodeFeature("mapper.synthetic_source_keep"); - static final String STORE_SOURCE_PARAM = "store_source"; + static final String SYNTHETIC_SOURCE_KEEP_PARAM = "synthetic_source_keep"; // Only relevant for synthetic source mode. public enum StoreSourceMode { @@ -48,7 +48,7 @@ static StoreSourceMode from(String input) { if (input.equals(ARRAYS.name)) { return ARRAYS; } - throw new IllegalArgumentException("Unknown " + STORE_SOURCE_PARAM + " value [" + input + "]"); + throw new IllegalArgumentException("Unknown " + SYNTHETIC_SOURCE_KEEP_PARAM + " value [" + input + "]"); } @Override @@ -62,13 +62,13 @@ public String toString() { // Only relevant for indexes configured with synthetic source mode. Otherwise, it has no effect. // Controls the default behavior for storing the source of leaf fields and objects, in singleton or array form. // Setting to StoreSourceMode.ALL is equivalent to disabling synthetic source, so this is not allowed. - public static final Setting STORE_ARRAY_SOURCE_SETTING = Setting.enumSetting( + public static final Setting SYNTHETIC_SOURCE_KEEP_INDEX_SETTING = Setting.enumSetting( StoreSourceMode.class, - "index.mapping.store_source", + "index.mapping.synthetic_source_keep", StoreSourceMode.NONE, value -> { if (value == StoreSourceMode.ALL) { - throw new IllegalArgumentException("index.mapping.store_source can't be set to [" + value.toString() + "]"); + throw new IllegalArgumentException("index.mapping.synthetic_source_keep can't be set to [" + value.toString() + "]"); } }, Setting.Property.IndexScope, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 639c300d12828..2e250726b98ca 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -35,7 +35,7 @@ public Set getFeatures() { ObjectMapper.SUBOBJECTS_AUTO, KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE, SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX, - Mapper.STORE_SOURCE_MAPPER_PARAM + Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 387798f8f3a8e..61c4068cedf4a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -49,7 +49,10 @@ private String getSyntheticSourceWithFieldLimit(CheckedConsumer Date: Wed, 4 Sep 2024 17:02:57 +0300 Subject: [PATCH 15/18] rename enum --- .../org/elasticsearch/index/IndexSettings.java | 4 ++-- .../elasticsearch/index/mapper/DocumentParser.java | 4 ++-- .../index/mapper/DocumentParserContext.java | 2 +- .../org/elasticsearch/index/mapper/Mapper.java | 14 +++++++------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 1ce7c839aa155..2dda4b0d9de46 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -798,11 +798,11 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { * The default mode for storing source, for all mappers not overriding this setting. * This is only relevant for indexes configured with synthetic-source code. */ - public Mapper.StoreSourceMode storeSourceMode() { + public Mapper.SourceKeepMode storeSourceMode() { return storeSourceMode; } - private final Mapper.StoreSourceMode storeSourceMode; + private final Mapper.SourceKeepMode storeSourceMode; /** * Returns the default search fields for this index. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index dba6183fa6ad8..aa3903cf13c62 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -687,13 +687,13 @@ private static void parseNonDynamicArray( if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper && (objectMapper.storeArraySource() - || (context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ARRAYS + || (context.storeSourceModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false) || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper - && context.storeSourceModeFromIndexSettings() == Mapper.StoreSourceMode.ARRAYS; + && context.storeSourceModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS; boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext || fieldWithStoredArraySource) { Tuple tuple = XContentDataHelper.cloneSubContext(context); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 16ddc6ab067e1..edc3cf76ddbfb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -329,7 +329,7 @@ public final boolean canAddIgnoredField() { return mappingLookup.isSourceSynthetic() && clonedSource == false; } - Mapper.StoreSourceMode storeSourceModeFromIndexSettings() { + Mapper.SourceKeepMode storeSourceModeFromIndexSettings() { return indexSettings().storeSourceMode(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 53efcc27e429e..4455decd71553 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -29,16 +29,16 @@ public abstract class Mapper implements ToXContentFragment, Iterable { static final String SYNTHETIC_SOURCE_KEEP_PARAM = "synthetic_source_keep"; // Only relevant for synthetic source mode. - public enum StoreSourceMode { + public enum SourceKeepMode { NONE("none"), // No source recording ARRAYS("arrays"), // Store source for arrays of mapped fields ALL("all"); // Store source for both singletons and arrays of mapped fields - StoreSourceMode(String name) { + SourceKeepMode(String name) { this.name = name; } - static StoreSourceMode from(String input) { + static SourceKeepMode from(String input) { if (input.equals(NONE.name)) { return NONE; } @@ -62,12 +62,12 @@ public String toString() { // Only relevant for indexes configured with synthetic source mode. Otherwise, it has no effect. // Controls the default behavior for storing the source of leaf fields and objects, in singleton or array form. // Setting to StoreSourceMode.ALL is equivalent to disabling synthetic source, so this is not allowed. - public static final Setting SYNTHETIC_SOURCE_KEEP_INDEX_SETTING = Setting.enumSetting( - StoreSourceMode.class, + public static final Setting SYNTHETIC_SOURCE_KEEP_INDEX_SETTING = Setting.enumSetting( + SourceKeepMode.class, "index.mapping.synthetic_source_keep", - StoreSourceMode.NONE, + SourceKeepMode.NONE, value -> { - if (value == StoreSourceMode.ALL) { + if (value == SourceKeepMode.ALL) { throw new IllegalArgumentException("index.mapping.synthetic_source_keep can't be set to [" + value.toString() + "]"); } }, From 950fcfb849964655ef3621cb9cd5e94d2ac89b04 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 17:06:01 +0300 Subject: [PATCH 16/18] rename enum --- ...ndardVersusLogsIndexModeRandomDataChallengeRestIT.java | 6 +++--- .../main/java/org/elasticsearch/index/IndexSettings.java | 8 ++++---- .../elasticsearch/index/mapper/DocumentParserContext.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index e1fa1ecdc8097..ad4302cb04b44 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -38,14 +38,14 @@ */ public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends StandardVersusLogsIndexModeChallengeRestIT { private final ObjectMapper.Subobjects subobjects; - private final boolean storeArraySource; + private final boolean keepArraySource; private final DataGenerator dataGenerator; public StandardVersusLogsIndexModeRandomDataChallengeRestIT() { super(); this.subobjects = randomFrom(ObjectMapper.Subobjects.values()); - this.storeArraySource = randomBoolean(); + this.keepArraySource = randomBoolean(); var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(randomBoolean()); if (subobjects != ObjectMapper.Subobjects.ENABLED) { @@ -126,7 +126,7 @@ public void contenderMappings(XContentBuilder builder) throws IOException { @Override public void contenderSettings(Settings.Builder builder) { - if (storeArraySource) { + if (keepArraySource) { builder.put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), "arrays"); } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 2dda4b0d9de46..509d37fd6077d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -798,11 +798,11 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { * The default mode for storing source, for all mappers not overriding this setting. * This is only relevant for indexes configured with synthetic-source code. */ - public Mapper.SourceKeepMode storeSourceMode() { - return storeSourceMode; + public Mapper.SourceKeepMode sourceKeepMode() { + return sourceKeepMode; } - private final Mapper.SourceKeepMode storeSourceMode; + private final Mapper.SourceKeepMode sourceKeepMode; /** * Returns the default search fields for this index. @@ -933,7 +933,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); - storeSourceMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING); + sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); scopedSettings.addSettingsUpdateConsumer( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index edc3cf76ddbfb..1caa33a56e776 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -330,7 +330,7 @@ public final boolean canAddIgnoredField() { } Mapper.SourceKeepMode storeSourceModeFromIndexSettings() { - return indexSettings().storeSourceMode(); + return indexSettings().sourceKeepMode(); } /** From 4348e3f2b9dbff6292c64515aa6fe2ceedaa18c1 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 17:08:17 +0300 Subject: [PATCH 17/18] rename enum --- .../java/org/elasticsearch/index/mapper/DocumentParser.java | 4 ++-- .../org/elasticsearch/index/mapper/DocumentParserContext.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index aa3903cf13c62..f020b8128bb13 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -687,13 +687,13 @@ private static void parseNonDynamicArray( if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper && (objectMapper.storeArraySource() - || (context.storeSourceModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS + || (context.sourceKeepModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false) || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper - && context.storeSourceModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS; + && context.sourceKeepModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS; boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext || fieldWithStoredArraySource) { Tuple tuple = XContentDataHelper.cloneSubContext(context); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 1caa33a56e776..3a84162b86c27 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -329,7 +329,7 @@ public final boolean canAddIgnoredField() { return mappingLookup.isSourceSynthetic() && clonedSource == false; } - Mapper.SourceKeepMode storeSourceModeFromIndexSettings() { + Mapper.SourceKeepMode sourceKeepModeFromIndexSettings() { return indexSettings().sourceKeepMode(); } From 7d0e84adb97c344e8d2cc45f5f8ad46e79f3963d Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 4 Sep 2024 17:10:27 +0300 Subject: [PATCH 18/18] rename enum --- server/src/main/java/org/elasticsearch/index/mapper/Mapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 4455decd71553..9469ee29ff0a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -61,7 +61,7 @@ public String toString() { // Only relevant for indexes configured with synthetic source mode. Otherwise, it has no effect. // Controls the default behavior for storing the source of leaf fields and objects, in singleton or array form. - // Setting to StoreSourceMode.ALL is equivalent to disabling synthetic source, so this is not allowed. + // Setting to SourceKeepMode.ALL is equivalent to disabling synthetic source, so this is not allowed. public static final Setting SYNTHETIC_SOURCE_KEEP_INDEX_SETTING = Setting.enumSetting( SourceKeepMode.class, "index.mapping.synthetic_source_keep",