diff --git a/muted-tests.yml b/muted-tests.yml index db47281f04d11..1c734b42f31ac 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -334,9 +334,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=ml/3rd_party_deployment/Test start deployment with low priority and max allocations more than one} issue: https://github.com/elastic/elasticsearch/issues/140610 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=security/authz_api_keys/30_field_level_security_synthetic_source/Fields with copy_to, field level security and synthetic source} - issue: https://github.com/elastic/elasticsearch/issues/142341 # Examples: # diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 812192d79cdce..eeb4fe9b868e7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -261,6 +261,9 @@ public record MappedNameValue(NameValue nameValue, XContentType type, Map> result = XContentHelper.convertToMap(bytes, true); - Map transformedSource = filter(result.v2(), filter, 0); - XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource); - visitor.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xContentBuilder))); + filterSourceField(fieldInfo, value); } else if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) { - // for _ignored_source, parse, filter out the field and its contents, and serialize back downstream - IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(value); - Map transformedField = filter(mappedNameValue.map(), filter, 0); - if (transformedField.isEmpty() == false) { - // The unfiltered map contains at least one element, the field name with its value. If the field contains - // an object or an array, the value of the first element is a map or a list, respectively. Otherwise, - // it's a single leaf value, e.g. a string or a number. - var topValue = mappedNameValue.map().values().iterator().next(); - if (topValue instanceof Map || topValue instanceof List) { - // The field contains an object or an array, reconstruct it from the transformed map in case - // any subfield has been filtered out. - visitor.binaryField(fieldInfo, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue, transformedField)); - } else { - // The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value. - visitor.binaryField(fieldInfo, value); - } - } + filterIgnoredSourceField(fieldInfo, value); + } else { + visitor.binaryField(fieldInfo, value); + } + } + + /** + * Filters the _source field by parsing the stored value, removing fields that should not be + * visible according to FLS rules, and serializing the filtered result back downstream. + */ + private void filterSourceField(FieldInfo fieldInfo, byte[] value) throws IOException { + BytesReference bytes = new BytesArray(value); + Tuple> result = XContentHelper.convertToMap(bytes, true); + Map transformedSource = filter(result.v2(), filter, 0); + XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource); + visitor.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xContentBuilder))); + } + + /** + * Filters the _ignored_source field by parsing the stored value, removing fields that should not be + * visible according to FLS rules, and serializing the filtered result back downstream. Entries with + * no data (VOID entries from copy_to targets) are skipped entirely. + */ + private void filterIgnoredSourceField(FieldInfo fieldInfo, byte[] value) throws IOException { + IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(value); + // NOTE: decodeAsMap returns null for VOID entries, which are placeholders for copy_to targets. + // These entries have no actual data (the value lives in the copy_to source field), so we skip them. + if (mappedNameValue == null) { + return; + } + Map transformedField = filter(mappedNameValue.map(), filter, 0); + if (transformedField.isEmpty()) { + return; + } + // The unfiltered map contains at least one element, the field name with its value. If the field contains + // an object or an array, the value of the first element is a map or a list, respectively. Otherwise, + // it's a single leaf value, e.g. a string or a number. + var topValue = mappedNameValue.map().values().iterator().next(); + if (topValue instanceof Map || topValue instanceof List) { + // The field contains an object or an array, reconstruct it from the transformed map in case + // any subfield has been filtered out. + visitor.binaryField(fieldInfo, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue, transformedField)); } else { + // The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value. visitor.binaryField(fieldInfo, value); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index eb650f57ea3fd..f3b060d3ed24d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -1558,6 +1558,42 @@ public void testProducesStoredFieldsReader() throws Exception { IOUtils.close(ir, iw, dir); } + public void testSyntheticSourceWithCopyToAndFLS() throws Exception { + final DocumentMapper mapper = createMapperService( + Settings.builder().put("index.mapping.source.mode", "synthetic").build(), + mapping(b -> { + b.startObject("user").field("type", "keyword").field("copy_to", "catch_all").endObject(); + b.startObject("domain").field("type", "keyword").field("copy_to", "catch_all").endObject(); + b.startObject("catch_all").field("type", "text").endObject(); + }) + ).documentMapper(); + + try (Directory directory = newDirectory()) { + final IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig()); + final ParsedDocument doc = mapper.parse(source(b -> { + b.field("user", "darth.vader"); + b.field("domain", "empire.gov"); + })); + writer.addDocuments(doc.docs()); + writer.commit(); + + final Automaton automaton = Automatons.patterns(Arrays.asList("user", "domain", IgnoredSourceFieldMapper.NAME)); + try ( + DirectoryReader reader = FieldSubsetReader.wrap( + DirectoryReader.open(writer), + new CharacterRunAutomaton(automaton), + (fieldName) -> false + ); + ) { + assertEquals( + "{\"domain\":\"empire.gov\",\"user\":\"darth.vader\"}", + syntheticSource(mapper, reader, doc.docs().size() - 1) + ); + } + IOUtils.close(writer, directory); + } + } + private static final String DOC_TEST_ITEM = """ { "field_text" : "text", diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml index c038c33f68f5a..3f16d80ad4f91 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml @@ -624,3 +624,135 @@ Field with ignored_malformed: - is_false: "hits.hits.0._source.secret" - match: { hits.hits.1._source.name: B } - is_false: "hits.hits.1._source.secret" + +--- +Fields with copy_to, field level security and synthetic source: + - do: + indices.create: + index: test-fls-copy-to-synthetic + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + user: + type: keyword + copy_to: catch_all + domain: + type: keyword + copy_to: catch_all + catch_all: + type: keyword + + - do: + bulk: + index: test-fls-copy-to-synthetic + refresh: true + body: + - '{"create": { }}' + - '{"user": "darth.vader", "domain": "empire.gov"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "test-fls-copy-to-synthetic" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "catch_all" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser + - do: + search: + index: test-fls-copy-to-synthetic + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.user: "darth.vader" } + - match: { hits.hits.0._source.domain: "empire.gov" } + - is_false: "hits.hits.0._source.catch_all" + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: test-fls-copy-to-synthetic + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.user: "darth.vader" } + - match: { hits.hits.0._source.domain: "empire.gov" } + - is_false: "hits.hits.0._source.catch_all" + +--- +Fields with copy_to and skip_ignored_source_read workaround: + # Setting skip_ignored_source_read skips _ignored_source during synthetic source reconstruction + - do: + indices.create: + index: test-fls-copy-to-skip-ignored + body: + settings: + index: + mapping.source.mode: synthetic + mapping.synthetic_source.skip_ignored_source_read: true + mappings: + properties: + user: + type: keyword + copy_to: catch_all + domain: + type: keyword + copy_to: catch_all + catch_all: + type: keyword + + - do: + bulk: + index: test-fls-copy-to-skip-ignored + refresh: true + body: + - '{"create": { }}' + - '{"user": "luke.skywalker", "domain": "tatooine.org"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls-skip-ignored" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "test-fls-copy-to-skip-ignored" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "catch_all" ] + - match: { name: "test-fls-skip-ignored" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser: search succeeds but source may be incomplete since _ignored_source is skipped + - do: + search: + index: test-fls-copy-to-skip-ignored + - match: { hits.total.value: 1 } + + # With FLS API Key: search succeeds without crash since _ignored_source VOID entries are not loaded + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: test-fls-copy-to-skip-ignored + - match: { hits.total.value: 1 }