[9.1] fix(mapper): handle VOID entries in _ignored_source with FLS (#141506)

muted-tests.yml

@@ -334,9 +334,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/3rd_party_deployment/Test start deployment with low priority and max allocations more than one}
   issue: https://github.com/elastic/elasticsearch/issues/140610
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=security/authz_api_keys/30_field_level_security_synthetic_source/Fields with copy_to, field level security and synthetic source}
-  issue: https://github.com/elastic/elasticsearch/issues/142341
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java

@@ -261,6 +261,9 @@ public record MappedNameValue(NameValue nameValue, XContentType type, Map<String
     public static MappedNameValue decodeAsMap(byte[] value) throws IOException {
         BytesRef bytes = new BytesRef(value);
         IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(bytes);
+        if (nameValue.hasValue() == false) {
+            return null;
+        }
         XContentBuilder xContentBuilder = XContentBuilder.builder(XContentDataHelper.getXContentType(nameValue.value()).xContent());
         xContentBuilder.startObject().field(nameValue.name());
         XContentDataHelper.decodeAndWrite(xContentBuilder, nameValue.value());

server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java

@@ -376,6 +376,19 @@ public void testEncodeArrayToMapAndDecode() throws IOException {
         assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue, mappedNameValue.map()));
     }
 
+    public void testDecodeAsMapReturnsNullForVoidEntry() throws IOException {
+        final IgnoredSourceFieldMapper.NameValue voidNameValue = new IgnoredSourceFieldMapper.NameValue(
+            "target_field",
+            0,
+            XContentDataHelper.voidValue(),
+            null
+        );
+        final IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(
+            IgnoredSourceFieldMapper.encode(voidNameValue)
+        );
+        assertNull(mappedNameValue);
+    }
+
     public void testMultipleIgnoredFieldsRootObject() throws IOException {
         boolean booleanValue = randomBoolean();
         int intValue = randomInt();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java

@@ -396,31 +396,52 @@ class FieldSubsetStoredFieldVisitor extends StoredFieldVisitor {
         @Override
         public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
             if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
-                // for _source, parse, filter out the fields we care about, and serialize back downstream
-                BytesReference bytes = new BytesArray(value);
-                Tuple<XContentType, Map<String, Object>> result = XContentHelper.convertToMap(bytes, true);
-                Map<String, Object> transformedSource = filter(result.v2(), filter, 0);
-                XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource);
-                visitor.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xContentBuilder)));
+                filterSourceField(fieldInfo, value);
             } else if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) {
-                // for _ignored_source, parse, filter out the field and its contents, and serialize back downstream
-                IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(value);
-                Map<String, Object> transformedField = filter(mappedNameValue.map(), filter, 0);
-                if (transformedField.isEmpty() == false) {
-                    // The unfiltered map contains at least one element, the field name with its value. If the field contains
-                    // an object or an array, the value of the first element is a map or a list, respectively. Otherwise,
-                    // it's a single leaf value, e.g. a string or a number.
-                    var topValue = mappedNameValue.map().values().iterator().next();
-                    if (topValue instanceof Map<?, ?> || topValue instanceof List<?>) {
-                        // The field contains an object or an array, reconstruct it from the transformed map in case
-                        // any subfield has been filtered out.
-                        visitor.binaryField(fieldInfo, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue, transformedField));
-                    } else {
-                        // The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value.
-                        visitor.binaryField(fieldInfo, value);
-                    }
-                }
+                filterIgnoredSourceField(fieldInfo, value);
+            } else {
+                visitor.binaryField(fieldInfo, value);
+            }
+        }
+
+        /**
+         * Filters the _source field by parsing the stored value, removing fields that should not be
+         * visible according to FLS rules, and serializing the filtered result back downstream.
+         */
+        private void filterSourceField(FieldInfo fieldInfo, byte[] value) throws IOException {
+            BytesReference bytes = new BytesArray(value);
+            Tuple<XContentType, Map<String, Object>> result = XContentHelper.convertToMap(bytes, true);
+            Map<String, Object> transformedSource = filter(result.v2(), filter, 0);
+            XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource);
+            visitor.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xContentBuilder)));
+        }
+
+        /**
+         * Filters the _ignored_source field by parsing the stored value, removing fields that should not be
+         * visible according to FLS rules, and serializing the filtered result back downstream. Entries with
+         * no data (VOID entries from copy_to targets) are skipped entirely.
+         */
+        private void filterIgnoredSourceField(FieldInfo fieldInfo, byte[] value) throws IOException {
+            IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(value);
+            // NOTE: decodeAsMap returns null for VOID entries, which are placeholders for copy_to targets.
+            // These entries have no actual data (the value lives in the copy_to source field), so we skip them.
+            if (mappedNameValue == null) {
+                return;
+            }
+            Map<String, Object> transformedField = filter(mappedNameValue.map(), filter, 0);
+            if (transformedField.isEmpty()) {
+                return;
+            }
+            // The unfiltered map contains at least one element, the field name with its value. If the field contains
+            // an object or an array, the value of the first element is a map or a list, respectively. Otherwise,
+            // it's a single leaf value, e.g. a string or a number.
+            var topValue = mappedNameValue.map().values().iterator().next();
+            if (topValue instanceof Map<?, ?> || topValue instanceof List<?>) {
+                // The field contains an object or an array, reconstruct it from the transformed map in case
+                // any subfield has been filtered out.
+                visitor.binaryField(fieldInfo, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue, transformedField));
             } else {
+                // The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value.
                 visitor.binaryField(fieldInfo, value);
             }
         }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java

@@ -1558,6 +1558,42 @@ public void testProducesStoredFieldsReader() throws Exception {
         IOUtils.close(ir, iw, dir);
     }
 
+    public void testSyntheticSourceWithCopyToAndFLS() throws Exception {
+        final DocumentMapper mapper = createMapperService(
+            Settings.builder().put("index.mapping.source.mode", "synthetic").build(),
+            mapping(b -> {
+                b.startObject("user").field("type", "keyword").field("copy_to", "catch_all").endObject();
+                b.startObject("domain").field("type", "keyword").field("copy_to", "catch_all").endObject();
+                b.startObject("catch_all").field("type", "text").endObject();
+            })
+        ).documentMapper();
+
+        try (Directory directory = newDirectory()) {
+            final IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());
+            final ParsedDocument doc = mapper.parse(source(b -> {
+                b.field("user", "darth.vader");
+                b.field("domain", "empire.gov");
+            }));
+            writer.addDocuments(doc.docs());
+            writer.commit();
+
+            final Automaton automaton = Automatons.patterns(Arrays.asList("user", "domain", IgnoredSourceFieldMapper.NAME));
+            try (
+                DirectoryReader reader = FieldSubsetReader.wrap(
+                    DirectoryReader.open(writer),
+                    new CharacterRunAutomaton(automaton),
+                    (fieldName) -> false
+                );
+            ) {
+                assertEquals(
+                    "{\"domain\":\"empire.gov\",\"user\":\"darth.vader\"}",
+                    syntheticSource(mapper, reader, doc.docs().size() - 1)
+                );
+            }
+            IOUtils.close(writer, directory);
+        }
+    }
+
     private static final String DOC_TEST_ITEM = """
         {
           "field_text" : "text",

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml

@@ -624,3 +624,135 @@ Field with ignored_malformed:
   - is_false: "hits.hits.0._source.secret"
   - match: { hits.hits.1._source.name: B }
   - is_false: "hits.hits.1._source.secret"
+
+---
+Fields with copy_to, field level security and synthetic source:
+  - do:
+      indices.create:
+        index: test-fls-copy-to-synthetic
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+          mappings:
+            properties:
+              user:
+                type: keyword
+                copy_to: catch_all
+              domain:
+                type: keyword
+                copy_to: catch_all
+              catch_all:
+                type: keyword
+
+  - do:
+      bulk:
+        index: test-fls-copy-to-synthetic
+        refresh: true
+        body:
+          - '{"create": { }}'
+          - '{"user": "darth.vader", "domain": "empire.gov"}'
+  - match: { errors: false }
+
+  - do:
+      security.create_api_key:
+        body:
+          name: "test-fls"
+          expiration: "1d"
+          role_descriptors:
+            index_access:
+              indices:
+                - names: [ "test-fls-copy-to-synthetic" ]
+                  privileges: [ "read" ]
+                  field_security:
+                    grant: [ "*" ]
+                    except: [ "catch_all" ]
+  - match: { name: "test-fls" }
+  - is_true: id
+  - set:
+      id: api_key_id
+      encoded: credentials
+
+  # With superuser
+  - do:
+      search:
+        index: test-fls-copy-to-synthetic
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._source.user: "darth.vader" }
+  - match: { hits.hits.0._source.domain: "empire.gov" }
+  - is_false: "hits.hits.0._source.catch_all"
+
+  # With FLS API Key
+  - do:
+      headers:
+        Authorization: "ApiKey ${credentials}"
+      search:
+        index: test-fls-copy-to-synthetic
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._source.user: "darth.vader" }
+  - match: { hits.hits.0._source.domain: "empire.gov" }
+  - is_false: "hits.hits.0._source.catch_all"
+
+---
+Fields with copy_to and skip_ignored_source_read workaround:
+  # Setting skip_ignored_source_read skips _ignored_source during synthetic source reconstruction
+  - do:
+      indices.create:
+        index: test-fls-copy-to-skip-ignored
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+              mapping.synthetic_source.skip_ignored_source_read: true
+          mappings:
+            properties:
+              user:
+                type: keyword
+                copy_to: catch_all
+              domain:
+                type: keyword
+                copy_to: catch_all
+              catch_all:
+                type: keyword
+
+  - do:
+      bulk:
+        index: test-fls-copy-to-skip-ignored
+        refresh: true
+        body:
+          - '{"create": { }}'
+          - '{"user": "luke.skywalker", "domain": "tatooine.org"}'
+  - match: { errors: false }
+
+  - do:
+      security.create_api_key:
+        body:
+          name: "test-fls-skip-ignored"
+          expiration: "1d"
+          role_descriptors:
+            index_access:
+              indices:
+                - names: [ "test-fls-copy-to-skip-ignored" ]
+                  privileges: [ "read" ]
+                  field_security:
+                    grant: [ "*" ]
+                    except: [ "catch_all" ]
+  - match: { name: "test-fls-skip-ignored" }
+  - is_true: id
+  - set:
+      id: api_key_id
+      encoded: credentials
+
+  # With superuser: search succeeds but source may be incomplete since _ignored_source is skipped
+  - do:
+      search:
+        index: test-fls-copy-to-skip-ignored
+  - match: { hits.total.value: 1 }
+
+  # With FLS API Key: search succeeds without crash since _ignored_source VOID entries are not loaded
+  - do:
+      headers:
+        Authorization: "ApiKey ${credentials}"
+      search:
+        index: test-fls-copy-to-skip-ignored
+  - match: { hits.total.value: 1 }