Skip automatically preserved request headers when rewriting (#79973) (#79985)

In #79412 we fixed a bug that request headers got dropped when the
request is sent across to a node of different version. The fix is to
restore all existing request headers during the threadContext rewriting.
However, there are headers that are always automatically preserved by
the ThreadContext infrastructure, e.g. x-opaque-id. This causes failures
when the code tries to re-add the x-opaque-id header since it already
exists. An example of this issue is for CCS where the remote cluster is
often on a different version compared to the local cluster.

Resolves: #79412

Author: ywangd

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java

@@ -21,7 +21,6 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
-import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
 import org.elasticsearch.xpack.core.security.authc.support.SecondaryAuthentication;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -167,7 +166,7 @@ public void executeAfterRewritingAuthentication(Consumer<StoredContext> consumer
                 authentication.getLookedUpBy(), version, authentication.getAuthenticationType(),
                 rewriteMetadataForApiKeyRoleDescriptors(version, authentication)));
             existingRequestHeaders.forEach((k, v) -> {
-                if (false == AuthenticationField.AUTHENTICATION_KEY.equals(k)) {
+                if (threadContext.getHeader(k) == null) {
                     threadContext.putHeader(k, v);
                 }
             });

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
 import org.elasticsearch.core.List;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -121,8 +122,14 @@ public void testExecuteAfterRewritingAuthentication() throws IOException {
         final Authentication original = new Authentication(user, authBy, authBy);
         original.writeToContext(threadContext);
         final Map<String, String> requestHeaders = org.elasticsearch.core.Map.of(
-            AuthenticationField.PRIVILEGE_CATEGORY_KEY, randomAlphaOfLengthBetween(3, 10),
-            randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)
+            AuthenticationField.PRIVILEGE_CATEGORY_KEY,
+            randomAlphaOfLengthBetween(3, 10),
+            randomAlphaOfLengthBetween(3, 8),
+            randomAlphaOfLengthBetween(3, 8),
+            Task.X_OPAQUE_ID,
+            randomAlphaOfLength(10),
+            Task.TRACE_ID,
+            randomAlphaOfLength(20)
         );
         threadContext.putHeader(requestHeaders);