From 1173919790264968fe99bfe9de92b2f4aab0835c Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Tue, 30 Sep 2025 10:19:19 +0200 Subject: [PATCH 01/10] Send cross cluster api key signature as headers --- .../add_cross_cluster_api_key_signature.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../authc/CrossClusterAccessSubjectInfo.java | 10 - .../CrossClusterAccessSubjectInfoTests.java | 15 +- ...ClusterSecurityBWCToRCS2ClusterRestIT.java | 5 + ...erSecurityCrossClusterApiKeySigningIT.java | 230 ++++++++++++++++++ ...lusterSecurityFcActionAuthorizationIT.java | 2 +- .../javaRestTest/resources/signing/root.crt | 20 ++ .../resources/signing/signing.crt | 19 ++ .../resources/signing/signing.key | 27 ++ ...usterApiKeySignatureManagerIntegTests.java | 11 +- ...ossClusterAccessAuthenticationService.java | 36 ++- .../authc/CrossClusterAccessHeaders.java | 60 ++++- .../CrossClusterAccessSecurityExtension.java | 10 +- ...ossClusterAccessServerTransportFilter.java | 24 +- ...rossClusterAccessTransportInterceptor.java | 35 ++- .../CrossClusterApiKeySignatureManager.java | 51 ++-- .../transport/X509CertificateSignature.java | 17 +- ...AccessAuthenticationServiceIntegTests.java | 8 +- ...usterAccessAuthenticationServiceTests.java | 126 +++++++++- .../authc/CrossClusterAccessHeadersTests.java | 51 +++- ...ossClusterApiKeySignatureManagerTests.java | 19 +- ...curityServerTransportInterceptorTests.java | 89 ++++++- 23 files changed, 761 insertions(+), 107 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCrossClusterApiKeySigningIT.java create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/root.crt create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.crt create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.key diff --git a/server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv b/server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv new file mode 100644 index 0000000000000..71ac710f9bfe1 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv @@ -0,0 +1 @@ +9187000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index dfb000bd20c3d..1bc1e329153aa 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -esql_plan_with_no_columns,9186000 +add_cross_cluster_api_key_signature,9187000 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java index 82bfc4b4a0dd4..0f2c95e52e18c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java @@ -76,16 +76,6 @@ public void writeToContext(final ThreadContext ctx) throws IOException { ctx.putHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY, encode()); } - public static CrossClusterAccessSubjectInfo readFromContext(final ThreadContext ctx) throws IOException { - final String header = ctx.getHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY); - if (header == null) { - throw new IllegalArgumentException( - "cross cluster access header [" + CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY + "] is required" - ); - } - return decode(header); - } - public Authentication getAuthentication() { return authentication; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfoTests.java index ec20e6e5fa2ff..cd1c8bb174a3e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfoTests.java @@ -48,7 +48,9 @@ public void testWriteReadContextRoundtrip() throws IOException { ); expectedCrossClusterAccessSubjectInfo.writeToContext(ctx); - final CrossClusterAccessSubjectInfo actual = CrossClusterAccessSubjectInfo.readFromContext(ctx); + final CrossClusterAccessSubjectInfo actual = CrossClusterAccessSubjectInfo.decode( + ctx.getHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY) + ); assertThat(actual.getAuthentication(), equalTo(expectedCrossClusterAccessSubjectInfo.getAuthentication())); final List> roleDescriptorsList = new ArrayList<>(); @@ -70,17 +72,6 @@ public void testRoleDescriptorsBytesToRoleDescriptors() throws IOException { assertThat(actualRoleDescriptors, equalTo(expectedRoleDescriptors)); } - public void testThrowsOnMissingEntry() { - var actual = expectThrows( - IllegalArgumentException.class, - () -> CrossClusterAccessSubjectInfo.readFromContext(new ThreadContext(Settings.EMPTY)) - ); - assertThat( - actual.getMessage(), - equalTo("cross cluster access header [" + CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY + "] is required") - ); - } - public void testCleanWithValidationForApiKeys() { final Map initialMetadata = newHashMapWithRandomMetadata(); final AuthenticationTestHelper.AuthenticationTestBuilder builder = AuthenticationTestHelper.builder() diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS2ClusterRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS2ClusterRestIT.java index c53cca6054fce..05d0218b65292 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS2ClusterRestIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS2ClusterRestIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.junit.Before; import org.junit.ClassRule; import org.junit.rules.RuleChain; @@ -54,6 +55,10 @@ public class RemoteClusterSecurityBWCToRCS2ClusterRestIT extends AbstractRemoteC .apply(commonClusterConfig) .setting("xpack.security.remote_cluster_client.ssl.enabled", "true") .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .configFile("signing.crt", Resource.fromClasspath("signing/signing.crt")) + .setting("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt") + .configFile("signing.key", Resource.fromClasspath("signing/signing.key")) + .setting("cluster.remote.my_remote_cluster.signing.key", "signing.key") .keystore("cluster.remote.my_remote_cluster.credentials", () -> { if (API_KEY_MAP_REF.get() == null) { final Map apiKeyMap = createCrossClusterAccessApiKey(""" diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCrossClusterApiKeySigningIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCrossClusterApiKeySigningIT.java new file mode 100644 index 0000000000000..446a6c546b509 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCrossClusterApiKeySigningIT.java @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import io.netty.handler.codec.http.HttpMethod; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Strings; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchResponseUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRemoteClusterSecurityTestCase { + + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .apply(commonClusterConfig) + .setting("remote_cluster_server.enabled", "true") + .setting("remote_cluster.port", "0") + .setting("xpack.security.remote_cluster_server.ssl.enabled", "true") + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .configFile("signing_ca.crt", Resource.fromClasspath("signing/root.crt")) + .setting("cluster.remote.signing.certificate_authorities", "signing_ca.crt") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .apply(commonClusterConfig) + .setting("xpack.security.remote_cluster_client.ssl.enabled", "true") + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .configFile("signing.crt", Resource.fromClasspath("signing/signing.crt")) + .setting("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt") + .configFile("signing.key", Resource.fromClasspath("signing/signing.key")) + .setting("cluster.remote.my_remote_cluster.signing.key", "signing.key") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["index*", "not_found_index"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + .keystore("cluster.remote.invalid_remote.credentials", randomEncodedApiKey()) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); + + public void testCrossClusterSearchWithCrossClusterApiKeySigning() throws Exception { + indexTestData(); + assertCrossClusterSearchSuccessfulWithResult(); + + // Change the CA to something that doesn't trust the signing cert + updateClusterSettingsFulfillingCluster( + Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build() + ); + assertCrossClusterAuthFail(); + + // Update settings on query cluster to ignore unavailable remotes + updateClusterSettings(Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build()); + + assertCrossClusterSearchSuccessfulWithoutResult(); + + // TODO add test for certificate identity configured for API key but no signature provided (should 401) + + // TODO add test for certificate identity not configured for API key but signature provided (should 200) + + // TODO add test for certificate identity not configured for API key but wrong signature provided (should 401) + + // TODO add test for certificate identity regex matching (should 200) + } + + private void assertCrossClusterAuthFail() { + var responseException = assertThrows(ResponseException.class, () -> simpleCrossClusterSearch(randomBoolean())); + assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(401)); + assertThat(responseException.getMessage(), containsString("Failed to verify cross cluster api key signature certificate from [(")); + } + + private void assertCrossClusterSearchSuccessfulWithoutResult() throws IOException { + boolean alsoSearchLocally = randomBoolean(); + final Response response = simpleCrossClusterSearch(alsoSearchLocally); + assertOK(response); + } + + private void assertCrossClusterSearchSuccessfulWithResult() throws IOException { + boolean alsoSearchLocally = randomBoolean(); + final Response response = simpleCrossClusterSearch(alsoSearchLocally); + assertOK(response); + final SearchResponse searchResponse; + try (var parser = responseAsParser(response)) { + searchResponse = SearchResponseUtils.parseSearchResponse(parser); + } + try { + final List actualIndices = Arrays.stream(searchResponse.getHits().getHits()) + .map(SearchHit::getIndex) + .collect(Collectors.toList()); + if (alsoSearchLocally) { + assertThat(actualIndices, containsInAnyOrder("index1", "local_index")); + } else { + assertThat(actualIndices, containsInAnyOrder("index1")); + } + } finally { + searchResponse.decRef(); + } + } + + private Response simpleCrossClusterSearch(boolean alsoSearchLocally) throws IOException { + final var searchRequest = new Request( + "GET", + String.format( + Locale.ROOT, + "/%s%s:%s/_search?ccs_minimize_roundtrips=%s", + alsoSearchLocally ? "local_index," : "", + randomFrom("my_remote_cluster", "*", "my_remote_*"), + randomFrom("index1", "*"), + randomBoolean() + ) + ); + return performRequestWithRemoteAccessUser(searchRequest); + } + + private void indexTestData() throws Exception { + configureRemoteCluster(); + + // Fulfilling cluster + { + // Index some documents, so we can attempt to search them from the querying cluster + final Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(Strings.format(""" + { "index": { "_index": "index1" } } + { "foo": "bar" } + { "index": { "_index": "index2" } } + { "bar": "foo" } + { "index": { "_index": "prefixed_index" } } + { "baz": "fee" }\n""")); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + } + + // Query cluster + { + // Index some documents, to use them in a mixed-cluster search + final var indexDocRequest = new Request("POST", "/local_index/_doc?refresh=true"); + indexDocRequest.setJsonEntity("{\"local_foo\": \"local_bar\"}"); + assertOK(client().performRequest(indexDocRequest)); + + // Create user role with privileges for remote and local indices + final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); + putRoleRequest.setJsonEntity(""" + { + "description": "role with privileges for remote and local indices", + "cluster": ["manage_own_api_key"], + "indices": [ + { + "names": ["local_index"], + "privileges": ["read"] + } + ], + "remote_indices": [ + { + "names": ["index1", "not_found_index", "prefixed_index"], + "privileges": ["read", "read_cross_cluster"], + "clusters": ["my_remote_cluster"] + } + ] + }"""); + assertOK(adminClient().performRequest(putRoleRequest)); + final var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER); + putUserRequest.setJsonEntity(""" + { + "password": "x-pack-test-password", + "roles" : ["remote_search"] + }"""); + assertOK(adminClient().performRequest(putUserRequest)); + } + } + + private void updateClusterSettingsFulfillingCluster(Settings settings) throws IOException { + final var request = newXContentRequest(HttpMethod.PUT, "/_cluster/settings", (builder, params) -> { + builder.startObject("persistent"); + settings.toXContent(builder, params); + return builder.endObject(); + }); + + performRequestWithAdminUser(fulfillingClusterClient, request); + } + + private Response performRequestWithRemoteAccessUser(final Request request) throws IOException { + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue(REMOTE_SEARCH_USER, PASS))); + return client().performRequest(request); + } +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java index 2092bf921a0a6..ac150e3f60ec3 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java @@ -619,7 +619,7 @@ private static MockTransportService startTransport( action, SystemUser.crossClusterAccessSubjectInfo(TransportVersion.current(), nodeName) ) - ).writeToContext(threadContext); + ).writeToContext(threadContext, null); connection.sendRequest(requestId, action, request, options); } }); diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/root.crt b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/root.crt new file mode 100644 index 0000000000000..17e837cc70c4d --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/root.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSzCCAjOgAwIBAgIUPw9V/LIrB5Y+Krhqp1mXhK/BQDIwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwIBcNMjUwOTE4MDczMzQ2WhgPMjA1MzAyMDMwNzMzNDZaMDQxMjAw +BgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuQkSVLDqxWT83K+gfljq +WWL5KQxiN/7XZ8ug6e5b+kY0MZQnaAUWNaj5RFgSMDB+2N6EWJMk5cmDXK7NB/xq +gTbC/o3o7B9AZMTpu5Wbj8chRBCTTirRaIh79/VdLWDHriIgxGBLdMm/A2b3IW6H +YeUGUdOszEjytCjslrrktnIMIQHlQQ5o/fSPslCunsm7P+rDyf3GjapflKtpcrIV +cZKExaaCaqJtQYn66JCyr/ZFmjiTRPjPBcFx2SqAvkPXSK4SghvhKX69K+qDQWjF +rPx9BUWgLnE00bJ27CCSCRzZ3dTgcZ86ou/2mOJqqpeCMacJGWnn7Cu1P3+LcgjT +cQIDAQABo1MwUTAdBgNVHQ4EFgQUlL1P7M7/YCDULUeMUHPxDMtHT9swHwYDVR0j +BBgwFoAUlL1P7M7/YCDULUeMUHPxDMtHT9swDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAQEATCKw10zkCI21nuNppQZKFbHf/m3IZR9mZYYU0tKBSIy7 +KoCTHZUTadbJuDzJ8eDRiqnUuXHUXNijykEphvfpckNDhb6ty5g707kET3EYDfkh +S1EKet2clM9DRqqcmFt3cyOmLJE3we7NjrNOuKNiwuXbGrqTTqNkqiiB3gWYOpSM +uwtCz1Syyl4y5sjocedkikqaeIKtl2htN3tEYd0BfLNVo5hN/syP8WDT6FdpCDpY +lZ2nqT622KDuusORCMTiC1qgUVR3RghPHy55Jq6Qq1+a1//E/Q9OfCs98JeUnoSp +W/q1hUVlSN0Edsn1T5LehGMjiH3UVszWvEThUqNHuA== +-----END CERTIFICATE----- diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.crt b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.crt new file mode 100644 index 0000000000000..ad9a4e6f6400d --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg2gAwIBAgIVAIXaEONwJyih5d7KhnPYfqW5eNgKMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMCAXDTI1MDkxODA5NDQwMVoYDzIwNTMwMjAzMDk0NDAxWjATMREw +DwYDVQQDEwhpbnN0YW5jZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AJCOS/v9I34GaRD4BoMpIa/zhL1TBc1tqIDFKDm4Y/g4a+KUMJlaqHFRZkNI35LF +TkgMkRJUrWEunXJ2wf2A2s+7xWykSoa+nJ2qghwDgBtoSBWSm6I2Hi9n400Mmde/ +xg912NzlfLJZH3la3/w3u7ENUY3GTNLeE5s5CpZAcOk+KQ/2/1Y7TgKPxyhbNtRA +2whWD862pnJypskQ9UGgB3Zq5h+2llQ2sB367pE77DyvXReKLHfCtA3lmTob6pLm +fK2cIBEJDwkFaAgrcWH5MwMkn+4v/Xw1PjAI4AVMOge+Rxt6waWxqQIJvOoyccXY +Vdvo8swUAjMPnR6E/5+bwykCAwEAAaNNMEswHQYDVR0OBBYEFH1XQX26JBIwvu95 +xPhSCqOFrz9IMB8GA1UdIwQYMBaAFJS9T+zO/2Ag1C1HjFBz8QzLR0/bMAkGA1Ud +EwQCMAAwDQYJKoZIhvcNAQELBQADggEBAIF/LkOYm52Q+buBqGS380HWkNitTLG2 +8qtICtXtLYd9673+c3RNIrW2CGFq3Z3TJ60FNvVT1z6NKiR8ZPUeqN+Avq5qN+dB +u9SPRFOrszlD6+2ZkNaZyRs2w6NQa6zBZWs0Zp3+ouu4fUEdsa/UmKud6njLaAGA +Rq8Sc7ckssykh1HKk8dOJt83GlvsBGXKALNv3vfHnMj+5XHC2NzZS5bn1IXWQE5z +0z5cHHD4NHiuGBnTl7MI8KzrF/Axwc2krsVO7WIQ/GpDVVwrCoKyvNm54GfpIAE/ +ndH7bu9hGVM6swzpAdhQC/HK6Vc0NoGfoARXVRtxuEZmoq2amixHJJU= +-----END CERTIFICATE----- diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.key b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.key new file mode 100644 index 0000000000000..f3b94a631596c --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/signing/signing.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAkI5L+/0jfgZpEPgGgykhr/OEvVMFzW2ogMUoObhj+Dhr4pQw +mVqocVFmQ0jfksVOSAyRElStYS6dcnbB/YDaz7vFbKRKhr6cnaqCHAOAG2hIFZKb +ojYeL2fjTQyZ17/GD3XY3OV8slkfeVrf/De7sQ1RjcZM0t4TmzkKlkBw6T4pD/b/ +VjtOAo/HKFs21EDbCFYPzramcnKmyRD1QaAHdmrmH7aWVDawHfrukTvsPK9dF4os +d8K0DeWZOhvqkuZ8rZwgEQkPCQVoCCtxYfkzAySf7i/9fDU+MAjgBUw6B75HG3rB +pbGpAgm86jJxxdhV2+jyzBQCMw+dHoT/n5vDKQIDAQABAoIBAA1EKeAF4sB5mSHY +CUz3NOK/cAKqAGHSewDaVy8451/L2cbQ/8bLJaNEq6RoJzCCkAUXtiafA8xj6Uos +cPAxZ6Nh4aPvTfGgw6HKmKc2gQbC4r6sFkFkQw/pslgLXIEK1gPsNktLek6p1DQg +bWbpvH1qsf3XYYyGmfkIWprgbhxRl0Z/9dji+T7f23JvwMm+18e4RG4ic4Xb40gU +J6oYQ4ogev4f0VML+r2YUke+wx5NdwoM2L3uMUM7tf/T75AJBxQdQGiBSHB5zakX +z2/LtdWYxb7IDKKrFkdY99TWPIgtHtzZzdYr4/FUGx6cRs25F9okP2ucQ0U6UU1V +161bbsECgYEAwmGOGAdXaRXlOSfvENjf6sf9npz2KGlv8kEuBU9w0Gtxfx1q8Xx9 +WUw4iI01LbYfBR/BWbCxOsn3SH5EQjQcReG0NyOo1G3a6S+NCfBKZms4DLWRA7fG +fF2ly9kvbvPz9089O7BWous9EqEnOEC+hkzMsb1lXYRDk8OQa2mlj1kCgYEAvmFR +EwcXuzfqJxHi6cEU+3bYrja3NOstWSBfvsulXc1G8tOcbqS4OGIRviQpA4+2JaK0 +S4/YMiT3hUF+2lzZcnGSSrToKKeKrxNLUXoE4QLRjcVNmOIBSQdN+xbZVdFXeRCM +UnqBuw+gGmOhHeVicVWEbSjUce0FhIdHiQd/qFECgYAFU68VMX5Pvu3dNx7yEz9v +q7NjmWGVke4jcW3Vb2vkCk298gxwOb0lqVUTSOtgKVGITmp6DsGMnuRL9EnilpL/ +x0OtDykdSTVqlocC8rbXP7D1iDRFKdAisF5Oy9Dk9YKGEIHZFOgK5u9xh0EP5ZZT +D9+8LziL64f+kKlwiCClYQKBgCB63+ccJatWPceOoKT6wQap3wvR3+3SVblH8a3O +dpcLR5h0C9NAnQFZkedbqfemlA/Vs2bU0rCzZ9s/MlI01xBUWf4O4TDWbK2z3/y1 +kZGF9pR2XefAXzHDYkV9P3UJsx+/eAE2T13Hq6v05W8BTItDaMVq2tvY8UEMB2NU +eS4RAoGASViwa3uI8avQts1Jf4M4jIHTLzyqY8hfA+06BALe69nw4vWNsIFQlNot +IA2+276jZp0eoFtnleo+y9PD+5zZBWXYypGfw3XgVW2m7RI0x7Bic9zGEJlmgyZH +hUEbMlQjY1F6xv7+WWQCuiZf4ns7uHeseu8bczp0d/jT2ZQtMWE= +-----END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerIntegTests.java index 554e1050b7248..3e5c57cf3b21a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerIntegTests.java @@ -13,6 +13,8 @@ import org.elasticsearch.common.ssl.PemKeyConfig; import org.elasticsearch.test.SecurityIntegTestCase; +import java.security.GeneralSecurityException; + import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.DIAGNOSE_TRUST_EXCEPTIONS; import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_CERTIFICATE_AUTHORITIES; import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_CERT_PATH; @@ -28,7 +30,7 @@ public class CrossClusterApiKeySignatureManagerIntegTests extends SecurityIntegT private static final String DYNAMIC_TEST_CLUSTER_ALIAS = "dynamic_test_cluster"; private static final String STATIC_TEST_CLUSTER_ALIAS = "static_test_cluster"; - public void testSignWithPemKeyConfig() { + public void testSignWithPemKeyConfig() throws GeneralSecurityException { final CrossClusterApiKeySignatureManager manager = getCrossClusterApiKeySignatureManagerInstance(); final String[] testHeaders = randomArray(5, String[]::new, () -> randomAlphanumericOfLength(randomInt(20))); @@ -49,9 +51,7 @@ public void testSignWithPemKeyConfig() { public void testSignUnknownClusterAlias() { final CrossClusterApiKeySignatureManager manager = getCrossClusterApiKeySignatureManagerInstance(); - final String[] testHeaders = randomArray(5, String[]::new, () -> randomAlphanumericOfLength(randomInt(20))); - X509CertificateSignature signature = manager.signerForClusterAlias("unknowncluster").sign(testHeaders); - assertNull(signature); + assertNull(manager.signerForClusterAlias("unknowncluster")); } public void testSeveralKeyStoreAliases() { @@ -72,8 +72,7 @@ public void testSeveralKeyStoreAliases() { { var signer = manager.signerForClusterAlias(DYNAMIC_TEST_CLUSTER_ALIAS); - X509CertificateSignature signature = signer.sign("test", "test"); - assertNull(signature); + assertNull(signer); } // Add an alias from the keystore diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java index 8fc912d40dfb5..d4fe3e14a6628 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.ClientHelper; @@ -22,7 +23,10 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; import org.elasticsearch.xpack.core.security.support.Exceptions; +import org.elasticsearch.xpack.security.transport.CrossClusterApiKeySignatureManager; +import org.elasticsearch.xpack.security.transport.X509CertificateSignature; +import java.security.GeneralSecurityException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -41,15 +45,18 @@ public class CrossClusterAccessAuthenticationService implements RemoteClusterAut private final ClusterService clusterService; private final ApiKeyService apiKeyService; private final AuthenticationService authenticationService; + private final CrossClusterApiKeySignatureManager.Verifier crossClusterApiKeySignatureVerifier; public CrossClusterAccessAuthenticationService( ClusterService clusterService, ApiKeyService apiKeyService, - AuthenticationService authenticationService + AuthenticationService authenticationService, + CrossClusterApiKeySignatureManager.Verifier crossClusterApiKeySignatureVerifier ) { this.clusterService = clusterService; this.apiKeyService = apiKeyService; this.authenticationService = authenticationService; + this.crossClusterApiKeySignatureVerifier = crossClusterApiKeySignatureVerifier; } @Override @@ -105,6 +112,14 @@ public void authenticate(final String action, final TransportRequest request, fi new ContextPreservingActionListener<>(storedContextSupplier, ActionListener.wrap(authentication -> { assert authentication.isApiKey() : "initial authentication for cross cluster access must be by API key"; assert false == authentication.isRunAs() : "initial authentication for cross cluster access cannot be run-as"; + + // TODO ALWAYS check if used api key has a certificate identity and do this verification conditionally based on that + var signature = crossClusterAccessHeaders.signature(); + // Always validate a signature if provided + if (signature != null) { + verifySignature(signature, crossClusterAccessHeaders); + } + // try-catch so any failure here is wrapped by `withRequestProcessingFailure`, whereas `authenticate` failures are not // we should _not_ wrap `authenticate` failures since this produces duplicate audit events try { @@ -118,6 +133,25 @@ public void authenticate(final String action, final TransportRequest request, fi } } + private void verifySignature(X509CertificateSignature signature, CrossClusterAccessHeaders crossClusterAccessHeaders) { + assert signature.certificates().length > 0 : "Signatures without certificates should not be considered for verification"; + try { + if (crossClusterApiKeySignatureVerifier.verify(signature, crossClusterAccessHeaders.signablePayload()) == false) { + logger.debug(Strings.format("Invalid cross cluster api key signature received [%s]", signature)); + throw Exceptions.authenticationError( + "Invalid cross cluster api key signature from [{}]", + X509CertificateSignature.certificateToString(signature.certificates()[0]) + ); + } + } catch (GeneralSecurityException securityException) { + logger.debug(Strings.format("Failed to verify cross cluster api key signature certificate [%s]", signature), securityException); + throw Exceptions.authenticationError( + "Failed to verify cross cluster api key signature certificate from [{}]", + X509CertificateSignature.certificateToString(signature.certificates()[0]) + ); + } + } + @Override public void authenticateHeaders(Map headers, ActionListener listener) { final ApiKeyService.ApiKeyCredentials credentials; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeaders.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeaders.java index 53ecec8b1cd25..7ab33d0646a1d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeaders.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeaders.java @@ -8,27 +8,52 @@ package org.elasticsearch.xpack.security.authc; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; +import org.elasticsearch.xpack.security.transport.CrossClusterApiKeySignatureManager; +import org.elasticsearch.xpack.security.transport.X509CertificateSignature; import java.io.IOException; +import java.util.Arrays; import java.util.Objects; +import static org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo.CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY; + public final class CrossClusterAccessHeaders { public static final String CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY = "_cross_cluster_access_credentials"; private final String credentialsHeader; private final CrossClusterAccessSubjectInfo crossClusterAccessSubjectInfo; + private final X509CertificateSignature signature; + private final String[] signablePayload; public CrossClusterAccessHeaders(String credentialsHeader, CrossClusterAccessSubjectInfo crossClusterAccessSubjectInfo) { + this(credentialsHeader, crossClusterAccessSubjectInfo, null); + } + + private CrossClusterAccessHeaders( + String credentialsHeader, + CrossClusterAccessSubjectInfo crossClusterAccessSubjectInfo, + @Nullable X509CertificateSignature signature, + String... signablePayload + ) { assert credentialsHeader.startsWith("ApiKey ") : "credentials header must start with [ApiKey ]"; this.credentialsHeader = credentialsHeader; this.crossClusterAccessSubjectInfo = crossClusterAccessSubjectInfo; + this.signature = signature; + this.signablePayload = signablePayload; } - public void writeToContext(final ThreadContext ctx) throws IOException { + public void writeToContext(final ThreadContext ctx, @Nullable CrossClusterApiKeySignatureManager.Signer signer) throws IOException { + var encodedSubjectInfo = crossClusterAccessSubjectInfo.encode(); + if (signer != null) { + var signature = signer.sign(encodedSubjectInfo, credentialsHeader); + X509CertificateSignature.writeToContext(ctx, signature); + } + + ctx.putHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY, encodedSubjectInfo); ctx.putHeader(CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY, credentialsHeader); - crossClusterAccessSubjectInfo.writeToContext(ctx); } public static CrossClusterAccessHeaders readFromContext(final ThreadContext ctx) throws IOException { @@ -41,13 +66,36 @@ public static CrossClusterAccessHeaders readFromContext(final ThreadContext ctx) // Invoke parsing logic to validate that the header decodes to a valid API key credential // Call `close` since the returned value is an auto-closable parseCredentialsHeader(credentialsHeader).close(); - return new CrossClusterAccessHeaders(credentialsHeader, CrossClusterAccessSubjectInfo.readFromContext(ctx)); + + final String subjectInfoHeader = ctx.getHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY); + if (subjectInfoHeader == null) { + throw new IllegalArgumentException( + "cross cluster access header [" + CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY + "] is required" + ); + } + var subjectInfo = CrossClusterAccessSubjectInfo.decode(subjectInfoHeader); + + return new CrossClusterAccessHeaders( + credentialsHeader, + subjectInfo, + X509CertificateSignature.readFromContext(ctx), + subjectInfoHeader, + credentialsHeader + ); } public ApiKeyService.ApiKeyCredentials credentials() { return parseCredentialsHeader(credentialsHeader); } + public X509CertificateSignature signature() { + return signature; + } + + public String[] signablePayload() { + return signablePayload; + } + static ApiKeyService.ApiKeyCredentials parseCredentialsHeader(final String header) { try { return Objects.requireNonNull(ApiKeyService.getCredentialsFromHeader(header, ApiKey.Type.CROSS_CLUSTER)); @@ -76,11 +124,13 @@ public boolean equals(Object obj) { if (obj == null || obj.getClass() != this.getClass()) return false; var that = (CrossClusterAccessHeaders) obj; return Objects.equals(this.credentialsHeader, that.credentialsHeader) - && Objects.equals(this.crossClusterAccessSubjectInfo, that.crossClusterAccessSubjectInfo); + && Objects.equals(this.crossClusterAccessSubjectInfo, that.crossClusterAccessSubjectInfo) + && Objects.equals(this.signature, that.signature) + && Arrays.equals(this.signablePayload, that.signablePayload); } @Override public int hashCode() { - return Objects.hash(credentialsHeader, crossClusterAccessSubjectInfo); + return Objects.hash(credentialsHeader, crossClusterAccessSubjectInfo, this.signature, Arrays.hashCode(this.signablePayload)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessSecurityExtension.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessSecurityExtension.java index dbf68011bc84d..02e4b651c3d50 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessSecurityExtension.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessSecurityExtension.java @@ -28,7 +28,6 @@ public class CrossClusterAccessSecurityExtension implements RemoteClusterSecurit private final CrossClusterAccessTransportInterceptor transportInterceptor; private final CrossClusterApiKeySigningConfigReloader crossClusterApiKeySignerReloader; - private final CrossClusterApiKeySignatureManager crossClusterApiKeySignatureManager; private CrossClusterAccessSecurityExtension(Components components) { this.crossClusterApiKeySignerReloader = new CrossClusterApiKeySigningConfigReloader( @@ -36,13 +35,16 @@ private CrossClusterAccessSecurityExtension(Components components) { components.resourceWatcherService(), components.clusterService().getClusterSettings() ); - this.crossClusterApiKeySignatureManager = new CrossClusterApiKeySignatureManager(components.environment()); + final CrossClusterApiKeySignatureManager crossClusterApiKeySignatureManager = new CrossClusterApiKeySignatureManager( + components.environment() + ); crossClusterApiKeySignerReloader.setSigningConfigLoader(crossClusterApiKeySignatureManager); this.authenticationService = new CrossClusterAccessAuthenticationService( components.clusterService(), components.apiKeyService(), - components.authenticationService() + components.authenticationService(), + crossClusterApiKeySignatureManager.verifier() ); this.transportInterceptor = new CrossClusterAccessTransportInterceptor( components.settings(), @@ -51,7 +53,7 @@ private CrossClusterAccessSecurityExtension(Components components) { components.authorizationService(), components.securityContext(), this.authenticationService, - this.crossClusterApiKeySignatureManager, + crossClusterApiKeySignatureManager, components.licenseState() ); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java index e3cd1d2f123d6..aad66795a6286 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java @@ -23,28 +23,30 @@ import org.elasticsearch.xpack.security.authc.CrossClusterAccessAuthenticationService; import org.elasticsearch.xpack.security.authz.AuthorizationService; -import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo.CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY; import static org.elasticsearch.xpack.security.authc.CrossClusterAccessHeaders.CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY; +import static org.elasticsearch.xpack.security.transport.X509CertificateSignature.CROSS_CLUSTER_ACCESS_SIGNATURE_HEADER_KEY; final class CrossClusterAccessServerTransportFilter extends ServerTransportFilter { private static final Logger logger = LogManager.getLogger(CrossClusterAccessServerTransportFilter.class); // pkg-private for testing - static final Set ALLOWED_TRANSPORT_HEADERS; - static { - final Set allowedHeaders = new HashSet<>( - Set.of(CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY, CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY) - ); - allowedHeaders.add(AuditUtil.AUDIT_REQUEST_ID); - allowedHeaders.add(Task.TRACE_STATE); - allowedHeaders.addAll(Task.HEADERS_TO_COPY); - ALLOWED_TRANSPORT_HEADERS = Set.copyOf(allowedHeaders); - } + static final Set ALLOWED_TRANSPORT_HEADERS = Stream.concat( + Stream.of( + CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY, + CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY, + CROSS_CLUSTER_ACCESS_SIGNATURE_HEADER_KEY, + AuditUtil.AUDIT_REQUEST_ID, + Task.TRACE_STATE + ), + Task.HEADERS_TO_COPY.stream() + ).collect(Collectors.toUnmodifiableSet()); private final CrossClusterAccessAuthenticationService crossClusterAccessAuthcService; private final XPackLicenseState licenseState; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptor.java index da93d67b7c542..b77bd7e2c7013 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptor.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.support.DestructiveOperations; @@ -17,6 +18,7 @@ import org.elasticsearch.common.ssl.SslConfiguration; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Nullable; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.TaskCancellationService; @@ -80,6 +82,9 @@ public class CrossClusterAccessTransportInterceptor implements RemoteClusterTran TaskCancellationService.REMOTE_CLUSTER_CANCEL_CHILD_ACTION_NAME ); + // Visible for testing + static final TransportVersion ADD_CROSS_CLUSTER_API_KEY_SIGNATURE = TransportVersion.fromName("add_cross_cluster_api_key_signature"); + private final Function> remoteClusterCredentialsResolver; private final CrossClusterAccessAuthenticationService crossClusterAccessAuthcService; private final CrossClusterApiKeySignatureManager crossClusterApiKeySignatureManager; @@ -217,7 +222,7 @@ private void sendWithCrossClusterAccessHeaders( final Authentication authentication = securityContext.getAuthentication(); assert authentication != null : "authentication must be present in security context"; - + var signer = crossClusterApiKeySignatureManager.signerForClusterAlias(remoteClusterCredentials.clusterAlias()); final User user = authentication.getEffectiveSubject().getUser(); if (user instanceof InternalUser && false == SystemUser.is(user)) { final String message = "Internal user [" + user.principal() + "] should not be used for cross cluster requests"; @@ -249,6 +254,7 @@ private void sendWithCrossClusterAccessHeaders( authentication.getEffectiveSubject().getRealm().getNodeName() ) ); + // To be able to enforce index-level privileges under the new remote cluster security model, // we switch from old-style internal actions to their new equivalent indices actions so that // they will be checked for index privileges against the index specified in the requests @@ -256,7 +262,15 @@ private void sendWithCrossClusterAccessHeaders( if (false == effectiveAction.equals(action)) { logger.trace("switching internal action from [{}] to [{}]", action, effectiveAction); } - sendWithCrossClusterAccessHeaders(crossClusterAccessHeaders, connection, effectiveAction, request, options, handler); + sendWithCrossClusterAccessHeaders( + crossClusterAccessHeaders, + connection, + effectiveAction, + signer, + request, + options, + handler + ); } else { assert false == action.startsWith("internal:") : "internal action must be sent with system user"; authzService.getRoleDescriptorsIntersectionForRemoteCluster( @@ -284,7 +298,15 @@ private void sendWithCrossClusterAccessHeaders( remoteClusterCredentials.credentials(), new CrossClusterAccessSubjectInfo(authentication, roleDescriptorsIntersection) ); - sendWithCrossClusterAccessHeaders(crossClusterAccessHeaders, connection, action, request, options, handler); + sendWithCrossClusterAccessHeaders( + crossClusterAccessHeaders, + connection, + action, + signer, + request, + options, + handler + ); }, // it's safe to not use a context restore handler here since `getRoleDescriptorsIntersectionForRemoteCluster` // uses a context preserving listener internally, and `sendWithCrossClusterAccessHeaders` uses a context restore // handler @@ -298,6 +320,7 @@ private void sendWithCrossClusterAccessHeaders( final CrossClusterAccessHeaders crossClusterAccessHeaders, final Transport.Connection connection, final String action, + @Nullable final CrossClusterApiKeySignatureManager.Signer signer, final TransportRequest request, final TransportRequestOptions options, final TransportResponseHandler handler @@ -308,7 +331,11 @@ private void sendWithCrossClusterAccessHeaders( handler ); try (ThreadContext.StoredContext ignored = threadContext.stashContextPreservingRequestHeaders(AuditUtil.AUDIT_REQUEST_ID)) { - crossClusterAccessHeaders.writeToContext(threadContext); + if (connection.getTransportVersion().supports(ADD_CROSS_CLUSTER_API_KEY_SIGNATURE)) { + crossClusterAccessHeaders.writeToContext(threadContext, signer); + } else { + crossClusterAccessHeaders.writeToContext(threadContext, null); + } sender.sendRequest(connection, action, request, options, contextRestoreHandler); } catch (Exception e) { contextRestoreHandler.handleException(new SendRequestTransportException(connection.getNode(), action, e)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManager.java index b9ba115ab4c24..acc50e029934a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManager.java @@ -160,13 +160,13 @@ public Verifier verifier() { } public Signer signerForClusterAlias(String clusterAlias) { - return new Signer(clusterAlias); + return keyPairByClusterAlias.containsKey(clusterAlias) ? new Signer(clusterAlias) : null; } public class Verifier { private Verifier() {} - public boolean verify(X509CertificateSignature signature, String... headers) { + public boolean verify(X509CertificateSignature signature, String... headers) throws GeneralSecurityException { assert signature.certificates().length > 0 : "Signature not valid without trusted certificate chain"; var authTrustManager = trustManager.get(); @@ -177,33 +177,28 @@ public boolean verify(X509CertificateSignature signature, String... headers) { ); } - try { - // Make sure the provided certificate chain is trusted - var leaf = signature.certificates()[0]; - if (logger.isTraceEnabled()) { - logger.trace( - "checking signing chain (len={}) [{}] with leaf subject [{}] using algorithm [{}]", - signature.certificates().length, - Arrays.stream(signature.certificates()) - .map(CrossClusterApiKeySignatureManager::calculateFingerprint) - .collect(Collectors.joining(",")), - leaf.getSubjectX500Principal().getName(X500Principal.RFC2253), - leaf.getPublicKey().getAlgorithm() - ); - } - authTrustManager.checkClientTrusted(signature.certificates(), signature.certificates()[0].getPublicKey().getAlgorithm()); - - // TODO Make sure the signing certificate belongs to the correct DN (the configured api key cert identity) - // TODO Make sure the signing certificate is valid - // Make sure signature is correct - final Signature signer = Signature.getInstance(signature.algorithm()); - signer.initVerify(signature.certificates()[0]); - signer.update(getSignableBytes(headers)); - return signer.verify(signature.signature().array()); - } catch (GeneralSecurityException e) { - logger.debug("failed certificate validation for Signature [" + signature + "]", e); - throw new ElasticsearchSecurityException("Failed to verify signature from [{}]", signature.certificates()[0], e); + // Make sure the provided certificate chain is trusted + var leaf = signature.certificates()[0]; + if (logger.isTraceEnabled()) { + logger.trace( + "checking signing chain (len={}) [{}] with leaf subject [{}] using algorithm [{}]", + signature.certificates().length, + Arrays.stream(signature.certificates()) + .map(CrossClusterApiKeySignatureManager::calculateFingerprint) + .collect(Collectors.joining(",")), + leaf.getSubjectX500Principal().getName(X500Principal.RFC2253), + leaf.getPublicKey().getAlgorithm() + ); } + authTrustManager.checkClientTrusted(signature.certificates(), signature.certificates()[0].getPublicKey().getAlgorithm()); + + // TODO Make sure the signing certificate belongs to the correct DN (the configured api key cert identity) + // TODO Make sure the signing certificate is valid + // Make sure signature is correct + final Signature signer = Signature.getInstance(signature.algorithm()); + signer.initVerify(signature.certificates()[0]); + signer.update(getSignableBytes(headers)); + return signer.verify(signature.signature().array()); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/X509CertificateSignature.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/X509CertificateSignature.java index f6abc28a0d73e..1e80239defa66 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/X509CertificateSignature.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/X509CertificateSignature.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.ssl.SslUtil; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.logging.LogManager; @@ -34,6 +35,7 @@ public final class X509CertificateSignature implements Writeable { private static final Logger logger = LogManager.getLogger(X509CertificateSignature.class); + public static final String CROSS_CLUSTER_ACCESS_SIGNATURE_HEADER_KEY = "_cross_cluster_access_signature"; private final X509Certificate[] certificateChain; private final String algorithm; @@ -99,7 +101,7 @@ public int hashCode() { public String toString() { return "X509CertificateSignature[" + "certificates=" - + Arrays.stream(certificateChain).map(this::certificateToString).collect(Collectors.joining(",")) + + Arrays.stream(certificateChain).map(X509CertificateSignature::certificateToString).collect(Collectors.joining(",")) + ", " + "algorithm=" + algorithm @@ -109,11 +111,11 @@ public String toString() { + ']'; } - private String certificateToString(X509Certificate certificate) { + public static String certificateToString(X509Certificate certificate) { return "(" + certificate.getSubjectX500Principal() + ";" + certificate.getType() + ";" + fingerprint(certificate) + ")"; } - private String fingerprint(X509Certificate certificate) { + private static String fingerprint(X509Certificate certificate) { try { return "SHA1:" + SslUtil.calculateFingerprint(certificate, "SHA-1"); } catch (CertificateEncodingException e) { @@ -142,6 +144,15 @@ public String encodeToString() throws IOException { return encoded; } + public static void writeToContext(ThreadContext ctx, X509CertificateSignature signature) throws IOException { + ctx.putHeader(CROSS_CLUSTER_ACCESS_SIGNATURE_HEADER_KEY, signature.encodeToString()); + } + + public static X509CertificateSignature readFromContext(ThreadContext ctx) throws IOException { + var encodedSignature = ctx.getHeader(CROSS_CLUSTER_ACCESS_SIGNATURE_HEADER_KEY); + return encodedSignature != null ? X509CertificateSignature.decode(encodedSignature) : null; + } + public static X509CertificateSignature decode(String encoded) throws IOException { logger.trace("Decoding [{}]", encoded); try { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceIntegTests.java index 904333805a2f4..c93641fdfddba 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceIntegTests.java @@ -71,7 +71,7 @@ public void testInvalidHeaders() throws IOException { new CrossClusterAccessHeaders( ApiKeyService.withApiKeyPrefix("abc"), AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo() - ).writeToContext(threadContext); + ).writeToContext(threadContext, null); authenticateAndAssertExpectedErrorMessage( service, msg -> assertThat( @@ -106,7 +106,7 @@ public void testInvalidHeaders() throws IOException { AuthenticationTestHelper.builder().internal(internalUser).build(), RoleDescriptorsIntersection.EMPTY ) - ).writeToContext(threadContext); + ).writeToContext(threadContext, null); authenticateAndAssertExpectedErrorMessage( service, msg -> assertThat( @@ -121,7 +121,7 @@ public void testInvalidHeaders() throws IOException { new CrossClusterAccessHeaders( encodedCrossClusterAccessApiKey, new CrossClusterAccessSubjectInfo(authentication, RoleDescriptorsIntersection.EMPTY) - ).writeToContext(threadContext); + ).writeToContext(threadContext, null); authenticateAndAssertExpectedErrorMessage( service, @@ -295,7 +295,7 @@ private void addRandomizedHeaders(ThreadContext threadContext, String validEncod RoleDescriptorsIntersection.EMPTY ) ) - ).writeToContext(threadContext); + ).writeToContext(threadContext, null); } else { if (randomBoolean()) { threadContext.putHeader(CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY, validEncodedApiKey); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java index 6e71b5f04fe7b..c6c37ebcf13ae 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java @@ -13,7 +13,9 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.ssl.PemUtils; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.TransportRequest; @@ -27,11 +29,16 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.transport.CrossClusterApiKeySignatureManager; +import org.elasticsearch.xpack.security.transport.X509CertificateSignature; import org.junit.Before; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.cert.X509Certificate; +import java.util.List; import java.util.concurrent.ExecutionException; import static org.hamcrest.Matchers.containsString; @@ -56,19 +63,24 @@ public class CrossClusterAccessAuthenticationServiceTests extends ESTestCase { private ApiKeyService apiKeyService; private AuthenticationService authenticationService; private CrossClusterAccessAuthenticationService crossClusterAccessAuthenticationService; + private CrossClusterApiKeySignatureManager.Verifier verifier; + private CrossClusterApiKeySignatureManager.Signer signer; @Before public void init() throws Exception { this.threadContext = new ThreadContext(Settings.EMPTY); this.apiKeyService = mock(ApiKeyService.class); this.authenticationService = mock(AuthenticationService.class); + this.verifier = mock(CrossClusterApiKeySignatureManager.Verifier.class); + this.signer = mock(CrossClusterApiKeySignatureManager.Signer.class); this.clusterService = mock(ClusterService.class, Mockito.RETURNS_DEEP_STUBS); when(clusterService.state().getMinTransportVersion()).thenReturn(TransportVersion.current()); when(clusterService.threadPool().getThreadContext()).thenReturn(threadContext); crossClusterAccessAuthenticationService = new CrossClusterAccessAuthenticationService( clusterService, apiKeyService, - authenticationService + authenticationService, + verifier ); } @@ -77,7 +89,7 @@ public void testAuthenticationSuccessOnSuccessfulAuthentication() throws IOExcep CrossClusterAccessHeadersTests.randomEncodedApiKeyHeader(), AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo() ); - crossClusterAccessHeaders.writeToContext(threadContext); + crossClusterAccessHeaders.writeToContext(threadContext, null); final AuthenticationService.AuditableRequest auditableRequest = mock(AuthenticationService.AuditableRequest.class); final ArgumentCaptor authenticationCapture = ArgumentCaptor.forClass(Authentication.class); doNothing().when(auditableRequest).authenticationSuccess(authenticationCapture.capture()); @@ -120,7 +132,7 @@ public void testExceptionProcessingRequestOnInvalidCrossClusterAccessSubjectInfo ) ) ); - crossClusterAccessHeaders.writeToContext(threadContext); + crossClusterAccessHeaders.writeToContext(threadContext, null); final AuthenticationService.AuditableRequest auditableRequest = mock(AuthenticationService.AuditableRequest.class); final ArgumentCaptor authenticationCapture = ArgumentCaptor.forClass(Authentication.class); doNothing().when(auditableRequest).authenticationSuccess(authenticationCapture.capture()); @@ -161,12 +173,118 @@ public void testExceptionProcessingRequestOnInvalidCrossClusterAccessSubjectInfo verifyNoMoreInteractions(auditableRequest); } + public void testAuthenticationSuccessfulCrossClusterApiKeySignature() throws IOException, GeneralSecurityException, ExecutionException, + InterruptedException { + var subjectInfo = AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo(); + var apiKeyHeader = CrossClusterAccessHeadersTests.randomEncodedApiKeyHeader(); + var certs = PemUtils.readCertificates(List.of(getDataPath("/org/elasticsearch/xpack/security/signature/signing_rsa.crt"))) + .stream() + .map(cert -> (X509Certificate) cert) + .toArray(X509Certificate[]::new); + final var crossClusterAccessHeaders = new CrossClusterAccessHeaders(apiKeyHeader, subjectInfo); + + when(signer.sign(anyString(), anyString())).thenReturn(new X509CertificateSignature(certs, "", mock(BytesReference.class))); + crossClusterAccessHeaders.writeToContext(threadContext, signer); + + when(verifier.verify(any(X509CertificateSignature.class), anyString(), anyString())).thenReturn(true); + + final AuthenticationService.AuditableRequest auditableRequest = mock(AuthenticationService.AuditableRequest.class); + final ArgumentCaptor authenticationCapture = ArgumentCaptor.forClass(Authentication.class); + doNothing().when(auditableRequest).authenticationSuccess(authenticationCapture.capture()); + + var authContext = new Authenticator.Context( + threadContext, + auditableRequest, + mock(Realms.class), + crossClusterAccessHeaders.credentials() + ); + var action = "action"; + var request = mock(TransportRequest.class); + when(authenticationService.newContext(anyString(), any(TransportRequest.class), any(ApiKeyService.ApiKeyCredentials.class))) + .thenReturn(authContext); + + @SuppressWarnings("unchecked") + final ArgumentCaptor> listenerCaptor = ArgumentCaptor.forClass(ActionListener.class); + doAnswer(i -> null).when(authenticationService).authenticate(any(Authenticator.Context.class), listenerCaptor.capture()); + + final PlainActionFuture future = new PlainActionFuture<>(); + crossClusterAccessAuthenticationService.authenticate(action, request, future); + + final Authentication apiKeyAuthentication = AuthenticationTestHelper.builder().apiKey().build(false); + listenerCaptor.getValue().onResponse(apiKeyAuthentication); + future.get(); + + final Authentication expectedAuthentication = apiKeyAuthentication.toCrossClusterAccess( + crossClusterAccessHeaders.getCleanAndValidatedSubjectInfo() + ); + verify(auditableRequest).authenticationSuccess(expectedAuthentication); + verifyNoMoreInteractions(auditableRequest); + } + + public void testAuthenticationExceptionOnBadCrossClusterApiKeySignature() throws IOException, GeneralSecurityException { + var subjectInfo = AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo(); + var apiKeyHeader = CrossClusterAccessHeadersTests.randomEncodedApiKeyHeader(); + var certs = PemUtils.readCertificates(List.of(getDataPath("/org/elasticsearch/xpack/security/signature/signing_rsa.crt"))) + .stream() + .map(cert -> (X509Certificate) cert) + .toArray(X509Certificate[]::new); + final var crossClusterAccessHeaders = new CrossClusterAccessHeaders(apiKeyHeader, subjectInfo); + + var verifyMock = when(verifier.verify(any(X509CertificateSignature.class), anyString(), anyString())); + boolean badCert = randomBoolean(); + + if (badCert) { + verifyMock.thenThrow(new GeneralSecurityException("bad certificate")); + } else { + verifyMock.thenReturn(false); + } + + when(signer.sign(anyString(), anyString())).thenReturn(new X509CertificateSignature(certs, "", mock(BytesReference.class))); + crossClusterAccessHeaders.writeToContext(threadContext, signer); + + final AuthenticationService.AuditableRequest auditableRequest = mock(AuthenticationService.AuditableRequest.class); + + var authContext = new Authenticator.Context( + threadContext, + auditableRequest, + mock(Realms.class), + crossClusterAccessHeaders.credentials() + ); + var action = "action"; + var request = mock(TransportRequest.class); + when(authenticationService.newContext(anyString(), any(TransportRequest.class), any(ApiKeyService.ApiKeyCredentials.class))) + .thenReturn(authContext); + + @SuppressWarnings("unchecked") + final ArgumentCaptor> listenerCaptor = ArgumentCaptor.forClass(ActionListener.class); + doAnswer(i -> null).when(authenticationService).authenticate(any(Authenticator.Context.class), listenerCaptor.capture()); + + final PlainActionFuture future = new PlainActionFuture<>(); + crossClusterAccessAuthenticationService.authenticate(action, request, future); + + final Authentication apiKeyAuthentication = AuthenticationTestHelper.builder().apiKey().build(false); + listenerCaptor.getValue().onResponse(apiKeyAuthentication); + + final ExecutionException actual = expectThrows(ExecutionException.class, future::get); + + assertThat(actual.getCause(), instanceOf(ElasticsearchSecurityException.class)); + assertThat( + actual.getCause().getMessage(), + containsString( + (badCert + ? "Failed to verify cross cluster api key signature certificate from [" + : "Invalid cross cluster api key signature from [") + X509CertificateSignature.certificateToString(certs[0]) + "]" + ) + ); + verifyNoMoreInteractions(auditableRequest); + } + public void testNoInteractionWithAuditableRequestOnInitialAuthenticationFailure() throws IOException { final var crossClusterAccessHeaders = new CrossClusterAccessHeaders( CrossClusterAccessHeadersTests.randomEncodedApiKeyHeader(), AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo() ); - crossClusterAccessHeaders.writeToContext(threadContext); + crossClusterAccessHeaders.writeToContext(threadContext, null); final AuthenticationService.AuditableRequest auditableRequest = mock(AuthenticationService.AuditableRequest.class); doAnswer(invocationOnMock -> { AuthenticationToken authenticationToken = (AuthenticationToken) invocationOnMock.getArguments()[2]; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeadersTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeadersTests.java index f567057d5b410..662f94f069d34 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeadersTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessHeadersTests.java @@ -8,20 +8,29 @@ package org.elasticsearch.xpack.security.authc; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.ssl.PemUtils; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; +import org.elasticsearch.xpack.security.transport.CrossClusterApiKeySignatureManager; +import org.elasticsearch.xpack.security.transport.X509CertificateSignature; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.Base64; +import java.util.List; import java.util.Set; import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomUniquelyNamedRoleDescriptors; import static org.elasticsearch.xpack.security.authc.CrossClusterAccessHeaders.CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class CrossClusterAccessHeadersTests extends ESTestCase { @@ -32,7 +41,7 @@ public void testWriteReadContextRoundtrip() throws IOException { AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo(randomRoleDescriptorsIntersection()) ); - expected.writeToContext(ctx); + expected.writeToContext(ctx, null); final CrossClusterAccessHeaders actual = CrossClusterAccessHeaders.readFromContext(ctx); assertThat(actual.getSubjectInfo(), equalTo(expected.getSubjectInfo())); @@ -41,6 +50,37 @@ public void testWriteReadContextRoundtrip() throws IOException { assertThat(actual.credentials().getKey().toString(), equalTo(expected.credentials().getKey().toString())); } + public void testWriteReadContextRoundtripWithSignature() throws IOException, CertificateException { + final ThreadContext ctx = new ThreadContext(Settings.EMPTY); + var encodedApiKeyHeader = randomEncodedApiKeyHeader(); + var subjectInfo = AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo(randomRoleDescriptorsIntersection()); + final var toWrite = new CrossClusterAccessHeaders(encodedApiKeyHeader, subjectInfo); + var testSignature = new X509CertificateSignature(getTestCertificates(), "MOCK", new BytesArray(new byte[] { 1, 2, 3 })); + var signer = mock(CrossClusterApiKeySignatureManager.Signer.class); + when(signer.sign(subjectInfo.encode(), encodedApiKeyHeader)).thenReturn(testSignature); + + toWrite.writeToContext(ctx, signer); + final CrossClusterAccessHeaders actual = CrossClusterAccessHeaders.readFromContext(ctx); + + assertThat(actual.getSubjectInfo(), equalTo(toWrite.getSubjectInfo())); + assertThat(actual.getCleanAndValidatedSubjectInfo(), equalTo(toWrite.getCleanAndValidatedSubjectInfo())); + assertThat(actual.credentials().getId(), equalTo(toWrite.credentials().getId())); + assertThat(actual.credentials().getKey().toString(), equalTo(toWrite.credentials().getKey().toString())); + assertThat(actual.signature(), equalTo(testSignature)); + assertThat(actual.signablePayload(), equalTo(new String[] { subjectInfo.encode(), encodedApiKeyHeader })); + } + + public void testThrowsOnMissingEntry() { + var actual = expectThrows( + IllegalArgumentException.class, + () -> CrossClusterAccessHeaders.readFromContext(new ThreadContext(Settings.EMPTY)) + ); + assertThat( + actual.getMessage(), + equalTo("cross cluster access header [" + CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY + "] is required") + ); + } + public void testClusterCredentialsReturnsValidApiKey() { final String id = UUIDs.randomBase64UUID(); final String key = UUIDs.randomBase64UUID(); @@ -63,7 +103,7 @@ public void testReadOnInvalidApiKeyValueThrows() throws IOException { AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo(randomRoleDescriptorsIntersection()) ); - expected.writeToContext(ctx); + expected.writeToContext(ctx, null); var actual = expectThrows(IllegalArgumentException.class, () -> CrossClusterAccessHeaders.readFromContext(ctx)); assertThat( @@ -100,6 +140,13 @@ public void testReadOnHeaderWithMalformedPrefixThrows() throws IOException { ); } + private X509Certificate[] getTestCertificates() throws CertificateException, IOException { + return PemUtils.readCertificates(List.of(getDataPath("/org/elasticsearch/xpack/security/signature/signing_rsa.crt"))) + .stream() + .map(cert -> (X509Certificate) cert) + .toArray(X509Certificate[]::new); + } + private static RoleDescriptorsIntersection randomRoleDescriptorsIntersection() { return new RoleDescriptorsIntersection(randomList(0, 3, () -> Set.copyOf(randomUniquelyNamedRoleDescriptors(0, 1)))); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java index 4bd99d6deb073..c6c14d0c892e3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.security.transport; -import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -17,6 +16,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; +import java.security.GeneralSecurityException; + import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -34,7 +35,7 @@ public void setUp() throws Exception { .put(Node.NODE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(3, 8)); } - public void testSignAndVerifyPKCS12orBCFKS() { + public void testSignAndVerifyPKCS12orBCFKS() throws GeneralSecurityException { var builder = Settings.builder() .put("cluster.remote.my_remote.signing.keystore.alias", "wholelottakey") .put("path.home", createTempDir()) @@ -48,7 +49,7 @@ public void testSignAndVerifyPKCS12orBCFKS() { assertTrue(manager.verifier().verify(signature, "a_header")); } - public void testSignAndVerifyDifferentPayloadFailsPKCS12orBCFKS() { + public void testSignAndVerifyDifferentPayloadFailsPKCS12orBCFKS() throws GeneralSecurityException { var builder = Settings.builder() .put("cluster.remote.my_remote.signing.keystore.alias", "wholelottakey") .put("path.home", createTempDir()) @@ -62,7 +63,7 @@ public void testSignAndVerifyDifferentPayloadFailsPKCS12orBCFKS() { assertFalse(manager.verifier().verify(signature, "another_header")); } - public void testSignAndVerifyRSAorEC() { + public void testSignAndVerifyRSAorEC() throws GeneralSecurityException { var builder = Settings.builder() .put("path.home", createTempDir()) .put(Node.NODE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(3, 8)); @@ -74,7 +75,7 @@ public void testSignAndVerifyRSAorEC() { assertTrue(manager.verifier().verify(signature, "a_header")); } - public void testSignAndVerifyDifferentPayloadFailsRSAorEC() { + public void testSignAndVerifyDifferentPayloadFailsRSAorEC() throws GeneralSecurityException { var builder = Settings.builder() .put("path.home", createTempDir()) .put(Node.NODE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(3, 8)); @@ -86,7 +87,7 @@ public void testSignAndVerifyDifferentPayloadFailsRSAorEC() { assertFalse(manager.verifier().verify(signature, "another_header")); } - public void testSignAndVerifyWrongKeyRSAorEC() { + public void testSignAndVerifyWrongKeyRSAorEC() throws GeneralSecurityException { var builder = Settings.builder() .put("cluster.remote.my_remote2.signing.keystore.alias", "ainttalkinboutkeys") .put("path.home", createTempDir()) @@ -111,7 +112,7 @@ public void testSignAndVerifyWrongKeyRSAorEC() { ); } - public void testSignAndVerifyManipulatedSignatureStringRSAorEC() { + public void testSignAndVerifyManipulatedSignatureStringRSAorEC() throws GeneralSecurityException { var builder = Settings.builder() .put("path.home", createTempDir()) .put(Node.NODE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(3, 8)); @@ -196,8 +197,8 @@ public void testSignAndVerifyFailsIntermediateCertMissing() { var signature = signer.sign("test"); assertThat(signature.certificates(), arrayWithSize(1)); - var exception = assertThrows(ElasticsearchSecurityException.class, () -> verifier.verify(signature, "test")); - assertThat(exception.getMessage(), containsString("Failed to verify signature from ")); + var exception = assertThrows(GeneralSecurityException.class, () -> verifier.verify(signature, "test")); + assertThat(exception.getMessage(), containsString("unable to find valid certification path to requested target")); } private void addStorePathToBuilder(String storeName, String password, String passwordFips, Settings.Builder builder) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java index 1e17459613a2c..cf76b46bf3436 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java @@ -17,12 +17,14 @@ import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.ssl.DefaultJdkTrustConfig; import org.elasticsearch.common.ssl.EmptyKeyConfig; +import org.elasticsearch.common.ssl.PemUtils; import org.elasticsearch.common.ssl.SslClientAuthenticationMode; import org.elasticsearch.common.ssl.SslConfiguration; import org.elasticsearch.common.ssl.SslKeyConfig; @@ -73,6 +75,8 @@ import org.mockito.Mockito; import java.io.IOException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.Collections; import java.util.List; import java.util.Map; @@ -102,6 +106,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -110,6 +115,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; public class SecurityServerTransportInterceptorTests extends ESTestCase { @@ -756,6 +762,16 @@ private void doTestSendWithCrossClusterAccessHeaders( String action, TransportRequest request, Authentication authentication + ) throws IOException { + doTestSendWithCrossClusterAccessHeaders(shouldAssertForSystemUser, action, request, authentication, TransportVersion.current()); + } + + private void doTestSendWithCrossClusterAccessHeaders( + boolean shouldAssertForSystemUser, + String action, + TransportRequest request, + Authentication authentication, + TransportVersion transportVersion ) throws IOException { authentication.writeToContext(threadContext); final String expectedRequestId = AuditUtil.getOrGenerateRequestId(threadContext); @@ -812,7 +828,9 @@ public void sendRequest( sentCredential.set(securityContext.getThreadContext().getHeader(CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY)); try { sentCrossClusterAccessSubjectInfo.set( - CrossClusterAccessSubjectInfo.readFromContext(securityContext.getThreadContext()) + CrossClusterAccessSubjectInfo.decode( + securityContext.getThreadContext().getHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY) + ) ); } catch (IOException e) { fail("no exceptions expected but got " + e); @@ -821,7 +839,7 @@ public void sendRequest( } }); final Connection connection = mock(Connection.class); - when(connection.getTransportVersion()).thenReturn(TransportVersion.current()); + when(connection.getTransportVersion()).thenReturn(transportVersion); sender.sendRequest(connection, action, request, null, new TransportResponseHandler<>() { @Override @@ -1062,6 +1080,62 @@ public TransportResponse read(StreamInput in) { assertThat(securityContext.getThreadContext().getHeader(CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY), nullValue()); } + public void testSendWithCrossClusterApiKeySignatureSkippedOnUnsupportedConnection() throws Exception { + final String action; + final TransportRequest request; + if (randomBoolean()) { + action = randomAlphaOfLengthBetween(5, 30); + request = mock(TransportRequest.class); + } else { + action = ClusterStateAction.NAME; + request = mock(ClusterStateRequest.class); + } + + var signer = mock(CrossClusterApiKeySignatureManager.Signer.class); + when(crossClusterApiKeySignatureManager.signerForClusterAlias(anyString())).thenReturn(signer); + var transportVersion = TransportVersionUtils.getPreviousVersion( + CrossClusterAccessTransportInterceptor.ADD_CROSS_CLUSTER_API_KEY_SIGNATURE + ); + doTestSendWithCrossClusterAccessHeaders( + true, + action, + request, + AuthenticationTestHelper.builder().internal(InternalUsers.SYSTEM_USER).transportVersion(transportVersion).build(), + transportVersion + ); + + verifyNoInteractions(signer); + } + + public void testSendWithCrossClusterApiKeySignatureSentOnSupportedConnection() throws Exception { + final String action; + final TransportRequest request; + if (randomBoolean()) { + action = randomAlphaOfLengthBetween(5, 30); + request = mock(TransportRequest.class); + } else { + action = ClusterStateAction.NAME; + request = mock(ClusterStateRequest.class); + } + + var testSignature = getTestSignature(); + var signer = mock(CrossClusterApiKeySignatureManager.Signer.class); + when(signer.sign(anyString(), anyString())).thenReturn(testSignature); + when(crossClusterApiKeySignatureManager.signerForClusterAlias(anyString())).thenReturn(signer); + + var transportVersion = CrossClusterAccessTransportInterceptor.ADD_CROSS_CLUSTER_API_KEY_SIGNATURE; + + doTestSendWithCrossClusterAccessHeaders( + true, + action, + request, + AuthenticationTestHelper.builder().internal(InternalUsers.SYSTEM_USER).transportVersion(transportVersion).build(), + transportVersion + ); + + verify(signer, times(1)).sign(anyString(), anyString()); + } + public void testSendRemoteRequestFailsIfUserHasNoRemoteIndicesPrivileges() throws Exception { final Authentication authentication = AuthenticationTestHelper.builder() .user(new User(randomAlphaOfLengthBetween(3, 10), randomRoles())) @@ -1308,4 +1382,15 @@ private static Consumer anyConsumer() { return any(Consumer.class); } + private X509CertificateSignature getTestSignature() throws CertificateException, IOException { + return new X509CertificateSignature(getTestCertificates(), "SHA256withRSA", new BytesArray(new byte[] { 1, 2, 3, 4 })); + } + + private X509Certificate[] getTestCertificates() throws CertificateException, IOException { + return PemUtils.readCertificates(List.of(getDataPath("/org/elasticsearch/xpack/security/signature/signing_rsa.crt"))) + .stream() + .map(cert -> (X509Certificate) cert) + .toArray(X509Certificate[]::new); + } + } From e2c0425f1b0667f4e6cfe193e50581feed87c685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Fri, 3 Oct 2025 15:27:22 +0200 Subject: [PATCH 02/10] Update docs/changelog/135674.yaml --- docs/changelog/135674.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135674.yaml diff --git a/docs/changelog/135674.yaml b/docs/changelog/135674.yaml new file mode 100644 index 0000000000000..3087f9223c9a2 --- /dev/null +++ b/docs/changelog/135674.yaml @@ -0,0 +1,5 @@ +pr: 135674 +summary: Send cross cluster api key signature as headers +area: Security +type: enhancement +issues: [] From ac844798afdb02d8088d82bfe65549513c2c5063 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Fri, 3 Oct 2025 16:07:30 +0200 Subject: [PATCH 03/10] Fix fips issue in test - different message --- .../transport/CrossClusterApiKeySignatureManagerTests.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java index c6c14d0c892e3..125cb9f82aa59 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterApiKeySignatureManagerTests.java @@ -198,7 +198,12 @@ public void testSignAndVerifyFailsIntermediateCertMissing() { var signature = signer.sign("test"); assertThat(signature.certificates(), arrayWithSize(1)); var exception = assertThrows(GeneralSecurityException.class, () -> verifier.verify(signature, "test")); - assertThat(exception.getMessage(), containsString("unable to find valid certification path to requested target")); + assertThat( + exception.getMessage(), + containsString( + inFipsJvm() ? "Unable to construct a valid chain" : "unable to find valid certification path to requested target" + ) + ); } private void addStorePathToBuilder(String storeName, String password, String passwordFips, Settings.Builder builder) { From f406c8b9cc51b7e6ba777582e633137d4111536a Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Fri, 3 Oct 2025 16:17:12 +0200 Subject: [PATCH 04/10] Fix signer behaviour change in test --- ...lusterSigningConfigReloaderIntegTests.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java index 064b558c2a19f..b97ad427e3059 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.test.SecurityIntegTestCase; +import javax.net.ssl.KeyManagerFactory; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -26,8 +27,6 @@ import java.util.Set; import java.util.function.Consumer; -import javax.net.ssl.KeyManagerFactory; - import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_CERTIFICATE_AUTHORITIES; import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_CERT_PATH; import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_KEYSTORE_ALGORITHM; @@ -128,14 +127,13 @@ public void testDependentKeyConfigFilesUpdated() throws Exception { var manager = getCrossClusterApiKeySignatureManagerInstance(); String testClusterAlias = "test_cluster"; - var signer = manager.signerForClusterAlias(testClusterAlias); try { // Write passphrase for ec key to keystore writeSecureSettingsToKeyStoreAndReload( Map.of(SIGNING_KEY_SECURE_PASSPHRASE.getConcreteSettingForNamespace(testClusterAlias).getKey(), "marshall".toCharArray()) ); - assertNull(signer.sign("a_header")); + assertNull(manager.signerForClusterAlias(testClusterAlias)); Path tempDir = createTempDir(); Path signingCert = tempDir.resolve("signing.crt"); Files.copy(getDataPath("/org/elasticsearch/xpack/security/signature/signing_rsa.crt"), signingCert); @@ -155,6 +153,7 @@ public void testDependentKeyConfigFilesUpdated() throws Exception { ); // Make sure a signature can be created + var signer = manager.signerForClusterAlias(testClusterAlias); var signatureBefore = signer.sign("test", "test"); assertNotNull(signatureBefore); @@ -183,9 +182,8 @@ public void testInitialBadDependentFileAvailableAfterUpdate() throws Exception { var manager = getCrossClusterApiKeySignatureManagerInstance(); String testClusterAlias = "test_cluster"; - var signer = manager.signerForClusterAlias(testClusterAlias); try { - assertNull(signer.sign("a_header")); + assertNull(manager.signerForClusterAlias(testClusterAlias)); Path tempDir = createTempDir(); Path emptyFile = createTempFile(); Path signingCert = tempDir.resolve("signing.crt"); @@ -202,8 +200,7 @@ public void testInitialBadDependentFileAvailableAfterUpdate() throws Exception { { // Make sure no signature can be created - var signature = signer.sign("test", "test"); - assertNull(signature); + assertNull(manager.signerForClusterAlias(testClusterAlias)); } // Overwrite the empty file with the actual signing cert Files.copy( @@ -214,6 +211,8 @@ public void testInitialBadDependentFileAvailableAfterUpdate() throws Exception { // Make sure config recovers and can generate a signature { assertBusy(() -> { + var signer = manager.signerForClusterAlias(testClusterAlias); + assertNotNull(signer); var signature = signer.sign("test", "test"); assertNotNull(signature); }); @@ -234,9 +233,8 @@ public void testInitialBadDependentFileAvailableAfterUpdate() throws Exception { public void testRemoveFileWithConfig() throws Exception { try { var manager = getCrossClusterApiKeySignatureManagerInstance(); - var signer = manager.signerForClusterAlias("test_cluster"); - assertNull(signer.sign("a_header")); + assertNull(manager.signerForClusterAlias("test_cluster")); Path tempDir = createTempDir(); Path signingCert = tempDir.resolve("signing.crt"); Files.copy(getDataPath("/org/elasticsearch/xpack/security/signature/signing_rsa.crt"), signingCert); @@ -251,6 +249,7 @@ public void testRemoveFileWithConfig() throws Exception { ); // Make sure a signature can be created + var signer = manager.signerForClusterAlias("test_cluster"); var signatureBefore = signer.sign("test", "test"); assertNotNull(signatureBefore); @@ -294,21 +293,20 @@ private void addAndRemoveClusterConfigsRuntime( try { for (var clusterAlias : clusterAliases) { - var signer = manager.signerForClusterAlias(clusterAlias); - var verfier = manager.verifier(); - // Try to create a signature for a remote cluster that doesn't exist - assertNull(signer.sign(testHeaders)); + var verifier = manager.verifier(); + // Try to create a signer for a remote cluster that doesn't exist + assertNull(manager.signerForClusterAlias(clusterAlias)); clusterCreator.accept(clusterAlias); // Make sure a signature can be created + var signer = manager.signerForClusterAlias(clusterAlias); var signature = signer.sign(testHeaders); assertNotNull(signature); - assertTrue(verfier.verify(signature, testHeaders)); + assertTrue(verifier.verify(signature, testHeaders)); } for (var clusterAlias : clusterAliases) { clusterRemover.accept(clusterAlias); - var signer = manager.signerForClusterAlias(clusterAlias); - // Make sure no signature was created - assertBusy(() -> assertNull(signer.sign(testHeaders))); + // Make sure no signer can be created + assertBusy(() -> assertNull(manager.signerForClusterAlias(clusterAlias))); } } finally { var builder = Settings.builder(); From 10b02817e9ce21c825cd87da20ab1ac89d5ff8cb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 3 Oct 2025 14:24:53 +0000 Subject: [PATCH 05/10] [CI] Auto commit changes from spotless --- .../transport/CrossClusterSigningConfigReloaderIntegTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java index b97ad427e3059..093a2ea5c60fb 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/CrossClusterSigningConfigReloaderIntegTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.test.SecurityIntegTestCase; -import javax.net.ssl.KeyManagerFactory; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -27,6 +26,8 @@ import java.util.Set; import java.util.function.Consumer; +import javax.net.ssl.KeyManagerFactory; + import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_CERTIFICATE_AUTHORITIES; import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_CERT_PATH; import static org.elasticsearch.xpack.security.transport.CrossClusterApiKeySigningSettings.SIGNING_KEYSTORE_ALGORITHM; From c865f507f5eec4070992005fdf885eaadf9b6426 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Thu, 9 Oct 2025 15:37:38 +0200 Subject: [PATCH 06/10] Review comments --- ...ossClusterAccessAuthenticationService.java | 34 +++++++++++++------ ...ossClusterAccessServerTransportFilter.java | 11 +++--- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java index d4fe3e14a6628..e65b3b05681c5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ContextPreservingActionListener; @@ -75,6 +76,14 @@ public void authenticate(final String action, final TransportRequest request, fi withRequestProcessingFailure(authenticationService.newContext(action, request, null), ex, listener); return; } + + // TODO ALWAYS check if used api key has a certificate identity and do this verification conditionally based on that + var signature = crossClusterAccessHeaders.signature(); + // Always validate a signature if provided + if (signature != null && verifySignature(authcContext, signature, crossClusterAccessHeaders, listener) == false) { + return; + } + try { apiKeyService.ensureEnabled(); } catch (Exception ex) { @@ -113,13 +122,6 @@ public void authenticate(final String action, final TransportRequest request, fi assert authentication.isApiKey() : "initial authentication for cross cluster access must be by API key"; assert false == authentication.isRunAs() : "initial authentication for cross cluster access cannot be run-as"; - // TODO ALWAYS check if used api key has a certificate identity and do this verification conditionally based on that - var signature = crossClusterAccessHeaders.signature(); - // Always validate a signature if provided - if (signature != null) { - verifySignature(signature, crossClusterAccessHeaders); - } - // try-catch so any failure here is wrapped by `withRequestProcessingFailure`, whereas `authenticate` failures are not // we should _not_ wrap `authenticate` failures since this produces duplicate audit events try { @@ -133,23 +135,35 @@ public void authenticate(final String action, final TransportRequest request, fi } } - private void verifySignature(X509CertificateSignature signature, CrossClusterAccessHeaders crossClusterAccessHeaders) { + private boolean verifySignature( + Authenticator.Context context, + X509CertificateSignature signature, + CrossClusterAccessHeaders crossClusterAccessHeaders, + ActionListener listener + ) { assert signature.certificates().length > 0 : "Signatures without certificates should not be considered for verification"; + ElasticsearchSecurityException authException = null; try { if (crossClusterApiKeySignatureVerifier.verify(signature, crossClusterAccessHeaders.signablePayload()) == false) { logger.debug(Strings.format("Invalid cross cluster api key signature received [%s]", signature)); - throw Exceptions.authenticationError( + authException = Exceptions.authenticationError( "Invalid cross cluster api key signature from [{}]", X509CertificateSignature.certificateToString(signature.certificates()[0]) ); } } catch (GeneralSecurityException securityException) { logger.debug(Strings.format("Failed to verify cross cluster api key signature certificate [%s]", signature), securityException); - throw Exceptions.authenticationError( + authException = Exceptions.authenticationError( "Failed to verify cross cluster api key signature certificate from [{}]", X509CertificateSignature.certificateToString(signature.certificates()[0]) ); } + if (authException != null) { + // TODO Verify this covers all audit logging scenarios + listener.onFailure(context.getRequest().exceptionProcessingRequest(authException, context.getMostRecentAuthenticationToken())); + return false; + } + return true; } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java index aad66795a6286..3ae2c22ffbc38 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessServerTransportFilter.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; @@ -24,8 +25,6 @@ import org.elasticsearch.xpack.security.authz.AuthorizationService; import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo.CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY; @@ -37,16 +36,16 @@ final class CrossClusterAccessServerTransportFilter extends ServerTransportFilte private static final Logger logger = LogManager.getLogger(CrossClusterAccessServerTransportFilter.class); // pkg-private for testing - static final Set ALLOWED_TRANSPORT_HEADERS = Stream.concat( - Stream.of( + static final Set ALLOWED_TRANSPORT_HEADERS = Sets.union( + Set.of( CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY, CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY, CROSS_CLUSTER_ACCESS_SIGNATURE_HEADER_KEY, AuditUtil.AUDIT_REQUEST_ID, Task.TRACE_STATE ), - Task.HEADERS_TO_COPY.stream() - ).collect(Collectors.toUnmodifiableSet()); + Task.HEADERS_TO_COPY + ); private final CrossClusterAccessAuthenticationService crossClusterAccessAuthcService; private final XPackLicenseState licenseState; From 113139ef347720cca768fc0132457c04824e4465 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Thu, 9 Oct 2025 15:37:54 +0200 Subject: [PATCH 07/10] fixup! Import --- .../security/authc/CrossClusterAccessAuthenticationService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java index e65b3b05681c5..e5a2b7a53a817 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ContextPreservingActionListener; From 2670f02e29e538c333461ec27f6c6e0ca94cb5b4 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Thu, 9 Oct 2025 16:27:30 +0200 Subject: [PATCH 08/10] Fix failing test --- ...ossClusterAccessAuthenticationServiceTests.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java index c6c37ebcf13ae..cb4f0ea7ed9a2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.transport.CrossClusterApiKeySignatureManager; import org.elasticsearch.xpack.security.transport.X509CertificateSignature; import org.junit.Before; @@ -242,7 +243,8 @@ public void testAuthenticationExceptionOnBadCrossClusterApiKeySignature() throws when(signer.sign(anyString(), anyString())).thenReturn(new X509CertificateSignature(certs, "", mock(BytesReference.class))); crossClusterAccessHeaders.writeToContext(threadContext, signer); - final AuthenticationService.AuditableRequest auditableRequest = mock(AuthenticationService.AuditableRequest.class); + var auditableRequest = mock(AuthenticationService.AuditableRequest.class); + doAnswer(invocationOnMock -> invocationOnMock.getArguments()[0]).when(auditableRequest).exceptionProcessingRequest(any(), any()); var authContext = new Authenticator.Context( threadContext, @@ -255,28 +257,20 @@ public void testAuthenticationExceptionOnBadCrossClusterApiKeySignature() throws when(authenticationService.newContext(anyString(), any(TransportRequest.class), any(ApiKeyService.ApiKeyCredentials.class))) .thenReturn(authContext); - @SuppressWarnings("unchecked") - final ArgumentCaptor> listenerCaptor = ArgumentCaptor.forClass(ActionListener.class); - doAnswer(i -> null).when(authenticationService).authenticate(any(Authenticator.Context.class), listenerCaptor.capture()); - final PlainActionFuture future = new PlainActionFuture<>(); crossClusterAccessAuthenticationService.authenticate(action, request, future); - final Authentication apiKeyAuthentication = AuthenticationTestHelper.builder().apiKey().build(false); - listenerCaptor.getValue().onResponse(apiKeyAuthentication); - final ExecutionException actual = expectThrows(ExecutionException.class, future::get); assertThat(actual.getCause(), instanceOf(ElasticsearchSecurityException.class)); assertThat( - actual.getCause().getMessage(), + actual.getMessage(), containsString( (badCert ? "Failed to verify cross cluster api key signature certificate from [" : "Invalid cross cluster api key signature from [") + X509CertificateSignature.certificateToString(certs[0]) + "]" ) ); - verifyNoMoreInteractions(auditableRequest); } public void testNoInteractionWithAuditableRequestOnInitialAuthenticationFailure() throws IOException { From 66791a09e93b8f809fb4d968f4331a2efbbfd10a Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Fri, 10 Oct 2025 10:02:21 +0200 Subject: [PATCH 09/10] Changes after merge main --- .../referable/add_cross_cluster_api_key_signature.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.3.csv | 2 +- .../transport/CrossClusterAccessTransportInterceptorTests.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv b/server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv index 71ac710f9bfe1..d617a6ff2f5d2 100644 --- a/server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv +++ b/server/src/main/resources/transport/definitions/referable/add_cross_cluster_api_key_signature.csv @@ -1 +1 @@ -9187000 +9191000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index c13b776b0c3a3..03e1a18b61d0a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -reordered_translog_operations,9190000 +add_cross_cluster_api_key_signature,9191000 diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptorTests.java index f1ecfecde42aa..6bfd902f53574 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/CrossClusterAccessTransportInterceptorTests.java @@ -270,7 +270,7 @@ public void sendRequest( } }); final Transport.Connection connection = mock(Transport.Connection.class); - when(connection.getTransportVersion()).thenReturn(TransportVersion.current()); + when(connection.getTransportVersion()).thenReturn(transportVersion); sender.sendRequest(connection, action, request, null, new TransportResponseHandler<>() { @Override From 531d35c72cd38cacd449e31901b93a1468030043 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 10 Oct 2025 08:11:20 +0000 Subject: [PATCH 10/10] [CI] Auto commit changes from spotless --- .../authc/CrossClusterAccessAuthenticationServiceTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java index cb4f0ea7ed9a2..72ffb6d894e2a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.transport.CrossClusterApiKeySignatureManager; import org.elasticsearch.xpack.security.transport.X509CertificateSignature; import org.junit.Before;