Enable run_as for all authentication schemes

x-pack/plugin/build.gradle

@@ -114,6 +114,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure{ task ->
   task.skipTest("indices.freeze/10_basic/Basic", "#70192 -- the freeze index API is removed from 8.0")
   task.skipTest("indices.freeze/10_basic/Test index options", "#70192 -- the freeze index API is removed from 8.0")
   task.skipTest("ml/categorization_agg/Test categorization aggregation with poor settings", "https://github.com/elastic/elasticsearch/pull/79586")
+  task.skipTest("service_accounts/10_basic/Test service account tokens", "mute till we decide whether to make the changes in 7.16")
 
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -113,11 +113,11 @@ public Map<String, Object> getMetadata() {
         return metadata;
     }
 
-    public boolean isServiceAccount() {
-        return ServiceAccountSettings.REALM_TYPE.equals(getAuthenticatedBy().getType()) && null == getLookedUpBy();
+    public boolean isAuthenticatedWithServiceAccount() {
+        return ServiceAccountSettings.REALM_TYPE.equals(getAuthenticatedBy().getType());
     }
 
-    public boolean isApiKey() {
+    public boolean isAuthenticatedWithApiKey() {
         return AuthenticationType.API_KEY.equals(getAuthenticationType());
     }
 
@@ -235,7 +235,7 @@ public void toXContentFragment(XContentBuilder builder) throws IOException {
         builder.array(User.Fields.ROLES.getPreferredName(), user.roles());
         builder.field(User.Fields.FULL_NAME.getPreferredName(), user.fullName());
         builder.field(User.Fields.EMAIL.getPreferredName(), user.email());
-        if (isServiceAccount()) {
+        if (isAuthenticatedWithServiceAccount()) {
             final String tokenName = (String) getMetadata().get(ServiceAccountSettings.TOKEN_NAME_FIELD);
             assert tokenName != null : "token name cannot be null";
             final String tokenSource = (String) getMetadata().get(ServiceAccountSettings.TOKEN_SOURCE_FIELD);
@@ -261,7 +261,7 @@ public void toXContentFragment(XContentBuilder builder) throws IOException {
         }
         builder.endObject();
         builder.field(User.Fields.AUTHENTICATION_TYPE.getPreferredName(), getAuthenticationType().name().toLowerCase(Locale.ROOT));
-        if (isApiKey()) {
+        if (isAuthenticatedWithApiKey()) {
             this.assertApiKeyMetadata();
             final String apiKeyId = (String) this.metadata.get(AuthenticationField.API_KEY_ID_KEY);
             final String apiKeyName = (String) this.metadata.get(AuthenticationField.API_KEY_NAME_KEY);

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java

@@ -126,10 +126,10 @@ public void testIsServiceAccount() {
         );
         final Authentication authentication = new Authentication(user, authRealm, lookupRealm);
 
-        if (authRealmIsForServiceAccount && lookupRealm == null) {
-            assertThat(authentication.isServiceAccount(), is(true));
+        if (authRealmIsForServiceAccount) {
+            assertThat(authentication.isAuthenticatedWithServiceAccount(), is(true));
         } else {
-            assertThat(authentication.isServiceAccount(), is(false));
+            assertThat(authentication.isAuthenticatedWithServiceAccount(), is(false));
         }
     }
 

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java

@@ -8,21 +8,35 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
+
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class RunAsIntegTests extends SecurityIntegTestCase {
 
     private static final String RUN_AS_USER = "run_as_user";
     private static final String CLIENT_USER = "transport_user";
-    private static final String ROLES = "run_as_role:\n" + "  run_as: [ '" + SecuritySettingsSource.TEST_USER_NAME + "', 'idontexist' ]\n";
+    private static final String NO_ROLE_USER = "no_role_user";
+    private static final String ROLES = "run_as_role:\n"
+        + "  cluster: ['manage_own_api_key', 'manage_token']\n"
+        + "  run_as: [ '"
+        + SecuritySettingsSource.TEST_USER_NAME
+        + "', '"
+        + NO_ROLE_USER
+        + "', 'idontexist' ]\n";
 
     // indicates whether the RUN_AS_USER that is being authenticated is also a superuser
     private static boolean runAsHasSuperUserRole;
@@ -52,6 +66,10 @@ public String configUsers() {
             + CLIENT_USER
             + ":"
             + SecuritySettingsSource.TEST_PASSWORD_HASHED
+            + "\n"
+            + NO_ROLE_USER
+            + ":"
+            + SecuritySettingsSource.TEST_PASSWORD_HASHED
             + "\n";
     }
 
@@ -64,6 +82,13 @@ public String configUsersRoles() {
         return roles;
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        final Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
+        builder.put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), "true");
+        return builder.build();
+    }
+
     @Override
     protected boolean transportSSLEnabled() {
         return false;
@@ -119,6 +144,107 @@ public void testNonExistentRunAsUserUsingHttp() throws Exception {
         }
     }
 
+    public void testRunAsUsingApiKey() throws IOException {
+        final Request createApiKeyRequest = new Request("PUT", "/_security/api_key");
+        createApiKeyRequest.setJsonEntity("{\"name\":\"k1\"}\n");
+        createApiKeyRequest.setOptions(
+            createApiKeyRequest.getOptions()
+                .toBuilder()
+                .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, TEST_PASSWORD_SECURE_STRING))
+        );
+        final Response createApiKeyResponse = getRestClient().performRequest(createApiKeyRequest);
+        final XContentTestUtils.JsonMapView apiKeyMapView = XContentTestUtils.createJsonMapView(
+            createApiKeyResponse.getEntity().getContent()
+        );
+
+        final boolean runAsTestUser = false;
+
+        final Request authenticateRequest = new Request("GET", "/_security/_authenticate");
+        authenticateRequest.setOptions(
+            authenticateRequest.getOptions()
+                .toBuilder()
+                .addHeader("Authorization", "ApiKey " + apiKeyMapView.get("encoded"))
+                .addHeader(
+                    AuthenticationServiceField.RUN_AS_USER_HEADER,
+                    runAsTestUser ? SecuritySettingsSource.TEST_USER_NAME : NO_ROLE_USER
+                )
+        );
+        final Response authenticateResponse = getRestClient().performRequest(authenticateRequest);
+        final XContentTestUtils.JsonMapView authenticateJsonView = XContentTestUtils.createJsonMapView(
+            authenticateResponse.getEntity().getContent()
+        );
+        assertThat(authenticateJsonView.get("username"), equalTo(runAsTestUser ? SecuritySettingsSource.TEST_USER_NAME : NO_ROLE_USER));
+        assertThat(authenticateJsonView.get("authentication_realm.type"), equalTo("_es_api_key"));
+        assertThat(authenticateJsonView.get("authentication_type"), equalTo("api_key"));
+
+        final Request getUserRequest = new Request("GET", "/_security/user");
+        getUserRequest.setOptions(
+            getUserRequest.getOptions()
+                .toBuilder()
+                .addHeader("Authorization", "ApiKey " + apiKeyMapView.get("encoded"))
+                .addHeader(
+                    AuthenticationServiceField.RUN_AS_USER_HEADER,
+                    runAsTestUser ? SecuritySettingsSource.TEST_USER_NAME : NO_ROLE_USER
+                )
+        );
+        if (runAsTestUser) {
+            assertThat(getRestClient().performRequest(getUserRequest).getStatusLine().getStatusCode(), equalTo(200));
+        } else {
+            final ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(getUserRequest));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+        }
+    }
+
+    public void testRunAsUsingOAuthToken() throws IOException {
+        final Request createTokenRequest = new Request("POST", "/_security/oauth2/token");
+        createTokenRequest.setJsonEntity("{\"grant_type\":\"client_credentials\"}");
+        createTokenRequest.setOptions(
+            createTokenRequest.getOptions()
+                .toBuilder()
+                .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, TEST_PASSWORD_SECURE_STRING))
+        );
+        final Response createTokenResponse = getRestClient().performRequest(createTokenRequest);
+        final XContentTestUtils.JsonMapView tokenMapView = XContentTestUtils.createJsonMapView(
+            createTokenResponse.getEntity().getContent()
+        );
+
+        final boolean runAsTestUser = randomBoolean();
+
+        final Request authenticateRequest = new Request("GET", "/_security/_authenticate");
+        authenticateRequest.setOptions(
+            authenticateRequest.getOptions()
+                .toBuilder()
+                .addHeader("Authorization", "Bearer " + tokenMapView.get("access_token"))
+                .addHeader(
+                    AuthenticationServiceField.RUN_AS_USER_HEADER,
+                    runAsTestUser ? SecuritySettingsSource.TEST_USER_NAME : NO_ROLE_USER
+                )
+        );
+        final Response authenticateResponse = getRestClient().performRequest(authenticateRequest);
+        final XContentTestUtils.JsonMapView authenticateJsonView = XContentTestUtils.createJsonMapView(
+            authenticateResponse.getEntity().getContent()
+        );
+        assertThat(authenticateJsonView.get("username"), equalTo(runAsTestUser ? SecuritySettingsSource.TEST_USER_NAME : NO_ROLE_USER));
+        assertThat(authenticateJsonView.get("authentication_type"), equalTo("token"));
+
+        final Request getUserRequest = new Request("GET", "/_security/user");
+        getUserRequest.setOptions(
+            getUserRequest.getOptions()
+                .toBuilder()
+                .addHeader("Authorization", "Bearer " + tokenMapView.get("access_token"))
+                .addHeader(
+                    AuthenticationServiceField.RUN_AS_USER_HEADER,
+                    runAsTestUser ? SecuritySettingsSource.TEST_USER_NAME : NO_ROLE_USER
+                )
+        );
+        if (runAsTestUser) {
+            assertThat(getRestClient().performRequest(getUserRequest).getStatusLine().getStatusCode(), equalTo(200));
+        } else {
+            final ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(getUserRequest));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+        }
+    }
+
     private static Request requestForUserRunAsUser(String user) {
         Request request = new Request("GET", "/_nodes");
         RequestOptions.Builder options = request.getOptions().toBuilder();

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java

@@ -75,7 +75,10 @@ protected void doExecute(Task task, CreateTokenRequest request, ActionListener<C
                 break;
             case CLIENT_CREDENTIALS:
                 Authentication authentication = securityContext.getAuthentication();
-                if (authentication.isServiceAccount()) {
+                if (authentication.isAuthenticatedWithServiceAccount() && false == authentication.getUser().isRunAs()) {
+                    // Service account itself cannot create OAuth2 tokens.
+                    // But it is possible to create an oauth2 token if the service account run-as a different user.
+                    // In this case, the token will be created for the run-as user (not the service account).
                     listener.onFailure(new ElasticsearchException("OAuth2 token creation is not supported for service accounts"));
                     return;
                 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -1512,7 +1512,7 @@ LogEntryBuilder withAuthentication(Authentication authentication) {
                     logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName());
                 }
             }
-            if (authentication.isServiceAccount()) {
+            if (authentication.isAuthenticatedWithServiceAccount()) {
                 logEntry.with(SERVICE_TOKEN_NAME_FIELD_NAME, (String) authentication.getMetadata().get(TOKEN_NAME_FIELD))
                     .with(
                         SERVICE_TOKEN_TYPE_FIELD_NAME,

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticatorChain.java

@@ -196,8 +196,7 @@ private void maybeLookupRunAsUser(
         Authentication authentication,
         ActionListener<Authentication> listener
     ) {
-        // TODO: only allow run as for realm authentication to maintain the existing behaviour
-        if (false == runAsEnabled || authentication.getAuthenticationType() != Authentication.AuthenticationType.REALM) {
+        if (false == runAsEnabled) {
             finishAuthentication(context, authentication, listener);
             return;
         }
@@ -227,9 +226,23 @@ private void maybeLookupRunAsUser(
             if (tuple == null) {
                 logger.debug("Cannot find run-as user [{}] for authenticated user [{}]", runAsUsername, user.principal());
                 // the user does not exist, but we still create a User object, which will later be rejected by authz
-                finalAuth = new Authentication(new User(runAsUsername, null, user), authentication.getAuthenticatedBy(), null);
+                finalAuth = new Authentication(
+                    new User(runAsUsername, null, user),
+                    authentication.getAuthenticatedBy(),
+                    null,
+                    authentication.getVersion(),
+                    authentication.getAuthenticationType(),
+                    authentication.getMetadata()
+                );
             } else {
-                finalAuth = new Authentication(new User(tuple.v1(), user), authentication.getAuthenticatedBy(), tuple.v2());
+                finalAuth = new Authentication(
+                    new User(tuple.v1(), user),
+                    authentication.getAuthenticatedBy(),
+                    tuple.v2(),
+                    authentication.getVersion(),
+                    authentication.getAuthenticationType(),
+                    authentication.getMetadata()
+                );
             }
             finishAuthentication(context, finalAuth, listener);
         }, listener::onFailure));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountService.java

@@ -164,7 +164,7 @@ public void findTokensFor(GetServiceAccountCredentialsRequest request, ActionLis
     }
 
     public void getRoleDescriptor(Authentication authentication, ActionListener<RoleDescriptor> listener) {
-        assert authentication.isServiceAccount() : "authentication is not for service account: " + authentication;
+        assert authentication.isAuthenticatedWithServiceAccount() : "authentication is not for service account: " + authentication;
         final String principal = authentication.getUser().principal();
         final ServiceAccount account = ACCOUNTS.get(principal);
         if (account == null) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java

@@ -882,7 +882,10 @@ private ElasticsearchSecurityException denialException(
             }
         }
 
-        String userText = "user [" + authUser.principal() + "]";
+        String userText = (authentication.isAuthenticatedWithServiceAccount() ? "service account" : "user")
+            + " ["
+            + authUser.principal()
+            + "]";
         // check for run as
         if (authentication.getUser().isRunAs()) {
             userText = userText + " run as [" + authentication.getUser().principal() + "]";
@@ -892,9 +895,14 @@ private ElasticsearchSecurityException denialException(
             final String apiKeyId = (String) authentication.getMetadata().get(AuthenticationField.API_KEY_ID_KEY);
             assert apiKeyId != null : "api key id must be present in the metadata";
             userText = "API key id [" + apiKeyId + "] of " + userText;
-        } else if (false == authentication.isServiceAccount()) {
-            // Don't print roles for API keys because they're not meaningful
-            // Also not printing roles for service accounts since they have no roles
+        }
+
+        // The run-as user is always from a realm. So it must have roles that can be printed.
+        // If the user is not run-as, we cannot print the roles if it's an API key or a service account (both do not have
+        // roles, but privileges)
+        if (authentication.getUser().isRunAs()
+            || (false == authentication.isAuthenticatedWithServiceAccount()
+                && AuthenticationType.API_KEY != authentication.getAuthenticationType())) {
             userText = userText + " with roles [" + Strings.arrayToCommaDelimitedString(authentication.getUser().roles()) + "]";
         }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -281,26 +281,37 @@ public void getRoles(User user, Authentication authentication, ActionListener<Ro
             return;
         }
 
-        if (authentication.isServiceAccount()) {
-            getRolesForServiceAccount(authentication, roleActionListener);
-        } else if (ApiKeyService.isApiKeyAuthentication(authentication)) {
-            getRolesForApiKey(authentication, roleActionListener);
+        if (user.isRunAs()) {
+            // The runas user currently must be from a realm and have regular roles
+            getRolesForUser(user, roleActionListener);
         } else {
-            Set<String> roleNames = new HashSet<>(Arrays.asList(user.roles()));
-            if (isAnonymousEnabled && anonymousUser.equals(user) == false) {
-                if (anonymousUser.roles().length == 0) {
-                    throw new IllegalStateException("anonymous is only enabled when the anonymous user has roles");
-                }
-                Collections.addAll(roleNames, anonymousUser.roles());
+            // The authenticated user may not come from a realm and they need to be handled specially
+            if (authentication.isAuthenticatedWithServiceAccount()) {
+                getRolesForServiceAccount(authentication, roleActionListener);
+            } else if (ApiKeyService.isApiKeyAuthentication(authentication)) {
+                // API key role descriptors are stored in the authentication metadata
+                getRolesForApiKey(authentication, roleActionListener);
+            } else {
+                getRolesForUser(user, roleActionListener);
             }
+        }
+    }
 
-            if (roleNames.isEmpty()) {
-                roleActionListener.onResponse(Role.EMPTY);
-            } else if (roleNames.contains(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName())) {
-                roleActionListener.onResponse(superuserRole);
-            } else {
-                roles(roleNames, roleActionListener);
+    private void getRolesForUser(User user, ActionListener<Role> roleActionListener) {
+        Set<String> roleNames = new HashSet<>(Arrays.asList(user.roles()));
+        if (isAnonymousEnabled && anonymousUser.equals(user) == false) {
+            if (anonymousUser.roles().length == 0) {
+                throw new IllegalStateException("anonymous is only enabled when the anonymous user has roles");
             }
+            Collections.addAll(roleNames, anonymousUser.roles());
+        }
+
+        if (roleNames.isEmpty()) {
+            roleActionListener.onResponse(Role.EMPTY);
+        } else if (roleNames.contains(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName())) {
+            roleActionListener.onResponse(superuserRole);
+        } else {
+            roles(roleNames, roleActionListener);
         }
     }