[7.x] Add kibana-system service account (#76449) (#76502)

This change introduces a Service Account for Kibana to use when
authenticating to Elasticsearch. The Service Account with
kibana service name under the elastic namespace,
uses the same RoleDescriptor as the existing kibana_system
built-in user and is its functional equivalent when it comes to
AuthZ.

Author: jkakavas

rest-api-spec/build.gradle

@@ -23,7 +23,6 @@ testClusters.all {
   module ':modules:mapper-extras'
 }
 
-
 tasks.named("test").configure {enabled = false }
 tasks.named("jarHell").configure {enabled = false }
 

x-pack/docs/en/security/authentication/service-accounts.asciidoc

@@ -18,12 +18,12 @@ prevents credential sharing between multiple instances of the same
 external service. Each instance can assume the same identity while using
 their own distinct service token for authentication.
 
-Service accounts provide flexibility over <<built-in-users,built-in users>> 
+Service accounts provide flexibility over <<built-in-users,built-in users>>
 because they:
 
 * Do not rely on the <<native-realm,internal `native` realm>>, and aren't
 always required to rely on the `.security` index
-* Use a role descriptor named after the service account principal instead of 
+* Use a role descriptor named after the service account principal instead of
 traditional roles
 * Support multiple credentials through service account tokens
 
@@ -40,11 +40,15 @@ the format of `<namespace>/<service>`, where the `namespace` is a top-level
 grouping of service accounts, and `service` is the name of the service and
 must be unique within its namespace.
 
-Service accounts are predefined in code. Currently, only one service account is available:
+Service accounts are predefined in code. The following service accounts are
+available:
 
 `elastic/fleet-server`:: The service account used by the {fleet} server to
 communicate with {es}.
 
+`elastic/kibana`:: The service account used by {kib} to communicate with
+{es}.
+
 // tag::service-accounts-usage[]
 IMPORTANT: Do not attempt to use service accounts for authenticating individual
 users. Service accounts can only be authenticated with service tokens, which are
@@ -83,11 +87,11 @@ the bearer token in the HTTP response
 
 Both of these methods create a service token with a guaranteed secret string
 length of `22`. The minimal, acceptable length of a secret string for a service
-token is `10`. If the secret string doesn't meet this minimal length, 
+token is `10`. If the secret string doesn't meet this minimal length,
 authentication with {es} will fail without even checking the value of the
 service token.
 
-Service tokens never expire. You must actively 
+Service tokens never expire. You must actively
 <<security-api-delete-service-token,delete>> them if they are no longer needed.
 
 [discrete]

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java

@@ -21,7 +21,7 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges.ManageApplicationPrivileges;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
-import org.elasticsearch.xpack.core.security.user.KibanaUser;
+import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
 import org.elasticsearch.xpack.core.security.user.UsernamesField;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.elasticsearch.xpack.core.watcher.execution.TriggeredWatchStoreField;
@@ -125,78 +125,7 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                         null, null,
                         MetadataUtils.getDeprecatedReservedMetadata("Please use Kibana feature privileges instead"),
                         null))
-                .put(KibanaUser.ROLE_NAME, new RoleDescriptor(KibanaUser.ROLE_NAME,
-                        new String[] {
-                            "monitor", "manage_index_templates", MonitoringBulkAction.NAME, "manage_saml", "manage_token", "manage_oidc",
-                            InvalidateApiKeyAction.NAME, "grant_api_key",
-                            GetBuiltinPrivilegesAction.NAME, "delegate_pki", GetLifecycleAction.NAME,  PutLifecycleAction.NAME,
-                            // To facilitate ML UI functionality being controlled using Kibana security privileges
-                            "manage_ml",
-                            // The symbolic constant for this one is in SecurityActionMapper, so not accessible from X-Pack core
-                            "cluster:admin/analyze",
-                            // To facilitate using the file uploader functionality
-                            "monitor_text_structure",
-                            // To cancel tasks and delete async searches
-                            "cancel_task"
-                        },
-                        new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".kibana*", ".reporting-*").privileges("all").build(),
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".monitoring-*").privileges("read", "read_cross_cluster").build(),
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".management-beats").privileges("create_index", "read", "write").build(),
-                                // To facilitate ML UI functionality being controlled using Kibana security privileges
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".ml-anomalies*", ".ml-stats-*")
-                                        .privileges("read").build(),
-                                RoleDescriptor.IndicesPrivileges.builder().indices(".ml-annotations*", ".ml-notifications*")
-                                        .privileges("read", "write").build(),
-                                // APM agent configuration
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".apm-agent-configuration").privileges("all").build(),
-                                // APM custom link index creation
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".apm-custom-link").privileges("all").build(),
-                                // APM telemetry queries APM indices in kibana task runner
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices("apm-*")
-                                    .privileges("read", "read_cross_cluster").build(),
-                                // Data telemetry reads mappings, metadata and stats of indices
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices("*")
-                                    .privileges("view_index_metadata", "monitor").build(),
-                                // Endpoint diagnostic information. Kibana reads from these indices to send telemetry
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices(".logs-endpoint.diagnostic.collection-*")
-                                    .privileges("read").build(),
-                                // Fleet Server indices. Kibana create this indice before Fleet Server use them.
-                                // Fleet Server indices. Kibana read and write to this indice to manage Elastic Agents
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices(".fleet*")
-                                    .privileges("all").build(),
-                                // Legacy "Alerts as data" index. Kibana user will create this index.
-                                // Kibana user will read / write to these indices
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices(ReservedRolesStore.LEGACY_ALERTS_INDEX)
-                                    .privileges("all").build(),
-                                // "Alerts as data" index. Kibana user will create this index.
-                                // Kibana user will read / write to these indices
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices(ReservedRolesStore.ALERTS_INDEX)
-                                    .privileges("all").build(),
-                                // Endpoint / Fleet policy responses. Kibana requires read access to send telemetry
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices("metrics-endpoint.policy-*")
-                                    .privileges("read").build(),
-                                // Endpoint metrics. Kibana requires read access to send telemetry
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                    .indices("metrics-endpoint.metrics-*")
-                                    .privileges("read").build()
-                        },
-                        null,
-                        new ConfigurableClusterPrivilege[] { new ManageApplicationPrivileges(Collections.singleton("kibana-*")) },
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA, null))
+                .put(KibanaSystemUser.ROLE_NAME, kibanaSystemRoleDescriptor(KibanaSystemUser.ROLE_NAME))
                 .put("logstash_system", new RoleDescriptor("logstash_system", new String[] { "monitor", MonitoringBulkAction.NAME},
                         null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("beats_admin", new RoleDescriptor("beats_admin",
@@ -442,6 +371,81 @@ private static RoleDescriptor kibanaAdminUser(String name, Map<String, Object> m
             null, null, metadata, null);
     }
 
+    public static RoleDescriptor kibanaSystemRoleDescriptor(String name) {
+        return new RoleDescriptor(name,
+            new String[] {
+                "monitor", "manage_index_templates", MonitoringBulkAction.NAME, "manage_saml", "manage_token", "manage_oidc",
+                InvalidateApiKeyAction.NAME, "grant_api_key",
+                GetBuiltinPrivilegesAction.NAME, "delegate_pki", GetLifecycleAction.NAME,  PutLifecycleAction.NAME,
+                // To facilitate ML UI functionality being controlled using Kibana security privileges
+                "manage_ml",
+                // The symbolic constant for this one is in SecurityActionMapper, so not accessible from X-Pack core
+                "cluster:admin/analyze",
+                // To facilitate using the file uploader functionality
+                "monitor_text_structure",
+                // To cancel tasks and delete async searches
+                "cancel_task"
+            },
+            new RoleDescriptor.IndicesPrivileges[] {
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".kibana*", ".reporting-*").privileges("all").build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".monitoring-*").privileges("read", "read_cross_cluster").build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".management-beats").privileges("create_index", "read", "write").build(),
+                // To facilitate ML UI functionality being controlled using Kibana security privileges
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".ml-anomalies*", ".ml-stats-*")
+                    .privileges("read").build(),
+                RoleDescriptor.IndicesPrivileges.builder().indices(".ml-annotations*", ".ml-notifications*")
+                    .privileges("read", "write").build(),
+                // APM agent configuration
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".apm-agent-configuration").privileges("all").build(),
+                // APM custom link index creation
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".apm-custom-link").privileges("all").build(),
+                // APM telemetry queries APM indices in kibana task runner
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("apm-*")
+                    .privileges("read", "read_cross_cluster").build(),
+                // Data telemetry reads mappings, metadata and stats of indices
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("*")
+                    .privileges("view_index_metadata", "monitor").build(),
+                // Endpoint diagnostic information. Kibana reads from these indices to send telemetry
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".logs-endpoint.diagnostic.collection-*")
+                    .privileges("read").build(),
+                // Fleet Server indices. Kibana create this indice before Fleet Server use them.
+                // Fleet Server indices. Kibana read and write to this indice to manage Elastic Agents
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".fleet*")
+                    .privileges("all").build(),
+                // Legacy "Alerts as data" index. Kibana user will create this index.
+                // Kibana user will read / write to these indices
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(ReservedRolesStore.LEGACY_ALERTS_INDEX)
+                    .privileges("all").build(),
+                // "Alerts as data" index. Kibana user will create this index.
+                // Kibana user will read / write to these indices
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(ReservedRolesStore.ALERTS_INDEX)
+                    .privileges("all").build(),
+                // Endpoint / Fleet policy responses. Kibana requires read access to send telemetry
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("metrics-endpoint.policy-*")
+                    .privileges("read").build(),
+                // Endpoint metrics. Kibana requires read access to send telemetry
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("metrics-endpoint.metrics-*")
+                    .privileges("read").build()
+            },
+            null,
+            new ConfigurableClusterPrivilege[] { new ManageApplicationPrivileges(Collections.singleton("kibana-*")) },
+            null, MetadataUtils.DEFAULT_RESERVED_METADATA, null);
+    }
+
     public static boolean isReserved(String role) {
         return RESERVED_ROLES.containsKey(role) || UsernamesField.SYSTEM_ROLE.equals(role) ||
                 UsernamesField.XPACK_ROLE.equals(role) || UsernamesField.ASYNC_SEARCH_ROLE.equals(role);

x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java

@@ -12,14 +12,20 @@
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
 import org.junit.BeforeClass;
 
 import java.io.FileNotFoundException;
@@ -165,6 +171,18 @@ public void testGetServiceAccount() throws IOException {
         assertServiceAccountRoleDescriptor(getServiceAccountResponse3,
             "elastic/fleet-server", ELASTIC_FLEET_SERVER_ROLE_DESCRIPTOR);
 
+        final Request getServiceAccountRequestKibana = new Request("GET", "_security/service/elastic/kibana");
+        final Response getServiceAccountResponseKibana = client().performRequest(getServiceAccountRequestKibana);
+        assertOK(getServiceAccountResponseKibana);
+        assertServiceAccountRoleDescriptor(
+            getServiceAccountResponseKibana,
+            "elastic/kibana",
+            Strings.toString(
+                ReservedRolesStore.kibanaSystemRoleDescriptor(KibanaSystemUser.ROLE_NAME)
+                    .toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS)
+            )
+        );
+
         final String requestPath = "_security/service/" + randomFrom("foo", "elastic/foo", "foo/bar");
         final Request getServiceAccountRequest4 = new Request("GET", requestPath);
         final Response getServiceAccountResponse4 = client().performRequest(getServiceAccountRequest4);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.List;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.User;
 
 import java.util.Map;
@@ -43,8 +44,10 @@ final class ElasticServiceAccounts {
             null,
             null
         ));
+    private static final ServiceAccount KIBANA_SYSTEM_ACCOUNT =
+        new ElasticServiceAccount("kibana", ReservedRolesStore.kibanaSystemRoleDescriptor(NAMESPACE + "/kibana"));
 
-    static final Map<String, ServiceAccount> ACCOUNTS = List.of(FLEET_ACCOUNT).stream()
+    static final Map<String, ServiceAccount> ACCOUNTS = List.of(FLEET_ACCOUNT, KIBANA_SYSTEM_ACCOUNT).stream()
         .collect(Collectors.toMap(a -> a.id().asPrincipal(), Function.identity()));;
 
     private ElasticServiceAccounts() {}