[kbn/es serverless] Allow ES serverless resources (ex. users, users_roles, etc) to be overwritten via yarn es serverless command

packages/kbn-es/src/cli_commands/serverless.ts

@@ -11,6 +11,8 @@ import getopts from 'getopts';
 import { ToolingLog } from '@kbn/tooling-log';
 import { getTimeReporter } from '@kbn/ci-stats-reporter';
 
+import { basename } from 'path';
+import { SERVERLESS_RESOURCES_PATHS } from '../paths';
 import { Cluster } from '../cluster';
 import {
   ES_SERVERLESS_REPO_ELASTICSEARCH,
@@ -37,6 +39,13 @@ export const serverless: Command = {
       --ssl               Enable HTTP SSL on the ES cluster
       --skipTeardown      If this process exits, leave the ES cluster running in the background
       --waitForReady      Wait for the ES cluster to be ready to serve requests
+      --resources         Overrides resources under ES 'config/' directory, which are by default
+                          mounted from 'packages/kbn-es/src/serverless_resources/users'. Value should
+                          be a valid file path (relative or absolute). This option can be used multiple
+                          times if needing to override multiple files. The following files can be overwritten:
+                          ${SERVERLESS_RESOURCES_PATHS.map((filePath) => basename(filePath)).join(
+                            ' | '
+                          )}
 
       -E                  Additional key=value settings to pass to ES
       -F                  Absolute paths for files to mount into containers
@@ -63,7 +72,7 @@ export const serverless: Command = {
         files: 'F',
       },
 
-      string: ['tag', 'image', 'basePath'],
+      string: ['tag', 'image', 'basePath', 'resources'],
       boolean: ['clean', 'ssl', 'kill', 'background', 'skipTeardown', 'waitForReady'],
 
       default: defaults,

packages/kbn-es/src/serverless_resources/README.md

@@ -14,7 +14,7 @@ password: changeme
 
 ### Adding users
 
-1. Add the user:encrypted_password to `users` file. The encrypted password for `elastic_serverless` is `changeme` if you want to reuse the value.
+1. Add the `user:encrypted_password` to `users` file. The encrypted password for `elastic_serverless` is `changeme` if you want to reuse the value.
 1. Set the new user's roles in `users_roles` file.
 1. Add the username to `operator_users.yml` in the array for file realm users.
 
@@ -46,4 +46,15 @@ If a node is configured to use this `service_tokens` file, then you can authenti
 curl -H "Authorization: Bearer AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA" http://localhost:9200/_security/_authenticate
 ```
 
-The name of the token (`kibana-dev`) is important because the `operator_users.yml` file designates that token as an operator and allows us to seed a serverless cluster with this token.
+The name of the token (`kibana-dev`) is important because the `operator_users.yml` file designates that token as an operator and allows us to seed a serverless cluster with this token.
+
+
+## Overriding resources
+
+The files found in this directory can be overwritten with customized versions by using the `--resources` option of the `yarn es serverless` command.
+Assuming a customized `users` and `users_roles` are located in `/tmp/my_es/` directory and executing the below command from the root of Kibana, here is an example:
+
+```shell
+yarn es serverless --resources=/tmp/my_es/users --resources=/tmp/my_es/users_roles 
+```
+

packages/kbn-es/src/serverless_resources/operator_users.yml

@@ -1,5 +1,5 @@
 operator:
-  - usernames: ["elastic_serverless", "system_indices_superuser"]
+  - usernames: ["elastic_serverless", "system_indices_superuser", "soc_manager"]
     realm_type: "file"
     auth_type: "realm"
   - usernames: [ "elastic/kibana" ]

packages/kbn-es/src/serverless_resources/users

@@ -1,2 +1,3 @@
 elastic_serverless:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW
 system_indices_superuser:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW
+soc_manager:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW

packages/kbn-es/src/serverless_resources/users_roles

@@ -1,2 +1,3 @@
 superuser:elastic_serverless
 system_indices_superuser:system_indices_superuser
+soc_manager:soc_manager

packages/kbn-es/src/utils/docker.test.ts

@@ -445,6 +445,35 @@ describe('setupServerlessVolumes()', () => {
     expect(volumeCmd).toHaveLength(20);
     expect(pathsNotIncludedInCmd).toEqual([]);
   });
+
+  test('should use resource overrides', async () => {
+    mockFs(existingObjectStore);
+    const volumeCmd = await setupServerlessVolumes(log, {
+      basePath: baseEsPath,
+      resources: ['./relative/path/users', '/absolute/path/users_roles'],
+    });
+
+    expect(volumeCmd).toContain(
+      '/absolute/path/users_roles:/usr/share/elasticsearch/config/users_roles'
+    );
+    expect(volumeCmd).toContain(
+      `${process.cwd()}/relative/path/users:/usr/share/elasticsearch/config/users`
+    );
+  });
+
+  test('should throw if an unknown resource override is used', async () => {
+    mockFs(existingObjectStore);
+
+    await expect(async () => {
+      await setupServerlessVolumes(log, {
+        basePath: baseEsPath,
+        resources: ['/absolute/path/invalid'],
+      });
+    }).rejects.toThrow(
+      'Unsupported ES serverless --resources value(s):\n  /absolute/path/invalid\n\n' +
+        'Valid resources: operator_users.yml | role_mapping.yml | roles.yml | service_tokens | users | users_roles'
+    );
+  });
 });
 
 describe('runServerlessEsNode()', () => {

packages/kbn-es/src/utils/docker.ts

@@ -66,6 +66,11 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions {
   background?: boolean;
   /** Wait for the ES cluster to be ready to serve requests */
   waitForReady?: boolean;
+  /**
+   * Resource file(s) to overwrite
+   * (see list of files that can be overwritten under `packages/kbn-es/src/serverless_resources/users`)
+   */
+  resources?: string | string[];
 }
 
 interface ServerlessEsNodeArgs {
@@ -470,7 +475,7 @@ export function getDockerFileMountPath(hostPath: string) {
  * Setup local volumes for Serverless ES
  */
 export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) {
-  const { basePath, clean, ssl, files } = options;
+  const { basePath, clean, ssl, files, resources } = options;
   const objectStorePath = resolve(basePath, 'stateless');
 
   log.info(chalk.bold(`Checking for local serverless ES object store at ${objectStorePath}`));
@@ -509,12 +514,38 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
     volumeCmds.push(...fileCmds);
   }
 
+  const resourceFileOverrides: Record<string, string> = resources
+    ? (Array.isArray(resources) ? resources : [resources]).reduce((acc, filePath) => {
+        acc[basename(filePath)] = resolve(process.cwd(), filePath);
+        return acc;
+      }, {} as Record<string, string>)
+    : {};
+
   const serverlessResources = SERVERLESS_RESOURCES_PATHS.reduce<string[]>((acc, path) => {
-    acc.push('--volume', `${path}:${SERVERLESS_CONFIG_PATH}${basename(path)}`);
+    const fileName = basename(path);
+    let localFilePath = path;
+
+    if (resourceFileOverrides[fileName]) {
+      localFilePath = resourceFileOverrides[fileName];
+      log.info(`'${fileName}' resource overridden with: ${localFilePath}`);
+      delete resourceFileOverrides[fileName];
+    }
+
+    acc.push('--volume', `${localFilePath}:${SERVERLESS_CONFIG_PATH}${fileName}`);
 
     return acc;
   }, []);
 
+  if (Object.keys(resourceFileOverrides).length > 0) {
+    throw new Error(
+      `Unsupported ES serverless --resources value(s):\n  ${Object.values(
+        resourceFileOverrides
+      ).join('  \n')}\n\nValid resources: ${SERVERLESS_RESOURCES_PATHS.map((filePath) =>
+        basename(filePath)
+      ).join(' | ')}`
+    );
+  }
+
   volumeCmds.push(
     ...getESp12Volume(),
     ...serverlessResources,

x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/README.md

@@ -0,0 +1,47 @@
+# Security Solution Serverless Resources
+
+Directory contains ES serverless resources that can be used to override the defaults that are loaded when ES is started in serverless mode. For more information on how these are used [packages/kbn-es/src/serverless_resources/README.md](https://github.com/elastic/kibana/blob/main/packages/kbn-es/src/serverless_resources/README.md)
+
+> **ℹ️ NOTE**
+> 
+> The files referenced via `--resources` argument will be bound and mounted to the ES docker containers that are running ES. This means that any changes to the files done on the host machine will be automatically (after a delay - 5s by default) picked up by Elasticsearch and applied to the ES docker nodes.
+
+## Usage
+
+Example executed from the root directory of Kibana: 
+
+```shell
+yarn es serverless \
+--clean \
+--kill \
+-E xpack.security.authc.api_key.enabled=true \
+-E http.host=0.0.0.0 \
+--resources=./x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/roles.yml \
+--resources=./x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/users \
+--resources=./x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/users_roles
+```
+
+> **💡️TIP**
+> 
+> If needing to make custom changes to any of the ES resources for personal dev. purposes, copy the files located in this folder to your own local directly, make changes there and then use those file paths when starting ES
+
+
+
+## Files
+
+### `roles.yml`
+
+The list of Roles that are loaded into security serverless projects. The values in this file should match those in the [project controller](https://github.com/elastic/project-controller/blob/main/internal/project/security/config/roles.yml) and should remain in sync.
+
+### `users`
+
+List of users that are loaded into ES for serverless. This file currently includes a user for each of the Security Project roles (same name as the role). All users in this file have their password set to `changeme`
+
+Format: `user:encrypted_password`
+
+### `users_roles`
+
+A map of role names (should match those define in the `roles.yml`) to list of users (values found in the `users` file). All Security serverless roles are listed in this file along with one user by the same name.
+
+Format: `role_name:username,username,username`
+