[Ai4DSoc][Serverless] Update search_ai_lake tier specific roles

.github/CODEOWNERS

@@ -1808,6 +1808,7 @@ x-pack/platform/plugins/shared/ml/server/models/data_recognizer/modules/security
 /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/es/roles.yml @elastic/appex-qa
 /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/oblt/roles.yml @elastic/appex-qa
 /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/security/roles.yml @elastic/appex-qa
+/src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/security/search_ai_lake/roles.yml @elastic/appex-qa
 /x-pack/platform/test/common/ftr_provider_context.ts @elastic/appex-qa
 /x-pack/platform/plugins/shared/maps/test/scout @elastic/appex-qa # temporarily
 /x-pack/platform/plugins/private/discover_enhanced/test/scout/ @elastic/appex-qa # temporarily

packages/kbn-mock-idp-plugin/server/plugin.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { existsSync } from 'fs';
 import { resolve } from 'path';
 
 import type { CloudSetup } from '@kbn/cloud-plugin/server';
@@ -18,6 +19,7 @@ import {
   SERVERLESS_ROLES_ROOT_PATH,
   STATEFUL_ROLES_ROOT_PATH,
 } from '@kbn/es';
+import type { ServerlessProductTier } from '@kbn/es/src/utils';
 import { createSAMLResponse, MOCK_IDP_LOGIN_PATH, MOCK_IDP_LOGOUT_PATH } from '@kbn/mock-idp-utils';
 
 export interface PluginSetupDependencies {
@@ -42,10 +44,25 @@ const projectToAlias = new Map<string, string>([
   // requires update of config/serverless.chat.yml (currently uses projectType 'search')
 ]);
 
-const readServerlessRoles = (projectType: string) => {
+const tierSpecificRolesFileExists = (filePath: string): boolean => {
+  try {
+    return existsSync(filePath);
+  } catch (e) {
+    return false;
+  }
+};
+
+const readServerlessRoles = (projectType: string, productTier?: ServerlessProductTier) => {
   if (projectToAlias.has(projectType)) {
     const alias = projectToAlias.get(projectType)!;
-    const rolesResourcePath = resolve(SERVERLESS_ROLES_ROOT_PATH, alias, 'roles.yml');
+
+    const tierSpecificRolesResourcePath =
+      productTier && resolve(SERVERLESS_ROLES_ROOT_PATH, alias, productTier, 'roles.yml');
+    const rolesResourcePath =
+      tierSpecificRolesResourcePath && tierSpecificRolesFileExists(tierSpecificRolesResourcePath)
+        ? tierSpecificRolesResourcePath
+        : resolve(SERVERLESS_ROLES_ROOT_PATH, alias, 'roles.yml');
+
     return readRolesFromResource(rolesResourcePath);
   } else {
     throw new Error(`Unsupported projectType: ${projectType}`);
@@ -93,7 +110,10 @@ export const plugin: PluginInitializer<
         try {
           if (roles.length === 0) {
             const projectType = plugins.cloud?.serverless?.projectType;
-            roles.push(...(projectType ? readServerlessRoles(projectType) : readStatefulRoles()));
+            const productTier = plugins.cloud?.serverless?.productTier;
+            roles.push(
+              ...(projectType ? readServerlessRoles(projectType, productTier) : readStatefulRoles())
+            );
           }
           return response.ok({
             body: {

src/platform/packages/shared/kbn-cypress-test-helper/src/auth/roles_and_users.ts

@@ -36,6 +36,8 @@ export const SECURITY_SERVERLESS_ROLE_NAMES = Object.freeze({
   endpoint_policy_manager: 'endpoint_policy_manager',
 });
 
+// TODO: Add support for serverless projects with different tiers
+// ref https://github.com/elastic/kibana/pull/229919
 export const ENDPOINT_SECURITY_ROLE_NAMES = Object.freeze({
   // --------------------------------------
   // Set of roles used in serverless

src/platform/packages/shared/kbn-cypress-test-helper/src/auth/saml_auth.ts

@@ -20,6 +20,8 @@ const ES_RESOURCES_DIR = resolve(
   'x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources'
 );
 
+// TODO: Add support for serverless projects with different tiers
+// ref https://github.com/elastic/kibana/pull/229919
 export const ES_RESOURCES = Object.freeze({
   roles: join(ES_RESOURCES_DIR, 'roles.yml'),
   users: join(ES_RESOURCES_DIR, 'users'),

src/platform/packages/shared/kbn-es/src/cli_commands/serverless.ts

@@ -24,11 +24,13 @@ import {
   ServerlessOptions,
   isServerlessProjectType,
   serverlessProjectTypes,
+  ServerlessProductTiers,
 } from '../utils';
 import { Command } from './types';
 import { createCliError } from '../errors';
 
 const supportedProjectTypesStr = Array.from(serverlessProjectTypes).join(' | ').trim();
+const supportedProductTiersStr = Array.from(ServerlessProductTiers).join(' | ').trim();
 
 export const serverless: Command = {
   description: 'Run Serverless Elasticsearch through Docker',
@@ -38,6 +40,7 @@ export const serverless: Command = {
     Options:
 
       --projectType       Serverless project type: ${supportedProjectTypesStr}
+      --productTier      Serverless product tier: ${supportedProductTiersStr}
       --tag               Image tag of ES serverless to run from ${ES_SERVERLESS_REPO_ELASTICSEARCH}
       --image             Full path of ES serverless image to run, has precedence over tag. [default: ${ES_SERVERLESS_DEFAULT_IMAGE}]
       --background        Start ES serverless without attaching to the first node's logs
@@ -120,6 +123,19 @@ export const serverless: Command = {
       );
     }
 
+    if (options.productTier) {
+      if (options.productTier === 'search_ai_lake' && options.projectType !== 'security') {
+        throw createCliError(
+          `--productTier flag 'search_ai_lake' can only be used with projectType 'security'`
+        );
+      }
+      if (!ServerlessProductTiers.has(options.productTier)) {
+        throw createCliError(
+          `--productTier flag and must be a string: ${supportedProductTiersStr}`
+        );
+      }
+    }
+
     if (!isServerlessProjectType(options.projectType)) {
       throw createCliError(
         `Invalid projectType '${options.projectType}', supported values: ${supportedProjectTypesStr}`

src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/security/roles.yml

@@ -1,6 +1,4 @@
-# -----
-# Copy from internal roles config in elasticsearch-controller
-# -----
+# ----- Copy from internal roles config in elasticsearch-controller
 # modeled after the t1_analyst minus osquery run saved queries privilege
 viewer:
   cluster: []

src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/security/search_ai_lake/roles.yml

@@ -0,0 +1,130 @@
+# Predefined roles for "Security" "SearchAiLake" (tier) project types
+# Source: https://github.com/elastic/elasticsearch-controller/blob/main/internal/config/roles/security.search_ai_lake.yaml
+
+_search_ai_lake_analyst:
+  metadata:
+    _public: true
+    _reserved: true
+  cluster:
+    - "monitor_ml"
+    - "monitor_connector"
+  indices:
+    - names:
+        - ".alerts-security*"
+        - ".siem-signals-*"
+        - ".preview.alerts-security*"
+        - ".internal.preview.alerts-security*"
+        - ".adhoc.alerts-security*"
+        - ".internal.adhoc.alerts-security*"
+        - ".internal.alerts-security*"
+        - "security_solution-alert-*"
+      privileges:
+        - "read"
+        - "view_index_metadata"
+    - names:
+        - ".lists*"
+        - ".items*"
+        - ".fleet-agents*"
+        - ".fleet-actions*"
+        - "risk-score.risk-score-*"
+        - ".entities.v1.latest.security_*"
+        - ".ml-anomalies-*"
+        - "security_solution-*.misconfiguration_latest*"
+        - ".elastic-connectors"
+      privileges:
+        - "read"
+    - names:
+        - "synthetics-*-"
+        - "metrics-*-"
+        - "logs-*-"
+        - "traces-*-"
+        - "profiling-*"
+      privileges:
+        - "view_index_metadata"
+        - "read"
+  applications:
+    - application: "kibana-.kibana"
+      privileges:
+        - "feature_ml.read"
+        - "feature_siemV3.all"
+        - "feature_securitySolutionCasesV2.all"
+        - "feature_securitySolutionAssistant.all"
+        - "feature_securitySolutionAttackDiscovery.all"
+        - "feature_builtInAlerts.all"
+        - "feature_actions.all"
+        - "feature_discover_v2.read"
+        - "feature_savedQueryManagement.read"
+        - "feature_indexPatterns.read"
+        - "feature_fleetv2.read"
+      resources: "*"
+  run_as: []
+
+_search_ai_lake_soc_manager:
+  metadata:
+    _public: true
+    _reserved: true
+  cluster:
+    - "monitor_ml"
+    - "monitor_connector"
+  indices:
+    - names:
+        - "apm-*-transaction*"
+        - "traces-apm*"
+        - "auditbeat-*"
+        - "endgame-*"
+        - "filebeat-*"
+        - "logs-*"
+        - "packetbeat-*"
+        - "winlogbeat-*"
+        - "logstash-*"
+        - ".asset-criticality.asset-criticality-*"
+        - "security_solution-*.misconfiguration_latest*"
+      privileges:
+        - "read"
+        - "write"
+    - names:
+        - ".alerts-security*"
+        - ".siem-signals-*"
+        - ".preview.alerts-security*"
+        - ".internal.preview.alerts-security*"
+        - ".adhoc.alerts-security*"
+        - ".internal.adhoc.alerts-security*"
+        - ".internal.alerts-security*"
+      privileges:
+        - "read"
+        - "write"
+        - "manage"
+        - "view_index_metadata"
+    - names:
+        - ".lists*"
+        - ".items*"
+      privileges:
+        - "read"
+        - "write"
+        - "view_index_metadata"
+    - names:
+        - "metrics-endpoint.metadata_current_*"
+        - ".fleet-agents*"
+        - ".fleet-actions*"
+        - "risk-score.risk-score-*"
+        - ".entities.v1.latest.security_*"
+        - ".ml-anomalies-*"
+      privileges:
+        - "read"
+  applications:
+    - application: "kibana-.kibana"
+      privileges:
+        - "feature_siemV3.all"
+        - "feature_siemV3.global_artifact_management_all"
+        - "feature_siemV3.workflow_insights_all"
+        - "feature_securitySolutionCasesV2.all"
+        - "feature_securitySolutionAssistant.all"
+        - "feature_securitySolutionAttackDiscovery.all"
+        - "feature_actions.all"
+        - "feature_builtInAlerts.all"
+        - "feature_indexPatterns.all"
+        - "feature_discover_v2.all"
+        - "feature_savedQueryManagement.all"
+        - "feature_ml.all"
+        - "feature_fleetv2.all"
+      resources: "*"

src/platform/packages/shared/kbn-es/src/utils/docker.ts

@@ -9,7 +9,7 @@
 
 import chalk from 'chalk';
 import execa from 'execa';
-import fs from 'fs';
+import fs, { existsSync } from 'fs';
 import Fsp from 'fs/promises';
 import pRetry from 'p-retry';
 import { resolve, basename, join } from 'path';
@@ -64,11 +64,22 @@ interface BaseOptions extends ImageOptions {
 }
 
 export const serverlessProjectTypes = new Set<string>(['es', 'oblt', 'security', 'chat']);
+export const ServerlessProductTiers = new Set<string>([
+  'essentials',
+  'logs_essentials',
+  'complete',
+  'search_ai_lake',
+]);
 export const isServerlessProjectType = (value: string): value is ServerlessProjectType => {
   return serverlessProjectTypes.has(value);
 };
 
 export type ServerlessProjectType = 'es' | 'oblt' | 'security' | 'chat';
+export type ServerlessProductTier =
+  | 'essentials'
+  | 'logs_essentials'
+  | 'complete'
+  | 'search_ai_lake';
 
 export interface DockerOptions extends EsClusterExecOptions, BaseOptions {
   dockerCmd?: string;
@@ -79,6 +90,8 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions {
   host?: string;
   /**  Serverless project type */
   projectType: ServerlessProjectType;
+  /** Product tier for serverless project */
+  productTier?: ServerlessProductTier;
   /** Clean (or delete) all data created by the ES cluster after it is stopped */
   clean?: boolean;
   /** Full path where the ES cluster will store data */
@@ -630,6 +643,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
     files,
     resources,
     projectType,
+    productTier,
     dataPath = 'stateless',
   } = options;
   const objectStorePath = resolve(basePath, dataPath);
@@ -689,8 +703,21 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
       }, {} as Record<string, string>)
     : {};
 
+  const tierSpecificRolesFileExists = (filePath: string): boolean => {
+    try {
+      return existsSync(filePath);
+    } catch (e) {
+      return false;
+    }
+  };
+
   // Read roles for the specified projectType
-  const rolesResourcePath = resolve(SERVERLESS_ROLES_ROOT_PATH, projectType, 'roles.yml');
+  const tierSpecificRolesResourcePath =
+    productTier && resolve(SERVERLESS_ROLES_ROOT_PATH, projectType, productTier, 'roles.yml');
+  const rolesResourcePath =
+    tierSpecificRolesResourcePath && tierSpecificRolesFileExists(tierSpecificRolesResourcePath)
+      ? tierSpecificRolesResourcePath
+      : resolve(SERVERLESS_ROLES_ROOT_PATH, projectType, 'roles.yml');
 
   const resourcesPaths = [...SERVERLESS_RESOURCES_PATHS, rolesResourcePath];
 

src/platform/packages/shared/kbn-ftr-common-functional-services/services/saml_auth/serverless/auth_provider.ts

@@ -58,6 +58,8 @@ export class ServerlessAuthProvider implements AuthProvider {
       throw new Error(`Unsupported serverless projectType: ${this.projectType}`);
     }
 
+    // TODO: Add support for serverless projects with different tiers
+    // ref https://github.com/elastic/kibana/pull/229919
     this.rolesDefinitionPath = resolve(SERVERLESS_ROLES_ROOT_PATH, this.projectType, 'roles.yml');
   }
 

src/platform/packages/shared/kbn-scout/src/common/services/saml_auth.ts

@@ -20,6 +20,8 @@ import { ScoutTestConfig } from '../../types';
 import { Protocol } from '../../playwright/types';
 import { ScoutLogger } from './logger';
 
+// TODO: Add support for serverless projects with different tiers
+// ref https://github.com/elastic/kibana/pull/229919
 const getResourceDirPath = (config: ScoutTestConfig) => {
   return config.serverless
     ? path.resolve(SERVERLESS_ROLES_ROOT_PATH, config.projectType!)

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/default_roles.ts

@@ -27,6 +27,8 @@ export const defaultRolesFixture = coreWorkerFixtures.extend<
 >({
   defaultRoles: [
     async ({ log, config }, use) => {
+      // TODO: Add support for serverless projects with different tiers
+      // ref https://github.com/elastic/kibana/pull/229919
       const resourcePath = config.serverless
         ? `${SERVERLESS_ROLES_ROOT_PATH}/${config.projectType!}/roles.yml`
         : `${STATEFUL_ROLES_ROOT_PATH}/roles.yml`;

x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/worker/roles_descriptors/index.ts

@@ -22,6 +22,8 @@ export const roleDescriptorsFixture = base.extend<
 >({
   roleDescriptors: [
     ({ log }, use) => {
+      // TODO: Add support for serverless projects with different tiers
+      // ref https://github.com/elastic/kibana/pull/229919
       const resourcePath = `${SERVERLESS_ROLES_ROOT_PATH}/security/roles.yml`;
       const serverless = new Map<string, ElasticsearchRoleDescriptor>(
         Object.entries(

x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts

@@ -64,6 +64,8 @@ export const SECURITY_SERVERLESS_ROLE_NAMES = Object.freeze({
   endpoint_policy_manager: 'endpoint_policy_manager',
 });
 
+// TODO: Add support for serverless projects with different tiers
+// ref https://github.com/elastic/kibana/pull/229919
 export const ENDPOINT_SECURITY_ROLE_NAMES = Object.freeze({
   // --------------------------------------
   // Set of roles used in serverless

x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/index.ts

@@ -11,6 +11,8 @@ import { REPO_ROOT } from '@kbn/repo-info';
 
 const ES_RESOURCES_DIR = resolve(__dirname, 'es_serverless_resources');
 
+// TODO: Add support for serverless projects with different tiers
+// ref https://github.com/elastic/kibana/pull/229919
 export const ES_RESOURCES = Object.freeze({
   roles: join(ES_RESOURCES_DIR, 'roles.yml'),
   users: join(ES_RESOURCES_DIR, 'users'),

x-pack/solutions/security/test/security_solution_cypress/cypress/support/saml_auth.ts

@@ -36,6 +36,8 @@ export const samlAuthentication = async (
     password: config.env.ELASTICSEARCH_PASSWORD,
   };
 
+  // TODO: Add support for serverless projects with different tiers
+  // ref https://github.com/elastic/kibana/pull/229919
   const rolesPath =
     '../../../../../../src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/security/roles.yml';