diff --git a/src/platform/packages/private/kbn-mock-idp-utils/src/constants.ts b/src/platform/packages/private/kbn-mock-idp-utils/src/constants.ts index 994341172cde6..a5ae7f7de05eb 100644 --- a/src/platform/packages/private/kbn-mock-idp-utils/src/constants.ts +++ b/src/platform/packages/private/kbn-mock-idp-utils/src/constants.ts @@ -35,6 +35,7 @@ export const MOCK_IDP_UIAM_COSMOS_DB_URL = export const MOCK_IDP_UIAM_ORGANIZATION_ID = 'org1234567890'; export const MOCK_IDP_UIAM_PROJECT_ID = 'abcdef12345678901234567890123456'; +export const MOCK_IDP_UIAM_PROJECT_ID2 = 'fedcba65432109876543210987654321'; // Sometimes it is useful or required to point local UIAM service clients, or clients operating within the same Docker // network (i.e., Elasticsearch), to a different UIAM service URL. For example, http://host.docker.internal:8080 can be diff --git a/src/platform/packages/private/kbn-mock-idp-utils/src/index.ts b/src/platform/packages/private/kbn-mock-idp-utils/src/index.ts index 307f4b30a77e6..219f05521c1bb 100644 --- a/src/platform/packages/private/kbn-mock-idp-utils/src/index.ts +++ b/src/platform/packages/private/kbn-mock-idp-utils/src/index.ts @@ -36,6 +36,7 @@ export { MOCK_IDP_UIAM_ORG_ADMIN_API_KEY, MOCK_IDP_UIAM_ORGANIZATION_ID, MOCK_IDP_UIAM_PROJECT_ID, + MOCK_IDP_UIAM_PROJECT_ID2, } from './constants'; export { diff --git a/src/platform/packages/shared/kbn-es/index.ts b/src/platform/packages/shared/kbn-es/index.ts index 9e3a1d80992ed..419dd1b32c9ef 100644 --- a/src/platform/packages/shared/kbn-es/index.ts +++ b/src/platform/packages/shared/kbn-es/index.ts @@ -14,6 +14,9 @@ export { ELASTIC_SERVERLESS_SUPERUSER, ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, SERVERLESS_NODES, + LINKED_CLUSTER_PORT_OFFSET, + getServerlessNodes, + getSharedServerlessParams, getDockerFileMountPath, verifyDockerInstalled, maybeCreateDockerNetwork, diff --git a/src/platform/packages/shared/kbn-es/src/utils/docker.test.ts b/src/platform/packages/shared/kbn-es/src/utils/docker.test.ts index cc84effe6ecb7..4c8ed1c11c452 100644 --- a/src/platform/packages/shared/kbn-es/src/utils/docker.test.ts +++ b/src/platform/packages/shared/kbn-es/src/utils/docker.test.ts @@ -33,6 +33,8 @@ import { teardownServerlessClusterSync, verifyDockerInstalled, getESp12Volume, + getServerlessNodes, + getSharedServerlessParams, } from './docker'; import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log'; import { CA_CERT_PATH, ES_P12_PATH } from '@kbn/dev-utils'; @@ -136,6 +138,59 @@ const volumeCmdTest = async (volumeCmd: string[]) => { expect((await Fsp.stat(serverlessObjectStorePath)).mode & 0o777).toBe(0o777); }; +describe('getServerlessNodes()', () => { + test('should return default node names and ports with no arguments', () => { + const nodes = getServerlessNodes(); + expect(nodes).toHaveLength(3); + expect(nodes[0].name).toBe('es01'); + expect(nodes[1].name).toBe('es02'); + expect(nodes[2].name).toBe('es03'); + expect(nodes[0].params).toEqual(expect.arrayContaining(['127.0.0.1:9300:9300'])); + expect(nodes[1].params).toEqual(expect.arrayContaining(['127.0.0.1:9202:9202'])); + expect(nodes[2].params).toEqual(expect.arrayContaining(['127.0.0.1:9203:9203'])); + }); + + test('should apply name suffix and port offset for linked cluster', () => { + const nodes = getServerlessNodes('-linked', 10); + expect(nodes).toHaveLength(3); + expect(nodes[0].name).toBe('es01-linked'); + expect(nodes[1].name).toBe('es02-linked'); + expect(nodes[2].name).toBe('es03-linked'); + expect(nodes[0].params).toEqual(expect.arrayContaining(['127.0.0.1:9310:9310'])); + expect(nodes[1].params).toEqual(expect.arrayContaining(['127.0.0.1:9212:9212'])); + expect(nodes[1].params).toEqual(expect.arrayContaining(['127.0.0.1:9312:9312'])); + expect(nodes[2].params).toEqual(expect.arrayContaining(['127.0.0.1:9213:9213'])); + expect(nodes[2].params).toEqual(expect.arrayContaining(['127.0.0.1:9313:9313'])); + }); + + test('should configure discovery hosts with suffixed names', () => { + const nodes = getServerlessNodes('-linked', 10); + expect(nodes[0].params).toEqual( + expect.arrayContaining([`discovery.seed_hosts=es02-linked,es03-linked`]) + ); + expect(nodes[1].params).toEqual( + expect.arrayContaining([`discovery.seed_hosts=es01-linked,es03-linked`]) + ); + expect(nodes[2].params).toEqual( + expect.arrayContaining([`discovery.seed_hosts=es01-linked,es02-linked`]) + ); + }); +}); + +describe('getSharedServerlessParams()', () => { + test('should return default master nodes with no arguments', () => { + const params = getSharedServerlessParams(); + expect(params).toEqual(expect.arrayContaining(['cluster.initial_master_nodes=es01,es02,es03'])); + }); + + test('should return suffixed master nodes for linked cluster', () => { + const params = getSharedServerlessParams('-linked'); + expect(params).toEqual( + expect.arrayContaining(['cluster.initial_master_nodes=es01-linked,es02-linked,es03-linked']) + ); + }); +}); + describe('resolveDockerImage()', () => { const defaultRepo = 'another/repo'; const defaultImg = 'default/reg/repo:tag'; @@ -637,6 +692,37 @@ describe('resolveEsArgs()', () => { `); }); + test('should use projectIdOverride when provided in UIAM mode', () => { + const overrideId = 'custom_project_id_123'; + const esArgs = resolveEsArgs( + [], + { + ssl: true, + kibanaUrl: 'http://localhost:5601/', + projectType, + basePath: baseEsPath, + uiam: true, + }, + overrideId + ); + + expect(findEnvValue(esArgs, 'serverless.project_id')).toBe(overrideId); + expect(findEnvValue(esArgs, 'serverless.organization_id')).toBeDefined(); + expect(findEnvValue(esArgs, 'serverless.universal_iam_service.enabled')).toBe('true'); + }); + + test('should use default project ID when no override is provided in UIAM mode', () => { + const esArgs = resolveEsArgs([], { + ssl: true, + kibanaUrl: 'http://localhost:5601/', + projectType, + basePath: baseEsPath, + uiam: true, + }); + + expect(findEnvValue(esArgs, 'serverless.project_id')).toBe('abcdef12345678901234567890123456'); + }); + test('should append refresh interval override when ES_JAVA_OPTS is provided', () => { const esArgs = resolveEsArgs([], { esArgs: 'ES_JAVA_OPTS=-Xms1g -Xmx1g' }); @@ -831,13 +917,13 @@ describe('runServerlessCluster()', () => { // docker version (1) // docker ps (1) - // docker container rm (5 = 3 for ES nodes, 2 for UIAM containers) + // docker container rm (8 = 3 for ES nodes, 3 for linked ES nodes, 2 for UIAM containers) // docker network create (1) // docker pull (1) // docker inspect (1) // docker run (3) // docker logs (1) - expect(execa.mock.calls).toHaveLength(14); + expect(execa.mock.calls).toHaveLength(17); // UIAM containers should not be started when `--uiam` is not passed expect(runUiamContainerMock).not.toHaveBeenCalled(); @@ -855,13 +941,13 @@ describe('runServerlessCluster()', () => { // docker version (1) // docker ps (1) - // docker container rm (5 = 3 for ES nodes, 2 for UIAM containers) + // docker container rm (8 = 3 for ES nodes, 3 for linked ES nodes, 2 for UIAM containers) // docker network create (1) // docker pull (3 = 1 for ES nodes, 2 for UIAM containers) // docker inspect (2 = image info call for ES nodes is memoized in the previous test, 2 for UIAM containers) // docker run (3) // docker logs (1) - expect(execa.mock.calls).toHaveLength(17); + expect(execa.mock.calls).toHaveLength(20); expect(runUiamContainerMock).toHaveBeenCalledTimes(2); expect(runUiamContainerMock).toHaveBeenCalledWith( @@ -1023,12 +1109,12 @@ describe('runDockerContainer()', () => { await expect(runDockerContainer(log, {})).resolves.toBeUndefined(); // docker version (1) // docker ps (1) - // docker container rm (5 = 3 for ES nodes, 2 for UIAM containers) + // docker container rm (8 = 3 for ES nodes, 3 for linked ES nodes, 2 for UIAM containers) // docker network create (1) // docker pull (1) // docker inspect (1) // docker run (1) - expect(execa.mock.calls).toHaveLength(11); + expect(execa.mock.calls).toHaveLength(14); }); }); diff --git a/src/platform/packages/shared/kbn-es/src/utils/docker.ts b/src/platform/packages/shared/kbn-es/src/utils/docker.ts index 934844477a136..1fad2528dc31c 100644 --- a/src/platform/packages/shared/kbn-es/src/utils/docker.ts +++ b/src/platform/packages/shared/kbn-es/src/utils/docker.ts @@ -159,6 +159,8 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions { resources?: string | string[]; /** Configure ES serverless with UIAM support */ uiam?: boolean; + /** Configuration for a linked project in Cross Project Search (CPS) mode */ + linkedProject?: { projectId: string; port: number }; } interface ServerlessEsNodeArgs { @@ -210,30 +212,36 @@ export const ES_SERVERLESS_DEFAULT_IMAGE = `${ES_SERVERLESS_REPO_KIBANA}:${ES_SE // See for default cluster settings // https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/kotlin/elasticsearch.serverless-run.gradle.kts -const SHARED_SERVERLESS_PARAMS = [ - 'run', +export function getSharedServerlessParams(nameSuffix = ''): string[] { + const n1 = `es01${nameSuffix}`; + const n2 = `es02${nameSuffix}`; + const n3 = `es03${nameSuffix}`; - '--detach', + return [ + 'run', - '--interactive', + '--detach', - '--tty', + '--interactive', - '--net', - 'elastic', + '--tty', - '--env', - 'path.repo=/objectstore', + '--net', + 'elastic', - '--env', - 'cluster.initial_master_nodes=es01,es02,es03', + '--env', + 'path.repo=/objectstore', - '--env', - 'stateless.enabled=true', + '--env', + `cluster.initial_master_nodes=${n1},${n2},${n3}`, - '--env', - 'stateless.object_store.type=fs', -]; + '--env', + 'stateless.enabled=true', + + '--env', + 'stateless.object_store.type=fs', + ]; +} // only allow certain ES args to be overwrote by options const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ @@ -315,62 +323,73 @@ const DOCKER_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.transport.ssl.keystore.password', ES_P12_PASSWORD], ]; -export const SERVERLESS_NODES: Array> = [ - { - name: 'es01', - params: [ - '-p', - '127.0.0.1:9300:9300', - - '--env', - 'discovery.seed_hosts=es02,es03', - - '--env', - 'node.roles=["master","remote_cluster_client","ingest","index"]', - ], - esArgs: [ - ['xpack.searchable.snapshot.shared_cache.size', '16MB'], - ['xpack.searchable.snapshot.shared_cache.region_size', '256K'], - ['ES_JAVA_OPTS', '-Xms1536m -Xmx1536m'], - ], - }, - { - name: 'es02', - params: [ - '-p', - '127.0.0.1:9202:9202', - - '-p', - '127.0.0.1:9302:9302', - - '--env', - 'discovery.seed_hosts=es01,es03', - - '--env', - 'node.roles=["master","remote_cluster_client","search"]', - ], - esArgs: [ - ['xpack.searchable.snapshot.shared_cache.size', '16MB'], - ['xpack.searchable.snapshot.shared_cache.region_size', '256K'], - ], - }, - { - name: 'es03', - params: [ - '-p', - '127.0.0.1:9203:9203', - - '-p', - '127.0.0.1:9303:9303', - - '--env', - 'discovery.seed_hosts=es01,es02', - - '--env', - 'node.roles=["master","remote_cluster_client","ml","transform"]', - ], - }, -]; +export function getServerlessNodes( + nameSuffix = '', + portOffset = 0 +): Array> { + const n1 = `es01${nameSuffix}`; + const n2 = `es02${nameSuffix}`; + const n3 = `es03${nameSuffix}`; + + return [ + { + name: n1, + params: [ + '-p', + `127.0.0.1:${9300 + portOffset}:${9300 + portOffset}`, + + '--env', + `discovery.seed_hosts=${n2},${n3}`, + + '--env', + 'node.roles=["master","remote_cluster_client","ingest","index"]', + ], + esArgs: [ + ['xpack.searchable.snapshot.shared_cache.size', '16MB'], + ['xpack.searchable.snapshot.shared_cache.region_size', '256K'], + ['ES_JAVA_OPTS', '-Xms1536m -Xmx1536m'], + ], + }, + { + name: n2, + params: [ + '-p', + `127.0.0.1:${9202 + portOffset}:${9202 + portOffset}`, + + '-p', + `127.0.0.1:${9302 + portOffset}:${9302 + portOffset}`, + + '--env', + `discovery.seed_hosts=${n1},${n3}`, + + '--env', + 'node.roles=["master","remote_cluster_client","search"]', + ], + esArgs: [ + ['xpack.searchable.snapshot.shared_cache.size', '16MB'], + ['xpack.searchable.snapshot.shared_cache.region_size', '256K'], + ], + }, + { + name: n3, + params: [ + '-p', + `127.0.0.1:${9203 + portOffset}:${9203 + portOffset}`, + + '-p', + `127.0.0.1:${9303 + portOffset}:${9303 + portOffset}`, + + '--env', + `discovery.seed_hosts=${n1},${n2}`, + + '--env', + 'node.roles=["master","remote_cluster_client","ml","transform"]', + ], + }, + ]; +} + +export const SERVERLESS_NODES = getServerlessNodes(); /** * Determine the Docker image from CLI options and defaults @@ -521,7 +540,8 @@ export async function cleanUpDanglingContainers(log: ToolingLog) { log.info(chalk.bold('Cleaning up dangling Docker containers.')); try { - const serverlessContainerNames = SERVERLESS_NODES.concat(UIAM_CONTAINERS).map( + const linkedNodes = getServerlessNodes('-linked', 10); + const serverlessContainerNames = SERVERLESS_NODES.concat(linkedNodes, UIAM_CONTAINERS).map( ({ name }) => name ); @@ -538,11 +558,11 @@ export async function cleanUpDanglingContainers(log: ToolingLog) { } export async function detectRunningNodes(log: ToolingLog, options: BaseOptions) { - const namesCmd = SERVERLESS_NODES.concat(UIAM_CONTAINERS).reduce((acc, { name }) => { - acc.push('--filter', `name=${name}`); - - return acc; - }, []); + const linkedNodes = getServerlessNodes('-linked', 10); + const namesCmd = SERVERLESS_NODES.concat(linkedNodes, UIAM_CONTAINERS).flatMap(({ name }) => [ + '--filter', + `name=${name}`, + ]); const { stdout } = await execa('docker', ['ps', '--quiet'].concat(namesCmd)); const runningNodeIds = stdout.split(/\r?\n/).filter((s) => s); @@ -584,7 +604,8 @@ async function setupDockerImage({ log, image }: { log: ToolingLog; image: string */ export function resolveEsArgs( defaultEsArgs: Array<[string, string]>, - options: ServerlessOptions | DockerOptions + options: ServerlessOptions | DockerOptions, + projectIdOverride?: string ) { const { esArgs: customEsArgs, password, ssl } = options; const esArgs = new Map(defaultEsArgs); @@ -680,11 +701,16 @@ export function resolveEsArgs( esArgs.set('serverless.organization_id', MOCK_IDP_UIAM_ORGANIZATION_ID); esArgs.set('serverless.project_type', esProjectTypeFromKbn.get(options.projectType)!); - esArgs.set('serverless.project_id', MOCK_IDP_UIAM_PROJECT_ID); + esArgs.set('serverless.project_id', projectIdOverride ?? MOCK_IDP_UIAM_PROJECT_ID); esArgs.set('serverless.universal_iam_service.enabled', 'true'); esArgs.set('serverless.universal_iam_service.url', MOCK_IDP_UIAM_SERVICE_INTERNAL_URL); esArgs.set('serverless.universal_iam_service.ssl.verification_mode', 'none'); + + if ('linkedProject' in options && options.linkedProject) { + esArgs.set('serverless.cross_project.enabled', 'true'); + esArgs.set('remote_cluster_server.enabled', 'true'); + } } } @@ -737,7 +763,11 @@ export function getDockerFileMountPath(hostPath: string) { /** * Setup local volumes for Serverless ES */ -export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) { +export async function setupServerlessVolumes( + log: ToolingLog, + options: ServerlessOptions, + overrides?: { projectId?: string; operatorPath?: string } +) { const { basePath, clean, @@ -867,7 +897,11 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles volumeCmds.push( ...getESp12Volume(), ...serverlessResources, - ...(await getOperatorVolume(esProjectTypeFromKbn.get(projectType)!)), + ...(await getOperatorVolume( + esProjectTypeFromKbn.get(projectType)!, + overrides?.projectId, + overrides?.operatorPath + )), '--volume', `${ @@ -897,9 +931,10 @@ function getServerlessImage({ image, tag }: ImageOptions) { */ export async function runServerlessEsNode( log: ToolingLog, - { params, name, image }: ServerlessEsNodeArgs + { params, name, image }: ServerlessEsNodeArgs, + sharedParams: string[] = getSharedServerlessParams() ) { - const dockerCmd = SHARED_SERVERLESS_PARAMS.concat( + const dockerCmd = sharedParams.concat( params, ['--name', name, '--env', `node.name=${name}`], image @@ -1047,6 +1082,163 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO return nodeNames; } +export const LINKED_CLUSTER_NAME_SUFFIX = '-linked'; +export const LINKED_CLUSTER_PORT_OFFSET = 10; + +/** + * Starts a linked ES Serverless cluster for Cross Project Search (CPS). + * Must be called AFTER the origin cluster and UIAM are fully ready. + * Reuses the same Docker network, ES image, and UIAM service -- does NOT start new UIAM containers. + */ +export async function runLinkedServerlessCluster(log: ToolingLog, options: ServerlessOptions) { + const { linkedProject } = options; + if (!linkedProject) { + return []; + } + + log.info(chalk.bold('Starting linked ES cluster for Cross Project Search...')); + + const esServerlessImage = getServerlessImage({ image: options.image, tag: options.tag }); + const linkedNodes = getServerlessNodes(LINKED_CLUSTER_NAME_SUFFIX, LINKED_CLUSTER_PORT_OFFSET); + const linkedSharedParams = getSharedServerlessParams(LINKED_CLUSTER_NAME_SUFFIX); + + const linkedBasePath = resolve(options.basePath, `linked`); + const linkedOptions: ServerlessOptions = { + ...options, + basePath: linkedBasePath, + port: linkedProject.port, + dataPath: `stateless${LINKED_CLUSTER_NAME_SUFFIX}`, + uiam: true, + }; + + const linkedOperatorPath = resolve(REPO_ROOT, '.es', `operator${LINKED_CLUSTER_NAME_SUFFIX}`); + const volumeCmd = await setupServerlessVolumes(log, linkedOptions, { + projectId: linkedProject.projectId, + operatorPath: linkedOperatorPath, + }); + const portCmd = resolvePort(linkedOptions); + + const linkedClusterEsArgs: Array<[string, string]> = DEFAULT_SERVERLESS_ESARGS.concat([ + ['cluster.name', `stateless${LINKED_CLUSTER_NAME_SUFFIX}`], + ]); + + const nodeNames = await Promise.all( + linkedNodes.map(async (node, i) => { + await runServerlessEsNode( + log, + { + ...node, + image: esServerlessImage, + params: node.params.concat( + resolveEsArgs( + linkedClusterEsArgs.concat(node.esArgs ?? []), + linkedOptions, + linkedProject.projectId + ), + i === 0 ? portCmd : [], + volumeCmd + ), + }, + linkedSharedParams + ); + return node.name; + }) + ); + + log.success(`Linked ES cluster running for CPS (Cross-Project Search). + Project ID: ${chalk.bold.cyan(linkedProject.projectId)} + HTTP port: ${chalk.bold.cyan(String(linkedProject.port))} + Stop the cluster: ${chalk.bold(`docker container stop ${nodeNames.join(' ')}`)} + `); + + const esNodeUrl = `${options.ssl ? 'https' : 'http'}://${portCmd[1].substring( + 0, + portCmd[1].lastIndexOf(':') + )}`; + + const client = getESClient({ + node: esNodeUrl, + auth: { + username: ELASTIC_SERVERLESS_SUPERUSER, + password: ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, + }, + ...(options.ssl + ? { + tls: { + ca: [fs.readFileSync(CA_CERT_PATH)], + checkServerIdentity: () => { + return undefined; + }, + }, + } + : {}), + }); + + await waitUntilClusterReady({ client, expectedStatus: 'green', log }); + + if (options.ssl && options.kibanaUrl) { + await ensureSAMLRoleMapping(client); + } + + if (!options.esArgs || !options.esArgs.includes('xpack.security.enabled=false')) { + await waitForSecurityIndex({ client, log }); + } + + log.success('Linked ES cluster is ready.'); + + await registerLinkedProjectInOriginSettings(log, options); + + return nodeNames; +} + +const REMOTE_CLUSTER_SERVER_PORT = 9400; + +/** + * Updates the origin cluster's operator settings.json to register the linked project, + * so ES can discover it for Cross Project Search via the /_project/tags API. + * + * The file is bind-mounted from the host, so writing it triggers an ES config reload. + */ +async function registerLinkedProjectInOriginSettings(log: ToolingLog, options: ServerlessOptions) { + const { linkedProject } = options; + if (!linkedProject) { + return; + } + + const settingsPath = join(SERVERLESS_OPERATOR_PATH, 'settings.json'); + log.info('Registering linked project in origin operator settings...'); + + const currentJson = JSON.parse(await Fsp.readFile(settingsPath, 'utf-8')); + + const esProjectType = esProjectTypeFromKbn.get(options.projectType) ?? options.projectType; + const linkedNodeName = `es01${LINKED_CLUSTER_NAME_SUFFIX}`; + const linkedEndpoint = `${linkedNodeName}:${REMOTE_CLUSTER_SERVER_PORT}`; + + const linkedProjectInfo = { + alias: 'linked_local_project', + type: esProjectType, + endpoint: linkedEndpoint, + server_name: 'linked-local-project', + tags: { + _alias: 'linked_local_project', + _id: linkedProject.projectId, + _organization: MOCK_IDP_UIAM_ORGANIZATION_ID, + _type: esProjectType, + env: 'local', + }, + }; + + currentJson.metadata.version = String(Number(currentJson.metadata.version) + 1); + currentJson.state.linked = { + projects: { + [linkedProject.projectId]: linkedProjectInfo, + }, + }; + + await Fsp.writeFile(settingsPath, JSON.stringify(currentJson, null, 2)); + log.success(`Linked project registered: ${linkedProject.projectId} -> ${linkedEndpoint}`); +} + /** * Stop a serverless ES cluster by node names */ @@ -1132,13 +1324,19 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions * A volume mount for the operator folder, that contains operator specific configuration files like settings.json. * We mount entire folder since Elasticsearch cannot properly watch changes in bind-mounted files. * @param projectType Type of the serverless project. + * @param projectId Override for the project ID (defaults to MOCK_IDP_UIAM_PROJECT_ID). + * @param operatorPath Override for the operator directory path on the host. */ -async function getOperatorVolume(projectType: string) { - await Fsp.mkdir(SERVERLESS_OPERATOR_PATH, { recursive: true }); +async function getOperatorVolume( + projectType: string, + projectId: string = MOCK_IDP_UIAM_PROJECT_ID, + operatorPath: string = SERVERLESS_OPERATOR_PATH +) { + await Fsp.mkdir(operatorPath, { recursive: true }); // Settings should include information about the project that's normally populated by the Elasticsearch Controller. const projectInfo = { - id: MOCK_IDP_UIAM_PROJECT_ID, + id: projectId, type: projectType, alias: 'local_project', organization: MOCK_IDP_UIAM_ORGANIZATION_ID, @@ -1149,7 +1347,7 @@ async function getOperatorVolume(projectType: string) { }; await Fsp.writeFile( - join(SERVERLESS_OPERATOR_PATH, 'settings.json'), + join(operatorPath, 'settings.json'), JSON.stringify( { metadata: { version: '100', compatibility: '' }, @@ -1159,7 +1357,7 @@ async function getOperatorVolume(projectType: string) { 2 ) ); - return ['--volume', `${SERVERLESS_OPERATOR_PATH}:${SERVERLESS_CONFIG_PATH}operator`]; + return ['--volume', `${operatorPath}:${SERVERLESS_CONFIG_PATH}operator`]; } // --------------------------------------------------------------------------- diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index 2875f441a27b2..ab485071eea68 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -121,6 +121,7 @@ The `fixtures/scope` directory contains core Scout capabilities required for tes - `kbnClient` - `esArchiver` - `samlAuth` +- `linkedProject` (Cross-Project Search only -- provides `esClient` and `esArchiver` for the linked cluster) ```ts test.beforeAll(async ({ kbnClient }) => { @@ -339,6 +340,50 @@ node scripts/scout start-server --arch --domain This command is useful for manual testing or running tests via an IDE. +#### Cross Project Search (CPS) Support + +Scout supports testing Cross Project Search by starting a second ("linked") Elasticsearch cluster alongside the origin. The linked cluster runs on a separate port, shares the same UIAM identity provider as the origin, and is intended exclusively for **reading data from** -- it has no Kibana instance. + +Important: When running tests locally, make sure your Docker Memory Allocation resources are set to **15 GB RAM** or above. + +To start servers with CPS enabled, use the `cps_local` server config set: + +```bash +node scripts/scout start-server --arch serverless --domain security_complete --serverConfigSet cps_local +``` + +This starts: + +- **Origin ES cluster** (3 nodes, port `9220`) with UIAM +- **Linked ES cluster** (3 nodes, port `9230`) connected to the same UIAM +- **Kibana** (port `5620`) + +The linked cluster port is derived from the origin ES port plus `LINKED_CLUSTER_PORT_OFFSET` (defined in `kbn-es`). If the origin port changes, the linked port adjusts automatically via the config set. + +**Ingesting data into the linked cluster** + +Use the `linkedProject` worker fixture in your tests: + +```ts +import { test } from '@kbn/scout'; + +test.beforeAll(async ({ linkedProject }) => { + // Load data archive into the linked ES cluster + await linkedProject.esArchiver.loadIfNeeded('path/to/data/archive'); +}); + +test('query across projects', async ({ linkedProject, page }) => { + // Access the linked ES client directly if needed + const result = await linkedProject.esClient.search({ index: 'my-index' }); + // ... +}); +``` + +The `linkedProject` fixture provides: + +- `esArchiver` -- data-only archiver that rejects `.kibana*` indices (use `kbnArchiver` for saved objects) +- `esClient` -- Elasticsearch client connected to the linked cluster + #### Running Servers and Tests Locally To start the servers locally and run tests in one step, use: diff --git a/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts b/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts index 0565dc185c8d6..6205b1e45a69e 100644 --- a/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts +++ b/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts @@ -53,6 +53,34 @@ export function getEsClient(config: ScoutTestConfig, log: ScoutLogger) { return esClientInstance; } +let linkedEsClientInstance: EsClient | null = null; + +export function getLinkedEsClient(config: ScoutTestConfig, log: ScoutLogger) { + if (!linkedEsClientInstance) { + const linkedProject = config.linkedProject; + if (!linkedProject) { + throw new Error('linkedProject is not configured in ScoutTestConfig'); + } + + const { username, password } = linkedProject.auth; + const elasticsearchUrl = createClientUrlWithAuth({ + serviceName: 'linkedEs', + url: linkedProject.hosts.elasticsearch, + username, + password, + log, + }); + + linkedEsClientInstance = createEsClientForTesting({ + esUrl: elasticsearchUrl, + isCloud: config.isCloud, + authOverride: { username, password }, + }); + } + + return linkedEsClientInstance; +} + export function getKbnClient(config: ScoutTestConfig, log: ScoutLogger) { if (!kbnClientInstance) { const kibanaUrl = createClientUrlWithAuth({ diff --git a/src/platform/packages/shared/kbn-scout/src/common/services/es_archiver.ts b/src/platform/packages/shared/kbn-scout/src/common/services/es_archiver.ts index bd912dd9417e5..7c880b0cbc2de 100644 --- a/src/platform/packages/shared/kbn-scout/src/common/services/es_archiver.ts +++ b/src/platform/packages/shared/kbn-scout/src/common/services/es_archiver.ts @@ -9,8 +9,8 @@ import { EsArchiver } from '@kbn/es-archiver'; import { REPO_ROOT } from '@kbn/repo-info'; -import type { ScoutLogger } from './logger'; import type { EsClient } from '../../types'; +import type { ScoutLogger } from './logger'; let esArchiverInstance: EsArchiver | undefined; @@ -28,3 +28,20 @@ export function getEsArchiver(esClient: EsClient, log: ScoutLogger) { return esArchiverInstance; } + +let linkedEsArchiverInstance: EsArchiver | undefined; + +export function getLinkedEsArchiver(esClient: EsClient, log: ScoutLogger) { + if (!linkedEsArchiverInstance) { + linkedEsArchiverInstance = new EsArchiver({ + log, + client: esClient, + baseDir: REPO_ROOT, + dataOnly: true, + }); + + log.serviceLoaded('linkedEsArchiver'); + } + + return linkedEsArchiverInstance; +} diff --git a/src/platform/packages/shared/kbn-scout/src/common/services/index.ts b/src/platform/packages/shared/kbn-scout/src/common/services/index.ts index 77f5067dcecd4..a7462d8598a13 100644 --- a/src/platform/packages/shared/kbn-scout/src/common/services/index.ts +++ b/src/platform/packages/shared/kbn-scout/src/common/services/index.ts @@ -7,9 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export { getEsClient, getKbnClient } from './clients'; +export { getEsClient, getLinkedEsClient, getKbnClient } from './clients'; export { createScoutConfig } from './config'; -export { getEsArchiver } from './es_archiver'; +export { getEsArchiver, getLinkedEsArchiver } from './es_archiver'; export { createKbnUrl } from './kibana_url'; export { createSamlSessionManager } from './saml_auth'; diff --git a/src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/index.ts b/src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/index.ts index 6c0daec67e45b..5d02cd3d3edbd 100644 --- a/src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/index.ts +++ b/src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/index.ts @@ -21,6 +21,9 @@ export type { export { esArchiverFixture } from './es_archiver'; export type { EsArchiverFixture } from './es_archiver'; +export { linkedEsFixtures } from './linked_es_archiver'; +export type { LinkedProjectFixture } from './linked_es_archiver'; + export { uiSettingsFixture } from './ui_settings'; export type { UiSettingsFixture } from './ui_settings'; diff --git a/src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/linked_es_archiver.ts b/src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/linked_es_archiver.ts new file mode 100644 index 0000000000000..afbf785506a84 --- /dev/null +++ b/src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/linked_es_archiver.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { Client } from '@elastic/elasticsearch'; +import type { LoadActionPerfOptions } from '@kbn/es-archiver'; +import { coreWorkerFixtures } from './core_fixtures'; +import { getLinkedEsClient, getLinkedEsArchiver } from '../../../../common/services'; +import type { EsArchiverFixture } from './es_archiver'; + +export interface LinkedProjectFixture { + esClient: Client; + esArchiver: EsArchiverFixture; +} + +export const linkedEsFixtures = coreWorkerFixtures.extend< + {}, + { linkedProject: LinkedProjectFixture } +>({ + /** + * Provides ES client and esArchiver for the linked CPS project. + * Usage in tests: `linkedProject.esArchiver.loadIfNeeded(...)` or `linkedProject.esClient`. + */ + linkedProject: [ + ({ config, log }, use) => { + if (!config.serverless || !config.linkedProject) { + throw new Error( + 'linkedProject fixture is only available in serverless mode with CPS enabled. ' + + 'Use --serverConfigSet cps_local to start servers with a linked cluster.' + ); + } + + const esClient = getLinkedEsClient(config, log); + const archiver = getLinkedEsArchiver(esClient, log); + const loadIfNeeded = async (name: string, performance?: LoadActionPerfOptions | undefined) => + archiver.loadIfNeeded(name, performance); + + use({ + esClient, + esArchiver: { loadIfNeeded }, + }); + }, + { scope: 'worker' }, + ], +}); diff --git a/src/platform/packages/shared/kbn-scout/src/playwright/index.ts b/src/platform/packages/shared/kbn-scout/src/playwright/index.ts index 7271bac7577ea..897afa6ab6635 100644 --- a/src/platform/packages/shared/kbn-scout/src/playwright/index.ts +++ b/src/platform/packages/shared/kbn-scout/src/playwright/index.ts @@ -44,6 +44,7 @@ export { synthtraceFixture } from './fixtures/scope/worker/synthtrace'; // Other worker types export type { + LinkedProjectFixture, SamlAuth, SynthtraceFixture, RequestAuthFixture, diff --git a/src/platform/packages/shared/kbn-scout/src/playwright/test/api/index.ts b/src/platform/packages/shared/kbn-scout/src/playwright/test/api/index.ts index 3ad60a71d4c56..65735737a019f 100644 --- a/src/platform/packages/shared/kbn-scout/src/playwright/test/api/index.ts +++ b/src/platform/packages/shared/kbn-scout/src/playwright/test/api/index.ts @@ -12,6 +12,7 @@ import { test as base, mergeTests } from 'playwright/test'; import { coreWorkerFixtures, esArchiverFixture, + linkedEsFixtures, apiClientFixture, apiServicesFixture, defaultRolesFixture, @@ -20,6 +21,7 @@ import { import type { CoreWorkerFixtures, EsArchiverFixture, + LinkedProjectFixture, RequestAuthFixture, ApiClientFixture, DefaultRolesFixture, @@ -35,6 +37,7 @@ export interface ApiWorkerFixtures extends CoreWorkerFixtures { defaultRolesFixture: DefaultRolesFixture; requestAuth: RequestAuthFixture; esArchiver: EsArchiverFixture; + linkedProject: LinkedProjectFixture; } // This disables browser-related fixtures by overriding them with undefined @@ -76,5 +79,6 @@ export const apiTest = mergeTests( apiServicesFixture, defaultRolesFixture, requestAuthFixture, - esArchiverFixture + esArchiverFixture, + linkedEsFixtures ) as unknown as TestType<{}, ApiWorkerFixtures>; diff --git a/src/platform/packages/shared/kbn-scout/src/playwright/test/ui/single_thread_fixtures.ts b/src/platform/packages/shared/kbn-scout/src/playwright/test/ui/single_thread_fixtures.ts index a126ade68809b..acdb1f8f17ab7 100644 --- a/src/platform/packages/shared/kbn-scout/src/playwright/test/ui/single_thread_fixtures.ts +++ b/src/platform/packages/shared/kbn-scout/src/playwright/test/ui/single_thread_fixtures.ts @@ -12,6 +12,7 @@ import { apiServicesFixture, coreWorkerFixtures, esArchiverFixture, + linkedEsFixtures, uiSettingsFixture, synthtraceFixture, lighthouseFixture, @@ -19,6 +20,7 @@ import { import type { ApiServicesFixture, EsArchiverFixture, + LinkedProjectFixture, EsClient, KbnClient, KibanaUrl, @@ -49,6 +51,7 @@ export const scoutFixtures = mergeTests( // worker scope fixtures coreWorkerFixtures, esArchiverFixture, + linkedEsFixtures, uiSettingsFixture, synthtraceFixture, // api fixtures @@ -77,6 +80,7 @@ export interface ScoutWorkerFixtures extends ApiServicesFixture { kbnClient: KbnClient; esClient: EsClient; esArchiver: EsArchiverFixture; + linkedProject: LinkedProjectFixture; uiSettings: UiSettingsFixture; apiServices: ApiServicesFixture; apmSynthtraceEsClient: SynthtraceFixture['apmSynthtraceEsClient']; diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config.test.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config.test.ts index 5f622ec216637..cda13f531a62c 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config.test.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config.test.ts @@ -188,4 +188,104 @@ describe('Config.getScoutTestConfig', () => { expect(scoutConfig).toEqual(expectedConfig); }); + + it(`should return a properly structured 'ScoutTestConfig' object for 'serverless=security' in UIAM+CPS mode`, async () => { + const config = new Config({ + serverless: true, + servers: { + elasticsearch: { + protocol: 'https', + hostname: 'localhost', + port: 9220, + username: 'elastic_serverless', + password: 'changeme', + }, + linkedElasticsearch: { + protocol: 'https', + hostname: 'localhost', + port: 9230, + username: 'elastic_serverless', + password: 'changeme', + }, + kibana: { + protocol: 'http', + hostname: 'localhost', + port: 5620, + username: 'elastic_serverless', + password: 'changeme', + }, + }, + dockerServers: {}, + esTestCluster: { + from: 'serverless', + files: [], + serverArgs: [], + ssl: true, + }, + esServerlessOptions: { uiam: true, cps: true }, + kbnTestServer: { + buildArgs: [], + env: {}, + sourceArgs: [], + serverArgs: ['--serverless=security', '--xpack.cloud.organization_id=org123'], + }, + }); + + const scoutConfig = config.getScoutTestConfig(); + + expect(scoutConfig.linkedProject).toBeDefined(); + expect(scoutConfig.linkedProject).toEqual({ + hosts: { + elasticsearch: 'https://localhost:9230', + }, + auth: { + username: 'elastic_serverless', + password: 'changeme', + }, + }); + + expect(scoutConfig.serverless).toBe(true); + expect(scoutConfig.uiam).toBe(true); + expect(scoutConfig.projectType).toBe('security'); + expect(scoutConfig.hosts.elasticsearch).toBe('https://localhost:9220'); + }); + + it(`should not include linkedProject when CPS is not enabled`, async () => { + const config = new Config({ + serverless: true, + servers: { + elasticsearch: { + protocol: 'https', + hostname: 'localhost', + port: 9220, + username: 'elastic_serverless', + password: 'changeme', + }, + kibana: { + protocol: 'http', + hostname: 'localhost', + port: 5620, + username: 'elastic_serverless', + password: 'changeme', + }, + }, + dockerServers: {}, + esTestCluster: { + from: 'serverless', + files: [], + serverArgs: [], + ssl: true, + }, + esServerlessOptions: { uiam: true }, + kbnTestServer: { + buildArgs: [], + env: {}, + sourceArgs: [], + serverArgs: ['--serverless=es'], + }, + }); + + const scoutConfig = config.getScoutTestConfig(); + expect(scoutConfig.linkedProject).toBeUndefined(); + }); }); diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config.ts index ec7329242839a..da367671bb1e1 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config.ts @@ -134,6 +134,24 @@ export class Config { password: this.get('servers.kibana.password'), }, + ...(this.get('esServerlessOptions.cps', false) && this.get('servers.linkedElasticsearch.port') + ? { + linkedProject: { + hosts: { + elasticsearch: Url.format({ + protocol: this.get('servers.linkedElasticsearch.protocol'), + hostname: this.get('servers.linkedElasticsearch.hostname'), + port: this.get('servers.linkedElasticsearch.port'), + }), + }, + auth: { + username: this.get('servers.linkedElasticsearch.username'), + password: this.get('servers.linkedElasticsearch.password'), + }, + }, + } + : {}), + metadata: { generatedOn: formatCurrentDate(), config: this.getAll(), diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/cps_local/serverless/security_complete.serverless.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/cps_local/serverless/security_complete.serverless.config.ts new file mode 100644 index 0000000000000..0bf6410eea4ae --- /dev/null +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/cps_local/serverless/security_complete.serverless.config.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { + ELASTIC_SERVERLESS_SUPERUSER, + ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, + LINKED_CLUSTER_PORT_OFFSET, +} from '@kbn/es'; +import { servers as uiamConfig } from '../../uiam_local/serverless/security_complete.serverless.config'; +import type { ScoutServerConfig } from '../../../../../types'; + +export const servers: ScoutServerConfig = { + ...uiamConfig, + servers: { + ...uiamConfig.servers, + linkedElasticsearch: { + ...uiamConfig.servers.elasticsearch, + port: (uiamConfig.servers.elasticsearch.port as number) + LINKED_CLUSTER_PORT_OFFSET, + username: ELASTIC_SERVERLESS_SUPERUSER, + password: ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, + }, + }, + esServerlessOptions: { + uiam: true, + cps: true, + }, +}; diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/serverless/serverless.base.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/serverless/serverless.base.config.ts index 8ce90ab83a442..05e43a0f4cca3 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/serverless/serverless.base.config.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/serverless/serverless.base.config.ts @@ -189,7 +189,9 @@ export const defaultConfig: ScoutServerConfig = { type: 'elasticsearch', is_default: true, is_default_monitoring: true, - hosts: ['https://localhost:9200'], + hosts: [ + `${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`, + ], }, ])}`, ], diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/stateful/base.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/stateful/base.config.ts index 783df16895045..195fd2f5d5f25 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/stateful/base.config.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/stateful/base.config.ts @@ -197,7 +197,9 @@ export const defaultConfig: ScoutServerConfig = { type: 'elasticsearch', is_default: true, is_default_monitoring: true, - hosts: ['https://localhost:9200'], + hosts: [ + `${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`, + ], }, ])}`, // Agent policies are now created via Fleet API using the helper function from @kbn-scout diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/schema/schema.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/schema/schema.ts index cbabc048851e5..83f9fe0a8f052 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/schema/schema.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/schema/schema.ts @@ -70,6 +70,8 @@ export const schema = Joi.object() elasticsearch: urlPartsSchema({ requiredKeys: ['port'], }), + // Only applicable for serverless CPS configs (cross-project search) + linkedElasticsearch: urlPartsSchema(), fleetserver: urlPartsSchema(), }) .default(), @@ -97,6 +99,7 @@ export const schema = Joi.object() host: Joi.string().ip(), resources: Joi.array().items(Joi.string()).default([]), uiam: Joi.boolean().default(false), + cps: Joi.boolean().default(false), }) .default(), diff --git a/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts b/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts index 2122cb6ab2025..a000848755508 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts @@ -8,7 +8,8 @@ */ import type { ArtifactLicense, ServerlessProjectType } from '@kbn/es'; -import { isServerlessProjectType } from '@kbn/es/src/utils'; +import { isServerlessProjectType, runLinkedServerlessCluster } from '@kbn/es/src/utils'; +import { MOCK_IDP_UIAM_PROJECT_ID2 } from '@kbn/mock-idp-utils'; import { REPO_ROOT } from '@kbn/repo-info'; import { cleanupElasticsearch, createTestEsCluster, esTestConfig } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; @@ -84,25 +85,24 @@ export async function runElasticsearch( config, }); - // Enable it to debug why SAML callback randomly returns 401 - // log.info('Enable authc debug logs for ES'); - // const clientUrl = new URL( - // Url.format({ - // protocol: options.config.get('servers.elasticsearch.protocol'), - // hostname: options.config.get('servers.elasticsearch.hostname'), - // port: options.config.get('servers.elasticsearch.port'), - // }) - // ); - // clientUrl.username = options.config.get('servers.kibana.username'); - // clientUrl.password = options.config.get('servers.kibana.password'); - // const esClient = createEsClientForTesting({ - // esUrl: clientUrl.toString(), - // }); - // await esClient.cluster.putSettings({ - // persistent: { - // 'logger.org.elasticsearch.xpack.security.authc': 'debug', - // }, - // }); + // Start linked cluster for Cross Project Search after origin is ready + if (config.esServerlessOptions?.linkedProject) { + const serverlessOptions = { + basePath: resolve(REPO_ROOT, '.es'), + esArgs: config.esArgs, + dataPath: `stateless-cluster-${name ?? 'scout'}`, + ...config.esServerlessOptions, + port: config.port, + clean: true, + background: true, + files: config.files, + ssl: config.ssl, + kill: false, + waitForReady: true, + }; + + await runLinkedServerlessCluster(log, serverlessOptions); + } return async () => { await cleanupElasticsearch(node, config.serverless, logsDir, log); @@ -160,6 +160,7 @@ interface EsServerlessOptions { kibanaUrl: string; tag?: string; image?: string; + linkedProject?: { projectId: string; port: number }; } function getESServerlessOptions( @@ -193,6 +194,8 @@ function getESServerlessOptions( throw new Error(`Unsupported serverless projectType: ${projectType}`); } + const cps: boolean = config.get('esServerlessOptions.cps', false); + const commonOptions = { projectType, host: serverlessHost, @@ -203,6 +206,14 @@ function getESServerlessOptions( hostname: config.get('servers.kibana.hostname'), port: config.get('servers.kibana.port'), }), + ...(cps && config.get('servers.linkedElasticsearch.port') + ? { + linkedProject: { + projectId: MOCK_IDP_UIAM_PROJECT_ID2, + port: config.get('servers.linkedElasticsearch.port') as number, + }, + } + : {}), }; if (esServerlessImageUrlOrTag) { diff --git a/src/platform/packages/shared/kbn-scout/src/types/server_config.d.ts b/src/platform/packages/shared/kbn-scout/src/types/server_config.d.ts index f91ebdc4f13d6..cce16b542b440 100644 --- a/src/platform/packages/shared/kbn-scout/src/types/server_config.d.ts +++ b/src/platform/packages/shared/kbn-scout/src/types/server_config.d.ts @@ -14,6 +14,7 @@ export interface ScoutServerConfig { servers: { kibana: UrlParts; elasticsearch: UrlParts; + linkedElasticsearch?: UrlParts; fleet?: UrlParts; }; dockerServers: any; @@ -25,7 +26,7 @@ export interface ScoutServerConfig { ssl: boolean; secureFiles?: string[]; }; - esServerlessOptions?: { uiam: boolean }; + esServerlessOptions?: { uiam: boolean; cps?: boolean }; kbnTestServer: { env: any; buildArgs: string[]; diff --git a/src/platform/packages/shared/kbn-scout/src/types/test_config.d.ts b/src/platform/packages/shared/kbn-scout/src/types/test_config.d.ts index 8237b892bfcfb..818a78a12fca3 100644 --- a/src/platform/packages/shared/kbn-scout/src/types/test_config.d.ts +++ b/src/platform/packages/shared/kbn-scout/src/types/test_config.d.ts @@ -26,5 +26,14 @@ export interface ScoutTestConfig { username: string; password: string; }; + linkedProject?: { + hosts: { + elasticsearch: string; + }; + auth: { + username: string; + password: string; + }; + }; metadata?: any; }