diff --git a/src/platform/packages/shared/kbn-es/src/cli_commands/snapshot.ts b/src/platform/packages/shared/kbn-es/src/cli_commands/snapshot.ts index 95d4310dda306..47d7b1fd1bf20 100644 --- a/src/platform/packages/shared/kbn-es/src/cli_commands/snapshot.ts +++ b/src/platform/packages/shared/kbn-es/src/cli_commands/snapshot.ts @@ -33,6 +33,11 @@ export const snapshot: Command = { --password.[user] Sets password for native realm user [default: ${password}] -E Additional key=value settings to pass to Elasticsearch --download-only Download the snapshot but don't actually start it + --docker Run in a Docker container instead of downloading the snapshot locally. + Supports the same options (--license, -E, --ssl, --password, etc.) + and maps -E path.data= to a Docker volume mount. + --port The port to bind to on 127.0.0.1 [default: 9200] (Docker mode only) + --kill Kill running ES Docker containers before starting (Docker mode only) --ssl Sets up SSL on Elasticsearch --use-cached Skips cache verification and use cached ES snapshot. --skip-ready-check Disable the ready check, @@ -44,6 +49,7 @@ export const snapshot: Command = { Example: es snapshot --version 5.6.8 -E cluster.name=test -E path.data=/tmp/es-data + es snapshot --docker --license=trial -E path.data=../my-data `; }, run: async (defaults = {}) => { @@ -69,13 +75,28 @@ export const snapshot: Command = { }, string: ['version', 'ready-timeout', 'es-log-level'], - boolean: ['download-only', 'use-cached', 'skip-ready-check'], + boolean: ['download-only', 'use-cached', 'skip-ready-check', 'docker', 'kill'], default: defaults, }); const cluster = new Cluster({ ssl: options.ssl }); - if (options['download-only']) { + + if (options.docker) { + await cluster.runDockerSnapshot({ + reportTime, + startTime: runStartTime, + license: options.license, + version: options.version, + password: options.password, + port: options.port ? Number(options.port) : undefined, + ssl: options.ssl, + kill: options.kill, + esArgs: options.esArgs, + skipReadyCheck: options.skipReadyCheck, + readyTimeout: parseTimeoutToMs(options.readyTimeout), + }); + } else if (options['download-only']) { await cluster.downloadSnapshot({ version: options.version, license: options.license, diff --git a/src/platform/packages/shared/kbn-es/src/cluster.ts b/src/platform/packages/shared/kbn-es/src/cluster.ts index 4eb65a2471c4a..5a78cdfc85348 100644 --- a/src/platform/packages/shared/kbn-es/src/cluster.ts +++ b/src/platform/packages/shared/kbn-es/src/cluster.ts @@ -22,13 +22,15 @@ import treeKill from 'tree-kill'; import { MOCK_IDP_REALM_NAME, ensureSAMLRoleMapping } from '@kbn/mock-idp-utils'; import { downloadSnapshot, installSnapshot, installSource, installArchive } from './install'; import { ES_BIN, ES_PLUGIN_BIN, ES_KEYSTORE_BIN } from './paths'; -import type { DockerOptions, ServerlessOptions } from './utils'; +import type { DockerOptions, DockerSnapshotOptions, ServerlessOptions } from './utils'; import { extractConfigFiles, log as defaultLog, NativeRealm, parseEsLog, runDockerContainer, + runDockerSnapshotContainer, + stopDockerSnapshotContainer, runServerlessCluster, stopServerlessCluster, teardownServerlessClusterSync, @@ -111,6 +113,7 @@ export class Cluster { private process: execa.ExecaChildProcess | null; private outcome: Promise | null; private serverlessNodes: string[]; + private dockerSnapshotContainerName: string | null; private setupPromise: Promise | null; private stdioTarget: NodeJS.WritableStream | null; @@ -120,6 +123,8 @@ export class Cluster { this.stopCalled = false; // Serverless Elasticsearch node names, started via Docker this.serverlessNodes = []; + // Docker snapshot container name, if running via Docker + this.dockerSnapshotContainerName = null; // properties used exclusively for the locally started Elasticsearch cluster this.process = null; this.outcome = null; @@ -317,6 +322,11 @@ export class Cluster { return await stopServerlessCluster(this.log, this.serverlessNodes); } + // Stop Docker snapshot container + if (this.dockerSnapshotContainerName) { + return await stopDockerSnapshotContainer(this.log, this.dockerSnapshotContainerName); + } + // Stop local ES process if (!this.process || !this.outcome) { throw new Error('ES has not been started'); @@ -487,11 +497,11 @@ export class Cluster { if (!skipSecuritySetup) { const nativeRealm = new NativeRealm({ log: this.log, - elasticPassword: options.password, + elasticPassword: options.password ?? 'changeme', client, }); - await nativeRealm.setPasswords(options); + await nativeRealm.setPasswords(options as Record); const samlRealmConfigPrefix = `authc.realms.saml.${MOCK_IDP_REALM_NAME}.`; if (args.some((arg) => arg.includes(samlRealmConfigPrefix))) { @@ -642,4 +652,16 @@ export class Cluster { await runDockerContainer(this.log, options); } + + /** + * Run an Elasticsearch Docker container with snapshot-equivalent semantics. + * Same defaults and native realm setup as the local snapshot flow. + */ + async runDockerSnapshot(options: DockerSnapshotOptions) { + if (this.process || this.outcome) { + throw new Error('ES stateful cluster has already been started'); + } + + this.dockerSnapshotContainerName = await runDockerSnapshotContainer(this.log, options); + } } diff --git a/src/platform/packages/shared/kbn-es/src/utils/docker.ts b/src/platform/packages/shared/kbn-es/src/utils/docker.ts index 73d72b24c2fd1..934844477a136 100644 --- a/src/platform/packages/shared/kbn-es/src/utils/docker.ts +++ b/src/platform/packages/shared/kbn-es/src/utils/docker.ts @@ -56,7 +56,7 @@ import { ELASTIC_SERVERLESS_SUPERUSER, ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, } from './serverless_file_realm'; -import { SYSTEM_INDICES_SUPERUSER } from './native_realm'; +import { NativeRealm, SYSTEM_INDICES_SUPERUSER } from './native_realm'; import { waitUntilClusterReady } from './wait_until_cluster_ready'; interface ImageOptions { @@ -195,16 +195,9 @@ const DEFAULT_DOCKER_ESARGS: Array<[string, string]> = [ ['discovery.type', 'single-node'], ['xpack.security.enabled', 'false'], + + ['ES_JAVA_OPTS', '-Xms1536m -Xmx1536m'], ]; -// Temporary workaround for https://github.com/elastic/elasticsearch/issues/118583 -if (process.arch === 'arm64') { - DEFAULT_DOCKER_ESARGS.push( - ['ES_JAVA_OPTS', '-Xms1536m -Xmx1536m -XX:UseSVE=0'], - ['CLI_JAVA_OPTS', '-XX:UseSVE=0'] - ); -} else { - DEFAULT_DOCKER_ESARGS.push(['ES_JAVA_OPTS', '-Xms1536m -Xmx1536m']); -} export const DOCKER_REPO = `${DOCKER_REGISTRY}/elasticsearch/elasticsearch`; export const DOCKER_TAG = `${pkg.version}-SNAPSHOT`; @@ -293,16 +286,9 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ], ['xpack.security.remote_cluster_server.ssl.verification_mode', 'certificate'], ['xpack.security.remote_cluster_server.ssl.client_authentication', 'required'], + + ['ES_JAVA_OPTS', '-Xms1g -Xmx1g'], ]; -// Temporary workaround for https://github.com/elastic/elasticsearch/issues/118583 -if (process.arch === 'arm64') { - DEFAULT_SERVERLESS_ESARGS.push( - ['ES_JAVA_OPTS', '-Xms1g -Xmx1g -XX:UseSVE=0'], - ['CLI_JAVA_OPTS', '-XX:UseSVE=0'] - ); -} else { - DEFAULT_SERVERLESS_ESARGS.push(['ES_JAVA_OPTS', '-Xms1g -Xmx1g']); -} const DEFAULT_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.http.ssl.enabled', 'true'], @@ -1175,3 +1161,235 @@ async function getOperatorVolume(projectType: string) { ); return ['--volume', `${SERVERLESS_OPERATOR_PATH}:${SERVERLESS_CONFIG_PATH}operator`]; } + +// --------------------------------------------------------------------------- +// Docker Snapshot Mode +// --------------------------------------------------------------------------- + +export interface DockerSnapshotOptions extends EsClusterExecOptions { + license?: string; + version?: string; + port?: number; + ssl?: boolean; + kill?: boolean; + tag?: string; + image?: string; + /** Container name. Defaults to 'es01'. Use unique names for parallel runs. */ + name?: string; + /** When true, returns immediately after ES is ready instead of tailing logs. */ + background?: boolean; + /** Host-side transport port to map to container port 9300. Defaults to port + 100. */ + transportPort?: number; +} + +/** + * Default esArgs for Docker snapshot mode. + * Mirrors the defaults applied by Cluster.exec() for the local snapshot flow, + * plus the Docker-specific settings required for single-node operation. + */ +const DEFAULT_DOCKER_SNAPSHOT_ESARGS: Array<[string, string]> = [ + ['ES_LOG_STYLE', 'file'], + ['discovery.type', 'single-node'], + ['action.destructive_requires_name', 'true'], + ['cluster.routing.allocation.disk.threshold_enabled', 'false'], + ['ingest.geoip.downloader.enabled', 'false'], + ['search.check_ccs_compatibility', 'true'], + + ['ES_JAVA_OPTS', '-Xms1536m -Xmx1536m'], +]; + +/** + * Runs an Elasticsearch Docker container with the same semantics as `yarn es snapshot`. + * + * - Applies the same default esArgs as the local snapshot flow + * - Maps `--license=trial` → `xpack.license.self_generated.type=trial` + * - Maps `-E path.data=` → a Docker volume mount + * - Waits for cluster readiness and sets up the native realm (passwords) + */ +export async function runDockerSnapshotContainer( + log: ToolingLog, + options: DockerSnapshotOptions +): Promise { + await verifyDockerInstalled(log); + await maybeCreateDockerNetwork(log); + + const tag = options.tag || (options.version ? `${options.version}-SNAPSHOT` : DOCKER_TAG); + const image = resolveDockerImage({ + image: options.image, + tag, + repo: DOCKER_REPO, + defaultImg: DOCKER_IMG, + }); + await setupDockerImage({ log, image }); + + const containerName = options.name || 'es01'; + const port = options.port || DEFAULT_PORT; + const password = options.password || 'changeme'; + const transportPort = options.transportPort ?? port + 100; + + await execa('docker', ['rm', '-f', containerName]).catch(() => { + // ignore if container doesn't exist + }); + + const esArgsMap = new Map(DEFAULT_DOCKER_SNAPSHOT_ESARGS); + + if (options.license === 'trial') { + esArgsMap.set('xpack.license.self_generated.type', 'trial'); + } + + esArgsMap.set('ELASTIC_PASSWORD', password); + + const volumeMounts: string[] = []; + const userEsArgs: string[] = options.esArgs + ? Array.isArray(options.esArgs) + ? options.esArgs + : [options.esArgs] + : []; + + for (const arg of userEsArgs) { + const [key, ...rest] = arg.split('='); + const k = key.trim(); + const v = rest.join('=').trim(); + + if (k === 'path.data') { + const hostPath = resolve(process.cwd(), v); + volumeMounts.push('--volume', `${hostPath}:/usr/share/elasticsearch/data`); + continue; + } + + if (v) { + const hostPath = resolve(REPO_ROOT, v); + if (fs.existsSync(hostPath) && fs.statSync(hostPath).isFile()) { + const containerPath = getDockerFileMountPath(hostPath); + volumeMounts.push('--volume', `${hostPath}:${containerPath}`); + esArgsMap.set(k, containerPath); + continue; + } + } + + esArgsMap.set(k, v); + } + + if (options.ssl) { + esArgsMap.set('xpack.security.http.ssl.enabled', 'true'); + esArgsMap.set( + 'xpack.security.http.ssl.keystore.path', + `${SERVERLESS_CONFIG_PATH}certs/elasticsearch.p12` + ); + esArgsMap.set('xpack.security.http.ssl.keystore.password', ES_P12_PASSWORD); + esArgsMap.set('xpack.security.transport.ssl.enabled', 'true'); + esArgsMap.set( + 'xpack.security.transport.ssl.keystore.path', + `${SERVERLESS_CONFIG_PATH}certs/elasticsearch.p12` + ); + esArgsMap.set('xpack.security.transport.ssl.verification_mode', 'certificate'); + esArgsMap.set('xpack.security.transport.ssl.keystore.password', ES_P12_PASSWORD); + volumeMounts.push(...getESp12Volume()); + } + + const envArgs = Array.from(esArgsMap).flatMap(([k, v]) => { + const value = + k.startsWith('cluster.remote.') && k.endsWith('.seeds') && v.includes('localhost') + ? v.replace(/localhost/g, 'host.docker.internal') + : v; + return ['--env', `${k}=${value}`]; + }); + + const dockerCmd = [ + 'run', + '--detach', + '-t', + '--net', + 'elastic', + '--add-host', + 'host.docker.internal:host-gateway', + '--name', + containerName, + '-p', + `${port}:9200`, + '-p', + `${transportPort}:9300`, + ...envArgs, + ...volumeMounts, + image, + ]; + + log.info(chalk.dim(`docker ${dockerCmd.join(' ')}`)); + const { stdout: containerId } = await execa('docker', dockerCmd); + + log.info(`Container ${containerName} started: ${containerId.substring(0, 12)}`); + + process.on('SIGINT', () => { + try { + execa.commandSync(`docker kill ${containerName}`); + } catch { + // container may already be stopped + } + }); + + const esUrl = `${options.ssl ? 'https' : 'http'}://127.0.0.1:${port}`; + const client = new Client({ + node: esUrl, + auth: { username: 'elastic', password }, + Connection: HttpConnection, + requestTimeout: 30_000, + ...(options.ssl + ? { + tls: { + ca: [fs.readFileSync(CA_CERT_PATH)], + checkServerIdentity: () => undefined, + }, + } + : {}), + }); + + if (!options.skipReadyCheck) { + log.info('Waiting for ES to be ready...'); + await waitUntilClusterReady({ + client, + expectedStatus: 'yellow', + log, + readyTimeout: options.readyTimeout, + }); + } + + const securityExplicitlyDisabled = esArgsMap.get('xpack.security.enabled') === 'false'; + if (!securityExplicitlyDisabled && !options.skipSecuritySetup) { + const nativeRealm = new NativeRealm({ elasticPassword: password, client, log }); + await nativeRealm.setPasswords(options as Record); + } + + log.success('ES is ready and native realm is set up'); + log.info(` View logs: ${chalk.bold(`docker logs -f ${containerName}`)}`); + log.info(` Shell: ${chalk.bold(`docker exec -it ${containerName} /bin/bash`)}`); + log.info(` Stop: ${chalk.bold(`docker container stop ${containerName}`)}`); + + if (!options.background) { + await execa('docker', ['logs', '-f', containerName], { + stdio: ['ignore', 'inherit', 'inherit'], + }).catch(() => { + // docker logs exits when the container stops + }); + } + + return containerName; +} + +export async function stopDockerSnapshotContainer( + log: ToolingLog, + containerName: string +): Promise { + try { + await execa('docker', ['kill', containerName]); + log.info(`Docker container ${containerName} killed`); + } catch { + // container may already be stopped + } + + try { + await execa('docker', ['rm', '-f', containerName]); + log.info(`Docker container ${containerName} removed`); + } catch { + // container may already be removed + } +} diff --git a/src/platform/packages/shared/kbn-es/src/utils/index.ts b/src/platform/packages/shared/kbn-es/src/utils/index.ts index e1a51ecb44685..aa2bcdf27568c 100644 --- a/src/platform/packages/shared/kbn-es/src/utils/index.ts +++ b/src/platform/packages/shared/kbn-es/src/utils/index.ts @@ -12,7 +12,6 @@ export { log } from './log'; export { parseEsLog } from './parse_es_log'; export { findMostRecentlyChanged } from './find_most_recently_changed'; export { extractConfigFiles, isFile, copyFileSync } from './extract_config_files'; -// @ts-expect-error not typed yet export { NativeRealm, SYSTEM_INDICES_SUPERUSER } from './native_realm'; export { buildSnapshot } from './build_snapshot'; export { archiveForPlatform } from './build_snapshot'; diff --git a/src/platform/packages/shared/kbn-es/src/utils/native_realm.test.js b/src/platform/packages/shared/kbn-es/src/utils/native_realm.test.ts similarity index 92% rename from src/platform/packages/shared/kbn-es/src/utils/native_realm.test.js rename to src/platform/packages/shared/kbn-es/src/utils/native_realm.test.ts index 07a6c3af93de1..87dc057d3d59e 100644 --- a/src/platform/packages/shared/kbn-es/src/utils/native_realm.test.js +++ b/src/platform/packages/shared/kbn-es/src/utils/native_realm.test.ts @@ -7,8 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -const { NativeRealm } = require('./native_realm'); -const { ToolingLog } = require('@kbn/tooling-log'); +import { NativeRealm } from './native_realm'; +import { ToolingLog } from '@kbn/tooling-log'; const mockClient = { xpack: { @@ -26,17 +26,21 @@ const mockClient = { }; const log = new ToolingLog(); -let nativeRealm; +let nativeRealm: NativeRealm; beforeEach(() => { - nativeRealm = new NativeRealm({ elasticPassword: 'changeme', client: mockClient, log }); + nativeRealm = new NativeRealm({ + elasticPassword: 'changeme', + client: mockClient as any, + log, + }); }); afterAll(() => { jest.clearAllMocks(); }); -function mockXPackInfo(available, enabled) { +function mockXPackInfo(available: boolean, enabled: boolean) { mockClient.xpack.info.mockImplementation(() => ({ features: { security: { @@ -47,7 +51,7 @@ function mockXPackInfo(available, enabled) { })); } -function mockClusterStatus(status) { +function mockClusterStatus(status: string) { mockClient.cluster.health.mockImplementation(() => { return status; }); @@ -72,7 +76,7 @@ describe('isSecurityEnabled', () => { test('returns false if 400 error returned', async () => { mockClient.xpack.info.mockImplementation(() => { const error = new Error('ResponseError'); - error.meta = { + (error as any).meta = { statusCode: 400, }; throw error; @@ -84,7 +88,7 @@ describe('isSecurityEnabled', () => { test('rejects if unexpected error is thrown', async () => { mockClient.xpack.info.mockImplementation(() => { const error = new Error('ResponseError'); - error.meta = { + (error as any).meta = { statusCode: 500, }; throw error; diff --git a/src/platform/packages/shared/kbn-es/src/utils/native_realm.js b/src/platform/packages/shared/kbn-es/src/utils/native_realm.ts similarity index 68% rename from src/platform/packages/shared/kbn-es/src/utils/native_realm.js rename to src/platform/packages/shared/kbn-es/src/utils/native_realm.ts index 618acbf43d28c..9a9088da0726e 100644 --- a/src/platform/packages/shared/kbn-es/src/utils/native_realm.js +++ b/src/platform/packages/shared/kbn-es/src/utils/native_realm.ts @@ -7,29 +7,51 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -const chalk = require('chalk'); +import chalk from 'chalk'; +import type { Client } from '@elastic/elasticsearch'; +import type { ToolingLog } from '@kbn/tooling-log'; -const { log: defaultLog } = require('./log'); +import { log as defaultLog } from './log'; export const SYSTEM_INDICES_SUPERUSER = process.env.TEST_ES_SYSTEM_INDICES_USER || 'system_indices_superuser'; -exports.NativeRealm = class NativeRealm { - constructor({ elasticPassword, log = defaultLog, client }) { +interface RetryOpts { + attempt?: number; + maxAttempts?: number; +} + +interface NativeRealmOptions { + elasticPassword: string; + log?: ToolingLog; + client: Client; +} + +export class NativeRealm { + private readonly _elasticPassword: string; + private readonly _client: Client; + private readonly _log: ToolingLog; + + constructor({ elasticPassword, log = defaultLog, client }: NativeRealmOptions) { this._elasticPassword = elasticPassword; this._client = client; this._log = log; } - async setPassword(username, password = this._elasticPassword, retryOpts = {}) { + async setPassword( + username: string, + password: string | undefined = this._elasticPassword, + retryOpts: RetryOpts = {} + ) { + const effectivePassword = password ?? this._elasticPassword; await this._autoRetry(retryOpts, async () => { try { await this._client.security.changePassword({ username, refresh: 'wait_for', - password, + password: effectivePassword, }); - } catch (err) { + } catch (err: any) { const isAnonymousUserPasswordChangeError = err.statusCode === 400 && err.body && @@ -46,7 +68,7 @@ exports.NativeRealm = class NativeRealm { }); } - async setPasswords(options) { + async setPasswords(options: Record) { if (!(await this.isSecurityEnabled())) { this._log.info('security is not enabled, unable to set native realm passwords'); return; @@ -56,16 +78,18 @@ exports.NativeRealm = class NativeRealm { this._log.info(`Set up ${reservedUsers.length} ES users`); await Promise.all([ ...reservedUsers.map(async (user) => { - await this.setPassword(user, options[`password.${user}`]); + await this.setPassword(user, options[`password.${user}`] as string | undefined); }), this._createSystemIndicesUser(), ]); } - async getReservedUsers(retryOpts = {}) { + async getReservedUsers(retryOpts: RetryOpts = {}): Promise { return await this._autoRetry(retryOpts, async () => { const resp = await this._client.security.getUser(); - const usernames = Object.keys(resp).filter((user) => resp[user].metadata._reserved === true); + const usernames = Object.keys(resp).filter( + (user) => (resp[user] as any).metadata._reserved === true + ); if (!usernames?.length) { throw new Error('no reserved users found, unable to set native realm passwords'); @@ -75,13 +99,14 @@ exports.NativeRealm = class NativeRealm { }); } - async isSecurityEnabled(retryOpts = {}) { + async isSecurityEnabled(retryOpts: RetryOpts = {}): Promise { try { return await this._autoRetry(retryOpts, async () => { - const { features } = await this._client.xpack.info({ categories: 'features' }); - return features.security && features.security.enabled && features.security.available; + const { features } = await this._client.xpack.info({ categories: ['features'] }); + const security = (features as any)?.security; + return Boolean(security && security.enabled && security.available); }); - } catch (error) { + } catch (error: any) { if (error.meta && error.meta.statusCode === 400) { return false; } @@ -90,7 +115,7 @@ exports.NativeRealm = class NativeRealm { } } - async _autoRetry(opts, fn) { + private async _autoRetry(opts: RetryOpts, fn: (attempt: number) => Promise): Promise { const { attempt = 1, maxAttempts = 3 } = opts; try { @@ -104,15 +129,11 @@ exports.NativeRealm = class NativeRealm { this._log.warning(`assuming ES isn't initialized completely, trying again in ${sec} seconds`); await new Promise((resolve) => setTimeout(resolve, sec * 1000)); - const nextOpts = { - ...opts, - attempt: attempt + 1, - }; - return await this._autoRetry(nextOpts, fn); + return await this._autoRetry({ ...opts, attempt: attempt + 1 }, fn); } } - async _createSystemIndicesUser() { + private async _createSystemIndicesUser() { if (!(await this.isSecurityEnabled())) { this._log.info('security is not enabled, unable to create role and user'); return; @@ -146,4 +167,4 @@ exports.NativeRealm = class NativeRealm { roles: [SYSTEM_INDICES_SUPERUSER], }); } -}; +} diff --git a/src/platform/packages/shared/kbn-test/src/es/test_es_cluster.ts b/src/platform/packages/shared/kbn-test/src/es/test_es_cluster.ts index 4864903c2568f..3384beeb9b38d 100644 --- a/src/platform/packages/shared/kbn-test/src/es/test_es_cluster.ts +++ b/src/platform/packages/shared/kbn-test/src/es/test_es_cluster.ts @@ -172,6 +172,20 @@ export interface CreateTestEsClusterOptions { secureFiles?: string[]; } +/** + * Resolves a transport port suitable for Docker (single numeric port). + * Falls back to TEST_ES_TRANSPORT_PORT (taking the first port from a range like '9300-9400'), + * matching the existing behavior in esTestConfig.getTransportPort(). + */ +const resolveTransportPort = (transportPort: number | string | undefined): number | undefined => { + const raw = transportPort ?? esTestConfig.getTransportPort(); + if (typeof raw === 'number') { + return raw; + } + const parsed = parseInt(String(raw).split('-')[0], 10); + return Number.isNaN(parsed) ? undefined : parsed; +}; + export function createTestEsCluster< Options extends CreateTestEsClusterOptions = CreateTestEsClusterOptions >(options: Options): EsTestCluster { @@ -244,7 +258,10 @@ export function createTestEsCluster< const second = 1000; const minute = second * 60; - return esFrom === 'snapshot' ? 3 * minute : 6 * minute; + if (esFrom === 'snapshot' || esFrom === 'docker') { + return 3 * minute; + } + return 6 * minute; } async start() { @@ -264,6 +281,19 @@ export function createTestEsCluster< })); } else if (esFrom === 'snapshot') { ({ installPath, disableEsTmpDir } = await firstNode.installSnapshot(config)); + } else if (esFrom === 'docker') { + await firstNode.runDockerSnapshot({ + port, + password, + license: testLicense, + esArgs: customEsArgs, + ssl, + name: `es-${clusterName}`, + background: true, + kill: true, + transportPort: resolveTransportPort(transportPort), + }); + return; } else if (esFrom === 'serverless') { if (!esServerlessOptions) { throw new Error( diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index a4b03db46fe31..f6955801a2ffe 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -291,8 +291,8 @@ const startFleetServerWithDocker = async ({ const hostname = `dev-fleet-server.${port}.${Math.random().toString(32).substring(2, 6)}`; let containerId = ''; - if (isLocalhost(esURL.hostname)) { - esURL.hostname = localhostRealIp; + if (isLocalhost(esURL.hostname) || esURL.hostname === localhostRealIp) { + esURL.hostname = 'host.docker.internal'; } if (isServerless) { diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 181c91df248be..2bf1c2786789a 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -230,23 +230,6 @@ ${chalk.red('NOTE:')} ${chalk.bold( return ''; }; -interface CreateVagrantVmOptions extends BaseVmCreateOptions { - type: SupportedVmManager & 'vagrant'; - - name: string; - /** - * The downloaded agent information. The Agent file will be uploaded to the Vagrant VM and - * made available under the default login home directory (`~/agent-filename`) - */ - agentDownload: DownloadedAgentInfo; - /** - * The path to the Vagrantfile to use to provision the VM. Defaults to Vagrantfile under: - * `x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile` - */ - vagrantFile?: string; - log?: ToolingLog; -} - const ensureVirtualBoxProvider = async (log: ToolingLog): Promise => { const isVboxKernelLoaded = async (): Promise => { try { @@ -270,7 +253,7 @@ const ensureVirtualBoxProvider = async (log: ToolingLog): Promise => { return; } - const loadModules = async (): Promise => { + const tryLoadModules = async (): Promise => { for (const cmd of [ 'sudo modprobe vboxdrv', 'sudo /sbin/vboxconfig', @@ -295,7 +278,7 @@ const ensureVirtualBoxProvider = async (log: ToolingLog): Promise => { await execa.command('sudo apt-get install -y --no-install-recommends virtualbox-7.1', { stdio: 'pipe', }); - return (await loadModules()) && (await isVboxKernelLoaded()); + return (await tryLoadModules()) && (await isVboxKernelLoaded()); } catch { log.debug('virtualbox-7.1 package not available or install failed'); return false; @@ -304,13 +287,14 @@ const ensureVirtualBoxProvider = async (log: ToolingLog): Promise => { log.warning('VirtualBox kernel module not loaded, attempting recovery...'); - if (await loadModules()) { + if (await tryLoadModules()) { return; } log.warning('Kernel module build failed, upgrading to VirtualBox 7.1...'); if (await upgradeToVbox71()) { + log.info('VirtualBox 7.1 upgrade successful'); return; } @@ -335,6 +319,23 @@ const ensureVirtualBoxProvider = async (log: ToolingLog): Promise => { ); }; +interface CreateVagrantVmOptions extends BaseVmCreateOptions { + type: SupportedVmManager & 'vagrant'; + + name: string; + /** + * The downloaded agent information. The Agent file will be uploaded to the Vagrant VM and + * made available under the default login home directory (`~/agent-filename`) + */ + agentDownload: DownloadedAgentInfo; + /** + * The path to the Vagrantfile to use to provision the VM. Defaults to Vagrantfile under: + * `x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile` + */ + vagrantFile?: string; + log?: ToolingLog; +} + /** * Creates a new VM using `vagrant` */ diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/solutions/security/plugins/security_solution/scripts/run_cypress/parallel.ts index d4e8739aee647..1a06c0539e556 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -331,6 +331,10 @@ ${JSON.stringify( let fleetServer: StartedFleetServer | undefined; let shutdownEs; + const esFromEnv = process.env.CYPRESS_ES_FROM; + const configEsFrom = config.get('esTestCluster.from'); + const esFrom = esFromEnv || (configEsFrom === 'serverless' ? 'serverless' : 'docker'); + try { shutdownEs = await pRetry( async () => @@ -338,7 +342,7 @@ ${JSON.stringify( config, log, name: `ftr-${esPort}`, - esFrom: config.get('esTestCluster')?.from || 'snapshot', + esFrom, onEarlyExit, }), { retries: 2, forever: false }