diff --git a/packages/kbn-dev-utils/src/proc_runner/proc.ts b/packages/kbn-dev-utils/src/proc_runner/proc.ts index 8238e29413309..4f9a48d3f34cc 100644 --- a/packages/kbn-dev-utils/src/proc_runner/proc.ts +++ b/packages/kbn-dev-utils/src/proc_runner/proc.ts @@ -168,5 +168,8 @@ export function startProc(name: string, options: ProcOptions, log: ToolingLog) { outcome$, outcomePromise, stop, + stopWasCalled() { + return stopCalled; + }, }; } diff --git a/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts b/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts index cb2ac2604e035..2d2e513dfb98d 100644 --- a/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts +++ b/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts @@ -22,6 +22,7 @@ const noop = () => {}; interface RunOptions extends ProcOptions { wait: true | RegExp; waitTimeout?: number | false; + onEarlyExit?: (msg: string) => void; } /** @@ -48,16 +49,6 @@ export class ProcRunner { /** * Start a process, tracking it by `name` - * @param {String} name - * @param {Object} options - * @property {String} options.cmd executable to run - * @property {Array?} options.args arguments to provide the executable - * @property {String?} options.cwd current working directory for the process - * @property {RegExp|Boolean} options.wait Should start() wait for some time? Use - * `true` will wait until the proc exits, - * a `RegExp` will wait until that log line - * is found - * @return {Promise} */ async run(name: string, options: RunOptions) { const { @@ -67,6 +58,7 @@ export class ProcRunner { wait = false, waitTimeout = 15 * MINUTE, env = process.env, + onEarlyExit, } = options; const cmd = options.cmd === 'node' ? process.execPath : options.cmd; @@ -90,6 +82,25 @@ export class ProcRunner { stdin, }); + if (onEarlyExit) { + proc.outcomePromise + .then( + (code) => { + if (!proc.stopWasCalled()) { + onEarlyExit(`[${name}] exitted early with ${code}`); + } + }, + (error) => { + if (!proc.stopWasCalled()) { + onEarlyExit(`[${name}] exitted early: ${error.message}`); + } + } + ) + .catch((error) => { + throw new Error(`Error handling early exit: ${error.stack}`); + }); + } + try { if (wait instanceof RegExp) { // wait for process to log matching line diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index a6faffc2cfcd7..cb3498d4e2d3e 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -215,6 +215,25 @@ exports.Cluster = class Cluster { }), ]); }); + + if (options.onEarlyExit) { + this._outcome + .then( + () => { + if (!this._stopCalled) { + options.onEarlyExit(`ES exitted unexpectedly`); + } + }, + (error) => { + if (!this._stopCalled) { + options.onEarlyExit(`ES exitted unexpectedly: ${error.stack}`); + } + } + ) + .catch((error) => { + throw new Error(`failure handling early exit: ${error.stack}`); + }); + } } /** diff --git a/packages/kbn-es/src/cluster_exec_options.ts b/packages/kbn-es/src/cluster_exec_options.ts index 8ef3b23cd8c51..da21aaf05b139 100644 --- a/packages/kbn-es/src/cluster_exec_options.ts +++ b/packages/kbn-es/src/cluster_exec_options.ts @@ -15,4 +15,5 @@ export interface EsClusterExecOptions { password?: string; skipReadyCheck?: boolean; readyTimeout?: number; + onEarlyExit?: (msg: string) => void; } diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 27f29ce6995a7..b030b244c22b3 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -145,6 +145,11 @@ export interface CreateTestEsClusterOptions { * defaults to the transport port from `packages/kbn-test/src/es/es_test_config.ts` */ transportPort?: number | string; + /** + * Report to the creator of the es-test-cluster that the es node has exitted before stop() was called, allowing + * this caller to react appropriately. If this is not passed then an uncatchable exception will be thrown + */ + onEarlyExit?: (msg: string) => void; } export function createTestEsCluster< @@ -164,6 +169,7 @@ export function createTestEsCluster< clusterName: customClusterName = 'es-test-cluster', ssl, transportPort, + onEarlyExit, } = options; const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`; @@ -257,6 +263,7 @@ export function createTestEsCluster< // set it up after the last node is started. skipNativeRealmSetup: this.nodes.length > 1 && i < this.nodes.length - 1, skipReadyCheck: this.nodes.length > 1 && i < this.nodes.length - 1, + onEarlyExit, }); }); } diff --git a/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts b/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts index 96ebcd79c4e43..506b6f139f736 100644 --- a/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts +++ b/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts @@ -37,6 +37,7 @@ export interface Test { export interface Runner extends EventEmitter { abort(): void; failures: any[]; + uncaught: (error: Error) => void; } export interface Mocha { diff --git a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts index c8caad3049f14..47134599a9c78 100644 --- a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts +++ b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts @@ -54,7 +54,7 @@ export class FunctionalTestRunner { : new EsVersion(esVersion); } - async run() { + async run(abortSignal?: AbortSignal) { return await this._run(async (config, coreProviders) => { SuiteTracker.startTracking(this.lifecycle, this.configFile); @@ -98,6 +98,10 @@ export class FunctionalTestRunner { reporter, reporterOptions ); + if (abortSignal?.aborted) { + this.log.warning('run aborted'); + return; + } // there's a bug in mocha's dry run, see https://github.com/mochajs/mocha/issues/4838 // until we can update to a mocha version where this is fixed, we won't actually @@ -109,9 +113,14 @@ export class FunctionalTestRunner { } await this.lifecycle.beforeTests.trigger(mocha.suite); - this.log.info('Starting tests'); - return await runTests(this.lifecycle, mocha); + if (abortSignal?.aborted) { + this.log.warning('run aborted'); + return; + } + + this.log.info('Starting tests'); + return await runTests(this.lifecycle, mocha, abortSignal); }); } diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts b/packages/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts index 89f0ea088cac8..d71365a07eac4 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts @@ -6,6 +6,8 @@ * Side Public License, v 1. */ +import * as Rx from 'rxjs'; +import { take } from 'rxjs/operators'; import { Lifecycle } from '../lifecycle'; import { Mocha } from '../../fake_mocha_types'; @@ -18,14 +20,23 @@ import { Mocha } from '../../fake_mocha_types'; * @param {Mocha} mocha * @return {Promise} resolves to the number of test failures */ -export async function runTests(lifecycle: Lifecycle, mocha: Mocha) { +export async function runTests(lifecycle: Lifecycle, mocha: Mocha, abortSignal?: AbortSignal) { let runComplete = false; const runner = mocha.run(() => { runComplete = true; }); - lifecycle.cleanup.add(() => { - if (!runComplete) runner.abort(); + Rx.race( + lifecycle.cleanup.before$, + abortSignal ? Rx.fromEvent(abortSignal, 'abort').pipe(take(1)) : Rx.NEVER + ).subscribe({ + next() { + if (!runComplete) { + runComplete = true; + runner.uncaught(new Error('Forcing mocha to abort')); + runner.abort(); + } + }, }); return new Promise((resolve) => { diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts index ba314e8325a65..5c5289a0d6d6e 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts @@ -17,6 +17,7 @@ interface RunElasticsearchOptions { log: ToolingLog; esFrom?: string; config: Config; + onEarlyExit?: (msg: string) => void; } interface CcsConfig { @@ -92,7 +93,8 @@ export async function runElasticsearch( async function startEsNode( log: ToolingLog, name: string, - config: EsConfig & { transportPort?: number } + config: EsConfig & { transportPort?: number }, + onEarlyExit?: (msg: string) => void ) { const cluster = createTestEsCluster({ clusterName: `cluster-${name}`, @@ -112,6 +114,7 @@ async function startEsNode( }, ], transportPort: config.transportPort, + onEarlyExit, }); await cluster.start(); diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.ts b/packages/kbn-test/src/functional_tests/lib/run_ftr.ts index e339347b07eb5..4b39ac0af5dee 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.ts @@ -81,8 +81,8 @@ async function createFtr({ }; } -export async function assertNoneExcluded({ configPath, options }: CreateFtrParams) { - const { config, ftr } = await createFtr({ configPath, options }); +export async function assertNoneExcluded(params: CreateFtrParams) { + const { config, ftr } = await createFtr(params); if (config.get('testRunner')) { // tests with custom test runners are not included in this check @@ -92,21 +92,21 @@ export async function assertNoneExcluded({ configPath, options }: CreateFtrParam const stats = await ftr.getTestStats(); if (stats.testsExcludedByTag.length > 0) { throw new CliError(` - ${stats.testsExcludedByTag.length} tests in the ${configPath} config + ${stats.testsExcludedByTag.length} tests in the ${params.configPath} config are excluded when filtering by the tags run on CI. Make sure that all suites are tagged with one of the following tags: - ${JSON.stringify(options.suiteTags)} + ${JSON.stringify(params.options.suiteTags)} - ${stats.testsExcludedByTag.join('\n - ')} `); } } -export async function runFtr({ configPath, options }: CreateFtrParams) { - const { ftr } = await createFtr({ configPath, options }); +export async function runFtr(params: CreateFtrParams, signal?: AbortSignal) { + const { ftr } = await createFtr(params); - const failureCount = await ftr.run(); + const failureCount = await ftr.run(signal); if (failureCount > 0) { throw new CliError( `${failureCount} functional test ${failureCount === 1 ? 'failure' : 'failures'}` @@ -114,8 +114,8 @@ export async function runFtr({ configPath, options }: CreateFtrParams) { } } -export async function hasTests({ configPath, options }: CreateFtrParams) { - const { ftr, config } = await createFtr({ configPath, options }); +export async function hasTests(params: CreateFtrParams) { + const { ftr, config } = await createFtr(params); if (config.get('testRunner')) { // configs with custom test runners are assumed to always have tests diff --git a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts index 6305e522b3929..85ed21eba669d 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts @@ -31,10 +31,12 @@ export async function runKibanaServer({ procs, config, options, + onEarlyExit, }: { procs: ProcRunner; config: Config; options: { installDir?: string; extraKbnOpts?: string[] }; + onEarlyExit?: (msg: string) => void; }) { const { installDir } = options; const runOptions = config.get('kbnTestServer.runOptions'); @@ -51,6 +53,7 @@ export async function runKibanaServer({ }, cwd: installDir || KIBANA_ROOT, wait: runOptions.wait, + onEarlyExit, }); } diff --git a/packages/kbn-test/src/functional_tests/tasks.ts b/packages/kbn-test/src/functional_tests/tasks.ts index b1213ceef2905..4ffbfc8e72ca2 100644 --- a/packages/kbn-test/src/functional_tests/tasks.ts +++ b/packages/kbn-test/src/functional_tests/tasks.ts @@ -107,14 +107,26 @@ export async function runTests(options: RunTestsParams) { await withProcRunner(log, async (procs) => { const config = await readConfigFile(log, options.esVersion, configPath); + const abortCtrl = new AbortController(); + + const onEarlyExit = (msg: string) => { + log.error(msg); + abortCtrl.abort(); + }; let shutdownEs; try { if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') { - shutdownEs = await runElasticsearch({ ...options, log, config }); + shutdownEs = await runElasticsearch({ ...options, log, config, onEarlyExit }); + if (abortCtrl.signal.aborted) { + return; + } + } + await runKibanaServer({ procs, config, options, onEarlyExit }); + if (abortCtrl.signal.aborted) { + return; } - await runKibanaServer({ procs, config, options }); - await runFtr({ configPath, options: { ...options, log } }); + await runFtr({ configPath, options: { ...options, log } }, abortCtrl.signal); } finally { try { const delay = config.get('kbnTestServer.delayShutdown');