From 3c8966c1c6b79159c2565fce914323102a3498e8 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 14 Oct 2025 16:36:17 +0200 Subject: [PATCH] [FTR] Track process duration + test stats in run-end (#238397) Currently we only measure the root suite duration, which does not include the time to start Elasticsearch, Kibana (which are started before Mocha runs) and the docker containers (which are in the `beforeTests` lifecycle). With this change, we start tracking A) total process duration which will capture these phases, B) test stats, so we can see the distribution of test counts across configs and the relation between test counts and process/test run duration (cherry picked from commit 086c2900f274de0239dccb82595b93a883fa946b) --- .../src/reporting/jest/reporter.ts | 9 +++++ .../playwright/events/playwright_reporter.ts | 39 +++++++++++++++++++ .../src/reporting/report/events/event.ts | 9 +++++ .../events/persistence/component_templates.ts | 2 +- .../report/events/persistence/mappings.ts | 17 ++++++++ .../lib/mocha/reporter/scout_ftr_reporter.ts | 13 +++++++ 6 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/reporter.ts b/src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/reporter.ts index b1db4b4b1b5c4..b36abe65c15cd 100644 --- a/src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/reporter.ts +++ b/src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/reporter.ts @@ -235,10 +235,19 @@ export class ScoutJestReporter extends BaseReporter { ...this.baseTestRunInfo, status: results.numFailedTests === 0 ? 'passed' : 'failed', duration: Date.now() - results.startTime || 0, + tests: { + failures: results.numFailedTests, + passes: results.numPassedTests, + pending: results.numPendingTests, + total: results.numTotalTests, + }, }, event: { action: ScoutReportEventAction.RUN_END, }, + process: { + uptime: Math.floor(process.uptime() * 1000), + }, }); // Save & conclude the report diff --git a/src/platform/packages/private/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts b/src/platform/packages/private/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts index 8fb4e04de5cfe..67f7afac4c604 100644 --- a/src/platform/packages/private/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts +++ b/src/platform/packages/private/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts @@ -48,6 +48,16 @@ export class ScoutPlaywrightReporter implements Reporter { private baseTestRunInfo: ScoutTestRunInfo; private readonly codeOwnersEntries: CodeOwnersEntry[]; + private readonly testStats: { + passes: number; + failures: number; + pending: number; + } = { + passes: 0, + failures: 0, + pending: 0, + }; + constructor(private reporterOptions: ScoutPlaywrightReporterOptions = {}) { this.log = new ToolingLog({ level: 'info', @@ -227,6 +237,26 @@ export class ScoutPlaywrightReporter implements Reporter { } onTestEnd(test: TestCase, result: TestResult) { + switch (result.status) { + case 'failed': + this.testStats.failures++; + break; + case 'interrupted': + this.testStats.failures++; + break; + case 'timedOut': + this.testStats.failures++; + break; + + case 'passed': + this.testStats.passes++; + break; + + case 'skipped': + this.testStats.pending++; + break; + } + this.report.logEvent({ ...environmentMetadata, reporter: { @@ -269,10 +299,19 @@ export class ScoutPlaywrightReporter implements Reporter { ...this.baseTestRunInfo, status: result.status, duration: result.duration, + tests: { + failures: this.testStats.failures, + passes: this.testStats.passes, + pending: this.testStats.pending, + total: this.testStats.failures + this.testStats.passes + this.testStats.pending, + }, }, event: { action: ScoutReportEventAction.RUN_END, }, + process: { + uptime: Math.floor(process.uptime() * 1000), + }, }); // Save, upload events & conclude the report diff --git a/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/event.ts b/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/event.ts index 9329ed7ea3a6d..6796c0fa3e2e6 100644 --- a/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/event.ts +++ b/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/event.ts @@ -71,6 +71,12 @@ export interface ScoutTestRunInfo { }; status?: string; duration?: number; + tests?: { + passes?: number; + pending?: number; + failures?: number; + total?: number; + }; } /** @@ -116,4 +122,7 @@ export interface ScoutReportEvent { test_run: ScoutTestRunInfo; suite?: ScoutSuiteInfo; test?: ScoutTestInfo; + process?: { + uptime?: number; + }; } diff --git a/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/component_templates.ts b/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/component_templates.ts index 3669698301ab5..9e58a506cf126 100644 --- a/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/component_templates.ts +++ b/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/component_templates.ts @@ -48,7 +48,7 @@ export const reporterMappings: ClusterPutComponentTemplateRequest = { export const testRunMappings: ClusterPutComponentTemplateRequest = { name: 'scout-test-event.mappings.test-run', - version: 3, + version: 4, template: { mappings: { properties: { diff --git a/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts b/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts index 131905d626ffb..a445c34ed8e31 100644 --- a/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts +++ b/src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts @@ -140,6 +140,23 @@ export const testRunProperties: Record = { duration: { type: 'long', }, + tests: { + type: 'object', + properties: { + passes: { + type: 'long', + }, + failures: { + type: 'long', + }, + pending: { + type: 'long', + }, + total: { + type: 'long', + }, + }, + }, config: { type: 'object', properties: { diff --git a/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts b/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts index 1da59c1a45d71..ec0484a40a37c 100644 --- a/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts +++ b/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts @@ -201,6 +201,10 @@ export class ScoutFTRReporter { /** * Root suite execution has ended */ + const passes = this.runner.stats?.passes ?? 0; + const failures = this.runner.stats?.failures ?? 0; + const pending = this.runner.stats?.pending ?? 0; + this.report.logEvent({ ...datasources.environmentMetadata, reporter: { @@ -211,10 +215,19 @@ export class ScoutFTRReporter { ...this.baseTestRunInfo, status: this.runner.stats?.failures === 0 ? 'passed' : 'failed', duration: this.runner.stats?.duration || 0, + tests: { + passes, + failures, + pending, + total: passes + failures + pending, + }, }, event: { action: ScoutReportEventAction.RUN_END, }, + process: { + uptime: Math.floor(process.uptime() * 1000), + }, }); // Save & conclude the report