From 892e2cee306537fd13efc960ebf06c0a46691882 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 11:09:54 +0100 Subject: [PATCH 01/28] feat: --debug pauses test --- packages/playwright/src/runner/testRunner.ts | 4 +- packages/playwright/src/worker/testInfo.ts | 6 +- tests/playwright-test/pause-at-end.spec.ts | 61 ++++++++++++++++++++ tests/playwright-test/timeout.spec.ts | 20 +++---- 4 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 tests/playwright-test/pause-at-end.spec.ts diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts index ec2397e31fee8..b7b80c6bca14b 100644 --- a/packages/playwright/src/runner/testRunner.ts +++ b/packages/playwright/src/runner/testRunner.ts @@ -470,7 +470,9 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }), ...createRunTestsTasks(config), ]; - const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout); + + const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug }); + const status = await runTasks(testRun, tasks, config.config.globalTimeout); // Calling process.exit() might truncate large stdout/stderr output. // See https://github.com/nodejs/node/issues/6456. diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index c9fd959dd9fab..ced3eec2bfe19 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -406,7 +406,7 @@ export class TestInfoImpl implements TestInfo { this._tracing.appendForError(serialized); } - async _runAsStep(stepInfo: { title: string, category: 'hook' | 'fixture', location?: Location, group?: string }, cb: () => Promise) { + async _runAsStep(stepInfo: { title: string, category: 'hook' | 'fixture' | 'test.step', location?: Location, group?: string }, cb: () => Promise) { const step = this._addStep(stepInfo); try { await cb(); @@ -465,7 +465,9 @@ export class TestInfoImpl implements TestInfo { const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure()); if (shouldPause) { this._onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }); - await this._interruptedPromise; + await this._runAsStep({ title: this._isFailure() ? 'Paused on Error' : 'Paused at End', category: 'test.step' }, async () => { + await this._interruptedPromise; + }); } await this._onDidFinishTestFunctionCallback?.(); } diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts new file mode 100644 index 0000000000000..a4cdfb2a97fb7 --- /dev/null +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -0,0 +1,61 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, parseTestRunnerOutput } from './playwright-test-fixtures'; + +test('--debug should pause at end', async ({ interactWithTestRunner }) => { + const testProcess = await interactWithTestRunner({ + 'playwright.config.js': ` + module.exports = {}; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('pass', () => { + }); + test.afterEach(() => { + console.log('teardown'.toUpperCase()); + }); + ` + }, { debug: true, reporter: 'list' }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); + expect(testProcess.output).toContain('TEARDOWN'); + + const result = parseTestRunnerOutput(testProcess.output); + expect(result.interrupted).toBe(1); +}); + +test('--debug should pause on error', async ({ interactWithTestRunner }) => { + const testProcess = await interactWithTestRunner({ + 'playwright.config.js': ` + module.exports = {}; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('pass', () => { + throw new Error('error'); + console.log('after error'.toUpperCase()); + }); + ` + }, { debug: true, reporter: 'list' }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused on Error'); + expect(testProcess.output).not.toContain('AFTER ERROR'); + await testProcess.kill('SIGINT'); + expect(testProcess.output).not.toContain('AFTER ERROR'); + + const result = parseTestRunnerOutput(testProcess.output); + expect(result.failed).toBe(1); +}); diff --git a/tests/playwright-test/timeout.spec.ts b/tests/playwright-test/timeout.spec.ts index 9d6001078d0c2..03b626d9cef79 100644 --- a/tests/playwright-test/timeout.spec.ts +++ b/tests/playwright-test/timeout.spec.ts @@ -143,8 +143,8 @@ test('should respect test.slow', async ({ runInlineTest }) => { expect(result.output).toContain('Test timeout of 1000ms exceeded.'); }); -test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) => { - const result = await runInlineTest({ +test('should ignore test.setTimeout when debugging', async ({ interactWithTestRunner }) => { + const testProcess = await interactWithTestRunner({ 'a.spec.ts': ` import { test as base, expect } from '@playwright/test'; const test = base.extend({ @@ -159,15 +159,15 @@ test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) = await new Promise(f => setTimeout(f, 2000)); }); ` - }, { debug: true }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); }); test('should ignore globalTimeout when debugging', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/34911' }, -}, async ({ runInlineTest }) => { - const result = await runInlineTest({ +}, async ({ interactWithTestRunner }) => { + const testProcess = await interactWithTestRunner({ 'playwright.config.ts': ` export default { globalTimeout: 100, @@ -179,9 +179,9 @@ test('should ignore globalTimeout when debugging', { await new Promise(f => setTimeout(f, 2000)); }); ` - }, { debug: true }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); }); test('should respect fixture timeout', async ({ runInlineTest }) => { From bcf2fec7bd960d07efb2a9965a8276a0f3f9c2db Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 11:11:33 +0100 Subject: [PATCH 02/28] location --- packages/playwright/src/worker/testInfo.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index ced3eec2bfe19..a10404c7154f0 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -465,7 +465,17 @@ export class TestInfoImpl implements TestInfo { const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure()); if (shouldPause) { this._onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }); - await this._runAsStep({ title: this._isFailure() ? 'Paused on Error' : 'Paused at End', category: 'test.step' }, async () => { + + let location: Location | undefined; + if (this.error) { + if (this.error.stack) + location = filteredStackTrace(this.error.stack.split('\n'))[0]; + } else { + const source = await fs.promises.readFile(this.file, 'utf-8'); + location = findTestEndPosition(source, this); + } + location ??= this; + await this._runAsStep({ title: this._isFailure() ? 'Paused on Error' : 'Paused at End', category: 'test.step', location }, async () => { await this._interruptedPromise; }); } From c40f10251b280cf7f5efe5e0b1b44e4ec975ecc1 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 11:28:07 +0100 Subject: [PATCH 03/28] proper step location --- .../playwright/src/transform/babelBundle.ts | 2 +- .../src/transform/babelHighlightUtils.ts | 71 +++++++++++++++++++ packages/playwright/src/worker/testInfo.ts | 22 +++--- tests/playwright-test/pause-at-end.spec.ts | 20 ++++-- 4 files changed, 100 insertions(+), 15 deletions(-) create mode 100644 packages/playwright/src/transform/babelHighlightUtils.ts diff --git a/packages/playwright/src/transform/babelBundle.ts b/packages/playwright/src/transform/babelBundle.ts index ce61eecc90a66..11ed0e4e22030 100644 --- a/packages/playwright/src/transform/babelBundle.ts +++ b/packages/playwright/src/transform/babelBundle.ts @@ -24,5 +24,5 @@ export type BabelTransformFunction = (code: string, filename: string, isModule: export const babelTransform: BabelTransformFunction = require('./babelBundleImpl').babelTransform; export type BabelParseFunction = (code: string, filename: string, isModule: boolean) => ParseResult; export const babelParse: BabelParseFunction = require('./babelBundleImpl').babelParse; -export type { NodePath, PluginObj, types as T } from '../../bundles/babel/node_modules/@types/babel__core'; +export type { NodePath, PluginObj, types as T, ParseResult } from '../../bundles/babel/node_modules/@types/babel__core'; export type { BabelAPI } from '../../bundles/babel/node_modules/@types/babel__helper-plugin-utils'; diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts new file mode 100644 index 0000000000000..f7018ac6c05e5 --- /dev/null +++ b/packages/playwright/src/transform/babelHighlightUtils.ts @@ -0,0 +1,71 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import { traverse, babelParse, ParseResult, T, types as t } from './babelBundle'; +import type { Location } from '../../types/testReporter'; + +const astCache = new Map(); + +export function pruneAstCaches(fsPathsToRetain: string[]) { + const retain = new Set(fsPathsToRetain); + for (const key of astCache.keys()) { + if (!retain.has(key)) + astCache.delete(key); + } +} + +function getAst(text: string, fsPath: string) { + const cached = astCache.get(fsPath); + let ast = cached?.ast; + if (!cached || cached.text !== text) { + try { + ast = babelParse(text, path.basename(fsPath), false); + astCache.set(fsPath, { text, ast }); + } catch (e) { + astCache.set(fsPath, { text, ast: undefined }); + } + } + return ast; +} + +function containsPosition(location: T.SourceLocation, position: Location): boolean { + if (position.line < location.start.line || position.line > location.end.line) + return false; + if (position.line === location.start.line && position.column < location.start.column) + return false; + if (position.line === location.end.line && position.column > location.end.column) + return false; + return true; +} + +export function findTestEndPosition(text: string, location: Location): Location | undefined { + const ast = getAst(text, location.file); + if (!ast) + return; + let result: Location | undefined; + traverse(ast, { + enter(path) { + if (t.isCallExpression(path.node) && path.node.loc && containsPosition(path.node.loc, location)) { + const callNode = path.node; + const funcNode = callNode.arguments[callNode.arguments.length - 1]; + if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc) + result = { file: location.file, ...funcNode.body.loc.end }; + } + } + }); + return result; +} diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index a10404c7154f0..d661e59ac60ca 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -24,6 +24,7 @@ import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAnd import { TestTracing } from './testTracing'; import { testInfoError } from './util'; import { wrapFunctionWithLocation } from '../transform/transform'; +import { findTestEndPosition } from '../transform/babelHighlightUtils'; import type { RunnableDescription } from './timeoutManager'; import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test'; @@ -464,17 +465,8 @@ export class TestInfoImpl implements TestInfo { async _didFinishTestFunction() { const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure()); if (shouldPause) { + const location = (this._isFailure() ? this._errorLocation() : await this._testEndLocation()) ?? { file: this.file, line: this.line, column: this.column }; this._onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }); - - let location: Location | undefined; - if (this.error) { - if (this.error.stack) - location = filteredStackTrace(this.error.stack.split('\n'))[0]; - } else { - const source = await fs.promises.readFile(this.file, 'utf-8'); - location = findTestEndPosition(source, this); - } - location ??= this; await this._runAsStep({ title: this._isFailure() ? 'Paused on Error' : 'Paused at End', category: 'test.step', location }, async () => { await this._interruptedPromise; }); @@ -482,6 +474,16 @@ export class TestInfoImpl implements TestInfo { await this._onDidFinishTestFunctionCallback?.(); } + _errorLocation(): Location | undefined { + if (this.error?.stack) + return filteredStackTrace(this.error.stack.split('\n'))[0]; + } + + async _testEndLocation() { + const source = await fs.promises.readFile(this.file, 'utf-8'); + return findTestEndPosition(source, this); + } + // ------------ TestInfo methods ------------ async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) { diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts index a4cdfb2a97fb7..7c504070b4f79 100644 --- a/tests/playwright-test/pause-at-end.spec.ts +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -14,12 +14,21 @@ * limitations under the License. */ +import { Reporter, TestCase, TestResult, TestStep } from 'packages/playwright-test/reporter'; import { test, expect, parseTestRunnerOutput } from './playwright-test-fixtures'; +class LocationReporter implements Reporter { + onStepBegin(test: TestCase, result: TestResult, step: TestStep): void { + if (step.title.startsWith('Paused')) + console.log(`\n%%${step.title} at :${step.location?.line}:${step.location?.column}\n`); + } +} + test('--debug should pause at end', async ({ interactWithTestRunner }) => { const testProcess = await interactWithTestRunner({ + 'location-reporter.js': `export default ${LocationReporter}`, 'playwright.config.js': ` - module.exports = {}; + module.exports = { reporter: [['list'], ['./location-reporter.js']] }; `, 'a.test.js': ` import { test, expect } from '@playwright/test'; @@ -29,10 +38,11 @@ test('--debug should pause at end', async ({ interactWithTestRunner }) => { console.log('teardown'.toUpperCase()); }); ` - }, { debug: true, reporter: 'list' }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); await testProcess.waitForOutput('Paused at End'); await testProcess.kill('SIGINT'); expect(testProcess.output).toContain('TEARDOWN'); + expect(testProcess.outputLines()).toContain('Paused at End at :4:7'); const result = parseTestRunnerOutput(testProcess.output); expect(result.interrupted).toBe(1); @@ -40,8 +50,9 @@ test('--debug should pause at end', async ({ interactWithTestRunner }) => { test('--debug should pause on error', async ({ interactWithTestRunner }) => { const testProcess = await interactWithTestRunner({ + 'location-reporter.js': `export default ${LocationReporter}`, 'playwright.config.js': ` - module.exports = {}; + module.exports = { reporter: [['list'], ['./location-reporter.js']] }; `, 'a.test.js': ` import { test, expect } from '@playwright/test'; @@ -50,11 +61,12 @@ test('--debug should pause on error', async ({ interactWithTestRunner }) => { console.log('after error'.toUpperCase()); }); ` - }, { debug: true, reporter: 'list' }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); await testProcess.waitForOutput('Paused on Error'); expect(testProcess.output).not.toContain('AFTER ERROR'); await testProcess.kill('SIGINT'); expect(testProcess.output).not.toContain('AFTER ERROR'); + expect(testProcess.outputLines()).toContain('Paused on Error at :4:15'); const result = parseTestRunnerOutput(testProcess.output); expect(result.failed).toBe(1); From 4b87e41b797be3d71068752eb224442189f382e8 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 12:10:38 +0100 Subject: [PATCH 04/28] progressively report errors --- packages/playwright/src/common/ipc.ts | 1 + .../src/reporters/internalReporter.ts | 1 + packages/playwright/src/runner/dispatcher.ts | 6 +++-- packages/playwright/src/worker/testInfo.ts | 4 ++++ packages/playwright/src/worker/workerMain.ts | 2 +- tests/playwright-test/pause-at-end.spec.ts | 23 +++++++++++++++---- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 4d9ca7f846280..7c9ba3fd719c9 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -120,6 +120,7 @@ export type StepBeginPayload = { category: string; wallTime: number; // milliseconds since unix epoch location?: { file: string, line: number, column: number }; + errors: TestInfoErrorImpl[]; }; export type StepEndPayload = { diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 946c16932ba2f..d4367f4797b90 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -94,6 +94,7 @@ export class InternalReporter implements ReporterV2 { } onStepBegin(test: TestCase, result: TestResult, step: TestStep) { + this._addSnippetToTestErrors(test, result); this._reporter.onStepBegin?.(test, result, step); } diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 358f639ac01aa..fc1801056c72c 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -333,7 +333,7 @@ class JobDispatcher { this._remainingByTestId.delete(params.testId); const { result, test } = data; result.duration = params.duration; - result.errors = params.errors; + result.errors.push(...params.errors); result.error = result.errors[0]; result.status = params.status; result.annotations = params.annotations; @@ -382,6 +382,8 @@ class JobDispatcher { }; steps.set(params.stepId, step); (parentStep || result).steps.push(step); + result.errors.push(...params.errors); + result.error = result.errors[0]; this._reporter.onStepBegin?.(test, result, step); } @@ -439,7 +441,7 @@ class JobDispatcher { result = test._appendTestResult(); this._reporter.onTestBegin?.(test, result); } - result.errors = [...errors]; + result.errors.push(...errors); result.error = result.errors[0]; result.status = errors.length ? 'failed' : 'skipped'; this._reportTestEnd(test, result); diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index d661e59ac60ca..88e3181764844 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -122,6 +122,7 @@ export class TestInfoImpl implements TestInfo { readonly outputDir: string; readonly snapshotDir: string; errors: TestInfoErrorImpl[] = []; + _reportedError = 0; readonly _attachmentsPush: (...items: TestInfo['attachments']) => number; private _workerParams: WorkerInitParams; @@ -361,6 +362,8 @@ export class TestInfoImpl implements TestInfo { this._stepMap.set(stepId, step); if (!step.group) { + const errors = this.errors.slice(this._reportedError); + this._reportedError = this.errors.length; const payload: StepBeginPayload = { testId: this.testId, stepId, @@ -369,6 +372,7 @@ export class TestInfoImpl implements TestInfo { category: step.category, wallTime: Date.now(), location: step.location, + errors, }; this._onStepBegin(payload); } diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index f1a96cbdd848d..9ffa76e91f1f7 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -622,7 +622,7 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { testId: testInfo.testId, duration: testInfo.duration, status: testInfo.status!, - errors: testInfo.errors, + errors: testInfo.errors.slice(testInfo._reportedError), hasNonRetriableError: testInfo._hasNonRetriableError, expectedStatus: testInfo.expectedStatus, annotations: testInfo.annotations, diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts index 7c504070b4f79..1dd27180e9595 100644 --- a/tests/playwright-test/pause-at-end.spec.ts +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -19,8 +19,15 @@ import { test, expect, parseTestRunnerOutput } from './playwright-test-fixtures' class LocationReporter implements Reporter { onStepBegin(test: TestCase, result: TestResult, step: TestStep): void { - if (step.title.startsWith('Paused')) - console.log(`\n%%${step.title} at :${step.location?.line}:${step.location?.column}\n`); + if (step.title.startsWith('Paused')) { + console.log('\n'); + console.log(`%%${step.title} at :${step.location?.line}:${step.location?.column}`); + if (result.error) + console.log(`%%result.error at :${result.error.location?.line}:${result.error.location?.column}`); + for (const [index, error] of result.errors.entries()) + console.log(`%%result.errors[${index}] at :${error.location?.line}:${error.location?.column}`); + console.log('\n'); + } } } @@ -42,7 +49,7 @@ test('--debug should pause at end', async ({ interactWithTestRunner }) => { await testProcess.waitForOutput('Paused at End'); await testProcess.kill('SIGINT'); expect(testProcess.output).toContain('TEARDOWN'); - expect(testProcess.outputLines()).toContain('Paused at End at :4:7'); + expect(testProcess.outputLines()).toEqual(['Paused at End at :4:7']); const result = parseTestRunnerOutput(testProcess.output); expect(result.interrupted).toBe(1); @@ -57,7 +64,8 @@ test('--debug should pause on error', async ({ interactWithTestRunner }) => { 'a.test.js': ` import { test, expect } from '@playwright/test'; test('pass', () => { - throw new Error('error'); + expect.soft(1).toBe(2); + expect(2).toBe(3); console.log('after error'.toUpperCase()); }); ` @@ -66,7 +74,12 @@ test('--debug should pause on error', async ({ interactWithTestRunner }) => { expect(testProcess.output).not.toContain('AFTER ERROR'); await testProcess.kill('SIGINT'); expect(testProcess.output).not.toContain('AFTER ERROR'); - expect(testProcess.outputLines()).toContain('Paused on Error at :4:15'); + expect(testProcess.outputLines()).toEqual([ + 'Paused on Error at :4:24', + 'result.error at :4:24', + 'result.errors[0] at :4:24', + 'result.errors[1] at :5:19', + ]); const result = parseTestRunnerOutput(testProcess.output); expect(result.failed).toBe(1); From b26031ff5dfc8a7c09d53607e4b5a9d4d1f670d0 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 12:23:03 +0100 Subject: [PATCH 05/28] tele --- packages/playwright/src/isomorphic/teleReceiver.ts | 9 ++++++--- packages/playwright/src/reporters/teleEmitter.ts | 12 ++++++++++-- tests/playwright-test/pause-at-end.spec.ts | 7 +++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index cef009c730941..030f3200dfaf2 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -181,6 +181,7 @@ export type JsonOnStepBeginEvent = { testId: string; resultId: string; step: JsonTestStepStart; + errors: reporterTypes.TestError[]; }; }; @@ -287,7 +288,7 @@ export class TeleReporterReceiver { return; } if (method === 'onStepBegin') { - this._onStepBegin(params.testId, params.resultId, params.step); + this._onStepBegin(params.testId, params.resultId, params.step, params.errors); return; } if (method === 'onAttach') { @@ -353,7 +354,7 @@ export class TeleReporterReceiver { const result = test.results.find(r => r._id === payload.id)!; result.duration = payload.duration; result.status = payload.status; - result.errors = payload.errors; + result.errors.push(...payload.errors); result.error = result.errors?.[0]; // Attachments are only present here from legacy blobs. These override all _onAttach events if (!!payload.attachments) @@ -368,7 +369,7 @@ export class TeleReporterReceiver { result._stepMap = new Map(); } - private _onStepBegin(testId: string, resultId: string, payload: JsonTestStepStart) { + private _onStepBegin(testId: string, resultId: string, payload: JsonTestStepStart, errors: reporterTypes.TestError[]) { const test = this._tests.get(testId)!; const result = test.results.find(r => r._id === resultId)!; const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined; @@ -380,6 +381,8 @@ export class TeleReporterReceiver { else result.steps.push(step); result._stepMap.set(payload.id, step); + result.errors.push(...errors); + result.error = result.errors[0]; this._reporter.onStepBegin?.(test, result, step); } diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index d191104b7099b..1f63890f959c7 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -38,6 +38,7 @@ export class TeleReporterEmitter implements ReporterV2 { // In case there is blob reporter and UI mode, make sure one does override // the id assigned by the other. private readonly _idSymbol = Symbol('id'); + private readonly _reportedErrorsSymbol = Symbol('reportedErrors'); constructor(messageSink: (message: teleReceiver.JsonEvent) => void, options: TeleReporterEmitterOptions = {}) { this._messageSink = messageSink; @@ -97,11 +98,18 @@ export class TeleReporterEmitter implements ReporterV2 { params: { testId: test.id, resultId: (result as any)[this._idSymbol], - step: this._serializeStepStart(step) + step: this._serializeStepStart(step), + errors: this._unreportedErrors(result), } }); } + private _unreportedErrors(result: reporterTypes.TestResult) { + const index = (result as any)[this._reportedErrorsSymbol] ?? 0; + (result as any)[this._reportedErrorsSymbol] = result.errors.length; + return result.errors.slice(index); + } + onStepEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { // Create synthetic onAttach event so we serialize the entire attachment along with the step const resultId = (result as any)[this._idSymbol] as string; @@ -248,7 +256,7 @@ export class TeleReporterEmitter implements ReporterV2 { id: (result as any)[this._idSymbol], duration: result.duration, status: result.status, - errors: result.errors, + errors: this._unreportedErrors(result), annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined, }; } diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts index 1dd27180e9595..9652622337d94 100644 --- a/tests/playwright-test/pause-at-end.spec.ts +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -55,11 +55,11 @@ test('--debug should pause at end', async ({ interactWithTestRunner }) => { expect(result.interrupted).toBe(1); }); -test('--debug should pause on error', async ({ interactWithTestRunner }) => { +test('--debug should pause on error', async ({ interactWithTestRunner, mergeReports }) => { const testProcess = await interactWithTestRunner({ 'location-reporter.js': `export default ${LocationReporter}`, 'playwright.config.js': ` - module.exports = { reporter: [['list'], ['./location-reporter.js']] }; + module.exports = { reporter: [['list'], ['blob'], ['./location-reporter.js']] }; `, 'a.test.js': ` import { test, expect } from '@playwright/test'; @@ -83,4 +83,7 @@ test('--debug should pause on error', async ({ interactWithTestRunner }) => { const result = parseTestRunnerOutput(testProcess.output); expect(result.failed).toBe(1); + + const merged = await mergeReports('blob-report', undefined, { additionalArgs: ['--reporter', 'location-reporter.js'] }); + expect(merged.outputLines).toEqual(testProcess.outputLines()); }); From c1c98679e5eb6a69e84eb5f6ead72462b26c2353 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 12:24:17 +0100 Subject: [PATCH 06/28] skip on windows --- tests/playwright-test/pause-at-end.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts index 9652622337d94..ca5eb0dda803d 100644 --- a/tests/playwright-test/pause-at-end.spec.ts +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -31,7 +31,8 @@ class LocationReporter implements Reporter { } } -test('--debug should pause at end', async ({ interactWithTestRunner }) => { +test('--debug should pause at end', async ({ interactWithTestRunner, }) => { + test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); const testProcess = await interactWithTestRunner({ 'location-reporter.js': `export default ${LocationReporter}`, 'playwright.config.js': ` @@ -56,6 +57,7 @@ test('--debug should pause at end', async ({ interactWithTestRunner }) => { }); test('--debug should pause on error', async ({ interactWithTestRunner, mergeReports }) => { + test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); const testProcess = await interactWithTestRunner({ 'location-reporter.js': `export default ${LocationReporter}`, 'playwright.config.js': ` From b515837b33de831486b3a2c5f763d427e77535bb Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 12:53:12 +0100 Subject: [PATCH 07/28] reporting more errors --- tests/playwright-test/max-failures.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.ts index fed320b8267f4..1ce7493092548 100644 --- a/tests/playwright-test/max-failures.spec.ts +++ b/tests/playwright-test/max-failures.spec.ts @@ -61,7 +61,7 @@ test('-x should work', async ({ runInlineTest }) => { }, { '-x': true }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.output.split('\n').filter(l => l.includes('expect(')).length).toBe(2); + expect(result.output.split('\n').filter(l => l.includes('expect(')).length).toBe(4); }); test('max-failures should work with retries', async ({ runInlineTest }) => { From 0a611bb2f7f292d6a323dc47ae1f5ff8b68bffa1 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 12:54:01 +0100 Subject: [PATCH 08/28] errors are optional --- packages/playwright/src/isomorphic/teleReceiver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 030f3200dfaf2..99b138970fe76 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -369,7 +369,7 @@ export class TeleReporterReceiver { result._stepMap = new Map(); } - private _onStepBegin(testId: string, resultId: string, payload: JsonTestStepStart, errors: reporterTypes.TestError[]) { + private _onStepBegin(testId: string, resultId: string, payload: JsonTestStepStart, errors: reporterTypes.TestError[] = []) { const test = this._tests.get(testId)!; const result = test.results.find(r => r._id === resultId)!; const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined; From db2419db64e4bd5c5ba8cfcbd0cbe41b6a1b8e88 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 12:54:38 +0100 Subject: [PATCH 09/28] mark optional for backwards compat --- packages/playwright/src/isomorphic/teleReceiver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 99b138970fe76..4281f89fd83db 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -181,7 +181,7 @@ export type JsonOnStepBeginEvent = { testId: string; resultId: string; step: JsonTestStepStart; - errors: reporterTypes.TestError[]; + errors?: reporterTypes.TestError[]; }; }; From 099653461d83a5508f393bb6e1c2786e0ab3b9db Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 13:41:03 +0100 Subject: [PATCH 10/28] extract into TestErrorPayload ipc --- packages/playwright/src/common/ipc.ts | 8 +++--- packages/playwright/src/runner/dispatcher.ts | 27 +++++++++++++------- packages/playwright/src/worker/testInfo.ts | 18 +++++++++---- packages/playwright/src/worker/workerMain.ts | 6 +++-- 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 7c9ba3fd719c9..bd6ebaea89974 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -84,11 +84,15 @@ export type AttachmentPayload = { stepId?: string; }; +export type TestErrorPayload = { + testId: string; + errors: TestInfoErrorImpl[]; +}; + export type TestInfoErrorImpl = TestInfoError; export type TestPausedPayload = { testId: string; - errors: TestInfoErrorImpl[]; }; export type CustomMessageRequestPayload = { @@ -105,7 +109,6 @@ export type TestEndPayload = { testId: string; duration: number; status: TestStatus; - errors: TestInfoErrorImpl[]; hasNonRetriableError: boolean; expectedStatus: TestStatus; annotations: { type: string, description?: string }[]; @@ -120,7 +123,6 @@ export type StepBeginPayload = { category: string; wallTime: number; // milliseconds since unix epoch location?: { file: string, line: number, column: number }; - errors: TestInfoErrorImpl[]; }; export type StepEndPayload = { diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index fc1801056c72c..3b0898fc80c8c 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -28,7 +28,7 @@ import type { ProcessExitData } from './processHost'; import type { TestGroup } from './testGroups'; import type { TestError, TestResult, TestStep } from '../../types/testReporter'; import type { FullConfigInternal } from '../common/config'; -import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc'; +import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc'; import type { Suite } from '../common/test'; import type { TestCase } from '../common/test'; import type { ReporterV2 } from '../reporters/reporterV2'; @@ -322,7 +322,6 @@ class JobDispatcher { // Do not show more than one error to avoid confusion, but report // as interrupted to indicate that we did actually start the test. params.status = 'interrupted'; - params.errors = []; } const data = this._dataByTestId.get(params.testId); if (!data) { @@ -333,8 +332,6 @@ class JobDispatcher { this._remainingByTestId.delete(params.testId); const { result, test } = data; result.duration = params.duration; - result.errors.push(...params.errors); - result.error = result.errors[0]; result.status = params.status; result.annotations = params.annotations; test.annotations = [...params.annotations]; // last test result wins @@ -382,8 +379,6 @@ class JobDispatcher { }; steps.set(params.stepId, step); (parentStep || result).steps.push(step); - result.errors.push(...params.errors); - result.error = result.errors[0]; this._reporter.onStepBegin?.(test, result, step); } @@ -431,6 +426,17 @@ class JobDispatcher { } } + private _onErrors(params: TestErrorPayload) { + const data = this._dataByTestId.get(params.testId)!; + if (!data) + return; + const { result } = data; + for (const error of params.errors) + addLocationAndSnippetToError(this._config.config, error); + result.errors.push(...params.errors); + result.error = result.errors[0]; + } + private _failTestWithErrors(test: TestCase, errors: TestError[]) { const runData = this._dataByTestId.get(test.id); // There might be a single test that has started but has not finished yet. @@ -580,6 +586,7 @@ class JobDispatcher { eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)), eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)), eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)), + eventsHelper.addEventListener(worker, 'errors', this._onErrors.bind(this)), eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)), eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)), eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)), @@ -602,9 +609,11 @@ class JobDispatcher { } }; - for (const error of params.errors) - addLocationAndSnippetToError(this._config.config, error); - this._failureTracker.onTestPaused?.({ ...params, sendMessage }); + const data = this._dataByTestId.get(params.testId); + if (!data) + return; + + this._failureTracker.onTestPaused?.({ errors: data.result.errors, sendMessage }); } skipWholeJob(): boolean { diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 88e3181764844..989be36271499 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -30,7 +30,7 @@ import type { RunnableDescription } from './timeoutManager'; import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test'; import type { FullConfig, Location } from '../../types/testReporter'; import type { FullConfigInternal, FullProjectInternal } from '../common/config'; -import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc'; +import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestErrorPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc'; import type { TestCase } from '../common/test'; import type { StackFrame } from '@protocol/channels'; @@ -70,6 +70,7 @@ export class TestInfoImpl implements TestInfo { private _onStepBegin: (payload: StepBeginPayload) => void; private _onStepEnd: (payload: StepEndPayload) => void; private _onAttach: (payload: AttachmentPayload) => void; + private _onError: (errors: TestErrorPayload) => void; private _onTestPaused: (payload: TestPausedPayload) => void; private _snapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; private _ariaSnapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; @@ -166,12 +167,14 @@ export class TestInfoImpl implements TestInfo { onStepBegin: (payload: StepBeginPayload) => void, onStepEnd: (payload: StepEndPayload) => void, onAttach: (payload: AttachmentPayload) => void, + onError: (payload: TestErrorPayload) => void, onTestPaused: (payload: TestPausedPayload) => void, ) { this.testId = test?.id ?? ''; this._onStepBegin = onStepBegin; this._onStepEnd = onStepEnd; this._onAttach = onAttach; + this._onError = onError; this._onTestPaused = onTestPaused; this._startTime = monotonicTime(); this._startWallTime = Date.now(); @@ -362,8 +365,6 @@ export class TestInfoImpl implements TestInfo { this._stepMap.set(stepId, step); if (!step.group) { - const errors = this.errors.slice(this._reportedError); - this._reportedError = this.errors.length; const payload: StepBeginPayload = { testId: this.testId, stepId, @@ -372,7 +373,6 @@ export class TestInfoImpl implements TestInfo { category: step.category, wallTime: Date.now(), location: step.location, - errors, }; this._onStepBegin(payload); } @@ -470,7 +470,8 @@ export class TestInfoImpl implements TestInfo { const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure()); if (shouldPause) { const location = (this._isFailure() ? this._errorLocation() : await this._testEndLocation()) ?? { file: this.file, line: this.line, column: this.column }; - this._onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }); + this._emitErrors(); + this._onTestPaused({ testId: this.testId }); await this._runAsStep({ title: this._isFailure() ? 'Paused on Error' : 'Paused at End', category: 'test.step', location }, async () => { await this._interruptedPromise; }); @@ -478,6 +479,13 @@ export class TestInfoImpl implements TestInfo { await this._onDidFinishTestFunctionCallback?.(); } + _emitErrors() { + const errors = this.errors.slice(this._reportedError); + this._reportedError = this.errors.length; + if (errors.length) + this._onError({ testId: this.testId, errors }); + } + _errorLocation(): Location | undefined { if (this.error?.stack) return filteredStackTrace(this.error.stack.split('\n'))[0]; diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index 9ffa76e91f1f7..260637606d91f 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -118,7 +118,7 @@ export class WorkerMain extends ProcessRunner { return; } // Ignore top-level errors, they are already inside TestInfo.errors. - const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {}); + const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => { }, () => { }, () => { }, () => { }, () => { }); const runnable = { type: 'teardown' } as const; // We have to load the project to get the right deadline below. await fakeTestInfo._runWithTimeout(runnable, () => this._loadIfNeeded()).catch(() => {}); @@ -283,6 +283,7 @@ export class WorkerMain extends ProcessRunner { stepBeginPayload => this.dispatchEvent('stepBegin', stepBeginPayload), stepEndPayload => this.dispatchEvent('stepEnd', stepEndPayload), attachment => this.dispatchEvent('attach', attachment), + errors => this.dispatchEvent('errors', errors), testPausedPayload => this.dispatchEvent('testPaused', testPausedPayload)); const processAnnotation = (annotation: TestAnnotation) => { @@ -331,6 +332,7 @@ export class WorkerMain extends ProcessRunner { if (isSkipped && nextTest && !hasAfterAllToRunBeforeNextTest) { // Fast path - this test is skipped, and there are more tests that will handle cleanup. testInfo.status = 'skipped'; + testInfo._emitErrors(); this.dispatchEvent('testEnd', buildTestEndPayload(testInfo)); return; } @@ -497,6 +499,7 @@ export class WorkerMain extends ProcessRunner { this._currentTest = null; setCurrentTestInfo(null); + testInfo._emitErrors(); this.dispatchEvent('testEnd', buildTestEndPayload(testInfo)); const preserveOutput = this._config.config.preserveOutput === 'always' || @@ -622,7 +625,6 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { testId: testInfo.testId, duration: testInfo.duration, status: testInfo.status!, - errors: testInfo.errors.slice(testInfo._reportedError), hasNonRetriableError: testInfo._hasNonRetriableError, expectedStatus: testInfo.expectedStatus, annotations: testInfo.annotations, From 8381924a25a656ed439d5bdafb9b4c7c9002d817 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 13:52:39 +0100 Subject: [PATCH 11/28] mirror attachments --- .../playwright/src/isomorphic/teleReceiver.ts | 39 ++++++++++++++----- .../playwright/src/reporters/teleEmitter.ts | 32 ++++++++++----- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 4281f89fd83db..6a352160ceae0 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -96,7 +96,8 @@ export type JsonTestResultEnd = { id: string; duration: number; status: reporterTypes.TestStatus; - errors: reporterTypes.TestError[]; + /** No longer emitted, but kept for backwards compatibility */ + errors?: reporterTypes.TestError[]; /** No longer emitted, but kept for backwards compatibility */ attachments?: JsonAttachment[]; annotations?: TestAnnotation[]; @@ -132,7 +133,7 @@ export type JsonFullResult = { }; export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent - | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnErrorEvent | JsonOnStdIOEvent; + | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorsEvent | JsonOnErrorEvent | JsonOnStdIOEvent; export type JsonOnConfigureEvent = { method: 'onConfigure'; @@ -181,7 +182,6 @@ export type JsonOnStepBeginEvent = { testId: string; resultId: string; step: JsonTestStepStart; - errors?: reporterTypes.TestError[]; }; }; @@ -199,6 +199,15 @@ export type JsonOnAttachEvent = { params: JsonTestResultOnAttach; }; +export type JsonOnTestErrorsEvent = { + method: 'onTestErrors'; + params: { + testId: string; + resultId: string; + errors: reporterTypes.TestError[]; + } +}; + export type JsonOnErrorEvent = { method: 'onError'; params: { @@ -288,13 +297,17 @@ export class TeleReporterReceiver { return; } if (method === 'onStepBegin') { - this._onStepBegin(params.testId, params.resultId, params.step, params.errors); + this._onStepBegin(params.testId, params.resultId, params.step); return; } if (method === 'onAttach') { this._onAttach(params.testId, params.resultId, params.attachments); return; } + if (method === 'onTestErrors') { + this._onTestErrors(params.testId, params.resultId, params.errors); + return; + } if (method === 'onStepEnd') { this._onStepEnd(params.testId, params.resultId, params.step); return; @@ -354,8 +367,11 @@ export class TeleReporterReceiver { const result = test.results.find(r => r._id === payload.id)!; result.duration = payload.duration; result.status = payload.status; - result.errors.push(...payload.errors); - result.error = result.errors?.[0]; + // Errors are only present here from legacy blobs. These override all _onTestErrors events + if (!!payload.errors) { + result.errors = payload.errors; + result.error = result.errors?.[0]; + } // Attachments are only present here from legacy blobs. These override all _onAttach events if (!!payload.attachments) result.attachments = this._parseAttachments(payload.attachments); @@ -369,7 +385,7 @@ export class TeleReporterReceiver { result._stepMap = new Map(); } - private _onStepBegin(testId: string, resultId: string, payload: JsonTestStepStart, errors: reporterTypes.TestError[] = []) { + private _onStepBegin(testId: string, resultId: string, payload: JsonTestStepStart) { const test = this._tests.get(testId)!; const result = test.results.find(r => r._id === resultId)!; const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined; @@ -381,8 +397,6 @@ export class TeleReporterReceiver { else result.steps.push(step); result._stepMap.set(payload.id, step); - result.errors.push(...errors); - result.error = result.errors[0]; this._reporter.onStepBegin?.(test, result, step); } @@ -407,6 +421,13 @@ export class TeleReporterReceiver { }))); } + private _onTestErrors(testId: string, resultId: string, errors: reporterTypes.TestError[]) { + const test = this._tests.get(testId)!; + const result = test.results.find(r => r._id === resultId)!; + result.errors.push(...errors); + result.error = result.errors[0]; + } + private _onError(error: reporterTypes.TestError) { this._reporter.onError?.(error); } diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 1f63890f959c7..d72f2c9aeaab7 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -34,11 +34,11 @@ export class TeleReporterEmitter implements ReporterV2 { private _messageSink: (message: teleReceiver.JsonEvent) => void; private _rootDir!: string; private _emitterOptions: TeleReporterEmitterOptions; + private _resultKnownErrorsCounts = new Map(); private _resultKnownAttachmentCounts = new Map(); // In case there is blob reporter and UI mode, make sure one does override // the id assigned by the other. private readonly _idSymbol = Symbol('id'); - private readonly _reportedErrorsSymbol = Symbol('reportedErrors'); constructor(messageSink: (message: teleReceiver.JsonEvent) => void, options: TeleReporterEmitterOptions = {}) { this._messageSink = messageSink; @@ -79,6 +79,7 @@ export class TeleReporterEmitter implements ReporterV2 { timeout: test.timeout, annotations: [] }; + this._sendNewErrors(result, test.id); this._sendNewAttachments(result, test.id); this._messageSink({ method: 'onTestEnd', @@ -88,28 +89,24 @@ export class TeleReporterEmitter implements ReporterV2 { } }); - this._resultKnownAttachmentCounts.delete((result as any)[this._idSymbol]); + const resultId = (result as any)[this._idSymbol] as string; + this._resultKnownAttachmentCounts.delete(resultId); + this._resultKnownErrorsCounts.delete(resultId); } onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { (step as any)[this._idSymbol] = createGuid(); + this._sendNewErrors(result, test.id); this._messageSink({ method: 'onStepBegin', params: { testId: test.id, resultId: (result as any)[this._idSymbol], step: this._serializeStepStart(step), - errors: this._unreportedErrors(result), } }); } - private _unreportedErrors(result: reporterTypes.TestResult) { - const index = (result as any)[this._reportedErrorsSymbol] ?? 0; - (result as any)[this._reportedErrorsSymbol] = result.errors.length; - return result.errors.slice(index); - } - onStepEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { // Create synthetic onAttach event so we serialize the entire attachment along with the step const resultId = (result as any)[this._idSymbol] as string; @@ -256,11 +253,26 @@ export class TeleReporterEmitter implements ReporterV2 { id: (result as any)[this._idSymbol], duration: result.duration, status: result.status, - errors: this._unreportedErrors(result), annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined, }; } + private _sendNewErrors(result: reporterTypes.TestResult, testId: string) { + const resultId = (result as any)[this._idSymbol] as string; + const knownErrorCount = this._resultKnownErrorsCounts.get(resultId) ?? 0; + if (result.errors.length > knownErrorCount) { + this._messageSink({ + method: 'onTestErrors', + params: { + testId, + resultId: (result as any)[this._idSymbol], + errors: result.errors.slice(knownErrorCount), + } + }); + } + this._resultKnownErrorsCounts.set(resultId, result.errors.length); + } + private _sendNewAttachments(result: reporterTypes.TestResult, testId: string) { const resultId = (result as any)[this._idSymbol] as string; // Track whether this step (or something else since the last step) has added attachments and send them From d0751475ad4ad1a28d157d1b0c2298f592aafdd5 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 13:54:17 +0100 Subject: [PATCH 12/28] typescript --- packages/playwright/src/isomorphic/teleReceiver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 6a352160ceae0..b89489bcf8808 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -370,7 +370,7 @@ export class TeleReporterReceiver { // Errors are only present here from legacy blobs. These override all _onTestErrors events if (!!payload.errors) { result.errors = payload.errors; - result.error = result.errors?.[0]; + result.error = result.errors[0]; } // Attachments are only present here from legacy blobs. These override all _onAttach events if (!!payload.attachments) From 7f90a0f2f4ad871d65d63177572dab178fe01f89 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 14:32:17 +0100 Subject: [PATCH 13/28] unflake --- tests/playwright-test/max-failures.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.ts index 1ce7493092548..88cbab4b3e82e 100644 --- a/tests/playwright-test/max-failures.spec.ts +++ b/tests/playwright-test/max-failures.spec.ts @@ -58,10 +58,10 @@ test('-x should work', async ({ runInlineTest }) => { }); } ` - }, { '-x': true }); + }, { '-x': true, 'workers': 1 }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.output.split('\n').filter(l => l.includes('expect(')).length).toBe(4); + expect(result.output.split('\n').filter(l => l.includes('expect(')).length).toBe(2); }); test('max-failures should work with retries', async ({ runInlineTest }) => { From 2242b4cc16cc343cec3c6c0ac1a23e5709aa6069 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 14:33:37 +0100 Subject: [PATCH 14/28] style --- packages/playwright/src/worker/workerMain.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index 260637606d91f..36c20b372eca5 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -118,7 +118,7 @@ export class WorkerMain extends ProcessRunner { return; } // Ignore top-level errors, they are already inside TestInfo.errors. - const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => { }, () => { }, () => { }, () => { }, () => { }); + const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {}, () => {}); const runnable = { type: 'teardown' } as const; // We have to load the project to get the right deadline below. await fakeTestInfo._runWithTimeout(runnable, () => this._loadIfNeeded()).catch(() => {}); From 6061a2993f34774b438821cdf77fc0aca87f168d Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 14:34:53 +0100 Subject: [PATCH 15/28] more feedback --- .../playwright/src/transform/babelHighlightUtils.ts | 8 ++++---- packages/playwright/src/worker/testInfo.ts | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts index f7018ac6c05e5..c81136b70e017 100644 --- a/packages/playwright/src/transform/babelHighlightUtils.ts +++ b/packages/playwright/src/transform/babelHighlightUtils.ts @@ -52,18 +52,18 @@ function containsPosition(location: T.SourceLocation, position: Location): boole return true; } -export function findTestEndPosition(text: string, location: Location): Location | undefined { - const ast = getAst(text, location.file); +export function findTestEndPosition(text: string, testStartLocation: Location): Location | undefined { + const ast = getAst(text, testStartLocation.file); if (!ast) return; let result: Location | undefined; traverse(ast, { enter(path) { - if (t.isCallExpression(path.node) && path.node.loc && containsPosition(path.node.loc, location)) { + if (t.isCallExpression(path.node) && path.node.loc && containsPosition(path.node.loc, testStartLocation)) { const callNode = path.node; const funcNode = callNode.arguments[callNode.arguments.length - 1]; if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc) - result = { file: location.file, ...funcNode.body.loc.end }; + result = { file: testStartLocation.file, ...funcNode.body.loc.end }; } } }); diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 989be36271499..4a0d512090756 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -123,7 +123,7 @@ export class TestInfoImpl implements TestInfo { readonly outputDir: string; readonly snapshotDir: string; errors: TestInfoErrorImpl[] = []; - _reportedError = 0; + private _reportedErrorCount = 0; readonly _attachmentsPush: (...items: TestInfo['attachments']) => number; private _workerParams: WorkerInitParams; @@ -480,20 +480,20 @@ export class TestInfoImpl implements TestInfo { } _emitErrors() { - const errors = this.errors.slice(this._reportedError); - this._reportedError = this.errors.length; + const errors = this.errors.slice(this._reportedErrorCount); + this._reportedErrorCount = this.errors.length; if (errors.length) this._onError({ testId: this.testId, errors }); } - _errorLocation(): Location | undefined { + private _errorLocation(): Location | undefined { if (this.error?.stack) return filteredStackTrace(this.error.stack.split('\n'))[0]; } async _testEndLocation() { const source = await fs.promises.readFile(this.file, 'utf-8'); - return findTestEndPosition(source, this); + return findTestEndPosition(source, { file: this.file, line: this.line, column: this.column }); } // ------------ TestInfo methods ------------ From bda266287e04b936da876ba5b36dd94bc805a895 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 14:44:40 +0100 Subject: [PATCH 16/28] feedback --- .../playwright/src/reporters/teleEmitter.ts | 3 ++ packages/playwright/src/runner/dispatcher.ts | 7 +++- .../playwright/src/transform/babelBundle.ts | 2 +- .../src/transform/babelHighlightUtils.ts | 30 ++-------------- packages/playwright/src/worker/testInfo.ts | 18 +++++----- packages/playwright/src/worker/workerMain.ts | 2 +- tests/playwright-test/max-failures.spec.ts | 2 +- tests/playwright-test/pause-at-end.spec.ts | 34 +++++++++++++++++++ 8 files changed, 58 insertions(+), 40 deletions(-) diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index d72f2c9aeaab7..73023d6ea5bc5 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -96,6 +96,9 @@ export class TeleReporterEmitter implements ReporterV2 { onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { (step as any)[this._idSymbol] = createGuid(); + // This is here to support "test paused" step, where we want to see all errors + // at the time of the pause. If we were to have `onTestError()` reporter api, this + // won't be needed. this._sendNewErrors(result, test.id); this._messageSink({ method: 'onStepBegin', diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 3b0898fc80c8c..b66e02ad7d444 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -427,6 +427,11 @@ class JobDispatcher { } private _onErrors(params: TestErrorPayload) { + if (this._failureTracker.hasReachedMaxFailures()) { + // Do not show more than one error to avoid confusion. + return; + } + const data = this._dataByTestId.get(params.testId)!; if (!data) return; @@ -586,7 +591,7 @@ class JobDispatcher { eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)), eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)), eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)), - eventsHelper.addEventListener(worker, 'errors', this._onErrors.bind(this)), + eventsHelper.addEventListener(worker, 'testErrors', this._onErrors.bind(this)), eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)), eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)), eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)), diff --git a/packages/playwright/src/transform/babelBundle.ts b/packages/playwright/src/transform/babelBundle.ts index 11ed0e4e22030..ce61eecc90a66 100644 --- a/packages/playwright/src/transform/babelBundle.ts +++ b/packages/playwright/src/transform/babelBundle.ts @@ -24,5 +24,5 @@ export type BabelTransformFunction = (code: string, filename: string, isModule: export const babelTransform: BabelTransformFunction = require('./babelBundleImpl').babelTransform; export type BabelParseFunction = (code: string, filename: string, isModule: boolean) => ParseResult; export const babelParse: BabelParseFunction = require('./babelBundleImpl').babelParse; -export type { NodePath, PluginObj, types as T, ParseResult } from '../../bundles/babel/node_modules/@types/babel__core'; +export type { NodePath, PluginObj, types as T } from '../../bundles/babel/node_modules/@types/babel__core'; export type { BabelAPI } from '../../bundles/babel/node_modules/@types/babel__helper-plugin-utils'; diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts index c81136b70e017..64ac4c1e7e8d2 100644 --- a/packages/playwright/src/transform/babelHighlightUtils.ts +++ b/packages/playwright/src/transform/babelHighlightUtils.ts @@ -15,33 +15,9 @@ */ import path from 'path'; -import { traverse, babelParse, ParseResult, T, types as t } from './babelBundle'; +import { traverse, babelParse, T, types as t } from './babelBundle'; import type { Location } from '../../types/testReporter'; -const astCache = new Map(); - -export function pruneAstCaches(fsPathsToRetain: string[]) { - const retain = new Set(fsPathsToRetain); - for (const key of astCache.keys()) { - if (!retain.has(key)) - astCache.delete(key); - } -} - -function getAst(text: string, fsPath: string) { - const cached = astCache.get(fsPath); - let ast = cached?.ast; - if (!cached || cached.text !== text) { - try { - ast = babelParse(text, path.basename(fsPath), false); - astCache.set(fsPath, { text, ast }); - } catch (e) { - astCache.set(fsPath, { text, ast: undefined }); - } - } - return ast; -} - function containsPosition(location: T.SourceLocation, position: Location): boolean { if (position.line < location.start.line || position.line > location.end.line) return false; @@ -53,9 +29,7 @@ function containsPosition(location: T.SourceLocation, position: Location): boole } export function findTestEndPosition(text: string, testStartLocation: Location): Location | undefined { - const ast = getAst(text, testStartLocation.file); - if (!ast) - return; + const ast = babelParse(text, path.basename(testStartLocation.file), false); let result: Location | undefined; traverse(ast, { enter(path) { diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 4a0d512090756..65cad97e7e400 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -70,7 +70,7 @@ export class TestInfoImpl implements TestInfo { private _onStepBegin: (payload: StepBeginPayload) => void; private _onStepEnd: (payload: StepEndPayload) => void; private _onAttach: (payload: AttachmentPayload) => void; - private _onError: (errors: TestErrorPayload) => void; + private _onErrors: (errors: TestErrorPayload) => void; private _onTestPaused: (payload: TestPausedPayload) => void; private _snapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; private _ariaSnapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; @@ -167,14 +167,14 @@ export class TestInfoImpl implements TestInfo { onStepBegin: (payload: StepBeginPayload) => void, onStepEnd: (payload: StepEndPayload) => void, onAttach: (payload: AttachmentPayload) => void, - onError: (payload: TestErrorPayload) => void, + onErrors: (payload: TestErrorPayload) => void, onTestPaused: (payload: TestPausedPayload) => void, ) { this.testId = test?.id ?? ''; this._onStepBegin = onStepBegin; this._onStepEnd = onStepEnd; this._onAttach = onAttach; - this._onError = onError; + this._onErrors = onErrors; this._onTestPaused = onTestPaused; this._startTime = monotonicTime(); this._startWallTime = Date.now(); @@ -481,9 +481,9 @@ export class TestInfoImpl implements TestInfo { _emitErrors() { const errors = this.errors.slice(this._reportedErrorCount); - this._reportedErrorCount = this.errors.length; + this._reportedErrorCount = Math.max(this._reportedErrorCount, this.errors.length); if (errors.length) - this._onError({ testId: this.testId, errors }); + this._onErrors({ testId: this.testId, errors }); } private _errorLocation(): Location | undefined { @@ -491,9 +491,11 @@ export class TestInfoImpl implements TestInfo { return filteredStackTrace(this.error.stack.split('\n'))[0]; } - async _testEndLocation() { - const source = await fs.promises.readFile(this.file, 'utf-8'); - return findTestEndPosition(source, { file: this.file, line: this.line, column: this.column }); + async _testEndLocation(): Promise { + try { + const source = await fs.promises.readFile(this.file, 'utf-8'); + return findTestEndPosition(source, { file: this.file, line: this.line, column: this.column }); + } catch {} } // ------------ TestInfo methods ------------ diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index 36c20b372eca5..b49d6482daf30 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -283,7 +283,7 @@ export class WorkerMain extends ProcessRunner { stepBeginPayload => this.dispatchEvent('stepBegin', stepBeginPayload), stepEndPayload => this.dispatchEvent('stepEnd', stepEndPayload), attachment => this.dispatchEvent('attach', attachment), - errors => this.dispatchEvent('errors', errors), + errors => this.dispatchEvent('testErrors', errors), testPausedPayload => this.dispatchEvent('testPaused', testPausedPayload)); const processAnnotation = (annotation: TestAnnotation) => { diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.ts index 88cbab4b3e82e..fed320b8267f4 100644 --- a/tests/playwright-test/max-failures.spec.ts +++ b/tests/playwright-test/max-failures.spec.ts @@ -58,7 +58,7 @@ test('-x should work', async ({ runInlineTest }) => { }); } ` - }, { '-x': true, 'workers': 1 }); + }, { '-x': true }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); expect(result.output.split('\n').filter(l => l.includes('expect(')).length).toBe(2); diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts index ca5eb0dda803d..c708b837d2556 100644 --- a/tests/playwright-test/pause-at-end.spec.ts +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -56,6 +56,40 @@ test('--debug should pause at end', async ({ interactWithTestRunner, }) => { expect(result.interrupted).toBe(1); }); +test('--debug should pause at end with setup project', async ({ interactWithTestRunner, }) => { + test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); + const testProcess = await interactWithTestRunner({ + 'location-reporter.js': `export default ${LocationReporter}`, + 'playwright.config.js': ` + module.exports = { + reporter: [['list'], ['./location-reporter.js']], + projects: [ + { name: 'setup', testMatch: /setup\\.test\\.js/ }, + { name: 'main', dependencies: ['setup'] } + ] + }; + `, + 'setup.test.js': ` + import { test } from '@playwright/test'; + test('setup', () => { + }); + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('pass', () => { + console.log('main test started'); + }); + ` + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('main test started'); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); + expect(testProcess.outputLines()).toEqual(['Paused at End at :5:7']); + + const result = parseTestRunnerOutput(testProcess.output); + expect(result.interrupted).toBe(1); +}); + test('--debug should pause on error', async ({ interactWithTestRunner, mergeReports }) => { test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); const testProcess = await interactWithTestRunner({ From 3999df2881b139cc9fc726dc97b8460a65f89300 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 14:54:09 +0100 Subject: [PATCH 17/28] add snippets in dispatcher --- .../playwright/src/reporters/internalReporter.ts | 13 ------------- packages/playwright/src/runner/dispatcher.ts | 6 +++++- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index d4367f4797b90..6d142543830bd 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -68,7 +68,6 @@ export class InternalReporter implements ReporterV2 { } onTestEnd(test: TestCase, result: TestResult) { - this._addSnippetToTestErrors(test, result); this._reporter.onTestEnd?.(test, result); } @@ -94,28 +93,16 @@ export class InternalReporter implements ReporterV2 { } onStepBegin(test: TestCase, result: TestResult, step: TestStep) { - this._addSnippetToTestErrors(test, result); this._reporter.onStepBegin?.(test, result, step); } onStepEnd(test: TestCase, result: TestResult, step: TestStep) { - this._addSnippetToStepError(test, step); this._reporter.onStepEnd?.(test, result, step); } printsToStdio() { return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true; } - - private _addSnippetToTestErrors(test: TestCase, result: TestResult) { - for (const error of result.errors) - addLocationAndSnippetToError(this._config, error, test.location.file); - } - - private _addSnippetToStepError(test: TestCase, step: TestStep) { - if (step.error) - addLocationAndSnippetToError(this._config, step.error, test.location.file); - } } export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file?: string) { diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index b66e02ad7d444..9827bc69d1db1 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -395,8 +395,10 @@ class JobDispatcher { return; } step.duration = params.wallTime - step.startTime.getTime(); - if (params.error) + if (params.error) { + addLocationAndSnippetToError(this._config.config, params.error); step.error = params.error; + } if (params.suggestedRebaseline) addSuggestedRebaseline(step.location!, params.suggestedRebaseline); step.annotations = params.annotations; @@ -452,6 +454,8 @@ class JobDispatcher { result = test._appendTestResult(); this._reporter.onTestBegin?.(test, result); } + for (const error of errors) + addLocationAndSnippetToError(this._config.config, error); result.errors.push(...errors); result.error = result.errors[0]; result.status = errors.length ? 'failed' : 'skipped'; From 807eafe8a883d3ef7cf867b36986487e19555a92 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 14:59:54 +0100 Subject: [PATCH 18/28] merge --- packages/playwright/src/reporters/merge.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index c0ed7fd6e90da..bfe50c4235eae 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -404,6 +404,7 @@ class IdsPatcher { case 'onProject': this._onProject(params.project); return; + case 'onTestErrors': case 'onAttach': case 'onTestBegin': case 'onStepBegin': @@ -498,7 +499,7 @@ class PathSeparatorPatcher { test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); const testResult = jsonEvent.params.result; testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); - testResult.errors.forEach(error => this._updateErrorLocations(error)); + testResult.errors?.forEach(error => this._updateErrorLocations(error)); (testResult.attachments ?? []).forEach(attachment => { if (attachment.path) attachment.path = this._updatePath(attachment.path); @@ -516,6 +517,10 @@ class PathSeparatorPatcher { step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); return; } + if (jsonEvent.method === 'onTestErrors') { + const errors = jsonEvent.params.errors; + errors.forEach(error => this._updateErrorLocations(error)); + } if (jsonEvent.method === 'onAttach') { const attach = jsonEvent.params; attach.attachments.forEach(attachment => { From c420d976d10784cd7020c516d31401706d5e9095 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 27 Nov 2025 16:05:03 +0100 Subject: [PATCH 19/28] pass file when possible --- .../src/reporters/internalReporter.ts | 6 +++--- packages/playwright/src/runner/dispatcher.ts | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 6d142543830bd..0d22093d13112 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -88,7 +88,7 @@ export class InternalReporter implements ReporterV2 { } onError(error: TestError) { - addLocationAndSnippetToError(this._config, error); + addLocationAndSnippetToError(this._config, error, undefined); this._reporter.onError?.(error); } @@ -105,7 +105,7 @@ export class InternalReporter implements ReporterV2 { } } -export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file?: string) { +export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file: string | undefined) { if (error.stack && !error.location) error.location = prepareErrorStack(error.stack).location; const location = error.location; @@ -117,7 +117,7 @@ export function addLocationAndSnippetToError(config: FullConfig, error: TestErro const source = fs.readFileSync(location.file, 'utf8'); const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode: true }); // Convert /var/folders to /private/var/folders on Mac. - if (!file || fs.realpathSync(file) !== location.file) { + if (!file || fs.realpathSync(file) !== location.file && false) { tokens.push(internalScreen.colors.gray(` at `) + `${relativeFilePath(internalScreen, config, location.file)}:${location.line}`); tokens.push(''); } diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 9827bc69d1db1..23fb491f41f4f 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -396,7 +396,7 @@ class JobDispatcher { } step.duration = params.wallTime - step.startTime.getTime(); if (params.error) { - addLocationAndSnippetToError(this._config.config, params.error); + addLocationAndSnippetToError(this._config.config, params.error, test.location.file); step.error = params.error; } if (params.suggestedRebaseline) @@ -439,7 +439,7 @@ class JobDispatcher { return; const { result } = data; for (const error of params.errors) - addLocationAndSnippetToError(this._config.config, error); + addLocationAndSnippetToError(this._config.config, error, data.test.location.file); result.errors.push(...params.errors); result.error = result.errors[0]; } @@ -455,7 +455,7 @@ class JobDispatcher { this._reporter.onTestBegin?.(test, result); } for (const error of errors) - addLocationAndSnippetToError(this._config.config, error); + addLocationAndSnippetToError(this._config.config, error, test.location.file); result.errors.push(...errors); result.error = result.errors[0]; result.status = errors.length ? 'failed' : 'skipped'; @@ -603,25 +603,25 @@ class JobDispatcher { } private _onTestPaused(worker: WorkerHost, params: TestPausedPayload) { + const data = this._dataByTestId.get(params.testId); + if (!data) + return; + const sendMessage = async (message: { request: any }) => { try { if (this.jobResult.isDone()) throw new Error('Test has already stopped'); const response = await worker.sendCustomMessage({ testId: params.testId, request: message.request }); if (response.error) - addLocationAndSnippetToError(this._config.config, response.error); + addLocationAndSnippetToError(this._config.config, response.error, data.test.location.file); return response; } catch (e) { const error = serializeError(e); - addLocationAndSnippetToError(this._config.config, error); + addLocationAndSnippetToError(this._config.config, error, data.test.location.file); return { response: undefined, error }; } }; - const data = this._dataByTestId.get(params.testId); - if (!data) - return; - this._failureTracker.onTestPaused?.({ errors: data.result.errors, sendMessage }); } From c8bd2451119e19031ba06556b86d8ed0b0b58b01 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 1 Dec 2025 12:56:02 +0100 Subject: [PATCH 20/28] add comment --- tests/playwright-test/pause-at-end.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts index c708b837d2556..6021f70225f8c 100644 --- a/tests/playwright-test/pause-at-end.spec.ts +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -43,7 +43,7 @@ test('--debug should pause at end', async ({ interactWithTestRunner, }) => { test('pass', () => { }); test.afterEach(() => { - console.log('teardown'.toUpperCase()); + console.log('teardown'.toUpperCase()); // uppercase so we dont confuse it with source snippets }); ` }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); From dea4b7d51bc9629b573d02dcf0d8617ea432799d Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 1 Dec 2025 13:20:45 +0100 Subject: [PATCH 21/28] address --- packages/playwright/src/common/ipc.ts | 2 +- .../src/reporters/internalReporter.ts | 8 +++++- .../playwright/src/reporters/reporterV2.ts | 1 + packages/playwright/src/runner/dispatcher.ts | 26 +++++++++---------- .../src/transform/babelHighlightUtils.ts | 10 +++---- packages/playwright/src/worker/testInfo.ts | 8 +++--- 6 files changed, 31 insertions(+), 24 deletions(-) diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index bd6ebaea89974..89f17bad4edb1 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -84,7 +84,7 @@ export type AttachmentPayload = { stepId?: string; }; -export type TestErrorPayload = { +export type TestErrorsPayload = { testId: string; errors: TestInfoErrorImpl[]; }; diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 0d22093d13112..2c96c8866eb11 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -67,6 +67,10 @@ export class InternalReporter implements ReporterV2 { this._reporter.onStdErr?.(chunk, test, result); } + onTestError(test: TestCase, result: TestResult, error: TestError): void { + addLocationAndSnippetToError(this._config, error, test.location.file); + } + onTestEnd(test: TestCase, result: TestResult) { this._reporter.onTestEnd?.(test, result); } @@ -97,6 +101,8 @@ export class InternalReporter implements ReporterV2 { } onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.error) + addLocationAndSnippetToError(this._config, step.error, test.location.file); this._reporter.onStepEnd?.(test, result, step); } @@ -117,7 +123,7 @@ export function addLocationAndSnippetToError(config: FullConfig, error: TestErro const source = fs.readFileSync(location.file, 'utf8'); const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode: true }); // Convert /var/folders to /private/var/folders on Mac. - if (!file || fs.realpathSync(file) !== location.file && false) { + if (!file || fs.realpathSync(file) !== location.file) { tokens.push(internalScreen.colors.gray(` at `) + `${relativeFilePath(internalScreen, config, location.file)}:${location.line}`); tokens.push(''); } diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts index 280ef2bfd2f19..5085a63a85bb9 100644 --- a/packages/playwright/src/reporters/reporterV2.ts +++ b/packages/playwright/src/reporters/reporterV2.ts @@ -22,6 +22,7 @@ export interface ReporterV2 { onTestBegin?(test: TestCase, result: TestResult): void; onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; + onTestError?(test: TestCase, result: TestResult, error: TestError): void; onTestEnd?(test: TestCase, result: TestResult): void; onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; onExit?(): void | Promise; diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 23fb491f41f4f..7ab719283670c 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -28,7 +28,7 @@ import type { ProcessExitData } from './processHost'; import type { TestGroup } from './testGroups'; import type { TestError, TestResult, TestStep } from '../../types/testReporter'; import type { FullConfigInternal } from '../common/config'; -import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc'; +import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorsPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc'; import type { Suite } from '../common/test'; import type { TestCase } from '../common/test'; import type { ReporterV2 } from '../reporters/reporterV2'; @@ -395,10 +395,8 @@ class JobDispatcher { return; } step.duration = params.wallTime - step.startTime.getTime(); - if (params.error) { - addLocationAndSnippetToError(this._config.config, params.error, test.location.file); + if (params.error) step.error = params.error; - } if (params.suggestedRebaseline) addSuggestedRebaseline(step.location!, params.suggestedRebaseline); step.annotations = params.annotations; @@ -428,7 +426,7 @@ class JobDispatcher { } } - private _onErrors(params: TestErrorPayload) { + private _onErrors(params: TestErrorsPayload) { if (this._failureTracker.hasReachedMaxFailures()) { // Do not show more than one error to avoid confusion. return; @@ -437,11 +435,11 @@ class JobDispatcher { const data = this._dataByTestId.get(params.testId)!; if (!data) return; - const { result } = data; - for (const error of params.errors) - addLocationAndSnippetToError(this._config.config, error, data.test.location.file); + const { test, result } = data; result.errors.push(...params.errors); result.error = result.errors[0]; + for (const error of result.errors) + this._reporter.onTestError?.(test, result, error); } private _failTestWithErrors(test: TestCase, errors: TestError[]) { @@ -454,10 +452,10 @@ class JobDispatcher { result = test._appendTestResult(); this._reporter.onTestBegin?.(test, result); } - for (const error of errors) - addLocationAndSnippetToError(this._config.config, error, test.location.file); result.errors.push(...errors); result.error = result.errors[0]; + for (const error of result.errors) + this._reporter.onTestError?.(test, result, error); result.status = errors.length ? 'failed' : 'skipped'; this._reportTestEnd(test, result); this._failedTests.add(test); @@ -607,22 +605,24 @@ class JobDispatcher { if (!data) return; + const { test, result } = data; + const sendMessage = async (message: { request: any }) => { try { if (this.jobResult.isDone()) throw new Error('Test has already stopped'); const response = await worker.sendCustomMessage({ testId: params.testId, request: message.request }); if (response.error) - addLocationAndSnippetToError(this._config.config, response.error, data.test.location.file); + addLocationAndSnippetToError(this._config.config, response.error, test.location.file); return response; } catch (e) { const error = serializeError(e); - addLocationAndSnippetToError(this._config.config, error, data.test.location.file); + addLocationAndSnippetToError(this._config.config, error, test.location.file); return { response: undefined, error }; } }; - this._failureTracker.onTestPaused?.({ errors: data.result.errors, sendMessage }); + this._failureTracker.onTestPaused?.({ errors: result.errors, sendMessage }); } skipWholeJob(): boolean { diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts index 64ac4c1e7e8d2..2d127f082871e 100644 --- a/packages/playwright/src/transform/babelHighlightUtils.ts +++ b/packages/playwright/src/transform/babelHighlightUtils.ts @@ -18,12 +18,12 @@ import path from 'path'; import { traverse, babelParse, T, types as t } from './babelBundle'; import type { Location } from '../../types/testReporter'; -function containsPosition(location: T.SourceLocation, position: Location): boolean { - if (position.line < location.start.line || position.line > location.end.line) +function containsLocation(range: T.SourceLocation, location: Location): boolean { + if (location.line < range.start.line || location.line > range.end.line) return false; - if (position.line === location.start.line && position.column < location.start.column) + if (location.line === range.start.line && location.column < range.start.column) return false; - if (position.line === location.end.line && position.column > location.end.column) + if (location.line === range.end.line && location.column > range.end.column) return false; return true; } @@ -33,7 +33,7 @@ export function findTestEndPosition(text: string, testStartLocation: Location): let result: Location | undefined; traverse(ast, { enter(path) { - if (t.isCallExpression(path.node) && path.node.loc && containsPosition(path.node.loc, testStartLocation)) { + if (t.isCallExpression(path.node) && path.node.loc && containsLocation(path.node.loc, testStartLocation)) { const callNode = path.node; const funcNode = callNode.arguments[callNode.arguments.length - 1]; if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc) diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 65cad97e7e400..5bb8d0721eab1 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -30,7 +30,7 @@ import type { RunnableDescription } from './timeoutManager'; import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test'; import type { FullConfig, Location } from '../../types/testReporter'; import type { FullConfigInternal, FullProjectInternal } from '../common/config'; -import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestErrorPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc'; +import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestErrorsPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc'; import type { TestCase } from '../common/test'; import type { StackFrame } from '@protocol/channels'; @@ -70,7 +70,7 @@ export class TestInfoImpl implements TestInfo { private _onStepBegin: (payload: StepBeginPayload) => void; private _onStepEnd: (payload: StepEndPayload) => void; private _onAttach: (payload: AttachmentPayload) => void; - private _onErrors: (errors: TestErrorPayload) => void; + private _onErrors: (errors: TestErrorsPayload) => void; private _onTestPaused: (payload: TestPausedPayload) => void; private _snapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; private _ariaSnapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; @@ -167,7 +167,7 @@ export class TestInfoImpl implements TestInfo { onStepBegin: (payload: StepBeginPayload) => void, onStepEnd: (payload: StepEndPayload) => void, onAttach: (payload: AttachmentPayload) => void, - onErrors: (payload: TestErrorPayload) => void, + onErrors: (payload: TestErrorsPayload) => void, onTestPaused: (payload: TestPausedPayload) => void, ) { this.testId = test?.id ?? ''; @@ -491,7 +491,7 @@ export class TestInfoImpl implements TestInfo { return filteredStackTrace(this.error.stack.split('\n'))[0]; } - async _testEndLocation(): Promise { + private async _testEndLocation(): Promise { try { const source = await fs.promises.readFile(this.file, 'utf-8'); return findTestEndPosition(source, { file: this.file, line: this.line, column: this.column }); From 22d074463ef003b3b5d52d8b83cbc56749b239c3 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 1 Dec 2025 13:22:38 +0100 Subject: [PATCH 22/28] rename --- packages/playwright/src/runner/dispatcher.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 7ab719283670c..e47f26b4203aa 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -426,7 +426,7 @@ class JobDispatcher { } } - private _onErrors(params: TestErrorsPayload) { + private _onTestErrors(params: TestErrorsPayload) { if (this._failureTracker.hasReachedMaxFailures()) { // Do not show more than one error to avoid confusion. return; @@ -593,7 +593,7 @@ class JobDispatcher { eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)), eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)), eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)), - eventsHelper.addEventListener(worker, 'testErrors', this._onErrors.bind(this)), + eventsHelper.addEventListener(worker, 'testErrors', this._onTestErrors.bind(this)), eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)), eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)), eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)), From dab17dd759210e3fa5bb6fed8f79afe80c5b0550 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 2 Dec 2025 08:20:48 +0100 Subject: [PATCH 23/28] rename --- packages/playwright/src/transform/babelHighlightUtils.ts | 2 +- packages/playwright/src/worker/testInfo.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts index 2d127f082871e..4611233686377 100644 --- a/packages/playwright/src/transform/babelHighlightUtils.ts +++ b/packages/playwright/src/transform/babelHighlightUtils.ts @@ -28,7 +28,7 @@ function containsLocation(range: T.SourceLocation, location: Location): boolean return true; } -export function findTestEndPosition(text: string, testStartLocation: Location): Location | undefined { +export function findTestEndLocation(text: string, testStartLocation: Location): Location | undefined { const ast = babelParse(text, path.basename(testStartLocation.file), false); let result: Location | undefined; traverse(ast, { diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 5bb8d0721eab1..5838b28a57a29 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -24,7 +24,7 @@ import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAnd import { TestTracing } from './testTracing'; import { testInfoError } from './util'; import { wrapFunctionWithLocation } from '../transform/transform'; -import { findTestEndPosition } from '../transform/babelHighlightUtils'; +import { findTestEndLocation } from '../transform/babelHighlightUtils'; import type { RunnableDescription } from './timeoutManager'; import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test'; @@ -494,7 +494,7 @@ export class TestInfoImpl implements TestInfo { private async _testEndLocation(): Promise { try { const source = await fs.promises.readFile(this.file, 'utf-8'); - return findTestEndPosition(source, { file: this.file, line: this.line, column: this.column }); + return findTestEndLocation(source, { file: this.file, line: this.line, column: this.column }); } catch {} } From 7bf8763dfe4faea3bab79e184189708b67f583b0 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 2 Dec 2025 08:28:38 +0100 Subject: [PATCH 24/28] maintain last error index --- packages/playwright/src/runner/dispatcher.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index e47f26b4203aa..1e8958569f96c 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -436,10 +436,11 @@ class JobDispatcher { if (!data) return; const { test, result } = data; - result.errors.push(...params.errors); - result.error = result.errors[0]; - for (const error of result.errors) + for (const error of params.errors) { + result.errors.push(error); + result.error = result.errors[0]; this._reporter.onTestError?.(test, result, error); + } } private _failTestWithErrors(test: TestCase, errors: TestError[]) { @@ -452,10 +453,11 @@ class JobDispatcher { result = test._appendTestResult(); this._reporter.onTestBegin?.(test, result); } - result.errors.push(...errors); - result.error = result.errors[0]; - for (const error of result.errors) + for (const error of errors) { + result.errors.push(error); + result.error = result.errors[0]; this._reporter.onTestError?.(test, result, error); + } result.status = errors.length ? 'failed' : 'skipped'; this._reportTestEnd(test, result); this._failedTests.add(test); From 5817b37490980f5d817c02cf94f19c8ef1205de8 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 2 Dec 2025 08:31:48 +0100 Subject: [PATCH 25/28] use onTestError --- .../playwright/src/isomorphic/teleReceiver.ts | 16 +++++----- packages/playwright/src/reporters/merge.ts | 8 ++--- .../playwright/src/reporters/teleEmitter.ts | 32 +++++++------------ 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index b89489bcf8808..5f87dd9843fe8 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -133,7 +133,7 @@ export type JsonFullResult = { }; export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent - | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorsEvent | JsonOnErrorEvent | JsonOnStdIOEvent; + | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorEvent | JsonOnErrorEvent | JsonOnStdIOEvent; export type JsonOnConfigureEvent = { method: 'onConfigure'; @@ -199,12 +199,12 @@ export type JsonOnAttachEvent = { params: JsonTestResultOnAttach; }; -export type JsonOnTestErrorsEvent = { - method: 'onTestErrors'; +export type JsonOnTestErrorEvent = { + method: 'onTestError'; params: { testId: string; resultId: string; - errors: reporterTypes.TestError[]; + error: reporterTypes.TestError; } }; @@ -304,8 +304,8 @@ export class TeleReporterReceiver { this._onAttach(params.testId, params.resultId, params.attachments); return; } - if (method === 'onTestErrors') { - this._onTestErrors(params.testId, params.resultId, params.errors); + if (method === 'onTestError') { + this._onTestError(params.testId, params.resultId, params.error); return; } if (method === 'onStepEnd') { @@ -421,10 +421,10 @@ export class TeleReporterReceiver { }))); } - private _onTestErrors(testId: string, resultId: string, errors: reporterTypes.TestError[]) { + private _onTestError(testId: string, resultId: string, error: reporterTypes.TestError) { const test = this._tests.get(testId)!; const result = test.results.find(r => r._id === resultId)!; - result.errors.push(...errors); + result.errors.push(error); result.error = result.errors[0]; } diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index bfe50c4235eae..3571f6297290c 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -404,7 +404,7 @@ class IdsPatcher { case 'onProject': this._onProject(params.project); return; - case 'onTestErrors': + case 'onTestError': case 'onAttach': case 'onTestBegin': case 'onStepBegin': @@ -517,9 +517,9 @@ class PathSeparatorPatcher { step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); return; } - if (jsonEvent.method === 'onTestErrors') { - const errors = jsonEvent.params.errors; - errors.forEach(error => this._updateErrorLocations(error)); + if (jsonEvent.method === 'onTestError') { + this._updateErrorLocations(jsonEvent.params.error); + return; } if (jsonEvent.method === 'onAttach') { const attach = jsonEvent.params; diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 73023d6ea5bc5..9dc635c4ac3eb 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -72,6 +72,17 @@ export class TeleReporterEmitter implements ReporterV2 { }); } + onTestError(test: reporterTypes.TestCase, result: reporterTypes.TestResult, error: reporterTypes.TestError): void { + this._messageSink({ + method: 'onTestError', + params: { + testId: test.id, + resultId: (result as any)[this._idSymbol], + error, + } + }); + } + onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void { const testEnd: teleReceiver.JsonTestEnd = { testId: test.id, @@ -79,7 +90,6 @@ export class TeleReporterEmitter implements ReporterV2 { timeout: test.timeout, annotations: [] }; - this._sendNewErrors(result, test.id); this._sendNewAttachments(result, test.id); this._messageSink({ method: 'onTestEnd', @@ -96,10 +106,6 @@ export class TeleReporterEmitter implements ReporterV2 { onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { (step as any)[this._idSymbol] = createGuid(); - // This is here to support "test paused" step, where we want to see all errors - // at the time of the pause. If we were to have `onTestError()` reporter api, this - // won't be needed. - this._sendNewErrors(result, test.id); this._messageSink({ method: 'onStepBegin', params: { @@ -260,22 +266,6 @@ export class TeleReporterEmitter implements ReporterV2 { }; } - private _sendNewErrors(result: reporterTypes.TestResult, testId: string) { - const resultId = (result as any)[this._idSymbol] as string; - const knownErrorCount = this._resultKnownErrorsCounts.get(resultId) ?? 0; - if (result.errors.length > knownErrorCount) { - this._messageSink({ - method: 'onTestErrors', - params: { - testId, - resultId: (result as any)[this._idSymbol], - errors: result.errors.slice(knownErrorCount), - } - }); - } - this._resultKnownErrorsCounts.set(resultId, result.errors.length); - } - private _sendNewAttachments(result: reporterTypes.TestResult, testId: string) { const resultId = (result as any)[this._idSymbol] as string; // Track whether this step (or something else since the last step) has added attachments and send them From f768a69ed073c16fad10cc600491f74b23035322 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 2 Dec 2025 08:32:39 +0100 Subject: [PATCH 26/28] remove unused map --- packages/playwright/src/reporters/teleEmitter.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 9dc635c4ac3eb..ae0b84a294e65 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -34,7 +34,6 @@ export class TeleReporterEmitter implements ReporterV2 { private _messageSink: (message: teleReceiver.JsonEvent) => void; private _rootDir!: string; private _emitterOptions: TeleReporterEmitterOptions; - private _resultKnownErrorsCounts = new Map(); private _resultKnownAttachmentCounts = new Map(); // In case there is blob reporter and UI mode, make sure one does override // the id assigned by the other. @@ -101,7 +100,6 @@ export class TeleReporterEmitter implements ReporterV2 { const resultId = (result as any)[this._idSymbol] as string; this._resultKnownAttachmentCounts.delete(resultId); - this._resultKnownErrorsCounts.delete(resultId); } onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { From 7020ccd12197c9e4aeb4d0014127534f428a3646 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 2 Dec 2025 08:34:14 +0100 Subject: [PATCH 27/28] revert linechange --- packages/playwright/src/reporters/teleEmitter.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index ae0b84a294e65..8a792d1849251 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -98,8 +98,7 @@ export class TeleReporterEmitter implements ReporterV2 { } }); - const resultId = (result as any)[this._idSymbol] as string; - this._resultKnownAttachmentCounts.delete(resultId); + this._resultKnownAttachmentCounts.delete((result as any)[this._idSymbol]); } onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { @@ -109,7 +108,7 @@ export class TeleReporterEmitter implements ReporterV2 { params: { testId: test.id, resultId: (result as any)[this._idSymbol], - step: this._serializeStepStart(step), + step: this._serializeStepStart(step) } }); } From fa8fa149a05bcc90a4192823100dabe20d48f814 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 2 Dec 2025 13:53:45 +0100 Subject: [PATCH 28/28] the multiplexer! --- packages/playwright/src/isomorphic/teleReceiver.ts | 2 +- packages/playwright/src/reporters/internalReporter.ts | 1 + packages/playwright/src/reporters/multiplexer.ts | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 5f87dd9843fe8..d0fced2ece384 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -367,7 +367,7 @@ export class TeleReporterReceiver { const result = test.results.find(r => r._id === payload.id)!; result.duration = payload.duration; result.status = payload.status; - // Errors are only present here from legacy blobs. These override all _onTestErrors events + // Errors are only present here from legacy blobs. These override all _onTestError events if (!!payload.errors) { result.errors = payload.errors; result.error = result.errors[0]; diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 2c96c8866eb11..c48bfdf70b609 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -69,6 +69,7 @@ export class InternalReporter implements ReporterV2 { onTestError(test: TestCase, result: TestResult, error: TestError): void { addLocationAndSnippetToError(this._config, error, test.location.file); + this._reporter.onTestError?.(test, result, error); } onTestEnd(test: TestCase, result: TestResult) { diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts index 2a11b9358319a..0c8bb5ff14f74 100644 --- a/packages/playwright/src/reporters/multiplexer.ts +++ b/packages/playwright/src/reporters/multiplexer.ts @@ -54,6 +54,11 @@ export class Multiplexer implements ReporterV2 { wrap(() => reporter.onStdErr?.(chunk, test, result)); } + onTestError(test: TestCase, result: TestResult, error: TestError) { + for (const reporter of this._reporters) + wrap(() => reporter.onTestError?.(test, result, error)); + } + onTestEnd(test: TestCase, result: TestResult) { for (const reporter of this._reporters) wrap(() => reporter.onTestEnd?.(test, result));