feat: pause on --debug

packages/playwright/src/common/ipc.ts

@@ -84,11 +84,15 @@ export type AttachmentPayload = {
   stepId?: string;
 };
 
+export type TestErrorsPayload = {
+  testId: string;
+  errors: TestInfoErrorImpl[];
+};
+
 export type TestInfoErrorImpl = TestInfoError;
 
 export type TestPausedPayload = {
   testId: string;
-  errors: TestInfoErrorImpl[];
 };
 
 export type CustomMessageRequestPayload = {
@@ -105,7 +109,6 @@ export type TestEndPayload = {
   testId: string;
   duration: number;
   status: TestStatus;
-  errors: TestInfoErrorImpl[];
   hasNonRetriableError: boolean;
   expectedStatus: TestStatus;
   annotations: { type: string, description?: string }[];

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -96,7 +96,8 @@ export type JsonTestResultEnd = {
   id: string;
   duration: number;
   status: reporterTypes.TestStatus;
-  errors: reporterTypes.TestError[];
+  /** No longer emitted, but kept for backwards compatibility */
+  errors?: reporterTypes.TestError[];
   /** No longer emitted, but kept for backwards compatibility */
   attachments?: JsonAttachment[];
   annotations?: TestAnnotation[];
@@ -132,7 +133,7 @@ export type JsonFullResult = {
 };
 
 export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent
-  | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
+  | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorsEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
 
 export type JsonOnConfigureEvent = {
   method: 'onConfigure';
@@ -198,6 +199,15 @@ export type JsonOnAttachEvent = {
   params: JsonTestResultOnAttach;
 };
 
+export type JsonOnTestErrorsEvent = {
+  method: 'onTestErrors';
+  params: {
+    testId: string;
+    resultId: string;
+    errors: reporterTypes.TestError[];
+  }
+};
+
 export type JsonOnErrorEvent = {
   method: 'onError';
   params: {
@@ -294,6 +304,10 @@ export class TeleReporterReceiver {
       this._onAttach(params.testId, params.resultId, params.attachments);
       return;
     }
+    if (method === 'onTestErrors') {
+      this._onTestErrors(params.testId, params.resultId, params.errors);
+      return;
+    }
     if (method === 'onStepEnd') {
       this._onStepEnd(params.testId, params.resultId, params.step);
       return;
@@ -353,8 +367,11 @@ export class TeleReporterReceiver {
     const result = test.results.find(r => r._id === payload.id)!;
     result.duration = payload.duration;
     result.status = payload.status;
-    result.errors = payload.errors;
-    result.error = result.errors?.[0];
+    // Errors are only present here from legacy blobs. These override all _onTestErrors events
+    if (!!payload.errors) {
+      result.errors = payload.errors;
+      result.error = result.errors[0];
+    }
     // Attachments are only present here from legacy blobs. These override all _onAttach events
     if (!!payload.attachments)
       result.attachments = this._parseAttachments(payload.attachments);
@@ -404,6 +421,13 @@ export class TeleReporterReceiver {
     })));
   }
 
+  private _onTestErrors(testId: string, resultId: string, errors: reporterTypes.TestError[]) {
+    const test = this._tests.get(testId)!;
+    const result = test.results.find(r => r._id === resultId)!;
+    result.errors.push(...errors);
+    result.error = result.errors[0];
+  }
+
   private _onError(error: reporterTypes.TestError) {
     this._reporter.onError?.(error);
   }

packages/playwright/src/reporters/internalReporter.ts

@@ -67,8 +67,11 @@ export class InternalReporter implements ReporterV2 {
     this._reporter.onStdErr?.(chunk, test, result);
   }
 
+  onTestError(test: TestCase, result: TestResult, error: TestError): void {
+    addLocationAndSnippetToError(this._config, error, test.location.file);
+  }
+
   onTestEnd(test: TestCase, result: TestResult) {
-    this._addSnippetToTestErrors(test, result);
     this._reporter.onTestEnd?.(test, result);
   }
 
@@ -89,7 +92,7 @@ export class InternalReporter implements ReporterV2 {
   }
 
   onError(error: TestError) {
-    addLocationAndSnippetToError(this._config, error);
+    addLocationAndSnippetToError(this._config, error, undefined);
     this._reporter.onError?.(error);
   }
 
@@ -98,26 +101,17 @@ export class InternalReporter implements ReporterV2 {
   }
 
   onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
-    this._addSnippetToStepError(test, step);
+    if (step.error)
+      addLocationAndSnippetToError(this._config, step.error, test.location.file);
     this._reporter.onStepEnd?.(test, result, step);
   }
 
   printsToStdio() {
     return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true;
   }
-
-  private _addSnippetToTestErrors(test: TestCase, result: TestResult) {
-    for (const error of result.errors)
-      addLocationAndSnippetToError(this._config, error, test.location.file);
-  }
-
-  private _addSnippetToStepError(test: TestCase, step: TestStep) {
-    if (step.error)
-      addLocationAndSnippetToError(this._config, step.error, test.location.file);
-  }
 }
 
-export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file?: string) {
+export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file: string | undefined) {
   if (error.stack && !error.location)
     error.location = prepareErrorStack(error.stack).location;
   const location = error.location;

packages/playwright/src/reporters/merge.ts

@@ -404,6 +404,7 @@ class IdsPatcher {
       case 'onProject':
         this._onProject(params.project);
         return;
+      case 'onTestErrors':
       case 'onAttach':
       case 'onTestBegin':
       case 'onStepBegin':
@@ -498,7 +499,7 @@ class PathSeparatorPatcher {
       test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
       const testResult = jsonEvent.params.result;
       testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
-      testResult.errors.forEach(error => this._updateErrorLocations(error));
+      testResult.errors?.forEach(error => this._updateErrorLocations(error));
       (testResult.attachments ?? []).forEach(attachment => {
         if (attachment.path)
           attachment.path = this._updatePath(attachment.path);
@@ -516,6 +517,10 @@ class PathSeparatorPatcher {
       step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
       return;
     }
+    if (jsonEvent.method === 'onTestErrors') {
+      const errors = jsonEvent.params.errors;
+      errors.forEach(error => this._updateErrorLocations(error));
+    }
     if (jsonEvent.method === 'onAttach') {
       const attach = jsonEvent.params;
       attach.attachments.forEach(attachment => {

packages/playwright/src/reporters/reporterV2.ts

@@ -22,6 +22,7 @@ export interface ReporterV2 {
   onTestBegin?(test: TestCase, result: TestResult): void;
   onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
+  onTestError?(test: TestCase, result: TestResult, error: TestError): void;
   onTestEnd?(test: TestCase, result: TestResult): void;
   onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
   onExit?(): void | Promise<void>;

packages/playwright/src/reporters/teleEmitter.ts

@@ -34,6 +34,7 @@ export class TeleReporterEmitter implements ReporterV2 {
   private _messageSink: (message: teleReceiver.JsonEvent) => void;
   private _rootDir!: string;
   private _emitterOptions: TeleReporterEmitterOptions;
+  private _resultKnownErrorsCounts = new Map<string, number>();
   private _resultKnownAttachmentCounts = new Map<string, number>();
   // In case there is blob reporter and UI mode, make sure one does override
   // the id assigned by the other.
@@ -78,6 +79,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       timeout: test.timeout,
       annotations: []
     };
+    this._sendNewErrors(result, test.id);
     this._sendNewAttachments(result, test.id);
     this._messageSink({
       method: 'onTestEnd',
@@ -87,17 +89,23 @@ export class TeleReporterEmitter implements ReporterV2 {
       }
     });
 
-    this._resultKnownAttachmentCounts.delete((result as any)[this._idSymbol]);
+    const resultId = (result as any)[this._idSymbol] as string;
+    this._resultKnownAttachmentCounts.delete(resultId);
+    this._resultKnownErrorsCounts.delete(resultId);
   }
 
   onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void {
     (step as any)[this._idSymbol] = createGuid();
+    // This is here to support "test paused" step, where we want to see all errors
+    // at the time of the pause. If we were to have `onTestError()` reporter api, this
+    // won't be needed.
+    this._sendNewErrors(result, test.id);
     this._messageSink({
       method: 'onStepBegin',
       params: {
         testId: test.id,
         resultId: (result as any)[this._idSymbol],
-        step: this._serializeStepStart(step)
+        step: this._serializeStepStart(step),
       }
     });
   }
@@ -248,11 +256,26 @@ export class TeleReporterEmitter implements ReporterV2 {
       id: (result as any)[this._idSymbol],
       duration: result.duration,
       status: result.status,
-      errors: result.errors,
       annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined,
     };
   }
 
+  private _sendNewErrors(result: reporterTypes.TestResult, testId: string) {
+    const resultId = (result as any)[this._idSymbol] as string;
+    const knownErrorCount = this._resultKnownErrorsCounts.get(resultId) ?? 0;
+    if (result.errors.length > knownErrorCount) {
+      this._messageSink({
+        method: 'onTestErrors',
+        params: {
+          testId,
+          resultId: (result as any)[this._idSymbol],
+          errors: result.errors.slice(knownErrorCount),
+        }
+      });
+    }
+    this._resultKnownErrorsCounts.set(resultId, result.errors.length);
+  }
+
   private _sendNewAttachments(result: reporterTypes.TestResult, testId: string) {
     const resultId = (result as any)[this._idSymbol] as string;
     // Track whether this step (or something else since the last step) has added attachments and send them

packages/playwright/src/runner/dispatcher.ts

@@ -28,7 +28,7 @@ import type { ProcessExitData } from './processHost';
 import type { TestGroup } from './testGroups';
 import type { TestError, TestResult, TestStep } from '../../types/testReporter';
 import type { FullConfigInternal } from '../common/config';
-import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
+import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorsPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
 import type { Suite } from '../common/test';
 import type { TestCase } from '../common/test';
 import type { ReporterV2 } from '../reporters/reporterV2';
@@ -322,7 +322,6 @@ class JobDispatcher {
       // Do not show more than one error to avoid confusion, but report
       // as interrupted to indicate that we did actually start the test.
       params.status = 'interrupted';
-      params.errors = [];
     }
     const data = this._dataByTestId.get(params.testId);
     if (!data) {
@@ -333,8 +332,6 @@ class JobDispatcher {
     this._remainingByTestId.delete(params.testId);
     const { result, test } = data;
     result.duration = params.duration;
-    result.errors = params.errors;
-    result.error = result.errors[0];
     result.status = params.status;
     result.annotations = params.annotations;
     test.annotations = [...params.annotations]; // last test result wins
@@ -429,6 +426,22 @@ class JobDispatcher {
     }
   }
 
+  private _onTestErrors(params: TestErrorsPayload) {
+    if (this._failureTracker.hasReachedMaxFailures()) {
+      // Do not show more than one error to avoid confusion.
+      return;
+    }
+
+    const data = this._dataByTestId.get(params.testId)!;
+    if (!data)
+      return;
+    const { test, result } = data;
+    result.errors.push(...params.errors);
+    result.error = result.errors[0];
+    for (const error of result.errors)
+      this._reporter.onTestError?.(test, result, error);
+  }
+
   private _failTestWithErrors(test: TestCase, errors: TestError[]) {
     const runData = this._dataByTestId.get(test.id);
     // There might be a single test that has started but has not finished yet.
@@ -439,8 +452,10 @@ class JobDispatcher {
       result = test._appendTestResult();
       this._reporter.onTestBegin?.(test, result);
     }
-    result.errors = [...errors];
+    result.errors.push(...errors);
     result.error = result.errors[0];
+    for (const error of result.errors)
+      this._reporter.onTestError?.(test, result, error);
     result.status = errors.length ? 'failed' : 'skipped';
     this._reportTestEnd(test, result);
     this._failedTests.add(test);
@@ -578,31 +593,36 @@ class JobDispatcher {
       eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),
       eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),
       eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)),
+      eventsHelper.addEventListener(worker, 'testErrors', this._onTestErrors.bind(this)),
       eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)),
       eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)),
       eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)),
     ];
   }
 
   private _onTestPaused(worker: WorkerHost, params: TestPausedPayload) {
+    const data = this._dataByTestId.get(params.testId);
+    if (!data)
+      return;
+
+    const { test, result } = data;
+
     const sendMessage = async (message: { request: any }) => {
       try {
         if (this.jobResult.isDone())
           throw new Error('Test has already stopped');
         const response = await worker.sendCustomMessage({ testId: params.testId, request: message.request });
         if (response.error)
-          addLocationAndSnippetToError(this._config.config, response.error);
+          addLocationAndSnippetToError(this._config.config, response.error, test.location.file);
         return response;
       } catch (e) {
         const error = serializeError(e);
-        addLocationAndSnippetToError(this._config.config, error);
+        addLocationAndSnippetToError(this._config.config, error, test.location.file);
         return { response: undefined, error };
       }
     };
 
-    for (const error of params.errors)
-      addLocationAndSnippetToError(this._config.config, error);
-    this._failureTracker.onTestPaused?.({ ...params, sendMessage });
+    this._failureTracker.onTestPaused?.({ errors: result.errors, sendMessage });
   }
 
   skipWholeJob(): boolean {

packages/playwright/src/runner/testRunner.ts

@@ -470,7 +470,9 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise
     createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }),
     ...createRunTestsTasks(config),
   ];
-  const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout);
+
+  const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug });
+  const status = await runTasks(testRun, tasks, config.config.globalTimeout);
 
   // Calling process.exit() might truncate large stdout/stderr output.
   // See https://github.com/nodejs/node/issues/6456.