chore: push action in context to the recorder app (#36611)

Author: pavelfeldman

packages/injected/src/recorder/recorder.ts

@@ -60,7 +60,6 @@ interface RecorderTool {
   onMouseLeave?(event: MouseEvent): void;
   onFocus?(event: Event): void;
   onScroll?(event: Event): void;
-  onLoad?(event: Event): void;
 }
 
 class NoneTool implements RecorderTool {
@@ -633,10 +632,6 @@ class JsonRecordActionTool implements RecorderTool {
     return 'pointer';
   }
 
-  install() {
-    this._pushSnapshot();
-  }
-
   onClick(event: MouseEvent) {
     // in webkit, sliding a range element may trigger a click event with a different target if the mouse is released outside the element bounding box.
     // So we check the hovered element instead, and if it is a range input, we skip click handling
@@ -769,20 +764,6 @@ class JsonRecordActionTool implements RecorderTool {
     });
   }
 
-  onLoad() {
-    this._pushSnapshot();
-  }
-
-  private _pushSnapshot() {
-    const { ariaSnapshot } = this._ariaSnapshot(this._recorder.document.body);
-    this._recorder.recordAction({
-      selector: '',
-      name: 'assertSnapshot',
-      signals: [],
-      ariaSnapshot,
-    });
-  }
-
   private _shouldIgnoreMouseEvent(event: MouseEvent): boolean {
     const target = this._recorder.deepEventTarget(event);
     const nodeName = target.nodeName;
@@ -1331,7 +1312,6 @@ export class Recorder {
       addEventListener(this.document, 'mouseenter', event => this._onMouseEnter(event as MouseEvent), true),
       addEventListener(this.document, 'focus', event => this._onFocus(event), true),
       addEventListener(this.document, 'scroll', event => this._onScroll(event), true),
-      addEventListener(this.injectedScript.window, 'load', event => this._onLoad(event), true),
     ];
 
     this.highlight.install();
@@ -1515,12 +1495,6 @@ export class Recorder {
     this._currentTool.onFocus?.(event);
   }
 
-  private _onLoad(event: Event) {
-    if (!event.isTrusted)
-      return;
-    this._currentTool.onLoad?.(event);
-  }
-
   private _onScroll(event: Event) {
     if (!event.isTrusted)
       return;

packages/playwright-core/src/client/browserContext.ts

@@ -47,6 +47,13 @@ import type * as api from '../../types/types';
 import type { URLMatch } from '../utils/isomorphic/urlMatch';
 import type { Platform } from './platform';
 import type * as channels from '@protocol/channels';
+import type * as actions from '@recorder/actions';
+
+interface RecorderEventSink {
+  actionAdded(page: Page, actionInContext: actions.ActionInContext): void;
+  actionUpdated(page: Page, actionInContext: actions.ActionInContext): void;
+  signalAdded(page: Page, signal: actions.SignalInContext): void;
+}
 
 export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel> implements api.BrowserContext {
   _pages = new Set<Page>();
@@ -71,7 +78,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
   _closingStatus: 'none' | 'closing' | 'closed' = 'none';
   private _closeReason: string | undefined;
   private _harRouters: HarRouter[] = [];
-  private _onRecorderEventSink: ((event: string, data: any) => void) | undefined;
+  private _onRecorderEventSink: RecorderEventSink | undefined;
 
   static from(context: channels.BrowserContextChannel): BrowserContext {
     return (context as any)._object;
@@ -141,7 +148,14 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     this._channel.on('requestFailed', ({ request, failureText, responseEndTiming, page }) => this._onRequestFailed(network.Request.from(request), responseEndTiming, failureText, Page.fromNullable(page)));
     this._channel.on('requestFinished', params => this._onRequestFinished(params));
     this._channel.on('response', ({ response, page }) => this._onResponse(network.Response.from(response), Page.fromNullable(page)));
-    this._channel.on('recorderEvent', ({ event, data }) => this._onRecorderEventSink?.(event, data));
+    this._channel.on('recorderEvent', ({ event, data, page }) => {
+      if (event === 'actionAdded')
+        this._onRecorderEventSink?.actionAdded(Page.from(page), data as actions.ActionInContext);
+      else if (event === 'actionUpdated')
+        this._onRecorderEventSink?.actionUpdated(Page.from(page), data as actions.ActionInContext);
+      else if (event === 'signalAdded')
+        this._onRecorderEventSink?.signalAdded(Page.from(page), data as actions.SignalInContext);
+    });
     this._closedPromise = new Promise(f => this.once(Events.BrowserContext.Close, f));
 
     this._setEventToSubscriptionMapping(new Map<string, channels.BrowserContextUpdateSubscriptionParams['event']>([
@@ -501,9 +515,9 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     await this._closedPromise;
   }
 
-  async _enableRecorder(params: channels.BrowserContextEnableRecorderParams, callback?: (event: string, data: any) => void) {
-    if (callback)
-      this._onRecorderEventSink = callback;
+  async _enableRecorder(params: channels.BrowserContextEnableRecorderParams, eventSink?: RecorderEventSink) {
+    if (eventSink)
+      this._onRecorderEventSink = eventSink;
     await this._channel.enableRecorder(params);
   }
 

packages/playwright-core/src/protocol/validator.ts

@@ -980,8 +980,9 @@ scheme.BrowserContextResponseEvent = tObject({
   page: tOptional(tChannel(['Page'])),
 });
 scheme.BrowserContextRecorderEventEvent = tObject({
-  event: tString,
+  event: tEnum(['actionAdded', 'actionUpdated', 'signalAdded']),
   data: tAny,
+  page: tChannel(['Page']),
 });
 scheme.BrowserContextAddCookiesParams = tObject({
   cookies: tArray(tType('SetNetworkCookie')),

packages/playwright-core/src/server/debugController.ts

@@ -221,10 +221,10 @@ function wireListeners(recorder: Recorder, debugController: DebugController) {
     actions.push(action);
     actionsChanged();
   });
-  recorder.on(RecorderEvent.SignalAdded, (signal: actions.Signal) => {
-    const lastAction = actions[actions.length - 1];
+  recorder.on(RecorderEvent.SignalAdded, (signal: actions.SignalInContext) => {
+    const lastAction = actions.findLast(a => a.frame.pageGuid === signal.frame.pageGuid);
     if (lastAction)
-      lastAction.action.signals.push(signal);
+      lastAction.action.signals.push(signal.signal);
     actionsChanged();
   });
 }

packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts

@@ -200,8 +200,8 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
         page: PageDispatcher.fromNullable(this, request.frame()?._page.initializedOrUndefined()),
       });
     });
-    this.addObjectListener(BrowserContext.Events.RecorderEvent, ({ event, data }: { event: string, data: any }) => {
-      this._dispatchEvent('recorderEvent', { event, data });
+    this.addObjectListener(BrowserContext.Events.RecorderEvent, ({ event, data, page }: { event: 'actionAdded' | 'actionUpdated' | 'signalAdded', data: any, page: Page }) => {
+      this._dispatchEvent('recorderEvent', { event, data, page: PageDispatcher.from(this, page) });
     });
   }
 

packages/playwright-core/src/server/recorder.ts

@@ -40,7 +40,6 @@ import type * as channels from '@protocol/channels';
 import type * as actions from '@recorder/actions';
 import type { CallLog, CallLogStatus, ElementInfo, Mode, OverlayState, Source, UIState } from '@recorder/recorderTypes';
 import type { RegisteredListener } from '../utils';
-import type { Signal } from '../../../recorder/src/actions';
 
 const recorderSymbol = Symbol('recorderSymbol');
 
@@ -65,7 +64,7 @@ export type RecorderEventMap = {
   [RecorderEvent.CallLogsUpdated]: [callLogs: CallLog[]];
   [RecorderEvent.UserSourcesChanged]: [sources: Source[]];
   [RecorderEvent.ActionAdded]: [action: actions.ActionInContext];
-  [RecorderEvent.SignalAdded]: [signal: actions.Signal];
+  [RecorderEvent.SignalAdded]: [signal: actions.SignalInContext];
   [RecorderEvent.PageNavigated]: [url: string];
   [RecorderEvent.ContextClosed]: [];
 };
@@ -120,14 +119,15 @@ export class Recorder extends EventEmitter<RecorderEventMap> implements Instrume
     this._recorderMode = params.recorderMode ?? 'default';
     this.handleSIGINT = params.handleSIGINT;
 
-    this._signalProcessor = new RecorderSignalProcessor();
-    this._signalProcessor.on('action', (actionInContext: actions.ActionInContext) => {
-      if (this._enabled)
-        this.emit(RecorderEvent.ActionAdded, actionInContext);
-    });
-    this._signalProcessor.on('signal', (signal: Signal) => {
-      if (this._enabled)
-        this.emit(RecorderEvent.SignalAdded, signal);
+    this._signalProcessor = new RecorderSignalProcessor({
+      addAction: (actionInContext: actions.ActionInContext) => {
+        if (this._enabled)
+          this.emit(RecorderEvent.ActionAdded, actionInContext);
+      },
+      addSignal: (signal: actions.SignalInContext) => {
+        if (this._enabled)
+          this.emit(RecorderEvent.SignalAdded, signal);
+      },
     });
 
     context.on(BrowserContext.Events.BeforeClose, () => {
@@ -491,13 +491,15 @@ export class Recorder extends EventEmitter<RecorderEventMap> implements Instrume
 
   private _describeMainFrame(page: Page): actions.FrameDescription {
     return {
+      pageGuid: page.guid,
       pageAlias: this._pageAliases.get(page)!,
       framePath: [],
     };
   }
 
   private async _describeFrame(frame: Frame): Promise<actions.FrameDescription> {
     return {
+      pageGuid: frame._page.guid,
       pageAlias: this._pageAliases.get(frame._page)!,
       framePath: await generateFrameSelector(frame),
     };

packages/playwright-core/src/server/recorder/recorderApp.ts

@@ -228,7 +228,7 @@ export class RecorderApp {
       this._onActionAdded(action);
     });
 
-    recorder.on(RecorderEvent.SignalAdded, (signal: actions.Signal) => {
+    recorder.on(RecorderEvent.SignalAdded, (signal: actions.SignalInContext) => {
       this._onSignalAdded(signal);
     });
 
@@ -266,10 +266,10 @@ export class RecorderApp {
     this._updateActions();
   }
 
-  private _onSignalAdded(signal: actions.Signal) {
-    const lastAction = this._actions[this._actions.length - 1];
+  private _onSignalAdded(signal: actions.SignalInContext) {
+    const lastAction = this._actions.findLast(a => a.frame.pageGuid === signal.frame.pageGuid);
     if (lastAction)
-      lastAction.action.signals.push(signal);
+      lastAction.action.signals.push(signal.signal);
     this._updateActions();
   }
 
@@ -367,16 +367,24 @@ export class ProgrammaticRecorderApp {
   static async run(inspectedContext: BrowserContext, recorder: Recorder) {
     let lastAction: actions.ActionInContext | null = null;
     recorder.on(RecorderEvent.ActionAdded, action => {
+      const page = findPageByGuid(inspectedContext, action.frame.pageGuid);
+      if (!page)
+        return;
       if (!lastAction || !shouldMergeAction(action, lastAction))
-        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionAdded', data: action });
+        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionAdded', data: action, page });
       else
-        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionUpdated', data: action });
+        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionUpdated', data: action, page });
       lastAction = action;
     });
     recorder.on(RecorderEvent.SignalAdded, signal => {
-      inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'signalAdded', data: signal });
+      const page = findPageByGuid(inspectedContext, signal.frame.pageGuid);
+      inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'signalAdded', data: signal, page });
     });
   }
 }
 
+function findPageByGuid(context: BrowserContext, guid: string) {
+  return context.pages().find(p => p.guid === guid);
+}
+
 const recorderAppSymbol = Symbol('recorderApp');

packages/playwright-core/src/server/recorder/recorderSignalProcessor.ts

@@ -14,26 +14,35 @@
  * limitations under the License.
  */
 
-import { EventEmitter } from 'events';
-
 import { isUnderTest } from '../utils/debug';
 import { monotonicTime } from '../../utils/isomorphic/time';
+import { generateFrameSelector } from './recorderUtils';
 
 import type { Signal } from '../../../../recorder/src/actions';
 import type { Frame } from '../frames';
 import type * as actions from '@recorder/actions';
 
-export class RecorderSignalProcessor extends EventEmitter {
+export interface ProcessorDelegate {
+  addAction(actionInContext: actions.ActionInContext): void;
+  addSignal(signalInContext: actions.SignalInContext): void;
+}
+
+export class RecorderSignalProcessor {
+  private _delegate: ProcessorDelegate;
   private _lastAction: actions.ActionInContext | null = null;
 
-  addAction(actionInContext: actions.ActionInContext, callback?: () => Promise<void>) {
+  constructor(actionSink: ProcessorDelegate) {
+    this._delegate = actionSink;
+  }
+
+  addAction(actionInContext: actions.ActionInContext) {
     this._lastAction = actionInContext;
-    this.emit('action', actionInContext);
+    this._delegate.addAction(actionInContext);
   }
 
   signal(pageAlias: string, frame: Frame, signal: Signal) {
+    const timestamp = monotonicTime();
     if (signal.name === 'navigation' && frame._page.mainFrame() === frame) {
-      const timestamp = monotonicTime();
       const lastAction = this._lastAction;
       const signalThreshold = isUnderTest() ? 500 : 5000;
 
@@ -48,6 +57,7 @@ export class RecorderSignalProcessor extends EventEmitter {
       if (generateGoto) {
         this.addAction({
           frame: {
+            pageGuid: frame._page.guid,
             pageAlias,
             framePath: [],
           },
@@ -63,6 +73,17 @@ export class RecorderSignalProcessor extends EventEmitter {
       return;
     }
 
-    this.emit('signal', signal);
+    generateFrameSelector(frame).then(framePath => {
+      const signalInContext: actions.SignalInContext = {
+        frame: {
+          pageGuid: frame._page.guid,
+          pageAlias,
+          framePath,
+        },
+        signal,
+        timestamp,
+      };
+      this._delegate.addSignal(signalInContext);
+    });
   }
 }

packages/playwright-mdd/src/recorderLoop.ts

@@ -22,25 +22,31 @@ import { runOneShot } from './loop';
 
 import type { BrowserContext } from '../../playwright-core/src/client/browserContext';
 import type * as actions from '@recorder/actions';
+import type * as playwright from 'playwright-core';
 
 export async function runRecorderLoop() {
   const browser = await chromium.launch({ headless: false });
   const context = await browser.newContext() as BrowserContext;
   await context._enableRecorder({
     mode: 'recording',
     recorderMode: 'api',
-  }, async (event, data) => {
-    if (event !== 'actionAdded')
-      return;
-    const action = data.action as actions.Action;
-    if (action.name !== 'click' && action.name !== 'press') {
-      console.log('============= action', action.name);
-      return;
-    }
-    const response = await runOneShot(prompt(action)).catch(e => {
-      console.error(e);
-    });
-    console.log(response);
+  }, {
+    actionAdded: (page: playwright.Page, actionInContext: actions.ActionInContext) => {
+      const action = actionInContext.action;
+      if (action.name !== 'click' && action.name !== 'press')
+        return;
+      runOneShot(prompt(action)).then(response => {
+        console.log(response);
+      }).catch(e => {
+        console.error(e);
+      });
+    },
+    actionUpdated: (page: playwright.Page, actionInContext: actions.ActionInContext) => {
+      console.log('actionUpdated', actionInContext);
+    },
+    signalAdded: (page: playwright.Page, signal: actions.SignalInContext) => {
+      console.log('signalAdded', signal);
+    },
   });
   const page = await context.newPage();
   await page.goto('https://playwright.dev/');

packages/protocol/src/channels.d.ts

@@ -1717,8 +1717,9 @@ export type BrowserContextResponseEvent = {
   page?: PageChannel,
 };
 export type BrowserContextRecorderEventEvent = {
-  event: string,
+  event: 'actionAdded' | 'actionUpdated' | 'signalAdded',
   data: any,
+  page: PageChannel,
 };
 export type BrowserContextAddCookiesParams = {
   cookies: SetNetworkCookie[],