feat(trace): show target point for raw mouse apis (#28459)

Fixes #27931.

packages/playwright-core/src/server/dispatchers/pageDispatcher.ts

@@ -235,27 +235,27 @@ export class PageDispatcher extends Dispatcher<Page, channels.PageChannel, Brows
   }
 
   async mouseMove(params: channels.PageMouseMoveParams, metadata: CallMetadata): Promise<void> {
-    await this._page.mouse.move(params.x, params.y, params);
+    await this._page.mouse.move(params.x, params.y, params, metadata);
   }
 
   async mouseDown(params: channels.PageMouseDownParams, metadata: CallMetadata): Promise<void> {
-    await this._page.mouse.down(params);
+    await this._page.mouse.down(params, metadata);
   }
 
   async mouseUp(params: channels.PageMouseUpParams, metadata: CallMetadata): Promise<void> {
-    await this._page.mouse.up(params);
+    await this._page.mouse.up(params, metadata);
   }
 
   async mouseClick(params: channels.PageMouseClickParams, metadata: CallMetadata): Promise<void> {
-    await this._page.mouse.click(params.x, params.y, params);
+    await this._page.mouse.click(params.x, params.y, params, metadata);
   }
 
   async mouseWheel(params: channels.PageMouseWheelParams, metadata: CallMetadata): Promise<void> {
     await this._page.mouse.wheel(params.deltaX, params.deltaY);
   }
 
   async touchscreenTap(params: channels.PageTouchscreenTapParams, metadata: CallMetadata): Promise<void> {
-    await this._page.touchscreen.tap(params.x, params.y);
+    await this._page.touchscreen.tap(params.x, params.y, metadata);
   }
 
   async accessibilitySnapshot(params: channels.PageAccessibilitySnapshotParams, metadata: CallMetadata): Promise<channels.PageAccessibilitySnapshotResult> {

packages/playwright-core/src/server/input.ts

@@ -18,6 +18,7 @@ import { assert } from '../utils';
 import * as keyboardLayout from './usKeyboardLayout';
 import type * as types from './types';
 import type { Page } from './page';
+import type { CallMetadata } from './instrumentation';
 
 export const keypadLocation = keyboardLayout.keypadLocation;
 
@@ -169,7 +170,9 @@ export class Mouse {
     this._keyboard = this._page.keyboard;
   }
 
-  async move(x: number, y: number, options: { steps?: number, forClick?: boolean } = {}) {
+  async move(x: number, y: number, options: { steps?: number, forClick?: boolean } = {}, metadata?: CallMetadata) {
+    if (metadata)
+      metadata.point = { x, y };
     const { steps = 1 } = options;
     const fromX = this._x;
     const fromY = this._y;
@@ -182,21 +185,27 @@ export class Mouse {
     }
   }
 
-  async down(options: { button?: types.MouseButton, clickCount?: number } = {}) {
+  async down(options: { button?: types.MouseButton, clickCount?: number } = {}, metadata?: CallMetadata) {
+    if (metadata)
+      metadata.point = { x: this._x, y: this._y };
     const { button = 'left', clickCount = 1 } = options;
     this._lastButton = button;
     this._buttons.add(button);
     await this._raw.down(this._x, this._y, this._lastButton, this._buttons, this._keyboard._modifiers(), clickCount);
   }
 
-  async up(options: { button?: types.MouseButton, clickCount?: number } = {}) {
+  async up(options: { button?: types.MouseButton, clickCount?: number } = {}, metadata?: CallMetadata) {
+    if (metadata)
+      metadata.point = { x: this._x, y: this._y };
     const { button = 'left', clickCount = 1 } = options;
     this._lastButton = 'none';
     this._buttons.delete(button);
     await this._raw.up(this._x, this._y, button, this._buttons, this._keyboard._modifiers(), clickCount);
   }
 
-  async click(x: number, y: number, options: { delay?: number, button?: types.MouseButton, clickCount?: number } = {}) {
+  async click(x: number, y: number, options: { delay?: number, button?: types.MouseButton, clickCount?: number } = {}, metadata?: CallMetadata) {
+    if (metadata)
+      metadata.point = { x: this._x, y: this._y };
     const { delay = null, clickCount = 1 } = options;
     if (delay) {
       this.move(x, y, { forClick: true });
@@ -300,7 +309,9 @@ export class Touchscreen {
     this._page = page;
   }
 
-  async tap(x: number, y: number) {
+  async tap(x: number, y: number, metadata?: CallMetadata) {
+    if (metadata)
+      metadata.point = { x, y };
     if (!this._page._browserContext._options.hasTouch)
       throw new Error('hasTouch must be enabled on the browser context before using the touchscreen.');
     await this._raw.tap(x, y, this._page.keyboard._modifiers());

packages/playwright-core/src/server/trace/recorder/tracing.ts

@@ -565,6 +565,7 @@ function createAfterActionTraceEvent(metadata: CallMetadata): trace.AfterActionT
     endTime: metadata.endTime,
     error: metadata.error?.error,
     result: metadata.result,
+    point: metadata.point,
   };
 }
 

packages/trace-viewer/src/snapshotRenderer.ts

@@ -251,7 +251,7 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
         if (!src) {
           iframe.setAttribute('src', 'data:text/html,<body style="background: #ddd"></body>');
         } else {
-          // Retain query parameters to inherit name=, time=, showPoint= and other values from parent.
+          // Retain query parameters to inherit name=, time=, pointX=, pointY= and other values from parent.
           const url = new URL(unwrapPopoutUrl(window.location.href));
           // We can be loading iframe from within iframe, reset base to be absolute.
           const index = url.pathname.lastIndexOf('/snapshot/');
@@ -305,8 +305,13 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
       document.styleSheets[0].disabled = true;
 
       const search = new URL(window.location.href).searchParams;
-      if (search.get('showPoint')) {
-        for (const target of targetElements) {
+
+      if (search.get('pointX') && search.get('pointY')) {
+        const pointX = +search.get('pointX')!;
+        const pointY = +search.get('pointY')!;
+        const hasTargetElements = targetElements.length > 0;
+        const roots = document.documentElement ? [document.documentElement] : [];
+        for (const target of (hasTargetElements ? targetElements : roots)) {
           const pointElement = document.createElement('x-pw-pointer');
           pointElement.style.position = 'fixed';
           pointElement.style.backgroundColor = '#f44336';
@@ -315,9 +320,24 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
           pointElement.style.borderRadius = '10px';
           pointElement.style.margin = '-10px 0 0 -10px';
           pointElement.style.zIndex = '2147483646';
-          const box = target.getBoundingClientRect();
-          pointElement.style.left = (box.left + box.width / 2) + 'px';
-          pointElement.style.top = (box.top + box.height / 2) + 'px';
+          if (hasTargetElements) {
+            // Sometimes there are layout discrepancies between recording and rendering, e.g. fonts,
+            // that may place the point at the wrong place. To avoid confusion, we just show the
+            // point in the middle of the target element.
+            const box = target.getBoundingClientRect();
+            const centerX = (box.left + box.width / 2);
+            const centerY = (box.top + box.height / 2);
+            pointElement.style.left = centerX + 'px';
+            pointElement.style.top = centerY + 'px';
+            // "Blue dot" to indicate that action point is not 100% correct.
+            if (Math.abs(centerX - pointX) >= 2 || Math.abs(centerY - pointY) >= 2)
+              pointElement.style.backgroundColor = '#3646f4';
+          } else {
+            // For actions without a target element, e.g. page.mouse.move(),
+            // show the point at the recorder location.
+            pointElement.style.left = pointX + 'px';
+            pointElement.style.top = pointY + 'px';
+          }
           document.documentElement.appendChild(pointElement);
         }
       }

packages/trace-viewer/src/traceModel.ts

@@ -206,6 +206,8 @@ export class TraceModel {
         existing!.result = event.result;
         existing!.error = event.error;
         existing!.attachments = event.attachments;
+        if (event.point)
+          existing!.point = event.point;
         for (const attachment of event.attachments?.filter(a => a.sha1) || [])
           this._attachments.set(attachment.sha1!, attachment);
         break;

packages/trace-viewer/src/ui/snapshotTab.tsx

@@ -42,7 +42,7 @@ export const SnapshotTab: React.FunctionComponent<{
   const [measure, ref] = useMeasure<HTMLDivElement>();
   const [snapshotTab, setSnapshotTab] = React.useState<'action'|'before'|'after'>('action');
 
-  type Snapshot = { action: ActionTraceEvent, snapshotName: string, showPoint?: boolean };
+  type Snapshot = { action: ActionTraceEvent, snapshotName: string, point?: { x: number, y: number } };
   const { snapshots } = React.useMemo(() => {
     if (!action)
       return { snapshots: {} };
@@ -55,7 +55,9 @@ export const SnapshotTab: React.FunctionComponent<{
       beforeSnapshot = a?.afterSnapshot ? { action: a, snapshotName: a?.afterSnapshot } : undefined;
     }
     const afterSnapshot: Snapshot | undefined = action.afterSnapshot ? { action, snapshotName: action.afterSnapshot } : beforeSnapshot;
-    const actionSnapshot: Snapshot | undefined = action.inputSnapshot ? { action, snapshotName: action.inputSnapshot, showPoint: !!action.point } : afterSnapshot;
+    const actionSnapshot: Snapshot | undefined = action.inputSnapshot ? { action, snapshotName: action.inputSnapshot } : afterSnapshot;
+    if (actionSnapshot)
+      actionSnapshot.point = action.point;
     return { snapshots: { action: actionSnapshot, before: beforeSnapshot, after: afterSnapshot } };
   }, [action]);
 
@@ -67,16 +69,20 @@ export const SnapshotTab: React.FunctionComponent<{
     const params = new URLSearchParams();
     params.set('trace', context(snapshot.action).traceUrl);
     params.set('name', snapshot.snapshotName);
-    if (snapshot.showPoint)
-      params.set('showPoint', '1');
+    if (snapshot.point) {
+      params.set('pointX', String(snapshot.point.x));
+      params.set('pointY', String(snapshot.point.y));
+    }
     const snapshotUrl = new URL(`snapshot/${snapshot.action.pageId}?${params.toString()}`, window.location.href).toString();
     const snapshotInfoUrl = new URL(`snapshotInfo/${snapshot.action.pageId}?${params.toString()}`, window.location.href).toString();
 
     const popoutParams = new URLSearchParams();
     popoutParams.set('r', snapshotUrl);
     popoutParams.set('trace', context(snapshot.action).traceUrl);
-    if (snapshot.showPoint)
-      popoutParams.set('showPoint', '1');
+    if (snapshot.point) {
+      popoutParams.set('pointX', String(snapshot.point.x));
+      popoutParams.set('pointY', String(snapshot.point.y));
+    }
     const popoutUrl = new URL(`snapshot.html?${popoutParams.toString()}`, window.location.href).toString();
     return { snapshots, snapshotInfoUrl, snapshotUrl, popoutUrl };
   }, [snapshots, snapshotTab]);

packages/trace/src/trace.ts

@@ -90,6 +90,7 @@ export type AfterActionTraceEvent = {
   error?: SerializedError['error'];
   attachments?: AfterActionTraceEventAttachment[];
   result?: any;
+  point?: Point;
 };
 
 export type LogTraceEvent = {

tests/library/trace-viewer.spec.ts

@@ -626,6 +626,7 @@ test('should highlight target elements', async ({ page, runAndTrace, browserName
     await page.locator('text=t5').innerText();
     await expect(page.locator('text=t6')).toHaveText(/t6/i);
     await expect(page.locator('text=multi')).toHaveText(['a', 'b'], { timeout: 1000 }).catch(() => {});
+    await page.mouse.move(123, 234);
   });
 
   async function highlightedDivs(frameLocator: FrameLocator) {
@@ -637,6 +638,14 @@ test('should highlight target elements', async ({ page, runAndTrace, browserName
 
   const framePageClick = await traceViewer.snapshotFrame('page.click');
   await expect.poll(() => highlightedDivs(framePageClick)).toEqual(['t1']);
+  const box1 = await framePageClick.getByText('t1').boundingBox();
+  const box2 = await framePageClick.locator('x-pw-pointer').boundingBox();
+  const x1 = box1!.x + box1!.width / 2;
+  const y1 = box1!.y + box1!.height / 2;
+  const x2 = box2!.x + box2!.width / 2;
+  const y2 = box2!.y + box2!.height / 2;
+  expect(Math.abs(x1 - x2) < 2).toBeTruthy();
+  expect(Math.abs(y1 - y2) < 2).toBeTruthy();
 
   const framePageInnerText = await traceViewer.snapshotFrame('page.innerText');
   await expect.poll(() => highlightedDivs(framePageInnerText)).toEqual(['t2']);
@@ -655,6 +664,10 @@ test('should highlight target elements', async ({ page, runAndTrace, browserName
 
   const frameExpect2 = await traceViewer.snapshotFrame('expect.toHaveText', 1);
   await expect.poll(() => highlightedDivs(frameExpect2)).toEqual(['multi', 'multi']);
+  await expect(frameExpect2.locator('x-pw-pointer')).not.toBeVisible();
+
+  const frameMouseMove = await traceViewer.snapshotFrame('mouse.move');
+  await expect(frameMouseMove.locator('x-pw-pointer')).toBeVisible();
 });
 
 test('should highlight target element in shadow dom', async ({ page, server, runAndTrace }) => {