fix(server): desktop-drawn preview snapshots match the page's scale and viewport

apps/server/src/preview/ServerBrowser.test.ts

@@ -1026,6 +1026,298 @@ it.live("viewers see the agent's pointer move to its target and click there", ()
   ).pipe(Effect.provide(layer)),
 );
 
+/** Makes a fake desktop page report its ratio, CSS viewport, and zoom, as a real one would. */
+const reportPage = (
+  context: ReturnType<typeof makeContext>,
+  page: {
+    readonly ratio: number;
+    readonly width: number;
+    readonly height: number;
+    readonly zoom?: number;
+    /** How far the page is scrolled down, in CSS pixels. */
+    readonly scrollY?: number;
+  },
+) => {
+  const { zoom = 1, scrollY = 0, ...reported } = page;
+  const replies: Record<string, Record<string, unknown>> = {
+    "Page.getFrameTree": { frameTree: { frame: { id: "main" } } },
+    "Page.createIsolatedWorld": { executionContextId: 7 },
+    "Page.getLayoutMetrics": { cssVisualViewport: { pageX: 0, pageY: scrollY, zoom } },
+  };
+  for (const session of context.sessions) {
+    const send = session.send.getMockImplementation()!;
+    session.send.mockImplementation(async (method, input) =>
+      method === "Runtime.evaluate" && (input as { contextId?: number })?.contextId === 7
+        ? { result: { value: reported } }
+        : (replies[method] ?? send(method, input)),
+    );
+  }
+};
+
+/** The clip of each scaled capture a context's sessions were asked for. Viewer stills are full size. */
+const captureClips = (context: ReturnType<typeof makeContext>) =>
+  context.sessions.flatMap((session) =>
+    session.send.mock.calls.flatMap(([method, input]) => {
+      const clip = (
+        input as
+          | {
+              readonly clip?: {
+                readonly width: number;
+                readonly height: number;
+                readonly scale: number;
+              };
+            }
+          | undefined
+      )?.clip;
+      return method === "Page.captureScreenshot" && clip !== undefined
+        ? [{ width: clip.width, height: clip.height, scale: clip.scale }]
+        : [];
+    }),
+  );
+
+it.live("sizes a snapshot by the display's ratio for a tab the desktop renders", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      // A 150% display, with the page at the panel's default 1280x800.
+      reportPage(context, { ratio: 1.5, width: 1280, height: 800 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("captures the viewport a desktop-drawn page actually has", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      // The desktop window was zoomed out, so the page lays out smaller than 1280x800.
+      reportPage(context, { ratio: 1.5, width: 1067, height: 667 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(context)).toEqual([
+        { width: 1067, height: 667, scale: 1280 / (1067 * 1.5) },
+      ]);
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("captures a desktop-drawn page resized to a phone preset without a 1280x800 clip", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      // An iPhone 12 Pro preset on a 100% display; Playwright still reports 1280x800.
+      reportPage(context, { ratio: 1, width: 390, height: 844 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      // At scale 1 the capture needs no clip, so nothing cuts it to the stale 1280x800.
+      expect(captureClips(context)).toEqual([]);
+      expect(snapshot.screenshot).toMatchObject({ width: 390, height: 844 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("leaves page zoom out of a desktop-drawn tab's snapshot scale", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      const manager = yield* Manager.PreviewManager;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      yield* manager.adjust({ threadId: scope.thread.threadId, tabId, zoomFactor: 2 });
+      // A 150% display with the page at 200% zoom: it lays out at half size in CSS pixels.
+      reportPage(context, { ratio: 3, width: 640, height: 400, zoom: 2 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+      // The desktop zooms its own page; the server sends it no emulation.
+      expect(
+        context.sessions.flatMap((session) => session.send.mock.calls.map(([method]) => method)),
+      ).not.toContain("Emulation.setDeviceMetricsOverride");
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("sizes a desktop-drawn snapshot by the zoom the page has, not one still on its way", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      const manager = yield* Manager.PreviewManager;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      yield* manager.adjust({ threadId: scope.thread.threadId, tabId, zoomFactor: 2 });
+      // The desktop has not applied the new zoom yet, so the page is still at 100%.
+      reportPage(context, { ratio: 1.5, width: 1280, height: 800, zoom: 1 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+// A 150% display at 100% zoom and the same display at 200%: both are 1280x800 in
+// device-independent pixels, so either state alone gives the same clip.
+const beforeZoom = { ratio: 1.5, width: 1280, height: 800, zoom: 1 };
+const afterZoom = { ratio: 3, width: 640, height: 400, zoom: 2 };
+
+/** A desktop page that applies its pending zoom right after the given read. */
+const applyZoomAfter = (
+  context: ReturnType<typeof makeContext>,
+  read: "evaluate" | "first-layout-metrics" | "every-evaluate",
+) => {
+  let current = beforeZoom;
+  let layoutReads = 0;
+  for (const session of context.sessions) {
+    const send = session.send.getMockImplementation()!;
+    session.send.mockImplementation(async (method, input) => {
+      if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } };
+      if (method === "Page.createIsolatedWorld") return { executionContextId: 7 };
+      if (method === "Runtime.evaluate" && (input as { contextId?: number })?.contextId === 7) {
+        const { zoom: _zoom, ...reported } = current;
+        if (read === "evaluate") current = afterZoom;
+        if (read === "every-evaluate") current = current === beforeZoom ? afterZoom : beforeZoom;
+        return { result: { value: reported } };
+      }
+      if (method === "Page.getLayoutMetrics") {
+        const reply = { cssVisualViewport: { pageX: 0, pageY: 0, zoom: current.zoom } };
+        layoutReads += 1;
+        if (read === "first-layout-metrics" && layoutReads === 1) current = afterZoom;
+        return reply;
+      }
+      return send(method, input);
+    });
+  }
+};
+
+it.live.each(["evaluate", "first-layout-metrics"] as const)(
+  "sizes a desktop-drawn snapshot from one page state when the zoom lands after its %s read",
+  (read) =>
+    Effect.scoped(
+      Effect.gen(function* () {
+        desktopRendersNext = true;
+        const { browser, broker, tabId } = yield* ready;
+        const manager = yield* Manager.PreviewManager;
+        yield* browser.attachViewer(viewerInput(tabId, false));
+        const context = desktopConnections[0]!.context;
+        yield* manager.adjust({ threadId: scope.thread.threadId, tabId, zoomFactor: 2 });
+        applyZoomAfter(context, read);
+        yield* broker.invoke<PreviewAutomationSnapshot>({
+          scope,
+          tabId,
+          operation: "snapshot",
+          input: {},
+        });
+        expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
+      }),
+    ).pipe(Effect.provide(layer)),
+);
+
+it.live("fails a desktop-drawn snapshot whose zoom keeps changing during its read", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      applyZoomAfter(context, "every-evaluate");
+      const failure = yield* Effect.flip(
+        broker.invoke<PreviewAutomationSnapshot>({
+          scope,
+          tabId,
+          operation: "snapshot",
+          input: {},
+        }),
+      );
+      expect(JSON.stringify(failure)).toContain("zoom kept changing");
+      expect(captureClips(context)).toEqual([]);
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("starts a zoomed desktop-drawn page's clip where it is scrolled to", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      // Scrolled 100 CSS pixels at 200% zoom is 200 device-independent pixels down.
+      reportPage(context, { ratio: 3, width: 640, height: 400, zoom: 2, scrollY: 100 });
+      yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      const origins = context.sessions.flatMap((session) =>
+        session.send.mock.calls.flatMap(([method, input]) => {
+          const clip = (input as { readonly clip?: { readonly x: number; readonly y: number } })
+            ?.clip;
+          return method === "Page.captureScreenshot" && clip ? [{ x: clip.x, y: clip.y }] : [];
+        }),
+      );
+      expect(origins).toEqual([{ x: 0, y: 200 }]);
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("keeps a headless tab's snapshot at its own render scale", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const { broker, tabId } = yield* ready;
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(contexts[0]!)).toEqual([{ width: 1280, height: 800, scale: 0.5 }]);
+      expect(
+        contexts[0]!.sessions.flatMap((session) =>
+          session.send.mock.calls.map(([method]) => method),
+        ),
+      ).not.toContain("Page.createIsolatedWorld");
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
 it.live("drives the desktop's own page for a tab the desktop renders", () =>
   Effect.scoped(
     Effect.gen(function* () {

apps/server/src/preview/ServerBrowser.ts

@@ -1270,6 +1270,25 @@ const make = Effect.gen(function* () {
     return started;
   };
 
+  /**
+   * What sizes a snapshot. Chromium sizes a capture in device-independent pixels times the
+   * display's scale, without page zoom. A headless tab renders at RENDER_SCALE in the viewport
+   * Playwright set. A tab the desktop draws has neither: its page reports the display's scale
+   * times its zoom, and its viewport in CSS pixels. The zoom comes from the page too, because
+   * the desktop applies a zoom change some time after the server publishes it.
+   */
+  const snapshotRendering = async (tab: ServerTab) => {
+    if (!tab.desktop) return { renderScale: RENDER_SCALE };
+    const page = await ServerBrowserPage.readPageMetrics(tab.cdp);
+    return {
+      renderScale: page.ratio / page.zoom,
+      viewport: {
+        width: Math.round(page.width * page.zoom),
+        height: Math.round(page.height * page.zoom),
+      },
+    };
+  };
+
   // Scaled captures repaint every screencast; pause them to avoid leaking that frame.
   const withScreencastsPaused = <A>(tab: ServerTab, capture: () => Promise<A>): Promise<A> =>
     withCaptureLock(tab, async () => {
@@ -1653,8 +1672,8 @@ const make = Effect.gen(function* () {
         return { tabId: tab.tabId, colorScheme };
       }
       case "snapshot": {
-        return withScreencastsPaused(tab, () =>
-          ServerBrowserPage.snapshot({ ...tab, renderScale: RENDER_SCALE }),
+        return withScreencastsPaused(tab, async () =>
+          ServerBrowserPage.snapshot({ ...tab, ...(await snapshotRendering(tab)) }),
         );
       }
       case "click": {