fix(server): desktop-drawn preview snapshots match the page's scale and viewport

apps/server/src/preview/ServerBrowser.test.ts

@@ -1026,6 +1026,129 @@ it.live("viewers see the agent's pointer move to its target and click there", ()
   ).pipe(Effect.provide(layer)),
 );
 
+/** Makes a fake desktop page report its ratio and CSS viewport, as a real one would. */
+const reportPage = (
+  context: ReturnType<typeof makeContext>,
+  page: { readonly ratio: number; readonly width: number; readonly height: number },
+) => {
+  const evaluate = context.page.evaluate.getMockImplementation()!;
+  context.page.evaluate.mockImplementation(((script?: unknown) =>
+    typeof script === "string" && script.includes("devicePixelRatio")
+      ? Promise.resolve(page)
+      : evaluate()) as typeof evaluate);
+};
+
+/** The clip of each scaled capture a context's sessions were asked for. Viewer stills are full size. */
+const captureClips = (context: ReturnType<typeof makeContext>) =>
+  context.sessions.flatMap((session) =>
+    session.send.mock.calls.flatMap(([method, input]) => {
+      const clip = (
+        input as
+          | {
+              readonly clip?: {
+                readonly width: number;
+                readonly height: number;
+                readonly scale: number;
+              };
+            }
+          | undefined
+      )?.clip;
+      return method === "Page.captureScreenshot" && clip !== undefined
+        ? [{ width: clip.width, height: clip.height, scale: clip.scale }]
+        : [];
+    }),
+  );
+
+it.live("sizes a snapshot by the display's ratio for a tab the desktop renders", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      // A 150% display, with the page at the panel's default 1280x800.
+      reportPage(context, { ratio: 1.5, width: 1280, height: 800 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("captures the viewport a desktop-drawn page actually has", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      // The desktop window was zoomed out, so the page lays out smaller than 1280x800.
+      reportPage(context, { ratio: 1.5, width: 1067, height: 667 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(context)).toEqual([
+        { width: 1067, height: 667, scale: 1280 / (1067 * 1.5) },
+      ]);
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("leaves page zoom out of a desktop-drawn tab's snapshot scale", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      desktopRendersNext = true;
+      const { browser, broker, tabId } = yield* ready;
+      const manager = yield* Manager.PreviewManager;
+      yield* browser.attachViewer(viewerInput(tabId, false));
+      const context = desktopConnections[0]!.context;
+      yield* manager.adjust({ threadId: scope.thread.threadId, tabId, zoomFactor: 2 });
+      // A 150% display with the page at 200% zoom: it lays out at half size in CSS pixels.
+      reportPage(context, { ratio: 3, width: 640, height: 400 });
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+      // The desktop zooms its own page; the server sends it no emulation.
+      expect(
+        context.sessions.flatMap((session) => session.send.mock.calls.map(([method]) => method)),
+      ).not.toContain("Emulation.setDeviceMetricsOverride");
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("keeps a headless tab's snapshot at its own render scale", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const { broker, tabId } = yield* ready;
+      const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
+        scope,
+        tabId,
+        operation: "snapshot",
+        input: {},
+      });
+      expect(captureClips(contexts[0]!)).toEqual([{ width: 1280, height: 800, scale: 0.5 }]);
+      expect(contexts[0]!.page.evaluate).not.toHaveBeenCalledWith(
+        expect.stringContaining("devicePixelRatio"),
+      );
+      expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
 it.live("drives the desktop's own page for a tab the desktop renders", () =>
   Effect.scoped(
     Effect.gen(function* () {

apps/server/src/preview/ServerBrowser.ts

@@ -631,10 +631,13 @@ const make = Effect.gen(function* () {
       await tab.page.emulateMedia({ colorScheme: colorScheme === "system" ? null : colorScheme });
     }
     const zoomFactor = snapshot.zoomFactor ?? 1;
-    if (zoomFactor !== tab.zoomFactor && !tab.desktop) {
+    if (zoomFactor !== tab.zoomFactor) {
       tab.zoomFactor = zoomFactor;
-      await applyZoom(tab);
-      broadcastViewport(tab);
+      // The desktop applies the zoom to a page it draws; it is kept here for snapshot sizing.
+      if (!tab.desktop) {
+        await applyZoom(tab);
+        broadcastViewport(tab);
+      }
     }
   };
 
@@ -1270,6 +1273,26 @@ const make = Effect.gen(function* () {
     return started;
   };
 
+  /**
+   * What sizes a snapshot. Chromium sizes a capture in device-independent pixels times the
+   * display's scale, without page zoom. A headless tab renders at RENDER_SCALE in the viewport
+   * Playwright set. A tab the desktop draws has neither: its page reports the display's scale
+   * times the zoom the desktop applied, and its viewport in CSS pixels.
+   */
+  const snapshotRendering = async (tab: ServerTab) => {
+    if (!tab.desktop) return { renderScale: RENDER_SCALE };
+    const page = (await tab.page.evaluate(
+      "({ ratio: devicePixelRatio, width: innerWidth, height: innerHeight })",
+    )) as { readonly ratio: number; readonly width: number; readonly height: number };
+    return {
+      renderScale: page.ratio / tab.zoomFactor,
+      viewport: {
+        width: Math.round(page.width * tab.zoomFactor),
+        height: Math.round(page.height * tab.zoomFactor),
+      },
+    };
+  };
+
   // Scaled captures repaint every screencast; pause them to avoid leaking that frame.
   const withScreencastsPaused = <A>(tab: ServerTab, capture: () => Promise<A>): Promise<A> =>
     withCaptureLock(tab, async () => {
@@ -1653,8 +1676,8 @@ const make = Effect.gen(function* () {
         return { tabId: tab.tabId, colorScheme };
       }
       case "snapshot": {
-        return withScreencastsPaused(tab, () =>
-          ServerBrowserPage.snapshot({ ...tab, renderScale: RENDER_SCALE }),
+        return withScreencastsPaused(tab, async () =>
+          ServerBrowserPage.snapshot({ ...tab, ...(await snapshotRendering(tab)) }),
         );
       }
       case "click": {

apps/server/src/preview/ServerBrowserPage.test.ts

@@ -246,3 +246,78 @@ describe("server browser element refs", () => {
     expect(await page.evaluate("window.seen")).toEqual(["click", "hover", "drop"]);
   });
 });
+
+describe("server browser snapshot size", () => {
+  /** Width and height from the PNG's IHDR chunk. */
+  const pngSize = (data: string) => {
+    const bytes = Buffer.from(data, "base64");
+    return { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
+  };
+
+  // Chromium sizes a scaled capture by the window's real ratio, so each case launches at its own,
+  // as a display does for a tab the desktop draws.
+  it.each([1, 1.5, 2])(
+    "captures a 1280x800 viewport at 1280x800 when the render scale is the page's %sx",
+    async (deviceScaleFactor) => {
+      const browser = await chromium.launch({
+        headless: true,
+        args: [`--force-device-scale-factor=${deviceScaleFactor}`],
+      });
+      try {
+        const context = await browser.newContext({
+          viewport: { width: 1280, height: 800 },
+          deviceScaleFactor,
+        });
+        const page = await context.newPage();
+        const cdp = await context.newCDPSession(page);
+        await page.setContent("<p>sized</p>");
+        // A just-launched browser can refuse a capture until it has painted.
+        await page.evaluate(
+          "new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+        );
+        const result = await ServerBrowserPage.snapshot({
+          page,
+          cdp,
+          renderScale: deviceScaleFactor,
+          consoleEntries: [],
+          networkEntries: [],
+          actionTimeline: [],
+        });
+        expect(pngSize(result.screenshot.data)).toEqual({ width: 1280, height: 800 });
+        expect(result.screenshot).toMatchObject({ width: 1280, height: 800 });
+      } finally {
+        await browser.close();
+      }
+    },
+  );
+
+  it("reports the captured image's size when the given viewport is a pixel off", async () => {
+    const browser = await chromium.launch({
+      headless: true,
+      args: ["--force-device-scale-factor=1"],
+    });
+    try {
+      const context = await browser.newContext({ viewport: { width: 539, height: 939 } });
+      const page = await context.newPage();
+      const cdp = await context.newCDPSession(page);
+      await page.setContent("<p>sized</p>");
+      await page.evaluate(
+        "new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
+      );
+      // A zoomed desktop page's integer CSS size times its zoom can miss its real width by one.
+      const result = await ServerBrowserPage.snapshot({
+        page,
+        cdp,
+        renderScale: 1,
+        viewport: { width: 538, height: 940 },
+        consoleEntries: [],
+        networkEntries: [],
+        actionTimeline: [],
+      });
+      expect(pngSize(result.screenshot.data)).toEqual({ width: 539, height: 939 });
+      expect(result.screenshot).toMatchObject({ width: 539, height: 939 });
+    } finally {
+      await browser.close();
+    }
+  });
+});

apps/server/src/preview/ServerBrowserPage.ts

@@ -150,15 +150,34 @@ const SNAPSHOT_SCRIPT = `(() => {
   };
 })()`;
 
+/**
+ * Width and height from a PNG's header, so the reported size is the image's own. Null for data
+ * that is not a PNG.
+ */
+const pngSize = (base64: string) => {
+  const header = Buffer.from(base64.slice(0, 32), "base64");
+  return header.length >= 24 && header.toString("latin1", 12, 16) === "IHDR"
+    ? { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }
+    : null;
+};
+
+/** A page's viewport in device-independent pixels, the units a capture clip is given in. */
+type Viewport = { readonly width: number; readonly height: number };
+
 // Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets.
 export const captureViewport = async (
   page: Page,
   cdp: CDPSession,
-  options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number },
+  options: {
+    readonly format: "png" | "jpeg";
+    readonly quality?: number;
+    readonly scale: number;
+    readonly viewport?: Viewport;
+  },
 ) => {
   let clip;
   if (options.scale < 1) {
-    const viewport = page.viewportSize() ?? { width: 1280, height: 800 };
+    const viewport = options.viewport ?? page.viewportSize() ?? { width: 1280, height: 800 };
     const { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics");
     clip = {
       x: cssVisualViewport.pageX,
@@ -179,11 +198,13 @@ export const snapshot = async (input: {
   readonly page: Page;
   readonly cdp: CDPSession;
   readonly renderScale: number;
+  /** For a page Playwright did not size, such as one the desktop draws. */
+  readonly viewport?: Viewport;
   readonly consoleEntries: ReadonlyArray<PreviewAutomationConsoleEntry>;
   readonly networkEntries: ReadonlyArray<PreviewAutomationNetworkEntry>;
   readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
 }): Promise<PreviewAutomationSnapshot> => {
-  const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 };
+  const viewport = input.viewport ?? input.page.viewportSize() ?? { width: 1280, height: 800 };
   const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * input.renderScale));
   const state = refsFor(input.page);
   invalidateRefs(input.page);
@@ -196,7 +217,7 @@ export const snapshot = async (input: {
       >
     >,
     input.page.ariaSnapshot({ mode: "ai", boxes: true, timeout: DEFAULT_TIMEOUT_MS }),
-    captureViewport(input.page, input.cdp, { format: "png", scale }),
+    captureViewport(input.page, input.cdp, { format: "png", scale, viewport }),
   ]);
   if (state.generation !== generation) {
     throw new ServerBrowserOperationError(
@@ -220,8 +241,10 @@ export const snapshot = async (input: {
     screenshot: {
       mimeType: "image/png",
       data,
-      width: Math.round(viewport.width * input.renderScale * scale),
-      height: Math.round(viewport.height * input.renderScale * scale),
+      ...(pngSize(data) ?? {
+        width: Math.round(viewport.width * input.renderScale * scale),
+        height: Math.round(viewport.height * input.renderScale * scale),
+      }),
     },
   };
 };