fix(desktop): capture background browser tabs without stalling

apps/desktop/src/ipc/DesktopIpcHandlers.ts

@@ -85,6 +85,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
   const ipc = yield* DesktopIpc.DesktopIpc;
   yield* installNotificationBadge();
   yield* PreviewIpc.installPreviewEventForwarding();
+  yield* ipc.handle(PreviewIpc.acknowledgeCapture);
 
   yield* ipc.handle(AppActivationIpc.setReady);
   yield* ipc.handle(AppActivationIpc.complete);

apps/desktop/src/ipc/channels.ts

@@ -109,6 +109,8 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";
 export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame";
 export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";
 export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
+export const PREVIEW_CAPTURE_REQUEST_CHANNEL = "desktop:preview-capture-request";
+export const PREVIEW_CAPTURE_READY_CHANNEL = "desktop:preview-capture-ready";
 
 export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";
 

apps/desktop/src/ipc/methods/preview.ts

@@ -27,6 +27,7 @@ import {
 import * as Effect from "effect/Effect";
 import * as Schema from "effect/Schema";
 import * as Option from "effect/Option";
+import * as Stream from "effect/Stream";
 import * as NodeURL from "node:url";
 
 import * as ElectronWindow from "../../electron/ElectronWindow.ts";
@@ -36,12 +37,20 @@ import * as DesktopClientSettings from "../../settings/DesktopClientSettings.ts"
 import { PREVIEW_WEBVIEW_PREFERENCES } from "../../preview/WebviewPreferences.ts";
 import * as IpcChannels from "../channels.ts";
 import * as DesktopIpc from "../DesktopIpc.ts";
+import * as DesktopBrowserHost from "../../preview/DesktopBrowserHost.ts";
 
 export const installPreviewEventForwarding = Effect.fn(
   "desktop.ipc.preview.installEventForwarding",
 )(function* () {
   const electronWindow = yield* ElectronWindow.ElectronWindow;
   const manager = yield* PreviewManager.PreviewManager;
+  const browserHost = yield* DesktopBrowserHost.DesktopBrowserHost;
+  yield* browserHost.captureRequests.pipe(
+    Stream.runForEach((event) =>
+      electronWindow.sendAll(IpcChannels.PREVIEW_CAPTURE_REQUEST_CHANNEL, event),
+    ),
+    Effect.forkScoped,
+  );
   yield* manager.subscribeStateChanges((tabId, state) =>
     electronWindow.sendAll(IpcChannels.PREVIEW_STATE_CHANGE_CHANNEL, tabId, state),
   );
@@ -56,6 +65,18 @@ export const installPreviewEventForwarding = Effect.fn(
   );
 });
 
+export const acknowledgeCapture = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.PREVIEW_CAPTURE_READY_CHANNEL,
+  payload: Schema.String,
+  result: Schema.Void,
+  handler: Effect.fn("desktop.ipc.preview.acknowledgeCapture")(function* (requestId, event) {
+    const main = yield* (yield* ElectronWindow.ElectronWindow).main;
+    if (!event || Option.isNone(main) || main.value.webContents.id !== event.sender.id) return;
+    const host = yield* DesktopBrowserHost.DesktopBrowserHost;
+    host.acknowledgeCapture(requestId);
+  }),
+});
+
 export const setForwardedShortcuts = DesktopIpc.makeIpcMethod({
   channel: IpcChannels.PREVIEW_SET_FORWARDED_SHORTCUTS_CHANNEL,
   payload: Schema.Array(PreviewForwardedShortcut).check(Schema.isMaxLength(MAX_KEYBINDINGS_COUNT)),

apps/desktop/src/preload.ts

@@ -1,6 +1,7 @@
 import type {
   DesktopBridge,
   DesktopPreviewPointerEvent,
+  DesktopPreviewCaptureRequest,
   DesktopPreviewRecordingInputEvent,
   DesktopPreviewRecordingFrame,
   DesktopPreviewTabState,
@@ -416,5 +417,16 @@ contextBridge.exposeInMainWorld("desktopBridge", {
       return () =>
         ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener);
     },
+    acknowledgeCapture: (requestId) =>
+      ipcRenderer.invoke(IpcChannels.PREVIEW_CAPTURE_READY_CHANNEL, requestId),
+    onCaptureRequest: (listener) => {
+      const wrappedListener = (
+        _event: Electron.IpcRendererEvent,
+        request: DesktopPreviewCaptureRequest,
+      ) => listener(request);
+      ipcRenderer.on(IpcChannels.PREVIEW_CAPTURE_REQUEST_CHANNEL, wrappedListener);
+      return () =>
+        ipcRenderer.removeListener(IpcChannels.PREVIEW_CAPTURE_REQUEST_CHANNEL, wrappedListener);
+    },
   },
 } satisfies DesktopBridge);

apps/desktop/src/preview/DesktopBrowserCapture.test.ts

@@ -0,0 +1,296 @@
+// @effect-diagnostics nodeBuiltinImport:off - Stands in for an Electron debugger.
+import { describe, expect, it } from "@effect/vitest";
+import { DesktopBrowserEvent, type DesktopPreviewCaptureRequest } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Queue from "effect/Queue";
+import * as Schema from "effect/Schema";
+import * as Stream from "effect/Stream";
+import { TestClock } from "effect/testing";
+import * as NodeEvents from "node:events";
+import { vi } from "vite-plus/test";
+
+import * as DesktopBrowserHost from "./DesktopBrowserHost.ts";
+
+const key = { threadId: "thread", tabId: "tab" };
+const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(DesktopBrowserEvent));
+const decodeReply = Schema.decodeUnknownSync(
+  Schema.fromJsonString(
+    Schema.Struct({
+      id: Schema.Number,
+      result: Schema.optional(Schema.Unknown),
+      error: Schema.optional(Schema.Struct({ message: Schema.String })),
+    }),
+  ),
+);
+
+const makeGuest = (id = 41) => {
+  const nativeStarted = Promise.withResolvers<void>();
+  const screenshotStarted = Promise.withResolvers<void>();
+  const capturePage = vi.fn(async (_rect: unknown, _options: unknown) => {
+    nativeStarted.resolve();
+    return {};
+  });
+  const screenshot = vi.fn(async (_parameters: object) => {
+    screenshotStarted.resolve();
+    return { data: "encoded-by-cdp" };
+  });
+  const sendCommand = vi.fn(async (method: string, parameters: object, _sessionId?: string) =>
+    method === "Page.captureScreenshot" ? screenshot(parameters) : { method },
+  );
+  return {
+    capturePage,
+    screenshot,
+    sendCommand,
+    nativeStarted,
+    screenshotStarted,
+    tab: {
+      withCaptureActivity: <A, E>(capture: Effect.Effect<A, E>) => capture,
+      webContents: {
+        id,
+        capturePage,
+        isDestroyed: vi.fn(() => false),
+        getURL: () => "http://localhost/",
+        getTitle: () => "Fixture",
+        getUserAgent: () => "Electron",
+      } as unknown as Electron.WebContents,
+      debugger: Object.assign(new NodeEvents.EventEmitter(), {
+        sendCommand,
+      }) as unknown as Electron.Debugger,
+    },
+  };
+};
+
+const setup = Effect.gen(function* () {
+  const host = yield* DesktopBrowserHost.make;
+  const guest = makeGuest();
+  host.attach(key, guest.tab);
+  const leases = yield* Queue.unbounded<DesktopPreviewCaptureRequest>();
+  const replies = yield* Queue.unbounded<ReturnType<typeof decodeReply>>();
+  yield* host.captureRequests.pipe(
+    Stream.runForEach((event) => Queue.offer(leases, event)),
+    Effect.forkScoped,
+  );
+  yield* host.events.pipe(
+    Stream.runForEach((line) => {
+      const event = decodeEvent(new TextDecoder().decode(line));
+      return event.type === "cdp" ? Queue.offer(replies, decodeReply(event.message)) : Effect.void;
+    }),
+    Effect.forkScoped,
+  );
+  yield* Effect.yieldNow;
+  const command = (
+    id: number,
+    method = "Page.captureScreenshot",
+    params: object = {},
+    sessionId = "t3-preview-page",
+  ) =>
+    host.handleCommandLine(
+      JSON.stringify({
+        type: "cdp",
+        ...key,
+        message: JSON.stringify({ id, method, params, sessionId }),
+      }),
+    );
+  const acknowledge = Effect.gen(function* () {
+    const event = yield* Queue.take(leases);
+    expect(event.active).toBe(true);
+    host.acknowledgeCapture(event.requestId);
+    return event;
+  });
+  return { host, guest, leases, replies, command, acknowledge };
+});
+
+describe("desktop agent screenshot rendering lease", () => {
+  it.effect("waits for placement, warms the compositor and preserves CDP parameters", () =>
+    Effect.gen(function* () {
+      const { host, guest, leases, replies, command } = yield* setup;
+      const parameters = {
+        format: "jpeg",
+        quality: 72,
+        clip: { x: 12, y: 340, width: 600, height: 400, scale: 0.5 },
+        captureBeyondViewport: false,
+      };
+      yield* command(1, "Page.captureScreenshot", parameters);
+      const active = yield* Queue.take(leases);
+      expect(guest.capturePage).not.toHaveBeenCalled();
+      expect(guest.sendCommand).not.toHaveBeenCalled();
+      host.acknowledgeCapture(active.requestId);
+      expect(yield* Queue.take(replies)).toEqual({ id: 1, result: { data: "encoded-by-cdp" } });
+      expect(guest.capturePage.mock.calls.length).toBeGreaterThanOrEqual(1);
+      expect(guest.capturePage.mock.calls.length).toBeLessThanOrEqual(2);
+      for (const args of guest.capturePage.mock.calls) {
+        expect(args).toEqual([undefined, { stayHidden: true, stayAwake: false }]);
+      }
+      expect(guest.screenshot).toHaveBeenCalledExactlyOnceWith(parameters);
+      expect(yield* Queue.take(leases)).toEqual({ ...active, active: false });
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect("releases a missing renderer acknowledgement and ignores a late one", () =>
+    Effect.gen(function* () {
+      const { host, guest, leases, replies, command } = yield* setup;
+      yield* command(1);
+      const active = yield* Queue.take(leases);
+      yield* TestClock.adjust("2 seconds");
+      expect((yield* Queue.take(replies)).error?.message).toContain("paintable within 2 seconds");
+      expect(yield* Queue.take(leases)).toEqual({ ...active, active: false });
+      host.acknowledgeCapture(active.requestId);
+      yield* command(2, "DOM.enable");
+      expect((yield* Queue.take(replies)).id).toBe(2);
+      expect(guest.capturePage).not.toHaveBeenCalled();
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect("reports a destroyed guest with one capture failure prefix", () =>
+    Effect.gen(function* () {
+      const { guest, leases, replies, command, acknowledge } = yield* setup;
+      yield* command(1);
+      vi.mocked(guest.tab.webContents.isDestroyed).mockReturnValue(true);
+      const active = yield* acknowledge;
+      expect((yield* Queue.take(replies)).error?.message).toBe(
+        "Desktop browser screenshot failed: the browser tab changed during capture.",
+      );
+      expect(yield* Queue.take(leases)).toEqual({ ...active, active: false });
+      expect(guest.capturePage).not.toHaveBeenCalled();
+      expect(guest.screenshot).not.toHaveBeenCalled();
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect.each(["native", "cdp"] as const)(
+    "reports a stable capture error when %s fails and allows a retry",
+    (failedStage) =>
+      Effect.gen(function* () {
+        const { guest, leases, replies, command, acknowledge } = yield* setup;
+        const cause = new Error("internal compositor failure detail");
+        const captureFailed = vi.fn<(error: unknown) => void>();
+        guest.tab.withCaptureActivity = (capture) =>
+          capture.pipe(Effect.tapError((error) => Effect.sync(() => captureFailed(error))));
+        if (failedStage === "native") guest.capturePage.mockRejectedValueOnce(cause);
+        else guest.screenshot.mockRejectedValueOnce(cause);
+        yield* command(1);
+        const active = yield* acknowledge;
+        expect((yield* Queue.take(replies)).error?.message).toBe(
+          "Desktop browser screenshot failed: the compositor could not capture a frame.",
+        );
+        expect(captureFailed).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ cause }));
+        expect(yield* Queue.take(leases)).toEqual({ ...active, active: false });
+        yield* command(2);
+        yield* acknowledge;
+        expect(yield* Queue.take(replies)).toEqual({ id: 2, result: { data: "encoded-by-cdp" } });
+      }).pipe(Effect.scoped),
+  );
+
+  it.effect.each(["native", "cdp"] as const)(
+    "bounds %s waiting without accumulating captures or blocking DOM commands",
+    (stalledStage) =>
+      Effect.gen(function* () {
+        const { guest, leases, replies, command, acknowledge } = yield* setup;
+        const pending = Promise.withResolvers<{ data: string }>();
+        if (stalledStage === "native")
+          guest.capturePage.mockImplementationOnce(() => {
+            guest.nativeStarted.resolve();
+            return pending.promise;
+          });
+        else
+          guest.screenshot.mockImplementationOnce(() => {
+            guest.screenshotStarted.resolve();
+            return pending.promise;
+          });
+        yield* command(1);
+        const active = yield* acknowledge;
+        yield* Effect.promise(
+          () => (stalledStage === "native" ? guest.nativeStarted : guest.screenshotStarted).promise,
+        );
+        yield* TestClock.adjust("8 seconds");
+        expect((yield* Queue.take(replies)).error?.message).toContain("within 8 seconds");
+        expect(yield* Queue.take(leases)).toEqual({ ...active, active: false });
+        yield* command(2);
+        expect((yield* Queue.take(replies)).error?.message).toContain(
+          "previous capture is still pending",
+        );
+        yield* command(3, "DOM.enable");
+        expect((yield* Queue.take(replies)).id).toBe(3);
+        expect(guest.capturePage).toHaveBeenCalledTimes(stalledStage === "cdp" ? 2 : 1);
+        pending.resolve({ data: "late-image" });
+        yield* Effect.promise(() => new Promise((resolve) => setImmediate(resolve)));
+        expect(guest.screenshot).toHaveBeenCalledTimes(1);
+        yield* command(4);
+        yield* acknowledge;
+        expect(yield* Queue.take(replies)).toEqual({ id: 4, result: { data: "encoded-by-cdp" } });
+        expect(guest.capturePage.mock.calls.length).toBeLessThanOrEqual(
+          stalledStage === "cdp" ? 4 : 3,
+        );
+      }).pipe(Effect.scoped),
+  );
+
+  it.effect("retains the pending CDP slot when native capture throws synchronously", () =>
+    Effect.gen(function* () {
+      const { guest, leases, replies, command, acknowledge } = yield* setup;
+      const pending = Promise.withResolvers<{ data: string }>();
+      guest.screenshot.mockImplementationOnce(() => {
+        guest.screenshotStarted.resolve();
+        return pending.promise;
+      });
+      guest.capturePage.mockImplementationOnce(() => {
+        throw new Error("native unavailable");
+      });
+      yield* command(1);
+      yield* acknowledge;
+      yield* Effect.promise(() => guest.screenshotStarted.promise);
+      yield* TestClock.adjust("8 seconds");
+      expect((yield* Queue.take(replies)).error?.message).toContain("within 8 seconds");
+      expect((yield* Queue.take(leases)).active).toBe(false);
+      yield* command(2);
+      expect((yield* Queue.take(replies)).error?.message).toContain(
+        "previous capture is still pending",
+      );
+      expect(guest.screenshot).toHaveBeenCalledTimes(1);
+      pending.resolve({ data: "discarded" });
+      yield* Effect.promise(() => new Promise((resolve) => setImmediate(resolve)));
+      yield* command(3);
+      yield* acknowledge;
+      expect((yield* Queue.take(replies)).result).toEqual({ data: "encoded-by-cdp" });
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect.each(["release", "detach", "replace"] as const)(
+    "releases placement on %s and discards late native work",
+    (lifecycle) =>
+      Effect.gen(function* () {
+        const { host, guest, leases, replies, command, acknowledge } = yield* setup;
+        const pending = Promise.withResolvers<object>();
+        guest.capturePage.mockImplementationOnce(() => {
+          guest.nativeStarted.resolve();
+          return pending.promise;
+        });
+        yield* command(1);
+        const active = yield* acknowledge;
+        yield* Effect.promise(() => guest.nativeStarted.promise);
+        if (lifecycle === "release")
+          yield* host.handleCommandLine(JSON.stringify({ type: "release", ...key }));
+        else if (lifecycle === "detach") host.detach(key);
+        else host.attach(key, makeGuest(42).tab);
+        expect(yield* Queue.take(leases)).toEqual({ ...active, active: false });
+        if (lifecycle === "detach") host.attach(key, guest.tab);
+        pending.resolve({});
+        yield* command(2, "DOM.enable");
+        expect((yield* Queue.take(replies)).id).toBe(2);
+        yield* Effect.promise(() => new Promise((resolve) => setImmediate(resolve)));
+        expect(guest.screenshot).toHaveBeenCalledTimes(1);
+      }).pipe(Effect.scoped),
+  );
+
+  it.effect("leaves child-session screenshots on their own CDP target", () =>
+    Effect.gen(function* () {
+      const { guest, replies, command } = yield* setup;
+      yield* command(1, "Page.captureScreenshot", { format: "png" }, "child-session");
+      expect((yield* Queue.take(replies)).result).toEqual({ data: "encoded-by-cdp" });
+      expect(guest.sendCommand).toHaveBeenCalledExactlyOnceWith(
+        "Page.captureScreenshot",
+        { format: "png" },
+        "child-session",
+      );
+      expect(guest.capturePage).not.toHaveBeenCalled();
+    }).pipe(Effect.scoped),
+  );
+});