From fe7270dbe8130250968e84bfdeaa39b634d14a1a Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Thu, 3 Sep 2026 10:23:16 -0400 Subject: [PATCH 1/2] fix(desktop): require Windows foreground activation --- apps/desktop/package.json | 1 + .../src/electron/ElectronWindow.test.ts | 31 ++++++ apps/desktop/src/electron/ElectronWindow.ts | 5 + .../src/electron/WindowsForeground.test.ts | 93 ++++++++++++++++ .../desktop/src/electron/WindowsForeground.ts | 101 ++++++++++++++++++ docs/internals/windows-window-capture.md | 15 +-- pnpm-lock.yaml | 3 + 7 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 apps/desktop/src/electron/WindowsForeground.test.ts create mode 100644 apps/desktop/src/electron/WindowsForeground.ts diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 80278f66cc8f..0c4798487f30 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -26,6 +26,7 @@ "electron": "44.1.0", "electron-store": "^8.2.0", "electron-updater": "^6.6.2", + "ffi-rs": "1.3.2", "get-windows": "9.3.0", "playwright-core": "1.60.0", "react-grab": "^0.1.32", diff --git a/apps/desktop/src/electron/ElectronWindow.test.ts b/apps/desktop/src/electron/ElectronWindow.test.ts index 0447b82c8685..a5d247a5c02d 100644 --- a/apps/desktop/src/electron/ElectronWindow.test.ts +++ b/apps/desktop/src/electron/ElectronWindow.test.ts @@ -9,6 +9,7 @@ import { beforeEach, vi } from "vite-plus/test"; const { activeWindowMock, + activateWindowsForegroundMock, appFocusMock, browserWindowMock, getAllWindowsMock, @@ -17,6 +18,7 @@ const { nativeAppListMock, } = vi.hoisted(() => ({ activeWindowMock: vi.fn(), + activateWindowsForegroundMock: vi.fn(), appFocusMock: vi.fn(), browserWindowMock: vi.fn(function BrowserWindowMock() {}), getAllWindowsMock: vi.fn(), @@ -27,6 +29,10 @@ const { vi.mock("get-windows", () => ({ activeWindow: activeWindowMock })); +vi.mock("./WindowsForeground.ts", () => ({ + activateWindowsForeground: activateWindowsForegroundMock, +})); + vi.mock("@crowecawcaw/xa11y", () => ({ App: { byPid: nativeAppByPidMock, @@ -78,6 +84,7 @@ function makeWindowsRevealWindow() { describe("ElectronWindow", () => { beforeEach(() => { activeWindowMock.mockReset().mockResolvedValue(undefined); + activateWindowsForegroundMock.mockReset().mockResolvedValue(undefined); appFocusMock.mockReset(); browserWindowMock.mockReset(); getAllWindowsMock.mockReset(); @@ -398,6 +405,30 @@ describe("ElectronWindow", () => { }).pipe(Effect.provide(testLayer("win32"))), ); + it.effect("fails reveal when Windows refuses foreground activation", () => + Effect.gen(function* () { + const cause = new Error("Windows refused foreground activation"); + const window = makeWindowsRevealWindow(); + activateWindowsForegroundMock.mockRejectedValue(cause); + const electronWindow = yield* ElectronWindow.ElectronWindow; + + const exit = yield* Effect.exit( + electronWindow.reveal(window as unknown as Electron.BrowserWindow), + ); + + assert.equal(exit._tag, "Failure"); + if (exit._tag === "Failure") { + const error = Cause.squash(exit.cause); + assert.instanceOf(error, ElectronWindow.ElectronWindowOperationError); + assert.equal(error.operation, "reveal-window"); + assert.strictEqual(error.cause, cause); + } + assert.deepEqual(activateWindowsForegroundMock.mock.calls, [ + [window.getNativeWindowHandle.mock.results[0]?.value], + ]); + }).pipe(Effect.provide(testLayer("win32"))), + ); + it.effect("cancels native focus when destroyed during the foreground query", () => Effect.gen(function* () { const window = makeWindowsRevealWindow(); diff --git a/apps/desktop/src/electron/ElectronWindow.ts b/apps/desktop/src/electron/ElectronWindow.ts index 956fdbb971aa..2bc95c245b5c 100644 --- a/apps/desktop/src/electron/ElectronWindow.ts +++ b/apps/desktop/src/electron/ElectronWindow.ts @@ -11,6 +11,8 @@ import * as Schema from "effect/Schema"; import * as Electron from "electron"; import { activeWindow } from "get-windows"; +import { activateWindowsForeground } from "./WindowsForeground.ts"; + async function isWindowsBrowserWindowForeground(window: Electron.BrowserWindow): Promise { const foreground = await activeWindow().catch(() => undefined); if (window.isDestroyed() || foreground?.owner.processId !== process.pid) return false; @@ -279,6 +281,9 @@ export const make = Effect.gen(function* () { if (platform === "win32") { await focusWindowsBrowserWindow(window).catch(() => undefined); + if (!window.isDestroyed()) { + await activateWindowsForeground(window.getNativeWindowHandle()); + } } }, catch: (cause) => diff --git a/apps/desktop/src/electron/WindowsForeground.test.ts b/apps/desktop/src/electron/WindowsForeground.test.ts new file mode 100644 index 000000000000..522f4da65ab1 --- /dev/null +++ b/apps/desktop/src/electron/WindowsForeground.test.ts @@ -0,0 +1,93 @@ +import { assert, describe, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; + +import { + activateWindowsForegroundWithApi, + type WindowsForegroundApi, +} from "./WindowsForeground.ts"; + +function nativeHandle(value: bigint, bytes = 8): Buffer { + const handle = Buffer.alloc(bytes); + if (bytes === 8) handle.writeBigUInt64LE(value); + else handle.writeUInt32LE(Number(value)); + return handle; +} + +function makeApi(input: { + readonly foregroundWindow?: bigint; + readonly currentThreadId?: number; + readonly foregroundThreadId?: number; + readonly attached?: boolean; + readonly activated?: boolean; +}) { + return { + getCurrentThreadId: vi.fn(() => input.currentThreadId ?? 10), + getForegroundWindow: vi.fn(() => input.foregroundWindow ?? 99n), + getWindowThreadId: vi.fn(() => input.foregroundThreadId ?? 20), + attachThreadInput: vi.fn((_source, _target, attach) => + attach ? (input.attached ?? true) : true, + ), + setForegroundWindow: vi.fn(() => input.activated ?? true), + } satisfies WindowsForegroundApi; +} + +describe("Windows foreground activation", () => { + it.each([4, 8])("activates a %i-byte native window handle", (bytes) => { + const api = makeApi({}); + const handle = nativeHandle(41n, bytes); + + assert.isTrue(activateWindowsForegroundWithApi(handle, api)); + + assert.deepEqual(api.attachThreadInput.mock.calls, [ + [10, 20, true], + [10, 20, false], + ]); + assert.deepEqual(api.setForegroundWindow.mock.calls, [[41n]]); + }); + + it("does not disturb input queues when T3 is already foreground", () => { + const api = makeApi({ foregroundWindow: 41n }); + + assert.isTrue(activateWindowsForegroundWithApi(nativeHandle(41n), api)); + + assert.lengthOf(api.getCurrentThreadId.mock.calls, 0); + assert.lengthOf(api.attachThreadInput.mock.calls, 0); + assert.lengthOf(api.setForegroundWindow.mock.calls, 0); + }); + + it("does not attach a thread to itself", () => { + const api = makeApi({ currentThreadId: 20, foregroundThreadId: 20 }); + + assert.isTrue(activateWindowsForegroundWithApi(nativeHandle(41n), api)); + + assert.lengthOf(api.attachThreadInput.mock.calls, 0); + assert.deepEqual(api.setForegroundWindow.mock.calls, [[41n]]); + }); + + it("uses SetForegroundWindow's result as the activation receipt", () => { + const api = makeApi({ activated: false }); + + assert.isFalse(activateWindowsForegroundWithApi(nativeHandle(41n), api)); + + assert.deepEqual(api.attachThreadInput.mock.calls, [ + [10, 20, true], + [10, 20, false], + ]); + }); + + it("still asks Windows directly when the input queues cannot be attached", () => { + const api = makeApi({ attached: false }); + + assert.isTrue(activateWindowsForegroundWithApi(nativeHandle(41n), api)); + + assert.deepEqual(api.attachThreadInput.mock.calls, [[10, 20, true]]); + assert.deepEqual(api.setForegroundWindow.mock.calls, [[41n]]); + }); + + it("rejects malformed native handles", () => { + const api = makeApi({}); + + assert.throws(() => activateWindowsForegroundWithApi(Buffer.alloc(6), api)); + assert.lengthOf(api.getForegroundWindow.mock.calls, 0); + }); +}); diff --git a/apps/desktop/src/electron/WindowsForeground.ts b/apps/desktop/src/electron/WindowsForeground.ts new file mode 100644 index 000000000000..045eb30b866e --- /dev/null +++ b/apps/desktop/src/electron/WindowsForeground.ts @@ -0,0 +1,101 @@ +export interface WindowsForegroundApi { + readonly getCurrentThreadId: () => number; + readonly getForegroundWindow: () => bigint; + readonly getWindowThreadId: (windowHandle: bigint) => number; + readonly attachThreadInput: ( + sourceThreadId: number, + targetThreadId: number, + attach: boolean, + ) => boolean; + readonly setForegroundWindow: (windowHandle: bigint) => boolean; +} + +function nativeWindowHandle(buffer: Buffer): bigint { + if (buffer.length === 8) return buffer.readBigUInt64LE(); + if (buffer.length === 4) return BigInt(buffer.readUInt32LE()); + throw new Error(`Unsupported Windows window handle size: ${String(buffer.length)} bytes.`); +} + +export function activateWindowsForegroundWithApi( + handleBuffer: Buffer, + api: WindowsForegroundApi, +): boolean { + const targetWindow = nativeWindowHandle(handleBuffer); + const foregroundWindow = api.getForegroundWindow(); + if (targetWindow === foregroundWindow) return true; + + const currentThreadId = api.getCurrentThreadId(); + const foregroundThreadId = foregroundWindow === 0n ? 0 : api.getWindowThreadId(foregroundWindow); + const shouldAttach = + currentThreadId !== 0 && foregroundThreadId !== 0 && currentThreadId !== foregroundThreadId; + const attached = shouldAttach && api.attachThreadInput(currentThreadId, foregroundThreadId, true); + + try { + return api.setForegroundWindow(targetWindow); + } finally { + if (attached) { + api.attachThreadInput(currentThreadId, foregroundThreadId, false); + } + } +} + +let windowsForegroundApiPromise: Promise | undefined; + +function loadWindowsForegroundApi(): Promise { + windowsForegroundApiPromise ??= import("ffi-rs").then(({ DataType, load, open }) => { + const kernel32 = "t3-kernel32"; + const user32 = "t3-user32"; + open({ library: kernel32, path: "kernel32.dll" }); + open({ library: user32, path: "user32.dll" }); + + return { + getCurrentThreadId: () => + load({ + library: kernel32, + funcName: "GetCurrentThreadId", + retType: DataType.U32, + paramsType: [], + paramsValue: [], + }), + getForegroundWindow: () => + load({ + library: user32, + funcName: "GetForegroundWindow", + retType: DataType.BigInt, + paramsType: [], + paramsValue: [], + }) as bigint, + getWindowThreadId: (windowHandle) => + load({ + library: user32, + funcName: "GetWindowThreadProcessId", + retType: DataType.U32, + paramsType: [DataType.BigInt, DataType.BigInt], + paramsValue: [windowHandle, 0n], + }), + attachThreadInput: (sourceThreadId, targetThreadId, attach) => + load({ + library: user32, + funcName: "AttachThreadInput", + retType: DataType.Boolean, + paramsType: [DataType.U32, DataType.U32, DataType.Boolean], + paramsValue: [sourceThreadId, targetThreadId, attach], + }), + setForegroundWindow: (windowHandle) => + load({ + library: user32, + funcName: "SetForegroundWindow", + retType: DataType.Boolean, + paramsType: [DataType.BigInt], + paramsValue: [windowHandle], + }), + } satisfies WindowsForegroundApi; + }); + return windowsForegroundApiPromise; +} + +export async function activateWindowsForeground(handleBuffer: Buffer): Promise { + const api = await loadWindowsForegroundApi(); + if (activateWindowsForegroundWithApi(handleBuffer, api)) return; + throw new Error("Windows refused to activate the T3 Code window."); +} diff --git a/docs/internals/windows-window-capture.md b/docs/internals/windows-window-capture.md index 6afad5d61301..f73913e573c7 100644 --- a/docs/internals/windows-window-capture.md +++ b/docs/internals/windows-window-capture.md @@ -43,13 +43,14 @@ the window active even when Windows denies its `SetForegroundWindow` request; se [`HWNDMessageHandler::Activate` and `IsActive`](https://github.com/chromium/chromium/blob/152.0.7977.65/ui/views/win/hwnd_message_handler.cc). Do not use `isFocused()` to skip the native activation fallback. -After Electron's normal show/focus calls, compare `get-windows.activeWindow()` with the destination's -native HWND and process ID. If it is still behind another window, find its xa11y UI Automation -element by process, title, and logical bounds, and await `IUIAutomationElement::SetFocus` before -dispatching the capture. xa11y already converts its UIA bounding rectangle to logical coordinates. -Recheck foreground ownership after asynchronous enumeration and avoid selecting another window -belonging to the same Electron process. Verify modifier-pair shortcuts as well as Electron global -key chords: they arrive through different native input paths. +UI Automation's `IUIAutomationElement::SetFocus` can focus an element without making its top-level +window visible or foreground. It remains a compatibility attempt, not the handoff receipt. After +Electron's normal show/focus calls, Windows reveal attaches T3's UI thread to the current foreground +thread's input queue, calls `SetForegroundWindow` for T3's native HWND, and detaches the queues. The +capture-started event is dispatched only when that call reports success, so a rejected activation +cannot start an animation behind another app. This is the activation operation's native result, not +a later foreground-window query. Verify modifier-pair shortcuts as well as Electron global key +chords: they arrive through different native input paths. `WindowCaptureTransition.ts` presents the frozen screenshot in transparent, non-activating Electron windows. Each overlay keeps its native bounds fixed; screenshot movement, scaling, cropping, and diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6a1a33a8ae9c..6567c1b5fde0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,6 +174,9 @@ importers: electron-updater: specifier: ^6.6.2 version: 6.8.3 + ffi-rs: + specifier: 1.3.2 + version: 1.3.2 get-windows: specifier: 9.3.0 version: 9.3.0(encoding@0.1.13) From 6186ab00ca90d7dc64e4734c14bdc62cff4ac2cd Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Thu, 3 Sep 2026 12:20:52 -0400 Subject: [PATCH 2/2] perf(desktop): speed up Windows capture reveal --- .../src/electron/ElectronWindow.test.ts | 53 ++++++++++++++- apps/desktop/src/electron/ElectronWindow.ts | 8 ++- .../DesktopWindowCapture.test.ts | 50 ++++++++++---- .../src/windowCapture/DesktopWindowCapture.ts | 36 +++++----- .../WindowCaptureAccessibilityProcess.test.ts | 65 ++++++++++++++++--- .../WindowCaptureAccessibilityProcess.ts | 60 ++++++++++++++++- .../WindowCaptureAccessibilityWorker.ts | 2 + 7 files changed, 234 insertions(+), 40 deletions(-) diff --git a/apps/desktop/src/electron/ElectronWindow.test.ts b/apps/desktop/src/electron/ElectronWindow.test.ts index a5d247a5c02d..9d5451eb9669 100644 --- a/apps/desktop/src/electron/ElectronWindow.test.ts +++ b/apps/desktop/src/electron/ElectronWindow.test.ts @@ -226,7 +226,30 @@ describe("ElectronWindow", () => { }).pipe(Effect.provide(TestLayer)), ); - it.effect("awaits native foreground focus even when Electron reports the window focused", () => + it.effect("uses native Windows activation without starting the accessibility fallback", () => + Effect.gen(function* () { + const operations: Array = []; + appFocusMock.mockImplementation(() => operations.push("app-focus")); + activateWindowsForegroundMock.mockImplementation(async () => { + operations.push("native-activation"); + }); + const window = { + ...makeWindowsRevealWindow(), + show: vi.fn(() => operations.push("show")), + moveTop: vi.fn(() => operations.push("move-top")), + focus: vi.fn(() => operations.push("focus")), + } as unknown as Electron.BrowserWindow; + const electronWindow = yield* ElectronWindow.ElectronWindow; + + yield* electronWindow.reveal(window); + + assert.deepEqual(operations, ["app-focus", "show", "move-top", "focus", "native-activation"]); + assert.lengthOf(activeWindowMock.mock.calls, 0); + assert.lengthOf(nativeAppListMock.mock.calls, 0); + }).pipe(Effect.provide(testLayer("win32"))), + ); + + it.effect("falls back to accessibility before retrying native Windows activation", () => Effect.gen(function* () { const operations: Array = []; const nativeFocusStarted = Promise.withResolvers(); @@ -240,6 +263,11 @@ describe("ElectronWindow", () => { }> >(); appFocusMock.mockImplementation(() => operations.push("app-focus")); + activateWindowsForegroundMock + .mockRejectedValueOnce(new Error("Windows initially refused foreground activation")) + .mockImplementationOnce(async () => { + operations.push("native-activation"); + }); nativeAppListMock.mockImplementation(() => { listingStarted.resolve(); return listedApps.promise; @@ -314,8 +342,10 @@ describe("ElectronWindow", () => { "move-top", "focus", "native-focus", + "native-activation", "revealed", ]); + assert.lengthOf(activateWindowsForegroundMock.mock.calls, 2); assert.lengthOf(nativeAppByPidMock.mock.calls, 0); assert.lengthOf(readNativeBounds.mock.calls, 1); }).pipe(Effect.provide(testLayer("win32"))), @@ -326,6 +356,9 @@ describe("ElectronWindow", () => { (handleBytes) => Effect.gen(function* () { const window = makeWindowsRevealWindow(); + activateWindowsForegroundMock.mockRejectedValueOnce( + new Error("Windows initially refused foreground activation"), + ); const hwnd = handleBytes === 4 ? 0xf123_4567 : 0x1_f123_4567; const handle = Buffer.alloc(handleBytes); if (handleBytes === 4) handle.writeUInt32LE(hwnd); @@ -348,6 +381,9 @@ describe("ElectronWindow", () => { Effect.gen(function* () { const window = makeWindowsRevealWindow(); const focus = vi.fn(async () => undefined); + activateWindowsForegroundMock.mockRejectedValueOnce( + new Error("Windows initially refused foreground activation"), + ); activeWindowMock.mockResolvedValue({ id: foreground.id, owner: { processId: foreground.processId }, @@ -370,6 +406,9 @@ describe("ElectronWindow", () => { Effect.gen(function* () { const window = makeWindowsRevealWindow(); const focus = vi.fn(async () => undefined); + activateWindowsForegroundMock.mockRejectedValueOnce( + new Error("Windows initially refused foreground activation"), + ); activeWindowMock.mockRejectedValue(new Error("Foreground query unavailable")); nativeAppListMock.mockResolvedValue([ { @@ -391,6 +430,9 @@ describe("ElectronWindow", () => { const focus = vi.fn(async () => { throw new Error("Focus rejected"); }); + activateWindowsForegroundMock.mockRejectedValueOnce( + new Error("Windows initially refused foreground activation"), + ); nativeAppListMock.mockResolvedValue([ { pid: process.pid, @@ -425,6 +467,7 @@ describe("ElectronWindow", () => { } assert.deepEqual(activateWindowsForegroundMock.mock.calls, [ [window.getNativeWindowHandle.mock.results[0]?.value], + [window.getNativeWindowHandle.mock.results[1]?.value], ]); }).pipe(Effect.provide(testLayer("win32"))), ); @@ -432,6 +475,9 @@ describe("ElectronWindow", () => { it.effect("cancels native focus when destroyed during the foreground query", () => Effect.gen(function* () { const window = makeWindowsRevealWindow(); + activateWindowsForegroundMock.mockRejectedValueOnce( + new Error("Windows initially refused foreground activation"), + ); const queryStarted = Promise.withResolvers(); const foreground = Promise.withResolvers(); activeWindowMock.mockImplementation(() => { @@ -449,7 +495,7 @@ describe("ElectronWindow", () => { yield* Fiber.join(revealFiber); assert.lengthOf(nativeAppListMock.mock.calls, 0); - assert.lengthOf(window.getNativeWindowHandle.mock.calls, 0); + assert.lengthOf(window.getNativeWindowHandle.mock.calls, 1); assert.lengthOf(window.getTitle.mock.calls, 0); }).pipe(Effect.provide(testLayer("win32"))), ); @@ -467,6 +513,9 @@ describe("ElectronWindow", () => { return listedApps.promise; }); const window = makeWindowsRevealWindow(); + activateWindowsForegroundMock.mockRejectedValueOnce( + new Error("Windows initially refused foreground activation"), + ); const electronWindow = yield* ElectronWindow.ElectronWindow; const revealFiber = yield* electronWindow diff --git a/apps/desktop/src/electron/ElectronWindow.ts b/apps/desktop/src/electron/ElectronWindow.ts index 2bc95c245b5c..d4cd8b63375c 100644 --- a/apps/desktop/src/electron/ElectronWindow.ts +++ b/apps/desktop/src/electron/ElectronWindow.ts @@ -280,9 +280,13 @@ export const make = Effect.gen(function* () { window.focus(); if (platform === "win32") { - await focusWindowsBrowserWindow(window).catch(() => undefined); - if (!window.isDestroyed()) { + try { await activateWindowsForeground(window.getNativeWindowHandle()); + } catch { + await focusWindowsBrowserWindow(window).catch(() => undefined); + if (!window.isDestroyed()) { + await activateWindowsForeground(window.getNativeWindowHandle()); + } } } }, diff --git a/apps/desktop/src/windowCapture/DesktopWindowCapture.test.ts b/apps/desktop/src/windowCapture/DesktopWindowCapture.test.ts index 1c1e3c1e7425..7e34e1df8b0a 100644 --- a/apps/desktop/src/windowCapture/DesktopWindowCapture.test.ts +++ b/apps/desktop/src/windowCapture/DesktopWindowCapture.test.ts @@ -31,8 +31,9 @@ beforeEach(() => { const { activeWindowMock, animationSettingsMock, - accessibilityProcessStartMock, + accessibilityProcessWarmMock, accessibilityProcessCloseMock, + accessibilityProcessCoolMock, accessibilityProcessReadMock, accessibilityByPidMock, accessibilityForegroundMock, @@ -70,8 +71,9 @@ const { prefersReducedMotion: true, shouldRenderRichAnimation: false, })), - accessibilityProcessStartMock: vi.fn(), + accessibilityProcessWarmMock: vi.fn(), accessibilityProcessCloseMock: vi.fn(), + accessibilityProcessCoolMock: vi.fn(), accessibilityProcessReadMock: vi.fn< (request: import("./WindowCaptureAccessibility.ts").WindowCaptureAccessibilityRequest) => { started: Promise; @@ -168,13 +170,12 @@ vi.mock("@crowecawcaw/xa11y", () => { }; }); vi.mock("./WindowCaptureAccessibilityProcess.ts", () => ({ - startWindowCaptureAccessibilityProcess: () => { - accessibilityProcessStartMock(); - return { - read: accessibilityProcessReadMock, - close: accessibilityProcessCloseMock, - }; - }, + makeWindowCaptureAccessibilityProcessPool: () => ({ + warm: accessibilityProcessWarmMock, + cool: accessibilityProcessCoolMock, + read: accessibilityProcessReadMock, + close: accessibilityProcessCloseMock, + }), })); vi.mock("get-windows", () => ({ activeWindow: activeWindowMock })); vi.mock("./WindowsCaptureFeedback.ts", () => ({ @@ -1261,7 +1262,7 @@ it.effect("skips accessibility capture when the setting is disabled", () => { } as const; activeWindowMock.mockReset().mockResolvedValue(active); screenshotMock.mockReset().mockResolvedValue({ width: 800, height: 600, toPng: () => png }); - accessibilityProcessStartMock.mockClear(); + accessibilityProcessWarmMock.mockClear(); accessibilityProcessReadMock.mockClear(); accessibilityByPidMock.mockClear(); const layer = testLayer("win32", { @@ -1280,13 +1281,40 @@ it.effect("skips accessibility capture when the setting is disabled", () => { }); yield* service.captureNow; - assert.lengthOf(accessibilityProcessStartMock.mock.calls, 0); + assert.lengthOf(accessibilityProcessWarmMock.mock.calls, 0); assert.lengthOf(accessibilityProcessReadMock.mock.calls, 0); assert.lengthOf(accessibilityByPidMock.mock.calls, 0); }), ).pipe(Effect.provide(layer)); }); +it.effect("keeps an accessibility helper warm only while capture data is enabled", () => { + accessibilityProcessWarmMock.mockClear(); + accessibilityProcessCoolMock.mockClear(); + + return Effect.scoped( + Effect.gen(function* () { + const service = yield* DesktopWindowCapture.make; + yield* service.configure({ + ...DEFAULT_CLIENT_SETTINGS, + windowCaptureEnabled: true, + windowCaptureIncludeAccessibility: true, + }); + + assert.lengthOf(accessibilityProcessWarmMock.mock.calls, 1); + assert.lengthOf(accessibilityProcessCoolMock.mock.calls, 0); + + yield* service.configure({ + ...DEFAULT_CLIENT_SETTINGS, + windowCaptureEnabled: false, + windowCaptureIncludeAccessibility: true, + }); + + assert.lengthOf(accessibilityProcessCoolMock.mock.calls, 1); + }), + ).pipe(Effect.provide(testLayer("win32"))); +}); + it.effect("rejects X11 capture without registering shortcuts or loading capture backends", () => { vi.stubEnv("XDG_SESSION_TYPE", "x11"); vi.stubEnv("WAYLAND_DISPLAY", ""); diff --git a/apps/desktop/src/windowCapture/DesktopWindowCapture.ts b/apps/desktop/src/windowCapture/DesktopWindowCapture.ts index 6bdb96a9ea4b..a30b933a742f 100644 --- a/apps/desktop/src/windowCapture/DesktopWindowCapture.ts +++ b/apps/desktop/src/windowCapture/DesktopWindowCapture.ts @@ -68,7 +68,10 @@ import { type WindowCaptureAnimationDestination, WindowCaptureTransition, } from "./WindowCaptureTransition.ts"; -import { startWindowCaptureAccessibilityProcess } from "./WindowCaptureAccessibilityProcess.ts"; +import { + type AccessibilityProcessPool, + makeWindowCaptureAccessibilityProcessPool, +} from "./WindowCaptureAccessibilityProcess.ts"; import { showWindowsCaptureOverlay } from "./WindowsCaptureFeedback.ts"; import { @@ -363,7 +366,7 @@ async function captureSource({ linuxAppId, kdeCapturePaths, hyprlandCapturePaths, - accessibilityWorkerPath, + accessibilityProcessPool, onLinuxFeedback, }: { target: WindowCaptureTarget; @@ -377,7 +380,7 @@ async function captureSource({ linuxAppId: string; kdeCapturePaths: KdeCapturePaths; hyprlandCapturePaths: HyprlandCapturePaths; - accessibilityWorkerPath: string; + accessibilityProcessPool: AccessibilityProcessPool; onLinuxFeedback: (feedback: LinuxCaptureFeedback) => void; }) { let active: ActiveWindow | undefined; @@ -390,10 +393,6 @@ async function captureSource({ focusedWindow ?? Electron.BrowserWindow.getAllWindows().find((window) => !window.isDestroyed()); const destinationWindowBounds = destinationWindow?.getBounds(); let hiddenWindowRestored = false; - const accessibilityProcess = settings.windowCaptureIncludeAccessibility - ? startWindowCaptureAccessibilityProcess(accessibilityWorkerPath) - : undefined; - let accessibilityProcessOwned = accessibilityProcess !== undefined; try { if (hiddenWindow) await hideAndWaitForBlur(hiddenWindow); if (mode === "direct") { @@ -474,8 +473,8 @@ async function captureSource({ } : undefined; const accessibilityRead = - accessibleIdentity && accessibilityProcess - ? accessibilityProcess.read({ + accessibleIdentity && settings.windowCaptureIncludeAccessibility + ? accessibilityProcessPool.read({ active: accessibleIdentity, platform, sourceTitle: source.name, @@ -483,11 +482,7 @@ async function captureSource({ }) : undefined; if (accessibilityRead) { - accessibilityProcessOwned = false; await accessibilityRead.started; - } else { - accessibilityProcess?.close(); - accessibilityProcessOwned = false; } const contextPromise = accessibilityRead?.result ?? Promise.resolve(undefined); if (platform !== "win32" && hiddenWindow && !hiddenWindow.isDestroyed()) { @@ -528,7 +523,6 @@ async function captureSource({ imageTempReady, }; } finally { - if (accessibilityProcessOwned) accessibilityProcess?.close(); if (!hiddenWindowRestored && hiddenWindow && !hiddenWindow.isDestroyed()) hiddenWindow.show(); } } @@ -747,6 +741,8 @@ export const make = Effect.gen(function* () { "windowCapture", "WindowCaptureAccessibilityWorker.cjs", ); + const accessibilityProcessPool = + makeWindowCaptureAccessibilityProcessPool(accessibilityWorkerPath); let registeredAccelerator: string | undefined; let portalShortcut: PortalCaptureShortcut | undefined; let shortcutGeneration = 0; @@ -859,7 +855,7 @@ export const make = Effect.gen(function* () { linuxAppId, kdeCapturePaths, hyprlandCapturePaths, - accessibilityWorkerPath, + accessibilityProcessPool, onLinuxFeedback: (feedback) => { linuxFeedback = { id, feedback }; }, @@ -1070,6 +1066,15 @@ export const make = Effect.gen(function* () { const mode = captureMode(environment.platform); const shortcut = settings.windowCaptureShortcut; + if ( + settings.windowCaptureEnabled && + settings.windowCaptureIncludeAccessibility && + mode !== "unavailable" + ) { + accessibilityProcessPool.warm(); + } else { + accessibilityProcessPool.cool(); + } if (!settings.windowCaptureEnabled || !settings.windowCaptureFlash || mode === "unavailable") { flash.dispose(); } @@ -1369,6 +1374,7 @@ export const make = Effect.gen(function* () { flash.dispose(); transition.dispose(); closeLinuxFeedback(); + accessibilityProcessPool.close(); }), ); diff --git a/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.test.ts b/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.test.ts index 658e2177d72c..a1c32b0cb9d4 100644 --- a/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.test.ts +++ b/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.test.ts @@ -10,12 +10,17 @@ const forkMock = vi.hoisted(() => vi.mock("node:child_process", () => ({ fork: forkMock })); -import { startWindowCaptureAccessibilityProcess } from "./WindowCaptureAccessibilityProcess.ts"; +import { + makeWindowCaptureAccessibilityProcessPool, + startWindowCaptureAccessibilityProcess, +} from "./WindowCaptureAccessibilityProcess.ts"; -const worker = Object.assign(new NodeEvents.EventEmitter(), { - kill: vi.fn(() => true), - send: vi.fn(), -}); +const makeWorker = () => + Object.assign(new NodeEvents.EventEmitter(), { + kill: vi.fn(() => true), + send: vi.fn(), + }); +let worker = makeWorker(); const request = { active: { title: "Zoom Meeting", @@ -28,9 +33,7 @@ const request = { }; beforeEach(() => { - worker.removeAllListeners(); - worker.kill.mockClear(); - worker.send.mockClear(); + worker = makeWorker(); forkMock.mockReset().mockReturnValue(worker); }); @@ -66,3 +69,49 @@ it("returns accessibility extracted by the helper", async () => { assert.deepEqual(await read.result, context); assert.strictEqual(forkMock.mock.calls[0]?.[2]?.env?.ELECTRON_RUN_AS_NODE, "1"); }); + +it("hands a warm helper to the capture and prewarms its replacement", async () => { + const warmWorker = makeWorker(); + const replacementWorker = makeWorker(); + forkMock.mockReset().mockReturnValueOnce(warmWorker).mockReturnValueOnce(replacementWorker); + const pool = makeWindowCaptureAccessibilityProcessPool("accessibility.cjs"); + + pool.warm(); + warmWorker.emit("message", "ready"); + const read = pool.read(request); + + assert.lengthOf(forkMock.mock.calls, 2); + assert.deepEqual(warmWorker.send.mock.calls[0]?.[0], request); + assert.lengthOf(replacementWorker.send.mock.calls, 0); + + warmWorker.emit("message", "started"); + warmWorker.emit("message", { type: "result", context: { accessibleText: "Ready" } }); + assert.deepEqual(await read.result, { accessibleText: "Ready" }); + + pool.close(); + assert.lengthOf(replacementWorker.kill.mock.calls, 1); +}); + +it("replaces a warm helper that exits before capture", async () => { + const exitedWorker = makeWorker(); + const captureWorker = makeWorker(); + const replacementWorker = makeWorker(); + forkMock + .mockReset() + .mockReturnValueOnce(exitedWorker) + .mockReturnValueOnce(captureWorker) + .mockReturnValueOnce(replacementWorker); + const pool = makeWindowCaptureAccessibilityProcessPool("accessibility.cjs"); + + pool.warm(); + exitedWorker.emit("exit", 1); + const read = pool.read(request); + captureWorker.emit("message", "ready"); + captureWorker.emit("message", "started"); + captureWorker.emit("message", { type: "result", context: { accessibleText: "Recovered" } }); + + assert.deepEqual(await read.result, { accessibleText: "Recovered" }); + assert.lengthOf(forkMock.mock.calls, 3); + assert.deepEqual(captureWorker.send.mock.calls[0]?.[0], request); + pool.close(); +}); diff --git a/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.ts b/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.ts index 140a8eec6196..c34292ac02f7 100644 --- a/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.ts +++ b/apps/desktop/src/windowCapture/WindowCaptureAccessibilityProcess.ts @@ -11,12 +11,20 @@ import type { const START_TIMEOUT_MS = 1_000; const RESULT_TIMEOUT_MS = 4_000; -type AccessibilityRead = { +export type AccessibilityRead = { readonly started: Promise; readonly result: Promise; }; -type AccessibilityProcess = { +export type AccessibilityProcess = { + readonly alive: boolean; + readonly read: (request: WindowCaptureAccessibilityRequest) => AccessibilityRead; + readonly close: () => void; +}; + +export type AccessibilityProcessPool = { + readonly warm: () => void; + readonly cool: () => void; readonly read: (request: WindowCaptureAccessibilityRequest) => AccessibilityRead; readonly close: () => void; }; @@ -32,6 +40,7 @@ const completedRead = (context?: CapturedWindowAccessibilityContext): Accessibil }); const unavailableProcess = (): AccessibilityProcess => ({ + alive: false, read: () => completedRead(), close: () => undefined, }); @@ -100,6 +109,9 @@ export function startWindowCaptureAccessibilityProcess(workerPath: string): Acce worker.once("exit", () => finish()); return { + get alive() { + return !settled; + }, read: (nextRequest) => { if (settled) return completedRead(settledContext); request = nextRequest; @@ -117,3 +129,47 @@ export function startWindowCaptureAccessibilityProcess(workerPath: string): Acce close: () => finish(), }; } + +export function makeWindowCaptureAccessibilityProcessPool( + workerPath: string, +): AccessibilityProcessPool { + let standby: AccessibilityProcess | undefined; + const active = new Set(); + let closed = false; + + const warm = () => { + if (closed || standby?.alive) return; + standby = startWindowCaptureAccessibilityProcess(workerPath); + }; + const cool = () => { + standby?.close(); + standby = undefined; + }; + + return { + warm, + cool, + read: (request) => { + if (closed) return completedRead(); + const workerProcess = standby?.alive + ? standby + : startWindowCaptureAccessibilityProcess(workerPath); + standby = undefined; + warm(); + active.add(workerProcess); + const read = workerProcess.read(request); + void read.result.then( + () => active.delete(workerProcess), + () => active.delete(workerProcess), + ); + return read; + }, + close: () => { + if (closed) return; + closed = true; + cool(); + for (const workerProcess of active) workerProcess.close(); + active.clear(); + }, + }; +} diff --git a/apps/desktop/src/windowCapture/WindowCaptureAccessibilityWorker.ts b/apps/desktop/src/windowCapture/WindowCaptureAccessibilityWorker.ts index 3f62d352e6fa..34a568498907 100644 --- a/apps/desktop/src/windowCapture/WindowCaptureAccessibilityWorker.ts +++ b/apps/desktop/src/windowCapture/WindowCaptureAccessibilityWorker.ts @@ -3,6 +3,8 @@ import { type WindowCaptureAccessibilityRequest, } from "./WindowCaptureAccessibility.ts"; +process.once("disconnect", () => process.exit(0)); + async function readAccessibility() { const { App } = await import("@crowecawcaw/xa11y"); process.send?.("ready");