diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 920dcd12e..45247bdbc 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1,3 +1,4 @@ +import * as NodeVM from "node:vm"; import { it as effectIt } from "@effect/vitest"; import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts"; import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; @@ -204,6 +205,7 @@ const { vi.mock("electron", () => ({ BrowserWindow: browserWindowConstructor, + // eslint-disable-next-line @typescript-eslint/no-extraneous-class -- Electron constructs this native boundary. ClipboardItem: class { constructor(data: Record) { clipboardItemConstructor(data); @@ -3997,29 +3999,92 @@ describe("PreviewManager", () => { withManager((manager) => Effect.gen(function* () { let failKeyDown = false; + let routeToIframe = false; + let interruptFrameKeyDown = false; + let holdKeyUp = false; + let releaseKeyUp: (() => void) | undefined; + let notifyKeyUpQueued: (() => void) | undefined; + const keyUpQueued = new Promise((resolve) => { + notifyKeyUpQueued = resolve; + }); + const listeners = new Map void>(); + const eventCounts = new Map(); + const animationFrames = new Map void>(); + let animationFrameId = 0; + const renderFrame = () => { + const callbacks = [...animationFrames.values()]; + animationFrames.clear(); + callbacks.forEach((callback) => callback()); + }; + const frameContext = NodeVM.createContext({ + performance: { eventCounts }, + requestAnimationFrame: (callback: () => void) => { + const id = ++animationFrameId; + animationFrames.set(id, callback); + return id; + }, + cancelAnimationFrame: (id: number) => animationFrames.delete(id), + window: { + addEventListener: (type: string, listener: (event: unknown) => void) => + listeners.set(type, listener), + removeEventListener: (type: string) => listeners.delete(type), + }, + }); + const frame = { + executeJavaScript: vi.fn(async (expression: string) => + NodeVM.runInContext(expression, frameContext), + ), + }; let humanInput: ((_event: unknown, signal: unknown) => void) | undefined; - const sendCommand = vi.fn(async (method: string, params?: Record) => { - if ( - failKeyDown && - method === "Input.dispatchKeyEvent" && - (params?.["type"] === "keyDown" || params?.["type"] === "rawKeyDown") - ) { - throw new Error("key dispatch failed"); - } - if ( - method === "Input.dispatchKeyEvent" && - (params?.["type"] === "keyDown" || params?.["type"] === "rawKeyDown") - ) { - humanInput?.( - {}, - { - kind: "key", - key: params["key"], - code: params["code"] ?? "Digit1", - }, - ); + const sendCommand = vi.fn( + async (method: string, params?: Record, sessionId?: string) => { + if (method === "Runtime.evaluate") { + if (params?.["returnByValue"] === true) return { result: { value: { ok: true } } }; + return { + result: + routeToIframe && !sessionId + ? { objectId: "focused-iframe-object" } + : { subtype: "null" }, + }; + } + if (method === "DOM.describeNode") return { node: { frameId: "focused-frame" } }; + if (method === "Target.getTargets") + return { + targetInfos: [ + { targetId: "unrelated-frame", type: "iframe" }, + { targetId: "focused-frame", type: "iframe" }, + ], + }; + if (method === "Target.attachToTarget") return { sessionId: "child-session" }; + if (method === "Input.dispatchKeyEvent" && params?.["type"] !== "keyUp") { + if (failKeyDown) throw new Error("key dispatch failed"); + if (interruptFrameKeyDown) + humanInput?.({}, { kind: "pointer", x: 80, y: 40, button: 0 }); + } + return undefined; + }, + ); + const sendInputEvent = vi.fn((input: Electron.KeyboardInputEvent) => { + const signal = { + kind: "key", + key: input.keyCode, + code: input.keyCode === "!" ? "Digit1" : `Key${input.keyCode.toUpperCase()}`, + }; + if (input.type === "keyUp") { + const deliver = () => { + eventCounts.set("keyup", (eventCounts.get("keyup") ?? 0) + 1); + renderFrame(); + }; + if (holdKeyUp) { + releaseKeyUp = deliver; + notifyKeyUpQueued?.(); + } else { + queueMicrotask(deliver); + } } - return method === "Runtime.evaluate" ? { result: { value: { ok: true } } } : undefined; + if (input.type !== "keyDown") return; + if (failKeyDown) throw new Error("key dispatch failed"); + humanInput?.({}, signal); }); const restoreFocus = vi.fn(); const focus = vi.fn(); @@ -4030,6 +4095,7 @@ describe("PreviewManager", () => { } as never); fromId.mockReturnValue({ id: 42, + mainFrame: { framesInSubtree: [frame] }, isDestroyed: () => false, getType: () => "webview", getURL: () => "https://example.com", @@ -4037,6 +4103,7 @@ describe("PreviewManager", () => { isLoading: () => false, isDevToolsOpened: () => false, focus, + sendInputEvent, getZoomFactor: () => 1, setZoomFactor: vi.fn(), setAudioMuted: vi.fn(), @@ -4075,13 +4142,6 @@ describe("PreviewManager", () => { ([method, params]) => method === "Emulation.setFocusEmulationEnabled" && params?.["enabled"] === true, ); - const keyDownIndex = calls.findIndex( - ([method, params]) => - method === "Input.dispatchKeyEvent" && params?.["type"] === "keyDown", - ); - const keyUpIndex = calls.findIndex( - ([method, params]) => method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp", - ); const focusOffIndex = calls.findIndex( ([method, params]) => method === "Emulation.setFocusEmulationEnabled" && params?.["enabled"] === false, @@ -4109,61 +4169,102 @@ describe("PreviewManager", () => { expect(clearOnlyEvaluation).toBeDefined(); expect(methods).not.toContain("Input.insertText"); expect(enableIndex).toBeGreaterThanOrEqual(0); - expect(focus).toHaveBeenCalledOnce(); - expect(restoreFocus).toHaveBeenCalledOnce(); - expect(methods).toContain("Page.bringToFront"); + expect(methods).not.toContain("Page.bringToFront"); + expect(methods).not.toContain("Input.dispatchKeyEvent"); expect(enableIndex).toBeLessThan(focusOnIndex); - expect(focusOnIndex).toBeLessThan(keyDownIndex); - expect(keyDownIndex).toBeLessThan(keyUpIndex); - expect(keyUpIndex).toBeLessThan(focusOffIndex); - expect( - calls.filter( - ([method, params]) => - method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp", - ), - ).toHaveLength(1); + expect(sendCommand.mock.invocationCallOrder[focusOnIndex]).toBeLessThan( + sendInputEvent.mock.invocationCallOrder[0]!, + ); + expect(sendInputEvent.mock.invocationCallOrder[2]).toBeLessThan( + sendCommand.mock.invocationCallOrder[focusOffIndex]!, + ); + expect(sendInputEvent.mock.calls.map(([input]) => input.type)).toEqual([ + "keyDown", + "char", + "keyUp", + ]); expect(sendCommand).toHaveBeenCalledWith("Input.setIgnoreInputEvents", { ignore: false }); + expect(listeners.size).toBe(0); + expect(animationFrames.size).toBe(0); sendCommand.mockClear(); - failKeyDown = true; - const failedPress = yield* Effect.exit(manager.automationPress("tab_input", { key: "y" })); - - expect(Exit.isFailure(failedPress)).toBe(true); - expect(sendCommand).toHaveBeenCalledWith("Input.dispatchKeyEvent", { - type: "keyUp", - key: "y", - code: "KeyY", - modifiers: 0, - windowsVirtualKeyCode: 89, - location: 0, - isKeypad: false, + sendInputEvent.mockClear(); + getFocusedWebContents.mockReturnValue(null); + holdKeyUp = true; + const backgroundPress = yield* manager + .automationPress("tab_input", { key: "x" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => keyUpQueued); + expect(sendCommand).not.toHaveBeenCalledWith("Emulation.setFocusEmulationEnabled", { + enabled: false, }); + renderFrame(); + expect(animationFrames.size).toBe(1); + releaseKeyUp?.(); + yield* Fiber.join(backgroundPress); + expect(listeners.size).toBe(0); + expect(animationFrames.size).toBe(0); expect(sendCommand).toHaveBeenCalledWith("Emulation.setFocusEmulationEnabled", { enabled: false, }); - expect(restoreFocus).toHaveBeenCalledTimes(2); - expect( - sendCommand.mock.calls.filter( - ([method, params]) => - method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp", - ), - ).toHaveLength(1); + holdKeyUp = false; + + // Both native failures and expected-input matching must leave focus emulation off. + for (const key of ["y", "!"]) { + sendCommand.mockClear(); + sendInputEvent.mockClear(); + failKeyDown = key === "y"; + const exit = yield* Effect.exit(manager.automationPress("tab_input", { key })); + expect(Exit.isFailure(exit)).toBe(failKeyDown); + expect(sendInputEvent.mock.calls.map(([input]) => input.type)).toEqual( + failKeyDown ? ["keyDown", "keyUp"] : ["keyDown", "char", "keyUp"], + ); + expect(sendCommand).toHaveBeenCalledWith("Emulation.setFocusEmulationEnabled", { + enabled: false, + }); + } - sendCommand.mockClear(); - failKeyDown = false; - yield* manager.automationPress("tab_input", { key: "!" }); - expect(sendCommand).toHaveBeenCalledWith("Input.dispatchKeyEvent", { - type: "keyDown", - key: "!", - code: "Digit1", - modifiers: 0, - windowsVirtualKeyCode: 49, - location: 0, - isKeypad: false, - text: "!", - unmodifiedText: "!", - }); - expect(restoreFocus).toHaveBeenCalledTimes(3); + routeToIframe = true; + sendInputEvent.mockClear(); + for (const outcome of ["success", "failure", "interrupted"]) { + sendCommand.mockClear(); + failKeyDown = outcome === "failure"; + interruptFrameKeyDown = outcome === "interrupted"; + const exit = yield* Effect.exit( + manager.automationPress("tab_input", { + key: outcome === "success" ? "Enter" : "x", + }), + ); + expect(Exit.isSuccess(exit)).toBe(outcome === "success"); + if (outcome === "interrupted" && Exit.isFailure(exit)) { + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + }); + } + expect(sendInputEvent).not.toHaveBeenCalled(); + expect(sendCommand).toHaveBeenCalledWith("Target.attachToTarget", { + targetId: "focused-frame", + flatten: true, + }); + expect( + sendCommand.mock.calls + .filter(([method]) => method === "Input.dispatchKeyEvent") + .map(([, params, sessionId]) => ({ type: params?.["type"], sessionId })), + ).toEqual([ + { type: "keyDown", sessionId: "child-session" }, + { type: "keyUp", sessionId: "child-session" }, + ]); + expect(sendCommand).toHaveBeenCalledWith( + "Emulation.setFocusEmulationEnabled", + { enabled: false }, + "child-session", + ); + expect(sendCommand).toHaveBeenCalledWith("Target.detachFromTarget", { + sessionId: "child-session", + }); + } + expect(focus).not.toHaveBeenCalled(); + expect(restoreFocus).not.toHaveBeenCalled(); }), ), ); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index f4ab55a12..2549494eb 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -5,6 +5,7 @@ * elements live in the renderer; we only attach listeners and forward state * here). Single layer-scoped browser session partition. */ +import * as NodeCrypto from "node:crypto"; import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts"; import type { DesktopPreviewAnnotationTheme, @@ -75,7 +76,11 @@ import { } from "./GuestProtocol.ts"; import { isPreviewAnnotationPayload } from "./PickedElementPayload.ts"; import { playwrightInjectedRuntimeInstallExpression } from "./PlaywrightInjectedRuntime.ts"; -import { makePreviewAutomationKeySequence } from "./PreviewKeyboard.ts"; +import { + makePreviewAutomationKeySequence, + makePreviewAutomationNativeKeySequence, + previewAutomationEditingCommandExpression, +} from "./PreviewKeyboard.ts"; import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts"; export type PreviewNavStatus = @@ -1403,6 +1408,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function type SendCommand = ( method: string, commandParams?: Record, + sessionId?: string, ) => Effect.Effect; const prepareAutomationInput = Effect.fn("PreviewManager.prepareAutomationInput")(function* ( @@ -1422,7 +1428,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, wc: Electron.WebContents, action: string, - use: (send: SendCommand, sendCleanup: SendCommand) => Effect.Effect, + use: ( + send: SendCommand, + sendCleanup: SendCommand, + checkControl: Effect.Effect, + ) => Effect.Effect, ) { const sequence = yield* nextCounter(actionSequenceRef); const startedAt = yield* currentIso; @@ -1438,28 +1448,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const control = yield* ensureControlSession(wc); const execute = Effect.fn("PreviewManager.executeControlAction")(function* () { yield* update(tabId, { controller: "agent" }); + const checkControl = Effect.gen(function* () { + const currentEpoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; + if (currentEpoch !== epoch) { + return yield* new PreviewAutomationControlInterruptedError({ + operation: action, + tabId, + webContentsId: wc.id, + }); + } + }); const send: SendCommand = Effect.fn("PreviewManager.sendCommand")( - function* (method, commandParams) { - const before = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - if (before !== epoch) { - return yield* new PreviewAutomationControlInterruptedError({ - operation: action, - tabId, - webContentsId: wc.id, - }); - } + function* (method, commandParams, sessionId) { + yield* checkControl; const result = yield* attemptPromise( { operation: `${action}.${method}`, tabId, webContentsId: wc.id }, - () => control.debugger.sendCommand(method, commandParams), + () => + sessionId === undefined + ? control.debugger.sendCommand(method, commandParams) + : control.debugger.sendCommand(method, commandParams, sessionId), ); - const after = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - if (after !== epoch) { - return yield* new PreviewAutomationControlInterruptedError({ - operation: action, - tabId, - webContentsId: wc.id, - }); - } + yield* checkControl; return result; }, ); @@ -1467,18 +1476,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // control epoch. Otherwise a partially dispatched input can leave Chromium // with a held key or focus emulation enabled for subsequent actions. const sendCleanup: SendCommand = Effect.fn("PreviewManager.sendCleanupCommand")( - function* (method, commandParams) { + function* (method, commandParams, sessionId) { return yield* attemptPromise( { operation: `${action}.cleanup.${method}`, tabId, webContentsId: wc.id, }, - () => control.debugger.sendCommand(method, commandParams), + () => + sessionId === undefined + ? control.debugger.sendCommand(method, commandParams) + : control.debugger.sendCommand(method, commandParams, sessionId), ); }, ); - return yield* use(send, sendCleanup); + return yield* use(send, sendCleanup, checkControl); }); const finalize = Effect.fn("PreviewManager.finalizeControlAction")(function* ( exit: Exit.Exit, @@ -3932,56 +3944,326 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); + const withNativeKeyReceipt = Effect.fn("PreviewManager.withNativeKeyReceipt")(function* ( + tabId: string, + wc: Electron.WebContents, + dispatch: Effect.Effect, + checkControl: Effect.Effect, + ) { + const context = { operation: "automationPress.awaitNativeKey", tabId, webContentsId: wc.id }; + const evaluate = (frame: Electron.WebFrameMain, expression: string) => + attemptPromise(context, () => frame.executeJavaScript(expression)); + const { frames, receiptKey } = yield* Effect.acquireRelease( + attempt(context, () => ({ + frames: wc.mainFrame.framesInSubtree, + receiptKey: JSON.stringify(`__t3NativeKey_${NodeCrypto.randomUUID()}`), + })), + ({ frames, receiptKey }) => + Effect.all( + frames.map((frame) => + evaluate(frame, `globalThis[${receiptKey}]?.dispose()`).pipe( + Effect.timeoutOption(1_000), + Effect.ignore, + ), + ), + { concurrency: "unbounded", discard: true }, + ), + ); + yield* Effect.gen(function* () { + for (const frame of frames) { + yield* checkControl; + yield* evaluate( + frame, + `(() => { + const receiptKey = ${receiptKey}; + const counts = performance.eventCounts; + if (!counts) throw new Error("Native key delivery counters are unavailable."); + const keyUpsBefore = counts.get("keyup") ?? 0; + const keyDownsBefore = counts.get("keydown") ?? 0; + let settle; + let animationFrame = 0; + const promise = new Promise(resolve => { settle = resolve; }); + const finish = delivered => { + cancelAnimationFrame(animationFrame); + window.removeEventListener("pagehide", onPageHide, true); + settle(delivered); + }; + // Chromium counts trusted keys before dispatching page listeners, + // so stopImmediatePropagation cannot hide completed input. + const observe = () => { + if ((counts.get("keyup") ?? 0) > keyUpsBefore) finish(true); + else animationFrame = requestAnimationFrame(observe); + }; + const onPageHide = () => finish( + (counts.get("keyup") ?? 0) > keyUpsBefore || + (counts.get("keydown") ?? 0) > keyDownsBefore, + ); + globalThis[receiptKey] = { promise, dispose: () => { + finish(false); + delete globalThis[receiptKey]; + }}; + window.addEventListener("pagehide", onPageHide, true); + animationFrame = requestAnimationFrame(observe); + })()`, + ); + } + yield* checkControl; + yield* dispatch; + yield* attemptPromise(context, () => + Promise.any( + frames.map(async (frame) => { + const delivered: unknown = await frame.executeJavaScript( + `globalThis[${receiptKey}]?.promise`, + ); + if (delivered !== true) + throw new Error( + "The preview document changed before native key delivery was confirmed.", + ); + }), + ), + ); + yield* checkControl; + }).pipe( + Effect.timeout(5_000), + Effect.catchTags({ + TimeoutError: () => + Effect.fail(new PreviewAutomationTimeoutError({ tabId, timeoutMs: 5_000 })), + }), + ); + }, Effect.scoped); + + const resolveKeyboardTarget = Effect.fn("PreviewManager.resolveKeyboardTarget")(function* ( + tabId: string, + send: SendCommand, + sendCleanup: SendCommand, + checkControl: Effect.Effect, + ) { + const context = { operation: "automationPress.resolveFocusedFrame", tabId }; + let sessionId: string | undefined; + let contextId: number | undefined; + while (true) { + const evaluated = (yield* send( + "Runtime.evaluate", + { + expression: `(() => { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + return element?.tagName === "IFRAME" ? element : null; + })()`, + ...(contextId === undefined ? {} : { contextId }), + }, + sessionId, + )) as { result?: { objectId?: string; subtype?: string } }; + if (evaluated.result?.subtype === "null") break; + const objectId = evaluated.result?.objectId; + if (!objectId) + return yield* new PreviewOperationError({ + ...context, + cause: new Error("The focused preview frame could not be resolved."), + }); + const described = (yield* send("DOM.describeNode", { objectId }, sessionId).pipe( + Effect.ensuring( + sendCleanup("Runtime.releaseObject", { objectId }, sessionId).pipe(Effect.ignore), + ), + )) as { node?: { frameId?: string } }; + const frameId = described.node?.frameId; + if (!frameId) + return yield* new PreviewOperationError({ + ...context, + cause: new Error("The focused preview iframe is unavailable."), + }); + const targets = (yield* send("Target.getTargets")) as { + targetInfos?: ReadonlyArray<{ targetId: string; type: string }>; + }; + if ( + targets.targetInfos?.some( + (target) => target.type === "iframe" && target.targetId === frameId, + ) + ) { + yield* checkControl; + // Register cleanup before checking the epoch again: a successful + // attach must be released even when human input interrupts its reply. + sessionId = yield* Effect.acquireRelease( + sendCleanup("Target.attachToTarget", { targetId: frameId, flatten: true }).pipe( + Effect.flatMap((response) => + attempt(context, () => { + const attached = response as { sessionId?: string }; + if (!attached.sessionId) + throw new Error("The focused preview iframe could not be attached."); + return attached.sessionId; + }), + ), + ), + (attachedSessionId) => + sendCleanup("Target.detachFromTarget", { sessionId: attachedSessionId }).pipe( + Effect.ignore, + ), + ); + yield* checkControl; + contextId = undefined; + } else { + const world = (yield* send( + "Page.createIsolatedWorld", + { + frameId, + worldName: "t3-preview-key-target", + }, + sessionId, + )) as { executionContextId?: number }; + if (typeof world.executionContextId !== "number") + return yield* new PreviewOperationError({ + ...context, + cause: new Error("The focused preview iframe context is unavailable."), + }); + contextId = world.executionContextId; + } + } + return { sessionId, contextId }; + }); + const performAutomationPress = Effect.fn("PreviewManager.performAutomationPress")(function* ( tabId: string, wc: Electron.WebContents, input: PreviewAutomationPressInput, send: SendCommand, sendCleanup: SendCommand, + checkControl: Effect.Effect, ) { yield* prepareAutomationInput(send, false); - const keySequence = makePreviewAutomationKeySequence(input, { + const keySequence = makePreviewAutomationNativeKeySequence(input, { isMac: hostPlatform === "darwin", }); - const previouslyFocused = yield* attempt( - { operation: "automationPress.getFocusedWebContents", tabId, webContentsId: wc.id }, - () => webContents.getFocusedWebContents(), - ); - let keyDownAttempted = false; - const releaseInput = Effect.gen(function* () { - if (keyDownAttempted) { - yield* sendCleanup("Input.dispatchKeyEvent", keySequence.keyUp).pipe(Effect.ignore); - } - yield* sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }).pipe( - Effect.ignore, + // CDP keyboard dispatch follows the embedder's focused renderer, and + // WebContents.focus() is a no-op for webview guests. Native input targets + // this guest's widget directly, so Enter cannot submit the host composer. + yield* Effect.gen(function* () { + const { sessionId, contextId } = yield* resolveKeyboardTarget( + tabId, + send, + sendCleanup, + checkControl, ); - if (previouslyFocused && previouslyFocused.id !== wc.id && !previouslyFocused.isDestroyed()) { - yield* attempt( - { - operation: "automationPress.restoreFocusedWebContents", + // Only descendant renderer sessions bypass Chromium's desktop focus lookup. + if (sessionId) { + const keys = makePreviewAutomationKeySequence(input, { isMac: hostPlatform === "darwin" }); + yield* Effect.acquireRelease(Effect.void, () => + sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }, sessionId).pipe( + Effect.ignore, + ), + ); + yield* send("Emulation.setFocusEmulationEnabled", { enabled: true }, sessionId); + yield* Effect.acquireRelease(Effect.void, () => + sendCleanup("Input.dispatchKeyEvent", keys.keyUp, sessionId).pipe(Effect.ignore), + ); + yield* send("Input.dispatchKeyEvent", keys.keyDown, sessionId); + return; + } + if (keySequence.commands?.length) { + const context = { + operation: "automationPress.editFocusedFrame", + tabId, + webContentsId: wc.id, + }; + const evaluate = (expression: string, cleanup = false) => + evaluateWithDebugger( tabId, - webContentsId: previouslyFocused.id, - }, - () => previouslyFocused.focus(), - ).pipe(Effect.ignore); + (method, params) => + (cleanup ? sendCleanup : send)(method, { + ...params, + ...(contextId === undefined ? {} : { contextId }), + }), + expression, + true, + ); + const clipboardData = keySequence.commands.includes("paste") + ? yield* attemptPromise(context, async () => { + const formats: Array<{ type: string; data: string }> = []; + for (const item of await clipboard.read()) { + for (const type of item.types) { + if (type.startsWith("electron ")) continue; + const blob = await item.getType(type); + if (!("arrayBuffer" in blob)) continue; + formats.push({ + type, + data: type.startsWith("text/") + ? await blob.text() + : Buffer.from(await blob.arrayBuffer()).toString("base64"), + }); + } + } + return formats; + }) + : []; + yield* checkControl; + const expression = previewAutomationEditingCommandExpression( + input, + keySequence, + clipboardData, + ); + const selectionKey = yield* encodeJson( + context, + `__t3EditingSelection_${NodeCrypto.randomUUID()}`, + ); + // Editing requires an active document. Preserve the target + // and selection across focus handlers without focusing the desktop. + yield* Effect.acquireUseRelease( + evaluate(`(() => { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + const selection = document.getSelection(); + const range = selection?.rangeCount ? selection.getRangeAt(0).cloneRange() : null; + const backward = selection?.direction === "backward"; + const start = element?.selectionStart; + const end = element?.selectionEnd; + const direction = element?.selectionDirection; + globalThis[${selectionKey}] = () => { + element?.focus({ preventScroll: true }); + if (typeof start === "number") element.setSelectionRange(start, end, direction); + else { + selection?.removeAllRanges(); + if (range && backward) selection.setBaseAndExtent( + range.endContainer, range.endOffset, range.startContainer, range.startOffset, + ); + else if (range) selection.addRange(range); + } + }; + })()`), + () => + Effect.gen(function* () { + yield* send("Emulation.setFocusEmulationEnabled", { enabled: true }); + yield* evaluate(`globalThis[${selectionKey}]();${expression}`); + }), + () => evaluate(`delete globalThis[${selectionKey}]`, true).pipe(Effect.ignore), + ); + yield* checkControl; + return; } - }); - - // Focus the guest WebContents itself, not its containing BrowserWindow. This - // activates native keyboard behavior for hidden/background previews without - // changing which thread is mounted in the UI. Restore the previous renderer - // after dispatch so automation never leaves the app's input focus behind. - yield* Effect.gen(function* () { - yield* attempt( - { operation: "automationPress.focusWebContents", tabId, webContentsId: wc.id }, - () => wc.focus(), - ); - yield* send("Page.bringToFront"); yield* send("Emulation.setFocusEmulationEnabled", { enabled: true }); - yield* expectAgentInput(tabId, keySequence.signal); - keyDownAttempted = true; - yield* send("Input.dispatchKeyEvent", keySequence.keyDown); - }).pipe(Effect.ensuring(releaseInput)); + yield* withNativeKeyReceipt( + tabId, + wc, + Effect.gen(function* () { + yield* expectAgentInput(tabId, keySequence.signal); + yield* attempt( + { operation: "automationPress.sendInputEvent", tabId, webContentsId: wc.id }, + () => { + try { + wc.sendInputEvent(keySequence.keyDown); + if (keySequence.char) wc.sendInputEvent(keySequence.char); + } finally { + wc.sendInputEvent(keySequence.keyUp); + } + }, + ); + }), + checkControl, + ); + }).pipe( + Effect.scoped, + Effect.ensuring( + sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }).pipe(Effect.ignore), + ), + ); }); const automationPress = Effect.fn("PreviewManager.automationPress")(function* ( @@ -3989,8 +4271,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input: PreviewAutomationPressInput, ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "press", (send, sendCleanup) => - performAutomationPress(tabId, wc, input, send, sendCleanup), + yield* withControlSession(tabId, wc, "press", (send, sendCleanup, checkControl) => + performAutomationPress(tabId, wc, input, send, sendCleanup, checkControl), ); }); diff --git a/apps/desktop/src/preview/PreviewKeyboard.test.ts b/apps/desktop/src/preview/PreviewKeyboard.test.ts index 7a9a7373f..39b542087 100644 --- a/apps/desktop/src/preview/PreviewKeyboard.test.ts +++ b/apps/desktop/src/preview/PreviewKeyboard.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vite-plus/test"; -import { makePreviewAutomationKeySequence } from "./PreviewKeyboard.ts"; +import { + makePreviewAutomationKeySequence, + makePreviewAutomationNativeKeySequence, +} from "./PreviewKeyboard.ts"; describe("preview keyboard packets", () => { it("includes the Chromium virtual key code and Enter text", () => { @@ -94,4 +97,88 @@ describe("preview keyboard packets", () => { }); expect(sequence.signal).toEqual({ kind: "key", key: "!", code: "Digit1" }); }); + + it.each([ + ["Enter", "\r"], + ["z", "z"], + ])("converts %s into native down, char, and up packets", (key, text) => { + const sequence = makePreviewAutomationNativeKeySequence({ key }); + const shared = { keyCode: key, modifiers: [], skipIfUnhandled: true }; + expect(sequence.keyDown).toEqual({ type: "keyDown", ...shared }); + expect(sequence.char).toEqual({ type: "char", ...shared, keyCode: text }); + expect(sequence.keyUp).toEqual({ type: "keyUp", ...shared }); + }); + + it("suppresses text for shortcuts and retains macOS editing commands", () => { + const sequence = makePreviewAutomationNativeKeySequence( + { key: "a", modifiers: ["Meta"] }, + { isMac: true }, + ); + expect(sequence.keyDown).toEqual({ + type: "keyDown", + keyCode: "a", + modifiers: ["meta"], + skipIfUnhandled: true, + }); + expect(sequence.char).toBeUndefined(); + expect(sequence.commands).toEqual(["selectAll"]); + }); + + it.each([ + ["ArrowLeft", "Left"], + ["ArrowRight", "Right"], + ["ArrowUp", "Up"], + ["ArrowDown", "Down"], + ])("maps %s to Electron's %s accelerator", (key, keyCode) => { + const sequence = makePreviewAutomationNativeKeySequence({ key }); + expect(sequence.keyDown.keyCode).toBe(keyCode); + expect(sequence.keyUp.keyCode).toBe(keyCode); + expect(sequence.signal.key).toBe(key); + expect(sequence.char).toBeUndefined(); + }); + + it("matches native uppercase key signals without inventing shortcut modifiers", () => { + const plain = makePreviewAutomationNativeKeySequence({ key: "X" }); + expect(plain.signal).toEqual({ kind: "key", key: "x", code: "KeyX" }); + expect(plain.char?.keyCode).toBe("X"); + const shortcut = makePreviewAutomationNativeKeySequence({ key: "A", modifiers: ["Control"] }); + expect(shortcut.signal).toEqual({ kind: "key", key: "a", code: "KeyA" }); + expect(shortcut.keyDown.modifiers).toEqual(["control"]); + expect(shortcut.char).toBeUndefined(); + expect( + makePreviewAutomationNativeKeySequence({ key: "X", modifiers: ["Shift"] }).signal, + ).toEqual({ + kind: "key", + key: "X", + code: "KeyX", + }); + }); + + it("matches native signals for Unicode text and literal spaces", () => { + const unicode = makePreviewAutomationNativeKeySequence({ key: "é" }); + expect(unicode.signal).toEqual({ kind: "key", key: "", code: "" }); + expect(unicode.char?.keyCode).toBe("é"); + expect(makePreviewAutomationNativeKeySequence({ key: " " }).signal).toEqual({ + kind: "key", + key: " ", + code: "Space", + }); + }); + + it("preserves text and editing commands for isolated child renderer targets", () => { + const text = makePreviewAutomationKeySequence({ key: "é" }); + expect(text.keyDown).toMatchObject({ type: "keyDown", text: "é", key: "é" }); + expect(text.keyUp).toMatchObject({ type: "keyUp", key: "é" }); + const shortcut = makePreviewAutomationKeySequence( + { key: "a", modifiers: ["Meta"] }, + { isMac: true }, + ); + expect(shortcut.keyDown).toMatchObject({ + type: "rawKeyDown", + modifiers: 4, + commands: ["selectAll"], + }); + expect(shortcut.keyDown).not.toHaveProperty("text"); + expect(shortcut.keyDown).not.toHaveProperty("nativeVirtualKeyCode"); + }); }); diff --git a/apps/desktop/src/preview/PreviewKeyboard.ts b/apps/desktop/src/preview/PreviewKeyboard.ts index 0d231b86f..69f2daad0 100644 --- a/apps/desktop/src/preview/PreviewKeyboard.ts +++ b/apps/desktop/src/preview/PreviewKeyboard.ts @@ -131,7 +131,7 @@ const modifierMask = (modifiers: PreviewAutomationPressInput["modifiers"]): numb }, 0); function resolveKeyDefinition(input: PreviewAutomationPressInput): KeyDefinition { - const named = NAMED_KEYS[input.key]; + const named = NAMED_KEYS[input.key === " " ? "Space" : input.key]; if (named) return named; const functionKey = /^F([1-9]|1[0-2])$/.exec(input.key); @@ -201,3 +201,163 @@ export function makePreviewAutomationKeySequence( signal: { kind: "key", key: definition.key, code: definition.code }, }; } + +/** Root CDP input can retarget the embedder; native packets address the guest widget. */ +export function makePreviewAutomationNativeKeySequence( + input: PreviewAutomationPressInput, + options?: { readonly isMac?: boolean }, +) { + const { keyDown, signal } = makePreviewAutomationKeySequence(input, options); + const modifiers = ( + [ + [1, "alt"], + [2, "control"], + [4, "meta"], + [8, "shift"], + ] as const + ) + .filter(([mask]) => keyDown.modifiers & mask) + .map(([, modifier]) => modifier); + const shared = { + keyCode: keyDown.key.startsWith("Arrow") ? keyDown.key.slice(5) : keyDown.key, + modifiers, + skipIfUnhandled: true as const, + }; + // Electron lowercases unshifted letters and reports no key for Unicode accelerators. + const key = + keyDown.windowsVirtualKeyCode === 0 && keyDown.key.length === 1 + ? "" + : /^[A-Z]$/.test(keyDown.key) && !modifiers.includes("shift") + ? keyDown.key.toLowerCase() + : keyDown.key; + return { + keyDown: { type: "keyDown" as const, ...shared }, + ...(keyDown.text ? { char: { type: "char" as const, ...shared, keyCode: keyDown.text } } : {}), + keyUp: { type: "keyUp" as const, ...shared }, + ...(keyDown.commands ? { commands: keyDown.commands } : {}), + signal: { ...signal, key }, + }; +} + +/** Keep macOS editing shortcuts inside the target page without native focus. */ +export function previewAutomationEditingCommandExpression( + input: PreviewAutomationPressInput, + sequence: ReturnType, + clipboardData: ReadonlyArray<{ readonly type: string; readonly data: string }> = [], +): string { + const definition = resolveKeyDefinition(input); + const event = { + key: definition.key, + code: definition.code, + keyCode: definition.keyCode, + which: definition.keyCode, + location: definition.location ?? 0, + altKey: input.modifiers?.includes("Alt") ?? false, + ctrlKey: input.modifiers?.includes("Control") ?? false, + metaKey: input.modifiers?.includes("Meta") ?? false, + shiftKey: input.modifiers?.includes("Shift") ?? false, + bubbles: true, + cancelable: true, + composed: true, + }; + return `(() => { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + if (!element) return; + const event = ${JSON.stringify(event)}; + try { + if (!element.dispatchEvent(new KeyboardEvent("keydown", event))) return; + for (const command of ${JSON.stringify(sequence.commands ?? [])}) { + // Main-process clipboard reads also work on insecure HTTP previews. + // Let the page's paste handler consume the clipboard MIME formats. + if (command === "paste") { + const transfer = new DataTransfer(); + for (const { type, data } of ${JSON.stringify(clipboardData)}) { + if (type === "text/html") { + // Match native paste sanitization before page handlers or insertion. + const container = document.createElement("div"); + container.setHTML(data); + transfer.setData(type, container.innerHTML); + } else if (type.startsWith("text/")) transfer.setData(type, data); + else { + const bytes = Uint8Array.from(atob(data), character => character.charCodeAt(0)); + transfer.items.add(new File([bytes], "clipboard", { type })); + } + } + if (!element.dispatchEvent(new ClipboardEvent("paste", { + clipboardData: transfer, bubbles: true, cancelable: true, composed: true, + }))) continue; + const text = transfer.getData("text/plain"); + if (!element.dispatchEvent(new InputEvent("beforeinput", { + inputType: "insertFromPaste", data: text, dataTransfer: transfer, + bubbles: true, cancelable: true, composed: true, + }))) continue; + const html = element.isContentEditable ? transfer.getData("text/html") : ""; + document.execCommand(html ? "insertHTML" : "insertText", false, html || text); + continue; + } + const inputType = command === "deleteToBeginningOfLine" ? "deleteSoftLineBackward" + : command === "undo" ? "historyUndo" + : command === "redo" ? "historyRedo" : null; + // execCommand emits input without beforeinput. Let controlled editors + // perform the edit before applying the browser's default operation. + if (inputType && !element.dispatchEvent(new InputEvent("beforeinput", { + inputType, bubbles: true, cancelable: true, composed: true, + }))) continue; + const selection = document.getSelection(); + if (command === "deleteToBeginningOfLine") { + const collapsed = typeof element.selectionStart === "number" + ? element.selectionStart === element.selectionEnd + : selection?.isCollapsed; + if (collapsed) selection?.modify("extend", "backward", "lineboundary"); + document.execCommand("delete"); + } else if (command.startsWith("moveTo")) { + const selectionElement = selection?.anchorNode?.nodeType === Node.ELEMENT_NODE + ? selection.anchorNode : selection?.anchorNode?.parentElement; + const editable = element.isContentEditable || selectionElement?.isContentEditable || + (((element instanceof HTMLInputElement && element.selectionStart !== null) || + element instanceof HTMLTextAreaElement) && + !element.readOnly && !element.disabled); + if (!editable && (command === "moveToBeginningOfDocument" || command === "moveToEndOfDocument")) { + let scrollable = element === document.body ? selectionElement ?? element : element; + while (scrollable && !(scrollable.scrollHeight > scrollable.clientHeight && + /^(auto|scroll|overlay)$/.test(getComputedStyle(scrollable).overflowY))) { + scrollable = scrollable.parentElement ?? scrollable.getRootNode().host; + } + scrollable ??= document.scrollingElement; + if (scrollable) scrollable.scrollTop = command === "moveToBeginningOfDocument" + ? 0 : scrollable.scrollHeight; + continue; + } + const direction = command.includes("Beginning") ? "backward" + : command.includes("Left") ? "left" + : command.includes("Right") ? "right" : "forward"; + selection?.modify( + command.endsWith("AndModifySelection") ? "extend" : "move", + direction, + command.includes("Document") ? "documentboundary" : "lineboundary", + ); + if (element instanceof HTMLInputElement && element.selectionStart !== null) { + if (command.includes("Left") || command.includes("Beginning")) element.scrollLeft = 0; + else element.scrollLeft = element.scrollWidth; + } + // Programmatic selection changes do not reveal the caret like native editing commands. + if (editable && command.includes("Document")) { + const beginning = command.includes("Beginning"); + if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) { + element.scrollTop = beginning ? 0 : element.scrollHeight; + } else { + const caretElement = selection?.focusNode?.nodeType === Node.ELEMENT_NODE + ? selection.focusNode : selection?.focusNode?.parentElement; + caretElement?.scrollIntoView({ block: beginning ? "start" : "end", inline: "nearest" }); + } + } + } else { + document.execCommand(command); + } + } + } finally { + element.dispatchEvent(new KeyboardEvent("keyup", event)); + } + })()`; +} diff --git a/apps/desktop/src/window/QuitHold.test.ts b/apps/desktop/src/window/QuitHold.test.ts index 58809d8eb..47f3ec529 100644 --- a/apps/desktop/src/window/QuitHold.test.ts +++ b/apps/desktop/src/window/QuitHold.test.ts @@ -178,14 +178,35 @@ describe("makeQuitShortcutHandler", () => { await harness.holdFor(QUIT_HOLD_DURATION_MS + 200); await harness.send(makeInput({ type: "keyUp", key: "Meta", meta: false })); harness.preventDefault.mockClear(); - await harness.send(makeInput({ meta: false, isAutoRepeat: true })); - expect(harness.preventDefault).toHaveBeenCalledTimes(1); - vi.advanceTimersByTime(QUIT_HOLD_RELEASE_GRACE_MS * 2); + // Repeats without the modifier prove Q is still down, so they hold the + // quit back for as long as they keep arriving. + await harness.holdFor(QUIT_HOLD_RELEASE_GRACE_MS * 2, { meta: false }); + expect(harness.preventDefault).toHaveBeenCalled(); expect(harness.quit).not.toHaveBeenCalled(); await harness.send(makeInput({ type: "keyUp", meta: false })); expect(harness.quit).toHaveBeenCalledTimes(1); }); + it("commits a concealed hold when the last Q repeat is never released", async () => { + // macOS can drop the final Q keyUp. The quit must land on its own once + // repeats stop, rather than sitting armed until an unrelated key arrives. + const harness = makeHarness(); + await harness.send(makeInput({})); + await harness.holdFor(QUIT_HOLD_DURATION_MS + 200); + await harness.send(makeInput({ type: "keyUp", key: "Meta", meta: false })); + await harness.send(makeInput({ meta: false, isAutoRepeat: true })); + + vi.advanceTimersByTime(QUIT_HOLD_RELEASE_GRACE_MS); + expect(harness.quit).toHaveBeenCalledTimes(1); + + // A lone Cmd tap afterwards must not quit a second time. + harness.quit.mockClear(); + await harness.send(makeInput({ key: "Meta" })); + await harness.send(makeInput({ type: "keyUp", key: "Meta", meta: false })); + vi.advanceTimersByTime(QUIT_HOLD_RELEASE_GRACE_MS * 4); + expect(harness.quit).not.toHaveBeenCalled(); + }); + it("does not quit when the hold stops before the duration", async () => { const harness = makeHarness(); await harness.send(makeInput({})); diff --git a/apps/desktop/src/window/QuitHold.ts b/apps/desktop/src/window/QuitHold.ts index a995184dd..f656183f1 100644 --- a/apps/desktop/src/window/QuitHold.ts +++ b/apps/desktop/src/window/QuitHold.ts @@ -125,13 +125,9 @@ export function makeQuitShortcutHandler( } if (quitOnRelease) { event.preventDefault(); - if (key === "q") { - if (modifierDown) { - quitAfterQuietPeriod(); - } else { - clearWatchdog(); - } - } + // A Q keydown proves the key is still down whether or not the modifier + // is still held, so it only pushes the quiet period back. + if (key === "q") quitAfterQuietPeriod(); return; }