diff --git a/apps/desktop/package.json b/apps/desktop/package.json index dd005aa9ee..634fce0294 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -11,11 +11,12 @@ "dev:hmr": "node scripts/dev.mjs", "storybook": "storybook dev -p 6006 -c .storybook", "build-storybook": "storybook build -c .storybook --output-dir storybook-static", - "build": "npm run build:main && npm run build:preload && npm run build:renderer", - "build:test": "npm run build:main && npm run build:preload", + "build": "npm run build:main && npm run build:preload && npm run build:overlay && npm run build:renderer", + "build:test": "npm run build:main && npm run build:preload && npm run build:overlay", "clean:main": "node ../../scripts/clean-paths.mjs dist/main tsconfig.main.tsbuildinfo", "build:main": "tsc -p tsconfig.main.json", "build:preload": "esbuild src/preload/preload.ts --bundle --platform=node --format=cjs --outfile=dist/preload/preload.cjs --external:electron", + "build:overlay": "node ../../scripts/build-cursor-overlay.mjs", "build:renderer": "vite build && node ../../scripts/check-third-party-notices.mjs", "typecheck": "tsc -p tsconfig.main.json --noEmit && tsc -p tsconfig.renderer.json --noEmit && tsc -p tsconfig.storybook.json --noEmit", "typecheck:stories": "tsc -p tsconfig.storybook.json --noEmit", diff --git a/apps/desktop/scripts/dev.mjs b/apps/desktop/scripts/dev.mjs index 46b96ac55f..2ef1865b41 100644 --- a/apps/desktop/scripts/dev.mjs +++ b/apps/desktop/scripts/dev.mjs @@ -21,6 +21,7 @@ import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createServer } from 'vite'; import { build as esbuildBuild } from 'esbuild'; +import { buildCursorOverlay } from '../../../scripts/build-cursor-overlay.mjs'; const DESKTOP_DIR = resolve(fileURLToPath(new URL('..', import.meta.url))); const REPO_ROOT = resolve(DESKTOP_DIR, '..', '..'); @@ -88,6 +89,10 @@ await Promise.all([ () => log('build', 'preload — done'), (e) => { log('build', `preload — FAILED: ${e.message}`); throw e; }, ), + buildCursorOverlay({ logLevel: 'warning' }).then( + () => log('build', 'cursor overlay — done'), + (e) => { log('build', `cursor overlay — FAILED: ${e.message}`); throw e; }, + ), ]); // Phase 2: main — esbuild bundle for dev startup. The full diff --git a/apps/desktop/src/main/__tests__/computer-use-capability.test.ts b/apps/desktop/src/main/__tests__/computer-use-capability.test.ts index 36d013e076..3bdebd2453 100644 --- a/apps/desktop/src/main/__tests__/computer-use-capability.test.ts +++ b/apps/desktop/src/main/__tests__/computer-use-capability.test.ts @@ -12,9 +12,10 @@ describe('Desktop Computer Use production wiring', () => { 'utf8', ); assert.match(main, /createComputerUseHost/); + assert.match(main, /createCursorOverlayController/); + assert.match(main, /createComputerUseOverlayHook/); assert.match(main, /computerUseTools/); assert.match(main, /id:\s*'computer_use'/); - assert.doesNotMatch(main, /createComputerUseOverlayHook/); assert.doesNotMatch(main, /createAnthropicComputerHarness|createKimiComputerHarness|createMiniMaxComputerHarness/); }); @@ -24,11 +25,17 @@ describe('Desktop Computer Use production wiring', () => { 'utf8', ); assert.match(main, /sessions:stop[\s\S]*computerUseTools\.clearSession/); + assert.match(main, /sessions:stop[\s\S]*computerUseOverlay\.clearForSession/); assert.match(main, /sessions:archive[\s\S]*computerUseTools\.clearSession/); + assert.match(main, /sessions:archive[\s\S]*computerUseOverlay\.clearForSession/); assert.match(main, /sessions:remove[\s\S]*computerUseTools\.clearSession/); + assert.match(main, /sessions:remove[\s\S]*computerUseOverlay\.clearForSession/); assert.match(main, /isTurnStatusChangingSessionEvent[\s\S]*computerUseTools\.clearSession/); assert.match(main, /catch \(error\)[\s\S]*computerUseTools\.clearSession/); assert.match(main, /Promise\.allSettled\(\[[\s\S]*computerUse\.backend\?\.dispose/); + assert.match(main, /Promise\.allSettled\(\[[\s\S]*computerUseOverlay\.destroyAll/); + assert.match(main, /window-all-closed[\s\S]*computerUseOverlay\.destroyAll/); + assert.match(main, /onMainWindowClose = \(\) => computerUseOverlay\.destroyAll/); }); it('reports scoped approval and live service health instead of binary-only healthy', async () => { diff --git a/apps/desktop/src/main/__tests__/cursor-engine.test.ts b/apps/desktop/src/main/__tests__/cursor-engine.test.ts new file mode 100644 index 0000000000..f481c2259b --- /dev/null +++ b/apps/desktop/src/main/__tests__/cursor-engine.test.ts @@ -0,0 +1,200 @@ +// Unit tests for the ported agent-cursor engine (palette + Dubins + tick/spring). +// Pure math — no DOM; `paint()` is exercised only by the visual demo. Faithful to +// trycua/cua's cursor-overlay Rust source these were ported from. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { CursorEngine } from '../../renderer/computer-use-overlay/engine/cursor-engine.js'; +import { planDirectPath, planPath } from '../../renderer/computer-use-overlay/engine/dubins.js'; +import { paletteForInstance, defaultPalette, gradientAt } from '../../renderer/computer-use-overlay/engine/palette.js'; + +const finite = (v: number): boolean => Number.isFinite(v); +const REST_HEADING = Math.PI / 4; +const ARROW_TIP_LENGTH = 14; + +test('Dubins path primitive remains finite for legacy callers', () => { + const path = planPath(0, 0, 0, 400, 200, Math.PI / 4, Math.PI / 4, 80); + assert.ok(finite(path.length) && path.length > 0, `length ${path.length}`); + const s0 = path.sample(0); + assert.ok(Math.hypot(s0.x, s0.y) < 0.01, 'sample(0) == start'); + const sEnd = path.sample(path.length); + assert.ok(Math.hypot(sEnd.x - 400, sEnd.y - 200) < 1.0, `sample(len) ≈ target (err ${Math.hypot(sEnd.x - 400, sEnd.y - 200)})`); + let prev = path.sample(0); + const N = 400; + let maxStep = 0; + for (let i = 1; i <= N; i++) { + const cur = path.sample((path.length * i) / N); + assert.ok(finite(cur.x) && finite(cur.y), 'no NaN along path'); + maxStep = Math.max(maxStep, Math.hypot(cur.x - prev.x, cur.y - prev.y)); + prev = cur; + } + assert.ok(maxStep < (path.length / N) * 3, `continuity: max step ${maxStep}`); +}); + +test('direct planner is the shortest path and ends exactly at the target', () => { + const path = planDirectPath(12, 34, 112, 84, REST_HEADING); + assert.ok(Math.abs(path.length - Math.hypot(100, 50)) < 0.001); + const end = path.sample(path.length); + assert.ok(Math.hypot(end.x - 112, end.y - 84) < 0.001); +}); + +test('speed profile peaks at 1.0 at u=0.5 (smootherstep)', () => { + const u = 0.5; + const profile = (30 * u * u * (1 - u) * (1 - u)) / 1.875; + assert.ok(Math.abs(profile - 1.0) < 1e-9, `profile ${profile}`); +}); + +test('engine glides directly onto target+offset, no NaN', () => { + const e = new CursorEngine(); + e.setSession('conv-test'); + const tx = 500, ty = 300; + e.moveTo(tx, ty); // center is offset so the 14px arrow tip lands on (tx, ty) + const offX = tx + Math.cos(REST_HEADING) * ARROW_TIP_LENGTH; + const offY = ty + Math.sin(REST_HEADING) * ARROW_TIP_LENGTH; + let frames = 0; + const dt = 1 / 60; + while (e.isMoving() && frames < 60 * 8) { + e.tick(dt); + assert.ok(finite(e.pos[0]) && finite(e.pos[1]) && finite(e.heading), 'no NaN'); + frames++; + } + assert.ok(!e.isMoving(), `settled (frames ${frames})`); + assert.ok(Math.hypot(e.pos[0] - offX, e.pos[1] - offY) < 1.0, 'final pos ≈ target+offset'); + assert.ok(frames > 20 && frames < 60 * 6, `glide duration sane (${(frames / 60).toFixed(2)}s)`); +}); + +test('direct cursor path has no lateral detour or in-flight rotation', () => { + const path = planDirectPath(100, 100, 700, 250, REST_HEADING); + const expectedHeading = Math.atan2(150, 600); + for (let index = 0; index <= 100; index++) { + const point = path.sample((path.length * index) / 100); + const progress = index / 100; + const expectedX = 100 + 600 * progress; + const expectedY = 100 + 150 * progress; + assert.ok(Math.hypot(point.x - expectedX, point.y - expectedY) < 0.01); + assert.ok(Math.abs(point.heading - expectedHeading) < 0.01); + } +}); + +test('first move glides IN from off-screen (not a pop) and converges to target', () => { + const e = new CursorEngine(); + assert.ok(e.pos[0] < -100, 'starts off-screen'); + e.moveTo(400, 400); + e.tick(1 / 60); + // Entered on-screen but NOT already at the target — it's gliding in. + assert.ok(e.pos[0] > 0 && e.pos[0] < 400, `entering, still gliding (pos ${e.pos[0]})`); + let frames = 1; + while (e.isMoving() && frames < 600) { e.tick(1 / 60); frames++; } + const tx = 400 + Math.cos(REST_HEADING) * ARROW_TIP_LENGTH; + const ty = 400 + Math.sin(REST_HEADING) * ARROW_TIP_LENGTH; + assert.ok(Math.hypot(e.pos[0] - tx, e.pos[1] - ty) < 1.5, 'converged to target+offset'); +}); + +test('click pulse is centered on the action coordinate, not the arrow body', () => { + const e = new CursorEngine(); + const targetX = 320; + const targetY = 240; + e.moveTo(targetX, targetY, undefined, true); + for (let frames = 0; e.isMoving() && frames < 600; frames++) e.tick(1 / 60); + e.triggerClick(targetX, targetY); + + const arcs: Array<{ x: number; y: number; radius: number }> = []; + const gradient = { addColorStop() {} }; + const ctx = { + createRadialGradient: () => gradient, + createLinearGradient: () => gradient, + beginPath() {}, + arc(x: number, y: number, radius: number) { arcs.push({ x, y, radius }); }, + fill() {}, + stroke() {}, + moveTo() {}, + lineTo() {}, + closePath() {}, + set fillStyle(_value: unknown) {}, + set strokeStyle(_value: unknown) {}, + set lineWidth(_value: number) {}, + set lineJoin(_value: CanvasLineJoin) {}, + } as unknown as CanvasRenderingContext2D; + + e.paint(ctx, 0, 0); + assert.ok( + arcs.some((arc) => Math.hypot(arc.x - targetX, arc.y - targetY) < 0.01), + `click pulse should include action coordinate (${targetX},${targetY}); arcs=${JSON.stringify(arcs)}`, + ); +}); + +test('completion snaps the arrow tip to the executed coordinate and cancels glide', () => { + const e = new CursorEngine(); + e.pos = [100, 100]; + e.moveTo(500, 300); + e.tick(1 / 60); + e.completeAt(320, 240, true); + + const tipX = e.pos[0] - Math.cos(REST_HEADING) * ARROW_TIP_LENGTH; + const tipY = e.pos[1] - Math.sin(REST_HEADING) * ARROW_TIP_LENGTH; + assert.ok(Math.hypot(tipX - 320, tipY - 240) < 0.01); + assert.ok(e.isMoving(), 'pulse remains active after glide is cancelled'); +}); + +test('cursor bloom is centered on the arrow hotspot', () => { + const e = new CursorEngine(); + e.completeAt(320, 240); + const gradients: number[][] = []; + const gradient = { addColorStop() {} }; + const ctx = { + createRadialGradient: (...args: number[]) => { + gradients.push(args); + return gradient; + }, + createLinearGradient: () => gradient, + beginPath() {}, + arc() {}, + fill() {}, + stroke() {}, + moveTo() {}, + lineTo() {}, + closePath() {}, + set fillStyle(_value: unknown) {}, + set strokeStyle(_value: unknown) {}, + set lineWidth(_value: number) {}, + set lineJoin(_value: CanvasLineJoin) {}, + } as unknown as CanvasRenderingContext2D; + + e.paint(ctx, 0, 0); + assert.deepEqual(gradients[0]?.slice(0, 5), [320, 240, 0, 320, 240]); +}); + +test('direct path planner never detours for short moves', () => { + const cases = [ + [100, 100, 120, 120], + [100, 100, 150, 100], + [100, 100, 180, 130], + ] as const; + for (const [x0, y0, x1, y1] of cases) { + const direct = Math.hypot(x1 - x0, y1 - y0); + const path = planDirectPath(x0, y0, x1, y1, REST_HEADING); + assert.ok(Math.abs(path.length - direct) < 0.001); + const end = path.sample(path.length); + assert.ok(Math.hypot(end.x - x1, end.y - y1) < 0.001); + } +}); + +test('click pulse clears over ~0.25s', () => { + const e = new CursorEngine(); + e.setSession('x'); + e.triggerClick(100, 100); + let ticks = 0; + while (e.isMoving() && ticks < 60) { e.tick(1 / 60); ticks++; } + assert.ok(ticks >= 14 && ticks <= 17, `~0.25s (${ticks} ticks)`); +}); + +test('palette: deterministic, default→default_blue, varied across ids', () => { + assert.equal(paletteForInstance('run-1').name, paletteForInstance('run-1').name); + assert.equal(paletteForInstance('default').name, 'default_blue'); + assert.equal(paletteForInstance('').name, 'default_blue'); + const names = new Set([1, 2, 3, 4, 5, 6, 7, 8, 9].map((n) => paletteForInstance(`run-${n}`).name)); + assert.ok(names.size >= 5, `varied (${names.size})`); + const g0 = gradientAt(defaultPalette(), 0).join(); + const g1 = gradientAt(defaultPalette(), 1).join(); + assert.notEqual(g0, g1, 'gradient endpoints differ'); +}); diff --git a/apps/desktop/src/main/__tests__/cursor-overlay-preload-contract.test.ts b/apps/desktop/src/main/__tests__/cursor-overlay-preload-contract.test.ts new file mode 100644 index 0000000000..6e01032aa0 --- /dev/null +++ b/apps/desktop/src/main/__tests__/cursor-overlay-preload-contract.test.ts @@ -0,0 +1,24 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; + +const source = await readFile( + new URL('../../../src/overlay/cursor-overlay-preload.ts', import.meta.url), + 'utf8', +); + +test('cursor overlay preload exposes only fixed presentation acknowledgements', () => { + assert.match(source, /ipcRenderer\.send\('overlay:presentation-phase'/); + assert.equal(source.match(/ipcRenderer\.send\(/g)?.length, 1); + assert.match( + source, + /ipcRenderer\.send\('overlay:presentation-phase', \{[\s\S]*sessionId,[\s\S]*generation,[\s\S]*actionId,[\s\S]*phase,[\s\S]*\}\)/, + ); + assert.match(source, /typeof sessionId !== 'string'/); + assert.match(source, /Number\.isInteger\(generation\)/); + assert.match(source, /phase !== 'readyForInteraction'/); + assert.match(source, /phase !== 'finished'/); + assert.match(source, /ipcRenderer\.on\('overlay:cancel'/); + assert.doesNotMatch(source, /ipcRenderer\.(?:invoke|sendSync)\(/); + assert.doesNotMatch(source, /screenX|screenY|CuAction/); +}); diff --git a/apps/desktop/src/main/__tests__/cursor-overlay-window.test.ts b/apps/desktop/src/main/__tests__/cursor-overlay-window.test.ts new file mode 100644 index 0000000000..16f8420dcc --- /dev/null +++ b/apps/desktop/src/main/__tests__/cursor-overlay-window.test.ts @@ -0,0 +1,442 @@ +// Behavior contract for the cursor overlay window controller. Drives it against a +// FakeCursorOverlayWindow (no Electron) and asserts the Path 18 invariants: +// - S14: focusable:false + setIgnoreMouseEvents(true,{forward:true}) armed BEFORE +// showInactive; never a .focus(); receive-only preload wired. +// - persistence: move() does NOT recreate the window (no teardown-per-move). +// - S15: coords are MAIN-computed window-local (screen − bounds.origin). +// - S13/S18: teardown is synchronous destroy() on clear/abort/destroyAll/supersede. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { createCursorOverlayController, cursorOverlayWindowOptions } from '../computer-use/cursor-overlay-window.js'; + +type Call = { m: string; args: unknown[] }; + +class FakeCursorOverlayWindow { + calls: Call[] = []; + sent: Array<{ channel: string; payload: unknown }> = []; + private readyCb: (() => void) | null = null; + private goneCb: (() => void) | null = null; + private presentationCb: + | ((payload: { + sessionId: string; + generation: number; + actionId: string; + phase: 'readyForInteraction' | 'finished'; + }) => void) + | null = null; + destroyed = false; + constructor(public options: Record) {} + private rec(m: string, ...args: unknown[]): void { this.calls.push({ m, args }); } + setIgnoreMouseEvents(ignore: boolean, opts?: unknown): void { this.rec('setIgnoreMouseEvents', ignore, opts); } + setAlwaysOnTop(flag: boolean, level?: unknown): void { this.rec('setAlwaysOnTop', flag, level); } + setVisibleOnAllWorkspaces(v: boolean, o?: unknown): void { this.rec('setVisibleOnAllWorkspaces', v, o); } + async loadFile(p: string): Promise { this.rec('loadFile', p); } + showInactive(): void { this.rec('showInactive'); } + isDestroyed(): boolean { return this.destroyed; } + destroy(): void { this.destroyed = true; this.rec('destroy'); } + send(channel: string, payload: unknown): void { this.sent.push({ channel, payload }); } + onReady(cb: () => void): void { this.readyCb = cb; } + onGone(cb: () => void): void { this.goneCb = cb; } + onPresentationPhase( + cb: (payload: { + sessionId: string; + generation: number; + actionId: string; + phase: 'readyForInteraction' | 'finished'; + }) => void, + ): void { + this.presentationCb = cb; + } + fireReady(): void { this.readyCb?.(); } + fireGone(): void { this.goneCb?.(); } + firePresentation( + actionId: string, + phase: 'readyForInteraction' | 'finished', + sessionId = 's', + generation = 1, + ): void { + this.presentationCb?.({ sessionId, generation, actionId, phase }); + } +} + +const BOUNDS = { x: 100, y: 50, width: 1440, height: 900 }; +function harness() { + const created: FakeCursorOverlayWindow[] = []; + let displayChanged: (() => void) | undefined; + const controller = createCursorOverlayController({ + createOverlayWindow: (options) => { + const w = new FakeCursorOverlayWindow(options as Record); + created.push(w); + return w as never; + }, + resolveOverlayBounds: () => BOUNDS, + preloadPath: '/fake/preload.cjs', + htmlPath: '/fake/overlay.html', + subscribeDisplayChanges: (cb) => { + displayChanged = cb; + return () => { displayChanged = undefined; }; + }, + }); + return { controller, created, fireDisplayChanged: () => displayChanged?.() }; +} + +test('S14 window options: focusable:false + non-interactive flags + receive-only preload', () => { + const opts = cursorOverlayWindowOptions(BOUNDS, '/p/preload.cjs') as Record; + assert.equal(opts.focusable, false); + assert.equal(opts.transparent, true); + assert.equal(opts.frame, false); + assert.equal(opts.alwaysOnTop, true); + assert.equal(opts.skipTaskbar, true); + assert.equal(opts.acceptFirstMouse, false); + assert.equal(opts.show, false); + assert.equal(opts.webPreferences.preload, '/p/preload.cjs'); + assert.equal(opts.webPreferences.sandbox, true); + assert.equal(opts.webPreferences.contextIsolation, true); + assert.equal(opts.webPreferences.nodeIntegration, false); +}); + +test('ensure(): arms click-through BEFORE showInactive, never focuses', () => { + const { controller, created } = harness(); + controller.ensure('sess-1'); + assert.equal(created.length, 1); + const w = created[0]; + assert.equal(w.options.focusable, false); + const order = w.calls.map((c) => c.m); + const armIdx = order.indexOf('setIgnoreMouseEvents'); + const showIdx = order.indexOf('showInactive'); + assert.equal(showIdx, -1, 'window stays hidden until did-finish-load'); + w.fireReady(); + const readyOrder = w.calls.map((c) => c.m); + assert.ok( + armIdx >= 0 && readyOrder.indexOf('showInactive') > armIdx, + `click-through armed before show (${readyOrder.join(',')})`, + ); + const arm = w.calls.find((c) => c.m === 'setIgnoreMouseEvents')!; + assert.deepEqual(arm.args, [true, { forward: true }]); + const aot = w.calls.find((c) => c.m === 'setAlwaysOnTop')!; + assert.deepEqual(aot.args, [true, 'screen-saver']); + assert.ok(!order.includes('focus'), 'never focuses'); +}); + +test('renderer loss and display changes teardown the live overlay', () => { + const gone = harness(); + gone.controller.ensure('s'); + gone.created[0].fireGone(); + assert.equal(gone.created[0].destroyed, true); + assert.equal(gone.controller.isActive(), false); + + const display = harness(); + display.controller.ensure('s'); + display.fireDisplayChanged(); + assert.equal(display.created[0].destroyed, true); + assert.equal(display.controller.isActive(), false); +}); + +test('persistence: move() does NOT recreate the window; sends window-local coords', () => { + const { controller, created } = harness(); + controller.move({ actionId: 'a0', sessionId: 's', screenX: 300, screenY: 250, kind: 'move' }); + controller.move({ actionId: 'a1', sessionId: 's', screenX: 500, screenY: 450, kind: 'click' }); + controller.move({ actionId: 'a2', sessionId: 's', screenX: 700, screenY: 650, kind: 'move' }); + assert.equal(created.length, 1, 'one window across 3 moves'); + const w = created[0]; + // before ready → queued; fire ready → reset first, then the 3 moves. + w.fireReady(); + assert.equal(w.sent[0].channel, 'overlay:reset'); + assert.deepEqual(w.sent[0].payload, { sessionId: 's', generation: 1 }); + const moves = w.sent.filter((s) => s.channel === 'overlay:move'); + assert.equal(moves.length, 3); + // window-local = screen − bounds.origin (100,50) + assert.deepEqual(moves[0].payload, { actionId: 'a0', x: 200, y: 200, kind: 'move', pressed: false }); + assert.deepEqual(moves[1].payload, { actionId: 'a1', x: 400, y: 400, kind: 'click', pressed: false }); + controller.move({ + actionId: 'a-instant', + sessionId: 's', + screenX: 520, + screenY: 470, + kind: 'click', + instant: true, + }); + const instantMove = w.sent.filter((s) => s.channel === 'overlay:move').at(-1); + assert.deepEqual(instantMove?.payload, { + actionId: 'a-instant', + x: 420, + y: 420, + kind: 'click', + pressed: false, + instant: true, + }); + // a post-ready move sends immediately + controller.move({ actionId: 'a3', sessionId: 's', screenX: 200, screenY: 150, kind: 'move' }); + const movesAfter = w.sent.filter((s) => s.channel === 'overlay:move'); + assert.equal(movesAfter.length, 5); + assert.deepEqual(movesAfter[4].payload, { actionId: 'a3', x: 100, y: 100, kind: 'move', pressed: false }); +}); + +test('multi-display union bounds preserve negative-origin secondary coordinates', () => { + const created: FakeCursorOverlayWindow[] = []; + const controller = createCursorOverlayController({ + createOverlayWindow: (options) => { + const window = new FakeCursorOverlayWindow(options as Record); + created.push(window); + return window as never; + }, + resolveOverlayBounds: () => ({ + x: -1920, + y: -180, + width: 4480, + height: 1620, + }), + preloadPath: '/fake/preload.cjs', + htmlPath: '/fake/overlay.html', + subscribeDisplayChanges: () => () => {}, + }); + controller.move({ + actionId: 'secondary', + sessionId: 's', + screenX: -960, + screenY: 540, + kind: 'click', + }); + created[0].fireReady(); + assert.deepEqual( + created[0].sent.find((message) => message.channel === 'overlay:move')?.payload, + { + actionId: 'secondary', + x: 960, + y: 720, + kind: 'click', + pressed: false, + }, + ); +}); + +test('presentation fence follows renderer phases and ignores stale action ids', async () => { + const { controller, created } = harness(); + const fence = controller.move({ + actionId: 'live', + sessionId: 's', + screenX: 300, + screenY: 250, + kind: 'move', + }); + const observed: string[] = []; + fence.readyForInteraction.then(() => observed.push('ready')); + fence.finished.then(() => observed.push('finished')); + created[0].firePresentation('stale', 'finished'); + created[0].firePresentation('live', 'finished', 'other', 1); + created[0].firePresentation('live', 'finished', 's', 2); + await Promise.resolve(); + assert.deepEqual(observed, []); + created[0].firePresentation('live', 'readyForInteraction'); + await Promise.resolve(); + assert.deepEqual(observed, ['ready']); + created[0].firePresentation('live', 'finished'); + await Promise.resolve(); + assert.deepEqual(observed, ['ready', 'finished']); +}); + +test('load failure tears down the window and releases pending fences', async () => { + const created: FakeCursorOverlayWindow[] = []; + const controller = createCursorOverlayController({ + createOverlayWindow: (options) => { + const window = new FakeCursorOverlayWindow(options as Record); + window.loadFile = async () => { + throw new Error('load failed'); + }; + created.push(window); + return window as never; + }, + resolveOverlayBounds: () => BOUNDS, + preloadPath: '/fake/preload.cjs', + htmlPath: '/fake/overlay.html', + subscribeDisplayChanges: () => () => {}, + }); + const fence = controller.move({ + actionId: 'failed-load', + sessionId: 's', + screenX: 300, + screenY: 250, + kind: 'move', + }); + await Promise.all([fence.readyForInteraction, fence.finished]); + assert.equal(created[0].destroyed, true); + assert.equal(controller.isActive(), false); +}); + +test('finished presentation releases ownership for a later semantic completion', async () => { + const { controller, created } = harness(); + const fence = controller.move({ + actionId: 'coordinate', + sessionId: 's', + screenX: 300, + screenY: 250, + kind: 'move', + }); + const w = created[0]; + w.fireReady(); + w.firePresentation('coordinate', 'finished'); + await fence.finished; + + controller.complete({ + actionId: 'semantic', + sessionId: 's', + screenX: 320, + screenY: 260, + kind: 'click', + pulse: true, + }); + assert.equal( + w.sent.filter((message) => message.channel === 'overlay:complete').at(-1)?.payload + && (w.sent.filter((message) => message.channel === 'overlay:complete').at(-1)?.payload as { actionId: string }).actionId, + 'semantic', + ); +}); + +test('supersede and teardown release pending presentation fences', async () => { + const { controller } = harness(); + const first = controller.move({ + actionId: 'first', + sessionId: 's', + screenX: 300, + screenY: 250, + kind: 'move', + }); + controller.move({ + actionId: 'second', + sessionId: 's', + screenX: 400, + screenY: 350, + kind: 'move', + }); + await Promise.all([first.readyForInteraction, first.finished]); + const second = controller.move({ + actionId: 'third', + sessionId: 's', + screenX: 500, + screenY: 450, + kind: 'click', + }); + controller.destroyAll(); + await Promise.all([second.readyForInteraction, second.finished]); +}); + +test('complete() sends exact backend coordinate only for the live action', () => { + const { controller, created } = harness(); + controller.move({ actionId: 'a1', sessionId: 's', screenX: 500, screenY: 450, kind: 'click' }); + const w = created[0]; + w.fireReady(); + + controller.complete({ + actionId: 'stale', + sessionId: 's', + screenX: 500, + screenY: 450, + kind: 'click', + pulse: true, + }); + assert.equal(w.sent.filter((message) => message.channel === 'overlay:complete').length, 0); + + controller.complete({ + actionId: 'a1', + sessionId: 's', + screenX: 500, + screenY: 450, + kind: 'click', + pulse: true, + }); + const completed = w.sent.filter((message) => message.channel === 'overlay:complete'); + assert.deepEqual(completed[0]?.payload, { + actionId: 'a1', + x: 400, + y: 400, + kind: 'click', + pulse: true, + }); +}); + +test('cancel() settles the live fence and sends no coordinate', async () => { + const { controller, created } = harness(); + const fence = controller.move({ + actionId: 'failed', + sessionId: 's', + screenX: 500, + screenY: 450, + kind: 'click', + instant: true, + }); + const w = created[0]; + w.fireReady(); + + controller.cancel({ actionId: 'stale', sessionId: 's' }); + assert.equal(w.sent.filter((message) => message.channel === 'overlay:cancel').length, 0); + controller.cancel({ actionId: 'failed', sessionId: 's' }); + await Promise.all([fence.readyForInteraction, fence.finished]); + + const cancelled = w.sent.filter((message) => message.channel === 'overlay:cancel'); + assert.deepEqual(cancelled, [{ + channel: 'overlay:cancel', + payload: { actionId: 'failed' }, + }]); + assert.equal('x' in (cancelled[0]?.payload as object), false); + assert.equal('y' in (cancelled[0]?.payload as object), false); +}); + +test('complete() can present an executor-resolved semantic point without a speculative begin move', () => { + const { controller, created } = harness(); + controller.ensure('s'); + const w = created[0]; + w.fireReady(); + + controller.complete({ + actionId: 'semantic', + sessionId: 's', + screenX: 320, + screenY: 260, + kind: 'click', + pulse: true, + }); + const completed = w.sent.filter((message) => message.channel === 'overlay:complete'); + assert.deepEqual(completed[0]?.payload, { + actionId: 'semantic', + x: 220, + y: 210, + kind: 'click', + pulse: true, + }); +}); + +test('teardown: clearForSession / abort / destroyAll destroy synchronously; supersede on session change', () => { + const { controller, created } = harness(); + controller.move({ actionId: 'a0', sessionId: 's1', screenX: 300, screenY: 250, kind: 'move' }); + controller.clearForSession('other'); // non-match ignored + assert.ok(!created[0].destroyed, 'non-matching clear ignored'); + controller.clearForSession('s1'); + assert.ok(created[0].destroyed, 'matching clear destroys'); + + // supersede: a different session destroys the old window and creates a new one + const h = harness(); + h.controller.ensure('sA'); + h.controller.ensure('sB'); + assert.ok(h.created[0].destroyed, 'old session window superseded'); + assert.equal(h.created.length, 2); + assert.ok(!h.created[1].destroyed); + + // abort keys on actionId + const h2 = harness(); + h2.controller.move({ actionId: 'act-9', sessionId: 's', screenX: 1, screenY: 1, kind: 'move' }); + h2.controller.abort('stale'); // ignored + assert.ok(!h2.created[0].destroyed); + h2.controller.abort('act-9'); + assert.ok(h2.created[0].destroyed); +}); + +test('fail-closed: empty ids and non-finite coords are no-ops', () => { + const { controller, created } = harness(); + controller.ensure(''); + assert.equal(created.length, 0, 'empty sessionId → no window'); + controller.move({ actionId: 'a', sessionId: '', screenX: 10, screenY: 10, kind: 'move' }); + assert.equal(created.length, 0, 'empty sessionId move → no window'); + controller.move({ actionId: 'a', sessionId: 's', screenX: NaN, screenY: 10, kind: 'move' }); + assert.equal(created.length, 0, 'NaN coord → no window'); +}); diff --git a/apps/desktop/src/main/computer-use-host.ts b/apps/desktop/src/main/computer-use-host.ts index 7b1421a9fa..256ff4d59f 100644 --- a/apps/desktop/src/main/computer-use-host.ts +++ b/apps/desktop/src/main/computer-use-host.ts @@ -14,6 +14,7 @@ import { selectComputerUseBackend, type SelectedComputerUseBackend, } from '@maka/computer-use'; +import type { CuOverlayHook } from '@maka/runtime'; export interface ComputerUseHostState { selected: SelectedComputerUseBackend; @@ -43,6 +44,7 @@ export function createComputerUseHost(input: { mimeType: string, ) => { base64: string; mimeType: 'image/png' | 'image/jpeg' }; physicalInputRecentlyActive?: () => boolean | Promise; + overlay?: CuOverlayHook; }): ComputerUseHostState { const manifestPath = input.manifestPath ?? (input.isPackaged ? join(input.resourcesPath, 'bundled-tools.json') @@ -99,6 +101,7 @@ export function createComputerUseHost(input: { ...(input.physicalInputRecentlyActive ? { physicalInputRecentlyActive: input.physicalInputRecentlyActive } : {}), + ...(input.overlay ? { overlay: input.overlay } : {}), }), binaryPath, expectedBinarySha256, diff --git a/apps/desktop/src/main/computer-use/cursor-overlay-window.ts b/apps/desktop/src/main/computer-use/cursor-overlay-window.ts new file mode 100644 index 0000000000..f41e35be3d --- /dev/null +++ b/apps/desktop/src/main/computer-use/cursor-overlay-window.ts @@ -0,0 +1,428 @@ +/** + * Cursor overlay window (main-process half) — the Maka-owned, Codex-style agent + * cursor. A transparent, always-on-top, click-through BrowserWindow that hosts a + * Canvas running the ported CursorEngine (Dubins glide + spring). MAIN drives it + * with per-action coordinates; the window persists across actions and repositions + * the cursor live over a one-way `overlay:move` channel (no teardown-per-move). + * + * Path 18 gates: + * - S13: action/session-scoped lifecycle; teardown is synchronous + event-driven, + * no timer keeps it alive. + * - S14 (load-bearing): `focusable:false` + `setIgnoreMouseEvents(true,{forward:true})` + * armed BEFORE show + `showInactive()` (never `.focus()`). The preload is + * RECEIVE-ONLY (main→renderer), so the overlay can never call back / inject. + * - S15: MAIN owns coordinates; the renderer only paints what MAIN sends. + * - S18: teardown is a single synchronous `destroy()`. + * + * Electron is required lazily so the module loads under `node --test`; tests + * inject a fake window factory + bounds resolver. + */ +import { createRequire } from 'node:module'; +import { basename, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { BrowserWindowConstructorOptions, Rectangle } from 'electron'; +import type { CuPresentationFence } from '@maka/runtime'; + +const requireElectron = createRequire(import.meta.url); + +// Shared cursor-move contract lives in @maka/computer-use so the CLI can drive the +// same hook against a headless sink. This controller is the Electron implementation +// of that sink (it also satisfies OverlayCursorSink structurally via ensure/move). +export type { CursorActionKind, CursorMoveInput } from '@maka/computer-use'; +import type { + CursorCancelInput, + CursorCompleteInput, + CursorMoveInput, +} from '@maka/computer-use'; + +/** Minimal window surface the controller drives (fake-able in node --test). */ +export interface CursorOverlayWindowLike { + setIgnoreMouseEvents(ignore: boolean, options?: { forward?: boolean }): void; + setAlwaysOnTop(flag: boolean, level?: string): void; + setVisibleOnAllWorkspaces(visible: boolean, options?: { visibleOnFullScreen?: boolean }): void; + loadFile(path: string): Promise; + showInactive(): void; + isDestroyed(): boolean; + destroy(): void; + /** webContents.send — the one-way main→renderer push. */ + send(channel: string, payload: unknown): void; + /** Fire cb once the page has loaded (webContents 'did-finish-load'). */ + onReady(cb: () => void): void; + onGone(cb: () => void): void; + onPresentationPhase( + cb: (payload: { + sessionId: string; + generation: number; + actionId: string; + phase: 'readyForInteraction' | 'finished'; + }) => void, + ): void; +} + +export interface CreateCursorOverlayControllerDeps { + createOverlayWindow?: (options: BrowserWindowConstructorOptions) => CursorOverlayWindowLike; + resolveOverlayBounds?: () => Rectangle; + /** Absolute path to the built overlay preload (dist/overlay/cursor-overlay-preload.cjs). */ + preloadPath?: string; + /** Absolute path to the built overlay html (dist/overlay/cursor-overlay.html). */ + htmlPath?: string; + onDisplayFrame?: (input: { actionId: string; completedAt: number; displayedAt: number }) => void; + subscribeDisplayChanges?: (cb: () => void) => () => void; +} + +export interface CursorOverlayController { + /** Lazily create/refresh the overlay for a session (palette from sessionId). */ + ensure(sessionId: string): void; + /** Move the cursor to a per-action screen coordinate (creates the window if needed). */ + move(input: CursorMoveInput): CuPresentationFence; + /** Reconcile the display with the coordinate where backend execution completed. */ + complete(input: CursorCompleteInput): void; + /** Finish a failed presentation without inventing a completion coordinate. */ + cancel(input: CursorCancelInput): void; + /** Per-session teardown (the clearComputerUseOverlay(sessionId) bag). */ + clearForSession(sessionId: string): void; + /** User abort (Esc) — tears down when actionId matches the live overlay. */ + abort(actionId: string): void; + /** Unconditional teardown (window close / quit). */ + destroyAll(): void; + isActive(): boolean; + getSessionId(): string | null; +} + +/** S14 window options — the focus/click-through contract surface, one literal. */ +export function cursorOverlayWindowOptions(bounds: Rectangle, preloadPath: string): BrowserWindowConstructorOptions { + return { + x: bounds.x, + y: bounds.y, + width: bounds.width, + height: bounds.height, + focusable: false, // S14: never take keyboard focus from the driven app + transparent: true, + frame: false, + hasShadow: false, + alwaysOnTop: true, + skipTaskbar: true, + resizable: false, + movable: false, + minimizable: false, + maximizable: false, + fullscreenable: false, + acceptFirstMouse: false, + show: false, // shown via showInactive() only after click-through is armed + backgroundColor: '#00000000', + enableLargerThanScreen: true, + webPreferences: { + // Receive-only preload: exposes ipcRenderer.on callbacks, never send/invoke. + preload: preloadPath, + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + webSecurity: true, + }, + }; +} + +function defaultOverlayDistDir(): string { + // Robust to BOTH layouts: prod tsc compiles this to dist/main/computer-use/*.js, + // while `npm run dev` esbuild-bundles it into dist/main/main.js — either way the + // overlay lives at /overlay. Walk up to the 'dist' root and join 'overlay'. + const start = dirname(fileURLToPath(import.meta.url)); + let dir = start; + for (let i = 0; i < 6; i++) { + if (basename(dir) === 'dist') return join(dir, 'overlay'); + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return join(start, '..', '..', 'overlay'); // fallback: assume dist/main/computer-use +} + +export function createCursorOverlayController( + deps: CreateCursorOverlayControllerDeps = {}, +): CursorOverlayController { + const createOverlayWindow = deps.createOverlayWindow ?? defaultCreateOverlayWindow; + const resolveOverlayBounds = deps.resolveOverlayBounds ?? defaultResolveOverlayBounds; + const preloadPath = deps.preloadPath ?? join(defaultOverlayDistDir(), 'cursor-overlay-preload.cjs'); + const htmlPath = deps.htmlPath ?? join(defaultOverlayDistDir(), 'cursor-overlay.html'); + const subscribeDisplayChanges = deps.subscribeDisplayChanges + ?? defaultSubscribeDisplayChanges; + + let win: CursorOverlayWindowLike | null = null; + let sessionId: string | null = null; + let actionId: string | null = null; + let bounds: Rectangle = { x: 0, y: 0, width: 0, height: 0 }; + let ready = false; + let generation = 0; + let queue: Array<{ channel: string; payload: unknown }> = []; + let unsubscribeDisplayChanges: (() => void) | undefined; + let presentation: + | { + actionId: string; + ready: () => void; + finish: () => void; + fence: CuPresentationFence; + completedAt?: number; + } + | undefined; + + function createPresentation(action: string): NonNullable { + let readyForInteraction!: () => void; + let finished!: () => void; + const fence: CuPresentationFence = { + readyForInteraction: new Promise((resolve) => { readyForInteraction = resolve; }), + finished: new Promise((resolve) => { finished = resolve; }), + }; + return { + actionId: action, + ready: readyForInteraction, + finish: finished, + fence, + }; + } + + function settlePresentation(): void { + presentation?.ready(); + presentation?.finish(); + presentation = undefined; + } + + function teardown(): void { + const w = win; + win = null; + sessionId = null; + actionId = null; + settlePresentation(); + ready = false; + queue = []; + unsubscribeDisplayChanges?.(); + unsubscribeDisplayChanges = undefined; + if (w && !w.isDestroyed()) w.destroy(); + } + + function push(channel: string, payload: unknown): void { + if (!win) return; + if (ready) win.send(channel, payload); + else queue.push({ channel, payload }); + } + + function ensure(nextSessionId: string): void { + if (typeof nextSessionId !== 'string' || nextSessionId.length === 0) return; + if (win && !win.isDestroyed() && sessionId === nextSessionId) return; + // Different session (or dead window) → supersede so no orphan survives. + if (win) teardown(); + + bounds = resolveOverlayBounds(); + generation += 1; + const windowGeneration = generation; + const w = createOverlayWindow(cursorOverlayWindowOptions(bounds, preloadPath)); + // S14 (load-bearing): arm click + focus pass-through BEFORE the window shows. + w.setIgnoreMouseEvents(true, { forward: true }); + w.setAlwaysOnTop(true, 'screen-saver'); + try { + w.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }); + } catch { + /* not supported everywhere; best-effort */ + } + win = w; + sessionId = nextSessionId; + ready = false; + queue = []; + unsubscribeDisplayChanges = subscribeDisplayChanges(() => { + if (win === w) teardown(); + }); + w.onReady(() => { + if (win !== w) return; // superseded during load + ready = true; + w.send('overlay:reset', { + sessionId: nextSessionId, + generation: windowGeneration, + }); + for (const m of queue) w.send(m.channel, m.payload); + queue = []; + w.showInactive(); + }); + w.onGone(() => { + if (win === w) teardown(); + }); + w.onPresentationPhase((payload) => { + if ( + win !== w + || payload.sessionId !== sessionId + || payload.generation !== windowGeneration + || !presentation + || payload.actionId !== presentation.actionId + ) return; + if (payload.phase === 'readyForInteraction') presentation.ready(); + else { + const completed = presentation; + presentation = undefined; + completed.ready(); + completed.finish(); + if (completed.completedAt !== undefined) { + deps.onDisplayFrame?.({ + actionId: completed.actionId, + completedAt: completed.completedAt, + displayedAt: Date.now(), + }); + } + if (actionId === completed.actionId) actionId = null; + } + }); + void w.loadFile(htmlPath).catch(() => { + if (win === w) teardown(); + }); + } + + function move(input: CursorMoveInput): CuPresentationFence { + if ( + typeof input.sessionId !== 'string' + || input.sessionId.length === 0 + || !Number.isFinite(input.screenX) + || !Number.isFinite(input.screenY) + ) { + return { readyForInteraction: Promise.resolve(), finished: Promise.resolve() }; + } + ensure(input.sessionId); + settlePresentation(); + actionId = input.actionId; + const nextPresentation = createPresentation(input.actionId); + presentation = nextPresentation; + // Screen → window-local so the renderer paints at origin (0,0). + push('overlay:move', { + actionId: input.actionId, + x: input.screenX - bounds.x, + y: input.screenY - bounds.y, + kind: input.kind, + pressed: input.pressed === true, + ...(input.instant === true ? { instant: true } : {}), + }); + return nextPresentation.fence; + } + + function complete(input: CursorCompleteInput): void { + if (typeof input.sessionId !== 'string' || input.sessionId.length === 0) return; + if (!Number.isFinite(input.screenX) || !Number.isFinite(input.screenY)) return; + if (input.sessionId !== sessionId) return; + if (presentation && input.actionId !== presentation.actionId) return; + actionId = input.actionId; + if (presentation?.actionId === input.actionId) { + presentation.completedAt = Date.now(); + } + push('overlay:complete', { + actionId: input.actionId, + x: input.screenX - bounds.x, + y: input.screenY - bounds.y, + kind: input.kind, + pulse: input.pulse, + }); + } + + function cancel(input: CursorCancelInput): void { + if (typeof input.sessionId !== 'string' || input.sessionId.length === 0) return; + if (input.sessionId !== sessionId || input.actionId !== actionId) return; + push('overlay:cancel', { actionId: input.actionId }); + settlePresentation(); + actionId = null; + } + + function clearForSession(id: string): void { + if (typeof id !== 'string' || id.length === 0) return; + if (id !== sessionId) return; + teardown(); + } + function abort(id: string): void { + if (typeof id !== 'string' || id.length === 0) return; + if (id !== actionId) return; + teardown(); + } + + return { + ensure, + move, + complete, + cancel, + clearForSession, + abort, + destroyAll: teardown, + isActive: () => win !== null, + getSessionId: () => sessionId, + }; +} + +function defaultCreateOverlayWindow(options: BrowserWindowConstructorOptions): CursorOverlayWindowLike { + const { BrowserWindow } = requireElectron('electron') as typeof import('electron'); + const bw = new BrowserWindow(options); + return { + setIgnoreMouseEvents: (ignore, opts) => bw.setIgnoreMouseEvents(ignore, opts), + setAlwaysOnTop: (flag, level) => bw.setAlwaysOnTop(flag, level as Parameters[1]), + setVisibleOnAllWorkspaces: (visible, opts) => bw.setVisibleOnAllWorkspaces(visible, opts), + loadFile: (path) => bw.loadFile(path), + showInactive: () => bw.showInactive(), + isDestroyed: () => bw.isDestroyed(), + destroy: () => bw.destroy(), + send: (channel, payload) => { if (!bw.isDestroyed()) bw.webContents.send(channel, payload); }, + onReady: (cb) => bw.webContents.once('did-finish-load', cb), + onGone: (cb) => { + bw.once('closed', cb); + bw.webContents.once('render-process-gone', cb); + bw.webContents.once('destroyed', cb); + }, + onPresentationPhase: (cb) => { + bw.webContents.on('ipc-message', (_event, channel, payload) => { + if (channel !== 'overlay:presentation-phase') return; + if (!payload || typeof payload !== 'object') return; + const candidate = payload as Record; + if ( + typeof candidate.actionId !== 'string' + || typeof candidate.sessionId !== 'string' + || !Number.isInteger(candidate.generation) + || ( + candidate.phase !== 'readyForInteraction' + && candidate.phase !== 'finished' + ) + ) return; + cb({ + sessionId: candidate.sessionId, + generation: candidate.generation as number, + actionId: candidate.actionId, + phase: candidate.phase, + }); + }); + }, + }; +} + +function defaultSubscribeDisplayChanges(cb: () => void): () => void { + const { screen } = requireElectron('electron') as typeof import('electron'); + const added = () => cb(); + const removed = () => cb(); + const changed = () => cb(); + screen.on('display-added', added); + screen.on('display-removed', removed); + screen.on('display-metrics-changed', changed); + return () => { + screen.removeListener('display-added', added); + screen.removeListener('display-removed', removed); + screen.removeListener('display-metrics-changed', changed); + }; +} + +function defaultResolveOverlayBounds(): Rectangle { + const { screen } = requireElectron('electron') as typeof import('electron'); + const displays = screen.getAllDisplays(); + if (displays.length === 0) return screen.getPrimaryDisplay().bounds; + const left = Math.min(...displays.map((display) => display.bounds.x)); + const top = Math.min(...displays.map((display) => display.bounds.y)); + const right = Math.max(...displays.map( + (display) => display.bounds.x + display.bounds.width, + )); + const bottom = Math.max(...displays.map( + (display) => display.bounds.y + display.bounds.height, + )); + return { + x: left, + y: top, + width: right - left, + height: bottom - top, + }; +} diff --git a/apps/desktop/src/main/main-window.ts b/apps/desktop/src/main/main-window.ts index 33681149ee..9796f1916d 100644 --- a/apps/desktop/src/main/main-window.ts +++ b/apps/desktop/src/main/main-window.ts @@ -44,6 +44,7 @@ interface MainWindowControllerDeps { // main.ts computes this from the same isE2e gate that also guards userData // and the fake backend, so main-window.ts owns no env policy of its own. startHidden: boolean; + onClose?: () => void; } let mainWindow: BrowserWindow | null = null; @@ -339,6 +340,7 @@ export function createMainWindowController(deps: MainWindowControllerDeps): Main mainWindow.on('unmaximize', scheduleSave); mainWindow.on('close', () => { clearShowFallbackTimer(); + deps.onClose?.(); if (saveTimer) clearTimeout(saveTimer); // The window owns the embedded-browser views (children of its contentView); // tear them down so their WebContents close with it instead of leaking. diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index ca24791a8a..3a42084033 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -172,6 +172,8 @@ import { computerUseServiceHealth, createComputerUseHost, } from './computer-use-host.js'; +import { createCursorOverlayController } from './computer-use/cursor-overlay-window.js'; +import { createComputerUseOverlayHook } from '@maka/computer-use'; import { releaseBrowserSession } from './browser/session.js'; import { createMainWindowController } from './main-window.js'; import { createDailyReviewMainService } from './daily-review-main.js'; @@ -479,11 +481,13 @@ const systemPromptService = createSystemPromptMainService({ // over 31s). The E2E harness sets it, not the workflow — see fixtures.ts. const startHidden = (Boolean(visualSmokeFixture) || isE2e) && process.env.MAKA_E2E_SHOW_WINDOW !== '1'; +let onMainWindowClose = (): void => {}; const mainWindowController = createMainWindowController({ workspaceRoot, visualSmokeFixture, settingsStore, startHidden, + onClose: () => onMainWindowClose(), }); // Shared by 'second-instance' and 'activate': focus the existing window, or // create one if all windows were closed while the app (macOS: still in the @@ -545,6 +549,8 @@ const officeTools: MakaTool[] = [buildOfficeDocumentTool(), buildOfficeDocumentE // WebContentsView via the BrowserViewHost the desktop provides in registerIpc; // outside the app (no host) they report the browser as unavailable. const browserTools: MakaTool[] = buildBrowserTools(); +const computerUseOverlay = createCursorOverlayController(); +onMainWindowClose = () => computerUseOverlay.destroyAll(); const computerUseHost = createComputerUseHost({ isPackaged: app.isPackaged, resourcesPath: process.resourcesPath, @@ -562,6 +568,7 @@ const computerUseHost = createComputerUseHost({ } }, physicalInputRecentlyActive: () => powerMonitor.getSystemIdleTime() < 1, + overlay: createComputerUseOverlayHook(computerUseOverlay), }); const computerUse = computerUseHost.selected; const computerUseTools = computerUse.tools; @@ -1369,6 +1376,7 @@ function registerIpc(): void { }); }); ipcMain.handle('sessions:stop', async (_event, sessionId: string, input?: { source?: 'stop_button' }) => { + computerUseOverlay.clearForSession(sessionId); computerUseTools.clearSession(sessionId); await runtime.stopSession(sessionId, normalizeStopSessionInput(input)); emitSessionsChanged('status-change', sessionId); @@ -1449,6 +1457,7 @@ function registerIpc(): void { return session; }); ipcMain.handle('sessions:archive', async (_event, sessionId: string) => { + computerUseOverlay.clearForSession(sessionId); computerUseTools.clearSession(sessionId); await runtime.archive(sessionId); // An archived conversation is no longer shown: drop its browser connection @@ -1525,6 +1534,7 @@ function registerIpc(): void { return next; }); ipcMain.handle('sessions:remove', async (_event, sessionId: string) => { + computerUseOverlay.clearForSession(sessionId); computerUseTools.clearSession(sessionId); await runtime.remove(sessionId); // Drop the conversation's browser connection and destroy its view (no-op @@ -1810,6 +1820,7 @@ async function streamEvents( } if (isTurnStatusChangingSessionEvent(event)) { emitSessionsChanged('turn-status-change', sessionId); + computerUseOverlay.clearForSession(sessionId); computerUseTools.clearSession(sessionId); } } @@ -1841,6 +1852,7 @@ async function streamEvents( openGateway.publishSessionEvent(sessionId, event); emitSessionsChanged('status-change', sessionId); emitSessionsChanged('turn-status-change', sessionId); + computerUseOverlay.clearForSession(sessionId); computerUseTools.clearSession(sessionId); if (!finalAppendBroadcasted) { emitSessionsChanged('message-appended', sessionId); @@ -2147,6 +2159,7 @@ async function runBackgroundStartup(): Promise { } app.on('window-all-closed', () => { + computerUseOverlay.destroyAll(); if (process.platform !== 'darwin') app.quit(); }); @@ -2171,6 +2184,7 @@ async function runBeforeQuitCleanup(): Promise { planReminders.stopTimers(); dailyReview.stopScheduler(); const results = await Promise.allSettled([ + Promise.resolve().then(() => computerUseOverlay.destroyAll()), Promise.resolve().then(() => computerUse.backend?.dispose?.()), botRegistry.stopAll(), openGateway.stop(), diff --git a/apps/desktop/src/overlay/cursor-overlay-preload.ts b/apps/desktop/src/overlay/cursor-overlay-preload.ts new file mode 100644 index 0000000000..282bab83f3 --- /dev/null +++ b/apps/desktop/src/overlay/cursor-overlay-preload.ts @@ -0,0 +1,35 @@ +// Overlay preload. Main owns all coordinates and actions. Renderer may send only +// a fixed presentation-phase acknowledgement keyed by the action id. +import { contextBridge, ipcRenderer } from 'electron'; + +contextBridge.exposeInMainWorld('cursorOverlay', { + onMove: (cb: (p: unknown) => void): void => { + ipcRenderer.on('overlay:move', (_e, payload) => cb(payload)); + }, + onReset: (cb: (p: unknown) => void): void => { + ipcRenderer.on('overlay:reset', (_e, payload) => cb(payload)); + }, + onComplete: (cb: (p: unknown) => void): void => { + ipcRenderer.on('overlay:complete', (_e, payload) => cb(payload)); + }, + onCancel: (cb: (p: unknown) => void): void => { + ipcRenderer.on('overlay:cancel', (_e, payload) => cb(payload)); + }, + reportPresentationPhase: ( + sessionId: string, + generation: number, + actionId: string, + phase: 'readyForInteraction' | 'finished', + ): void => { + if (typeof sessionId !== 'string' || sessionId.length === 0) return; + if (!Number.isInteger(generation) || generation < 1) return; + if (typeof actionId !== 'string' || actionId.length === 0) return; + if (phase !== 'readyForInteraction' && phase !== 'finished') return; + ipcRenderer.send('overlay:presentation-phase', { + sessionId, + generation, + actionId, + phase, + }); + }, +}); diff --git a/apps/desktop/src/overlay/cursor-overlay.html b/apps/desktop/src/overlay/cursor-overlay.html new file mode 100644 index 0000000000..f334a22d37 --- /dev/null +++ b/apps/desktop/src/overlay/cursor-overlay.html @@ -0,0 +1,16 @@ + + + + + + + + + + + diff --git a/apps/desktop/src/overlay/cursor-overlay.ts b/apps/desktop/src/overlay/cursor-overlay.ts new file mode 100644 index 0000000000..80fe3c4c17 --- /dev/null +++ b/apps/desktop/src/overlay/cursor-overlay.ts @@ -0,0 +1,133 @@ +// Overlay renderer entry — hosts the ported CursorEngine on a full-window canvas. +// Receives MAIN-computed, window-local coordinates over a one-way bridge and +// animates the agent cursor. Display-only: it never sends anything back (S15). +// The rAF loop blocks on idle (stops when the engine is at rest; the last frame +// persists), so a resting cursor costs no CPU. +import { CursorEngine } from '../renderer/computer-use-overlay/engine/cursor-engine.js'; + +interface MovePayload { + actionId: string; + x: number; + y: number; + kind?: 'move' | 'click' | 'drag' | 'scroll'; + pressed?: boolean; + instant?: boolean; +} +interface CompletePayload { actionId?: string; x: number; y: number; kind?: 'move' | 'click' | 'drag' | 'scroll'; pulse?: boolean } +interface CancelPayload { actionId: string } +interface ResetPayload { sessionId: string; generation: number } +declare global { + interface Window { + cursorOverlay?: { + onMove(cb: (p: MovePayload) => void): void; + onComplete(cb: (p: CompletePayload) => void): void; + onCancel(cb: (p: CancelPayload) => void): void; + onReset(cb: (p: ResetPayload) => void): void; + reportPresentationPhase( + sessionId: string, + generation: number, + actionId: string, + phase: 'readyForInteraction' | 'finished', + ): void; + }; + } +} + +const canvas = document.getElementById('cursor') as HTMLCanvasElement; +const ctx = canvas.getContext('2d')!; +let dpr = window.devicePixelRatio || 1; + +function resize(): void { + dpr = window.devicePixelRatio || 1; + canvas.width = Math.floor(window.innerWidth * dpr); + canvas.height = Math.floor(window.innerHeight * dpr); + canvas.style.width = `${window.innerWidth}px`; + canvas.style.height = `${window.innerHeight}px`; +} +resize(); +window.addEventListener('resize', resize); + +const engine = new CursorEngine(); +let running = false; +let last = 0; +let activeActionId: string | null = null; +let readySent = false; +let waitForNativeCompletion = false; +let sessionId = ''; +let generation = 0; + +function reportPhase(phase: 'readyForInteraction' | 'finished'): void { + if (!activeActionId) return; + window.cursorOverlay?.reportPresentationPhase( + sessionId, + generation, + activeActionId, + phase, + ); +} + +function loop(now: number): void { + const dt = Math.min(0.05, (now - last) / 1000); + last = now; + engine.tick(dt); + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + engine.paint(ctx, 0, 0); // MAIN sends window-local coords, so origin is (0,0) + if ( + !readySent + && ( + !engine.hasMotionPath() + || engine.motionProgress() >= 0.82 + || engine.motionDistanceRemaining() <= 24 + ) + ) { + readySent = true; + reportPhase('readyForInteraction'); + } + if (engine.isMoving()) { + requestAnimationFrame(loop); + } else { + running = false; // block on idle — leave the last frame painted + if (!waitForNativeCompletion) { + reportPhase('finished'); + activeActionId = null; + } + } +} +function kick(): void { + if (!running) { + running = true; + last = performance.now(); + requestAnimationFrame(loop); + } +} + +window.cursorOverlay?.onReset((p) => { + sessionId = p.sessionId; + generation = p.generation; + engine.setSession(sessionId); + kick(); +}); +window.cursorOverlay?.onMove((p) => { + activeActionId = p.actionId; + readySent = false; + waitForNativeCompletion = true; + if (p.instant === true) engine.completeAt(p.x, p.y); + else engine.moveTo(p.x, p.y); + engine.pressed = p.pressed === true; + kick(); +}); +window.cursorOverlay?.onComplete((p) => { + if (p.actionId && activeActionId && p.actionId !== activeActionId) return; + if (p.actionId) activeActionId = p.actionId; + waitForNativeCompletion = false; + engine.completeAt(p.x, p.y, p.pulse === true); + kick(); +}); +window.cursorOverlay?.onCancel((p) => { + if (!activeActionId || p.actionId !== activeActionId) return; + waitForNativeCompletion = false; + reportPhase('finished'); + activeActionId = null; +}); diff --git a/apps/desktop/src/renderer/computer-use-overlay/engine/cursor-engine.ts b/apps/desktop/src/renderer/computer-use-overlay/engine/cursor-engine.ts new file mode 100644 index 0000000000..0219897272 --- /dev/null +++ b/apps/desktop/src/renderer/computer-use-overlay/engine/cursor-engine.ts @@ -0,0 +1,223 @@ +// Agent-cursor render engine. The arrow and palette derive from trycua/cua's +// cursor overlay; Maka owns the direct-motion and backend-hotspot semantics. +// +// MoveTo uses a direct smootherstep glide (300→900→200 pts/s). Pointer actions +// snap to the backend coordinate; only explicit mouse_move requests animate. +// The real system cursor is NEVER touched — this only paints a fake cursor into a +// click-through overlay (see the empirical 0px-move finding). +// +// All units are logical points. The caller scales the canvas by devicePixelRatio +// once, then paints in logical px. +import { planDirectPath, PlannedPath } from './dubins.js'; +import { makaBrandPalette, type Palette, type Rgb, rgba } from './palette.js'; + +const PI = Math.PI; + +const PEAK_SPEED = 900; +const MIN_START_SPEED = 300; +const MIN_END_SPEED = 200; +const ARROW_TIP_LENGTH = 14; +const SENTINEL = -200; // off-screen start; paint hidden while pos.x < -100 + +/** Resting arrow heading: 45° so the tip points up-left like a normal cursor. */ +const REST_HEADING = PI / 4; + +export class CursorEngine { + pos: [number, number] = [SENTINEL, SENTINEL]; + heading = REST_HEADING; + private path: PlannedPath | null = null; + private dist = 0; + private clickT: number | null = null; + private clickPoint: [number, number] | null = null; + private clickOnArrive = false; + pressed = false; + private palette: Palette = makaBrandPalette(); + + setSession(_sessionId: string): void { + this.palette = makaBrandPalette(); + } + setPalette(p: Palette): void { + this.palette = p; + } + + /** Queue a glide to (x,y). The cursor arrow always rests at REST_HEADING + * (tip up-left, standard macOS cursor). `clickOnArrive` fires the click + * pulse the moment the cursor lands. */ + moveTo(x: number, y: number, endHeading: number = REST_HEADING, clickOnArrive = false): void { + // Shift the target so the arrow TIP (not center) lands at + // (x,y) when the arrow rests at endHeading (tip up-left). + const tx = x + Math.cos(endHeading) * ARROW_TIP_LENGTH; + const ty = y + Math.sin(endHeading) * ARROW_TIP_LENGTH; + if (clickOnArrive) this.clickPoint = [x, y]; + + if (this.pos[0] < -50) { + // First appearance starts off-screen and glides directly into view. + this.pos = [tx - 240, ty - 170]; + } + + const [x0, y0] = this.pos; + this.path = planDirectPath(x0, y0, tx, ty, endHeading); + this.dist = 0; + this.clickOnArrive = clickOnArrive; + } + + /** Snap the arrow tip to the coordinate where backend execution completed. */ + completeAt(x: number, y: number, pulse = false, endHeading: number = REST_HEADING): void { + this.pos = [ + x + Math.cos(endHeading) * ARROW_TIP_LENGTH, + y + Math.sin(endHeading) * ARROW_TIP_LENGTH, + ]; + this.heading = endHeading; + this.path = null; + this.dist = 0; + this.clickOnArrive = false; + this.pressed = false; + this.clickT = null; + this.clickPoint = null; + if (pulse) this.triggerClick(x, y); + } + + /** Fire the expanding click-pulse ring (and optionally hold pressed). */ + triggerClick(x?: number, y?: number): void { + if (typeof x === 'number' && typeof y === 'number' && this.pos[0] < -50) { + this.pos = [x, y]; + } + if (typeof x === 'number' && typeof y === 'number') { + this.clickPoint = [x, y]; + } + this.clickT = 0; + } + + /** True while a direct glide or click pulse is in progress. */ + isMoving(): boolean { + return this.path !== null || this.clickT !== null; + } + isVisible(): boolean { + return this.pos[0] >= -100; + } + motionProgress(): number { + if (!this.path) return 1; + return Math.min(1, this.dist / Math.max(this.path.length, 1)); + } + motionDistanceRemaining(): number { + if (!this.path) return 0; + return Math.max(0, this.path.length - this.dist); + } + hasMotionPath(): boolean { + return this.path !== null; + } + + /** Advance the animation by dt seconds. */ + tick(dt: number): void { + if (this.path) { + const pathLen = Math.max(this.path.length, 1); + const u = Math.min(this.dist / pathLen, 1); + const profile = (30 * u * u * (1 - u) * (1 - u)) / 1.875; // smootherstep, peak 1.0 @ u=0.5 + const floor = u < 0.5 ? MIN_START_SPEED : MIN_END_SPEED; + const speed = floor + (PEAK_SPEED - floor) * profile; + this.dist += speed * dt; + if (this.dist >= pathLen) { + const end = this.path.sample(pathLen); + const endHeading = this.path.endVisualHeading; + this.pos = [end.x, end.y]; + this.heading = endHeading; + this.path = null; + this.dist = 0; + if (this.clickOnArrive) { + this.clickT = 0; + this.clickOnArrive = false; + } + } else { + const s = this.path.sample(this.dist); + this.pos = [s.x, s.y]; + this.heading = s.heading + PI; // tip tracks the trajectory + } + } + if (this.clickT !== null) { + const next = this.clickT + dt * 4; // full pulse over 0.25s + if (next >= 1) { + this.clickT = null; + this.clickPoint = null; + } else { + this.clickT = next; + } + } + } + + /** Paint the cursor into a 2D context. (px,py) = pos − origin, in logical px. */ + paint(ctx: CanvasRenderingContext2D, originX: number, originY: number): void { + if (!this.isVisible()) return; + const px = this.pos[0] - originX; + const py = this.pos[1] - originY; + const hotspotX = px - Math.cos(this.heading) * ARROW_TIP_LENGTH; + const hotspotY = py - Math.sin(this.heading) * ARROW_TIP_LENGTH; + const p = this.palette; + + // --- Bloom (centered on the arrow hotspot / backend action point) --- + const bloomR = this.pressed ? 34 : 22; + const grad = ctx.createRadialGradient(hotspotX, hotspotY, 0, hotspotX, hotspotY, bloomR); + grad.addColorStop(0, rgba(p.bloomInner, 115 / 255)); + grad.addColorStop(0.5, rgba(p.bloomOuter, 26 / 255)); + grad.addColorStop(1, rgba(p.bloomOuter, 0)); + ctx.fillStyle = grad; + ctx.beginPath(); + ctx.arc(px, py, bloomR, 0, 2 * PI); + ctx.fill(); + + // --- Pressed state (dot + ring) --- + if (this.pressed) { + ctx.fillStyle = rgba(p.cursorMid, 110 / 255); + ctx.beginPath(); + ctx.arc(hotspotX, hotspotY, 6.5, 0, 2 * PI); + ctx.fill(); + ctx.strokeStyle = rgba(p.cursorMid, 210 / 255); + ctx.lineWidth = 3; + ctx.beginPath(); + ctx.arc(hotspotX, hotspotY, 13, 0, 2 * PI); + ctx.stroke(); + } + + // --- Click pulse ring --- + if (this.clickT !== null) { + const t = this.clickT; + const ringR = (bloomR + 20 * t) * (1 - t * 0.5); + const ringX = (this.clickPoint?.[0] ?? this.pos[0]) - originX; + const ringY = (this.clickPoint?.[1] ?? this.pos[1]) - originY; + ctx.strokeStyle = rgba(p.cursorMid, (1 - t) * 180 / 255); + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.arc(ringX, ringY, ringR, 0, 2 * PI); + ctx.stroke(); + } + + // --- Arrow glyph (procedural, gradient tip→tail, white outline) --- + this.paintArrow(ctx, px, py); + } + + private paintArrow(ctx: CanvasRenderingContext2D, px: number, py: number): void { + const verts: ReadonlyArray = [[14, 0], [-8, -9], [-3, 0], [-8, 9]]; + const angle = this.heading + PI; // tip points along motion (draw_default_arrow) + const ca = Math.cos(angle), sa = Math.sin(angle); + const pts = verts.map(([vx, vy]) => [px + ca * vx - sa * vy, py + sa * vx + ca * vy] as const); + const p = this.palette; + const tip = pts[0]; + const tail: readonly [number, number] = [(pts[1][0] + pts[3][0]) / 2, (pts[1][1] + pts[3][1]) / 2]; + + ctx.beginPath(); + ctx.moveTo(tip[0], tip[1]); + for (let i = 1; i < pts.length; i++) ctx.lineTo(pts[i][0], pts[i][1]); + ctx.closePath(); + + const g = ctx.createLinearGradient(tip[0], tip[1], tail[0], tail[1]); + const c = (rgb: Rgb): string => rgba(rgb, 1); + g.addColorStop(0.0, c(p.cursorStart)); + g.addColorStop(0.53, c(p.cursorMid)); + g.addColorStop(1.0, c(p.cursorEnd)); + ctx.fillStyle = g; + ctx.fill(); + ctx.strokeStyle = 'rgba(255,255,255,1)'; + ctx.lineWidth = 1.5; + ctx.lineJoin = 'round'; + ctx.stroke(); + } +} diff --git a/apps/desktop/src/renderer/computer-use-overlay/engine/dubins.ts b/apps/desktop/src/renderer/computer-use-overlay/engine/dubins.ts new file mode 100644 index 0000000000..4b35f0bb49 --- /dev/null +++ b/apps/desktop/src/renderer/computer-use-overlay/engine/dubins.ts @@ -0,0 +1,207 @@ +// Dubins path planner — faithful 1:1 port of trycua/cua's +// cursor-overlay/src/path_planner.rs (itself a port of the Swift AgentCursorRenderer). +// Plans a minimum-turning-radius arc–straight–arc path from (x0,y0,th0) to +// (x1,y1,th1) with turn radius R, then samples it at any arc-length. The curved, +// banked approach — not a straight line — is what gives the cursor its drift-in feel. +const PI = Math.PI; +const TAU = 2 * PI; + +export interface PathState { + x: number; + y: number; + heading: number; +} + +type SegType = 'L' | 'R' | 'S'; + +interface DubinsSol { + t: number; + p: number; + q: number; + types: [SegType, SegType, SegType]; +} + +function mod2pi(x: number): number { + const r = x - TAU * Math.floor(x / TAU); + return r < 0 ? r + TAU : r; +} + +export class PlannedPath { + readonly length: number; + readonly endVisualHeading: number; + private readonly kind: 'dubins' | 'straight'; + private readonly x0: number; + private readonly y0: number; + private readonly th0: number; + private readonly r: number; + private readonly seg1: number; + private readonly seg2: number; + private readonly seg3: number; + private readonly types: [SegType, SegType, SegType]; + private readonly x1: number; + private readonly y1: number; + private readonly th1: number; + + constructor(f: { + length: number; endVisualHeading: number; kind: 'dubins' | 'straight'; + x0: number; y0: number; th0: number; r: number; + seg1: number; seg2: number; seg3: number; types: [SegType, SegType, SegType]; + x1: number; y1: number; th1: number; + }) { + this.length = f.length; this.endVisualHeading = f.endVisualHeading; this.kind = f.kind; + this.x0 = f.x0; this.y0 = f.y0; this.th0 = f.th0; this.r = f.r; + this.seg1 = f.seg1; this.seg2 = f.seg2; this.seg3 = f.seg3; this.types = f.types; + this.x1 = f.x1; this.y1 = f.y1; this.th1 = f.th1; + } + + sample(distance: number): PathState { + return this.kind === 'straight' ? this.sampleStraight(distance) : this.sampleDubins(distance); + } + + private sampleStraight(s: number): PathState { + const len = Math.max(this.length, 1); + const u = Math.min(1, Math.max(0, s / len)); + let diff = this.th1 - this.th0; + while (diff > PI) diff -= TAU; + while (diff < -PI) diff += TAU; + return { x: this.x0 + (this.x1 - this.x0) * u, y: this.y0 + (this.y1 - this.y0) * u, heading: this.th0 + diff * u }; + } + + private sampleDubins(sIn: number): PathState { + if (sIn <= 0) return { x: this.x0, y: this.y0, heading: this.th0 }; + const r = this.r; + const l1 = this.seg1 * r, l2 = this.seg2 * r, l3 = this.seg3 * r; + const s = Math.min(sIn, l1 + l2 + l3); + let x = this.x0, y = this.y0, th = this.th0; + const advance = (len: number, seg: SegType): void => { + if (seg === 'S') { + x += Math.cos(th) * len; + y += Math.sin(th) * len; + } else { + const dth = (len / r) * (seg === 'L' ? 1 : -1); + const perp = seg === 'L' ? PI / 2 : -PI / 2; + const cx = x + Math.cos(th + perp) * r; + const cy = y + Math.sin(th + perp) * r; + const ang = Math.atan2(y - cy, x - cx); + x = cx + Math.cos(ang + dth) * r; + y = cy + Math.sin(ang + dth) * r; + th += dth; + } + }; + if (s <= l1) { advance(s, this.types[0]); return { x, y, heading: th }; } + advance(l1, this.types[0]); + if (s <= l1 + l2) { advance(s - l1, this.types[1]); return { x, y, heading: th }; } + advance(l2, this.types[1]); + advance(s - l1 - l2, this.types[2]); + return { x, y, heading: th }; + } +} + +function lsl(d: number, a: number, b: number): DubinsSol | null { + const tmp0 = d + Math.sin(a) - Math.sin(b); + const p2 = 2 + d * d - 2 * Math.cos(a - b) + 2 * d * (Math.sin(a) - Math.sin(b)); + if (p2 < 0) return null; + const tmp1 = Math.atan2(Math.cos(b) - Math.cos(a), tmp0); + return { t: mod2pi(-a + tmp1), p: Math.sqrt(p2), q: mod2pi(b - tmp1), types: ['L', 'S', 'L'] }; +} +function rsr(d: number, a: number, b: number): DubinsSol | null { + const tmp0 = d - Math.sin(a) + Math.sin(b); + const p2 = 2 + d * d - 2 * Math.cos(a - b) + 2 * d * (Math.sin(b) - Math.sin(a)); + if (p2 < 0) return null; + const tmp1 = Math.atan2(Math.cos(a) - Math.cos(b), tmp0); + return { t: mod2pi(a - tmp1), p: Math.sqrt(p2), q: mod2pi(-b + tmp1), types: ['R', 'S', 'R'] }; +} +function lsr(d: number, a: number, b: number): DubinsSol | null { + const p2 = -2 + d * d + 2 * Math.cos(a - b) + 2 * d * (Math.sin(a) + Math.sin(b)); + if (p2 < 0) return null; + const p = Math.sqrt(p2); + const tmp1 = Math.atan2(-(Math.cos(a) + Math.cos(b)), d + Math.sin(a) + Math.sin(b)) - Math.atan2(-2, p); + return { t: mod2pi(-a + tmp1), p, q: mod2pi(-mod2pi(b) + tmp1), types: ['L', 'S', 'R'] }; +} +function rsl(d: number, a: number, b: number): DubinsSol | null { + const p2 = d * d - 2 + 2 * Math.cos(a - b) - 2 * d * (Math.sin(a) + Math.sin(b)); + if (p2 < 0) return null; + const p = Math.sqrt(p2); + const tmp1 = Math.atan2(Math.cos(a) + Math.cos(b), d - Math.sin(a) - Math.sin(b)) - Math.atan2(2, p); + return { t: mod2pi(a - tmp1), p, q: mod2pi(b - tmp1), types: ['R', 'S', 'L'] }; +} +function rlr(d: number, a: number, b: number): DubinsSol | null { + const tmp = (6 - d * d + 2 * Math.cos(a - b) + 2 * d * (Math.sin(a) - Math.sin(b))) / 8; + if (Math.abs(tmp) > 1) return null; + const p = mod2pi(TAU - Math.acos(tmp)); + const t = mod2pi(a - Math.atan2(Math.cos(a) - Math.cos(b), d - Math.sin(a) + Math.sin(b)) + p / 2); + return { t, p, q: mod2pi(a - b - t + p), types: ['R', 'L', 'R'] }; +} +function lrl(d: number, a: number, b: number): DubinsSol | null { + const tmp = (6 - d * d + 2 * Math.cos(a - b) + 2 * d * (Math.sin(b) - Math.sin(a))) / 8; + if (Math.abs(tmp) > 1) return null; + const p = mod2pi(TAU - Math.acos(tmp)); + const t = mod2pi(-a + Math.atan2(-Math.cos(a) + Math.cos(b), d + Math.sin(a) - Math.sin(b)) + p / 2); + return { t, p, q: mod2pi(mod2pi(b) - a - t + p), types: ['L', 'R', 'L'] }; +} + +const SOLVERS = [lsl, rsr, lsr, rsl, rlr, lrl] as const; + +function planDubins(x0: number, y0: number, th0: number, x1: number, y1: number, th1: number, r: number, endVisualHeading: number): PlannedPath | null { + const dx = x1 - x0, dy = y1 - y0; + const dDist = Math.hypot(dx, dy); + if (dDist < 0.5) return null; + const d = dDist / r; + const theta = mod2pi(Math.atan2(dy, dx)); + const a = mod2pi(th0 - theta); + const b = mod2pi(th1 - theta); + let bestLen = Infinity; + let best: DubinsSol | null = null; + for (const solver of SOLVERS) { + const sol = solver(d, a, b); + if (sol) { + const len = sol.t + sol.p + sol.q; + if (Number.isFinite(len) && len >= 0 && len < bestLen) { bestLen = len; best = sol; } + } + } + if (!best) return null; + return new PlannedPath({ + length: (best.t + best.p + best.q) * r, endVisualHeading, kind: 'dubins', + x0, y0, th0, r, seg1: best.t, seg2: best.p, seg3: best.q, types: best.types, x1, y1, th1, + }); +} + +/** Plan a Dubins cursor path; falls back to a straight line if Dubins fails. */ +export function planPath(x0: number, y0: number, th0: number, x1: number, y1: number, th1: number, endVisualHeading: number, turnRadius: number): PlannedPath { + const r = Math.max(turnRadius, 1); + const dubins = planDubins(x0, y0, th0, x1, y1, th1, r, endVisualHeading); + if (dubins) return dubins; + const d = Math.max(Math.hypot(x1 - x0, y1 - y0), 1); + return new PlannedPath({ + length: d, endVisualHeading, kind: 'straight', + x0, y0, th0, r, seg1: 0, seg2: 0, seg3: 0, types: ['S', 'S', 'S'], x1, y1, th1, + }); +} + +/** Direct visual cursor motion with no arc detour or in-flight rotation. */ +export function planDirectPath( + x0: number, + y0: number, + x1: number, + y1: number, + endVisualHeading: number, +): PlannedPath { + const d = Math.max(Math.hypot(x1 - x0, y1 - y0), 1); + const heading = Math.atan2(y1 - y0, x1 - x0); + return new PlannedPath({ + length: d, + endVisualHeading, + kind: 'straight', + x0, + y0, + th0: heading, + r: 1, + seg1: 0, + seg2: 0, + seg3: 0, + types: ['S', 'S', 'S'], + x1, + y1, + th1: heading, + }); +} diff --git a/apps/desktop/src/renderer/computer-use-overlay/engine/palette.ts b/apps/desktop/src/renderer/computer-use-overlay/engine/palette.ts new file mode 100644 index 0000000000..fb2ba47e17 --- /dev/null +++ b/apps/desktop/src/renderer/computer-use-overlay/engine/palette.ts @@ -0,0 +1,107 @@ +// Agent-cursor colour palettes — faithful 1:1 port of trycua/cua's +// cursor-overlay/src/palette.rs (itself a port of AgentCursorPalette.cs). +// Colours are [R,G,B] 0-255. The overlay picks a palette from the session id so +// distinct agent runs are visually distinct but a given id is stable. +export type Rgb = readonly [number, number, number]; + +export interface Palette { + name: string; + /** Tip colour (lightest, gradient position 0.0). */ + cursorStart: Rgb; + /** Mid-gradient colour (position 0.53). */ + cursorMid: Rgb; + /** Tail colour (position 1.0). */ + cursorEnd: Rgb; + /** Outer bloom layer. */ + bloomOuter: Rgb; + /** Inner bloom layer (brighter core). */ + bloomInner: Rgb; +} + +type PaletteData = readonly [string, Rgb, Rgb, Rgb, Rgb, Rgb]; + +// (name, cursorStart, cursorMid, cursorEnd, bloomOuter, bloomInner) +const PALETTE_DATA: readonly PaletteData[] = [ + ['default_blue', [219, 238, 255], [94, 192, 232], [84, 205, 160], [188, 232, 252], [238, 248, 255]], + ['soft_purple', [238, 226, 255], [178, 132, 255], [118, 194, 255], [214, 188, 255], [246, 238, 255]], + ['rose_gold', [255, 231, 238], [247, 132, 170], [255, 181, 108], [255, 190, 211], [255, 243, 232]], + ['mint_lime', [226, 255, 240], [96, 218, 174], [178, 229, 72], [178, 245, 217], [241, 255, 231]], + ['amber', [255, 244, 214], [244, 178, 66], [255, 126, 92], [255, 219, 140], [255, 248, 225]], + ['aqua', [221, 252, 255], [76, 204, 224], [63, 222, 166], [172, 241, 249], [236, 255, 251]], + ['orchid', [252, 228, 255], [221, 113, 236], [255, 139, 196], [237, 181, 246], [255, 239, 252]], + ['crimson', [255, 226, 226], [232, 82, 98], [150, 94, 255], [255, 168, 178], [255, 240, 241]], + ['chartreuse', [247, 255, 218], [184, 220, 54], [72, 190, 119], [224, 247, 128], [249, 255, 232]], + ['cobalt', [226, 235, 255], [80, 126, 236], [91, 219, 222], [170, 195, 255], [239, 246, 255]], +]; + +function fromData(d: PaletteData): Palette { + return { name: d[0], cursorStart: d[1], cursorMid: d[2], cursorEnd: d[3], bloomOuter: d[4], bloomInner: d[5] }; +} + +export function defaultPalette(): Palette { + return fromData(PALETTE_DATA[0]); +} + +/** + * Maka's brand cursor palette, derived from the app's primary token + * `--action` = oklch(0.62 0.19 264) (a blue/indigo). Gradient tip→tail around it + * plus a soft brand bloom, so the agent cursor reads as "Maka" rather than a + * random per-session hue. (FOLLOW-UP: thread the live --primary from the renderer + * so it tracks theme changes instead of this baked snapshot.) + */ +export function makaBrandPalette(): Palette { + return { + name: 'maka_brand', + cursorStart: [144, 182, 255], // lightest at the tip + cursorMid: [73, 126, 247], // the primary + cursorEnd: [71, 97, 228], // deeper at the tail + bloomOuter: [157, 189, 255], + bloomInner: [212, 229, 255], + }; +} + +/** + * Select a palette for an instance id using the same stable-hash logic as the + * Rust `Palette::for_instance` (a port of C# `AgentCursorPalette.ForInstance`). + * Same id → same colour, always. + */ +export function paletteForInstance(instanceId: string): Palette { + if (instanceId === '' || instanceId === 'default') return defaultPalette(); + const exact = PALETTE_DATA.find((d) => d[0] === instanceId); + if (exact) return fromData(exact); + const alternates = PALETTE_DATA.slice(1); // all except default_blue + return fromData(alternates[stableIndex(instanceId, alternates.length)]); +} + +function stableIndex(id: string, count: number): number { + const sepIdx = Math.max(id.lastIndexOf('-'), id.lastIndexOf('_'), id.lastIndexOf('.')); + const suffix = sepIdx >= 0 ? id.slice(sepIdx + 1) : id; + const n = Number.parseInt(suffix, 10); + if (Number.isInteger(n) && String(n) === suffix.trim() && n > 0) return (n - 1) % count; + if (suffix.length === 1) { + const c = suffix.toLowerCase().charCodeAt(0); + if (c >= 97 && c <= 122) return (c - 97) % count; + } + // FNV-1a over the full id. + let hash = 2_166_136_261 >>> 0; + for (const ch of id) { + hash ^= ch.codePointAt(0)!; + hash = Math.imul(hash, 16_777_619) >>> 0; + } + return hash % count; +} + +const lerp = (a: number, b: number, t: number): number => Math.round(a + (b - a) * t); + +/** Lerp cursorStart → cursorMid → cursorEnd at t ∈ [0,1] (mid at 0.53). */ +export function gradientAt(p: Palette, t: number): Rgb { + const c = Math.min(1, Math.max(0, t)); + if (c <= 0.53) { + const u = c / 0.53; + return [lerp(p.cursorStart[0], p.cursorMid[0], u), lerp(p.cursorStart[1], p.cursorMid[1], u), lerp(p.cursorStart[2], p.cursorMid[2], u)]; + } + const u = (c - 0.53) / 0.47; + return [lerp(p.cursorMid[0], p.cursorEnd[0], u), lerp(p.cursorMid[1], p.cursorEnd[1], u), lerp(p.cursorMid[2], p.cursorEnd[2], u)]; +} + +export const rgba = (c: Rgb, a: number): string => `rgba(${c[0]},${c[1]},${c[2]},${a})`; diff --git a/knip.json b/knip.json index de5ef069f8..f2be70ba46 100644 --- a/knip.json +++ b/knip.json @@ -7,6 +7,8 @@ "entry": [ "src/main/main.ts", "src/preload/preload.ts", + "src/overlay/cursor-overlay.ts", + "src/overlay/cursor-overlay-preload.ts", "src/renderer/main.tsx", "src/main/**/*.test.ts", "src/renderer/**/*.test.ts", diff --git a/packages/computer-use/src/__tests__/computer-use-overlay-hook.test.ts b/packages/computer-use/src/__tests__/computer-use-overlay-hook.test.ts new file mode 100644 index 0000000000..3339fd8281 --- /dev/null +++ b/packages/computer-use/src/__tests__/computer-use-overlay-hook.test.ts @@ -0,0 +1,128 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { CuAction } from '@maka/core'; +import { createComputerUseOverlayHook } from '../computer-use-overlay-hook.js'; + +function fakeController() { + const moves: unknown[] = []; + const completions: unknown[] = []; + const cancellations: unknown[] = []; + const ensured: string[] = []; + return { + controller: { + ensure: (sessionId: string) => { ensured.push(sessionId); }, + move: (input: unknown) => { moves.push(input); }, + complete: (input: unknown) => { completions.push(input); }, + cancel: (input: unknown) => { cancellations.push(input); }, + }, + moves, + completions, + cancellations, + ensured, + }; +} + +test('presentation starts from the Runtime-bound screen point', () => { + const { controller, moves } = fakeController(); + const hook = createComputerUseOverlayHook(controller as never); + hook.onActionBegin( + { type: 'left_click', coordinate: { x: 400, y: 300 } }, + { + sessionId: 's1', + toolCallId: 'a1', + presentationScreenPoint: { x: 201, y: 151 }, + }, + ); + assert.deepEqual(moves, [{ + actionId: 'a1', + sessionId: 's1', + screenX: 201, + screenY: 151, + kind: 'click', + instant: true, + }]); +}); + +test('completion uses only the executor-resolved point', () => { + const { controller, completions } = fakeController(); + const hook = createComputerUseOverlayHook(controller as never); + hook.onActionEnd?.( + { type: 'left_click', coordinate: { x: 400, y: 300 } }, + { + outcome: { ok: true, tier: 'semantic-background', verified: true }, + resolvedScreenPoint: { x: 202, y: 152 }, + }, + { sessionId: 's1', toolCallId: 'a1' }, + ); + assert.deepEqual(completions, [{ + actionId: 'a1', + sessionId: 's1', + screenX: 202, + screenY: 152, + kind: 'click', + pulse: true, + }]); +}); + +test('failed pointer action without a resolved point cancels presentation', () => { + const { controller, completions, cancellations } = fakeController(); + const hook = createComputerUseOverlayHook(controller as never); + hook.onActionEnd?.( + { type: 'left_click', coordinate: { x: 40, y: 30 } }, + { outcome: { ok: false, error: 'capture_failed', message: 'no effect' } }, + { sessionId: 's1', toolCallId: 'a1' }, + ); + assert.deepEqual(completions, []); + assert.deepEqual(cancellations, [{ actionId: 'a1', sessionId: 's1' }]); +}); + +test('failed pointer action with a diagnostic point still cancels', () => { + const { controller, completions, cancellations } = fakeController(); + const hook = createComputerUseOverlayHook(controller as never); + hook.onActionEnd?.( + { type: 'left_click', coordinate: { x: 40, y: 30 } }, + { + outcome: { ok: false, error: 'target_changed', message: 'moved' }, + resolvedScreenPoint: { x: 140, y: 130 }, + }, + { sessionId: 's1', toolCallId: 'a1' }, + ); + assert.deepEqual(completions, []); + assert.deepEqual(cancellations, [{ actionId: 'a1', sessionId: 's1' }]); +}); + +test('mouse_move completion is reconciled from executor evidence', () => { + const { controller, completions } = fakeController(); + const hook = createComputerUseOverlayHook(controller as never); + hook.onActionEnd?.( + { type: 'mouse_move', coordinate: { x: 40, y: 30 } }, + { + outcome: { ok: true, tier: 'coordinate-background' }, + resolvedScreenPoint: { x: 140, y: 130 }, + }, + { sessionId: 's1', toolCallId: 'move1' }, + ); + assert.deepEqual(completions, [{ + actionId: 'move1', + sessionId: 's1', + screenX: 140, + screenY: 130, + kind: 'move', + pulse: false, + }]); +}); + +test('non-pointer actions keep the session cursor without moving it', () => { + const { controller, moves, ensured } = fakeController(); + const hook = createComputerUseOverlayHook(controller as never); + for (const action of [ + { type: 'type', text: 'hi' }, + { type: 'key', text: 'Return' }, + { type: 'screenshot' }, + { type: 'wait', durationMs: 100 }, + ] as CuAction[]) { + hook.onActionBegin(action, { sessionId: 's1', toolCallId: 'a1' }); + } + assert.deepEqual(moves, []); + assert.deepEqual(ensured, ['s1', 's1', 's1', 's1']); +}); diff --git a/packages/computer-use/src/computer-use-overlay-hook.ts b/packages/computer-use/src/computer-use-overlay-hook.ts new file mode 100644 index 0000000000..0efb9d415e --- /dev/null +++ b/packages/computer-use/src/computer-use-overlay-hook.ts @@ -0,0 +1,139 @@ +import type { CuAction, CuPoint } from '@maka/core'; +import type { CuOverlayHook, CuPresentationFence } from '@maka/runtime'; + +export type CursorActionKind = 'move' | 'click' | 'drag' | 'scroll'; + +export interface CursorMoveInput { + actionId: string; + sessionId: string; + screenX: number; + screenY: number; + kind: CursorActionKind; + pressed?: boolean; + instant?: boolean; +} + +export interface CursorCompleteInput extends CursorMoveInput { + pulse: boolean; +} + +export interface CursorCancelInput { + actionId: string; + sessionId: string; +} + +export interface OverlayCursorSink { + ensure(sessionId: string): void; + move(input: CursorMoveInput): CuPresentationFence | void; + complete(input: CursorCompleteInput): void; + cancel(input: CursorCancelInput): void; +} + +const RESOLVED_PRESENTATION_FENCE: CuPresentationFence = { + readyForInteraction: Promise.resolve(), + finished: Promise.resolve(), +}; + +function beginCoordinateOf(action: CuAction): CuPoint | undefined { + switch (action.type) { + case 'left_click_drag': + return action.startCoordinate; + case 'mouse_move': + case 'left_click': + case 'right_click': + case 'middle_click': + case 'double_click': + case 'triple_click': + case 'left_mouse_down': + case 'left_mouse_up': + case 'scroll': + return action.coordinate; + default: + return undefined; + } +} + +function endCoordinateOf(action: CuAction): CuPoint | undefined { + switch (action.type) { + case 'mouse_move': + case 'left_click': + case 'right_click': + case 'middle_click': + case 'double_click': + case 'triple_click': + case 'left_mouse_down': + case 'left_mouse_up': + case 'scroll': + case 'left_click_drag': + return action.coordinate; + default: + return undefined; + } +} + +function kindOf(action: CuAction): CursorActionKind { + switch (action.type) { + case 'left_click': + case 'right_click': + case 'middle_click': + case 'double_click': + case 'triple_click': + case 'left_mouse_down': + case 'left_mouse_up': + return 'click'; + case 'left_click_drag': + return 'drag'; + case 'scroll': + return 'scroll'; + default: + return 'move'; + } +} + +export function createComputerUseOverlayHook(controller: OverlayCursorSink): CuOverlayHook { + return { + onActionBegin(action, context) { + const declaredPoint = beginCoordinateOf(action); + const screenPoint = context.presentationScreenPoint; + if (!declaredPoint || !screenPoint) { + controller.ensure(context.sessionId); + return RESOLVED_PRESENTATION_FENCE; + } + return controller.move({ + actionId: context.toolCallId, + sessionId: context.sessionId, + screenX: screenPoint.x, + screenY: screenPoint.y, + kind: kindOf(action), + instant: action.type !== 'mouse_move', + }); + }, + onActionEnd(action, result, context) { + if (!endCoordinateOf(action)) return; + if (!result?.outcome.ok) { + controller.cancel({ + actionId: context.toolCallId, + sessionId: context.sessionId, + }); + return; + } + const screenPoint = result?.resolvedScreenPoint; + if (!screenPoint) { + controller.cancel({ + actionId: context.toolCallId, + sessionId: context.sessionId, + }); + return; + } + const kind = kindOf(action); + controller.complete({ + actionId: context.toolCallId, + sessionId: context.sessionId, + screenX: screenPoint.x, + screenY: screenPoint.y, + kind, + pulse: result.outcome.ok && (kind === 'click' || kind === 'drag'), + }); + }, + }; +} diff --git a/packages/computer-use/src/index.ts b/packages/computer-use/src/index.ts index f8e72b03f8..8c3ac94124 100644 --- a/packages/computer-use/src/index.ts +++ b/packages/computer-use/src/index.ts @@ -54,3 +54,11 @@ export type { } from './cua-driver-snapshot.js'; export { resolveCuaDisplaySnapshots } from './display-snapshot.js'; export type { CuaHostDisplay } from './display-snapshot.js'; +export { createComputerUseOverlayHook } from './computer-use-overlay-hook.js'; +export type { + CursorActionKind, + CursorCancelInput, + CursorCompleteInput, + CursorMoveInput, + OverlayCursorSink, +} from './computer-use-overlay-hook.js'; diff --git a/packages/computer-use/src/select-backend.ts b/packages/computer-use/src/select-backend.ts index 3182793990..72e3d66e7c 100644 --- a/packages/computer-use/src/select-backend.ts +++ b/packages/computer-use/src/select-backend.ts @@ -1,5 +1,6 @@ import { buildComputerUseTools, + type CuOverlayHook, type ComputerUseToolSet, type CuDispatchBackend, } from '@maka/runtime'; @@ -63,6 +64,7 @@ export function selectComputerUseBackend(deps?: { mimeType: string, ) => { base64: string; mimeType: 'image/png' | 'image/jpeg' }; physicalInputRecentlyActive?: () => boolean | Promise; + overlay?: CuOverlayHook; }): SelectedComputerUseBackend { if (process.platform !== 'darwin') return NONE; if (!deps?.binaryPath || !deps.expectedBinarySha256) return NONE; @@ -87,7 +89,10 @@ export function selectComputerUseBackend(deps?: { }); return { backend, - tools: buildComputerUseTools({ backend }), + tools: buildComputerUseTools({ + backend, + ...(deps.overlay ? { overlay: deps.overlay } : {}), + }), backendId: 'cua-driver', }; } catch { diff --git a/packages/runtime/src/__tests__/computer-use-tools.test.ts b/packages/runtime/src/__tests__/computer-use-tools.test.ts index bce8ef23b3..8ee9bea829 100644 --- a/packages/runtime/src/__tests__/computer-use-tools.test.ts +++ b/packages/runtime/src/__tests__/computer-use-tools.test.ts @@ -198,6 +198,359 @@ describe('buildComputerUseTools — the `maka_computer` MakaTool', () => { assert.ok(tool.parameters, 'carries a zod parameter schema'); }); + test('waits for presentation readiness before dispatch without waiting for finish', async () => { + const events: string[] = []; + let ready!: () => void; + const readyForInteraction = new Promise((resolve) => { ready = resolve; }); + const [tool] = buildComputerUseTools({ + backend: { + async preflight() { + return { accessibility: true, screenRecording: true }; + }, + async run() { + events.push('dispatch'); + return { outcome: { ok: true, tier: 'ax', verified: true } }; + }, + }, + overlay: { + onActionBegin() { + events.push('presentation'); + return { + readyForInteraction, + finished: new Promise(() => {}), + }; + }, + onActionEnd() { + events.push('end'); + }, + }, + presentationReadyTimeoutMs: 10_000, + }); + + const pending = tool.impl({ action: 'wait' } as never, ctx()); + while (events.length === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } + assert.deepEqual(events, ['presentation']); + ready(); + await pending; + assert.deepEqual(events, ['presentation', 'dispatch', 'end']); + }); + + test('presentation readiness timeout fails open', async () => { + const events: string[] = []; + const [tool] = buildComputerUseTools({ + backend: { + async preflight() { + return { accessibility: true, screenRecording: true }; + }, + async run() { + events.push('dispatch'); + return { outcome: { ok: true, tier: 'ax', verified: true } }; + }, + }, + overlay: { + onActionBegin() { + return { + readyForInteraction: new Promise(() => {}), + finished: new Promise(() => {}), + }; + }, + }, + presentationReadyTimeoutMs: 5, + }); + await tool.impl({ action: 'wait' } as never, ctx()); + assert.deepEqual(events, ['dispatch']); + }); + + test('user stop while presentation is pending prevents native dispatch', async () => { + const readyForInteraction = new Promise(() => {}); + let dispatchCount = 0; + const backend = fakeBackend() as CuDispatchBackend & { + observeApp: NonNullable; + }; + backend.observeApp = async () => observation(); + backend.run = async () => { + dispatchCount += 1; + return { outcome: { ok: true, tier: 'ax', verified: true } }; + }; + const tools = buildComputerUseTools({ + backend, + overlay: { + onActionBegin() { + return { + readyForInteraction, + finished: new Promise(() => {}), + }; + }, + }, + presentationReadyTimeoutMs: 10_000, + }); + const [tool] = tools; + const observed = await tool.impl({ + action: 'observe', + app: 'Fixture', + } as never, ctx()) as { + modelText?: string; + }; + const observationId = JSON.parse(observed.modelText ?? '{}').observation_id; + const pending = tool.impl({ + action: 'left_click', + observation_id: observationId, + coordinate: [10, 10], + } as never, ctx()); + await Promise.resolve(); + tools.clearSession('s1'); + const result = await pending as { error?: string }; + assert.equal(dispatchCount, 0); + assert.ok( + result.error === 'user_stopped' || result.error === 'no_active_frame', + `unexpected stop rejection: ${result.error}`, + ); + }); + + test('abort while presentation is pending prevents native dispatch', async () => { + const abortController = new AbortController(); + let dispatchCount = 0; + const [tool] = buildComputerUseTools({ + backend: { + async preflight() { + return { accessibility: true, screenRecording: true }; + }, + async run() { + dispatchCount += 1; + return { outcome: { ok: true, tier: 'ax', verified: true } }; + }, + }, + overlay: { + onActionBegin() { + return { + readyForInteraction: new Promise(() => {}), + finished: new Promise(() => {}), + }; + }, + }, + presentationReadyTimeoutMs: 10_000, + }); + const pending = tool.impl( + { action: 'wait' } as never, + ctx(abortController.signal), + ); + await Promise.resolve(); + abortController.abort(new Error('stopped')); + await assert.rejects(Promise.resolve(pending), /stopped/); + assert.equal(dispatchCount, 0); + }); + + test('presentation receives the observation-bound screen point', async () => { + let point: { x: number; y: number } | undefined; + const backend = fakeBackend() as CuDispatchBackend & { + observeApp: NonNullable; + }; + backend.observeApp = async () => observation({ + windowBounds: { x: 100, y: 50, width: 400, height: 300 }, + sourceBoundsPx: { x: 0, y: 0, width: 800, height: 600 }, + }); + const [tool] = buildComputerUseTools({ + backend, + overlay: { + onActionBegin(_action, context) { + point = context.presentationScreenPoint; + }, + }, + }); + const observed = await tool.impl({ + action: 'observe', + app: 'Fixture', + } as never, ctx()) as { + modelText?: string; + }; + const observationId = JSON.parse(observed.modelText ?? '{}').observation_id; + await tool.impl({ + action: 'left_click', + observation_id: observationId, + coordinate: [400, 300], + } as never, ctx()); + assert.deepEqual(point, { x: 300, y: 200 }); + }); + + test('discarded dispatch result cancels presentation instead of showing success', async () => { + const ended: Array = []; + let tools: ReturnType; + const backend = fakeBackend() as CuDispatchBackend & { + observeApp: NonNullable; + captureObservation: NonNullable; + }; + backend.observeApp = async () => observation(); + backend.captureObservation = async () => observation({ + observationId: 'backend-obs-2', + }); + backend.run = async () => { + tools.sessionEvents.physicalUserIntervened('s1'); + return { outcome: { ok: true, tier: 'ax', verified: true } }; + }; + tools = buildComputerUseTools({ + backend, + overlay: { + onActionBegin() { + return { + readyForInteraction: Promise.resolve(), + finished: Promise.resolve(), + }; + }, + onActionEnd(_action, result) { + ended.push(result?.outcome.ok); + }, + }, + }); + const [tool] = tools; + const observed = await tool.impl({ + action: 'observe', + app: 'Fixture', + } as never, ctx()) as { + modelText?: string; + }; + const observationId = JSON.parse(observed.modelText ?? '{}').observation_id; + const result = await tool.impl({ + action: 'left_click', + observation_id: observationId, + coordinate: [10, 10], + } as never, ctx()) as { error?: string }; + assert.equal(result.error, 'user_intervened'); + assert.deepEqual(ended, [undefined]); + }); + + test('presentation promise rejections are isolated from execution', async () => { + const [tool] = buildComputerUseTools({ + backend: fakeBackend(), + overlay: { + onActionBegin() { + return { + readyForInteraction: Promise.resolve(), + finished: Promise.reject(new Error('finished failed')), + }; + }, + async onActionEnd() { + throw new Error('end failed'); + }, + }, + }); + const result = await tool.impl({ action: 'wait' } as never, ctx()) as { + text: string; + }; + assert.match(result.text, /computer\.wait ok/); + await new Promise((resolve) => setImmediate(resolve)); + }); + + test('one visual overlay serializes presentation across independent sessions', async () => { + const events: string[] = []; + const ready = new Map void>(); + const backend: CuDispatchBackend = { + async preflight() { + return { accessibility: true, screenRecording: true }; + }, + async run(_action, _signal, context) { + events.push(`dispatch:${context.sessionId}`); + return { outcome: { ok: true, tier: 'ax', verified: true } }; + }, + }; + const [tool] = buildComputerUseTools({ + backend, + overlay: { + onActionBegin(_action, context) { + events.push(`presentation:${context.sessionId}`); + return { + readyForInteraction: new Promise((resolve) => { + ready.set(context.sessionId, resolve); + }), + finished: Promise.resolve(), + }; + }, + }, + presentationReadyTimeoutMs: 10_000, + }); + const first = tool.impl( + { action: 'wait' } as never, + ctx(undefined, { sessionId: 's1', toolCallId: 'a1' }), + ); + const second = tool.impl( + { action: 'wait' } as never, + ctx(undefined, { sessionId: 's2', toolCallId: 'a2' }), + ); + while (!ready.has('s1')) { + await new Promise((resolve) => setImmediate(resolve)); + } + assert.deepEqual(events, ['presentation:s1']); + ready.get('s1')?.(); + await first; + while (!ready.has('s2')) { + await new Promise((resolve) => setImmediate(resolve)); + } + assert.deepEqual(events, [ + 'presentation:s1', + 'dispatch:s1', + 'presentation:s2', + ]); + ready.get('s2')?.(); + await second; + }); + + test('clearSession releases an action queued behind another presentation', async () => { + let releaseFirst!: () => void; + const firstReady = new Promise((resolve) => { releaseFirst = resolve; }); + let firstDispatchStarted!: () => void; + const dispatchStarted = new Promise((resolve) => { + firstDispatchStarted = resolve; + }); + let releaseDispatch!: () => void; + const dispatchGate = new Promise((resolve) => { + releaseDispatch = resolve; + }); + const backend: CuDispatchBackend = { + async preflight() { + return { accessibility: true, screenRecording: true }; + }, + async run(_action, _signal, context) { + if (context.sessionId === 's1') { + firstDispatchStarted(); + await dispatchGate; + } + return { outcome: { ok: true, tier: 'ax', verified: true } }; + }, + }; + const tools = buildComputerUseTools({ + backend, + overlay: { + onActionBegin(_action, context) { + return { + readyForInteraction: context.sessionId === 's1' + ? firstReady + : Promise.resolve(), + finished: Promise.resolve(), + }; + }, + }, + presentationReadyTimeoutMs: 10_000, + }); + const [tool] = tools; + const first = tool.impl( + { action: 'wait' } as never, + ctx(undefined, { sessionId: 's1', toolCallId: 'a1' }), + ); + releaseFirst(); + await dispatchStarted; + const second = tool.impl( + { action: 'wait' } as never, + ctx(undefined, { sessionId: 's2', toolCallId: 'a2' }), + ); + await Promise.resolve(); + tools.clearSession('s2'); + const secondResult = await second as { error?: string }; + assert.equal(secondResult.error, 'user_stopped'); + releaseDispatch(); + await first; + }); + test('list_apps and observe expose one provider-neutral Sky-like surface', async () => { const backend = fakeBackend() as CuDispatchBackend & { listApps: NonNullable; diff --git a/packages/runtime/src/computer-use-tools.ts b/packages/runtime/src/computer-use-tools.ts index 826a079d28..c4efc8abae 100644 --- a/packages/runtime/src/computer-use-tools.ts +++ b/packages/runtime/src/computer-use-tools.ts @@ -161,6 +161,29 @@ export interface CuRunContext { boundAction?: CuaBoundAction; } +export interface CuPresentationFence { + readyForInteraction: Promise; + finished: Promise; +} + +export interface CuOverlayHookContext { + sessionId: string; + toolCallId: string; + presentationScreenPoint?: CuPoint; +} + +export interface CuOverlayHook { + onActionBegin( + action: CuAction, + context: CuOverlayHookContext, + ): CuPresentationFence | void; + onActionEnd?( + action: CuAction, + result: CuRunResult | undefined, + context: CuOverlayHookContext, + ): void | Promise; +} + /** * The host dispatch seam. Implemented in @maka/computer-use by the cua-driver * backend, which spawns trycua/cua-driver and speaks its JSON-RPC protocol over @@ -536,8 +559,15 @@ function persistedObservationText(observation: CuObservation): string { export function buildComputerUseTools(deps: { backend: CuDispatchBackend; + overlay?: CuOverlayHook; + presentationReadyTimeoutMs?: number; }): ComputerUseToolSet { + const presentationReadyTimeoutMs = deps.presentationReadyTimeoutMs ?? 1_000; const invocationQueues = new Map>(); + const presentationWaiters = new Map void>>(); + const presentationQueueWaiters = new Map void>>(); + const presentationGenerations = new Map(); + let presentationQueue = Promise.resolve(); interface SessionObservationRecord { turnId: string; state: CuaFrameState; @@ -796,6 +826,172 @@ export function buildComputerUseTools(deps: { } } + function presentationScreenPoint( + boundAction: CuaBoundAction | undefined, + ): CuPoint | undefined { + const source = boundAction?.sourceStartCoordinate + ?? boundAction?.sourceCoordinate; + const sourceBounds = boundAction?.target.sourceBoundsPx; + const windowBounds = boundAction?.target.bounds; + if (!source || !sourceBounds || !windowBounds) return undefined; + if (sourceBounds.width <= 0 || sourceBounds.height <= 0) return undefined; + return { + x: windowBounds.x + + source.x / sourceBounds.width * windowBounds.width, + y: windowBounds.y + + source.y / sourceBounds.height * windowBounds.height, + }; + } + + async function waitForPresentationReady( + fence: CuPresentationFence | undefined, + signal: AbortSignal, + sessionId: string, + ): Promise { + if (!fence) return; + if (signal.aborted) throw signal.reason ?? new Error('aborted'); + await new Promise((resolve, reject) => { + let settled = false; + const finish = (error?: unknown) => { + if (settled) return; + settled = true; + clearTimeout(timer); + signal.removeEventListener('abort', onAbort); + const waiters = presentationWaiters.get(sessionId); + waiters?.delete(wake); + if (waiters?.size === 0) presentationWaiters.delete(sessionId); + if (error) reject(error); + else resolve(); + }; + const timer = setTimeout(finish, presentationReadyTimeoutMs); + const onAbort = () => finish(signal.reason ?? new Error('aborted')); + const wake = () => finish(); + const waiters = presentationWaiters.get(sessionId) ?? new Set(); + waiters.add(wake); + presentationWaiters.set(sessionId, waiters); + signal.addEventListener('abort', onAbort, { once: true }); + fence.readyForInteraction.then( + () => finish(), + () => finish(), + ); + }); + } + + async function runWithPresentation( + action: CuAction, + context: CuRunContext, + signal: AbortSignal, + dispatch: () => Promise, + beforeDispatch?: () => ComputerToolResult | undefined, + invocationGeneration = 0, + ): Promise<{ + result?: CuRunResult; + blocked?: ComputerToolResult; + finish(result?: CuRunResult): void; + }> { + let releasePresentation!: () => void; + const previousPresentation = presentationQueue; + const presentationGate = new Promise((resolve) => { + releasePresentation = resolve; + }); + if (deps.overlay) { + presentationQueue = previousPresentation.then(() => presentationGate); + if ( + (presentationGenerations.get(context.sessionId) ?? 0) + !== invocationGeneration + ) { + releasePresentation(); + return { + blocked: sessionFailure('user_stopped'), + finish: () => {}, + }; + } + let queuedCancelled = false; + await Promise.race([ + previousPresentation, + new Promise((resolve) => { + const cancel = () => { + queuedCancelled = true; + resolve(); + }; + const waiters = presentationQueueWaiters.get(context.sessionId) + ?? new Set(); + waiters.add(cancel); + presentationQueueWaiters.set(context.sessionId, waiters); + if ( + (presentationGenerations.get(context.sessionId) ?? 0) + !== invocationGeneration + ) { + cancel(); + } + void previousPresentation.finally(() => { + waiters.delete(cancel); + if (waiters.size === 0) { + presentationQueueWaiters.delete(context.sessionId); + } + }); + }), + ]); + if ( + queuedCancelled + || (presentationGenerations.get(context.sessionId) ?? 0) + !== invocationGeneration + ) { + releasePresentation(); + return { + blocked: sessionFailure('user_stopped'), + finish: () => {}, + }; + } + } + const overlayContext: CuOverlayHookContext = { + sessionId: context.sessionId, + toolCallId: context.toolCallId, + ...(context.boundAction + ? { + presentationScreenPoint: presentationScreenPoint( + context.boundAction, + ), + } + : {}), + }; + let fence: CuPresentationFence | undefined; + try { + fence = deps.overlay?.onActionBegin(action, overlayContext) ?? undefined; + void fence?.finished.catch(() => {}); + } catch { + fence = undefined; + } + let finished = false; + const finish = (result?: CuRunResult) => { + if (finished) return; + finished = true; + try { + void Promise.resolve( + deps.overlay?.onActionEnd?.(action, result, overlayContext), + ).catch(() => {}); + } catch { + // Presentation is best-effort and cannot change execution outcome. + } + releasePresentation?.(); + }; + try { + if (fence) { + await waitForPresentationReady(fence, signal, context.sessionId); + } + const blocked = beforeDispatch?.(); + if (blocked) { + finish(); + return { blocked, finish }; + } + const result = await dispatch(); + return { result, finish }; + } catch (error) { + finish(); + throw error; + } + } + const tool: MakaTool = { name: 'maka_computer', displayName: 'Maka Computer', @@ -854,6 +1050,7 @@ export function buildComputerUseTools(deps: { }): Promise => { if (abortSignal.aborted) return { text: 'computer aborted before start' }; const input = snapshotComputerParams(computerParams.parse(args)); + const invocationGeneration = presentationGenerations.get(sessionId) ?? 0; return withInvocationQueue(sessionId, abortSignal, async () => { const state = sessionState(sessionId, turnId); const observationLease = input.action === 'observe' @@ -1067,49 +1264,83 @@ export function buildComputerUseTools(deps: { ...modelAction, observationId: record.backendObservationId, }; + const summaryAction: CuAction = semanticAction.type === 'click_element' + ? { + type: 'left_click', + coordinate: binding.sourceCoordinate ?? { x: 0, y: 0 }, + } + : semanticAction.type === 'press_key' + ? { type: 'key', text: semanticAction.key } + : semanticAction.type === 'set_value' + ? { type: 'type', text: semanticAction.value } + : semanticAction.type === 'select_text' + ? { type: 'type', text: semanticAction.text } + : { type: 'key', text: semanticAction.action }; let result: CuRunResult | undefined; let consumeFailure: ComputerToolResult | undefined; + let presentation: + | Awaited> + | undefined; try { if (!actionLease) return sessionFailure('no_active_frame'); const leaseFailure = validateActionLease(state, actionLease); if (leaseFailure) return leaseFailure; - result = await deps.backend.runSemantic( - semanticAction, + const operationContext = { ...runCtx, boundAction: binding }; + presentation = await runWithPresentation( + summaryAction, + operationContext, abortSignal, - { ...runCtx, boundAction: binding }, + () => deps.backend.runSemantic!( + semanticAction, + abortSignal, + operationContext, + ), + () => validateActionLease(state, actionLease), + invocationGeneration, ); + if (presentation.blocked) return presentation.blocked; + if (!presentation.result) return bindingFailure('capture_failed'); + result = presentation.result; const postDispatchFailure = validateActionLease(state, actionLease); - if (postDispatchFailure) return postDispatchFailure; + if (postDispatchFailure) { + presentation.finish(); + return postDispatchFailure; + } } finally { consumeFailure = consumeBoundAction(record, binding); if (actionLease && state.validateLease(actionLease).ok) { state.reobserveRequired(); } } - if (consumeFailure) return consumeFailure; - if (!result) return bindingFailure('capture_failed'); - const summaryAction: CuAction = semanticAction.type === 'click_element' - ? { type: 'left_click', coordinate: { x: 0, y: 0 } } - : semanticAction.type === 'press_key' - ? { type: 'key', text: semanticAction.key } - : semanticAction.type === 'set_value' - ? { type: 'type', text: semanticAction.value } - : semanticAction.type === 'select_text' - ? { type: 'type', text: semanticAction.text } - : { type: 'key', text: semanticAction.action }; - const text = summarize(summaryAction, result); - const freshObservation = result.outcome.ok - ? await freshFullObservation( - state, - record, - result, - abortSignal, - { ...runCtx, boundAction: binding }, - ) - : undefined; + if (consumeFailure) { + presentation?.finish(); + return consumeFailure; + } + if (!result) { + presentation?.finish(); + return bindingFailure('capture_failed'); + } + let freshObservation: CuObservation | undefined; + try { + freshObservation = result.outcome.ok + ? await freshFullObservation( + state, + record, + result, + abortSignal, + { ...runCtx, boundAction: binding }, + ) + : undefined; + } catch (error) { + presentation?.finish(); + throw error; + } if (result.outcome.ok && !freshObservation) { + presentation?.finish(); return bindingFailure('capture_failed'); } + presentation?.finish(result); + const text = summarize(summaryAction, result); const freshModelState = freshObservation ? `\nFresh observation:\n${observationText(freshObservation)}` : ''; @@ -1153,20 +1384,41 @@ export function buildComputerUseTools(deps: { return { text: 'computer failed: permission_missing — Screen Recording not granted (System Settings → Privacy & Security → Screen Recording)' }; } let result: CuRunResult | undefined; + let presentation: + | Awaited> + | undefined; { try { if (actionLease) { const leaseFailure = validateActionLease(state, actionLease); if (leaseFailure) return leaseFailure; } - result = await deps.backend.run( + const operationContext = { + ...runCtx, + ...(boundAction ? { boundAction } : {}), + }; + presentation = await runWithPresentation( action, + operationContext, abortSignal, - { ...runCtx, ...(boundAction ? { boundAction } : {}) }, + () => deps.backend.run( + action, + abortSignal, + operationContext, + ), + actionLease + ? () => validateActionLease(state, actionLease) + : undefined, + invocationGeneration, ); + if (presentation.blocked) return presentation.blocked; + result = presentation.result; if (actionLease) { const leaseFailure = validateActionLease(state, actionLease); - if (leaseFailure) return leaseFailure; + if (leaseFailure) { + presentation.finish(); + return leaseFailure; + } } } finally { if (actionLease && state.validateLease(actionLease).ok) { @@ -1180,19 +1432,34 @@ export function buildComputerUseTools(deps: { // bounded frame never bloats history. let bindingResult: ComputerToolResult | undefined; if (boundAction) bindingResult = consumeBoundAction(record, boundAction); - if (bindingResult) return bindingResult; - const freshObservation = actionLease && result.outcome.ok - ? await freshFullObservation( - state, - record, - result, - abortSignal, - { ...runCtx, boundAction }, - ) - : undefined; + if (bindingResult) { + presentation?.finish(); + return bindingResult; + } + if (!result) { + presentation?.finish(); + return bindingFailure('capture_failed'); + } + let freshObservation: CuObservation | undefined; + try { + freshObservation = actionLease && result.outcome.ok + ? await freshFullObservation( + state, + record, + result, + abortSignal, + { ...runCtx, boundAction }, + ) + : undefined; + } catch (error) { + presentation?.finish(); + throw error; + } if (actionLease && result.outcome.ok && !freshObservation) { + presentation?.finish(); return bindingFailure('capture_failed'); } + presentation?.finish(result); const modelRefresh = freshObservation ? `\nFresh observation:\n${observationText(freshObservation)}` : actionLease @@ -1243,6 +1510,12 @@ export function buildComputerUseTools(deps: { }; const tools = [tool] as ComputerUseToolSet; tools.clearSession = (sessionId: string) => { + presentationGenerations.set( + sessionId, + (presentationGenerations.get(sessionId) ?? 0) + 1, + ); + for (const wake of presentationQueueWaiters.get(sessionId) ?? []) wake(); + for (const wake of presentationWaiters.get(sessionId) ?? []) wake(); sessionStates.get(sessionId)?.state.userStopped(); invalidateObservation(sessionId); observations.delete(sessionId); diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 2a38d5aef2..2da479e63e 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -80,6 +80,9 @@ export type { CuDispatchOutcome, CuObservedElement, CuObservation, + CuOverlayHook, + CuOverlayHookContext, + CuPresentationFence, CuRunContext, CuRunResult, CuScreenshot, diff --git a/scripts/build-cursor-overlay.mjs b/scripts/build-cursor-overlay.mjs new file mode 100644 index 0000000000..f959d9af7f --- /dev/null +++ b/scripts/build-cursor-overlay.mjs @@ -0,0 +1,55 @@ +// Build the cursor overlay renderer bundle + preload into apps/desktop/dist/overlay. +// - cursor-overlay.js: the Canvas engine host (IIFE, browser). The `js→ts` resolve +// shim lets us bundle the engine's NodeNext `./x.js` imports straight from source. +// - cursor-overlay-preload.cjs: receive-only main→renderer bridge (CJS, electron external). +// - cursor-overlay.html: copied verbatim. +import * as esbuild from 'esbuild'; +import { resolve, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { mkdir, copyFile } from 'node:fs/promises'; + +const here = dirname(fileURLToPath(import.meta.url)); +const desktop = resolve(here, '..', 'apps', 'desktop'); +const srcOverlay = join(desktop, 'src', 'overlay'); +const outDir = join(desktop, 'dist', 'overlay'); + +const jsToTs = { + name: 'js-to-ts', + setup(build) { + build.onResolve({ filter: /^\.\.?\/.*\.js$/ }, (args) => ({ + path: resolve(args.resolveDir, args.path.replace(/\.js$/, '.ts')), + })); + }, +}; + +/** Build the overlay renderer bundle + preload + html into dist/overlay. */ +export async function buildCursorOverlay({ logLevel = 'info' } = {}) { + await mkdir(outDir, { recursive: true }); + await esbuild.build({ + entryPoints: [join(srcOverlay, 'cursor-overlay.ts')], + bundle: true, + format: 'iife', + platform: 'browser', + target: 'chrome120', + outfile: join(outDir, 'cursor-overlay.js'), + plugins: [jsToTs], + logLevel, + }); + await esbuild.build({ + entryPoints: [join(srcOverlay, 'cursor-overlay-preload.ts')], + bundle: true, + format: 'cjs', + platform: 'node', + external: ['electron'], + outfile: join(outDir, 'cursor-overlay-preload.cjs'), + logLevel, + }); + await copyFile(join(srcOverlay, 'cursor-overlay.html'), join(outDir, 'cursor-overlay.html')); + return outDir; +} + +// Run directly (npm run build:overlay) or import buildCursorOverlay (dev.mjs). +if (import.meta.url === `file://${process.argv[1]}`) { + const dir = await buildCursorOverlay(); + console.log('cursor overlay built →', dir); +}