From ce207d58daf5b06b11254a302cbfcd54eda4b724 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Tue, 14 Jul 2026 01:26:54 +0800 Subject: [PATCH] test(cu): add no-focus process restart e2e --- docs/computer-use-process-restart-e2e.md | 139 ++++++ package.json | 5 +- .../cu-process-restart-e2e-contract.test.mjs | 61 +++ scripts/cu-process-restart-e2e-launcher.mjs | 406 +++++++++++++++++ scripts/cu-process-restart-e2e.mjs | 427 ++++++++++++++++++ scripts/cu-real-e2e-monitor.swift | 23 + 6 files changed, 1059 insertions(+), 2 deletions(-) create mode 100644 docs/computer-use-process-restart-e2e.md create mode 100644 scripts/cu-process-restart-e2e-contract.test.mjs create mode 100644 scripts/cu-process-restart-e2e-launcher.mjs create mode 100644 scripts/cu-process-restart-e2e.mjs diff --git a/docs/computer-use-process-restart-e2e.md b/docs/computer-use-process-restart-e2e.md new file mode 100644 index 0000000000..b6a01da8f1 --- /dev/null +++ b/docs/computer-use-process-restart-e2e.md @@ -0,0 +1,139 @@ +# Computer Use Process-Restart E2E + +## Goal + +Prove that a Computer Use observation from one live application process cannot +authorize an action after the same canonical application path is relaunched as +a new process. + +The expected lifetime follows the recovered Codex boundary: + +```text +canonical application path + + current live process instance +``` + +Conversation or transport continuity must not extend an observation across a +real target-process restart. + +## Incident And Investigation + +The first real-machine harness attempts exposed several independent test +assumptions before reaching the process boundary: + +1. A fresh worktree had no workspace `node_modules`, so downstream packages + could not resolve `@maka/core`. Running `npm ci` established the same + dependency layout used by CI. +2. The ignored cua-driver artifact was absent in the new worktree. The launcher + now runs `prepare:cua-driver` and then verifies the pinned artifact before + starting the fixture. +3. WKWebView AX nodes can appear late and can be mirrored with duplicate labels + and different frames. The restart gate therefore uses the native + `CUA Lab Coordinate Target`; OOP behavior remains covered by the dedicated + guarded E2E. +4. Activating the fixture made the test pass but stole the user's focus. + `open -g` avoided activation but kept the app hidden from the driver's + on-screen window set. The final fixture mode uses LaunchServices background + launch plus `unhideWithoutActivation`, `moveToActiveSpace`, and + `orderFrontRegardless` so the window is on-screen without becoming the + frontmost application. +5. Runtime tool failures are projected through model-visible text rather than a + top-level `error` property. The gate verifies both the backend typed outcome + and the model-visible `target_missing` result. + +These were harness defects or environment prerequisites. None dispatched an +action before the target-process identity gate was reached. + +## Real Sequence + +The launcher and harness keep one cua-driver backend and one Runtime tool +instance alive across five target restarts: + +```text +repeat 5 times with one backend/Runtime instance: + -> observe current PID/window + -> bind coordinate action + -> terminate current app + -> launch same canonical app path + -> require globally new host PID and WebContent PID + -> attempt old observation + -> require target_missing, no dispatch, mutation 0 -> 0 + -> clear session + -> observe new PID/window + -> execute fresh coordinate action + -> if visible, require px dispatch, target 0 -> 1, decoy 0 -> 0 + -> if covered by the user's window, require target_occluded and mutation 0 -> 0 + -> require cua-driver generations stable and restartAttempts == 0 +``` + +The launcher owns sleep prevention, fixture cleanup, bounded child +termination, and private temporary handshake/report files. The synthetic app +is launched with `CUA_LAB_BACKGROUND=1`. In that mode the fixture orders its +window visible without activating the application or making it the user's +frontmost app. A continuous Swift sentinel allows the user to switch among +their own applications, type, and move the pointer, but fails immediately if +the synthetic fixture ever becomes frontmost or the screen locks. Cleanup +preserves the user's current application; restoration is only an emergency +path if the fixture itself stole focus. + +Physical user pointer movement is allowed and reported as observation data. +When the user's window occludes the target, the stronger non-interference +proof is that the backend emits no dispatch and both target and decoy mutation +remain zero. + +## Verified Result + +The July 14, 2026 five-round no-focus soak proved: + +```text +restart rounds: 5 +distinct host PIDs: 6 +distinct WebContent PIDs: 6 + +old observation per round: + outcome: target_missing + native dispatch: none + target mutation: 0 -> 0 + +fresh observation: + background px success: 4 rounds + fail_closed_occluded: 1 round + decoy mutation: always 0 -> 0 + +cua-driver service: + action generation: 1 throughout + capture generation: 0 throughout + restartAttempts: 0 throughout + +desktop concurrency: + fixture became frontmost: never + user pointer moved: 151.2 logical points + user app switching: allowed +``` + +The occluded round proved: + +```text +outcome: target_occluded +native dispatch: none +target mutation: 0 -> 0 +decoy mutation: 0 -> 0 +``` + +The test command is: + +```bash +npm run e2e:computer-use-process-restart +``` + +## Remaining Boundary + +This proves ordinary process restart isolation. It does not force the operating +system to reuse the old numeric PID. PID-reuse safety still requires either: + +- a native atomic process-instance identity exposed by the executor; or +- a deterministic driver/host fixture that can substitute a new process under + the same PID-shaped identity. + +The current fail-closed old-window lookup is sufficient for the observed real +restart, but it is not evidence for deliberate PID reuse. diff --git a/package.json b/package.json index 07855334c5..760ed2647e 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "typecheck": "npm run typecheck --workspaces --if-present", "test": "npm run test:scripts && npm --workspace @maka/core test && npm --workspace @maka/storage test && npm --workspace @maka/runtime test && npm --workspace @maka/computer-use test && npm --workspace @maka/headless test && npm --workspace maka-agent test && npm --workspace @maka/ui test && npm --workspace @maka/desktop test", "test:dist": "npm run test:scripts && npm exec -w @maka/core -- node --test \"dist/**/*.test.js\" && npm exec -w @maka/storage -- node --test \"dist/**/*.test.js\" && npm exec -w @maka/runtime -- node --test \"dist/**/*.test.js\" && npm exec -w @maka/computer-use -- node --test \"dist/**/*.test.js\" && npm exec -w @maka/headless -- node ../../scripts/run-headless-tests.mjs && npm exec -w maka-agent -- node --test \"dist/**/*.test.js\" && npm exec -w @maka/ui -- node --test \"dist/**/*.test.js\" && npm --workspace @maka/desktop run test:dist", - "test:scripts": "node --test scripts/run-headless-tests.test.mjs scripts/sync-model-metadata.test.mjs scripts/cua-driver-provenance.test.mjs scripts/cu-real-e2e-contract.test.mjs", + "test:scripts": "node --test scripts/run-headless-tests.test.mjs scripts/sync-model-metadata.test.mjs scripts/cua-driver-provenance.test.mjs scripts/cu-real-e2e-contract.test.mjs scripts/cu-process-restart-e2e-contract.test.mjs", "dev": "npm --workspace @maka/desktop run dev:hmr --", "dev:full": "npm run build && npm --workspace @maka/desktop run start", "build": "npm --workspace @maka/core run build && npm --workspace @maka/storage run build && npm --workspace @maka/runtime run build && npm --workspace @maka/computer-use run build && npm --workspace @maka/headless run build && npm --workspace maka-agent run build && npm --workspace @maka/ui run build && npm --workspace @maka/desktop run build", @@ -35,7 +35,8 @@ "prepare:cua-driver": "node scripts/prepare-cua-driver.mjs", "check:cua-driver-artifact": "node scripts/check-cua-driver-bundle.mjs", "e2e:computer-use-real": "node scripts/cu-real-e2e-launcher.mjs", - "e2e:computer-use-concurrent": "node scripts/cu-real-e2e-launcher.mjs --concurrent-user" + "e2e:computer-use-concurrent": "node scripts/cu-real-e2e-launcher.mjs --concurrent-user", + "e2e:computer-use-process-restart": "node scripts/cu-process-restart-e2e-launcher.mjs" }, "devDependencies": { "@types/node": "^25.0.0", diff --git a/scripts/cu-process-restart-e2e-contract.test.mjs b/scripts/cu-process-restart-e2e-contract.test.mjs new file mode 100644 index 0000000000..689467d874 --- /dev/null +++ b/scripts/cu-process-restart-e2e-contract.test.mjs @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; + +const [packageJson, launcher, harness] = await Promise.all([ + readFile(new URL('../package.json', import.meta.url), 'utf8'), + readFile(new URL('./cu-process-restart-e2e-launcher.mjs', import.meta.url), 'utf8'), + readFile(new URL('./cu-process-restart-e2e.mjs', import.meta.url), 'utf8'), +]); + +test('process restart E2E owns the real fixture lifecycle', () => { + assert.match(packageJson, /e2e:computer-use-process-restart/); + assert.match(launcher, /runFixtureScript\('stop\.sh'\)/); + assert.match(launcher, /runFixtureScript\('launch\.sh', \{/); + assert.match(launcher, /prepare:cua-driver/); + assert.doesNotMatch(launcher, /activateFixture/); + assert.match(launcher, /CUA_LAB_BACKGROUND/); + assert.match(launcher, /finalFrontmost\?\.bundleIdentifier === FIXTURE_BUNDLE_ID/); + assert.match(launcher, /waitForRestartedState\([\s\S]*currentPID,[\s\S]*request\.oldWebContentPID/); + assert.match(launcher, /waitForInitialState\(\)/); + assert.match(launcher, /pointerBefore/); + assert.match(launcher, /Concurrent user pointer displacement observed/); + assert.match(launcher, /Synthetic fixture never became frontmost/); + assert.match(launcher, /--snapshot/); + assert.match(launcher, /restart-request-\$\{round\}\.json/); + assert.match(launcher, /restart-complete-\$\{round\}\.json/); + assert.match(launcher, /SOAK_ROUNDS = 5/); + assert.match(launcher, /--deny-frontmost-bundle/); + assert.match(launcher, /--concurrent-user',[\s\S]*'0'/); + assert.match(launcher, /for \(let round = 1; round <= SOAK_ROUNDS; round \+= 1\)/); + assert.match(launcher, /caffeinate', \['-dimsu'\]/); +}); + +test('old observation is rejected and fresh-process actions succeed or fail occluded', () => { + assert.match(harness, /old-observation-after-restart/); + assert.match(harness, /oldRunResult\?\.error !== 'target_missing'/); + assert.match(harness, /staleAttempt\.modelText/); + assert.match(harness, /newState\.coordinate\.clickCount !== 0/); + assert.match(harness, /fresh-process-coordinate-click/); + assert.match(harness, /observeUntilElement/); + assert.match(harness, /invalidApp: no visible window matched/); + assert.match(harness, /CUA Lab Coordinate Target/); + assert.match(harness, /candidateCount/); + assert.match(harness, /freshSucceeded/); + assert.match(harness, /freshOccluded/); + assert.match(harness, /fail_closed_occluded/); + assert.match(harness, /background_dispatch_succeeded/); + assert.match(harness, /currentPID === newPID/); + assert.match(harness, /currentWebContentPID === newWebContentPID/); + assert.match(harness, /for \(let round = 1; round <= soakRounds; round \+= 1\)/); + assert.match(harness, /seenHostPIDs/); + assert.match(harness, /seenWebContentPIDs/); + assert.match(harness, /serviceState/); +}); + +test('restart reports are private launcher-owned temporary files', () => { + assert.match(harness, /MAKA_CU_RESTART_TEMP_DIR/); + assert.match(harness, /relative\(resolve\(tmpdir\(\)\)/); + assert.match(harness, /flag: 'wx'/); + assert.match(launcher, /rm\(temporaryDirectory, \{ recursive: true, force: true \}\)/); +}); diff --git a/scripts/cu-process-restart-e2e-launcher.mjs b/scripts/cu-process-restart-e2e-launcher.mjs new file mode 100644 index 0000000000..e048259ac7 --- /dev/null +++ b/scripts/cu-process-restart-e2e-launcher.mjs @@ -0,0 +1,406 @@ +import { spawn } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(here, '..'); +const harnessPath = join(here, 'cu-process-restart-e2e.mjs'); +const monitorPath = join(here, 'cu-real-e2e-monitor.swift'); +const labRoot = '/Users/haoqing/Documents/Learning/codex-computer-use-lab'; +const statePath = join(labRoot, 'test-app', 'runtime', 'state.json'); +const SOAK_ROUNDS = 5; +const FIXTURE_BUNDLE_ID = 'com.openai.codex.cualab'; + +const delay = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +function runChild(file, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(file, args, { + cwd: options.cwd, + env: options.env, + stdio: options.stdio ?? ['ignore', 'inherit', 'inherit'], + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) resolve(); + else reject(new Error(`${file} failed (${signal ?? code})`)); + }); + }); +} + +async function runFixtureScript(name, options = {}) { + await runChild(join(labRoot, 'test-app', name), [], options); +} + +async function runBuilds() { + for (const workspace of ['@maka/core', '@maka/runtime', '@maka/computer-use']) { + await runChild('npm', ['--workspace', workspace, 'run', 'build'], { + cwd: repoRoot, + }); + } + await runChild('npm', ['run', 'prepare:cua-driver'], { cwd: repoRoot }); + await runChild('npm', ['run', 'check:cua-driver-artifact'], { cwd: repoRoot }); +} + +async function frontmostApplication() { + const script = [ + 'tell application "System Events"', + 'set frontProcess to first application process whose frontmost is true', + 'return (unix id of frontProcess as text) & tab & (bundle identifier of frontProcess as text)', + 'end tell', + ].join('\n'); + let output = ''; + await new Promise((resolve, reject) => { + const child = spawn('/usr/bin/osascript', ['-e', script], { + stdio: ['ignore', 'pipe', 'inherit'], + }); + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { output += chunk; }); + child.once('error', reject); + child.once('exit', (code) => { + if (code === 0) resolve(); + else reject(new Error(`failed to capture frontmost application (${code})`)); + }); + }); + const [pid, bundleIdentifier] = output.trim().split('\t'); + if (!Number.isInteger(Number(pid)) || !bundleIdentifier) { + throw new Error(`invalid frontmost application identity: ${output.trim()}`); + } + return { pid: Number(pid), bundleIdentifier }; +} + +async function restoreFrontmost(application) { + const escaped = application.bundleIdentifier + .replaceAll('\\', '\\\\') + .replaceAll('"', '\\"'); + const script = [ + 'tell application "System Events"', + `if exists (first application process whose unix id is ${application.pid}) then`, + `set frontmost of first application process whose unix id is ${application.pid} to true`, + 'else', + `tell application id "${escaped}" to activate`, + 'end if', + 'end tell', + ].join('\n'); + await runChild('/usr/bin/osascript', ['-e', script], { + stdio: ['ignore', 'ignore', 'ignore'], + }); +} + +async function pointerLocation() { + let output = ''; + await new Promise((resolve, reject) => { + const child = spawn('swift', [monitorPath, '--snapshot'], { + stdio: ['ignore', 'pipe', 'inherit'], + }); + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { output += chunk; }); + child.once('error', reject); + child.once('exit', (code) => { + if (code === 0) resolve(); + else reject(new Error(`failed to capture desktop snapshot (${code})`)); + }); + }); + const [kind, , , xText, yText] = output.trim().split('\t'); + const x = Number(xText); + const y = Number(yText); + if (kind !== 'READY' || !Number.isFinite(x) || !Number.isFinite(y)) { + throw new Error(`invalid desktop snapshot: ${output.trim()}`); + } + return { x, y }; +} + +async function terminateChild(child, label, timeoutMs = 3_000) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((resolve) => child.once('exit', resolve)); + child.kill('SIGTERM'); + if (await Promise.race([exited.then(() => true), delay(timeoutMs).then(() => false)])) { + return; + } + child.kill('SIGKILL'); + if (!await Promise.race([exited.then(() => true), delay(timeoutMs).then(() => false)])) { + throw new Error(`${label} did not exit after SIGKILL`); + } +} + +async function waitForJson(path, label, timeoutMs = 15_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + return JSON.parse(await readFile(path, 'utf8')); + } catch (error) { + if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error; + } + await delay(50); + } + throw new Error(`${label} timeout`); +} + +async function waitForRestartedState(oldPID, oldWebContentPID, timeoutMs = 15_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const state = JSON.parse(await readFile(statePath, 'utf8')); + const newPID = state?.oop?.hostPID; + const newWebContentPID = state?.oop?.webContentPID; + if ( + Number.isInteger(newPID) + && newPID > 0 + && newPID !== oldPID + && Number.isInteger(newWebContentPID) + && newWebContentPID > 0 + && newWebContentPID !== oldWebContentPID + ) { + try { + process.kill(newPID, 0); + return state; + } catch { + // The state raced process exit; wait for the next launched instance. + } + } + } catch (error) { + if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error; + } + await delay(50); + } + throw new Error( + `fixture restart state did not advance from host/WebContent ` + + `${oldPID}/${oldWebContentPID}`, + ); +} + +async function waitForInitialState(timeoutMs = 15_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const state = JSON.parse(await readFile(statePath, 'utf8')); + const hostPID = state?.oop?.hostPID; + const webContentPID = state?.oop?.webContentPID; + if ( + Number.isInteger(hostPID) + && hostPID > 0 + && Number.isInteger(webContentPID) + && webContentPID > 0 + ) { + process.kill(hostPID, 0); + return state; + } + } catch (error) { + if ( + error?.code !== 'ENOENT' + && error?.code !== 'ESRCH' + && !(error instanceof SyntaxError) + ) throw error; + } + await delay(50); + } + throw new Error('initial fixture state did not publish live host/WebContent PIDs'); +} + +function startFocusMonitor() { + const child = spawn('swift', [ + monitorPath, + '--concurrent-user', + '0', + '--deny-frontmost-bundle', + FIXTURE_BUNDLE_ID, + ], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let buffer = ''; + let stderr = ''; + let readyResolve; + let readyReject; + let failureResolve; + let readySettled = false; + let failureSettled = false; + const ready = new Promise((resolve, reject) => { + readyResolve = resolve; + readyReject = reject; + }); + const failure = new Promise((resolve) => { failureResolve = resolve; }); + const fail = (error) => { + const normalized = error instanceof Error ? error : new Error(String(error)); + if (!readySettled) { + readySettled = true; + readyReject(normalized); + } + if (!failureSettled) { + failureSettled = true; + failureResolve(normalized); + } + }; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + buffer += chunk; + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + for (const line of lines.map((value) => value.trim()).filter(Boolean)) { + const [kind, ...fields] = line.split('\t'); + if (kind === 'READY') { + readySettled = true; + readyResolve({ + frontmostPID: Number(fields[1]), + pointer: { x: Number(fields[2]), y: Number(fields[3]) }, + bundleIdentifier: fields[4], + }); + } else if (kind === 'CHANGE' || kind === 'ERROR') { + fail(new Error(fields.join('\t') || line)); + } + } + }); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('error', fail); + child.on('exit', (code, signal) => { + if (!failureSettled && code !== 0 && signal !== 'SIGTERM') { + fail(new Error( + `focus monitor exited (${signal ?? code})` + + `${stderr.trim() ? `: ${stderr.trim()}` : ''}`, + )); + } + }); + return { + child, + ready, + failure, + stop: () => terminateChild(child, 'restart focus monitor'), + }; +} + +async function run() { + const originalFrontmost = await frontmostApplication(); + const temporaryDirectory = await mkdtemp(join(tmpdir(), 'maka-cu-restart-e2e-')); + let fixtureTouched = false; + let caffeinate; + let harness; + let monitor; + try { + caffeinate = spawn('/usr/bin/caffeinate', ['-dimsu'], { + stdio: ['ignore', 'ignore', 'inherit'], + }); + await runBuilds(); + fixtureTouched = true; + await runFixtureScript('stop.sh'); + await runFixtureScript('reset.sh'); + const pointerBefore = await pointerLocation(); + monitor = startFocusMonitor(); + await Promise.race([ + monitor.ready, + delay(10_000).then(() => { + throw new Error('restart focus monitor startup timeout'); + }), + ]); + await runFixtureScript('launch.sh', { + env: { ...process.env, CUA_LAB_BACKGROUND: '1' }, + }); + const oldState = await waitForInitialState(); + const oldPID = oldState?.oop?.hostPID; + if (!Number.isInteger(oldPID) || oldPID <= 0) { + throw new Error('old synthetic fixture did not publish a valid PID'); + } + harness = spawn(process.execPath, [harnessPath], { + cwd: repoRoot, + env: { + ...process.env, + MAKA_CU_RESTART_OLD_PID: String(oldPID), + MAKA_CU_RESTART_SOAK_ROUNDS: String(SOAK_ROUNDS), + MAKA_CU_RESTART_TEMP_DIR: temporaryDirectory, + }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + const exit = new Promise((resolve, reject) => { + harness.once('error', reject); + harness.once('exit', (code, signal) => resolve({ code, signal })); + }); + let currentPID = oldPID; + for (let round = 1; round <= SOAK_ROUNDS; round += 1) { + const requestPath = join( + temporaryDirectory, + `restart-request-${round}.json`, + ); + const completePath = join( + temporaryDirectory, + `restart-complete-${round}.json`, + ); + const request = await Promise.race([ + waitForJson(requestPath, `restart request ${round}`), + exit.then((result) => { + throw new Error( + `restart harness exited before round ${round} ` + + `(${result.signal ?? result.code})`, + ); + }), + monitor.failure.then((error) => { throw error; }), + ]); + if (request.round !== round || request.oldPID !== currentPID) { + throw new Error(`restart request ${round} identity mismatch`); + } + if (!Number.isInteger(request.oldWebContentPID) || request.oldWebContentPID <= 0) { + throw new Error(`restart request ${round} has no old WebContent PID`); + } + await runFixtureScript('stop.sh'); + await runFixtureScript('launch.sh', { + env: { ...process.env, CUA_LAB_BACKGROUND: '1' }, + }); + const newState = await waitForRestartedState( + currentPID, + request.oldWebContentPID, + ); + const newPID = newState?.oop?.hostPID; + if (!Number.isInteger(newPID) || newPID <= 0 || currentPID === newPID) { + throw new Error( + `fixture restart ${round} did not create a new process: ` + + `${currentPID} -> ${newPID}`, + ); + } + await writeFile( + completePath, + `${JSON.stringify({ + round, + oldPID: currentPID, + newPID, + newWebContentPID: newState.oop.webContentPID, + })}\n`, + { flag: 'wx', mode: 0o600 }, + ); + currentPID = newPID; + } + const result = await Promise.race([ + exit, + monitor.failure.then((error) => { throw error; }), + ]); + if (result.code !== 0) { + throw new Error(`restart soak E2E failed (${result.signal ?? result.code})`); + } + const pointerAfter = await pointerLocation(); + const displacement = Math.hypot( + pointerAfter.x - pointerBefore.x, + pointerAfter.y - pointerBefore.y, + ); + process.stdout.write( + `Synthetic fixture never became frontmost across ${SOAK_ROUNDS} rounds\n`, + ); + process.stdout.write( + `Concurrent user pointer displacement observed: ${displacement}\n`, + ); + } finally { + await terminateChild(harness, 'restart E2E harness').catch(() => {}); + await monitor?.stop().catch(() => {}); + if (fixtureTouched) await runFixtureScript('stop.sh').catch(() => {}); + const finalFrontmost = await frontmostApplication().catch(() => undefined); + if (finalFrontmost?.bundleIdentifier === FIXTURE_BUNDLE_ID) { + await restoreFrontmost(originalFrontmost).catch(() => {}); + } + await terminateChild(caffeinate, 'restart caffeinate').catch(() => {}); + await rm(temporaryDirectory, { recursive: true, force: true }); + } +} + +run().catch((error) => { + console.error('Computer Use process restart E2E failed:', error); + process.exitCode = 1; +}); diff --git a/scripts/cu-process-restart-e2e.mjs b/scripts/cu-process-restart-e2e.mjs new file mode 100644 index 0000000000..2c081ffa4a --- /dev/null +++ b/scripts/cu-process-restart-e2e.mjs @@ -0,0 +1,427 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, relative, resolve } from 'node:path'; +import { createCuaDriverBackend } from '../packages/computer-use/dist/index.js'; +import { buildComputerUseTools } from '../packages/runtime/dist/index.js'; + +const repoRoot = new URL('..', import.meta.url).pathname; +const binaryPath = join(repoRoot, 'apps/desktop/resources/bin/cua-driver'); +const labRoot = '/Users/haoqing/Documents/Learning/codex-computer-use-lab'; +const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app'); +const statePath = join(labRoot, 'test-app/runtime/state.json'); +const temporaryDirectory = process.env.MAKA_CU_RESTART_TEMP_DIR; +const oldPID = Number(process.env.MAKA_CU_RESTART_OLD_PID); +const soakRounds = Number(process.env.MAKA_CU_RESTART_SOAK_ROUNDS); +const expectedBinarySha256 = + '683dad5cccb47dd0a8bb5d534d62fbb9e6edfb1cded232509cf4c2b190066040'; + +const resolvedTemporaryDirectory = resolve(temporaryDirectory ?? ''); +const temporaryRelativePath = relative(resolve(tmpdir()), resolvedTemporaryDirectory); +if ( + !temporaryDirectory + || temporaryRelativePath.startsWith('..') + || temporaryRelativePath === '' + || !Number.isInteger(oldPID) + || oldPID <= 0 + || !Number.isInteger(soakRounds) + || soakRounds < 1 + || soakRounds > 20 +) { + throw new Error('process restart E2E requires launcher-owned inputs'); +} + +const reportPath = join(resolvedTemporaryDirectory, 'report.json'); +const restartRequestPath = (round) => + join(resolvedTemporaryDirectory, `restart-request-${round}.json`); +const restartCompletePath = (round) => + join(resolvedTemporaryDirectory, `restart-complete-${round}.json`); +const delay = (milliseconds) => + new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)); + +async function waitForJson(path, label, timeoutMs = 15_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + return JSON.parse(await readFile(path, 'utf8')); + } catch (error) { + if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error; + } + await delay(50); + } + throw new Error(`${label} timeout`); +} + +const traces = []; +const idFlow = []; +let latestObservationGeometry; +const backend = createCuaDriverBackend({ + binaryPath, + hostBundleId: 'com.maka.desktop', + expectedBinarySha256, + expectedServerName: 'cua-driver', + expectedServerVersion: '0.7.1', + expectedProtocolVersion: '2025-06-18', + timeoutMs: 10_000, + onTrace(event) { + traces.push(event); + }, +}); +const instrumentedBackend = { + ...backend, + async observeApp(input, signal, context) { + const observation = await backend.observeApp(input, signal, context); + latestObservationGeometry = { + windowBounds: observation.windowBounds, + sourceBoundsPx: observation.sourceBoundsPx, + }; + idFlow.push({ + phase: 'observe', + observationId: observation.observationId, + pid: observation.pid, + sessionId: context.sessionId, + turnId: context.turnId, + }); + return observation; + }, + async run(action, signal, context) { + const result = await backend.run(action, signal, context); + idFlow.push({ + phase: 'runResult', + toolCallId: context.toolCallId, + ok: result.outcome.ok, + ...(!result.outcome.ok + ? { error: result.outcome.error, message: result.outcome.message } + : {}), + }); + return result; + }, +}; +const tools = buildComputerUseTools({ backend: instrumentedBackend }); +const [tool] = tools; +const context = (toolCallId, turnId) => ({ + sessionId: 'process-restart-e2e', + turnId, + toolCallId, + cwd: repoRoot, + abortSignal: new AbortController().signal, + emitOutput() {}, +}); +const call = (input, toolCallId, turnId) => + tool.impl(input, context(toolCallId, turnId)); +const parseModel = (result) => JSON.parse(result.modelText ?? '{}'); +const readState = async () => JSON.parse(await readFile(statePath, 'utf8')); + +async function observe(pid, toolCallId, turnId) { + const result = await call({ + action: 'observe', + app: `pid:${pid}`, + include_screenshot: true, + }, toolCallId, turnId); + if (result.error || !result.modelText) { + throw new Error(`observe failed: ${result.error ?? result.text}`); + } + return { + result, + model: parseModel(result), + persisted: JSON.parse(result.text), + geometry: latestObservationGeometry, + }; +} + +async function observeUntilElement(pid, label, toolCallPrefix, turnId, timeoutMs = 8_000) { + const deadline = Date.now() + timeoutMs; + let lastObserved; + let lastObservationError; + while (Date.now() < deadline) { + try { + lastObserved = await observe( + pid, + `${toolCallPrefix}-${Date.now()}`, + turnId, + ); + lastObservationError = undefined; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/invalidApp: no visible window matched/.test(message)) throw error; + lastObservationError = message; + await delay(200); + continue; + } + const matches = lastObserved.model.elements + .filter((element) => element.label === label) + .sort((left, right) => Number(left.element_id) - Number(right.element_id)); + if (matches.length > 0) { + return { + observed: lastObserved, + element: matches[0], + candidateCount: matches.length, + }; + } + await delay(200); + } + const labels = lastObserved?.model.elements + ?.map((element) => element.label) + .filter(Boolean) + .slice(0, 30) ?? []; + throw new Error( + `${label} did not appear before timeout; ` + + `lastObservationError=${lastObservationError ?? 'none'}; ` + + `labels=${JSON.stringify(labels)}`, + ); +} + +function coordinateForElement(observed, element) { + const screenshot = observed.persisted.screenshot; + const frame = element.frame; + const window = observed.geometry?.windowBounds; + if (!screenshot || !frame || !window) { + throw new Error('restart coordinate evidence is incomplete'); + } + return [ + Math.round( + (frame.x + frame.width / 2 - window.x) + / window.width * screenshot.width_px, + ), + Math.round( + (frame.y + frame.height / 2 - window.y) + / window.height * screenshot.height_px, + ), + ]; +} + +async function writeReport(report) { + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, { + flag: 'wx', + mode: 0o600, + }); +} + +const report = { + schemaVersion: 1, + soakRounds, + fixture: { + bundleIdentifier: 'com.openai.codex.cualab', + canonicalAppPath: expectedAppPath, + oldPID, + }, + cases: [], +}; + +try { + const seenHostPIDs = new Set(); + const seenWebContentPIDs = new Set(); + let currentPID = oldPID; + let serviceGenerations; + for (let round = 1; round <= soakRounds; round += 1) { + const currentState = await readState(); + if ( + currentState.synthetic !== true + || currentState.syntheticMarker !== 'CUA Lab Synthetic Surface' + || currentState.appPath !== expectedAppPath + || currentState.oop.hostPID !== currentPID + || !Number.isInteger(currentState.oop.webContentPID) + || currentState.oop.webContentPID <= 0 + ) { + throw new Error(`round ${round} fixture provenance mismatch`); + } + const currentWebContentPID = currentState.oop.webContentPID; + if ( + seenHostPIDs.has(currentPID) + || seenWebContentPIDs.has(currentWebContentPID) + ) { + throw new Error(`round ${round} reused a prior process identity`); + } + seenHostPIDs.add(currentPID); + seenWebContentPIDs.add(currentWebContentPID); + + const oldReady = await observeUntilElement( + currentPID, + 'CUA Lab Coordinate Target', + `observe-old-process-r${round}`, + `turn-old-r${round}`, + ); + const oldObserved = oldReady.observed; + const oldCoordinate = coordinateForElement( + oldObserved, + oldReady.element, + ); + await writeFile( + restartRequestPath(round), + `${JSON.stringify({ + round, + oldPID: currentPID, + oldWebContentPID: currentWebContentPID, + observationId: oldObserved.model.observation_id, + candidateCount: oldReady.candidateCount, + })}\n`, + { flag: 'wx', mode: 0o600 }, + ); + + const restart = await waitForJson( + restartCompletePath(round), + `fixture restart ${round}`, + ); + const newPID = restart.newPID; + const newWebContentPID = restart.newWebContentPID; + if ( + restart.round !== round + || restart.oldPID !== currentPID + || !Number.isInteger(newPID) + || newPID <= 0 + || currentPID === newPID + || !Number.isInteger(newWebContentPID) + || newWebContentPID <= 0 + || currentWebContentPID === newWebContentPID + || seenHostPIDs.has(newPID) + || seenWebContentPIDs.has(newWebContentPID) + ) { + throw new Error(`round ${round} restart identity did not advance`); + } + + const staleToolCallId = `old-observation-after-restart-r${round}`; + const staleAttempt = await call({ + action: 'left_click', + observation_id: oldObserved.model.observation_id, + coordinate: oldCoordinate, + }, staleToolCallId, `turn-old-r${round}`); + await delay(150); + const newState = await readState(); + const oldRunResult = idFlow.find( + (event) => event.phase === 'runResult' + && event.toolCallId === staleToolCallId, + ); + const staleDispatch = traces.find( + (event) => event.type === 'dispatch' + && event.toolCallId === staleToolCallId, + ); + if ( + oldRunResult?.error !== 'target_missing' + || !/target_missing/.test(staleAttempt.modelText ?? '') + || staleDispatch + || newState.coordinate.clickCount !== 0 + || newState.coordinate.decoyClickCount !== 0 + ) { + throw new Error(`round ${round} old observation crossed process restart`); + } + + tools.clearSession('process-restart-e2e'); + const freshReady = await observeUntilElement( + newPID, + 'CUA Lab Coordinate Target', + `observe-new-process-r${round}`, + `turn-new-r${round}`, + ); + const freshCoordinate = coordinateForElement( + freshReady.observed, + freshReady.element, + ); + const freshToolCallId = `fresh-process-coordinate-click-r${round}`; + const freshClick = await call({ + action: 'left_click', + observation_id: freshReady.observed.model.observation_id, + coordinate: freshCoordinate, + }, freshToolCallId, `turn-new-r${round}`); + await delay(150); + const newStateAfterClick = await readState(); + const dispatch = traces.find( + (event) => event.type === 'dispatch' + && event.toolCallId === freshToolCallId, + ); + const freshRunResult = idFlow.find( + (event) => event.phase === 'runResult' + && event.toolCallId === freshToolCallId, + ); + const freshOccluded = freshRunResult?.error === 'target_occluded'; + const freshSucceeded = !freshClick.error + && dispatch?.address === 'px' + && newStateAfterClick.coordinate.clickCount === 1 + && newStateAfterClick.coordinate.decoyClickCount === 0; + if ( + (!freshSucceeded && !freshOccluded) + || (freshOccluded && ( + dispatch + || newStateAfterClick.coordinate.clickCount !== 0 + || newStateAfterClick.coordinate.decoyClickCount !== 0 + )) + || newStateAfterClick.oop.hostPID !== newPID + || newStateAfterClick.oop.webContentPID !== newWebContentPID + ) { + throw new Error(`round ${round} fresh process action violated its oracle`); + } + + const serviceState = backend.serviceState(); + for (const role of ['action', 'capture']) { + if (serviceState[role].restartAttempts !== 0) { + throw new Error(`round ${round} ${role} service consumed restart budget`); + } + } + const generations = { + action: serviceState.action.generation, + capture: serviceState.capture.generation, + }; + if (!serviceGenerations) { + serviceGenerations = generations; + } else if ( + generations.action !== serviceGenerations.action + || generations.capture !== serviceGenerations.capture + ) { + throw new Error(`round ${round} unexpectedly restarted cua-driver services`); + } + + report.cases.push({ + round, + oldPID: currentPID, + oldWebContentPID: currentWebContentPID, + newPID, + newWebContentPID, + stale: { + error: oldRunResult.error, + mutation: [0, newState.coordinate.clickCount], + dispatch: false, + }, + fresh: { + outcome: freshOccluded + ? 'fail_closed_occluded' + : 'background_dispatch_succeeded', + dispatchAddress: dispatch?.address, + mutation: [ + newState.coordinate.clickCount, + newStateAfterClick.coordinate.clickCount, + ], + decoyMutation: [ + newState.coordinate.decoyClickCount, + newStateAfterClick.coordinate.decoyClickCount, + ], + }, + serviceGenerations: generations, + }); + currentPID = newPID; + tools.clearSession('process-restart-e2e'); + } + + report.ok = true; + report.claim = + 'old observations never cross repeated real app-process restarts; fresh background actions either dispatch to the new process or fail occluded without side effects'; + report.evidence = { + seenHostPIDs: [...seenHostPIDs], + seenWebContentPIDs: [...seenWebContentPIDs], + serviceGenerations, + idFlow, + traceTypes: traces.map((event) => event.type), + }; + await writeReport(report); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} catch (error) { + const failureReport = { + ...report, + ok: false, + failure: error instanceof Error ? error.message : String(error), + evidence: { idFlow, traces }, + }; + await writeReport(failureReport).catch(() => {}); + process.stderr.write(`${JSON.stringify(failureReport, null, 2)}\n`); + throw error; +} finally { + tools.clearSession('process-restart-e2e'); + backend.dispose(); +} diff --git a/scripts/cu-real-e2e-monitor.swift b/scripts/cu-real-e2e-monitor.swift index d0c749d57a..7eb6f21500 100644 --- a/scripts/cu-real-e2e-monitor.swift +++ b/scripts/cu-real-e2e-monitor.swift @@ -5,6 +5,17 @@ import Darwin setbuf(stdout, nil) let concurrentUserMode = CommandLine.arguments.contains("--concurrent-user") +let snapshotMode = CommandLine.arguments.contains("--snapshot") +let deniedFrontmostBundle: String? = { + guard let flagIndex = CommandLine.arguments.firstIndex( + of: "--deny-frontmost-bundle" + ), + CommandLine.arguments.indices.contains(flagIndex + 1) + else { + return nil + } + return CommandLine.arguments[flagIndex + 1] +}() let mode = concurrentUserMode ? "concurrent_user" : "isolated" let fixturePID: pid_t? = { guard concurrentUserMode, @@ -61,6 +72,9 @@ print( "READY\t\(mode)\t\(frontmostPID)\t\(initialPointer.x)\t\(initialPointer.y)" + "\t\(bundleIdentifier)\t\(bundlePath)" ) +if snapshotMode { + exit(0) +} while true { autoreleasepool { @@ -70,7 +84,16 @@ while true { } let currentPID = NSWorkspace.shared.frontmostApplication?.processIdentifier ?? -1 + let currentBundle = + NSWorkspace.shared.frontmostApplication?.bundleIdentifier let currentPointer = NSEvent.mouseLocation + if let deniedFrontmostBundle, currentBundle == deniedFrontmostBundle { + print( + "CHANGE\tdenied bundle became frontmost during E2E: " + + deniedFrontmostBundle + ) + exit(3) + } if concurrentUserMode && currentPID == fixturePID { print("CHANGE\tsynthetic fixture became frontmost during concurrent E2E") exit(3)