diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 7e21884540e3..775d8d9bd4d6 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -61,7 +61,9 @@ "check": "npm run check:lint && npm run check:test:ui && npm run test:desktop:platforms && npm run test:desktop:all", "test:e2e": "npm run build && playwright test e2e/", "test:e2e:visual": "npm run build && WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list", - "test:e2e:update-snapshots": "npm run build && WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list --update-snapshots" + "test:e2e:update-snapshots": "npm run build && WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list --update-snapshots", + "repro:short-session-hang": "node scripts/run-short-session-hang-repro.mjs", + "repro:short-session-hang:test": "node --test scripts/run-short-session-hang-repro.test.mjs" }, "dependencies": { "@assistant-ui/core": "0.2.23", diff --git a/apps/desktop/scripts/run-short-session-hang-repro.mjs b/apps/desktop/scripts/run-short-session-hang-repro.mjs new file mode 100644 index 000000000000..5645bd5e2c81 --- /dev/null +++ b/apps/desktop/scripts/run-short-session-hang-repro.mjs @@ -0,0 +1,1702 @@ +#!/usr/bin/env node + +import { spawn, spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + appendFileSync, + cpSync, + copyFileSync, + createWriteStream, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync +} from 'node:fs' +import http from 'node:http' +import { createRequire } from 'node:module' +import { createServer } from 'node:net' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { finished } from 'node:stream/promises' +import { fileURLToPath } from 'node:url' + +import { CDP, discoverTarget, sleep } from './perf/lib/cdp.mjs' + +const HERE = dirname(fileURLToPath(import.meta.url)) +const DESKTOP_ROOT = resolve(HERE, '..') +const REPO_ROOT = resolve(DESKTOP_ROOT, '..', '..') +const HARNESS_SOURCE = resolve(DESKTOP_ROOT, 'src/app/chat/short-session-hang-repro.tsx') +const UPSTREAM_URL = 'https://github.com/NousResearch/hermes-agent.git' +const DEFAULT_BASELINE = '3651627d88858912e8460e6f949b7125725600c3' +const DEFAULT_CANDIDATE = '3651627d88858912e8460e6f949b7125725600c3' +const FREEZE_MS = 5_000 +const JOURNAL_SEED_TIMEOUT_MS = 30_000 +const STREAM_RESPONSE_TIMEOUT_MS = 30_000 +const STREAM_RESPONSE_EVALUATION_TIMEOUT_MS = 30_000 +const STREAM_PAYLOAD_BYTES = 512 * 1024 +const LEGACY_STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' +const LEGACY_MIGRATION_KEY = 'hermes.desktop.inflightTurnJournal.v2.migrated' +const LEGACY_SEED_ENTRY_COUNT = 15 +const MAX_LEGACY_SEED_BYTES = 1.9 * 1024 * 1024 +// Reserve space for the JSON envelope and per-entry metadata so the generated +// fixture remains below the guard even when the entry count or shape changes. +const LEGACY_SEED_METADATA_BUDGET_BYTES = 16 * 1024 +const LEGACY_SEED_PAYLOAD_CHARS = Math.floor( + (MAX_LEGACY_SEED_BYTES - LEGACY_SEED_METADATA_BUDGET_BYTES) / LEGACY_SEED_ENTRY_COUNT +) +const EXCHANGE_COUNT = 5 +const NATIVE_VISIBILITY_PAUSE_MS = 750 +const EXCHANGE_FREEZE_TIMEOUTS = 9 +const POST_EXCHANGE_FREEZE_TIMEOUTS = 13 +const WATCHDOG_OVERHEAD_MS = 120_000 +// The watchdog is a last-resort guard around the complete five-exchange run. +// Inner timeouts still identify actual renderer hangs; this budget prevents a +// slow but valid stream/response sequence from being misclassified by the old +// fixed 120s outer limit. The overhead covers renderer startup/cleanup and +// native visibility calls not represented by the per-operation timers. +const OUTER_WATCHDOG_MS = + EXCHANGE_COUNT * + (EXCHANGE_FREEZE_TIMEOUTS * FREEZE_MS + + STREAM_RESPONSE_TIMEOUT_MS + + STREAM_RESPONSE_EVALUATION_TIMEOUT_MS + + NATIVE_VISIBILITY_PAUSE_MS) + + POST_EXCHANGE_FREEZE_TIMEOUTS * FREEZE_MS + + 2_000 + + WATCHDOG_OVERHEAD_MS +const SECRET_ENV_RE = /(credential|token|secret|password|(^|_)key($|_)|auth|cookie)/i +const HEX_OBJECT_RE = /^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$/ +const PULL_REF_RE = /^refs\/pull\/[1-9][0-9]*\/(?:head|merge)$/ +const HEAD_REF_RE = /^refs\/heads\/[A-Za-z0-9][A-Za-z0-9._/-]*$/ +const HEAD_COMPONENT_RE = /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/ +const require = createRequire(import.meta.url) + +class ReproductionError extends Error { + name = 'ReproductionError' +} + +function usage() { + console.log(`Usage: node scripts/run-short-session-hang-repro.mjs [options] + +Options: + --baseline baseline ref (default: ${DEFAULT_BASELINE}) + --candidate candidate ref (default: ${DEFAULT_CANDIDATE}) + --repetitions measured repetitions per ref (default: 5) + --output artifact directory (default: short-session-hang-artifacts) + --keep-worktrees retain ephemeral source copies + --dry-run validate refs, lockfile/Electron parity, and print the plan + --help show this help + +Each ref gets one warm-up plus N measured fresh-app runs. Measured A/B order is +counterbalanced. A run is a reproduction only when a renderer/main operation, +heartbeat, or event-loop gap exceeds ${FREEZE_MS}ms, or Electron becomes +unresponsive, loses the renderer, or exits unexpectedly.`) +} + +function parseArgs(argv) { + const out = { + baseline: DEFAULT_BASELINE, + candidate: DEFAULT_CANDIDATE, + repetitions: 5, + output: resolve(process.cwd(), 'short-session-hang-artifacts'), + dryRun: false, + keepWorktrees: false + } + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i] + const next = argv[i + 1] + + if (arg === '--help') { + usage() + process.exit(0) + } else if (arg === '--dry-run') { + out.dryRun = true + } else if (arg === '--keep-worktrees') { + out.keepWorktrees = true + } else if (arg === '--baseline' || arg === '--candidate' || arg === '--repetitions' || arg === '--output') { + if (!next || next.startsWith('--')) { + throw new Error(`${arg} requires a value`) + } + + const key = arg.slice(2) + out[key] = key === 'repetitions' ? Number(next) : key === 'output' ? resolve(next) : next + i += 1 + } else { + throw new Error(`unknown option: ${arg}`) + } + } + + if (!Number.isInteger(out.repetitions) || out.repetitions < 1 || out.repetitions > 20) { + throw new Error('--repetitions must be an integer from 1 to 20') + } + + return out +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd ?? REPO_ROOT, + encoding: 'utf8', + env: options.env ?? process.env, + maxBuffer: 64 * 1024 * 1024 + }) + + if (options.logPath) { + writeFileSync(options.logPath, `${result.stdout ?? ''}${result.stderr ?? ''}`) + } + + if (result.error) { + throw new Error(`${command} ${args.join(' ')} failed to run: ${result.error.message}`) + } + + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(' ')} exited ${result.status ?? `signal ${result.signal}`}:\n${result.stderr || result.stdout}` + ) + } + + return String(result.stdout ?? '').trim() +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex') +} + +function validateRef(ref) { + const headParts = ref.startsWith('refs/heads/') ? ref.slice('refs/heads/'.length).split('/') : [] + const invalidHead = + ref.includes('..') || + headParts.length === 0 || + headParts.some(part => !HEAD_COMPONENT_RE.test(part) || part.endsWith('.lock')) + + if (HEX_OBJECT_RE.test(ref) || PULL_REF_RE.test(ref) || (HEAD_REF_RE.test(ref) && !invalidHead)) { + return ref + } + + throw new Error( + `unsafe ref ${JSON.stringify(ref)}; use a 40/64-digit hex object ID, refs/heads/, or refs/pull//(head|merge)` + ) +} + +function sanitizedEnv(extra = {}) { + const clean = Object.fromEntries(Object.entries(process.env).filter(([name]) => !SECRET_ENV_RE.test(name))) + + return { ...clean, ...extra } +} + +function resolveRef(rawRef) { + const ref = validateRef(rawRef) + const tryResolve = () => { + const result = spawnSync('git', ['rev-parse', '--verify', '--end-of-options', `${ref}^{commit}`], { + cwd: REPO_ROOT, + encoding: 'utf8' + }) + + return result.status === 0 ? result.stdout.trim() : null + } + + const local = HEX_OBJECT_RE.test(ref) ? tryResolve() : null + + if (local) { + return local + } + + run('git', ['fetch', '--no-tags', UPSTREAM_URL, ref], { + env: sanitizedEnv({ GCM_INTERACTIVE: 'never', GIT_TERMINAL_PROMPT: '0' }) + }) + + if (ref.startsWith('refs/')) { + const fetched = run('git', ['rev-parse', '--verify', '--end-of-options', 'FETCH_HEAD^{commit}']) + + return fetched + } + + const resolved = tryResolve() + + if (!resolved) { + throw new Error(`cannot resolve ref ${ref}`) + } + + return resolved +} + +function readTargetMetadata(sha) { + const lock = run('git', ['show', '--end-of-options', `${sha}:package-lock.json`]) + const pyproject = run('git', ['show', '--end-of-options', `${sha}:pyproject.toml`]) + const uvLock = run('git', ['show', '--end-of-options', `${sha}:uv.lock`]) + const pkg = JSON.parse(run('git', ['show', '--end-of-options', `${sha}:apps/desktop/package.json`])) + + return { + electron: pkg.devDependencies?.electron, + electronBuild: pkg.build?.electronVersion, + lockSha256: sha256(lock), + pyprojectSha256: sha256(pyproject), + uvLockSha256: sha256(uvLock) + } +} + +function readJournalContract(sha) { + const source = run('git', ['show', '--end-of-options', `${sha}:apps/desktop/src/lib/inflight-turn-journal.ts`]) + const storageKey = source.match(/const (?:LEGACY_STORAGE_KEY|STORAGE_KEY) = '([^']+)'/)?.[1] ?? null + const migrationKey = source.match(/const LEGACY_MIGRATION_KEY = '([^']+)'/)?.[1] ?? null + const legacyStoreLimit = source.match(/const MAX_LEGACY_STORE_CHARS = (\d+) \* 1024 \* 1024/)?.[1] + + return { + legacyStoreLimit: legacyStoreLimit ? Number(legacyStoreLimit) * 1024 * 1024 : null, + migrationKey, + storageKey + } +} + +function injectHarness(targetRoot) { + const targetHarness = join(targetRoot, 'apps/desktop/src/app/chat/short-session-hang-repro.tsx') + const targetMain = join(targetRoot, 'apps/desktop/src/main.tsx') + const main = readFileSync(targetMain, 'utf8') + + mkdirSync(dirname(targetHarness), { recursive: true }) + copyFileSync(HARNESS_SOURCE, targetHarness) + + if (!main.includes("import('./app/chat/short-session-hang-repro')")) { + writeFileSync( + targetMain, + `${main}\nif (import.meta.env.VITE_SHORT_SESSION_HANG_REPRO === '1') {\n import('./app/chat/short-session-hang-repro')\n}\n` + ) + } +} + +function linkShared(targetRoot, relativePath) { + const source = join(REPO_ROOT, relativePath) + const target = join(targetRoot, relativePath) + + if (existsSync(source) && !existsSync(target)) { + mkdirSync(dirname(target), { recursive: true }) + symlinkSync(source, target, 'dir') + } +} + +function prepareTarget(label, sha, root, output) { + const targetRoot = join(root, label) + run('git', ['worktree', 'add', '--detach', targetRoot, sha]) + + try { + injectHarness(targetRoot) + linkShared(targetRoot, 'node_modules') + linkShared(targetRoot, 'apps/desktop/node_modules') + linkShared(targetRoot, '.venv') + + const targetDesktop = join(targetRoot, 'apps/desktop') + const buildLog = join(output, `${label}-build.log`) + run('npm', ['run', '--prefix', 'apps/desktop', 'build'], { + cwd: targetRoot, + env: sanitizedEnv({ VITE_SHORT_SESSION_HANG_REPRO: '1' }), + logPath: buildLog + }) + + return { journalContract: readJournalContract(sha), label, sha, targetDesktop, targetRoot } + } catch (error) { + try { + run('git', ['worktree', 'remove', '--force', targetRoot]) + } catch { + // The original build error is more useful; cleanup is retried manually. + } + + throw error + } +} + +function startMockInference() { + const reply = `deterministic-tool-heavy-output:${'x'.repeat(STREAM_PAYLOAD_BYTES)}` + let streamingCompletionRequests = 0 + let activeStreamingRequests = 0 + let listening = false + const transportErrors = [] + const requests = [] + const recordTransportError = (source, error) => { + transportErrors.push({ + at: new Date().toISOString(), + message: error instanceof Error ? error.message : String(error), + source + }) + } + const server = http.createServer((request, response) => { + requests.push({ method: request.method, url: request.url ?? null }) + request.on('error', error => recordTransportError('request', error)) + response.on('error', error => recordTransportError('response', error)) + + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: 'short-session-model', object: 'model' }], object: 'list' })) + + return + } + + if (request.method === 'POST' && request.url?.startsWith('/v1/chat/completions')) { + let body = '' + request.on('data', chunk => { + body += String(chunk) + }) + request.on('end', () => { + let stream = false + + try { + stream = JSON.parse(body).stream === true + } catch { + stream = false + } + + if (stream) { + streamingCompletionRequests += 1 + activeStreamingRequests += 1 + let finishedStream = false + const finishStream = () => { + if (finishedStream) return + finishedStream = true + activeStreamingRequests = Math.max(0, activeStreamingRequests - 1) + } + response.once('close', finishStream) + response.writeHead(200, { 'content-type': 'text/event-stream' }) + let offset = 0 + const writeNext = () => { + if (response.destroyed) { + finishStream() + return + } + if (offset >= reply.length) { + response.write( + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: 'stop', index: 0 }], id: 'short-session', object: 'chat.completion.chunk' })}\n\n` + ) + response.end('data: [DONE]\n\n') + finishStream() + return + } + const chunk = reply.slice(offset, offset + 16 * 1024) + offset += chunk.length + response.write( + `data: ${JSON.stringify({ choices: [{ delta: { content: chunk }, finish_reason: null, index: 0 }], id: 'short-session', object: 'chat.completion.chunk' })}\n\n` + ) + setTimeout(writeNext, 10) + } + writeNext() + } else { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [{ finish_reason: 'stop', index: 0, message: { content: reply, role: 'assistant' } }], + id: 'short-session', + object: 'chat.completion' + }) + ) + } + }) + + return + } + + response.writeHead(404, { 'content-type': 'application/json' }) + response.end('{"error":"not found"}') + }) + + return new Promise((resolveStart, reject) => { + server.on('error', error => { + recordTransportError('server', error) + + if (!listening) reject(error) + }) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + + if (!address || typeof address === 'string') { + reject(new Error('mock inference server has no TCP address')) + return + } + + listening = true + resolveStart({ + activeStreamingRequests: () => activeStreamingRequests, + close: () => + new Promise(resolveClose => { + let settled = false + const settle = () => { + if (settled) return + settled = true + resolveClose() + } + const timeout = setTimeout(() => { + server.closeAllConnections?.() + settle() + }, 2_000) + + server.close(() => { + clearTimeout(timeout) + settle() + }) + server.closeAllConnections?.() + }), + requestSummary: () => requests.map(request => ({ ...request })), + streamingCompletionRequests: () => streamingCompletionRequests, + transportErrors: () => transportErrors.map(error => ({ ...error })), + waitForStreaming: async (timeoutMs = 30_000, baseline = 0) => { + const deadline = Date.now() + timeoutMs + while (activeStreamingRequests === 0 && streamingCompletionRequests <= baseline && Date.now() < deadline) { + await sleep(20) + } + if (activeStreamingRequests === 0 && streamingCompletionRequests <= baseline) { + throw new Error('streaming response did not start') + } + }, + url: `http://127.0.0.1:${address.port}` + }) + }) + }) +} +function writeSandboxConfig(home, mockUrl) { + mkdirSync(home, { recursive: true }) + writeFileSync( + join(home, 'config.yaml'), + `model:\n default: short-session-model\n provider: custom:short-session\nauxiliary:\n title_generation:\n enabled: false\nproviders:\n short-session:\n api: ${mockUrl}/v1\n transport: chat_completions\n default_model: short-session-model\n key_env: SHORT_SESSION_API_KEY\n` + ) + writeFileSync(join(home, '.env'), 'SHORT_SESSION_API_KEY=local-diagnostic-only\n') +} + +async function waitFor(cdp, expression, timeoutMs, label, ErrorType = Error) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + try { + if (await cdp.eval(expression)) { + return + } + } catch { + // Renderer is still loading. + } + + await sleep(Math.min(250, Math.max(1, deadline - Date.now()))) + } + + throw new ErrorType(`timed out waiting for ${label}`) +} + +async function screenshot(cdp, path) { + try { + await cdp.send('Page.enable') + const shot = await cdp.send('Page.captureScreenshot', { format: 'png', fromSurface: true }) + writeFileSync(path, Buffer.from(shot.data, 'base64')) + } catch { + // A frozen renderer may not service screenshot capture. + } +} + +function reliablePid(pid) { + return Number.isSafeInteger(pid) && pid > 1 ? pid : null +} + +function processRows() { + const result = spawnSync('ps', ['-axo', 'pid=,ppid=,%cpu=,%mem=,state=,etime=,command='], { encoding: 'utf8' }) + + if (result.error || result.status !== 0) { + const detail = + result.error?.message ?? (String(result.stderr ?? '').trim() || `exit status ${result.status ?? 'unknown'}`) + throw new Error(`process discovery failed: ${detail}`) + } + + return String(result.stdout ?? '') + .split(/\r?\n/) + .map(line => { + const match = line.match(/^\s*(\d+)\s+(\d+)\s+([\d.]+)\s+([\d.]+)\s+(\S+)\s+(\S+)\s+(.*)$/) + + return match + ? { + command: match[7], + cpu: Number(match[3]), + elapsed: match[6], + memory: Number(match[4]), + pid: Number(match[1]), + ppid: Number(match[2]), + state: match[5] + } + : null + }) + .filter(Boolean) +} + +function processTree(rootPid) { + if (!reliablePid(rootPid)) { + return [] + } + + let rows + let discoveryError = null + + try { + rows = processRows() + } catch (error) { + rows = [] + discoveryError = error instanceof Error ? error.message : String(error) + } + + const selected = new Set([rootPid]) + let changed = true + + while (changed) { + changed = false + + for (const row of rows) { + if (!selected.has(row.pid) && selected.has(row.ppid)) { + selected.add(row.pid) + changed = true + } + } + } + + const tree = rows.filter(row => selected.has(row.pid)) + const rootMissing = !tree.some(row => row.pid === rootPid) + + if (rootMissing) { + tree.unshift({ + command: '', + cpu: 0, + elapsed: '', + memory: 0, + pid: rootPid, + ppid: 0, + state: '?', + synthetic: true + }) + } + + if (discoveryError) { + tree.discoveryError = discoveryError + } + + if (rootMissing) { + tree.rootMissing = true + } + + return tree +} + +function redactCommand(command) { + return command + .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+(?::[^\s/@]*)?@/gi, '$1[REDACTED]@') + .replace(/\b([A-Za-z_][A-Za-z0-9_]*(?:TOKEN|SECRET|PASSWORD|KEY|AUTH|COOKIE)[A-Za-z0-9_]*)=\S+/gi, '$1=[REDACTED]') + .replace(/(--?\S*(?:token|secret|password|key|auth|cookie)\S*)(?:=|\s+)\S+/gi, '$1=[REDACTED]') +} + +function processSnapshot(path, rootPid) { + const tree = processTree(rootPid) + const rows = tree.map(row => ({ ...row, command: redactCommand(row.command) })) + writeFileSync( + path, + `${JSON.stringify({ rootPid, rows, discoveryError: tree.discoveryError ?? null, rootMissing: tree.rootMissing ?? false }, null, 2)}\n` + ) + + return rows +} + +function sampleOne(path, pid) { + if (process.platform !== 'darwin' || !reliablePid(pid)) { + return + } + + const result = spawnSync('sample', [String(pid), '5', '1'], { encoding: 'utf8', timeout: 8_000 }) + writeFileSync(path, `${result.stdout ?? ''}${result.stderr ?? ''}`) +} + +function sampleTree(runDir, rootPid, prefix) { + const rows = processTree(rootPid) + sampleOne(join(runDir, `${prefix}-main.sample.txt`), rootPid) + const renderer = rows + .filter(row => row.pid !== rootPid && /(?:^|\s)--type=renderer(?:\s|$)/.test(row.command)) + .sort((a, b) => b.cpu - a.cpu)[0] + + if (renderer) { + sampleOne(join(runDir, `${prefix}-renderer-${renderer.pid}.sample.txt`), renderer.pid) + } +} + +function captureDiagnostics(runDir, rootPid, prefix) { + try { + processSnapshot(join(runDir, `${prefix}-processes.txt`), rootPid) + } catch { + // Diagnostics are best effort and must not replace the original result. + } + + try { + sampleTree(runDir, rootPid, prefix) + } catch { + // Diagnostics are best effort and must not replace the original result. + } +} + +function liveCaptured(captured) { + let current + + try { + current = new Map(processRows().map(row => [row.pid, row])) + } catch (error) { + const live = captured.filter(original => pidIsLive(original.pid)) + live.discoveryError = error instanceof Error ? error.message : String(error) + return live + } + + return captured.filter(original => { + if (original.synthetic) { + return pidIsLive(original.pid) + } + + const row = current.get(original.pid) + + return row && !row.state.startsWith('Z') && row.command === original.command + }) +} + +function pidIsLive(pid) { + try { + process.kill(pid, 0) + return true + } catch (error) { + return error?.code === 'EPERM' + } +} + +async function stopProcessTree(rootPid) { + const captured = processTree(rootPid) + const processDiscoveryErrors = new Set() + + if (captured.discoveryError) { + processDiscoveryErrors.add(captured.discoveryError) + } + + const currentLive = () => { + const live = liveCaptured(captured) + + if (live.discoveryError) { + processDiscoveryErrors.add(live.discoveryError) + } + + return live + } + + for (const { pid } of [...captured].reverse()) { + try { + process.kill(pid, 'SIGTERM') + } catch { + // It already exited. + } + } + + const deadline = Date.now() + 3_000 + + while (Date.now() < deadline && currentLive().length > 0) { + await sleep(100) + } + + const remaining = currentLive() + + for (const { pid } of remaining.reverse()) { + try { + process.kill(pid, 'SIGKILL') + } catch { + // It exited between the liveness check and signal. + } + } + + const killDeadline = Date.now() + 2_000 + + while (Date.now() < killDeadline && currentLive().length > 0) { + await sleep(100) + } + + return { + captured: captured.map(row => row.pid), + processDiscoveryErrors: [...processDiscoveryErrors], + remainingAfterKill: currentLive().map(row => row.pid) + } +} + +function withWatchdog(task, onTimeout) { + let timer + + return Promise.race([ + task, + new Promise((_, reject) => { + timer = setTimeout(() => { + onTimeout() + reject(new ReproductionError(`outer watchdog exceeded ${OUTER_WATCHDOG_MS}ms`)) + }, OUTER_WATCHDOG_MS) + }) + ]).finally(() => clearTimeout(timer)) +} + +function withTimeout(task, timeoutMs, label, ErrorType = Error) { + let timer + + return Promise.race([ + task, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new ErrorType(`${label} exceeded ${timeoutMs}ms`)), timeoutMs) + }) + ]).finally(() => clearTimeout(timer)) +} + +async function waitForResponsive(cdp, expression, timeoutMs, label, evaluationTimeoutMs = FREEZE_MS) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + try { + if ( + await withTimeout(cdp.eval(expression), evaluationTimeoutMs, `${label} renderer evaluation`, ReproductionError) + ) { + return + } + } catch (error) { + if (error instanceof ReproductionError) { + throw error + } + + // Match the initial renderer-readiness polling: CDP can fail transiently while a document is replaced. + } + + await sleep(Math.min(250, Math.max(1, deadline - Date.now()))) + } + + throw new Error(`timed out waiting for ${label} while the renderer remained responsive`) +} + +async function waitForPredicate(predicate, timeoutMs, label) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + if (await predicate()) { + return + } + + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))) + } + + throw new Error(`timed out waiting for ${label}`) +} + +async function verifyInteractiveSurfaces(cdp, timed, measure, label) { + const sentinel = `short-session-sentinel-${label}` + const composerPainted = await timed(`composer.paint.${label}`, async () => { + const focused = await cdp.eval( + `(() => { const el = document.querySelector('[data-slot="composer-rich-input"]'); if (!el || el.contentEditable !== 'true') return false; el.focus(); return true })()` + ) + + if (!focused) { + return false + } + + await cdp.send('Input.insertText', { text: sentinel }) + + return cdp.eval( + `document.querySelector('[data-slot="composer-rich-input"]')?.textContent?.includes(${JSON.stringify(sentinel)}) === true` + ) + }) + + if (!composerPainted) { + throw new Error(`composer did not paint sentinel at ${label}`) + } + + const version = await timed(`version.ipc.${label}`, () => cdp.eval('window.hermesDesktop.getVersion()')) + + if (typeof version?.appVersion !== 'string' || version.appVersion.length === 0) { + throw new Error(`version IPC returned no appVersion at ${label}: ${JSON.stringify(version)}`) + } + + await timed(`about.open.${label}`, () => cdp.eval("location.hash = '#/settings?tab=about'; true")) + await measure(`about.ready.${label}`, () => + waitForResponsive( + cdp, + `document.body.textContent.includes(${JSON.stringify(version.appVersion)}) && !!document.querySelector('button[aria-label]')`, + FREEZE_MS, + `About settings at ${label}` + ) + ) + const aboutClosed = await timed(`about.close.${label}`, () => + cdp.eval( + `(() => { const close = document.querySelector('div[role="presentation"] > div > div:first-child button[aria-label]'); if (!close) return false; close.click(); return true })()` + ) + ) + + if (!aboutClosed) { + throw new Error(`About settings close control unavailable at ${label}`) + } + + await measure(`about.closed.${label}`, () => + waitForResponsive(cdp, "!location.hash.includes('/settings')", FREEZE_MS, `About settings close at ${label}`) + ) + const interactive = await timed(`transcript.interactive.${label}`, () => + cdp.eval( + `(() => { const viewport = document.querySelector('[data-slot="aui_thread-viewport"]'); const row = document.querySelector('[data-message-id]'); if (!viewport || !row) return false; const before = viewport.scrollTop; viewport.scrollTop = Math.min(viewport.scrollHeight, before + 40); viewport.dispatchEvent(new Event('scroll', { bubbles: true })); return getComputedStyle(row).pointerEvents !== 'none' })()` + ) + ) + + if (!interactive) { + throw new Error(`transcript was not interactive at ${label}`) + } + + return { sentinel, version } +} + +function nativeWindowVisibilitySource() { + return [ + 'import AppKit', + 'import CoreGraphics', + 'import Foundation', + 'import Darwin', + 'guard CommandLine.arguments.count >= 3,', + ' let processId = Int32(CommandLine.arguments[1]),', + ' let application = NSRunningApplication(processIdentifier: pid_t(processId)) else {', + ' fputs("target application was not found", stderr)', + ' exit(2)', + '}', + 'let process = pid_t(processId)', + 'let visible = CommandLine.arguments[2] == "visible"', + 'var changed: Bool', + 'if visible {', + ' changed = application.unhide() || application.activate(options: [.activateIgnoringOtherApps, .activateAllWindows])', + '} else {', + ' let _ = application.activate(options: [.activateIgnoringOtherApps, .activateAllWindows])', + ' usleep(100_000)', + ' changed = application.hide()', + ' if !changed, let source = CGEventSource(stateID: .hidSystemState),', + ' let down = CGEvent(keyboardEventSource: source, virtualKey: 4, keyDown: true),', + ' let up = CGEvent(keyboardEventSource: source, virtualKey: 4, keyDown: false) {', + ' down.flags = .maskCommand', + ' up.flags = .maskCommand', + ' down.postToPid(process)', + ' up.postToPid(process)', + ' changed = true', + ' }', + '}', + 'if !changed {', + ' fputs("native visibility request was rejected", stderr)', + ' exit(3)', + '}' + ].join(String.fromCharCode(10)) +} + +function compileNativeWindowVisibilityHelper(helperPath) { + const sourcePath = `${helperPath}.swift` + writeFileSync(sourcePath, nativeWindowVisibilitySource()) + + try { + const result = spawnSync('/usr/bin/swiftc', [sourcePath, '-o', helperPath], { + encoding: 'utf8', + timeout: 30_000 + }) + + if (result.status !== 0) { + const detail = result.error?.message || result.stderr || result.stdout || String(result.status) + throw new Error(`native macOS visibility helper compilation failed: ${detail}`) + } + } finally { + rmSync(sourcePath, { force: true }) + } + + return helperPath +} + +function setNativeWindowVisibility(helperPath, pid, visible) { + if (process.platform !== 'darwin' || !helperPath || !Number.isSafeInteger(pid) || pid <= 1) { + throw new Error(`native macOS visibility requires a helper and valid app pid, got ${pid}`) + } + + const result = spawnSync(helperPath, [String(pid), visible ? 'visible' : 'hidden'], { + encoding: 'utf8', + timeout: 10_000 + }) + + if (result.status !== 0) { + const detail = result.error?.message || result.stderr || result.stdout || String(result.status) + throw new Error(`native macOS visibility change failed: ${detail}`) + } +} + +async function runRealChatChecks(cdp, timed, measure, mock, runDir, appPid, nativeVisibilityHelper, getOperationCount) { + const requestCountBefore = mock.streamingCompletionRequests() + + for (let exchange = 1; exchange <= EXCHANGE_COUNT; exchange += 1) { + const streamingRequestsBefore = mock.streamingCompletionRequests() + const beforeAssistant = await timed(`real-chat.assistant-count.${exchange}`, () => + cdp.eval( + `document.querySelectorAll('[data-slot="aui_assistant-message-root"]:not([data-streaming="true"])').length` + ) + ) + const composer = await timed(`real-chat.composer-focus.${exchange}`, () => + cdp.eval( + `(() => { const el = document.querySelector('[data-slot="composer-rich-input"]'); if (!el || el.contentEditable !== 'true') return false; el.focus(); return true })()` + ) + ) + + if (!composer) { + throw new Error(`real chat composer unavailable at exchange ${exchange}`) + } + + const prompt = `Deterministic real chat exchange ${exchange}` + await timed(`real-chat.insert.${exchange}`, () => cdp.send('Input.insertText', { text: prompt })) + const inserted = await timed(`real-chat.inserted.${exchange}`, () => + cdp.eval( + `document.querySelector('[data-slot="composer-rich-input"]')?.textContent?.includes(${JSON.stringify(prompt)}) === true` + ) + ) + + if (!inserted) { + throw new Error(`real chat prompt did not reach the composer at exchange ${exchange}`) + } + + await measure(`real-chat.submit-ready.${exchange}`, () => + waitForResponsive( + cdp, + `!!document.querySelector('[data-slot="composer-root"] button[type="submit"]:not(:disabled)')`, + FREEZE_MS, + `real chat submit control ${exchange}` + ) + ) + const submitted = await timed(`real-chat.submit.${exchange}`, () => + cdp.eval( + `(() => { const button = document.querySelector('[data-slot="composer-root"] button[type="submit"]:not(:disabled)'); if (!button) return false; window.setTimeout(() => button.click(), 0); return true })()` + ) + ) + + if (!submitted) { + throw new Error(`real chat submit control unavailable at exchange ${exchange}`) + } + + try { + await measure(`real-chat.stream-start.${exchange}`, () => mock.waitForStreaming(30_000, streamingRequestsBefore)) + } catch (error) { + const state = await timed(`real-chat.dispatch-probe.${exchange}`, () => + cdp.eval(`({ + assistantMessages: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length, + assistantText: document.querySelector('[data-slot="aui_assistant-message-root"]')?.textContent ?? null, + composerText: document.querySelector('[data-slot="composer-rich-input"]')?.textContent ?? null, + harness: window.__SHORT_SESSION_HANG_REPRO__.summary(), + submitDisabled: document.querySelector('[data-slot="composer-root"] button[type="submit"]')?.disabled ?? null, + userMessages: document.querySelectorAll('[data-slot="aui_user-message-root"]').length + })`) + ) + + throw new Error( + `${error instanceof Error ? error.message : String(error)}; mock=${JSON.stringify(mock.requestSummary())}; dispatch probe responded with state: ${JSON.stringify(state)}` + ) + } + await measure(`real-chat.native-hide.${exchange}`, () => + setNativeWindowVisibility(nativeVisibilityHelper, appPid, false) + ) + await sleep(NATIVE_VISIBILITY_PAUSE_MS) + await measure(`real-chat.native-restore.${exchange}`, () => + setNativeWindowVisibility(nativeVisibilityHelper, appPid, true) + ) + + try { + await measure(`real-chat.mock-request.${exchange}`, () => + waitForPredicate( + () => mock.streamingCompletionRequests() >= requestCountBefore + exchange, + FREEZE_MS, + `mock inference request ${exchange}` + ) + ) + } catch (error) { + const state = await timed(`real-chat.dispatch-probe.${exchange}`, () => + cdp.eval(`({ + assistantMessages: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length, + composerText: document.querySelector('[data-slot="composer-rich-input"]')?.textContent ?? null, + harness: window.__SHORT_SESSION_HANG_REPRO__.summary(), + userMessages: document.querySelectorAll('[data-slot="aui_user-message-root"]').length + })`) + ) + + throw new Error( + `${error instanceof Error ? error.message : String(error)}; dispatch probe responded with state: ${JSON.stringify(state)}` + ) + } + + await measure(`real-chat.assistant-response.${exchange}`, () => + waitForResponsive( + cdp, + `document.querySelectorAll('[data-slot="aui_assistant-message-root"]:not([data-streaming="true"])').length > ${beforeAssistant}`, + STREAM_RESPONSE_TIMEOUT_MS, + `real assistant response ${exchange}`, + STREAM_RESPONSE_EVALUATION_TIMEOUT_MS + ) + ) + + if (mock.streamingCompletionRequests() < requestCountBefore + exchange) { + throw new Error(`mock inference request count did not advance for exchange ${exchange}`) + } + } + + const requestDelta = mock.streamingCompletionRequests() - requestCountBefore + + if (requestDelta !== EXCHANGE_COUNT) { + throw new Error(`expected exactly ${EXCHANGE_COUNT} mock completion requests, observed ${requestDelta}`) + } + + const streamHeartbeat = await timed('renderer.heartbeat.stream', () => + cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.summary()') + ) + await timed('renderer.heartbeat.checkpoint', () => cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.checkpoint()')) + const postStreamOperationStart = getOperationCount() + const surfaces = await verifyInteractiveSurfaces(cdp, timed, measure, 'real-chat-exchange-5') + await withTimeout(screenshot(cdp, join(runDir, 'real-chat-exchange-5.png')), 2_000, 'real chat screenshot').catch( + () => {} + ) + + return { + assistantResponses: EXCHANGE_COUNT, + exchanges: EXCHANGE_COUNT, + messageRecords: EXCHANGE_COUNT * 2, + mockCompletionRequests: requestDelta, + postStreamOperationStart, + surfaces, + streamMaxGapMs: Number(streamHeartbeat.maxGapMs || 0) + } +} + +async function runRendererChecks(cdp, label, runDir, mock, appPid, journalContract) { + const operations = [] + const measure = async (name, body) => { + const started = performance.now() + const value = await Promise.resolve().then(body) + const latencyMs = performance.now() - started + operations.push({ name, latencyMs }) + + return value + } + const timed = (name, body) => + measure(name, () => withTimeout(Promise.resolve().then(body), FREEZE_MS, name, ReproductionError)) + + await cdp.send('Runtime.enable') + await cdp.send('Profiler.enable') + await cdp.send('Profiler.start') + + if (journalContract.storageKey !== LEGACY_STORAGE_KEY) { + throw new Error( + `journal storage-key contract mismatch: target=${journalContract.storageKey ?? 'missing'} harness=${LEGACY_STORAGE_KEY}` + ) + } + + if (journalContract.migrationKey !== null && journalContract.migrationKey !== LEGACY_MIGRATION_KEY) { + throw new Error( + `journal migration-key contract mismatch: target=${journalContract.migrationKey} harness=${LEGACY_MIGRATION_KEY}` + ) + } + + const checkpoints = [] + const fixtureManifest = null + const nativeVisibilityHelper = compileNativeWindowVisibilityHelper(join(runDir, 'native-window-visibility')) + const journalSeed = await measure('journal.seed.legacy', () => + withTimeout( + cdp.eval(`(() => { + const payload = 'j'.repeat(${LEGACY_SEED_PAYLOAD_CHARS}) + const entries = Object.fromEntries(Array.from({ length: ${LEGACY_SEED_ENTRY_COUNT} }, (_, index) => { + const assistantId = 'legacy-seed-a-' + index + return [ + 'legacy-seed-' + index, + { + messages: [ + { id: 'legacy-seed-u-' + index, role: 'user', parts: [{ type: 'text', text: 'legacy seed prompt ' + index }] }, + { id: assistantId, role: 'assistant', parts: [{ type: 'text', text: payload }], pending: true } + ], + streamId: assistantId, + turnStartedAt: Date.now(), + updatedAt: Date.now() + } + ] + })) + const raw = JSON.stringify({ entries, version: 1 }) + localStorage.removeItem(${JSON.stringify(LEGACY_MIGRATION_KEY)}) + localStorage.setItem(${JSON.stringify(LEGACY_STORAGE_KEY)}, raw) + return { bytes: raw.length, entries: Object.keys(entries).length } + })()`), + JOURNAL_SEED_TIMEOUT_MS, + 'journal.seed.legacy' + ) + ) + + if (journalSeed.bytes >= MAX_LEGACY_SEED_BYTES) { + throw new Error(`legacy journal seed exceeded migration limit: ${journalSeed.bytes} bytes`) + } + + if (journalContract.legacyStoreLimit !== null && journalSeed.bytes >= journalContract.legacyStoreLimit) { + throw new Error( + `legacy journal seed exceeds target migration limit: seed=${journalSeed.bytes} target=${journalContract.legacyStoreLimit}` + ) + } + + await timed('harness.reset', () => cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.reset()')) + const realChat = await runRealChatChecks( + cdp, + timed, + measure, + mock, + runDir, + appPid, + nativeVisibilityHelper, + () => operations.length + ) + + const heartbeat = await timed('renderer.heartbeat.summary', () => + cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.summary()') + ) + const profile = await cdp.send('Profiler.stop') + writeFileSync(join(runDir, 'renderer.cpuprofile'), JSON.stringify(profile.profile)) + + const maxOperationMs = operations + .slice(realChat.postStreamOperationStart) + .reduce((max, operation) => Math.max(max, operation.latencyMs), 0) + const maxGapMs = Number(heartbeat.maxGapMs || 0) + const reproduced = maxOperationMs > FREEZE_MS || maxGapMs > FREEZE_MS + + return { + checkpoints, + fixtureManifest, + hardFailure: reproduced, + maxGapMs, + maxOperationMs, + operations, + outcome: reproduced ? 'reproduced' : 'not-reproduced', + journalSeed, + realChat, + syntheticScenario: 'disabled; real-chat-only' + } +} + +async function withTemporarySandbox(label, body) { + const sandbox = mkdtempSync(join(tmpdir(), `hermes-short-session-${label}-`)) + + try { + return await body(sandbox) + } finally { + rmSync(sandbox, { force: true, recursive: true }) + } +} + +async function allocateDebugPort() { + const server = createServer() + + try { + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + + const address = server.address() + + if (!address || typeof address === 'string') { + throw new Error('temporary debug-port allocation returned no numeric address') + } + + return address.port + } finally { + if (server.listening) { + await new Promise(resolve => server.close(() => resolve())) + } + } +} + +function resultForError(error) { + const reproduced = error instanceof ReproductionError + + return { + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + hardFailure: true, + lifecycleSignals: [], + maxGapMs: null, + maxOperationMs: null, + operations: [], + outcome: reproduced ? 'reproduced' : 'harness-error' + } +} + +async function executeRun(target, index, warmup, mock, output) { + return withTemporarySandbox(target.label, sandbox => + executeRunInSandbox(target, index, warmup, mock, output, sandbox) + ) +} + +function runDirForAttempt(output, label, index, warmup, attempt = 1) { + const base = join(output, label, warmup ? 'warmup' : `run-${index + 1}`) + + return attempt > 1 ? `${base}-attempt-${attempt}` : base +} + +function promoteAttemptArtifacts(output, label, index, warmup, attempt) { + if (attempt <= 1) { + return + } + + cpSync(runDirForAttempt(output, label, index, warmup, attempt), runDirForAttempt(output, label, index, warmup), { + force: true, + recursive: true + }) +} + +async function executeRunInSandboxAttempt(target, index, warmup, mock, output, sandbox, attempt) { + const runDir = runDirForAttempt(output, target.label, index, warmup, attempt) + const hermesHome = join(sandbox, 'hermes-home') + const userData = join(sandbox, 'electron-user-data') + const desktopLog = join(hermesHome, 'logs', 'desktop.log') + const stdoutPath = join(runDir, 'electron.stdout.log') + const stderrPath = join(runDir, 'electron.stderr.log') + const eventsPath = join(runDir, 'events.jsonl') + const port = await allocateDebugPort() + + mkdirSync(runDir, { recursive: true }) + mkdirSync(userData, { recursive: true }) + writeSandboxConfig(hermesHome, mock.url) + + const electron = require('electron') + const stdoutLog = createWriteStream(stdoutPath) + const stderrLog = createWriteStream(stderrPath) + const stdoutFlushed = finished(stdoutLog).then( + () => null, + error => error + ) + const stderrFlushed = finished(stderrLog).then( + () => null, + error => error + ) + const child = spawn( + electron, + [target.targetDesktop, `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`], + { + cwd: target.targetDesktop, + env: sanitizedEnv({ + HERMES_DESKTOP_APP_NAME: `HermesShortSession-${target.label}-${process.pid}-${index}-${warmup ? 'w' : 'm'}-${attempt}`, + HERMES_DESKTOP_HERMES_ROOT: target.targetRoot, + HERMES_DESKTOP_IGNORE_EXISTING: '1', + HERMES_DESKTOP_USER_DATA_DIR: userData, + HERMES_HOME: hermesHome, + SHORT_SESSION_API_KEY: 'local-diagnostic-only' + }), + stdio: ['ignore', 'pipe', 'pipe'] + } + ) + + let exited = null + let spawnFailed = false + let stopping = false + let resolveSpawn + let rejectSpawn + const spawnReady = new Promise((resolve, reject) => { + resolveSpawn = resolve + rejectSpawn = reject + }) + child.once('spawn', () => resolveSpawn()) + child.once('error', error => { + spawnFailed = true + exited = exited ?? { code: null, signal: null } + appendFileSync( + eventsPath, + `${JSON.stringify({ at: new Date().toISOString(), error: error.message, type: 'spawn-error' })}\n` + ) + rejectSpawn(error) + }) + if (child.stdout) { + child.stdout.pipe(stdoutLog) + } else { + stdoutLog.end() + } + + if (child.stderr) { + child.stderr.pipe(stderrLog) + } else { + stderrLog.end() + } + child.once('exit', (code, signal) => { + exited = { code, signal } + appendFileSync( + eventsPath, + `${JSON.stringify({ at: new Date().toISOString(), code, signal, type: stopping ? 'teardown-exit' : 'unexpected-exit' })}\n` + ) + }) + + let cdp + let result + + try { + await withTimeout(spawnReady, FREEZE_MS, 'Electron spawn') + const targetInfo = await withTimeout(discoverTarget({ port, timeoutMs: 60_000 }), 65_000, 'CDP target discovery') + cdp = await withTimeout(CDP.open(targetInfo.webSocketDebuggerUrl), 10_000, 'CDP connection') + await waitFor(cdp, '!!window.__SHORT_SESSION_HANG_REPRO__', 60_000, 'short-session renderer harness') + await waitFor( + cdp, + `document.querySelector('[data-slot="composer-rich-input"]')?.contentEditable === 'true'`, + 60_000, + 'interactive composer' + ) + + result = await withWatchdog( + runRendererChecks(cdp, `${target.label}-${index + 1}`, runDir, mock, child.pid, target.journalContract), + () => { + captureDiagnostics(runDir, child.pid, 'hard-timeout') + } + ) + } catch (error) { + result = resultForError(error) + captureDiagnostics(runDir, child.pid, 'failure') + if (cdp) { + await withTimeout(screenshot(cdp, join(runDir, 'failure.png')), 2_000, 'failure screenshot').catch(() => {}) + } + } finally { + cdp?.close() + const logText = existsSync(desktopLog) ? readFileSync(desktopLog, 'utf8') : '' + const lifecycleSignals = logText + .split(/\r?\n/) + .filter(line => /webContents became unresponsive|render-process-gone/i.test(line)) + const lifecycleReproduced = lifecycleSignals.length > 0 || Boolean(exited && !spawnFailed) + + result = { ...(result ?? resultForError(new Error('run produced no result'))), lifecycleSignals } + + if (lifecycleReproduced) { + result.hardFailure = true + result.outcome = 'reproduced' + } + + stopping = true + const teardown = await stopProcessTree(child.pid) + const logFlushErrors = await withTimeout( + Promise.all([stdoutFlushed, stderrFlushed]), + FREEZE_MS, + 'Electron log flush' + ).catch(error => [error]) + writeFileSync(join(runDir, 'teardown.json'), `${JSON.stringify(teardown, null, 2)}\n`) + + if (existsSync(desktopLog)) { + copyFileSync(desktopLog, join(runDir, 'desktop.log')) + } + + if (teardown.remainingAfterKill.length > 0) { + result = { + ...(result ?? {}), + error: `${result?.error ? `${result.error}\n` : ''}process teardown leaked PIDs ${teardown.remainingAfterKill.join(', ')}`, + hardFailure: true, + outcome: result?.outcome === 'reproduced' ? 'reproduced' : 'harness-error' + } + } + + if (teardown.processDiscoveryErrors.length > 0) { + result = { + ...(result ?? {}), + error: `${result?.error ? `${result.error}\n` : ''}${teardown.processDiscoveryErrors.join('\n')}`, + hardFailure: true, + outcome: result?.outcome === 'reproduced' ? 'reproduced' : 'harness-error' + } + } + + const logFlushError = logFlushErrors.find(Boolean) + + if (logFlushError) { + result = { + ...(result ?? {}), + error: `${result?.error ? `${result.error}\n` : ''}Electron log flush failed: ${logFlushError instanceof Error ? logFlushError.message : String(logFlushError)}`, + hardFailure: true, + outcome: result?.outcome === 'reproduced' ? 'reproduced' : 'harness-error' + } + } + } + + writeFileSync(join(runDir, 'result.json'), `${JSON.stringify(result, null, 2)}\n`) + appendFileSync(eventsPath, `${JSON.stringify({ at: new Date().toISOString(), result, type: 'run-complete' })}\n`) + + return result +} + +function isDebugStartupFailure(result) { + return !result.lifecycleSignals?.length && /CDP (?:target discovery|connection)/i.test(result.error ?? '') +} + +async function executeRunInSandbox(target, index, warmup, mock, output, sandbox) { + for (let attempt = 1; attempt <= 2; attempt += 1) { + const result = await executeRunInSandboxAttempt(target, index, warmup, mock, output, sandbox, attempt) + + if (attempt === 1 && isDebugStartupFailure(result)) { + rmSync(join(sandbox, 'electron-user-data'), { force: true, recursive: true }) + continue + } + + promoteAttemptArtifacts(output, target.label, index, warmup, attempt) + + return result + } + + throw new Error('unreachable debug startup retry state') +} + +function classify(results, warmup) { + const invalid = [warmup, ...results].some(result => result.outcome === 'harness-error') + const reproduced = results.filter(result => result.outcome === 'reproduced').length + const reproducedThreshold = Math.ceil(results.length * 0.8) + + return { + classification: invalid + ? 'invalid' + : reproduced >= reproducedThreshold + ? 'reproduced' + : reproduced === 0 + ? 'not-reproduced' + : 'intermittent', + invalid, + reproduced, + reproducedThreshold, + total: results.length + } +} + +function pairedSoftSignal(baseline, candidate) { + if (baseline.length === 0 || candidate.length === 0) { + return { material: false, materialPairs: 0, materialThreshold: 0, reason: 'insufficient-runs', thresholdPct: 30 } + } + + if ([...baseline, ...candidate].some(result => result.outcome !== 'not-reproduced')) { + return { material: false, materialPairs: 0, materialThreshold: 0, reason: 'hard-or-invalid-run', thresholdPct: 30 } + } + + let materialPairs = 0 + + for (let i = 0; i < Math.min(baseline.length, candidate.length); i += 1) { + const base = Math.max(1, baseline[i].maxGapMs || 0, baseline[i].maxOperationMs || 0) + const next = Math.max(candidate[i].maxGapMs || 0, candidate[i].maxOperationMs || 0) + + if (next >= base * 1.3) { + materialPairs += 1 + } + } + + const materialThreshold = Math.ceil(Math.min(baseline.length, candidate.length) * 0.8) + + return { material: materialPairs >= materialThreshold, materialPairs, materialThreshold, thresholdPct: 30 } +} + +function validateArtifactBundle(output, repetitions) { + const required = ['environment.json', 'summary.json', 'baseline-build.log', 'candidate-build.log'] + + for (const label of ['baseline', 'candidate']) { + for (const runName of ['warmup', ...Array.from({ length: repetitions }, (_, index) => `run-${index + 1}`)]) { + for (const artifact of ['events.jsonl', 'result.json', 'teardown.json']) { + required.push(join(label, runName, artifact)) + } + } + } + + for (const relativePath of required) { + const path = join(output, relativePath) + + if (!existsSync(path) || readFileSync(path).byteLength === 0) { + throw new Error(`missing or empty required diagnostic artifact: ${relativePath}`) + } + } + + const summary = JSON.parse(readFileSync(join(output, 'summary.json'), 'utf8')) + + validateSummary(summary, repetitions) +} + +function validateSummary(summary, repetitions) { + if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 20) { + throw new Error('invalid summary repetitions') + } + + let invalid = false + + for (const label of ['baseline', 'candidate']) { + if (!summary[label]?.warmup || summary[label].runs?.length !== repetitions) { + throw new Error(`invalid ${label} summary shape`) + } + + for (const result of [summary[label].warmup, ...summary[label].runs]) { + if (!['harness-error', 'not-reproduced', 'reproduced'].includes(result.outcome)) { + throw new Error(`invalid ${label} run outcome`) + } + + const expectedHardFailure = result.outcome !== 'not-reproduced' + + if (result.hardFailure !== expectedHardFailure) { + throw new Error(`inconsistent ${label} run outcome and hardFailure`) + } + } + + const derived = classify(summary[label].runs, summary[label].warmup) + + for (const field of ['classification', 'invalid', 'reproduced', 'reproducedThreshold', 'total']) { + if (summary[label][field] !== derived[field]) { + throw new Error(`inconsistent ${label} summary ${field}`) + } + } + + invalid ||= derived.invalid + } + + if (summary.invalid !== invalid) { + throw new Error('inconsistent summary invalid state') + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)) + + if (!existsSync(HARNESS_SOURCE)) { + throw new Error(`renderer harness missing: ${HARNESS_SOURCE}`) + } + + validateRef(options.baseline) + validateRef(options.candidate) + const baselineSha = resolveRef(options.baseline) + const candidateSha = resolveRef(options.candidate) + const mergeBase = run('git', ['merge-base', '--', baselineSha, candidateSha]) + const ancestor = spawnSync('git', ['merge-base', '--is-ancestor', '--', baselineSha, candidateSha], { + cwd: REPO_ROOT, + encoding: 'utf8' + }) + + if (mergeBase !== baselineSha || ancestor.status !== 0) { + throw new Error( + `baseline must be the exact merge-base and an ancestor of candidate; baseline=${baselineSha} merge-base=${mergeBase} candidate=${candidateSha}` + ) + } + + const baselineMetadata = readTargetMetadata(baselineSha) + const candidateMetadata = readTargetMetadata(candidateSha) + const harnessHead = run('git', ['rev-parse', '--verify', '--end-of-options', 'HEAD^{commit}']) + const harnessMetadata = readTargetMetadata(harnessHead) + + if ( + JSON.stringify(baselineMetadata) !== JSON.stringify(candidateMetadata) || + JSON.stringify(baselineMetadata) !== JSON.stringify(harnessMetadata) + ) { + throw new Error( + `shared dependency mismatch; harness checkout, baseline, and candidate Electron/lockfile metadata must match:\nharness ${JSON.stringify(harnessMetadata)}\nbaseline ${JSON.stringify(baselineMetadata)}\ncandidate ${JSON.stringify(candidateMetadata)}` + ) + } + + const plan = { + baseline: { ref: options.baseline, sha: baselineSha }, + candidate: { ref: options.candidate, sha: candidateSha }, + harness: { head: harnessHead, metadata: harnessMetadata }, + harnessSha256: sha256(readFileSync(HARNESS_SOURCE)), + mergeBase, + metadata: baselineMetadata, + repetitions: options.repetitions, + runner: { arch: process.arch, platform: process.platform, versions: process.versions } + } + + if (options.dryRun) { + console.log(JSON.stringify(plan, null, 2)) + + return + } + + if (process.platform !== 'darwin' || process.arch !== 'arm64') { + throw new Error( + `the short-session hang diagnostic requires macOS arm64; got ${process.platform}-${process.arch} (use --dry-run for preflight)` + ) + } + + mkdirSync(options.output, { recursive: true }) + writeFileSync(join(options.output, 'environment.json'), `${JSON.stringify(plan, null, 2)}\n`) + + const ephemeralRoot = mkdtempSync(join(tmpdir(), 'hermes-short-session-ab-')) + const prepared = [] + const mock = await startMockInference() + + try { + prepared.push(prepareTarget('baseline', baselineSha, ephemeralRoot, options.output)) + prepared.push(prepareTarget('candidate', candidateSha, ephemeralRoot, options.output)) + const byLabel = Object.fromEntries(prepared.map(target => [target.label, target])) + + const warmups = {} + + for (const label of ['baseline', 'candidate']) { + warmups[label] = await executeRun(byLabel[label], 0, true, mock, options.output) + } + + const measured = { baseline: [], candidate: [] } + + for (let index = 0; index < options.repetitions; index += 1) { + const order = index % 2 === 0 ? ['baseline', 'candidate'] : ['candidate', 'baseline'] + + for (const label of order) { + measured[label].push(await executeRun(byLabel[label], index, false, mock, options.output)) + } + } + + const baselineClassification = classify(measured.baseline, warmups.baseline) + const candidateClassification = classify(measured.candidate, warmups.candidate) + const invalid = baselineClassification.invalid || candidateClassification.invalid + const summary = { + ...plan, + invalid, + baseline: { ...plan.baseline, ...baselineClassification, runs: measured.baseline, warmup: warmups.baseline }, + candidate: { + ...plan.candidate, + ...candidateClassification, + runs: measured.candidate, + warmup: warmups.candidate + }, + softSignal: invalid + ? { material: false, materialPairs: 0, materialThreshold: 0, reason: 'invalid-run', thresholdPct: 30 } + : pairedSoftSignal(measured.baseline, measured.candidate), + mockRequests: mock.requestSummary(), + mockTransportErrors: mock.transportErrors() + } + writeFileSync(join(options.output, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`) + validateArtifactBundle(options.output, options.repetitions) + console.log(JSON.stringify(summary, null, 2)) + + // A warm-up reproduction can be the one-shot v1 migration or first-paint + // cost. It is retained in the summary, but only a warm-up harness error or + // a measured hard failure makes the A/B job fail. + if ( + warmups.baseline.outcome === 'harness-error' || + warmups.candidate.outcome === 'harness-error' || + measured.baseline.some(result => result.hardFailure) || + measured.candidate.some(result => result.hardFailure) + ) { + process.exitCode = 1 + } + } finally { + await mock.close() + + if (!options.keepWorktrees) { + for (const target of prepared.reverse()) { + try { + run('git', ['worktree', 'remove', '--force', target.targetRoot]) + } catch (error) { + console.error(`warning: failed to remove ${target.targetRoot}: ${error.message}`) + } + } + + rmSync(ephemeralRoot, { force: true, recursive: true }) + } else { + console.log(`kept ephemeral worktrees at ${ephemeralRoot}`) + } + } +} + +if (resolve(process.argv[1] ?? '') === fileURLToPath(import.meta.url)) { + main().catch(error => { + console.error(error instanceof Error ? (error.stack ?? error.message) : String(error)) + process.exitCode = 2 + }) +} + +export { + ReproductionError, + classify, + pairedSoftSignal, + resultForError, + validateArtifactBundle, + validateSummary, + waitFor, + waitForPredicate, + waitForResponsive, + withTemporarySandbox, + withTimeout +} diff --git a/apps/desktop/scripts/run-short-session-hang-repro.test.mjs b/apps/desktop/scripts/run-short-session-hang-repro.test.mjs new file mode 100644 index 000000000000..95526c16d2d1 --- /dev/null +++ b/apps/desktop/scripts/run-short-session-hang-repro.test.mjs @@ -0,0 +1,171 @@ +import assert from 'node:assert/strict' +import { existsSync, writeFileSync } from 'node:fs' +import test from 'node:test' +import { join } from 'node:path' + +import { + ReproductionError, + classify, + pairedSoftSignal, + resultForError, + validateSummary, + waitFor, + waitForPredicate, + waitForResponsive, + withTemporarySandbox, + withTimeout +} from './run-short-session-hang-repro.mjs' + +const result = (outcome, latency = 10) => ({ + hardFailure: outcome !== 'not-reproduced', + maxGapMs: latency, + maxOperationMs: latency, + outcome +}) + +test('separates harness errors from reproduction timeouts', () => { + const harness = resultForError(new Error('fixture mismatch')) + const reproduced = resultForError(new ReproductionError('renderer operation exceeded 5000ms')) + + assert.equal(harness.outcome, 'harness-error') + assert.equal(harness.maxGapMs, null) + assert.equal(harness.maxOperationMs, null) + assert.equal(reproduced.outcome, 'reproduced') +}) + +test('preserves timeout semantics through the actual nested helpers', async () => { + const immediate = await withTimeout(Promise.reject(new Error('precondition')), 30, 'outer', ReproductionError).catch( + error => error + ) + assert.equal(resultForError(immediate).outcome, 'harness-error') + + const stalledCdp = { eval: async () => false } + const inner = await withTimeout( + waitFor(stalledCdp, 'false', 10, 'inner product operation', ReproductionError), + 50, + 'outer product operation', + ReproductionError + ).catch(error => error) + assert.ok(inner instanceof ReproductionError) + assert.match(inner.message, /inner product operation/) + + const outer = await withTimeout( + waitFor(stalledCdp, 'false', 50, 'inner product operation', ReproductionError), + 10, + 'outer product operation', + ReproductionError + ).catch(error => error) + assert.ok(outer instanceof ReproductionError) + assert.match(outer.message, /outer product operation/) + + const nearDeadline = await withTimeout( + new Promise(resolve => setTimeout(() => resolve('responsive'), 20)), + 50, + 'responsive operation', + ReproductionError + ) + assert.equal(nearDeadline, 'responsive') +}) + +test('distinguishes a responsive false condition from a stalled renderer evaluation', { timeout: 2_000 }, async () => { + let transientEvaluations = 0 + await waitForResponsive( + { + eval: async () => { + transientEvaluations += 1 + + if (transientEvaluations === 1) { + throw new Error('execution context was destroyed') + } + + return true + } + }, + 'true', + 500, + 'transient condition', + 50 + ) + assert.equal(transientEvaluations, 2) + + await assert.rejects( + waitForResponsive({ eval: async () => false }, 'false', 10, 'responsive condition', 1_000), + error => !(error instanceof ReproductionError) && /renderer remained responsive/.test(error.message) + ) + await assert.rejects( + waitForResponsive({ eval: () => new Promise(() => {}) }, 'false', 50, 'stalled condition', 10), + ReproductionError + ) + await assert.rejects( + waitForPredicate(() => false, 10, 'provider request'), + /provider request/ + ) +}) + +test('invalidates a target when warmup or measured runs have harness errors', () => { + const passing = Array.from({ length: 5 }, () => result('not-reproduced')) + + assert.equal(classify(passing, result('harness-error')).classification, 'invalid') + assert.equal( + classify([result('harness-error'), ...passing.slice(1)], result('not-reproduced')).classification, + 'invalid' + ) + assert.equal(classify(passing, result('not-reproduced')).classification, 'not-reproduced') + assert.equal( + classify( + [result('reproduced'), result('reproduced'), result('reproduced'), result('reproduced'), passing[0]], + passing[0] + ).classification, + 'reproduced' + ) +}) + +test('suppresses soft-signal comparisons when a run is invalid or reproduced', () => { + const passing = Array.from({ length: 5 }, () => result('not-reproduced', 10)) + + assert.equal(pairedSoftSignal([], passing).reason, 'insufficient-runs') + assert.equal(pairedSoftSignal(passing, []).reason, 'insufficient-runs') + assert.equal(pairedSoftSignal(passing, passing).reason, undefined) + assert.equal(pairedSoftSignal([result('harness-error'), ...passing.slice(1)], passing).reason, 'hard-or-invalid-run') + assert.equal(pairedSoftSignal([result('reproduced'), ...passing.slice(1)], passing).reason, 'hard-or-invalid-run') +}) + +test('rejects contradictory summary semantics', () => { + const passing = Array.from({ length: 5 }, () => result('not-reproduced')) + const classification = classify(passing, result('not-reproduced')) + const target = { ...classification, runs: passing, warmup: result('not-reproduced') } + const summary = { baseline: target, candidate: target, invalid: false } + + assert.doesNotThrow(() => validateSummary(summary, 5)) + assert.throws( + () => validateSummary({ ...summary, baseline: { ...target, classification: 'reproduced' } }, 5), + /inconsistent baseline summary classification/ + ) + assert.throws( + () => + validateSummary( + { + ...summary, + baseline: { ...target, runs: [{ ...passing[0], hardFailure: true }, ...passing.slice(1)] } + }, + 5 + ), + /inconsistent baseline run outcome and hardFailure/ + ) +}) + +test('removes the exact temporary sandbox when the run body throws', async () => { + let sandbox + + await assert.rejects( + withTemporarySandbox('cleanup-test', path => { + sandbox = path + writeFileSync(join(path, 'diagnostic.txt'), 'temporary') + throw new Error('teardown report failed') + }), + /teardown report failed/ + ) + + assert.ok(sandbox) + assert.equal(existsSync(sandbox), false) +}) diff --git a/apps/desktop/src/app/chat/short-session-hang-repro.tsx b/apps/desktop/src/app/chat/short-session-hang-repro.tsx new file mode 100644 index 000000000000..af75555869fe --- /dev/null +++ b/apps/desktop/src/app/chat/short-session-hang-repro.tsx @@ -0,0 +1,388 @@ +import type { ChatMessage } from '@/lib/chat-messages' +import { setBusy, setMessages } from '@/store/session' + +const HEARTBEAT_INTERVAL_MS = 100 +const PAINT_VALIDATION_TIMEOUT_MS = 5_000 + +interface HeartbeatSample { + at: number + gapMs: number + source: 'interval' | 'animation-frame' | 'long-task' +} + +interface FixtureTurn { + class: 'plain' | 'code' | 'tools' | 'mixed' + messages: ChatMessage[] +} + +interface ShortSessionDriver { + fixtures: FixtureTurn[] + heartbeatSamples: HeartbeatSample[] + load: (count: number) => Promise<{ + discoveredAssistantMessages: number + expectedAssistantMessages: number + expectedCodeCards: number + expectedTools: number + expectedUserIds: string[] + messageRecords: number + paintedAssistantMessages: number + paintedCodeCards: number + paintedTools: number + paintedUserIds: string[] + syntheticTurns: number + }> + manifest: () => Promise<{ + bytes: number + classes: FixtureTurn['class'][] + messageRecords: number + parts: number + sha256: string + tools: number + syntheticTurns: number + }> + checkpoint: () => void + reset: () => void + summary: () => { maxGapMs: number; samples: HeartbeatSample[] } +} + +declare global { + interface Window { + __SHORT_SESSION_HANG_REPRO__?: ShortSessionDriver + } +} + +const text = (value: string) => ({ text: value, type: 'text' as const }) + +function turn(index: number, fixtureClass: FixtureTurn['class'], assistantParts: ChatMessage['parts']): FixtureTurn { + const timestamp = 1_700_000_000_000 + index * 1_000 + + return { + class: fixtureClass, + messages: [ + { + id: `short-session-u-${index}`, + parts: [text(`Deterministic ${fixtureClass} prompt ${index}: preserve renderer responsiveness.`)], + role: 'user', + timestamp + }, + { + id: `short-session-a-${index}`, + parts: assistantParts, + pending: false, + role: 'assistant', + timestamp: timestamp + 1 + } + ] + } +} + +const fixtures: FixtureTurn[] = [ + turn(1, 'plain', [text('Plain response one. The transcript should remain selectable and scrollable.')]), + turn(2, 'code', [ + text('Code response two.\n\n```ts\nexport const bounded = (n: number) => Math.min(8, Math.max(1, n))\n```') + ]), + turn(3, 'tools', [ + { + args: { path: '/tmp/hermes-short-session-fixture.txt' }, + argsText: '{"path":"/tmp/hermes-short-session-fixture.txt"}', + result: { content: 'deterministic fixture output', ok: true }, + toolCallId: 'short-session-tool-3', + toolName: 'read_file', + type: 'tool-call' + } + ]), + turn(4, 'mixed', [ + text( + 'Mixed response four starts with prose and a compact table.\n\n| field | value |\n|---|---|\n| stable | yes |' + ), + { + args: { command: 'printf deterministic' }, + argsText: '{"command":"printf deterministic"}', + result: { output: 'deterministic', status: 0 }, + toolCallId: 'short-session-tool-4', + toolName: 'terminal', + type: 'tool-call' + }, + text('\nThe renderer must paint narration after the tool result.') + ]), + turn(5, 'plain', [text('Plain response five is the first required responsiveness checkpoint.')]), + turn(6, 'code', [ + text('Code response six.\n\n```python\ndef heartbeat(now, previous):\n return max(0, now - previous)\n```') + ]), + turn(7, 'tools', [ + { + args: { query: 'deterministic local fixture' }, + argsText: '{"query":"deterministic local fixture"}', + result: { matches: ['fixture-7'], ok: true }, + toolCallId: 'short-session-tool-7', + toolName: 'search_files', + type: 'tool-call' + } + ]), + turn(8, 'mixed', [ + text('Mixed response eight is the final checkpoint. `inline code` and **markdown** remain interactive.'), + { + args: { path: '/tmp' }, + argsText: '{"path":"/tmp"}', + result: { entries: ['hermes-short-session-fixture.txt'], ok: true }, + toolCallId: 'short-session-tool-8', + toolName: 'list_directory', + type: 'tool-call' + } + ]) +] + +if (typeof window !== 'undefined' && !window.__SHORT_SESSION_HANG_REPRO__) { + const heartbeatSamples: HeartbeatSample[] = [] + let lastInterval = performance.now() + let lastFrame = performance.now() + let maxGapMsSoFar = 0 + let sampleWindowStartedAt = performance.now() + + const record = (sample: HeartbeatSample) => { + if (document.visibilityState !== 'visible' || sample.at < sampleWindowStartedAt) { + return + } + + maxGapMsSoFar = Math.max(maxGapMsSoFar, sample.gapMs) + heartbeatSamples.push(sample) + + if (heartbeatSamples.length > 2_000) { + heartbeatSamples.splice(0, heartbeatSamples.length - 2_000) + } + } + + window.setInterval(() => { + const now = performance.now() + + if (document.visibilityState === 'visible') { + record({ at: now, gapMs: now - lastInterval, source: 'interval' }) + } + + lastInterval = now + }, HEARTBEAT_INTERVAL_MS) + + const frame = (now: number) => { + if (document.visibilityState === 'visible') { + record({ at: now, gapMs: now - lastFrame, source: 'animation-frame' }) + } + + lastFrame = now + requestAnimationFrame(frame) + } + + requestAnimationFrame(frame) + + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible') { + const now = performance.now() + lastInterval = now + lastFrame = now + } + }) + + const afterTwoFramesWhileVisible = () => + new Promise((resolve, reject) => { + if (document.visibilityState !== 'visible') { + reject(new Error('renderer became hidden before paint validation')) + + return + } + + let settled = false + let deadline: number | undefined + + const finish = (error?: Error) => { + if (settled) { + return + } + + settled = true + + if (deadline !== undefined) { + window.clearTimeout(deadline) + } + + document.removeEventListener('visibilitychange', onVisibilityChange) + + if (error) { + reject(error) + } else { + resolve() + } + } + + const onVisibilityChange = () => { + if (document.visibilityState !== 'visible') { + finish(new Error('renderer became hidden during paint validation')) + } + } + + document.addEventListener('visibilitychange', onVisibilityChange) + deadline = window.setTimeout( + () => finish(new Error('paint validation did not observe two frames')), + PAINT_VALIDATION_TIMEOUT_MS + ) + requestAnimationFrame(() => { + if (document.visibilityState !== 'visible') { + finish(new Error('renderer became hidden before the first paint frame')) + + return + } + + requestAnimationFrame(() => { + if (document.visibilityState !== 'visible') { + finish(new Error('renderer became hidden before the second paint frame')) + + return + } + + finish() + }) + }) + }) + + try { + const observer = new PerformanceObserver(list => { + for (const entry of list.getEntries()) { + record({ at: entry.startTime, gapMs: entry.duration, source: 'long-task' }) + } + }) + + observer.observe({ entryTypes: ['longtask'] }) + } catch { + // Long Task API availability is a soft diagnostic signal. + } + + const checkpoint = () => { + const now = performance.now() + + sampleWindowStartedAt = now + heartbeatSamples.length = 0 + maxGapMsSoFar = 0 + lastInterval = now + lastFrame = now + } + + const reset = () => { + checkpoint() + setBusy(false) + setMessages([]) + } + + window.__SHORT_SESSION_HANG_REPRO__ = { + fixtures, + heartbeatSamples, + load: async count => { + if (!Number.isFinite(count)) { + throw new RangeError('count must be a finite number') + } + + const bounded = Math.min(fixtures.length, Math.max(1, Math.trunc(count))) + const next = fixtures.slice(0, bounded).flatMap(fixture => fixture.messages) + const expectedUserIds = next.filter(message => message.role === 'user').map(message => message.id) + const expectedAssistantIds = next.filter(message => message.role === 'assistant').map(message => message.id) + const expectedAssistantMessages = expectedAssistantIds.length + const expectedTools = next.flatMap(message => message.parts).filter(part => part.type === 'tool-call').length + + const expectedCodeCards = next + .flatMap(message => message.parts) + .filter(part => part.type === 'text') + .reduce((total, part) => total + Math.floor((part.text.match(/```/g)?.length ?? 0) / 2), 0) + + setBusy(false) + setMessages(next) + + await afterTwoFramesWhileVisible() + + const isPainted = (element: Element) => { + const style = getComputedStyle(element) + + return style.display !== 'none' && style.visibility !== 'hidden' && element.getClientRects().length > 0 + } + + const paintedUserIds = expectedUserIds.filter(id => { + const element = document.querySelector(`[data-message-id="${CSS.escape(id)}"]`) + + return element ? isPainted(element) : false + }) + + const assistantRoots = [ + ...new Set( + expectedUserIds.flatMap(id => { + const user = document.querySelector(`[data-message-id="${CSS.escape(id)}"]`) + const turn = user?.closest('[data-slot="aui_turn-pair"]') + + return turn ? [...turn.querySelectorAll('[data-slot="aui_assistant-message-root"]')] : [] + }) + ) + ] + + const paintedTools = assistantRoots + .flatMap(root => [...root.querySelectorAll('[data-tool-row]')]) + .filter(isPainted).length + + const paintedCodeCards = assistantRoots + .flatMap(root => [...root.querySelectorAll('[data-slot="code-card"]')]) + .filter(isPainted).length + + const paintedAssistantMessages = assistantRoots.filter(root => { + if (!isPainted(root)) { + return false + } + + const hasText = Boolean(root.textContent?.trim()) + const hasPaintedTool = [...root.querySelectorAll('[data-tool-row]')].some(isPainted) + const hasPaintedCode = [...root.querySelectorAll('[data-slot="code-card"]')].some(isPainted) + + return hasText || hasPaintedTool || hasPaintedCode + }).length + + return { + discoveredAssistantMessages: assistantRoots.length, + expectedAssistantMessages, + expectedCodeCards, + expectedTools, + expectedUserIds, + messageRecords: next.length, + paintedAssistantMessages, + paintedCodeCards, + paintedTools, + paintedUserIds, + syntheticTurns: bounded + } + }, + manifest: async () => { + const serialized = JSON.stringify(fixtures) + const bytes = new TextEncoder().encode(serialized) + const digest = await crypto.subtle.digest('SHA-256', bytes) + + return { + bytes: bytes.byteLength, + classes: fixtures.map(fixture => fixture.class), + messageRecords: fixtures.reduce((total, fixture) => total + fixture.messages.length, 0), + parts: fixtures.reduce( + (total, fixture) => total + fixture.messages.reduce((count, message) => count + message.parts.length, 0), + 0 + ), + sha256: [...new Uint8Array(digest)].map(value => value.toString(16).padStart(2, '0')).join(''), + tools: fixtures.reduce( + (total, fixture) => + total + + fixture.messages.reduce( + (count, message) => count + message.parts.filter(part => part.type === 'tool-call').length, + 0 + ), + 0 + ), + syntheticTurns: fixtures.length + } + }, + checkpoint, + reset, + summary: () => ({ + maxGapMs: maxGapMsSoFar, + samples: [...heartbeatSamples] + }) + } +} diff --git a/apps/desktop/src/lib/inflight-turn-journal.test.ts b/apps/desktop/src/lib/inflight-turn-journal.test.ts index 206a12228894..6a0bed4bf22a 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.test.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.test.ts @@ -7,11 +7,15 @@ import { mergeInFlightMessages, persistInFlightTurnState, readInFlightTurnJournal, - recoverInFlightTurnJournal + recoverInFlightTurnJournal, + resetInFlightTurnJournalStateForTests } from '@/lib/inflight-turn-journal' +const STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' const STORAGE_PREFIX = 'hermes.desktop.inflightTurnJournal.v2:' -const LEGACY_STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' +const MIGRATION_KEY = 'hermes.desktop.inflightTurnJournal.v2.migrated' + +const sessionStorageKey = (storedSessionId: string) => `${STORAGE_PREFIX}${encodeURIComponent(storedSessionId)}` function user(id: string, text: string): ChatMessage { return { id, role: 'user', parts: [{ type: 'text', text }] } @@ -46,16 +50,94 @@ function journalState(overrides: Partial = {}): Journal } beforeEach(() => { + resetInFlightTurnJournalStateForTests() vi.useFakeTimers() window.localStorage.clear() }) afterEach(() => { + vi.restoreAllMocks() clearInFlightTurnJournal('stored-1') vi.useRealTimers() }) describe('persistInFlightTurnState', () => { + it('sweeps expired and oldest session entries once before the first write', () => { + const now = Date.now() + + for (let index = 0; index < 25; index += 1) { + const sessionId = `old-${index}` + + const snapshot = { + messages: [ + user(`u-${index}`, `prompt-${index}`), + assistant(`a-${index}`, `partial-${index}`, { pending: true }) + ], + streamId: `a-${index}`, + turnStartedAt: index, + updatedAt: now - index * 1_000 + } + + window.localStorage.setItem(sessionStorageKey(sessionId), JSON.stringify(snapshot)) + } + + window.localStorage.setItem( + sessionStorageKey('expired'), + JSON.stringify({ + messages: [user('expired-u', 'expired'), assistant('expired-a', 'expired', { pending: true })], + streamId: 'expired-a', + turnStartedAt: 0, + updatedAt: now - 8 * 24 * 60 * 60 * 1_000 + }) + ) + + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + const sessionKeys = Array.from({ length: window.localStorage.length }, (_, index) => + window.localStorage.key(index) + ).filter((key): key is string => key?.startsWith(STORAGE_PREFIX) === true) + + expect(sessionKeys).toHaveLength(24) + expect(window.localStorage.getItem(sessionStorageKey('expired'))).toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('old-24'))).toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('stored-1'))).not.toBeNull() + }) + + it('writes only the current session instead of reading and rewriting the aggregate journal', () => { + const localStorage = window.localStorage + const storageConstructor = window.Storage + + const spyTarget = + typeof storageConstructor === 'function' && localStorage instanceof storageConstructor + ? storageConstructor.prototype + : localStorage + + const getItem = vi.spyOn(spyTarget, 'getItem') + const setItem = vi.spyOn(spyTarget, 'setItem') + + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + expect(getItem).not.toHaveBeenCalledWith(STORAGE_KEY) + expect(setItem).not.toHaveBeenCalledWith(STORAGE_KEY, expect.any(String)) + expect(setItem).toHaveBeenCalledWith(sessionStorageKey('stored-1'), expect.any(String)) + }) + + it('keeps another session snapshot when one session settles', () => { + persistInFlightTurnState(journalState()) + persistInFlightTurnState(journalState({ storedSessionId: 'stored-2' })) + vi.advanceTimersByTime(400) + + expect(window.localStorage.getItem(sessionStorageKey('stored-1'))).not.toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('stored-2'))).not.toBeNull() + + clearInFlightTurnJournal('stored-2') + + expect(readInFlightTurnJournal('stored-1')).not.toBeNull() + expect(readInFlightTurnJournal('stored-2')).toBeNull() + }) + it('journals the running turn tail after the throttle window', () => { persistInFlightTurnState(journalState()) @@ -88,6 +170,182 @@ describe('persistInFlightTurnState', () => { expect(tail?.parts).toEqual([{ type: 'text', text: 'partial answer grew' }]) }) + it('preserves a long user prompt exactly so recovery still matches its transcript row', () => { + const prompt = 'prompt '.repeat(8_000) + + persistInFlightTurnState( + journalState({ + messages: [user('u1', prompt), assistant('assistant-stream-1', 'partial', { pending: true })] + }) + ) + vi.advanceTimersByTime(400) + + const result = recoverInFlightTurnJournal('stored-1', [user('db-u1', prompt)]) + + expect(result.messages.map(message => message.id)).toEqual(['db-u1', 'assistant-stream-1']) + }) + + it('preserves user attachment refs exactly so recovery still matches its transcript row', () => { + const attachmentRefs = Array.from({ length: 25 }, (_, index) => `@file:/tmp/input-${index}.txt`) + const prompt = user('u1', 'inspect these files') + prompt.attachmentRefs = attachmentRefs + + persistInFlightTurnState( + journalState({ messages: [prompt, assistant('assistant-stream-1', 'partial', { pending: true })] }) + ) + vi.advanceTimersByTime(400) + + const restoredPrompt = user('db-u1', 'inspect these files') + restoredPrompt.attachmentRefs = attachmentRefs + const result = recoverInFlightTurnJournal('stored-1', [restoredPrompt]) + + expect(result.messages.map(message => message.id)).toEqual(['db-u1', 'assistant-stream-1']) + }) + + it('trims oldest sealed rows when bounded parts exceed the entry cap', () => { + const text = 'x'.repeat(60 * 1024) + + const messages = [ + user('u1', 'do the thing'), + assistant('a1', text, { pending: false }), + assistant('a2', text, { pending: false }), + assistant('a3', text, { pending: false }), + assistant('a4', text, { pending: true }) + ] + + persistInFlightTurnState(journalState({ messages, streamId: 'a4' })) + vi.advanceTimersByTime(400) + + const raw = window.localStorage.getItem(sessionStorageKey('stored-1')) + const snapshot = JSON.parse(raw!) + + expect(raw?.length).toBeLessThanOrEqual(160 * 1024) + expect(snapshot.messages.map((message: ChatMessage) => message.id)).toEqual(['u1', 'a3', 'a4']) + }) + + it('keeps an older recoverable snapshot when the newest row alone is too large', () => { + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + const hugeAssistant: ChatMessage = { + id: 'assistant-stream-1', + role: 'assistant', + parts: Array.from({ length: 3 }, () => ({ type: 'text' as const, text: 'x'.repeat(64 * 1024) })), + pending: true + } + + persistInFlightTurnState( + journalState({ messages: [user('u1', 'do the thing'), hugeAssistant], streamId: hugeAssistant.id }) + ) + vi.advanceTimersByTime(400) + + const snapshot = JSON.parse(window.localStorage.getItem(sessionStorageKey('stored-1'))!) + + expect(snapshot.messages[1].parts).toEqual([{ type: 'text', text: 'partial answer' }]) + }) + + it('skips a pathological user prompt instead of truncating its recovery join key', () => { + const prompt = 'x'.repeat(64 * 1024 + 1) + + persistInFlightTurnState( + journalState({ + messages: [user('u1', prompt), assistant('assistant-stream-1', 'partial', { pending: true })] + }) + ) + vi.advanceTimersByTime(400) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) + + it('does not parse the legacy aggregate when a pathological write discards its session', () => { + const legacy = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() + } + + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { 'stored-1': legacy }, version: 1 })) + const getItem = vi.spyOn(Storage.prototype, 'getItem') + const prompt = 'x'.repeat(64 * 1024 + 1) + + persistInFlightTurnState( + journalState({ + messages: [user('u1', prompt), assistant('assistant-stream-1', 'partial', { pending: true })] + }) + ) + vi.advanceTimersByTime(400) + + expect(getItem).not.toHaveBeenCalledWith(STORAGE_KEY) + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) + + it('preserves a tombstone while sweeping before legacy migration', () => { + const legacy = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() + } + + window.localStorage.setItem(sessionStorageKey('stored-1'), '0') + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { 'stored-1': legacy }, version: 1 })) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(window.localStorage.getItem(STORAGE_KEY)).toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('stored-1'))).toBeNull() + }) + + it('removes tombstones after the one-shot legacy migration has completed', () => { + const key = sessionStorageKey('stored-1') + + window.localStorage.setItem(MIGRATION_KEY, '1') + window.localStorage.setItem(key, '0') + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(window.localStorage.getItem(key)).toBeNull() + }) + + it('strips pathological 5 MiB tool payloads before attempting a storage write', () => { + const setItem = vi.spyOn(Storage.prototype, 'setItem') + + const oversized: ChatMessage = { + id: 'assistant-stream-1', + role: 'assistant', + parts: [ + { + type: 'tool-call', + toolCallId: 'tc-1', + toolName: 'terminal', + args: { command: 'x'.repeat(5 * 1024 * 1024) }, + result: 'x'.repeat(5 * 1024 * 1024), + isError: true + }, + { type: 'text', text: 'still useful' } + ], + pending: true + } + + persistInFlightTurnState(journalState({ messages: [user('u1', 'do the thing'), oversized] })) + vi.advanceTimersByTime(400) + + const raw = window.localStorage.getItem(sessionStorageKey('stored-1')) + const snapshot = JSON.parse(raw!) + const persistedTool = snapshot.messages[1].parts[0] + + expect(raw?.length).toBeLessThan(256 * 1024) + expect(persistedTool).toEqual({ + args: {}, + isError: true, + result: {}, + toolCallId: 'tc-1', + toolName: 'terminal', + type: 'tool-call' + }) + expect(snapshot.messages[1].parts[1]).toEqual({ type: 'text', text: 'still useful' }) + expect(setItem.mock.calls.every(([, value]) => value.length <= 256 * 1024)).toBe(true) + }) + it('clears the entry the moment the turn settles, cancelling pending writes', () => { persistInFlightTurnState(journalState()) vi.advanceTimersByTime(400) @@ -113,52 +371,170 @@ describe('persistInFlightTurnState', () => { persistInFlightTurnState(journalState()) vi.advanceTimersByTime(400) - const raw = JSON.parse(window.localStorage.getItem(`${STORAGE_PREFIX}stored-1`)!) + const key = sessionStorageKey('stored-1') + const raw = JSON.parse(window.localStorage.getItem(key)!) raw.updatedAt = Date.now() - 8 * 24 * 60 * 60 * 1000 - window.localStorage.setItem(`${STORAGE_PREFIX}stored-1`, JSON.stringify(raw)) + window.localStorage.setItem(key, JSON.stringify(raw)) expect(readInFlightTurnJournal('stored-1')).toBeNull() }) - it('writes each session under its own key, untouched by other sessions settling', () => { - persistInFlightTurnState(journalState()) - persistInFlightTurnState(journalState({ storedSessionId: 'stored-2' })) - vi.advanceTimersByTime(400) + it('isolates storage read, write, and removal failures', () => { + const getItem = vi.spyOn(Storage.prototype, 'getItem').mockImplementation(() => { + throw new Error('read denied') + }) - expect(window.localStorage.getItem(`${STORAGE_PREFIX}stored-1`)).not.toBeNull() - expect(window.localStorage.getItem(`${STORAGE_PREFIX}stored-2`)).not.toBeNull() + expect(() => readInFlightTurnJournal('stored-1')).not.toThrow() + getItem.mockRestore() - clearInFlightTurnJournal('stored-2') + const setItem = vi.spyOn(Storage.prototype, 'setItem').mockImplementation(() => { + throw new Error('quota') + }) - expect(readInFlightTurnJournal('stored-1')).not.toBeNull() - expect(readInFlightTurnJournal('stored-2')).toBeNull() + expect(() => { + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + }).not.toThrow() + setItem.mockRestore() + + const removeItem = vi.spyOn(Storage.prototype, 'removeItem').mockImplementation(() => { + throw new Error('remove denied') + }) + + expect(() => clearInFlightTurnJournal('stored-1')).not.toThrow() }) - it('recovers entries journaled by the v1 single-key store', () => { - // A pre-upgrade crash leaves a v1 store behind; the first journal touch - // after the upgrade must still recover its turns. + it('discards malformed optional message metadata instead of throwing during recovery', () => { window.localStorage.setItem( - LEGACY_STORAGE_KEY, + sessionStorageKey('stored-1'), JSON.stringify({ - entries: { - 'stored-legacy': { - messages: [user('u1', 'legacy prompt'), assistant('a1', 'legacy partial', { pending: true })], - streamId: 'a1', - turnStartedAt: 500, - updatedAt: Date.now() - } - }, - version: 1 + messages: [ + { id: 'u1', role: 'user', parts: [{ type: 'text', text: 'prompt' }], attachmentRefs: '@file:bad' }, + { id: 'a1', role: 'assistant', parts: [{ type: 'text', text: 'partial' }], pending: true } + ], + streamId: 'a1', + turnStartedAt: 1, + updatedAt: Date.now() }) ) + const base = [user('db-u1', 'prompt')] + + expect(() => recoverInFlightTurnJournal('stored-1', base)).not.toThrow() + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) +}) - const entry = readInFlightTurnJournal('stored-legacy') +describe('legacy journal migration', () => { + it('migrates the bounded v1 aggregate once and recovers its sessions', () => { + const first = { + messages: [user('u1', 'one'), assistant('a1', 'partial one', { pending: true })], + streamId: 'a1', + turnStartedAt: 1, + updatedAt: Date.now() + } - expect(entry?.streamId).toBe('a1') - expect(entry?.messages).toHaveLength(2) - expect(window.localStorage.getItem(LEGACY_STORAGE_KEY)).toBeNull() + const second = { + messages: [ + user('u2', 'two'), + { + id: 'a2', + role: 'assistant' as const, + parts: [ + { + type: 'tool-call' as const, + toolCallId: 'tc-legacy', + toolName: 'terminal', + args: { command: 'large-output' }, + result: 'x'.repeat(1024 * 1024) + }, + { type: 'text' as const, text: 'partial two' } + ], + pending: true + } + ], + streamId: 'a2', + turnStartedAt: 2, + updatedAt: Date.now() + } + + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { one: first, two: second }, version: 1 })) + + expect(readInFlightTurnJournal('one')).toEqual(first) + const migratedSecondRaw = window.localStorage.getItem(sessionStorageKey('two'))! + const migratedSecond = JSON.parse(migratedSecondRaw) + + expect(migratedSecondRaw.length).toBeLessThan(256 * 1024) + expect(migratedSecond.messages[1].parts).toEqual([ + { args: {}, result: {}, toolCallId: 'tc-legacy', toolName: 'terminal', type: 'tool-call' }, + { text: 'partial two', type: 'text' } + ]) + expect(window.localStorage.getItem(STORAGE_KEY)).toBeNull() + expect(window.localStorage.getItem(MIGRATION_KEY)).toBe('1') + + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { three: first }, version: 1 })) + expect(readInFlightTurnJournal('three')).toBeNull() + expect(window.localStorage.getItem(STORAGE_KEY)).not.toBeNull() + }) - clearInFlightTurnJournal('stored-legacy') + it('drops an oversized legacy aggregate without parsing it', () => { + window.localStorage.setItem(STORAGE_KEY, 'x'.repeat(2 * 1024 * 1024 + 1)) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(window.localStorage.getItem(STORAGE_KEY)).toBeNull() + expect(window.localStorage.getItem(MIGRATION_KEY)).toBe('1') + }) + + it('does not overwrite a newer per-session snapshot while migrating another legacy session', () => { + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + const legacyCurrent = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() - 1_000 + } + + const legacyOther = { + messages: [user('u2', 'other prompt'), assistant('a2', 'other partial', { pending: true })], + streamId: 'a2', + turnStartedAt: 2, + updatedAt: Date.now() + } + + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify({ entries: { 'stored-1': legacyCurrent, other: legacyOther }, version: 1 }) + ) + + expect(readInFlightTurnJournal('other')).toEqual(legacyOther) + expect(readInFlightTurnJournal('stored-1')?.messages[0]).toEqual(user('u1', 'do the thing')) + }) + + it('does not resurrect a legacy entry after that session settles before migration', () => { + const legacyCurrent = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() + } + + const legacyOther = { + messages: [user('u2', 'other prompt'), assistant('a2', 'other partial', { pending: true })], + streamId: 'a2', + turnStartedAt: 2, + updatedAt: Date.now() + } + + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify({ entries: { 'stored-1': legacyCurrent, other: legacyOther }, version: 1 }) + ) + + persistInFlightTurnState(journalState({ busy: false, awaitingResponse: false, streamId: null })) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(readInFlightTurnJournal('other')).toEqual(legacyOther) }) }) diff --git a/apps/desktop/src/lib/inflight-turn-journal.ts b/apps/desktop/src/lib/inflight-turn-journal.ts index 27912970ff67..46103b9c9736 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.ts @@ -16,20 +16,34 @@ import { type ChatMessage, type ChatMessagePart, chatMessageText } from '@/lib/c * Best-effort by design: storage failures must never break chat streaming. */ -/** One localStorage key PER SESSION. The v1 single-key store meant every - * throttled write re-parsed and re-stringified EVERY busy session's tail — - * with a grid of concurrent streams that was a whole-store JSON round-trip - * dozens of times a second, all on the main thread. Per-session keys make a - * write O(own tail) regardless of how many other sessions are streaming. */ -const STORAGE_PREFIX = 'hermes.desktop.inflightTurnJournal.v2:' const LEGACY_STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' -const MAX_ENTRIES = 24 +const STORAGE_PREFIX = 'hermes.desktop.inflightTurnJournal.v2:' +const LEGACY_MIGRATION_KEY = 'hermes.desktop.inflightTurnJournal.v2.migrated' +const DISCARDED_SNAPSHOT_RAW = '0' +const STORE_VERSION = 1 +const MAX_SESSION_STORE_CHARS = 4 * 1024 * 1024 const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000 +// Keep the worst-case v2 namespace below a conservative localStorage budget +// while retaining the 24 newest session slots for ordinary small snapshots. +const MAX_ENTRY_CHARS = 160 * 1024 +const MAX_ENTRIES = Math.min(24, Math.floor(MAX_SESSION_STORE_CHARS / MAX_ENTRY_CHARS)) +const MAX_LEGACY_STORE_CHARS = 2 * 1024 * 1024 +const MAX_SESSION_KEY_CHARS = 512 +const MAX_JOURNALED_MESSAGES = 24 +const MAX_TEXT_PART_CHARS = 64 * 1024 +const MAX_METADATA_CHARS = 2 * 1024 +const MAX_USER_ATTACHMENT_REFS = 256 +const MAX_USER_ATTACHMENT_REF_CHARS = 64 * 1024 /** Streaming repaints arrive every ~33ms; localStorage writes are synchronous. * Trailing-edge throttle keeps the journal off the hot path — a crash costs at * most this much of the newest tail. */ const PERSIST_THROTTLE_MS = 400 +// A renderer can accumulate one entry per session over its lifetime. Sweep the +// bounded v2 namespace once on first journal access; never scan it on the +// 400ms streaming write path. +let sessionStoreSwept = false + export interface InFlightTurnSnapshot { messages: ChatMessage[] streamId: null | string @@ -46,6 +60,11 @@ export interface JournalableSessionState { turnStartedAt: null | number } +interface JournalStore { + entries: Record + version: typeof STORE_VERSION +} + export interface InFlightRecoveryResult { applied: boolean /** The base transcript already contains the journaled turn's completed @@ -64,133 +83,431 @@ function storage(): Storage | null { } } -const entryKey = (storedSessionId: string) => `${STORAGE_PREFIX}${storedSessionId}` - -function isExpired(entry: InFlightTurnSnapshot, now = Date.now()): boolean { - return now - entry.updatedAt > MAX_AGE_MS -} - -function loadEntry(storedSessionId: string): InFlightTurnSnapshot | null { - const store = storage() +function sessionStorageKey(storedSessionId: string): null | string { + try { + const encoded = encodeURIComponent(storedSessionId) - if (!store) { + return encoded.length > 0 && encoded.length <= MAX_SESSION_KEY_CHARS ? `${STORAGE_PREFIX}${encoded}` : null + } catch { return null } +} +function readRaw(store: Storage, key: string): null | string { try { - const raw = store.getItem(entryKey(storedSessionId)) - const parsed = raw ? (JSON.parse(raw) as InFlightTurnSnapshot) : null - - return parsed && typeof parsed.updatedAt === 'number' && Array.isArray(parsed.messages) ? parsed : null + return store.getItem(key) } catch { return null } } -function saveEntry(storedSessionId: string, entry: InFlightTurnSnapshot): void { +function removeRaw(store: Storage, key: string): void { try { - storage()?.setItem(entryKey(storedSessionId), JSON.stringify(entry)) + store.removeItem(key) } catch { - // Quota/private-mode failures: the journal is a recovery aid, not truth. + // Best-effort recovery state must not interrupt chat streaming. } } -function removeEntry(storedSessionId: string): void { +function writeRaw(store: Storage, key: string, value: string): boolean { try { - storage()?.removeItem(entryKey(storedSessionId)) + store.setItem(key, value) + + return true } catch { - // Same best-effort stance as saveEntry. + return false } } -// Split a v1 single-key store into per-session entries. Checked on every -// journal touch (a null getItem is free); a populated v1 store exists at most -// once, right after the upgrade. -function migrateLegacyStore(store: Storage): void { - try { - const legacy = store.getItem(LEGACY_STORAGE_KEY) +function isSnapshot(value: unknown): value is InFlightTurnSnapshot { + if (!value || typeof value !== 'object') { + return false + } - if (!legacy) { - return - } + const snapshot = value as Partial - const parsed = JSON.parse(legacy) + return ( + Array.isArray(snapshot.messages) && + snapshot.messages.every( + message => + Boolean(message) && + typeof message === 'object' && + typeof message.id === 'string' && + ['assistant', 'system', 'tool', 'user'].includes(message.role) && + Array.isArray(message.parts) && + message.parts.every( + part => + Boolean(part) && + typeof part === 'object' && + typeof part.type === 'string' && + (part.type !== 'text' && part.type !== 'reasoning' + ? part.type !== 'tool-call' || + (typeof part.toolName === 'string' && + (part.toolCallId === undefined || typeof part.toolCallId === 'string') && + (part.isError === undefined || typeof part.isError === 'boolean')) + : typeof part.text === 'string' && (part.parentId === undefined || typeof part.parentId === 'string')) + ) && + (message.timestamp === undefined || + (typeof message.timestamp === 'number' && Number.isFinite(message.timestamp))) && + (message.pending === undefined || typeof message.pending === 'boolean') && + (message.error === undefined || typeof message.error === 'string') && + (message.branchGroupId === undefined || typeof message.branchGroupId === 'string') && + (message.hidden === undefined || typeof message.hidden === 'boolean') && + (message.interim === undefined || typeof message.interim === 'boolean') && + (message.attachmentRefs === undefined || + (Array.isArray(message.attachmentRefs) && message.attachmentRefs.every(ref => typeof ref === 'string'))) && + (message.rowId === undefined || (typeof message.rowId === 'number' && Number.isFinite(message.rowId))) + ) && + (snapshot.streamId === null || typeof snapshot.streamId === 'string') && + (snapshot.turnStartedAt === null || typeof snapshot.turnStartedAt === 'number') && + typeof snapshot.updatedAt === 'number' && + Number.isFinite(snapshot.updatedAt) + ) +} - if (parsed && typeof parsed.entries === 'object' && !Array.isArray(parsed.entries)) { - for (const [id, entry] of Object.entries(parsed.entries as Record)) { - saveEntry(id, entry) - } - } - } catch { - // A corrupt v1 store has nothing worth carrying over. +function parseSnapshot(raw: string): InFlightTurnSnapshot | null { + if (raw.length > MAX_ENTRY_CHARS) { + return null } try { - store.removeItem(LEGACY_STORAGE_KEY) + const parsed = JSON.parse(raw) + + return isSnapshot(parsed) ? parsed : null } catch { - // Best-effort, like every other journal write. + return null } } -// One-time prune per renderer: drop expired/overflow entries. Startup-only on -// purpose — entries clear on settle, so anything left over is crash residue, -// and enumerating localStorage on the write path would defeat the point. -let housekeepingDone = false +function serializeSnapshot(snapshot: InFlightTurnSnapshot): string | null { + let messages = snapshot.messages -function ensureHousekeeping(): void { - const store = storage() + while (messages.length > 0) { + try { + const raw = JSON.stringify({ ...snapshot, messages }) - if (!store) { - return + if (raw.length <= MAX_ENTRY_CHARS) { + return raw + } + } catch { + return null + } + + // Keep the join-key row and newest assistant progress while dropping the + // oldest sealed rows. If those two rows alone do not fit, the caller must + // avoid replacing an older recoverable snapshot with a tombstone. + if (messages.length <= 2) { + return null + } + + messages = [messages[0], ...messages.slice(2)] } - migrateLegacyStore(store) + return null +} - if (housekeepingDone) { +function sweepSessionStore(store: Storage, reserveSlot = false): void { + if (sessionStoreSwept) { return } - housekeepingDone = true + sessionStoreSwept = true try { - const keys: string[] = [] + const sessionKeys: string[] = [] for (let index = 0; index < store.length; index += 1) { const key = store.key(index) if (key?.startsWith(STORAGE_PREFIX)) { - keys.push(key) + sessionKeys.push(key) } } - const live: { key: string; updatedAt: number }[] = [] + const liveEntries: Array<{ key: string; snapshot: InFlightTurnSnapshot }> = [] + const migrated = readRaw(store, LEGACY_MIGRATION_KEY) !== null - for (const key of keys) { - let entry: InFlightTurnSnapshot | null = null + for (const key of sessionKeys) { + const raw = readRaw(store, key) - try { - entry = JSON.parse(store.getItem(key) ?? '') as InFlightTurnSnapshot - } catch { - // Unparseable — prune below. + // A tombstone is intentional state. It suppresses the stale v1 + // predecessor until the one-shot migration removes the aggregate. + if (raw === DISCARDED_SNAPSHOT_RAW) { + if (migrated) { + removeRaw(store, key) + } + + continue } - if (!entry || typeof entry.updatedAt !== 'number' || isExpired(entry)) { - store.removeItem(key) - } else { - live.push({ key, updatedAt: entry.updatedAt }) + const snapshot = raw ? parseSnapshot(raw) : null + + if (!snapshot || isExpired(snapshot)) { + removeRaw(store, key) + + continue } + + liveEntries.push({ key, snapshot }) + } + + liveEntries + .sort((left, right) => right.snapshot.updatedAt - left.snapshot.updatedAt) + .slice(reserveSlot ? MAX_ENTRIES - 1 : MAX_ENTRIES) + .forEach(entry => removeRaw(store, entry.key)) + } catch { + // The journal is best effort; a storage enumeration failure must not + // interrupt renderer work or turn persistence. + } +} + +function boundedString(value: string, maxChars: number): string { + return value.length <= maxChars ? value : value.slice(0, maxChars) +} + +function boundedPart(part: ChatMessagePart): ChatMessagePart | null { + if (part.type === 'text') { + return { + type: 'text', + text: boundedString(part.text, MAX_TEXT_PART_CHARS), + ...(part.parentId === undefined ? {} : { parentId: boundedString(part.parentId, MAX_METADATA_CHARS) }) + } + } + + if (part.type === 'reasoning') { + return { + type: 'reasoning', + text: boundedString(part.text, MAX_TEXT_PART_CHARS), + ...(part.parentId === undefined ? {} : { parentId: boundedString(part.parentId, MAX_METADATA_CHARS) }) + } + } + + if (part.type === 'tool-call') { + // Tool payloads can contain multi-megabyte command output. Recovery only + // needs invocation identity and failure state; args/results are available + // from the backend transcript when it survives. + return { + type: 'tool-call', + toolName: boundedString(part.toolName, MAX_METADATA_CHARS), + args: {}, + ...(part.toolCallId === undefined ? {} : { toolCallId: boundedString(part.toolCallId, MAX_METADATA_CHARS) }), + ...(part.result === undefined ? {} : { result: {} }), + ...(part.isError === undefined ? {} : { isError: part.isError }) + } + } + + // Rich file/image/data/source parts can embed large payloads. They are not + // required for in-flight text/tool recovery and remain backend-owned. + return null +} + +function boundedMessages(messages: ChatMessage[]): ChatMessage[] | null { + const bounded = + messages.length <= MAX_JOURNALED_MESSAGES + ? messages + : [messages[0], ...messages.slice(-(MAX_JOURNALED_MESSAGES - 1))] + + // User text and attachment refs are the recovery join key. Truncating either + // could attach a journal tail to the wrong transcript row, so pathological + // prompts skip journaling instead of weakening the match. + if ( + bounded.some(message => { + if (message.role !== 'user') { + return false + } + + if ( + message.parts.some( + part => (part.type === 'text' || part.type === 'reasoning') && part.text.length > MAX_TEXT_PART_CHARS + ) + ) { + return true + } + + const refs = message.attachmentRefs + + if (!refs) { + return false + } + + if (refs.length > MAX_USER_ATTACHMENT_REFS) { + return true + } + + let chars = 0 + + for (const ref of refs) { + chars += ref.length + + if (chars > MAX_USER_ATTACHMENT_REF_CHARS) { + return true + } + } + + return false + }) + ) { + return null + } + + return bounded.map(message => ({ + id: boundedString(message.id, MAX_METADATA_CHARS), + role: message.role, + parts: message.parts.map(boundedPart).filter((part): part is ChatMessagePart => part !== null), + ...(message.timestamp === undefined ? {} : { timestamp: message.timestamp }), + ...(message.pending === undefined ? {} : { pending: message.pending }), + ...(message.error === undefined ? {} : { error: boundedString(message.error, MAX_METADATA_CHARS) }), + ...(message.branchGroupId === undefined + ? {} + : { branchGroupId: boundedString(message.branchGroupId, MAX_METADATA_CHARS) }), + ...(message.hidden === undefined ? {} : { hidden: message.hidden }), + ...(message.interim === undefined ? {} : { interim: message.interim }), + ...(message.attachmentRefs === undefined + ? {} + : { + attachmentRefs: + message.role === 'user' + ? [...message.attachmentRefs] + : message.attachmentRefs + .slice(0, MAX_USER_ATTACHMENT_REFS) + .map(ref => boundedString(ref, MAX_METADATA_CHARS)) + }), + ...(message.rowId === undefined ? {} : { rowId: message.rowId }) + })) +} + +function migrateLegacyStore(store: Storage): void { + if (readRaw(store, LEGACY_MIGRATION_KEY) !== null) { + return + } + + const raw = readRaw(store, LEGACY_STORAGE_KEY) + + if (raw === null) { + return + } + + // Claim the migration before touching the aggregate. If storage is failing, + // skip legacy recovery rather than retrying an expensive parse on every read. + if (!writeRaw(store, LEGACY_MIGRATION_KEY, '1')) { + return + } + + // Release the multi-megabyte aggregate before allocating per-session v2 + // entries. The captured string remains available for this one migration. + removeRaw(store, LEGACY_STORAGE_KEY) + + if (!raw) { + return + } + + if (raw.length > MAX_LEGACY_STORE_CHARS) { + return + } + + try { + const parsed = JSON.parse(raw) as Partial + + if ( + parsed.version !== STORE_VERSION || + !parsed.entries || + typeof parsed.entries !== 'object' || + Array.isArray(parsed.entries) + ) { + return } - live.sort((a, b) => b.updatedAt - a.updatedAt) + const existingV2Keys = new Set() - for (const { key } of live.slice(MAX_ENTRIES)) { - store.removeItem(key) + for (let index = 0; index < store.length; index += 1) { + const key = store.key(index) + + if (key?.startsWith(STORAGE_PREFIX)) { + existingV2Keys.add(key) + } + } + + const entries = Object.entries(parsed.entries) + .filter((entry): entry is [string, InFlightTurnSnapshot] => isSnapshot(entry[1]) && !isExpired(entry[1])) + .sort((a, b) => b[1].updatedAt - a[1].updatedAt) + .slice(0, Math.max(0, MAX_ENTRIES - existingV2Keys.size)) + + for (const [storedSessionId, snapshot] of entries) { + const key = sessionStorageKey(storedSessionId) + const messages = boundedMessages(snapshot.messages) + const value = messages ? serializeSnapshot({ ...snapshot, messages }) : null + + // A v2 snapshot may have been written before the one-shot migration ran. + // Never replace newer per-session state with its stale v1 predecessor. + if (key && value && readRaw(store, key) === null) { + if (writeRaw(store, key, value)) { + existingV2Keys.add(key) + } + } } } catch { - // Best-effort, like every other journal write. + // Malformed legacy data is discarded below. } } +function discardSnapshot(store: Storage, key: string): void { + // Migrate first so an existing v2 key suppresses its stale v1 predecessor, + // then remove the current session. This keeps every discard path from + // resurrecting legacy state on a later read. + migrateLegacyStore(store) + removeRaw(store, key) +} + +function readSnapshot(storedSessionId: string): InFlightTurnSnapshot | null { + const store = storage() + const key = sessionStorageKey(storedSessionId) + + if (!store || !key) { + return null + } + + sweepSessionStore(store) + + let raw = readRaw(store, key) + + if (!raw) { + migrateLegacyStore(store) + raw = readRaw(store, key) + } + + if (!raw) { + return null + } + + const snapshot = parseSnapshot(raw) + + if (!snapshot || isExpired(snapshot)) { + discardSnapshot(store, key) + + return null + } + + return snapshot +} + +function removeSnapshot(storedSessionId: string): void { + const store = storage() + const key = sessionStorageKey(storedSessionId) + + if (store && key) { + sweepSessionStore(store) + + // Settling a session before the one-shot migration must clear its legacy + // entry too; otherwise a later read can migrate and resurrect stale state. + // This aggregate parse is terminal-transition work, never a stream write. + discardSnapshot(store, key) + } +} + +function isExpired(entry: InFlightTurnSnapshot, now = Date.now()): boolean { + return now - entry.updatedAt > MAX_AGE_MS +} + function cloneMessages(messages: ChatMessage[]): ChatMessage[] { try { return JSON.parse(JSON.stringify(messages)) as ChatMessage[] @@ -286,7 +603,7 @@ function recoverableTail(messages: ChatMessage[], streamId: null | string): Chat } } - return cloneMessages(visible.slice(start)) + return visible.slice(start) } function normalizeRecoveredTail(tail: ChatMessage[], keepPending: boolean): ChatMessage[] { @@ -470,6 +787,17 @@ export function mergeInFlightMessages( const persistTimers = new Map>() const persistLatest = new Map() +/** @internal Test-only reset for module-scoped throttles and sweep state. */ +export function resetInFlightTurnJournalStateForTests(): void { + for (const timer of persistTimers.values()) { + clearTimeout(timer) + } + + persistTimers.clear() + persistLatest.clear() + sessionStoreSwept = false +} + function writeSnapshot(storedSessionId: string, state: JournalableSessionState): void { const tail = recoverableTail(state.messages, state.streamId) @@ -477,13 +805,63 @@ function writeSnapshot(storedSessionId: string, state: JournalableSessionState): return } - ensureHousekeeping() - saveEntry(storedSessionId, { - messages: tail, + const store = storage() + const key = sessionStorageKey(storedSessionId) + + if (!store || !key) { + return + } + + sweepSessionStore(store, true) + + const messages = boundedMessages(tail) + + if (!messages) { + // Keep the timer write path free of aggregate migration. This tiny invalid + // v2 value suppresses the stale v1 predecessor until read/settle performs + // the one-shot migration and removes it. + tombstoneUnlessRecoverable(store, key) + + return + } + + const raw = serializeSnapshot({ + messages, streamId: state.streamId, turnStartedAt: state.turnStartedAt, updatedAt: Date.now() }) + + if (!raw) { + // Preserve an older bounded snapshot if the newest assistant row alone is + // too large. A tombstone is only needed when there is no recoverable v2 + // value, so stale v1 state cannot be resurrected on a later read. + tombstoneUnlessRecoverable(store, key) + + return + } + + if (!writeRaw(store, key, raw)) { + // A quota failure must not leave an older, misleading snapshot behind, or + // let the stale v1 predecessor be resurrected on a later read. + tombstoneUnlessRecoverable(store, key) + } +} + +function tombstoneUnlessRecoverable(store: Storage, key: string): void { + const previous = readRaw(store, key) + + if (previous) { + const snapshot = parseSnapshot(previous) + + if (snapshot && !isExpired(snapshot)) { + return + } + } + + if (!writeRaw(store, key, DISCARDED_SNAPSHOT_RAW)) { + removeRaw(store, key) + } } /** Persist the running turn's visible tail (throttled), or clear the entry the @@ -527,20 +905,7 @@ export function readInFlightTurnJournal(storedSessionId: null | string): InFligh return null } - ensureHousekeeping() - const entry = loadEntry(storedSessionId) - - if (!entry) { - return null - } - - if (isExpired(entry)) { - removeEntry(storedSessionId) - - return null - } - - return entry + return readSnapshot(storedSessionId) } /** Fold a journaled in-flight tail back onto a restored transcript. A no-op @@ -588,6 +953,6 @@ export function clearInFlightTurnJournal(storedSessionId: null | string): void { } persistLatest.delete(storedSessionId) - ensureHousekeeping() - removeEntry(storedSessionId) + + removeSnapshot(storedSessionId) } diff --git a/apps/desktop/vitest.config.ts b/apps/desktop/vitest.config.ts index dd2f38ff3193..70835d91f03a 100644 --- a/apps/desktop/vitest.config.ts +++ b/apps/desktop/vitest.config.ts @@ -1,4 +1,4 @@ -import type { TestProjectConfiguration } from 'vitest/config'; +import type { TestProjectConfiguration } from 'vitest/config' import { defineConfig } from 'vitest/config' const reactUi: TestProjectConfiguration = { @@ -20,7 +20,8 @@ const electronNative: TestProjectConfiguration = { test: { name: 'electron', environment: 'node', - include: ['electron/**/*.test.ts', 'scripts/**.test.{ts,mjs}'] + include: ['electron/**/*.test.ts', 'scripts/**.test.{ts,mjs}'], + exclude: ['scripts/run-short-session-hang-repro.test.mjs'] } }