From dbd1b2ac509a45e668caae09a763b91257b19d1c Mon Sep 17 00:00:00 2001 From: Eugeniusz Gilewski Date: Mon, 10 Aug 2026 02:11:34 +0200 Subject: [PATCH 1/4] fix(desktop): bound inflight journal persistence (#63047) The desktop journal synchronously read, parsed, cloned, and rewrote one aggregate localStorage value while streamed turns were repainting. Large tool results and multi-session state could therefore block the renderer and leave the app unresponsive, while the existing macOS diagnostic path lacked a real native hide/restore regression check. Store bounded recovery projections under per-session keys, migrate legacy v1 data once, isolate quota and storage failures, and preserve the newest recoverable tail without allowing oversized writes to replace valid state. Add a real Electron/CDP macOS-arm64 A/B harness with native visibility control, renderer heartbeat and Settings/composer/transcript checks, plus focused regressions. Keep bulk tool payloads, diagnostics, and the existing recovery merge behavior out of the hot path. Fixes #63047 --- apps/desktop/package.json | 4 +- .../scripts/run-short-session-hang-repro.mjs | 1702 +++++++++++++++++ .../run-short-session-hang-repro.test.mjs | 171 ++ .../src/app/chat/short-session-hang-repro.tsx | 388 ++++ .../src/lib/inflight-turn-journal.test.ts | 430 ++++- apps/desktop/src/lib/inflight-turn-journal.ts | 551 +++++- apps/desktop/vitest.config.ts | 5 +- 7 files changed, 3124 insertions(+), 127 deletions(-) create mode 100644 apps/desktop/scripts/run-short-session-hang-repro.mjs create mode 100644 apps/desktop/scripts/run-short-session-hang-repro.test.mjs create mode 100644 apps/desktop/src/app/chat/short-session-hang-repro.tsx diff --git a/apps/desktop/package.json b/apps/desktop/package.json index b80c74068b5d..caed49330ad4 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -70,7 +70,9 @@ "check": "npm run check:lint && npm run test:ui && npm run test:desktop:platforms && npm run test:desktop:all", "test:e2e": "npm run build && playwright test e2e/", "test:e2e:visual": "npm run build && WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list", - "test:e2e:update-snapshots": "npm run build && WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list --update-snapshots" + "test:e2e:update-snapshots": "npm run build && WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list --update-snapshots", + "repro:short-session-hang": "node scripts/run-short-session-hang-repro.mjs", + "repro:short-session-hang:test": "node --test scripts/run-short-session-hang-repro.test.mjs" }, "dependencies": { "@assistant-ui/core": "0.2.23", diff --git a/apps/desktop/scripts/run-short-session-hang-repro.mjs b/apps/desktop/scripts/run-short-session-hang-repro.mjs new file mode 100644 index 000000000000..5645bd5e2c81 --- /dev/null +++ b/apps/desktop/scripts/run-short-session-hang-repro.mjs @@ -0,0 +1,1702 @@ +#!/usr/bin/env node + +import { spawn, spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + appendFileSync, + cpSync, + copyFileSync, + createWriteStream, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync +} from 'node:fs' +import http from 'node:http' +import { createRequire } from 'node:module' +import { createServer } from 'node:net' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { finished } from 'node:stream/promises' +import { fileURLToPath } from 'node:url' + +import { CDP, discoverTarget, sleep } from './perf/lib/cdp.mjs' + +const HERE = dirname(fileURLToPath(import.meta.url)) +const DESKTOP_ROOT = resolve(HERE, '..') +const REPO_ROOT = resolve(DESKTOP_ROOT, '..', '..') +const HARNESS_SOURCE = resolve(DESKTOP_ROOT, 'src/app/chat/short-session-hang-repro.tsx') +const UPSTREAM_URL = 'https://github.com/NousResearch/hermes-agent.git' +const DEFAULT_BASELINE = '3651627d88858912e8460e6f949b7125725600c3' +const DEFAULT_CANDIDATE = '3651627d88858912e8460e6f949b7125725600c3' +const FREEZE_MS = 5_000 +const JOURNAL_SEED_TIMEOUT_MS = 30_000 +const STREAM_RESPONSE_TIMEOUT_MS = 30_000 +const STREAM_RESPONSE_EVALUATION_TIMEOUT_MS = 30_000 +const STREAM_PAYLOAD_BYTES = 512 * 1024 +const LEGACY_STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' +const LEGACY_MIGRATION_KEY = 'hermes.desktop.inflightTurnJournal.v2.migrated' +const LEGACY_SEED_ENTRY_COUNT = 15 +const MAX_LEGACY_SEED_BYTES = 1.9 * 1024 * 1024 +// Reserve space for the JSON envelope and per-entry metadata so the generated +// fixture remains below the guard even when the entry count or shape changes. +const LEGACY_SEED_METADATA_BUDGET_BYTES = 16 * 1024 +const LEGACY_SEED_PAYLOAD_CHARS = Math.floor( + (MAX_LEGACY_SEED_BYTES - LEGACY_SEED_METADATA_BUDGET_BYTES) / LEGACY_SEED_ENTRY_COUNT +) +const EXCHANGE_COUNT = 5 +const NATIVE_VISIBILITY_PAUSE_MS = 750 +const EXCHANGE_FREEZE_TIMEOUTS = 9 +const POST_EXCHANGE_FREEZE_TIMEOUTS = 13 +const WATCHDOG_OVERHEAD_MS = 120_000 +// The watchdog is a last-resort guard around the complete five-exchange run. +// Inner timeouts still identify actual renderer hangs; this budget prevents a +// slow but valid stream/response sequence from being misclassified by the old +// fixed 120s outer limit. The overhead covers renderer startup/cleanup and +// native visibility calls not represented by the per-operation timers. +const OUTER_WATCHDOG_MS = + EXCHANGE_COUNT * + (EXCHANGE_FREEZE_TIMEOUTS * FREEZE_MS + + STREAM_RESPONSE_TIMEOUT_MS + + STREAM_RESPONSE_EVALUATION_TIMEOUT_MS + + NATIVE_VISIBILITY_PAUSE_MS) + + POST_EXCHANGE_FREEZE_TIMEOUTS * FREEZE_MS + + 2_000 + + WATCHDOG_OVERHEAD_MS +const SECRET_ENV_RE = /(credential|token|secret|password|(^|_)key($|_)|auth|cookie)/i +const HEX_OBJECT_RE = /^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$/ +const PULL_REF_RE = /^refs\/pull\/[1-9][0-9]*\/(?:head|merge)$/ +const HEAD_REF_RE = /^refs\/heads\/[A-Za-z0-9][A-Za-z0-9._/-]*$/ +const HEAD_COMPONENT_RE = /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/ +const require = createRequire(import.meta.url) + +class ReproductionError extends Error { + name = 'ReproductionError' +} + +function usage() { + console.log(`Usage: node scripts/run-short-session-hang-repro.mjs [options] + +Options: + --baseline baseline ref (default: ${DEFAULT_BASELINE}) + --candidate candidate ref (default: ${DEFAULT_CANDIDATE}) + --repetitions measured repetitions per ref (default: 5) + --output artifact directory (default: short-session-hang-artifacts) + --keep-worktrees retain ephemeral source copies + --dry-run validate refs, lockfile/Electron parity, and print the plan + --help show this help + +Each ref gets one warm-up plus N measured fresh-app runs. Measured A/B order is +counterbalanced. A run is a reproduction only when a renderer/main operation, +heartbeat, or event-loop gap exceeds ${FREEZE_MS}ms, or Electron becomes +unresponsive, loses the renderer, or exits unexpectedly.`) +} + +function parseArgs(argv) { + const out = { + baseline: DEFAULT_BASELINE, + candidate: DEFAULT_CANDIDATE, + repetitions: 5, + output: resolve(process.cwd(), 'short-session-hang-artifacts'), + dryRun: false, + keepWorktrees: false + } + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i] + const next = argv[i + 1] + + if (arg === '--help') { + usage() + process.exit(0) + } else if (arg === '--dry-run') { + out.dryRun = true + } else if (arg === '--keep-worktrees') { + out.keepWorktrees = true + } else if (arg === '--baseline' || arg === '--candidate' || arg === '--repetitions' || arg === '--output') { + if (!next || next.startsWith('--')) { + throw new Error(`${arg} requires a value`) + } + + const key = arg.slice(2) + out[key] = key === 'repetitions' ? Number(next) : key === 'output' ? resolve(next) : next + i += 1 + } else { + throw new Error(`unknown option: ${arg}`) + } + } + + if (!Number.isInteger(out.repetitions) || out.repetitions < 1 || out.repetitions > 20) { + throw new Error('--repetitions must be an integer from 1 to 20') + } + + return out +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd ?? REPO_ROOT, + encoding: 'utf8', + env: options.env ?? process.env, + maxBuffer: 64 * 1024 * 1024 + }) + + if (options.logPath) { + writeFileSync(options.logPath, `${result.stdout ?? ''}${result.stderr ?? ''}`) + } + + if (result.error) { + throw new Error(`${command} ${args.join(' ')} failed to run: ${result.error.message}`) + } + + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(' ')} exited ${result.status ?? `signal ${result.signal}`}:\n${result.stderr || result.stdout}` + ) + } + + return String(result.stdout ?? '').trim() +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex') +} + +function validateRef(ref) { + const headParts = ref.startsWith('refs/heads/') ? ref.slice('refs/heads/'.length).split('/') : [] + const invalidHead = + ref.includes('..') || + headParts.length === 0 || + headParts.some(part => !HEAD_COMPONENT_RE.test(part) || part.endsWith('.lock')) + + if (HEX_OBJECT_RE.test(ref) || PULL_REF_RE.test(ref) || (HEAD_REF_RE.test(ref) && !invalidHead)) { + return ref + } + + throw new Error( + `unsafe ref ${JSON.stringify(ref)}; use a 40/64-digit hex object ID, refs/heads/, or refs/pull//(head|merge)` + ) +} + +function sanitizedEnv(extra = {}) { + const clean = Object.fromEntries(Object.entries(process.env).filter(([name]) => !SECRET_ENV_RE.test(name))) + + return { ...clean, ...extra } +} + +function resolveRef(rawRef) { + const ref = validateRef(rawRef) + const tryResolve = () => { + const result = spawnSync('git', ['rev-parse', '--verify', '--end-of-options', `${ref}^{commit}`], { + cwd: REPO_ROOT, + encoding: 'utf8' + }) + + return result.status === 0 ? result.stdout.trim() : null + } + + const local = HEX_OBJECT_RE.test(ref) ? tryResolve() : null + + if (local) { + return local + } + + run('git', ['fetch', '--no-tags', UPSTREAM_URL, ref], { + env: sanitizedEnv({ GCM_INTERACTIVE: 'never', GIT_TERMINAL_PROMPT: '0' }) + }) + + if (ref.startsWith('refs/')) { + const fetched = run('git', ['rev-parse', '--verify', '--end-of-options', 'FETCH_HEAD^{commit}']) + + return fetched + } + + const resolved = tryResolve() + + if (!resolved) { + throw new Error(`cannot resolve ref ${ref}`) + } + + return resolved +} + +function readTargetMetadata(sha) { + const lock = run('git', ['show', '--end-of-options', `${sha}:package-lock.json`]) + const pyproject = run('git', ['show', '--end-of-options', `${sha}:pyproject.toml`]) + const uvLock = run('git', ['show', '--end-of-options', `${sha}:uv.lock`]) + const pkg = JSON.parse(run('git', ['show', '--end-of-options', `${sha}:apps/desktop/package.json`])) + + return { + electron: pkg.devDependencies?.electron, + electronBuild: pkg.build?.electronVersion, + lockSha256: sha256(lock), + pyprojectSha256: sha256(pyproject), + uvLockSha256: sha256(uvLock) + } +} + +function readJournalContract(sha) { + const source = run('git', ['show', '--end-of-options', `${sha}:apps/desktop/src/lib/inflight-turn-journal.ts`]) + const storageKey = source.match(/const (?:LEGACY_STORAGE_KEY|STORAGE_KEY) = '([^']+)'/)?.[1] ?? null + const migrationKey = source.match(/const LEGACY_MIGRATION_KEY = '([^']+)'/)?.[1] ?? null + const legacyStoreLimit = source.match(/const MAX_LEGACY_STORE_CHARS = (\d+) \* 1024 \* 1024/)?.[1] + + return { + legacyStoreLimit: legacyStoreLimit ? Number(legacyStoreLimit) * 1024 * 1024 : null, + migrationKey, + storageKey + } +} + +function injectHarness(targetRoot) { + const targetHarness = join(targetRoot, 'apps/desktop/src/app/chat/short-session-hang-repro.tsx') + const targetMain = join(targetRoot, 'apps/desktop/src/main.tsx') + const main = readFileSync(targetMain, 'utf8') + + mkdirSync(dirname(targetHarness), { recursive: true }) + copyFileSync(HARNESS_SOURCE, targetHarness) + + if (!main.includes("import('./app/chat/short-session-hang-repro')")) { + writeFileSync( + targetMain, + `${main}\nif (import.meta.env.VITE_SHORT_SESSION_HANG_REPRO === '1') {\n import('./app/chat/short-session-hang-repro')\n}\n` + ) + } +} + +function linkShared(targetRoot, relativePath) { + const source = join(REPO_ROOT, relativePath) + const target = join(targetRoot, relativePath) + + if (existsSync(source) && !existsSync(target)) { + mkdirSync(dirname(target), { recursive: true }) + symlinkSync(source, target, 'dir') + } +} + +function prepareTarget(label, sha, root, output) { + const targetRoot = join(root, label) + run('git', ['worktree', 'add', '--detach', targetRoot, sha]) + + try { + injectHarness(targetRoot) + linkShared(targetRoot, 'node_modules') + linkShared(targetRoot, 'apps/desktop/node_modules') + linkShared(targetRoot, '.venv') + + const targetDesktop = join(targetRoot, 'apps/desktop') + const buildLog = join(output, `${label}-build.log`) + run('npm', ['run', '--prefix', 'apps/desktop', 'build'], { + cwd: targetRoot, + env: sanitizedEnv({ VITE_SHORT_SESSION_HANG_REPRO: '1' }), + logPath: buildLog + }) + + return { journalContract: readJournalContract(sha), label, sha, targetDesktop, targetRoot } + } catch (error) { + try { + run('git', ['worktree', 'remove', '--force', targetRoot]) + } catch { + // The original build error is more useful; cleanup is retried manually. + } + + throw error + } +} + +function startMockInference() { + const reply = `deterministic-tool-heavy-output:${'x'.repeat(STREAM_PAYLOAD_BYTES)}` + let streamingCompletionRequests = 0 + let activeStreamingRequests = 0 + let listening = false + const transportErrors = [] + const requests = [] + const recordTransportError = (source, error) => { + transportErrors.push({ + at: new Date().toISOString(), + message: error instanceof Error ? error.message : String(error), + source + }) + } + const server = http.createServer((request, response) => { + requests.push({ method: request.method, url: request.url ?? null }) + request.on('error', error => recordTransportError('request', error)) + response.on('error', error => recordTransportError('response', error)) + + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: 'short-session-model', object: 'model' }], object: 'list' })) + + return + } + + if (request.method === 'POST' && request.url?.startsWith('/v1/chat/completions')) { + let body = '' + request.on('data', chunk => { + body += String(chunk) + }) + request.on('end', () => { + let stream = false + + try { + stream = JSON.parse(body).stream === true + } catch { + stream = false + } + + if (stream) { + streamingCompletionRequests += 1 + activeStreamingRequests += 1 + let finishedStream = false + const finishStream = () => { + if (finishedStream) return + finishedStream = true + activeStreamingRequests = Math.max(0, activeStreamingRequests - 1) + } + response.once('close', finishStream) + response.writeHead(200, { 'content-type': 'text/event-stream' }) + let offset = 0 + const writeNext = () => { + if (response.destroyed) { + finishStream() + return + } + if (offset >= reply.length) { + response.write( + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: 'stop', index: 0 }], id: 'short-session', object: 'chat.completion.chunk' })}\n\n` + ) + response.end('data: [DONE]\n\n') + finishStream() + return + } + const chunk = reply.slice(offset, offset + 16 * 1024) + offset += chunk.length + response.write( + `data: ${JSON.stringify({ choices: [{ delta: { content: chunk }, finish_reason: null, index: 0 }], id: 'short-session', object: 'chat.completion.chunk' })}\n\n` + ) + setTimeout(writeNext, 10) + } + writeNext() + } else { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [{ finish_reason: 'stop', index: 0, message: { content: reply, role: 'assistant' } }], + id: 'short-session', + object: 'chat.completion' + }) + ) + } + }) + + return + } + + response.writeHead(404, { 'content-type': 'application/json' }) + response.end('{"error":"not found"}') + }) + + return new Promise((resolveStart, reject) => { + server.on('error', error => { + recordTransportError('server', error) + + if (!listening) reject(error) + }) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + + if (!address || typeof address === 'string') { + reject(new Error('mock inference server has no TCP address')) + return + } + + listening = true + resolveStart({ + activeStreamingRequests: () => activeStreamingRequests, + close: () => + new Promise(resolveClose => { + let settled = false + const settle = () => { + if (settled) return + settled = true + resolveClose() + } + const timeout = setTimeout(() => { + server.closeAllConnections?.() + settle() + }, 2_000) + + server.close(() => { + clearTimeout(timeout) + settle() + }) + server.closeAllConnections?.() + }), + requestSummary: () => requests.map(request => ({ ...request })), + streamingCompletionRequests: () => streamingCompletionRequests, + transportErrors: () => transportErrors.map(error => ({ ...error })), + waitForStreaming: async (timeoutMs = 30_000, baseline = 0) => { + const deadline = Date.now() + timeoutMs + while (activeStreamingRequests === 0 && streamingCompletionRequests <= baseline && Date.now() < deadline) { + await sleep(20) + } + if (activeStreamingRequests === 0 && streamingCompletionRequests <= baseline) { + throw new Error('streaming response did not start') + } + }, + url: `http://127.0.0.1:${address.port}` + }) + }) + }) +} +function writeSandboxConfig(home, mockUrl) { + mkdirSync(home, { recursive: true }) + writeFileSync( + join(home, 'config.yaml'), + `model:\n default: short-session-model\n provider: custom:short-session\nauxiliary:\n title_generation:\n enabled: false\nproviders:\n short-session:\n api: ${mockUrl}/v1\n transport: chat_completions\n default_model: short-session-model\n key_env: SHORT_SESSION_API_KEY\n` + ) + writeFileSync(join(home, '.env'), 'SHORT_SESSION_API_KEY=local-diagnostic-only\n') +} + +async function waitFor(cdp, expression, timeoutMs, label, ErrorType = Error) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + try { + if (await cdp.eval(expression)) { + return + } + } catch { + // Renderer is still loading. + } + + await sleep(Math.min(250, Math.max(1, deadline - Date.now()))) + } + + throw new ErrorType(`timed out waiting for ${label}`) +} + +async function screenshot(cdp, path) { + try { + await cdp.send('Page.enable') + const shot = await cdp.send('Page.captureScreenshot', { format: 'png', fromSurface: true }) + writeFileSync(path, Buffer.from(shot.data, 'base64')) + } catch { + // A frozen renderer may not service screenshot capture. + } +} + +function reliablePid(pid) { + return Number.isSafeInteger(pid) && pid > 1 ? pid : null +} + +function processRows() { + const result = spawnSync('ps', ['-axo', 'pid=,ppid=,%cpu=,%mem=,state=,etime=,command='], { encoding: 'utf8' }) + + if (result.error || result.status !== 0) { + const detail = + result.error?.message ?? (String(result.stderr ?? '').trim() || `exit status ${result.status ?? 'unknown'}`) + throw new Error(`process discovery failed: ${detail}`) + } + + return String(result.stdout ?? '') + .split(/\r?\n/) + .map(line => { + const match = line.match(/^\s*(\d+)\s+(\d+)\s+([\d.]+)\s+([\d.]+)\s+(\S+)\s+(\S+)\s+(.*)$/) + + return match + ? { + command: match[7], + cpu: Number(match[3]), + elapsed: match[6], + memory: Number(match[4]), + pid: Number(match[1]), + ppid: Number(match[2]), + state: match[5] + } + : null + }) + .filter(Boolean) +} + +function processTree(rootPid) { + if (!reliablePid(rootPid)) { + return [] + } + + let rows + let discoveryError = null + + try { + rows = processRows() + } catch (error) { + rows = [] + discoveryError = error instanceof Error ? error.message : String(error) + } + + const selected = new Set([rootPid]) + let changed = true + + while (changed) { + changed = false + + for (const row of rows) { + if (!selected.has(row.pid) && selected.has(row.ppid)) { + selected.add(row.pid) + changed = true + } + } + } + + const tree = rows.filter(row => selected.has(row.pid)) + const rootMissing = !tree.some(row => row.pid === rootPid) + + if (rootMissing) { + tree.unshift({ + command: '', + cpu: 0, + elapsed: '', + memory: 0, + pid: rootPid, + ppid: 0, + state: '?', + synthetic: true + }) + } + + if (discoveryError) { + tree.discoveryError = discoveryError + } + + if (rootMissing) { + tree.rootMissing = true + } + + return tree +} + +function redactCommand(command) { + return command + .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+(?::[^\s/@]*)?@/gi, '$1[REDACTED]@') + .replace(/\b([A-Za-z_][A-Za-z0-9_]*(?:TOKEN|SECRET|PASSWORD|KEY|AUTH|COOKIE)[A-Za-z0-9_]*)=\S+/gi, '$1=[REDACTED]') + .replace(/(--?\S*(?:token|secret|password|key|auth|cookie)\S*)(?:=|\s+)\S+/gi, '$1=[REDACTED]') +} + +function processSnapshot(path, rootPid) { + const tree = processTree(rootPid) + const rows = tree.map(row => ({ ...row, command: redactCommand(row.command) })) + writeFileSync( + path, + `${JSON.stringify({ rootPid, rows, discoveryError: tree.discoveryError ?? null, rootMissing: tree.rootMissing ?? false }, null, 2)}\n` + ) + + return rows +} + +function sampleOne(path, pid) { + if (process.platform !== 'darwin' || !reliablePid(pid)) { + return + } + + const result = spawnSync('sample', [String(pid), '5', '1'], { encoding: 'utf8', timeout: 8_000 }) + writeFileSync(path, `${result.stdout ?? ''}${result.stderr ?? ''}`) +} + +function sampleTree(runDir, rootPid, prefix) { + const rows = processTree(rootPid) + sampleOne(join(runDir, `${prefix}-main.sample.txt`), rootPid) + const renderer = rows + .filter(row => row.pid !== rootPid && /(?:^|\s)--type=renderer(?:\s|$)/.test(row.command)) + .sort((a, b) => b.cpu - a.cpu)[0] + + if (renderer) { + sampleOne(join(runDir, `${prefix}-renderer-${renderer.pid}.sample.txt`), renderer.pid) + } +} + +function captureDiagnostics(runDir, rootPid, prefix) { + try { + processSnapshot(join(runDir, `${prefix}-processes.txt`), rootPid) + } catch { + // Diagnostics are best effort and must not replace the original result. + } + + try { + sampleTree(runDir, rootPid, prefix) + } catch { + // Diagnostics are best effort and must not replace the original result. + } +} + +function liveCaptured(captured) { + let current + + try { + current = new Map(processRows().map(row => [row.pid, row])) + } catch (error) { + const live = captured.filter(original => pidIsLive(original.pid)) + live.discoveryError = error instanceof Error ? error.message : String(error) + return live + } + + return captured.filter(original => { + if (original.synthetic) { + return pidIsLive(original.pid) + } + + const row = current.get(original.pid) + + return row && !row.state.startsWith('Z') && row.command === original.command + }) +} + +function pidIsLive(pid) { + try { + process.kill(pid, 0) + return true + } catch (error) { + return error?.code === 'EPERM' + } +} + +async function stopProcessTree(rootPid) { + const captured = processTree(rootPid) + const processDiscoveryErrors = new Set() + + if (captured.discoveryError) { + processDiscoveryErrors.add(captured.discoveryError) + } + + const currentLive = () => { + const live = liveCaptured(captured) + + if (live.discoveryError) { + processDiscoveryErrors.add(live.discoveryError) + } + + return live + } + + for (const { pid } of [...captured].reverse()) { + try { + process.kill(pid, 'SIGTERM') + } catch { + // It already exited. + } + } + + const deadline = Date.now() + 3_000 + + while (Date.now() < deadline && currentLive().length > 0) { + await sleep(100) + } + + const remaining = currentLive() + + for (const { pid } of remaining.reverse()) { + try { + process.kill(pid, 'SIGKILL') + } catch { + // It exited between the liveness check and signal. + } + } + + const killDeadline = Date.now() + 2_000 + + while (Date.now() < killDeadline && currentLive().length > 0) { + await sleep(100) + } + + return { + captured: captured.map(row => row.pid), + processDiscoveryErrors: [...processDiscoveryErrors], + remainingAfterKill: currentLive().map(row => row.pid) + } +} + +function withWatchdog(task, onTimeout) { + let timer + + return Promise.race([ + task, + new Promise((_, reject) => { + timer = setTimeout(() => { + onTimeout() + reject(new ReproductionError(`outer watchdog exceeded ${OUTER_WATCHDOG_MS}ms`)) + }, OUTER_WATCHDOG_MS) + }) + ]).finally(() => clearTimeout(timer)) +} + +function withTimeout(task, timeoutMs, label, ErrorType = Error) { + let timer + + return Promise.race([ + task, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new ErrorType(`${label} exceeded ${timeoutMs}ms`)), timeoutMs) + }) + ]).finally(() => clearTimeout(timer)) +} + +async function waitForResponsive(cdp, expression, timeoutMs, label, evaluationTimeoutMs = FREEZE_MS) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + try { + if ( + await withTimeout(cdp.eval(expression), evaluationTimeoutMs, `${label} renderer evaluation`, ReproductionError) + ) { + return + } + } catch (error) { + if (error instanceof ReproductionError) { + throw error + } + + // Match the initial renderer-readiness polling: CDP can fail transiently while a document is replaced. + } + + await sleep(Math.min(250, Math.max(1, deadline - Date.now()))) + } + + throw new Error(`timed out waiting for ${label} while the renderer remained responsive`) +} + +async function waitForPredicate(predicate, timeoutMs, label) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + if (await predicate()) { + return + } + + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))) + } + + throw new Error(`timed out waiting for ${label}`) +} + +async function verifyInteractiveSurfaces(cdp, timed, measure, label) { + const sentinel = `short-session-sentinel-${label}` + const composerPainted = await timed(`composer.paint.${label}`, async () => { + const focused = await cdp.eval( + `(() => { const el = document.querySelector('[data-slot="composer-rich-input"]'); if (!el || el.contentEditable !== 'true') return false; el.focus(); return true })()` + ) + + if (!focused) { + return false + } + + await cdp.send('Input.insertText', { text: sentinel }) + + return cdp.eval( + `document.querySelector('[data-slot="composer-rich-input"]')?.textContent?.includes(${JSON.stringify(sentinel)}) === true` + ) + }) + + if (!composerPainted) { + throw new Error(`composer did not paint sentinel at ${label}`) + } + + const version = await timed(`version.ipc.${label}`, () => cdp.eval('window.hermesDesktop.getVersion()')) + + if (typeof version?.appVersion !== 'string' || version.appVersion.length === 0) { + throw new Error(`version IPC returned no appVersion at ${label}: ${JSON.stringify(version)}`) + } + + await timed(`about.open.${label}`, () => cdp.eval("location.hash = '#/settings?tab=about'; true")) + await measure(`about.ready.${label}`, () => + waitForResponsive( + cdp, + `document.body.textContent.includes(${JSON.stringify(version.appVersion)}) && !!document.querySelector('button[aria-label]')`, + FREEZE_MS, + `About settings at ${label}` + ) + ) + const aboutClosed = await timed(`about.close.${label}`, () => + cdp.eval( + `(() => { const close = document.querySelector('div[role="presentation"] > div > div:first-child button[aria-label]'); if (!close) return false; close.click(); return true })()` + ) + ) + + if (!aboutClosed) { + throw new Error(`About settings close control unavailable at ${label}`) + } + + await measure(`about.closed.${label}`, () => + waitForResponsive(cdp, "!location.hash.includes('/settings')", FREEZE_MS, `About settings close at ${label}`) + ) + const interactive = await timed(`transcript.interactive.${label}`, () => + cdp.eval( + `(() => { const viewport = document.querySelector('[data-slot="aui_thread-viewport"]'); const row = document.querySelector('[data-message-id]'); if (!viewport || !row) return false; const before = viewport.scrollTop; viewport.scrollTop = Math.min(viewport.scrollHeight, before + 40); viewport.dispatchEvent(new Event('scroll', { bubbles: true })); return getComputedStyle(row).pointerEvents !== 'none' })()` + ) + ) + + if (!interactive) { + throw new Error(`transcript was not interactive at ${label}`) + } + + return { sentinel, version } +} + +function nativeWindowVisibilitySource() { + return [ + 'import AppKit', + 'import CoreGraphics', + 'import Foundation', + 'import Darwin', + 'guard CommandLine.arguments.count >= 3,', + ' let processId = Int32(CommandLine.arguments[1]),', + ' let application = NSRunningApplication(processIdentifier: pid_t(processId)) else {', + ' fputs("target application was not found", stderr)', + ' exit(2)', + '}', + 'let process = pid_t(processId)', + 'let visible = CommandLine.arguments[2] == "visible"', + 'var changed: Bool', + 'if visible {', + ' changed = application.unhide() || application.activate(options: [.activateIgnoringOtherApps, .activateAllWindows])', + '} else {', + ' let _ = application.activate(options: [.activateIgnoringOtherApps, .activateAllWindows])', + ' usleep(100_000)', + ' changed = application.hide()', + ' if !changed, let source = CGEventSource(stateID: .hidSystemState),', + ' let down = CGEvent(keyboardEventSource: source, virtualKey: 4, keyDown: true),', + ' let up = CGEvent(keyboardEventSource: source, virtualKey: 4, keyDown: false) {', + ' down.flags = .maskCommand', + ' up.flags = .maskCommand', + ' down.postToPid(process)', + ' up.postToPid(process)', + ' changed = true', + ' }', + '}', + 'if !changed {', + ' fputs("native visibility request was rejected", stderr)', + ' exit(3)', + '}' + ].join(String.fromCharCode(10)) +} + +function compileNativeWindowVisibilityHelper(helperPath) { + const sourcePath = `${helperPath}.swift` + writeFileSync(sourcePath, nativeWindowVisibilitySource()) + + try { + const result = spawnSync('/usr/bin/swiftc', [sourcePath, '-o', helperPath], { + encoding: 'utf8', + timeout: 30_000 + }) + + if (result.status !== 0) { + const detail = result.error?.message || result.stderr || result.stdout || String(result.status) + throw new Error(`native macOS visibility helper compilation failed: ${detail}`) + } + } finally { + rmSync(sourcePath, { force: true }) + } + + return helperPath +} + +function setNativeWindowVisibility(helperPath, pid, visible) { + if (process.platform !== 'darwin' || !helperPath || !Number.isSafeInteger(pid) || pid <= 1) { + throw new Error(`native macOS visibility requires a helper and valid app pid, got ${pid}`) + } + + const result = spawnSync(helperPath, [String(pid), visible ? 'visible' : 'hidden'], { + encoding: 'utf8', + timeout: 10_000 + }) + + if (result.status !== 0) { + const detail = result.error?.message || result.stderr || result.stdout || String(result.status) + throw new Error(`native macOS visibility change failed: ${detail}`) + } +} + +async function runRealChatChecks(cdp, timed, measure, mock, runDir, appPid, nativeVisibilityHelper, getOperationCount) { + const requestCountBefore = mock.streamingCompletionRequests() + + for (let exchange = 1; exchange <= EXCHANGE_COUNT; exchange += 1) { + const streamingRequestsBefore = mock.streamingCompletionRequests() + const beforeAssistant = await timed(`real-chat.assistant-count.${exchange}`, () => + cdp.eval( + `document.querySelectorAll('[data-slot="aui_assistant-message-root"]:not([data-streaming="true"])').length` + ) + ) + const composer = await timed(`real-chat.composer-focus.${exchange}`, () => + cdp.eval( + `(() => { const el = document.querySelector('[data-slot="composer-rich-input"]'); if (!el || el.contentEditable !== 'true') return false; el.focus(); return true })()` + ) + ) + + if (!composer) { + throw new Error(`real chat composer unavailable at exchange ${exchange}`) + } + + const prompt = `Deterministic real chat exchange ${exchange}` + await timed(`real-chat.insert.${exchange}`, () => cdp.send('Input.insertText', { text: prompt })) + const inserted = await timed(`real-chat.inserted.${exchange}`, () => + cdp.eval( + `document.querySelector('[data-slot="composer-rich-input"]')?.textContent?.includes(${JSON.stringify(prompt)}) === true` + ) + ) + + if (!inserted) { + throw new Error(`real chat prompt did not reach the composer at exchange ${exchange}`) + } + + await measure(`real-chat.submit-ready.${exchange}`, () => + waitForResponsive( + cdp, + `!!document.querySelector('[data-slot="composer-root"] button[type="submit"]:not(:disabled)')`, + FREEZE_MS, + `real chat submit control ${exchange}` + ) + ) + const submitted = await timed(`real-chat.submit.${exchange}`, () => + cdp.eval( + `(() => { const button = document.querySelector('[data-slot="composer-root"] button[type="submit"]:not(:disabled)'); if (!button) return false; window.setTimeout(() => button.click(), 0); return true })()` + ) + ) + + if (!submitted) { + throw new Error(`real chat submit control unavailable at exchange ${exchange}`) + } + + try { + await measure(`real-chat.stream-start.${exchange}`, () => mock.waitForStreaming(30_000, streamingRequestsBefore)) + } catch (error) { + const state = await timed(`real-chat.dispatch-probe.${exchange}`, () => + cdp.eval(`({ + assistantMessages: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length, + assistantText: document.querySelector('[data-slot="aui_assistant-message-root"]')?.textContent ?? null, + composerText: document.querySelector('[data-slot="composer-rich-input"]')?.textContent ?? null, + harness: window.__SHORT_SESSION_HANG_REPRO__.summary(), + submitDisabled: document.querySelector('[data-slot="composer-root"] button[type="submit"]')?.disabled ?? null, + userMessages: document.querySelectorAll('[data-slot="aui_user-message-root"]').length + })`) + ) + + throw new Error( + `${error instanceof Error ? error.message : String(error)}; mock=${JSON.stringify(mock.requestSummary())}; dispatch probe responded with state: ${JSON.stringify(state)}` + ) + } + await measure(`real-chat.native-hide.${exchange}`, () => + setNativeWindowVisibility(nativeVisibilityHelper, appPid, false) + ) + await sleep(NATIVE_VISIBILITY_PAUSE_MS) + await measure(`real-chat.native-restore.${exchange}`, () => + setNativeWindowVisibility(nativeVisibilityHelper, appPid, true) + ) + + try { + await measure(`real-chat.mock-request.${exchange}`, () => + waitForPredicate( + () => mock.streamingCompletionRequests() >= requestCountBefore + exchange, + FREEZE_MS, + `mock inference request ${exchange}` + ) + ) + } catch (error) { + const state = await timed(`real-chat.dispatch-probe.${exchange}`, () => + cdp.eval(`({ + assistantMessages: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length, + composerText: document.querySelector('[data-slot="composer-rich-input"]')?.textContent ?? null, + harness: window.__SHORT_SESSION_HANG_REPRO__.summary(), + userMessages: document.querySelectorAll('[data-slot="aui_user-message-root"]').length + })`) + ) + + throw new Error( + `${error instanceof Error ? error.message : String(error)}; dispatch probe responded with state: ${JSON.stringify(state)}` + ) + } + + await measure(`real-chat.assistant-response.${exchange}`, () => + waitForResponsive( + cdp, + `document.querySelectorAll('[data-slot="aui_assistant-message-root"]:not([data-streaming="true"])').length > ${beforeAssistant}`, + STREAM_RESPONSE_TIMEOUT_MS, + `real assistant response ${exchange}`, + STREAM_RESPONSE_EVALUATION_TIMEOUT_MS + ) + ) + + if (mock.streamingCompletionRequests() < requestCountBefore + exchange) { + throw new Error(`mock inference request count did not advance for exchange ${exchange}`) + } + } + + const requestDelta = mock.streamingCompletionRequests() - requestCountBefore + + if (requestDelta !== EXCHANGE_COUNT) { + throw new Error(`expected exactly ${EXCHANGE_COUNT} mock completion requests, observed ${requestDelta}`) + } + + const streamHeartbeat = await timed('renderer.heartbeat.stream', () => + cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.summary()') + ) + await timed('renderer.heartbeat.checkpoint', () => cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.checkpoint()')) + const postStreamOperationStart = getOperationCount() + const surfaces = await verifyInteractiveSurfaces(cdp, timed, measure, 'real-chat-exchange-5') + await withTimeout(screenshot(cdp, join(runDir, 'real-chat-exchange-5.png')), 2_000, 'real chat screenshot').catch( + () => {} + ) + + return { + assistantResponses: EXCHANGE_COUNT, + exchanges: EXCHANGE_COUNT, + messageRecords: EXCHANGE_COUNT * 2, + mockCompletionRequests: requestDelta, + postStreamOperationStart, + surfaces, + streamMaxGapMs: Number(streamHeartbeat.maxGapMs || 0) + } +} + +async function runRendererChecks(cdp, label, runDir, mock, appPid, journalContract) { + const operations = [] + const measure = async (name, body) => { + const started = performance.now() + const value = await Promise.resolve().then(body) + const latencyMs = performance.now() - started + operations.push({ name, latencyMs }) + + return value + } + const timed = (name, body) => + measure(name, () => withTimeout(Promise.resolve().then(body), FREEZE_MS, name, ReproductionError)) + + await cdp.send('Runtime.enable') + await cdp.send('Profiler.enable') + await cdp.send('Profiler.start') + + if (journalContract.storageKey !== LEGACY_STORAGE_KEY) { + throw new Error( + `journal storage-key contract mismatch: target=${journalContract.storageKey ?? 'missing'} harness=${LEGACY_STORAGE_KEY}` + ) + } + + if (journalContract.migrationKey !== null && journalContract.migrationKey !== LEGACY_MIGRATION_KEY) { + throw new Error( + `journal migration-key contract mismatch: target=${journalContract.migrationKey} harness=${LEGACY_MIGRATION_KEY}` + ) + } + + const checkpoints = [] + const fixtureManifest = null + const nativeVisibilityHelper = compileNativeWindowVisibilityHelper(join(runDir, 'native-window-visibility')) + const journalSeed = await measure('journal.seed.legacy', () => + withTimeout( + cdp.eval(`(() => { + const payload = 'j'.repeat(${LEGACY_SEED_PAYLOAD_CHARS}) + const entries = Object.fromEntries(Array.from({ length: ${LEGACY_SEED_ENTRY_COUNT} }, (_, index) => { + const assistantId = 'legacy-seed-a-' + index + return [ + 'legacy-seed-' + index, + { + messages: [ + { id: 'legacy-seed-u-' + index, role: 'user', parts: [{ type: 'text', text: 'legacy seed prompt ' + index }] }, + { id: assistantId, role: 'assistant', parts: [{ type: 'text', text: payload }], pending: true } + ], + streamId: assistantId, + turnStartedAt: Date.now(), + updatedAt: Date.now() + } + ] + })) + const raw = JSON.stringify({ entries, version: 1 }) + localStorage.removeItem(${JSON.stringify(LEGACY_MIGRATION_KEY)}) + localStorage.setItem(${JSON.stringify(LEGACY_STORAGE_KEY)}, raw) + return { bytes: raw.length, entries: Object.keys(entries).length } + })()`), + JOURNAL_SEED_TIMEOUT_MS, + 'journal.seed.legacy' + ) + ) + + if (journalSeed.bytes >= MAX_LEGACY_SEED_BYTES) { + throw new Error(`legacy journal seed exceeded migration limit: ${journalSeed.bytes} bytes`) + } + + if (journalContract.legacyStoreLimit !== null && journalSeed.bytes >= journalContract.legacyStoreLimit) { + throw new Error( + `legacy journal seed exceeds target migration limit: seed=${journalSeed.bytes} target=${journalContract.legacyStoreLimit}` + ) + } + + await timed('harness.reset', () => cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.reset()')) + const realChat = await runRealChatChecks( + cdp, + timed, + measure, + mock, + runDir, + appPid, + nativeVisibilityHelper, + () => operations.length + ) + + const heartbeat = await timed('renderer.heartbeat.summary', () => + cdp.eval('window.__SHORT_SESSION_HANG_REPRO__.summary()') + ) + const profile = await cdp.send('Profiler.stop') + writeFileSync(join(runDir, 'renderer.cpuprofile'), JSON.stringify(profile.profile)) + + const maxOperationMs = operations + .slice(realChat.postStreamOperationStart) + .reduce((max, operation) => Math.max(max, operation.latencyMs), 0) + const maxGapMs = Number(heartbeat.maxGapMs || 0) + const reproduced = maxOperationMs > FREEZE_MS || maxGapMs > FREEZE_MS + + return { + checkpoints, + fixtureManifest, + hardFailure: reproduced, + maxGapMs, + maxOperationMs, + operations, + outcome: reproduced ? 'reproduced' : 'not-reproduced', + journalSeed, + realChat, + syntheticScenario: 'disabled; real-chat-only' + } +} + +async function withTemporarySandbox(label, body) { + const sandbox = mkdtempSync(join(tmpdir(), `hermes-short-session-${label}-`)) + + try { + return await body(sandbox) + } finally { + rmSync(sandbox, { force: true, recursive: true }) + } +} + +async function allocateDebugPort() { + const server = createServer() + + try { + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + + const address = server.address() + + if (!address || typeof address === 'string') { + throw new Error('temporary debug-port allocation returned no numeric address') + } + + return address.port + } finally { + if (server.listening) { + await new Promise(resolve => server.close(() => resolve())) + } + } +} + +function resultForError(error) { + const reproduced = error instanceof ReproductionError + + return { + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + hardFailure: true, + lifecycleSignals: [], + maxGapMs: null, + maxOperationMs: null, + operations: [], + outcome: reproduced ? 'reproduced' : 'harness-error' + } +} + +async function executeRun(target, index, warmup, mock, output) { + return withTemporarySandbox(target.label, sandbox => + executeRunInSandbox(target, index, warmup, mock, output, sandbox) + ) +} + +function runDirForAttempt(output, label, index, warmup, attempt = 1) { + const base = join(output, label, warmup ? 'warmup' : `run-${index + 1}`) + + return attempt > 1 ? `${base}-attempt-${attempt}` : base +} + +function promoteAttemptArtifacts(output, label, index, warmup, attempt) { + if (attempt <= 1) { + return + } + + cpSync(runDirForAttempt(output, label, index, warmup, attempt), runDirForAttempt(output, label, index, warmup), { + force: true, + recursive: true + }) +} + +async function executeRunInSandboxAttempt(target, index, warmup, mock, output, sandbox, attempt) { + const runDir = runDirForAttempt(output, target.label, index, warmup, attempt) + const hermesHome = join(sandbox, 'hermes-home') + const userData = join(sandbox, 'electron-user-data') + const desktopLog = join(hermesHome, 'logs', 'desktop.log') + const stdoutPath = join(runDir, 'electron.stdout.log') + const stderrPath = join(runDir, 'electron.stderr.log') + const eventsPath = join(runDir, 'events.jsonl') + const port = await allocateDebugPort() + + mkdirSync(runDir, { recursive: true }) + mkdirSync(userData, { recursive: true }) + writeSandboxConfig(hermesHome, mock.url) + + const electron = require('electron') + const stdoutLog = createWriteStream(stdoutPath) + const stderrLog = createWriteStream(stderrPath) + const stdoutFlushed = finished(stdoutLog).then( + () => null, + error => error + ) + const stderrFlushed = finished(stderrLog).then( + () => null, + error => error + ) + const child = spawn( + electron, + [target.targetDesktop, `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`], + { + cwd: target.targetDesktop, + env: sanitizedEnv({ + HERMES_DESKTOP_APP_NAME: `HermesShortSession-${target.label}-${process.pid}-${index}-${warmup ? 'w' : 'm'}-${attempt}`, + HERMES_DESKTOP_HERMES_ROOT: target.targetRoot, + HERMES_DESKTOP_IGNORE_EXISTING: '1', + HERMES_DESKTOP_USER_DATA_DIR: userData, + HERMES_HOME: hermesHome, + SHORT_SESSION_API_KEY: 'local-diagnostic-only' + }), + stdio: ['ignore', 'pipe', 'pipe'] + } + ) + + let exited = null + let spawnFailed = false + let stopping = false + let resolveSpawn + let rejectSpawn + const spawnReady = new Promise((resolve, reject) => { + resolveSpawn = resolve + rejectSpawn = reject + }) + child.once('spawn', () => resolveSpawn()) + child.once('error', error => { + spawnFailed = true + exited = exited ?? { code: null, signal: null } + appendFileSync( + eventsPath, + `${JSON.stringify({ at: new Date().toISOString(), error: error.message, type: 'spawn-error' })}\n` + ) + rejectSpawn(error) + }) + if (child.stdout) { + child.stdout.pipe(stdoutLog) + } else { + stdoutLog.end() + } + + if (child.stderr) { + child.stderr.pipe(stderrLog) + } else { + stderrLog.end() + } + child.once('exit', (code, signal) => { + exited = { code, signal } + appendFileSync( + eventsPath, + `${JSON.stringify({ at: new Date().toISOString(), code, signal, type: stopping ? 'teardown-exit' : 'unexpected-exit' })}\n` + ) + }) + + let cdp + let result + + try { + await withTimeout(spawnReady, FREEZE_MS, 'Electron spawn') + const targetInfo = await withTimeout(discoverTarget({ port, timeoutMs: 60_000 }), 65_000, 'CDP target discovery') + cdp = await withTimeout(CDP.open(targetInfo.webSocketDebuggerUrl), 10_000, 'CDP connection') + await waitFor(cdp, '!!window.__SHORT_SESSION_HANG_REPRO__', 60_000, 'short-session renderer harness') + await waitFor( + cdp, + `document.querySelector('[data-slot="composer-rich-input"]')?.contentEditable === 'true'`, + 60_000, + 'interactive composer' + ) + + result = await withWatchdog( + runRendererChecks(cdp, `${target.label}-${index + 1}`, runDir, mock, child.pid, target.journalContract), + () => { + captureDiagnostics(runDir, child.pid, 'hard-timeout') + } + ) + } catch (error) { + result = resultForError(error) + captureDiagnostics(runDir, child.pid, 'failure') + if (cdp) { + await withTimeout(screenshot(cdp, join(runDir, 'failure.png')), 2_000, 'failure screenshot').catch(() => {}) + } + } finally { + cdp?.close() + const logText = existsSync(desktopLog) ? readFileSync(desktopLog, 'utf8') : '' + const lifecycleSignals = logText + .split(/\r?\n/) + .filter(line => /webContents became unresponsive|render-process-gone/i.test(line)) + const lifecycleReproduced = lifecycleSignals.length > 0 || Boolean(exited && !spawnFailed) + + result = { ...(result ?? resultForError(new Error('run produced no result'))), lifecycleSignals } + + if (lifecycleReproduced) { + result.hardFailure = true + result.outcome = 'reproduced' + } + + stopping = true + const teardown = await stopProcessTree(child.pid) + const logFlushErrors = await withTimeout( + Promise.all([stdoutFlushed, stderrFlushed]), + FREEZE_MS, + 'Electron log flush' + ).catch(error => [error]) + writeFileSync(join(runDir, 'teardown.json'), `${JSON.stringify(teardown, null, 2)}\n`) + + if (existsSync(desktopLog)) { + copyFileSync(desktopLog, join(runDir, 'desktop.log')) + } + + if (teardown.remainingAfterKill.length > 0) { + result = { + ...(result ?? {}), + error: `${result?.error ? `${result.error}\n` : ''}process teardown leaked PIDs ${teardown.remainingAfterKill.join(', ')}`, + hardFailure: true, + outcome: result?.outcome === 'reproduced' ? 'reproduced' : 'harness-error' + } + } + + if (teardown.processDiscoveryErrors.length > 0) { + result = { + ...(result ?? {}), + error: `${result?.error ? `${result.error}\n` : ''}${teardown.processDiscoveryErrors.join('\n')}`, + hardFailure: true, + outcome: result?.outcome === 'reproduced' ? 'reproduced' : 'harness-error' + } + } + + const logFlushError = logFlushErrors.find(Boolean) + + if (logFlushError) { + result = { + ...(result ?? {}), + error: `${result?.error ? `${result.error}\n` : ''}Electron log flush failed: ${logFlushError instanceof Error ? logFlushError.message : String(logFlushError)}`, + hardFailure: true, + outcome: result?.outcome === 'reproduced' ? 'reproduced' : 'harness-error' + } + } + } + + writeFileSync(join(runDir, 'result.json'), `${JSON.stringify(result, null, 2)}\n`) + appendFileSync(eventsPath, `${JSON.stringify({ at: new Date().toISOString(), result, type: 'run-complete' })}\n`) + + return result +} + +function isDebugStartupFailure(result) { + return !result.lifecycleSignals?.length && /CDP (?:target discovery|connection)/i.test(result.error ?? '') +} + +async function executeRunInSandbox(target, index, warmup, mock, output, sandbox) { + for (let attempt = 1; attempt <= 2; attempt += 1) { + const result = await executeRunInSandboxAttempt(target, index, warmup, mock, output, sandbox, attempt) + + if (attempt === 1 && isDebugStartupFailure(result)) { + rmSync(join(sandbox, 'electron-user-data'), { force: true, recursive: true }) + continue + } + + promoteAttemptArtifacts(output, target.label, index, warmup, attempt) + + return result + } + + throw new Error('unreachable debug startup retry state') +} + +function classify(results, warmup) { + const invalid = [warmup, ...results].some(result => result.outcome === 'harness-error') + const reproduced = results.filter(result => result.outcome === 'reproduced').length + const reproducedThreshold = Math.ceil(results.length * 0.8) + + return { + classification: invalid + ? 'invalid' + : reproduced >= reproducedThreshold + ? 'reproduced' + : reproduced === 0 + ? 'not-reproduced' + : 'intermittent', + invalid, + reproduced, + reproducedThreshold, + total: results.length + } +} + +function pairedSoftSignal(baseline, candidate) { + if (baseline.length === 0 || candidate.length === 0) { + return { material: false, materialPairs: 0, materialThreshold: 0, reason: 'insufficient-runs', thresholdPct: 30 } + } + + if ([...baseline, ...candidate].some(result => result.outcome !== 'not-reproduced')) { + return { material: false, materialPairs: 0, materialThreshold: 0, reason: 'hard-or-invalid-run', thresholdPct: 30 } + } + + let materialPairs = 0 + + for (let i = 0; i < Math.min(baseline.length, candidate.length); i += 1) { + const base = Math.max(1, baseline[i].maxGapMs || 0, baseline[i].maxOperationMs || 0) + const next = Math.max(candidate[i].maxGapMs || 0, candidate[i].maxOperationMs || 0) + + if (next >= base * 1.3) { + materialPairs += 1 + } + } + + const materialThreshold = Math.ceil(Math.min(baseline.length, candidate.length) * 0.8) + + return { material: materialPairs >= materialThreshold, materialPairs, materialThreshold, thresholdPct: 30 } +} + +function validateArtifactBundle(output, repetitions) { + const required = ['environment.json', 'summary.json', 'baseline-build.log', 'candidate-build.log'] + + for (const label of ['baseline', 'candidate']) { + for (const runName of ['warmup', ...Array.from({ length: repetitions }, (_, index) => `run-${index + 1}`)]) { + for (const artifact of ['events.jsonl', 'result.json', 'teardown.json']) { + required.push(join(label, runName, artifact)) + } + } + } + + for (const relativePath of required) { + const path = join(output, relativePath) + + if (!existsSync(path) || readFileSync(path).byteLength === 0) { + throw new Error(`missing or empty required diagnostic artifact: ${relativePath}`) + } + } + + const summary = JSON.parse(readFileSync(join(output, 'summary.json'), 'utf8')) + + validateSummary(summary, repetitions) +} + +function validateSummary(summary, repetitions) { + if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 20) { + throw new Error('invalid summary repetitions') + } + + let invalid = false + + for (const label of ['baseline', 'candidate']) { + if (!summary[label]?.warmup || summary[label].runs?.length !== repetitions) { + throw new Error(`invalid ${label} summary shape`) + } + + for (const result of [summary[label].warmup, ...summary[label].runs]) { + if (!['harness-error', 'not-reproduced', 'reproduced'].includes(result.outcome)) { + throw new Error(`invalid ${label} run outcome`) + } + + const expectedHardFailure = result.outcome !== 'not-reproduced' + + if (result.hardFailure !== expectedHardFailure) { + throw new Error(`inconsistent ${label} run outcome and hardFailure`) + } + } + + const derived = classify(summary[label].runs, summary[label].warmup) + + for (const field of ['classification', 'invalid', 'reproduced', 'reproducedThreshold', 'total']) { + if (summary[label][field] !== derived[field]) { + throw new Error(`inconsistent ${label} summary ${field}`) + } + } + + invalid ||= derived.invalid + } + + if (summary.invalid !== invalid) { + throw new Error('inconsistent summary invalid state') + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)) + + if (!existsSync(HARNESS_SOURCE)) { + throw new Error(`renderer harness missing: ${HARNESS_SOURCE}`) + } + + validateRef(options.baseline) + validateRef(options.candidate) + const baselineSha = resolveRef(options.baseline) + const candidateSha = resolveRef(options.candidate) + const mergeBase = run('git', ['merge-base', '--', baselineSha, candidateSha]) + const ancestor = spawnSync('git', ['merge-base', '--is-ancestor', '--', baselineSha, candidateSha], { + cwd: REPO_ROOT, + encoding: 'utf8' + }) + + if (mergeBase !== baselineSha || ancestor.status !== 0) { + throw new Error( + `baseline must be the exact merge-base and an ancestor of candidate; baseline=${baselineSha} merge-base=${mergeBase} candidate=${candidateSha}` + ) + } + + const baselineMetadata = readTargetMetadata(baselineSha) + const candidateMetadata = readTargetMetadata(candidateSha) + const harnessHead = run('git', ['rev-parse', '--verify', '--end-of-options', 'HEAD^{commit}']) + const harnessMetadata = readTargetMetadata(harnessHead) + + if ( + JSON.stringify(baselineMetadata) !== JSON.stringify(candidateMetadata) || + JSON.stringify(baselineMetadata) !== JSON.stringify(harnessMetadata) + ) { + throw new Error( + `shared dependency mismatch; harness checkout, baseline, and candidate Electron/lockfile metadata must match:\nharness ${JSON.stringify(harnessMetadata)}\nbaseline ${JSON.stringify(baselineMetadata)}\ncandidate ${JSON.stringify(candidateMetadata)}` + ) + } + + const plan = { + baseline: { ref: options.baseline, sha: baselineSha }, + candidate: { ref: options.candidate, sha: candidateSha }, + harness: { head: harnessHead, metadata: harnessMetadata }, + harnessSha256: sha256(readFileSync(HARNESS_SOURCE)), + mergeBase, + metadata: baselineMetadata, + repetitions: options.repetitions, + runner: { arch: process.arch, platform: process.platform, versions: process.versions } + } + + if (options.dryRun) { + console.log(JSON.stringify(plan, null, 2)) + + return + } + + if (process.platform !== 'darwin' || process.arch !== 'arm64') { + throw new Error( + `the short-session hang diagnostic requires macOS arm64; got ${process.platform}-${process.arch} (use --dry-run for preflight)` + ) + } + + mkdirSync(options.output, { recursive: true }) + writeFileSync(join(options.output, 'environment.json'), `${JSON.stringify(plan, null, 2)}\n`) + + const ephemeralRoot = mkdtempSync(join(tmpdir(), 'hermes-short-session-ab-')) + const prepared = [] + const mock = await startMockInference() + + try { + prepared.push(prepareTarget('baseline', baselineSha, ephemeralRoot, options.output)) + prepared.push(prepareTarget('candidate', candidateSha, ephemeralRoot, options.output)) + const byLabel = Object.fromEntries(prepared.map(target => [target.label, target])) + + const warmups = {} + + for (const label of ['baseline', 'candidate']) { + warmups[label] = await executeRun(byLabel[label], 0, true, mock, options.output) + } + + const measured = { baseline: [], candidate: [] } + + for (let index = 0; index < options.repetitions; index += 1) { + const order = index % 2 === 0 ? ['baseline', 'candidate'] : ['candidate', 'baseline'] + + for (const label of order) { + measured[label].push(await executeRun(byLabel[label], index, false, mock, options.output)) + } + } + + const baselineClassification = classify(measured.baseline, warmups.baseline) + const candidateClassification = classify(measured.candidate, warmups.candidate) + const invalid = baselineClassification.invalid || candidateClassification.invalid + const summary = { + ...plan, + invalid, + baseline: { ...plan.baseline, ...baselineClassification, runs: measured.baseline, warmup: warmups.baseline }, + candidate: { + ...plan.candidate, + ...candidateClassification, + runs: measured.candidate, + warmup: warmups.candidate + }, + softSignal: invalid + ? { material: false, materialPairs: 0, materialThreshold: 0, reason: 'invalid-run', thresholdPct: 30 } + : pairedSoftSignal(measured.baseline, measured.candidate), + mockRequests: mock.requestSummary(), + mockTransportErrors: mock.transportErrors() + } + writeFileSync(join(options.output, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`) + validateArtifactBundle(options.output, options.repetitions) + console.log(JSON.stringify(summary, null, 2)) + + // A warm-up reproduction can be the one-shot v1 migration or first-paint + // cost. It is retained in the summary, but only a warm-up harness error or + // a measured hard failure makes the A/B job fail. + if ( + warmups.baseline.outcome === 'harness-error' || + warmups.candidate.outcome === 'harness-error' || + measured.baseline.some(result => result.hardFailure) || + measured.candidate.some(result => result.hardFailure) + ) { + process.exitCode = 1 + } + } finally { + await mock.close() + + if (!options.keepWorktrees) { + for (const target of prepared.reverse()) { + try { + run('git', ['worktree', 'remove', '--force', target.targetRoot]) + } catch (error) { + console.error(`warning: failed to remove ${target.targetRoot}: ${error.message}`) + } + } + + rmSync(ephemeralRoot, { force: true, recursive: true }) + } else { + console.log(`kept ephemeral worktrees at ${ephemeralRoot}`) + } + } +} + +if (resolve(process.argv[1] ?? '') === fileURLToPath(import.meta.url)) { + main().catch(error => { + console.error(error instanceof Error ? (error.stack ?? error.message) : String(error)) + process.exitCode = 2 + }) +} + +export { + ReproductionError, + classify, + pairedSoftSignal, + resultForError, + validateArtifactBundle, + validateSummary, + waitFor, + waitForPredicate, + waitForResponsive, + withTemporarySandbox, + withTimeout +} diff --git a/apps/desktop/scripts/run-short-session-hang-repro.test.mjs b/apps/desktop/scripts/run-short-session-hang-repro.test.mjs new file mode 100644 index 000000000000..95526c16d2d1 --- /dev/null +++ b/apps/desktop/scripts/run-short-session-hang-repro.test.mjs @@ -0,0 +1,171 @@ +import assert from 'node:assert/strict' +import { existsSync, writeFileSync } from 'node:fs' +import test from 'node:test' +import { join } from 'node:path' + +import { + ReproductionError, + classify, + pairedSoftSignal, + resultForError, + validateSummary, + waitFor, + waitForPredicate, + waitForResponsive, + withTemporarySandbox, + withTimeout +} from './run-short-session-hang-repro.mjs' + +const result = (outcome, latency = 10) => ({ + hardFailure: outcome !== 'not-reproduced', + maxGapMs: latency, + maxOperationMs: latency, + outcome +}) + +test('separates harness errors from reproduction timeouts', () => { + const harness = resultForError(new Error('fixture mismatch')) + const reproduced = resultForError(new ReproductionError('renderer operation exceeded 5000ms')) + + assert.equal(harness.outcome, 'harness-error') + assert.equal(harness.maxGapMs, null) + assert.equal(harness.maxOperationMs, null) + assert.equal(reproduced.outcome, 'reproduced') +}) + +test('preserves timeout semantics through the actual nested helpers', async () => { + const immediate = await withTimeout(Promise.reject(new Error('precondition')), 30, 'outer', ReproductionError).catch( + error => error + ) + assert.equal(resultForError(immediate).outcome, 'harness-error') + + const stalledCdp = { eval: async () => false } + const inner = await withTimeout( + waitFor(stalledCdp, 'false', 10, 'inner product operation', ReproductionError), + 50, + 'outer product operation', + ReproductionError + ).catch(error => error) + assert.ok(inner instanceof ReproductionError) + assert.match(inner.message, /inner product operation/) + + const outer = await withTimeout( + waitFor(stalledCdp, 'false', 50, 'inner product operation', ReproductionError), + 10, + 'outer product operation', + ReproductionError + ).catch(error => error) + assert.ok(outer instanceof ReproductionError) + assert.match(outer.message, /outer product operation/) + + const nearDeadline = await withTimeout( + new Promise(resolve => setTimeout(() => resolve('responsive'), 20)), + 50, + 'responsive operation', + ReproductionError + ) + assert.equal(nearDeadline, 'responsive') +}) + +test('distinguishes a responsive false condition from a stalled renderer evaluation', { timeout: 2_000 }, async () => { + let transientEvaluations = 0 + await waitForResponsive( + { + eval: async () => { + transientEvaluations += 1 + + if (transientEvaluations === 1) { + throw new Error('execution context was destroyed') + } + + return true + } + }, + 'true', + 500, + 'transient condition', + 50 + ) + assert.equal(transientEvaluations, 2) + + await assert.rejects( + waitForResponsive({ eval: async () => false }, 'false', 10, 'responsive condition', 1_000), + error => !(error instanceof ReproductionError) && /renderer remained responsive/.test(error.message) + ) + await assert.rejects( + waitForResponsive({ eval: () => new Promise(() => {}) }, 'false', 50, 'stalled condition', 10), + ReproductionError + ) + await assert.rejects( + waitForPredicate(() => false, 10, 'provider request'), + /provider request/ + ) +}) + +test('invalidates a target when warmup or measured runs have harness errors', () => { + const passing = Array.from({ length: 5 }, () => result('not-reproduced')) + + assert.equal(classify(passing, result('harness-error')).classification, 'invalid') + assert.equal( + classify([result('harness-error'), ...passing.slice(1)], result('not-reproduced')).classification, + 'invalid' + ) + assert.equal(classify(passing, result('not-reproduced')).classification, 'not-reproduced') + assert.equal( + classify( + [result('reproduced'), result('reproduced'), result('reproduced'), result('reproduced'), passing[0]], + passing[0] + ).classification, + 'reproduced' + ) +}) + +test('suppresses soft-signal comparisons when a run is invalid or reproduced', () => { + const passing = Array.from({ length: 5 }, () => result('not-reproduced', 10)) + + assert.equal(pairedSoftSignal([], passing).reason, 'insufficient-runs') + assert.equal(pairedSoftSignal(passing, []).reason, 'insufficient-runs') + assert.equal(pairedSoftSignal(passing, passing).reason, undefined) + assert.equal(pairedSoftSignal([result('harness-error'), ...passing.slice(1)], passing).reason, 'hard-or-invalid-run') + assert.equal(pairedSoftSignal([result('reproduced'), ...passing.slice(1)], passing).reason, 'hard-or-invalid-run') +}) + +test('rejects contradictory summary semantics', () => { + const passing = Array.from({ length: 5 }, () => result('not-reproduced')) + const classification = classify(passing, result('not-reproduced')) + const target = { ...classification, runs: passing, warmup: result('not-reproduced') } + const summary = { baseline: target, candidate: target, invalid: false } + + assert.doesNotThrow(() => validateSummary(summary, 5)) + assert.throws( + () => validateSummary({ ...summary, baseline: { ...target, classification: 'reproduced' } }, 5), + /inconsistent baseline summary classification/ + ) + assert.throws( + () => + validateSummary( + { + ...summary, + baseline: { ...target, runs: [{ ...passing[0], hardFailure: true }, ...passing.slice(1)] } + }, + 5 + ), + /inconsistent baseline run outcome and hardFailure/ + ) +}) + +test('removes the exact temporary sandbox when the run body throws', async () => { + let sandbox + + await assert.rejects( + withTemporarySandbox('cleanup-test', path => { + sandbox = path + writeFileSync(join(path, 'diagnostic.txt'), 'temporary') + throw new Error('teardown report failed') + }), + /teardown report failed/ + ) + + assert.ok(sandbox) + assert.equal(existsSync(sandbox), false) +}) diff --git a/apps/desktop/src/app/chat/short-session-hang-repro.tsx b/apps/desktop/src/app/chat/short-session-hang-repro.tsx new file mode 100644 index 000000000000..af75555869fe --- /dev/null +++ b/apps/desktop/src/app/chat/short-session-hang-repro.tsx @@ -0,0 +1,388 @@ +import type { ChatMessage } from '@/lib/chat-messages' +import { setBusy, setMessages } from '@/store/session' + +const HEARTBEAT_INTERVAL_MS = 100 +const PAINT_VALIDATION_TIMEOUT_MS = 5_000 + +interface HeartbeatSample { + at: number + gapMs: number + source: 'interval' | 'animation-frame' | 'long-task' +} + +interface FixtureTurn { + class: 'plain' | 'code' | 'tools' | 'mixed' + messages: ChatMessage[] +} + +interface ShortSessionDriver { + fixtures: FixtureTurn[] + heartbeatSamples: HeartbeatSample[] + load: (count: number) => Promise<{ + discoveredAssistantMessages: number + expectedAssistantMessages: number + expectedCodeCards: number + expectedTools: number + expectedUserIds: string[] + messageRecords: number + paintedAssistantMessages: number + paintedCodeCards: number + paintedTools: number + paintedUserIds: string[] + syntheticTurns: number + }> + manifest: () => Promise<{ + bytes: number + classes: FixtureTurn['class'][] + messageRecords: number + parts: number + sha256: string + tools: number + syntheticTurns: number + }> + checkpoint: () => void + reset: () => void + summary: () => { maxGapMs: number; samples: HeartbeatSample[] } +} + +declare global { + interface Window { + __SHORT_SESSION_HANG_REPRO__?: ShortSessionDriver + } +} + +const text = (value: string) => ({ text: value, type: 'text' as const }) + +function turn(index: number, fixtureClass: FixtureTurn['class'], assistantParts: ChatMessage['parts']): FixtureTurn { + const timestamp = 1_700_000_000_000 + index * 1_000 + + return { + class: fixtureClass, + messages: [ + { + id: `short-session-u-${index}`, + parts: [text(`Deterministic ${fixtureClass} prompt ${index}: preserve renderer responsiveness.`)], + role: 'user', + timestamp + }, + { + id: `short-session-a-${index}`, + parts: assistantParts, + pending: false, + role: 'assistant', + timestamp: timestamp + 1 + } + ] + } +} + +const fixtures: FixtureTurn[] = [ + turn(1, 'plain', [text('Plain response one. The transcript should remain selectable and scrollable.')]), + turn(2, 'code', [ + text('Code response two.\n\n```ts\nexport const bounded = (n: number) => Math.min(8, Math.max(1, n))\n```') + ]), + turn(3, 'tools', [ + { + args: { path: '/tmp/hermes-short-session-fixture.txt' }, + argsText: '{"path":"/tmp/hermes-short-session-fixture.txt"}', + result: { content: 'deterministic fixture output', ok: true }, + toolCallId: 'short-session-tool-3', + toolName: 'read_file', + type: 'tool-call' + } + ]), + turn(4, 'mixed', [ + text( + 'Mixed response four starts with prose and a compact table.\n\n| field | value |\n|---|---|\n| stable | yes |' + ), + { + args: { command: 'printf deterministic' }, + argsText: '{"command":"printf deterministic"}', + result: { output: 'deterministic', status: 0 }, + toolCallId: 'short-session-tool-4', + toolName: 'terminal', + type: 'tool-call' + }, + text('\nThe renderer must paint narration after the tool result.') + ]), + turn(5, 'plain', [text('Plain response five is the first required responsiveness checkpoint.')]), + turn(6, 'code', [ + text('Code response six.\n\n```python\ndef heartbeat(now, previous):\n return max(0, now - previous)\n```') + ]), + turn(7, 'tools', [ + { + args: { query: 'deterministic local fixture' }, + argsText: '{"query":"deterministic local fixture"}', + result: { matches: ['fixture-7'], ok: true }, + toolCallId: 'short-session-tool-7', + toolName: 'search_files', + type: 'tool-call' + } + ]), + turn(8, 'mixed', [ + text('Mixed response eight is the final checkpoint. `inline code` and **markdown** remain interactive.'), + { + args: { path: '/tmp' }, + argsText: '{"path":"/tmp"}', + result: { entries: ['hermes-short-session-fixture.txt'], ok: true }, + toolCallId: 'short-session-tool-8', + toolName: 'list_directory', + type: 'tool-call' + } + ]) +] + +if (typeof window !== 'undefined' && !window.__SHORT_SESSION_HANG_REPRO__) { + const heartbeatSamples: HeartbeatSample[] = [] + let lastInterval = performance.now() + let lastFrame = performance.now() + let maxGapMsSoFar = 0 + let sampleWindowStartedAt = performance.now() + + const record = (sample: HeartbeatSample) => { + if (document.visibilityState !== 'visible' || sample.at < sampleWindowStartedAt) { + return + } + + maxGapMsSoFar = Math.max(maxGapMsSoFar, sample.gapMs) + heartbeatSamples.push(sample) + + if (heartbeatSamples.length > 2_000) { + heartbeatSamples.splice(0, heartbeatSamples.length - 2_000) + } + } + + window.setInterval(() => { + const now = performance.now() + + if (document.visibilityState === 'visible') { + record({ at: now, gapMs: now - lastInterval, source: 'interval' }) + } + + lastInterval = now + }, HEARTBEAT_INTERVAL_MS) + + const frame = (now: number) => { + if (document.visibilityState === 'visible') { + record({ at: now, gapMs: now - lastFrame, source: 'animation-frame' }) + } + + lastFrame = now + requestAnimationFrame(frame) + } + + requestAnimationFrame(frame) + + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible') { + const now = performance.now() + lastInterval = now + lastFrame = now + } + }) + + const afterTwoFramesWhileVisible = () => + new Promise((resolve, reject) => { + if (document.visibilityState !== 'visible') { + reject(new Error('renderer became hidden before paint validation')) + + return + } + + let settled = false + let deadline: number | undefined + + const finish = (error?: Error) => { + if (settled) { + return + } + + settled = true + + if (deadline !== undefined) { + window.clearTimeout(deadline) + } + + document.removeEventListener('visibilitychange', onVisibilityChange) + + if (error) { + reject(error) + } else { + resolve() + } + } + + const onVisibilityChange = () => { + if (document.visibilityState !== 'visible') { + finish(new Error('renderer became hidden during paint validation')) + } + } + + document.addEventListener('visibilitychange', onVisibilityChange) + deadline = window.setTimeout( + () => finish(new Error('paint validation did not observe two frames')), + PAINT_VALIDATION_TIMEOUT_MS + ) + requestAnimationFrame(() => { + if (document.visibilityState !== 'visible') { + finish(new Error('renderer became hidden before the first paint frame')) + + return + } + + requestAnimationFrame(() => { + if (document.visibilityState !== 'visible') { + finish(new Error('renderer became hidden before the second paint frame')) + + return + } + + finish() + }) + }) + }) + + try { + const observer = new PerformanceObserver(list => { + for (const entry of list.getEntries()) { + record({ at: entry.startTime, gapMs: entry.duration, source: 'long-task' }) + } + }) + + observer.observe({ entryTypes: ['longtask'] }) + } catch { + // Long Task API availability is a soft diagnostic signal. + } + + const checkpoint = () => { + const now = performance.now() + + sampleWindowStartedAt = now + heartbeatSamples.length = 0 + maxGapMsSoFar = 0 + lastInterval = now + lastFrame = now + } + + const reset = () => { + checkpoint() + setBusy(false) + setMessages([]) + } + + window.__SHORT_SESSION_HANG_REPRO__ = { + fixtures, + heartbeatSamples, + load: async count => { + if (!Number.isFinite(count)) { + throw new RangeError('count must be a finite number') + } + + const bounded = Math.min(fixtures.length, Math.max(1, Math.trunc(count))) + const next = fixtures.slice(0, bounded).flatMap(fixture => fixture.messages) + const expectedUserIds = next.filter(message => message.role === 'user').map(message => message.id) + const expectedAssistantIds = next.filter(message => message.role === 'assistant').map(message => message.id) + const expectedAssistantMessages = expectedAssistantIds.length + const expectedTools = next.flatMap(message => message.parts).filter(part => part.type === 'tool-call').length + + const expectedCodeCards = next + .flatMap(message => message.parts) + .filter(part => part.type === 'text') + .reduce((total, part) => total + Math.floor((part.text.match(/```/g)?.length ?? 0) / 2), 0) + + setBusy(false) + setMessages(next) + + await afterTwoFramesWhileVisible() + + const isPainted = (element: Element) => { + const style = getComputedStyle(element) + + return style.display !== 'none' && style.visibility !== 'hidden' && element.getClientRects().length > 0 + } + + const paintedUserIds = expectedUserIds.filter(id => { + const element = document.querySelector(`[data-message-id="${CSS.escape(id)}"]`) + + return element ? isPainted(element) : false + }) + + const assistantRoots = [ + ...new Set( + expectedUserIds.flatMap(id => { + const user = document.querySelector(`[data-message-id="${CSS.escape(id)}"]`) + const turn = user?.closest('[data-slot="aui_turn-pair"]') + + return turn ? [...turn.querySelectorAll('[data-slot="aui_assistant-message-root"]')] : [] + }) + ) + ] + + const paintedTools = assistantRoots + .flatMap(root => [...root.querySelectorAll('[data-tool-row]')]) + .filter(isPainted).length + + const paintedCodeCards = assistantRoots + .flatMap(root => [...root.querySelectorAll('[data-slot="code-card"]')]) + .filter(isPainted).length + + const paintedAssistantMessages = assistantRoots.filter(root => { + if (!isPainted(root)) { + return false + } + + const hasText = Boolean(root.textContent?.trim()) + const hasPaintedTool = [...root.querySelectorAll('[data-tool-row]')].some(isPainted) + const hasPaintedCode = [...root.querySelectorAll('[data-slot="code-card"]')].some(isPainted) + + return hasText || hasPaintedTool || hasPaintedCode + }).length + + return { + discoveredAssistantMessages: assistantRoots.length, + expectedAssistantMessages, + expectedCodeCards, + expectedTools, + expectedUserIds, + messageRecords: next.length, + paintedAssistantMessages, + paintedCodeCards, + paintedTools, + paintedUserIds, + syntheticTurns: bounded + } + }, + manifest: async () => { + const serialized = JSON.stringify(fixtures) + const bytes = new TextEncoder().encode(serialized) + const digest = await crypto.subtle.digest('SHA-256', bytes) + + return { + bytes: bytes.byteLength, + classes: fixtures.map(fixture => fixture.class), + messageRecords: fixtures.reduce((total, fixture) => total + fixture.messages.length, 0), + parts: fixtures.reduce( + (total, fixture) => total + fixture.messages.reduce((count, message) => count + message.parts.length, 0), + 0 + ), + sha256: [...new Uint8Array(digest)].map(value => value.toString(16).padStart(2, '0')).join(''), + tools: fixtures.reduce( + (total, fixture) => + total + + fixture.messages.reduce( + (count, message) => count + message.parts.filter(part => part.type === 'tool-call').length, + 0 + ), + 0 + ), + syntheticTurns: fixtures.length + } + }, + checkpoint, + reset, + summary: () => ({ + maxGapMs: maxGapMsSoFar, + samples: [...heartbeatSamples] + }) + } +} diff --git a/apps/desktop/src/lib/inflight-turn-journal.test.ts b/apps/desktop/src/lib/inflight-turn-journal.test.ts index 206a12228894..2604c60d7a1d 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.test.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.test.ts @@ -7,11 +7,15 @@ import { mergeInFlightMessages, persistInFlightTurnState, readInFlightTurnJournal, - recoverInFlightTurnJournal + recoverInFlightTurnJournal, + resetInFlightTurnJournalStateForTests } from '@/lib/inflight-turn-journal' +const STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' const STORAGE_PREFIX = 'hermes.desktop.inflightTurnJournal.v2:' -const LEGACY_STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' +const MIGRATION_KEY = 'hermes.desktop.inflightTurnJournal.v2.migrated' + +const sessionStorageKey = (storedSessionId: string) => `${STORAGE_PREFIX}${encodeURIComponent(storedSessionId)}` function user(id: string, text: string): ChatMessage { return { id, role: 'user', parts: [{ type: 'text', text }] } @@ -46,16 +50,86 @@ function journalState(overrides: Partial = {}): Journal } beforeEach(() => { + resetInFlightTurnJournalStateForTests() vi.useFakeTimers() window.localStorage.clear() }) afterEach(() => { + vi.restoreAllMocks() clearInFlightTurnJournal('stored-1') vi.useRealTimers() }) describe('persistInFlightTurnState', () => { + it('sweeps expired and oldest session entries once before the first write', () => { + const now = Date.now() + + for (let index = 0; index < 25; index += 1) { + const sessionId = `old-${index}` + + const snapshot = { + messages: [ + user(`u-${index}`, `prompt-${index}`), + assistant(`a-${index}`, `partial-${index}`, { pending: true }) + ], + streamId: `a-${index}`, + turnStartedAt: index, + updatedAt: now - index * 1_000 + } + + window.localStorage.setItem(sessionStorageKey(sessionId), JSON.stringify(snapshot)) + } + + window.localStorage.setItem( + sessionStorageKey('expired'), + JSON.stringify({ + messages: [user('expired-u', 'expired'), assistant('expired-a', 'expired', { pending: true })], + streamId: 'expired-a', + turnStartedAt: 0, + updatedAt: now - 8 * 24 * 60 * 60 * 1_000 + }) + ) + + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + const sessionKeys = Array.from({ length: window.localStorage.length }, (_, index) => + window.localStorage.key(index) + ).filter((key): key is string => key?.startsWith(STORAGE_PREFIX) === true) + + expect(sessionKeys).toHaveLength(24) + expect(window.localStorage.getItem(sessionStorageKey('expired'))).toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('old-24'))).toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('stored-1'))).not.toBeNull() + }) + + it('writes only the current session instead of reading and rewriting the aggregate journal', () => { + const getItem = vi.spyOn(Storage.prototype, 'getItem') + const setItem = vi.spyOn(Storage.prototype, 'setItem') + + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + expect(getItem).not.toHaveBeenCalledWith(STORAGE_KEY) + expect(setItem).not.toHaveBeenCalledWith(STORAGE_KEY, expect.any(String)) + expect(setItem).toHaveBeenCalledWith(sessionStorageKey('stored-1'), expect.any(String)) + }) + + it('keeps another session snapshot when one session settles', () => { + persistInFlightTurnState(journalState()) + persistInFlightTurnState(journalState({ storedSessionId: 'stored-2' })) + vi.advanceTimersByTime(400) + + expect(window.localStorage.getItem(sessionStorageKey('stored-1'))).not.toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('stored-2'))).not.toBeNull() + + clearInFlightTurnJournal('stored-2') + + expect(readInFlightTurnJournal('stored-1')).not.toBeNull() + expect(readInFlightTurnJournal('stored-2')).toBeNull() + }) + it('journals the running turn tail after the throttle window', () => { persistInFlightTurnState(journalState()) @@ -88,6 +162,182 @@ describe('persistInFlightTurnState', () => { expect(tail?.parts).toEqual([{ type: 'text', text: 'partial answer grew' }]) }) + it('preserves a long user prompt exactly so recovery still matches its transcript row', () => { + const prompt = 'prompt '.repeat(8_000) + + persistInFlightTurnState( + journalState({ + messages: [user('u1', prompt), assistant('assistant-stream-1', 'partial', { pending: true })] + }) + ) + vi.advanceTimersByTime(400) + + const result = recoverInFlightTurnJournal('stored-1', [user('db-u1', prompt)]) + + expect(result.messages.map(message => message.id)).toEqual(['db-u1', 'assistant-stream-1']) + }) + + it('preserves user attachment refs exactly so recovery still matches its transcript row', () => { + const attachmentRefs = Array.from({ length: 25 }, (_, index) => `@file:/tmp/input-${index}.txt`) + const prompt = user('u1', 'inspect these files') + prompt.attachmentRefs = attachmentRefs + + persistInFlightTurnState( + journalState({ messages: [prompt, assistant('assistant-stream-1', 'partial', { pending: true })] }) + ) + vi.advanceTimersByTime(400) + + const restoredPrompt = user('db-u1', 'inspect these files') + restoredPrompt.attachmentRefs = attachmentRefs + const result = recoverInFlightTurnJournal('stored-1', [restoredPrompt]) + + expect(result.messages.map(message => message.id)).toEqual(['db-u1', 'assistant-stream-1']) + }) + + it('trims oldest sealed rows when bounded parts exceed the entry cap', () => { + const text = 'x'.repeat(60 * 1024) + + const messages = [ + user('u1', 'do the thing'), + assistant('a1', text, { pending: false }), + assistant('a2', text, { pending: false }), + assistant('a3', text, { pending: false }), + assistant('a4', text, { pending: true }) + ] + + persistInFlightTurnState(journalState({ messages, streamId: 'a4' })) + vi.advanceTimersByTime(400) + + const raw = window.localStorage.getItem(sessionStorageKey('stored-1')) + const snapshot = JSON.parse(raw!) + + expect(raw?.length).toBeLessThanOrEqual(160 * 1024) + expect(snapshot.messages.map((message: ChatMessage) => message.id)).toEqual(['u1', 'a3', 'a4']) + }) + + it('keeps an older recoverable snapshot when the newest row alone is too large', () => { + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + const hugeAssistant: ChatMessage = { + id: 'assistant-stream-1', + role: 'assistant', + parts: Array.from({ length: 3 }, () => ({ type: 'text' as const, text: 'x'.repeat(64 * 1024) })), + pending: true + } + + persistInFlightTurnState( + journalState({ messages: [user('u1', 'do the thing'), hugeAssistant], streamId: hugeAssistant.id }) + ) + vi.advanceTimersByTime(400) + + const snapshot = JSON.parse(window.localStorage.getItem(sessionStorageKey('stored-1'))!) + + expect(snapshot.messages[1].parts).toEqual([{ type: 'text', text: 'partial answer' }]) + }) + + it('skips a pathological user prompt instead of truncating its recovery join key', () => { + const prompt = 'x'.repeat(64 * 1024 + 1) + + persistInFlightTurnState( + journalState({ + messages: [user('u1', prompt), assistant('assistant-stream-1', 'partial', { pending: true })] + }) + ) + vi.advanceTimersByTime(400) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) + + it('does not parse the legacy aggregate when a pathological write discards its session', () => { + const legacy = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() + } + + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { 'stored-1': legacy }, version: 1 })) + const getItem = vi.spyOn(Storage.prototype, 'getItem') + const prompt = 'x'.repeat(64 * 1024 + 1) + + persistInFlightTurnState( + journalState({ + messages: [user('u1', prompt), assistant('assistant-stream-1', 'partial', { pending: true })] + }) + ) + vi.advanceTimersByTime(400) + + expect(getItem).not.toHaveBeenCalledWith(STORAGE_KEY) + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) + + it('preserves a tombstone while sweeping before legacy migration', () => { + const legacy = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() + } + + window.localStorage.setItem(sessionStorageKey('stored-1'), '0') + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { 'stored-1': legacy }, version: 1 })) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(window.localStorage.getItem(STORAGE_KEY)).toBeNull() + expect(window.localStorage.getItem(sessionStorageKey('stored-1'))).toBeNull() + }) + + it('removes tombstones after the one-shot legacy migration has completed', () => { + const key = sessionStorageKey('stored-1') + + window.localStorage.setItem(MIGRATION_KEY, '1') + window.localStorage.setItem(key, '0') + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(window.localStorage.getItem(key)).toBeNull() + }) + + it('strips pathological 5 MiB tool payloads before attempting a storage write', () => { + const setItem = vi.spyOn(Storage.prototype, 'setItem') + + const oversized: ChatMessage = { + id: 'assistant-stream-1', + role: 'assistant', + parts: [ + { + type: 'tool-call', + toolCallId: 'tc-1', + toolName: 'terminal', + args: { command: 'x'.repeat(5 * 1024 * 1024) }, + result: 'x'.repeat(5 * 1024 * 1024), + isError: true + }, + { type: 'text', text: 'still useful' } + ], + pending: true + } + + persistInFlightTurnState(journalState({ messages: [user('u1', 'do the thing'), oversized] })) + vi.advanceTimersByTime(400) + + const raw = window.localStorage.getItem(sessionStorageKey('stored-1')) + const snapshot = JSON.parse(raw!) + const persistedTool = snapshot.messages[1].parts[0] + + expect(raw?.length).toBeLessThan(256 * 1024) + expect(persistedTool).toEqual({ + args: {}, + isError: true, + result: {}, + toolCallId: 'tc-1', + toolName: 'terminal', + type: 'tool-call' + }) + expect(snapshot.messages[1].parts[1]).toEqual({ type: 'text', text: 'still useful' }) + expect(setItem.mock.calls.every(([, value]) => value.length <= 256 * 1024)).toBe(true) + }) + it('clears the entry the moment the turn settles, cancelling pending writes', () => { persistInFlightTurnState(journalState()) vi.advanceTimersByTime(400) @@ -113,52 +363,170 @@ describe('persistInFlightTurnState', () => { persistInFlightTurnState(journalState()) vi.advanceTimersByTime(400) - const raw = JSON.parse(window.localStorage.getItem(`${STORAGE_PREFIX}stored-1`)!) + const key = sessionStorageKey('stored-1') + const raw = JSON.parse(window.localStorage.getItem(key)!) raw.updatedAt = Date.now() - 8 * 24 * 60 * 60 * 1000 - window.localStorage.setItem(`${STORAGE_PREFIX}stored-1`, JSON.stringify(raw)) + window.localStorage.setItem(key, JSON.stringify(raw)) expect(readInFlightTurnJournal('stored-1')).toBeNull() }) - it('writes each session under its own key, untouched by other sessions settling', () => { - persistInFlightTurnState(journalState()) - persistInFlightTurnState(journalState({ storedSessionId: 'stored-2' })) - vi.advanceTimersByTime(400) + it('isolates storage read, write, and removal failures', () => { + const getItem = vi.spyOn(Storage.prototype, 'getItem').mockImplementation(() => { + throw new Error('read denied') + }) - expect(window.localStorage.getItem(`${STORAGE_PREFIX}stored-1`)).not.toBeNull() - expect(window.localStorage.getItem(`${STORAGE_PREFIX}stored-2`)).not.toBeNull() + expect(() => readInFlightTurnJournal('stored-1')).not.toThrow() + getItem.mockRestore() - clearInFlightTurnJournal('stored-2') + const setItem = vi.spyOn(Storage.prototype, 'setItem').mockImplementation(() => { + throw new Error('quota') + }) - expect(readInFlightTurnJournal('stored-1')).not.toBeNull() - expect(readInFlightTurnJournal('stored-2')).toBeNull() + expect(() => { + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + }).not.toThrow() + setItem.mockRestore() + + const removeItem = vi.spyOn(Storage.prototype, 'removeItem').mockImplementation(() => { + throw new Error('remove denied') + }) + + expect(() => clearInFlightTurnJournal('stored-1')).not.toThrow() }) - it('recovers entries journaled by the v1 single-key store', () => { - // A pre-upgrade crash leaves a v1 store behind; the first journal touch - // after the upgrade must still recover its turns. + it('discards malformed optional message metadata instead of throwing during recovery', () => { window.localStorage.setItem( - LEGACY_STORAGE_KEY, + sessionStorageKey('stored-1'), JSON.stringify({ - entries: { - 'stored-legacy': { - messages: [user('u1', 'legacy prompt'), assistant('a1', 'legacy partial', { pending: true })], - streamId: 'a1', - turnStartedAt: 500, - updatedAt: Date.now() - } - }, - version: 1 + messages: [ + { id: 'u1', role: 'user', parts: [{ type: 'text', text: 'prompt' }], attachmentRefs: '@file:bad' }, + { id: 'a1', role: 'assistant', parts: [{ type: 'text', text: 'partial' }], pending: true } + ], + streamId: 'a1', + turnStartedAt: 1, + updatedAt: Date.now() }) ) + const base = [user('db-u1', 'prompt')] + + expect(() => recoverInFlightTurnJournal('stored-1', base)).not.toThrow() + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) +}) - const entry = readInFlightTurnJournal('stored-legacy') +describe('legacy journal migration', () => { + it('migrates the bounded v1 aggregate once and recovers its sessions', () => { + const first = { + messages: [user('u1', 'one'), assistant('a1', 'partial one', { pending: true })], + streamId: 'a1', + turnStartedAt: 1, + updatedAt: Date.now() + } - expect(entry?.streamId).toBe('a1') - expect(entry?.messages).toHaveLength(2) - expect(window.localStorage.getItem(LEGACY_STORAGE_KEY)).toBeNull() + const second = { + messages: [ + user('u2', 'two'), + { + id: 'a2', + role: 'assistant' as const, + parts: [ + { + type: 'tool-call' as const, + toolCallId: 'tc-legacy', + toolName: 'terminal', + args: { command: 'large-output' }, + result: 'x'.repeat(1024 * 1024) + }, + { type: 'text' as const, text: 'partial two' } + ], + pending: true + } + ], + streamId: 'a2', + turnStartedAt: 2, + updatedAt: Date.now() + } + + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { one: first, two: second }, version: 1 })) + + expect(readInFlightTurnJournal('one')).toEqual(first) + const migratedSecondRaw = window.localStorage.getItem(sessionStorageKey('two'))! + const migratedSecond = JSON.parse(migratedSecondRaw) + + expect(migratedSecondRaw.length).toBeLessThan(256 * 1024) + expect(migratedSecond.messages[1].parts).toEqual([ + { args: {}, result: {}, toolCallId: 'tc-legacy', toolName: 'terminal', type: 'tool-call' }, + { text: 'partial two', type: 'text' } + ]) + expect(window.localStorage.getItem(STORAGE_KEY)).toBeNull() + expect(window.localStorage.getItem(MIGRATION_KEY)).toBe('1') + + window.localStorage.setItem(STORAGE_KEY, JSON.stringify({ entries: { three: first }, version: 1 })) + expect(readInFlightTurnJournal('three')).toBeNull() + expect(window.localStorage.getItem(STORAGE_KEY)).not.toBeNull() + }) - clearInFlightTurnJournal('stored-legacy') + it('drops an oversized legacy aggregate without parsing it', () => { + window.localStorage.setItem(STORAGE_KEY, 'x'.repeat(2 * 1024 * 1024 + 1)) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(window.localStorage.getItem(STORAGE_KEY)).toBeNull() + expect(window.localStorage.getItem(MIGRATION_KEY)).toBe('1') + }) + + it('does not overwrite a newer per-session snapshot while migrating another legacy session', () => { + persistInFlightTurnState(journalState()) + vi.advanceTimersByTime(400) + + const legacyCurrent = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() - 1_000 + } + + const legacyOther = { + messages: [user('u2', 'other prompt'), assistant('a2', 'other partial', { pending: true })], + streamId: 'a2', + turnStartedAt: 2, + updatedAt: Date.now() + } + + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify({ entries: { 'stored-1': legacyCurrent, other: legacyOther }, version: 1 }) + ) + + expect(readInFlightTurnJournal('other')).toEqual(legacyOther) + expect(readInFlightTurnJournal('stored-1')?.messages[0]).toEqual(user('u1', 'do the thing')) + }) + + it('does not resurrect a legacy entry after that session settles before migration', () => { + const legacyCurrent = { + messages: [user('legacy-u1', 'old prompt'), assistant('legacy-a1', 'old partial', { pending: true })], + streamId: 'legacy-a1', + turnStartedAt: 1, + updatedAt: Date.now() + } + + const legacyOther = { + messages: [user('u2', 'other prompt'), assistant('a2', 'other partial', { pending: true })], + streamId: 'a2', + turnStartedAt: 2, + updatedAt: Date.now() + } + + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify({ entries: { 'stored-1': legacyCurrent, other: legacyOther }, version: 1 }) + ) + + persistInFlightTurnState(journalState({ busy: false, awaitingResponse: false, streamId: null })) + + expect(readInFlightTurnJournal('stored-1')).toBeNull() + expect(readInFlightTurnJournal('other')).toEqual(legacyOther) }) }) diff --git a/apps/desktop/src/lib/inflight-turn-journal.ts b/apps/desktop/src/lib/inflight-turn-journal.ts index 27912970ff67..46103b9c9736 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.ts @@ -16,20 +16,34 @@ import { type ChatMessage, type ChatMessagePart, chatMessageText } from '@/lib/c * Best-effort by design: storage failures must never break chat streaming. */ -/** One localStorage key PER SESSION. The v1 single-key store meant every - * throttled write re-parsed and re-stringified EVERY busy session's tail — - * with a grid of concurrent streams that was a whole-store JSON round-trip - * dozens of times a second, all on the main thread. Per-session keys make a - * write O(own tail) regardless of how many other sessions are streaming. */ -const STORAGE_PREFIX = 'hermes.desktop.inflightTurnJournal.v2:' const LEGACY_STORAGE_KEY = 'hermes.desktop.inflightTurnJournal.v1' -const MAX_ENTRIES = 24 +const STORAGE_PREFIX = 'hermes.desktop.inflightTurnJournal.v2:' +const LEGACY_MIGRATION_KEY = 'hermes.desktop.inflightTurnJournal.v2.migrated' +const DISCARDED_SNAPSHOT_RAW = '0' +const STORE_VERSION = 1 +const MAX_SESSION_STORE_CHARS = 4 * 1024 * 1024 const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000 +// Keep the worst-case v2 namespace below a conservative localStorage budget +// while retaining the 24 newest session slots for ordinary small snapshots. +const MAX_ENTRY_CHARS = 160 * 1024 +const MAX_ENTRIES = Math.min(24, Math.floor(MAX_SESSION_STORE_CHARS / MAX_ENTRY_CHARS)) +const MAX_LEGACY_STORE_CHARS = 2 * 1024 * 1024 +const MAX_SESSION_KEY_CHARS = 512 +const MAX_JOURNALED_MESSAGES = 24 +const MAX_TEXT_PART_CHARS = 64 * 1024 +const MAX_METADATA_CHARS = 2 * 1024 +const MAX_USER_ATTACHMENT_REFS = 256 +const MAX_USER_ATTACHMENT_REF_CHARS = 64 * 1024 /** Streaming repaints arrive every ~33ms; localStorage writes are synchronous. * Trailing-edge throttle keeps the journal off the hot path — a crash costs at * most this much of the newest tail. */ const PERSIST_THROTTLE_MS = 400 +// A renderer can accumulate one entry per session over its lifetime. Sweep the +// bounded v2 namespace once on first journal access; never scan it on the +// 400ms streaming write path. +let sessionStoreSwept = false + export interface InFlightTurnSnapshot { messages: ChatMessage[] streamId: null | string @@ -46,6 +60,11 @@ export interface JournalableSessionState { turnStartedAt: null | number } +interface JournalStore { + entries: Record + version: typeof STORE_VERSION +} + export interface InFlightRecoveryResult { applied: boolean /** The base transcript already contains the journaled turn's completed @@ -64,133 +83,431 @@ function storage(): Storage | null { } } -const entryKey = (storedSessionId: string) => `${STORAGE_PREFIX}${storedSessionId}` - -function isExpired(entry: InFlightTurnSnapshot, now = Date.now()): boolean { - return now - entry.updatedAt > MAX_AGE_MS -} - -function loadEntry(storedSessionId: string): InFlightTurnSnapshot | null { - const store = storage() +function sessionStorageKey(storedSessionId: string): null | string { + try { + const encoded = encodeURIComponent(storedSessionId) - if (!store) { + return encoded.length > 0 && encoded.length <= MAX_SESSION_KEY_CHARS ? `${STORAGE_PREFIX}${encoded}` : null + } catch { return null } +} +function readRaw(store: Storage, key: string): null | string { try { - const raw = store.getItem(entryKey(storedSessionId)) - const parsed = raw ? (JSON.parse(raw) as InFlightTurnSnapshot) : null - - return parsed && typeof parsed.updatedAt === 'number' && Array.isArray(parsed.messages) ? parsed : null + return store.getItem(key) } catch { return null } } -function saveEntry(storedSessionId: string, entry: InFlightTurnSnapshot): void { +function removeRaw(store: Storage, key: string): void { try { - storage()?.setItem(entryKey(storedSessionId), JSON.stringify(entry)) + store.removeItem(key) } catch { - // Quota/private-mode failures: the journal is a recovery aid, not truth. + // Best-effort recovery state must not interrupt chat streaming. } } -function removeEntry(storedSessionId: string): void { +function writeRaw(store: Storage, key: string, value: string): boolean { try { - storage()?.removeItem(entryKey(storedSessionId)) + store.setItem(key, value) + + return true } catch { - // Same best-effort stance as saveEntry. + return false } } -// Split a v1 single-key store into per-session entries. Checked on every -// journal touch (a null getItem is free); a populated v1 store exists at most -// once, right after the upgrade. -function migrateLegacyStore(store: Storage): void { - try { - const legacy = store.getItem(LEGACY_STORAGE_KEY) +function isSnapshot(value: unknown): value is InFlightTurnSnapshot { + if (!value || typeof value !== 'object') { + return false + } - if (!legacy) { - return - } + const snapshot = value as Partial - const parsed = JSON.parse(legacy) + return ( + Array.isArray(snapshot.messages) && + snapshot.messages.every( + message => + Boolean(message) && + typeof message === 'object' && + typeof message.id === 'string' && + ['assistant', 'system', 'tool', 'user'].includes(message.role) && + Array.isArray(message.parts) && + message.parts.every( + part => + Boolean(part) && + typeof part === 'object' && + typeof part.type === 'string' && + (part.type !== 'text' && part.type !== 'reasoning' + ? part.type !== 'tool-call' || + (typeof part.toolName === 'string' && + (part.toolCallId === undefined || typeof part.toolCallId === 'string') && + (part.isError === undefined || typeof part.isError === 'boolean')) + : typeof part.text === 'string' && (part.parentId === undefined || typeof part.parentId === 'string')) + ) && + (message.timestamp === undefined || + (typeof message.timestamp === 'number' && Number.isFinite(message.timestamp))) && + (message.pending === undefined || typeof message.pending === 'boolean') && + (message.error === undefined || typeof message.error === 'string') && + (message.branchGroupId === undefined || typeof message.branchGroupId === 'string') && + (message.hidden === undefined || typeof message.hidden === 'boolean') && + (message.interim === undefined || typeof message.interim === 'boolean') && + (message.attachmentRefs === undefined || + (Array.isArray(message.attachmentRefs) && message.attachmentRefs.every(ref => typeof ref === 'string'))) && + (message.rowId === undefined || (typeof message.rowId === 'number' && Number.isFinite(message.rowId))) + ) && + (snapshot.streamId === null || typeof snapshot.streamId === 'string') && + (snapshot.turnStartedAt === null || typeof snapshot.turnStartedAt === 'number') && + typeof snapshot.updatedAt === 'number' && + Number.isFinite(snapshot.updatedAt) + ) +} - if (parsed && typeof parsed.entries === 'object' && !Array.isArray(parsed.entries)) { - for (const [id, entry] of Object.entries(parsed.entries as Record)) { - saveEntry(id, entry) - } - } - } catch { - // A corrupt v1 store has nothing worth carrying over. +function parseSnapshot(raw: string): InFlightTurnSnapshot | null { + if (raw.length > MAX_ENTRY_CHARS) { + return null } try { - store.removeItem(LEGACY_STORAGE_KEY) + const parsed = JSON.parse(raw) + + return isSnapshot(parsed) ? parsed : null } catch { - // Best-effort, like every other journal write. + return null } } -// One-time prune per renderer: drop expired/overflow entries. Startup-only on -// purpose — entries clear on settle, so anything left over is crash residue, -// and enumerating localStorage on the write path would defeat the point. -let housekeepingDone = false +function serializeSnapshot(snapshot: InFlightTurnSnapshot): string | null { + let messages = snapshot.messages -function ensureHousekeeping(): void { - const store = storage() + while (messages.length > 0) { + try { + const raw = JSON.stringify({ ...snapshot, messages }) - if (!store) { - return + if (raw.length <= MAX_ENTRY_CHARS) { + return raw + } + } catch { + return null + } + + // Keep the join-key row and newest assistant progress while dropping the + // oldest sealed rows. If those two rows alone do not fit, the caller must + // avoid replacing an older recoverable snapshot with a tombstone. + if (messages.length <= 2) { + return null + } + + messages = [messages[0], ...messages.slice(2)] } - migrateLegacyStore(store) + return null +} - if (housekeepingDone) { +function sweepSessionStore(store: Storage, reserveSlot = false): void { + if (sessionStoreSwept) { return } - housekeepingDone = true + sessionStoreSwept = true try { - const keys: string[] = [] + const sessionKeys: string[] = [] for (let index = 0; index < store.length; index += 1) { const key = store.key(index) if (key?.startsWith(STORAGE_PREFIX)) { - keys.push(key) + sessionKeys.push(key) } } - const live: { key: string; updatedAt: number }[] = [] + const liveEntries: Array<{ key: string; snapshot: InFlightTurnSnapshot }> = [] + const migrated = readRaw(store, LEGACY_MIGRATION_KEY) !== null - for (const key of keys) { - let entry: InFlightTurnSnapshot | null = null + for (const key of sessionKeys) { + const raw = readRaw(store, key) - try { - entry = JSON.parse(store.getItem(key) ?? '') as InFlightTurnSnapshot - } catch { - // Unparseable — prune below. + // A tombstone is intentional state. It suppresses the stale v1 + // predecessor until the one-shot migration removes the aggregate. + if (raw === DISCARDED_SNAPSHOT_RAW) { + if (migrated) { + removeRaw(store, key) + } + + continue } - if (!entry || typeof entry.updatedAt !== 'number' || isExpired(entry)) { - store.removeItem(key) - } else { - live.push({ key, updatedAt: entry.updatedAt }) + const snapshot = raw ? parseSnapshot(raw) : null + + if (!snapshot || isExpired(snapshot)) { + removeRaw(store, key) + + continue } + + liveEntries.push({ key, snapshot }) + } + + liveEntries + .sort((left, right) => right.snapshot.updatedAt - left.snapshot.updatedAt) + .slice(reserveSlot ? MAX_ENTRIES - 1 : MAX_ENTRIES) + .forEach(entry => removeRaw(store, entry.key)) + } catch { + // The journal is best effort; a storage enumeration failure must not + // interrupt renderer work or turn persistence. + } +} + +function boundedString(value: string, maxChars: number): string { + return value.length <= maxChars ? value : value.slice(0, maxChars) +} + +function boundedPart(part: ChatMessagePart): ChatMessagePart | null { + if (part.type === 'text') { + return { + type: 'text', + text: boundedString(part.text, MAX_TEXT_PART_CHARS), + ...(part.parentId === undefined ? {} : { parentId: boundedString(part.parentId, MAX_METADATA_CHARS) }) + } + } + + if (part.type === 'reasoning') { + return { + type: 'reasoning', + text: boundedString(part.text, MAX_TEXT_PART_CHARS), + ...(part.parentId === undefined ? {} : { parentId: boundedString(part.parentId, MAX_METADATA_CHARS) }) + } + } + + if (part.type === 'tool-call') { + // Tool payloads can contain multi-megabyte command output. Recovery only + // needs invocation identity and failure state; args/results are available + // from the backend transcript when it survives. + return { + type: 'tool-call', + toolName: boundedString(part.toolName, MAX_METADATA_CHARS), + args: {}, + ...(part.toolCallId === undefined ? {} : { toolCallId: boundedString(part.toolCallId, MAX_METADATA_CHARS) }), + ...(part.result === undefined ? {} : { result: {} }), + ...(part.isError === undefined ? {} : { isError: part.isError }) + } + } + + // Rich file/image/data/source parts can embed large payloads. They are not + // required for in-flight text/tool recovery and remain backend-owned. + return null +} + +function boundedMessages(messages: ChatMessage[]): ChatMessage[] | null { + const bounded = + messages.length <= MAX_JOURNALED_MESSAGES + ? messages + : [messages[0], ...messages.slice(-(MAX_JOURNALED_MESSAGES - 1))] + + // User text and attachment refs are the recovery join key. Truncating either + // could attach a journal tail to the wrong transcript row, so pathological + // prompts skip journaling instead of weakening the match. + if ( + bounded.some(message => { + if (message.role !== 'user') { + return false + } + + if ( + message.parts.some( + part => (part.type === 'text' || part.type === 'reasoning') && part.text.length > MAX_TEXT_PART_CHARS + ) + ) { + return true + } + + const refs = message.attachmentRefs + + if (!refs) { + return false + } + + if (refs.length > MAX_USER_ATTACHMENT_REFS) { + return true + } + + let chars = 0 + + for (const ref of refs) { + chars += ref.length + + if (chars > MAX_USER_ATTACHMENT_REF_CHARS) { + return true + } + } + + return false + }) + ) { + return null + } + + return bounded.map(message => ({ + id: boundedString(message.id, MAX_METADATA_CHARS), + role: message.role, + parts: message.parts.map(boundedPart).filter((part): part is ChatMessagePart => part !== null), + ...(message.timestamp === undefined ? {} : { timestamp: message.timestamp }), + ...(message.pending === undefined ? {} : { pending: message.pending }), + ...(message.error === undefined ? {} : { error: boundedString(message.error, MAX_METADATA_CHARS) }), + ...(message.branchGroupId === undefined + ? {} + : { branchGroupId: boundedString(message.branchGroupId, MAX_METADATA_CHARS) }), + ...(message.hidden === undefined ? {} : { hidden: message.hidden }), + ...(message.interim === undefined ? {} : { interim: message.interim }), + ...(message.attachmentRefs === undefined + ? {} + : { + attachmentRefs: + message.role === 'user' + ? [...message.attachmentRefs] + : message.attachmentRefs + .slice(0, MAX_USER_ATTACHMENT_REFS) + .map(ref => boundedString(ref, MAX_METADATA_CHARS)) + }), + ...(message.rowId === undefined ? {} : { rowId: message.rowId }) + })) +} + +function migrateLegacyStore(store: Storage): void { + if (readRaw(store, LEGACY_MIGRATION_KEY) !== null) { + return + } + + const raw = readRaw(store, LEGACY_STORAGE_KEY) + + if (raw === null) { + return + } + + // Claim the migration before touching the aggregate. If storage is failing, + // skip legacy recovery rather than retrying an expensive parse on every read. + if (!writeRaw(store, LEGACY_MIGRATION_KEY, '1')) { + return + } + + // Release the multi-megabyte aggregate before allocating per-session v2 + // entries. The captured string remains available for this one migration. + removeRaw(store, LEGACY_STORAGE_KEY) + + if (!raw) { + return + } + + if (raw.length > MAX_LEGACY_STORE_CHARS) { + return + } + + try { + const parsed = JSON.parse(raw) as Partial + + if ( + parsed.version !== STORE_VERSION || + !parsed.entries || + typeof parsed.entries !== 'object' || + Array.isArray(parsed.entries) + ) { + return } - live.sort((a, b) => b.updatedAt - a.updatedAt) + const existingV2Keys = new Set() - for (const { key } of live.slice(MAX_ENTRIES)) { - store.removeItem(key) + for (let index = 0; index < store.length; index += 1) { + const key = store.key(index) + + if (key?.startsWith(STORAGE_PREFIX)) { + existingV2Keys.add(key) + } + } + + const entries = Object.entries(parsed.entries) + .filter((entry): entry is [string, InFlightTurnSnapshot] => isSnapshot(entry[1]) && !isExpired(entry[1])) + .sort((a, b) => b[1].updatedAt - a[1].updatedAt) + .slice(0, Math.max(0, MAX_ENTRIES - existingV2Keys.size)) + + for (const [storedSessionId, snapshot] of entries) { + const key = sessionStorageKey(storedSessionId) + const messages = boundedMessages(snapshot.messages) + const value = messages ? serializeSnapshot({ ...snapshot, messages }) : null + + // A v2 snapshot may have been written before the one-shot migration ran. + // Never replace newer per-session state with its stale v1 predecessor. + if (key && value && readRaw(store, key) === null) { + if (writeRaw(store, key, value)) { + existingV2Keys.add(key) + } + } } } catch { - // Best-effort, like every other journal write. + // Malformed legacy data is discarded below. } } +function discardSnapshot(store: Storage, key: string): void { + // Migrate first so an existing v2 key suppresses its stale v1 predecessor, + // then remove the current session. This keeps every discard path from + // resurrecting legacy state on a later read. + migrateLegacyStore(store) + removeRaw(store, key) +} + +function readSnapshot(storedSessionId: string): InFlightTurnSnapshot | null { + const store = storage() + const key = sessionStorageKey(storedSessionId) + + if (!store || !key) { + return null + } + + sweepSessionStore(store) + + let raw = readRaw(store, key) + + if (!raw) { + migrateLegacyStore(store) + raw = readRaw(store, key) + } + + if (!raw) { + return null + } + + const snapshot = parseSnapshot(raw) + + if (!snapshot || isExpired(snapshot)) { + discardSnapshot(store, key) + + return null + } + + return snapshot +} + +function removeSnapshot(storedSessionId: string): void { + const store = storage() + const key = sessionStorageKey(storedSessionId) + + if (store && key) { + sweepSessionStore(store) + + // Settling a session before the one-shot migration must clear its legacy + // entry too; otherwise a later read can migrate and resurrect stale state. + // This aggregate parse is terminal-transition work, never a stream write. + discardSnapshot(store, key) + } +} + +function isExpired(entry: InFlightTurnSnapshot, now = Date.now()): boolean { + return now - entry.updatedAt > MAX_AGE_MS +} + function cloneMessages(messages: ChatMessage[]): ChatMessage[] { try { return JSON.parse(JSON.stringify(messages)) as ChatMessage[] @@ -286,7 +603,7 @@ function recoverableTail(messages: ChatMessage[], streamId: null | string): Chat } } - return cloneMessages(visible.slice(start)) + return visible.slice(start) } function normalizeRecoveredTail(tail: ChatMessage[], keepPending: boolean): ChatMessage[] { @@ -470,6 +787,17 @@ export function mergeInFlightMessages( const persistTimers = new Map>() const persistLatest = new Map() +/** @internal Test-only reset for module-scoped throttles and sweep state. */ +export function resetInFlightTurnJournalStateForTests(): void { + for (const timer of persistTimers.values()) { + clearTimeout(timer) + } + + persistTimers.clear() + persistLatest.clear() + sessionStoreSwept = false +} + function writeSnapshot(storedSessionId: string, state: JournalableSessionState): void { const tail = recoverableTail(state.messages, state.streamId) @@ -477,13 +805,63 @@ function writeSnapshot(storedSessionId: string, state: JournalableSessionState): return } - ensureHousekeeping() - saveEntry(storedSessionId, { - messages: tail, + const store = storage() + const key = sessionStorageKey(storedSessionId) + + if (!store || !key) { + return + } + + sweepSessionStore(store, true) + + const messages = boundedMessages(tail) + + if (!messages) { + // Keep the timer write path free of aggregate migration. This tiny invalid + // v2 value suppresses the stale v1 predecessor until read/settle performs + // the one-shot migration and removes it. + tombstoneUnlessRecoverable(store, key) + + return + } + + const raw = serializeSnapshot({ + messages, streamId: state.streamId, turnStartedAt: state.turnStartedAt, updatedAt: Date.now() }) + + if (!raw) { + // Preserve an older bounded snapshot if the newest assistant row alone is + // too large. A tombstone is only needed when there is no recoverable v2 + // value, so stale v1 state cannot be resurrected on a later read. + tombstoneUnlessRecoverable(store, key) + + return + } + + if (!writeRaw(store, key, raw)) { + // A quota failure must not leave an older, misleading snapshot behind, or + // let the stale v1 predecessor be resurrected on a later read. + tombstoneUnlessRecoverable(store, key) + } +} + +function tombstoneUnlessRecoverable(store: Storage, key: string): void { + const previous = readRaw(store, key) + + if (previous) { + const snapshot = parseSnapshot(previous) + + if (snapshot && !isExpired(snapshot)) { + return + } + } + + if (!writeRaw(store, key, DISCARDED_SNAPSHOT_RAW)) { + removeRaw(store, key) + } } /** Persist the running turn's visible tail (throttled), or clear the entry the @@ -527,20 +905,7 @@ export function readInFlightTurnJournal(storedSessionId: null | string): InFligh return null } - ensureHousekeeping() - const entry = loadEntry(storedSessionId) - - if (!entry) { - return null - } - - if (isExpired(entry)) { - removeEntry(storedSessionId) - - return null - } - - return entry + return readSnapshot(storedSessionId) } /** Fold a journaled in-flight tail back onto a restored transcript. A no-op @@ -588,6 +953,6 @@ export function clearInFlightTurnJournal(storedSessionId: null | string): void { } persistLatest.delete(storedSessionId) - ensureHousekeeping() - removeEntry(storedSessionId) + + removeSnapshot(storedSessionId) } diff --git a/apps/desktop/vitest.config.ts b/apps/desktop/vitest.config.ts index dd2f38ff3193..70835d91f03a 100644 --- a/apps/desktop/vitest.config.ts +++ b/apps/desktop/vitest.config.ts @@ -1,4 +1,4 @@ -import type { TestProjectConfiguration } from 'vitest/config'; +import type { TestProjectConfiguration } from 'vitest/config' import { defineConfig } from 'vitest/config' const reactUi: TestProjectConfiguration = { @@ -20,7 +20,8 @@ const electronNative: TestProjectConfiguration = { test: { name: 'electron', environment: 'node', - include: ['electron/**/*.test.ts', 'scripts/**.test.{ts,mjs}'] + include: ['electron/**/*.test.ts', 'scripts/**.test.{ts,mjs}'], + exclude: ['scripts/run-short-session-hang-repro.test.mjs'] } } From 606699121d7e809397e434f5c9b213cef09cb24d Mon Sep 17 00:00:00 2001 From: Eugeniusz Gilewski Date: Wed, 12 Aug 2026 22:53:55 +0200 Subject: [PATCH 2/4] test(desktop): spy on the active localStorage implementation (#82832) The inflight journal regression test always spied on Storage.prototype, but Node 26's jsdom setup can provide a plain in-memory localStorage fallback. That left the test unable to observe the setItem call in CI even though the per-session journal write was correct. Select the native window.Storage prototype when available and otherwise spy on the active localStorage object, preserving the assertion across both storage implementations. Refs #82832 --- apps/desktop/src/lib/inflight-turn-journal.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/lib/inflight-turn-journal.test.ts b/apps/desktop/src/lib/inflight-turn-journal.test.ts index 2604c60d7a1d..6a0bed4bf22a 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.test.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.test.ts @@ -105,8 +105,16 @@ describe('persistInFlightTurnState', () => { }) it('writes only the current session instead of reading and rewriting the aggregate journal', () => { - const getItem = vi.spyOn(Storage.prototype, 'getItem') - const setItem = vi.spyOn(Storage.prototype, 'setItem') + const localStorage = window.localStorage + const storageConstructor = window.Storage + + const spyTarget = + typeof storageConstructor === 'function' && localStorage instanceof storageConstructor + ? storageConstructor.prototype + : localStorage + + const getItem = vi.spyOn(spyTarget, 'getItem') + const setItem = vi.spyOn(spyTarget, 'setItem') persistInFlightTurnState(journalState()) vi.advanceTimersByTime(400) From d7b17f22f48747297fd9a8d98db8e8b9d6a29e17 Mon Sep 17 00:00:00 2001 From: Nicolas Formenton Date: Tue, 11 Aug 2026 14:04:09 -0300 Subject: [PATCH 3/4] fix(desktop): skip stale journal duplicate folds The inflight-turn journal can outlive the turn it recorded (reclaim, reconnect or restart races skip the settle that clears it). On session resume the fold then re-appends journaled assistant rows to a transcript that already holds the committed replies, so the conversation ends with duplicate answers in scrambled order. The fold also carried the stale entry's streamId onto the resumed state on an idle resume, which kept the journal entry alive (persistInFlightTurnState only clears when streamId is null) and re-folded the same tail on every open. Detect text-level staleness before the append path: when every recoverable journaled assistant row already exists as committed text in the base transcript, treat the entry as caught up and clear it. Only keep a stream target when the resumed session is genuinely running (keepPending), so an idle resume self-heals instead of re-folding. --- .../src/lib/inflight-turn-journal.test.ts | 72 ++++++++++++++++++- apps/desktop/src/lib/inflight-turn-journal.ts | 61 ++++++++++++++-- .../nformenton@Nicolass-MacBook-Air.local | 2 + 3 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 contributors/emails/nformenton@Nicolass-MacBook-Air.local diff --git a/apps/desktop/src/lib/inflight-turn-journal.test.ts b/apps/desktop/src/lib/inflight-turn-journal.test.ts index 6a0bed4bf22a..552793a0e4ff 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.test.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.test.ts @@ -559,7 +559,7 @@ describe('recoverInFlightTurnJournal', () => { ]) const base = [user('u0', 'earlier turn'), assistant('a0', 'earlier reply')] - const result = recoverInFlightTurnJournal('stored-1', base) + const result = recoverInFlightTurnJournal('stored-1', base, { keepPending: true }) expect(result.applied).toBe(true) expect(result.messages.map(m => m.id)).toEqual(['u0', 'a0', 'u1', 'assistant-stream-1']) @@ -659,6 +659,76 @@ describe('recoverInFlightTurnJournal', () => { expect(merged.id).toBe('assistant-stream-rt9') expect(merged.parts[1]).toMatchObject({ type: 'text', text: 'a much longer locally journaled partial answer' }) }) + + // ── Scrambled-transcript regression (duplicate trailing answers) ─────────── + // The journal can outlive the turn it recorded (reclaim/reconnect/restart + // races skip the settle that would clear it). On resume the fold then + // re-appends content that the committed transcript ALREADY holds, rendering + // the same answers twice at the end of the conversation. Reported on the + // desktop as "the answer was already there, but it was inputted again". + + it('does not re-append committed answers when the journaled user row never persisted', () => { + // A resume projection can journal a `user-inflight-*` row that was never + // written to the DB (and may even belong to a different conversation). + // Because no base user matches it, the fold used to treat the whole tail + // as unknown and append it — duplicating the assistant answers below. + journalEntry([ + user('user-inflight-a3c2beb1', 'a stray user bubble that never persisted'), + assistant('assistant-stream-1', 'the committed answer') + ]) + + const base = [user('db-u1', 'the real prompt'), assistant('db-a1', 'the committed answer')] + const result = recoverInFlightTurnJournal('stored-1', base, { keepPending: false }) + + expect(result.caughtUp).toBe(true) + expect(result.applied).toBe(false) + expect(result.messages).toBe(base) + expect(result.messages.map(m => m.id)).toEqual(['db-u1', 'db-a1']) + // The stale entry is cleared so the next resume stays clean. + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) + + it('does not re-append committed answers when the journal tail has no user row', () => { + // A tail captured after a partial hydrate can end on assistant rows with + // no user prompt before them. The old code appended them verbatim, so the + // transcript ended with a duplicate of an answer that was already settled. + journalEntry([assistant('assistant-stream-1', 'the committed answer')]) + + const base = [user('db-u1', 'the real prompt'), assistant('db-a1', 'the committed answer')] + const result = recoverInFlightTurnJournal('stored-1', base, { keepPending: false }) + + expect(result.caughtUp).toBe(true) + expect(result.messages).toBe(base) + expect(readInFlightTurnJournal('stored-1')).toBeNull() + }) + + it('keeps appending a genuinely unknown turn (crash recovery still works)', () => { + // The staleness check must not swallow a tail the base never saw: that is + // the crash-recovery path the journal exists for. + journalEntry([user('u1', 'the live prompt'), assistant('assistant-stream-1', 'partial answer', { pending: true })]) + + const base = [user('db-u0', 'an earlier turn'), assistant('db-a0', 'earlier reply')] + const result = recoverInFlightTurnJournal('stored-1', base, { keepPending: true }) + + expect(result.applied).toBe(true) + expect(result.caughtUp).toBe(false) + expect(result.messages.map(m => m.id)).toEqual(['db-u0', 'db-a0', 'u1', 'assistant-stream-1']) + expect(readInFlightTurnJournal('stored-1')).not.toBeNull() + }) + + it('does not resurrect the journal streamId on a not-running resume (journal self-clear)', () => { + // The fold used to carry the stale entry's streamId onto the resumed state + // even when the backend reported the session idle. persistInFlightTurnState + // then re-wrote the journal instead of clearing it, so the same stale tail + // was folded again on every open — the scramble never healed. + journalEntry([user('u1', 'do the thing'), assistant('assistant-stream-1', 'partial answer', { pending: true })]) + + const base = [user('db-u0', 'an earlier turn'), assistant('db-a0', 'earlier reply')] + const result = recoverInFlightTurnJournal('stored-1', base, { keepPending: false }) + + expect(result.applied).toBe(true) + expect(result.streamId).toBeNull() + }) }) describe('mergeInFlightMessages', () => { diff --git a/apps/desktop/src/lib/inflight-turn-journal.ts b/apps/desktop/src/lib/inflight-turn-journal.ts index 46103b9c9736..82397579b0d4 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.ts @@ -693,6 +693,32 @@ function withoutBaseIds(rows: ChatMessage[], baseMessages: ChatMessage[]): ChatM return rows.filter(row => !baseIds.has(row.id)) } +/** Whether every recoverable assistant row in the journal tail already exists + * as committed text in the base transcript. When true, the journal outlived + * the turn it recorded and appending it would re-render the same answers at + * the end of the transcript (the "scrambled conversation" regression). */ +function journalTailAlreadyCommitted(tailAssistants: ChatMessage[], baseMessages: ChatMessage[]): boolean { + const recoverable = tailAssistants.filter(assistantHasRecoverableContent) + + if (recoverable.length === 0) { + return false + } + + const baseTexts = new Set( + baseMessages + .filter(message => message.role === 'assistant' && !message.hidden) + .map(message => normalizedText(chatMessageText(message))) + ) + + return recoverable.every(message => { + const text = normalizedText(chatMessageText(message)) + + // Error-only rows carry no text to verify against — keep the conservative + // append path rather than risk dropping a recoverable failure. + return text.length > 0 && baseTexts.has(text) + }) +} + export function mergeInFlightMessages( baseMessages: ChatMessage[], tailMessages: ChatMessage[], @@ -719,15 +745,27 @@ export function mergeInFlightMessages( const matchingUserIndex = tailUser ? baseMessages.findLastIndex(message => userMessagesMatch(message, tailUser)) : -1 if (matchingUserIndex < 0) { - // Base doesn't know this turn at all (user row was never persisted): - // append the whole tail. + // No base user matches the tail's user row (a projected user-inflight row + // that never persisted, or a tail captured without its user prompt). If the + // tail's answers are already committed in the transcript, the journal is + // stale — appending it would re-render the same replies at the end of the + // conversation. Otherwise, the base never saw this turn at all: append the + // whole tail (the crash-recovery path the journal exists for). + if (journalTailAlreadyCommitted(tailAssistants, baseMessages)) { + return { ...noop, caughtUp: true } + } + const streamId = lastJournalRow?.id ?? null return { applied: true, caughtUp: false, messages: [...baseMessages, ...withoutBaseIds(tail, baseMessages)], - streamId, + // Only a genuinely running turn keeps a live stream target. On an idle + // resume, carrying the stale streamId would keep the journal entry alive + // (persistInFlightTurnState only clears when streamId is null) and the + // same tail would be folded again on every open. + streamId: options.keepPending ? streamId : null, turnStartedAt: null } } @@ -781,7 +819,16 @@ export function mergeInFlightMessages( ...baseMessages.slice(projectionIndex + 1) ] - return { applied: true, caughtUp: false, messages, streamId: merged.id, turnStartedAt: null } + return { + applied: true, + caughtUp: false, + messages, + // Same idle-resume rule as the append path: only a running turn keeps the + // stream target alive, so an idle resume clears the journal instead of + // re-folding the same tail on every open. + streamId: options.keepPending ? merged.id : null, + turnStartedAt: null + } } const persistTimers = new Map>() @@ -935,7 +982,11 @@ export function recoverInFlightTurnJournal( return { ...recovered, - streamId: recovered.applied ? (recovered.streamId ?? snapshot.streamId) : null, + // Never resurrect a stale stream target on an idle resume: with + // keepPending=false the session is not running, so the recovered rows are + // settled history and the journal must clear on the next state update — + // otherwise the same stale tail is folded again on every open. + streamId: recovered.applied ? (recovered.streamId ?? (options.keepPending ? snapshot.streamId : null)) : null, turnStartedAt: recovered.applied ? snapshot.turnStartedAt : null } } diff --git a/contributors/emails/nformenton@Nicolass-MacBook-Air.local b/contributors/emails/nformenton@Nicolass-MacBook-Air.local new file mode 100644 index 000000000000..9b9caaa5824c --- /dev/null +++ b/contributors/emails/nformenton@Nicolass-MacBook-Air.local @@ -0,0 +1,2 @@ +Nicolas-Formenton +# PR #84137 / #84021 (desktop fixes from a second machine) From bfd423d0c940991cdc8365483e28efcf0f4e0dfb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:47:01 -0700 Subject: [PATCH 4/4] fix(desktop): clear the stale stream target on the assistant-tail append path too Sibling site of the idle-resume rule from the stale-fold fix: the assistant-tail append exit (user row persisted, no projection row) still carried the journal's streamId onto a not-running resume, which kept the journal entry alive (persistInFlightTurnState only clears when streamId is null) and re-folded the same tail on every open. Apply the same keepPending gate and pin it with a regression assertion. Refs #85308 --- apps/desktop/src/lib/inflight-turn-journal.test.ts | 3 +++ apps/desktop/src/lib/inflight-turn-journal.ts | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src/lib/inflight-turn-journal.test.ts b/apps/desktop/src/lib/inflight-turn-journal.test.ts index 552793a0e4ff..8041db1059a6 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.test.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.test.ts @@ -577,6 +577,9 @@ describe('recoverInFlightTurnJournal', () => { expect(result.applied).toBe(true) expect(result.messages.map(m => m.id)).toEqual(['db-u1', 'assistant-stream-1']) + // Idle resume: the assistant-tail append path must not resurrect the + // stream target either, or the journal entry re-folds on every open. + expect(result.streamId).toBeNull() const tail = result.messages.at(-1)! expect(tail.pending).toBe(false) expect(tail.parts[0]).toMatchObject({ type: 'tool-call' }) diff --git a/apps/desktop/src/lib/inflight-turn-journal.ts b/apps/desktop/src/lib/inflight-turn-journal.ts index 82397579b0d4..31a314bb052d 100644 --- a/apps/desktop/src/lib/inflight-turn-journal.ts +++ b/apps/desktop/src/lib/inflight-turn-journal.ts @@ -797,7 +797,11 @@ export function mergeInFlightMessages( applied: true, caughtUp: false, messages: [...baseMessages, ...withoutBaseIds(tailAssistants, baseMessages)], - streamId, + // Same idle-resume rule as the other exit paths: only a running turn + // keeps the stream target alive. Carrying the stale streamId here kept + // the journal entry alive (persistInFlightTurnState only clears when + // streamId is null), so the same tail was folded again on every open. + streamId: options.keepPending ? streamId : null, turnStartedAt: null } }