fix(tui): auto-recover session on unexpected gateway death (+ persist lifecycle breadcrumbs)

ui-tui/src/__tests__/createGatewayEventHandler.test.ts

@@ -681,6 +681,31 @@ describe('createGatewayEventHandler', () => {
     expect(resumeById).not.toHaveBeenCalled()
   })
 
+  it('on gateway.ready after a crash, resumes the recovered session once and skips forge', async () => {
+    const appended: Msg[] = []
+    const newSession = vi.fn()
+    const resumeById = vi.fn()
+    const ctx = buildCtx(appended)
+
+    ctx.session.newSession = newSession
+    // Mimic resumeById's synchronous status write so the test proves the
+    // "recovering session…" label is applied *after* (and survives) it.
+    ctx.session.resumeById = resumeById.mockImplementation(() => patchUiState({ status: 'resuming…' }))
+    ctx.session.STARTUP_RESUME_ID = ''
+    ctx.session.recoverSidRef = ref<null | string>('sess-crashed')
+
+    const onEvent = createGatewayEventHandler(ctx)
+
+    onEvent({ payload: {}, type: 'gateway.ready' } as any)
+
+    await vi.waitFor(() => expect(resumeById).toHaveBeenCalledWith('sess-crashed'))
+    expect(newSession).not.toHaveBeenCalled()
+    // One-shot: the ref is consumed so a later ordinary restart forges/resumes
+    // per config instead of re-resuming the recovered session.
+    expect(ctx.session.recoverSidRef.current).toBeNull()
+    expect(getUiState().status).toBe('recovering session…')
+  })
+
   it('on gateway.ready with auto_resume on and a recent session, resumes it', async () => {
     const appended: Msg[] = []
     const newSession = vi.fn()

ui-tui/src/__tests__/gatewayRecovery.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest'
+
+import { GATEWAY_RECOVERY_LIMIT, GATEWAY_RECOVERY_WINDOW_MS, planGatewayRecovery } from '../app/gatewayRecovery.js'
+
+describe('planGatewayRecovery', () => {
+  it('recovers the live session and records the attempt', () => {
+    const plan = planGatewayRecovery('sess-1', null, [], 1000)
+
+    expect(plan).toEqual({ attempts: [1000], recover: true, sid: 'sess-1' })
+  })
+
+  it('does not recover when there is no session to resume', () => {
+    expect(planGatewayRecovery(null, null, [], 1000)).toEqual({ attempts: [], recover: false, sid: null })
+  })
+
+  it('keeps retrying the recovery target through a startup crash-loop, bounded by the budget', () => {
+    // First exit: live sid present.
+    let attempts: number[] = []
+    let plan = planGatewayRecovery('sess-1', null, attempts, 0)
+
+    expect(plan.recover).toBe(true)
+    expect(plan.sid).toBe('sess-1')
+    attempts = plan.attempts
+
+    // Respawn crash-loops before gateway.ready: live sid is now null, but the
+    // recovery target carries it forward so we keep trying up to the budget.
+    for (let i = 1; i < GATEWAY_RECOVERY_LIMIT; i++) {
+      plan = planGatewayRecovery(null, 'sess-1', attempts, i)
+      expect(plan.recover).toBe(true)
+      expect(plan.sid).toBe('sess-1')
+      attempts = plan.attempts
+    }
+
+    // Budget exhausted: fall back to the inert state instead of spawn-storming.
+    plan = planGatewayRecovery(null, 'sess-1', attempts, GATEWAY_RECOVERY_LIMIT)
+    expect(plan.recover).toBe(false)
+    expect(plan.sid).toBe('sess-1')
+  })
+
+  it('prunes attempts older than the window so recovery re-arms', () => {
+    const old = Array.from({ length: GATEWAY_RECOVERY_LIMIT }, (_, i) => i)
+    const plan = planGatewayRecovery('sess-1', null, old, GATEWAY_RECOVERY_WINDOW_MS + 100)
+
+    expect(plan.attempts).toEqual([GATEWAY_RECOVERY_WINDOW_MS + 100])
+    expect(plan.recover).toBe(true)
+  })
+})

ui-tui/src/__tests__/parentLog.test.ts

@@ -0,0 +1,75 @@
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+// parentLog gates itself off under VITEST so unit tests can't pollute a real
+// ~/.hermes. To exercise the real persistence path we clear that gate, point
+// HERMES_HOME at a temp dir, and re-import the module fresh (path + enabled
+// flag are captured at module load).
+const loadFresh = async (home: string) => {
+  vi.resetModules()
+  vi.stubEnv('VITEST', '')
+  vi.stubEnv('HERMES_HOME', home)
+
+  return import('../lib/parentLog.js')
+}
+
+describe('recordParentLifecycle', () => {
+  let home: string
+
+  beforeEach(() => {
+    home = mkdtempSync(join(tmpdir(), 'hermes-parentlog-'))
+  })
+
+  afterEach(() => {
+    vi.unstubAllEnvs()
+    rmSync(home, { force: true, recursive: true })
+  })
+
+  it('appends a timestamped breadcrumb to logs/tui_gateway_crash.log', async () => {
+    const { recordParentLifecycle } = await loadFresh(home)
+
+    recordParentLifecycle('graceful-exit received signal=SIGHUP → killing gateway')
+
+    const contents = readFileSync(join(home, 'logs', 'tui_gateway_crash.log'), 'utf8')
+
+    expect(contents).toContain('[tui-parent]')
+    expect(contents).toContain('graceful-exit received signal=SIGHUP → killing gateway')
+    expect(contents).toMatch(/\d{4}-\d{2}-\d{2}T/)
+  })
+
+  it('collapses embedded newlines so a value stays one breadcrumb', async () => {
+    const { recordParentLifecycle } = await loadFresh(home)
+
+    recordParentLifecycle('uncaughtException: boom\n  at foo()\r\n  at bar()')
+
+    const lines = readFileSync(join(home, 'logs', 'tui_gateway_crash.log'), 'utf8').trimEnd().split('\n')
+
+    expect(lines).toHaveLength(1)
+    expect(lines[0]).toContain('boom ↵   at foo() ↵   at bar()')
+  })
+
+  it('caps an oversized breadcrumb so it cannot bloat the shared crash log', async () => {
+    const { recordParentLifecycle } = await loadFresh(home)
+
+    recordParentLifecycle('x'.repeat(10_000))
+
+    const line = readFileSync(join(home, 'logs', 'tui_gateway_crash.log'), 'utf8')
+
+    expect(line).toContain('[truncated 10000 chars]')
+    expect(line.length).toBeLessThan(4_500)
+  })
+
+  it('is a no-op under VITEST so tests stay hermetic', async () => {
+    vi.resetModules()
+    vi.stubEnv('VITEST', 'true')
+    vi.stubEnv('HERMES_HOME', home)
+
+    const { recordParentLifecycle } = await import('../lib/parentLog.js')
+
+    expect(() => recordParentLifecycle('should not be written')).not.toThrow()
+    expect(() => readFileSync(join(home, 'logs', 'tui_gateway_crash.log'), 'utf8')).toThrow()
+  })
+})

ui-tui/src/app/createGatewayEventHandler.ts

@@ -76,7 +76,7 @@ const normalizeSubagentStatus = (status: unknown, fallback: SubagentStatus): Sub
 
 export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: GatewayEvent) => void {
   const { rpc } = ctx.gateway
-  const { STARTUP_RESUME_ID, newSession, resumeById, setCatalog } = ctx.session
+  const { STARTUP_RESUME_ID, newSession, recoverSidRef, resumeById, setCatalog } = ctx.session
   const { bellOnComplete, stdout, sys } = ctx.system
   const { appendMessage, panel, setHistoryItems } = ctx.transcript
   const { setInput } = ctx.composer
@@ -303,6 +303,23 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev:
       })
       .catch((e: unknown) => turnController.pushActivity(`command catalog unavailable: ${rpcErrorMessage(e)}`, 'info'))
 
+    // Crash recovery: a respawn triggered by an unexpected gateway death
+    // resumes the session that was live, not a brand-new one. One-shot — the
+    // ref is cleared so an ordinary later restart still forges/resumes per
+    // config. No startup prompt here (this is mid-session, not a cold boot).
+    const recoverSid = recoverSidRef?.current
+
+    if (recoverSidRef && recoverSid) {
+      recoverSidRef.current = null
+      resumeById(recoverSid)
+      // After resumeById: it synchronously sets status to 'resuming…' on entry,
+      // so override it here to keep the distinct "recovering" label visible for
+      // the duration of the resume RPC (which later flips status to 'ready').
+      patchUiState({ status: 'recovering session…' })
+
+      return
+    }
+
     if (STARTUP_RESUME_ID) {
       patchUiState({ status: 'resuming…' })
       resumeById(STARTUP_RESUME_ID)

ui-tui/src/app/gatewayRecovery.ts

@@ -0,0 +1,35 @@
+// Crash-recovery budget for the gateway exit handler. A gateway that
+// crash-loops on startup must not let the TUI spawn-storm, so respawn+resume
+// attempts are capped to GATEWAY_RECOVERY_LIMIT within a sliding
+// GATEWAY_RECOVERY_WINDOW_MS; past the budget the app falls back to the inert
+// "gateway exited" state. Kept pure (no refs/UI) so the bound — including the
+// crash-loop case — is unit-testable.
+export const GATEWAY_RECOVERY_LIMIT = 3
+export const GATEWAY_RECOVERY_WINDOW_MS = 60_000
+
+export interface RecoveryPlan {
+  // Attempt timestamps to persist (the pruned window, plus `now` iff recovering).
+  attempts: number[]
+  recover: boolean
+  // Session to resume — the live sid, or the not-yet-consumed recovery target
+  // when the live sid was already cleared by a prior exit.
+  sid: null | string
+}
+
+// Decide whether to respawn+resume after a gateway death. `liveSid` is the
+// current session (nulled on the first exit); `recoverSid` is a pending
+// recovery target carried across a respawn that died before gateway.ready —
+// so a startup crash-loop keeps retrying the same session up to the budget
+// instead of stranding it after one attempt.
+export function planGatewayRecovery(
+  liveSid: null | string,
+  recoverSid: null | string,
+  attempts: number[],
+  now: number
+): RecoveryPlan {
+  const sid = liveSid ?? recoverSid
+  const recent = attempts.filter(t => now - t < GATEWAY_RECOVERY_WINDOW_MS)
+  const recover = Boolean(sid) && recent.length < GATEWAY_RECOVERY_LIMIT
+
+  return { attempts: recover ? [...recent, now] : recent, recover, sid }
+}

ui-tui/src/app/interfaces.ts

@@ -239,6 +239,10 @@ export interface GatewayEventHandlerContext {
     STARTUP_RESUME_ID: string
     colsRef: MutableRefObject<number>
     newSession: (msg?: string, title?: string) => void
+    // Set by useMainApp's exit handler to the session that was live when the
+    // gateway died unexpectedly; consumed once by the next `gateway.ready` so a
+    // respawn resumes that session instead of forging a fresh one.
+    recoverSidRef?: MutableRefObject<null | string>
     resetSession: () => void
     resumeById: (id: string) => void
     setCatalog: StateSetter<null | SlashCatalog>

ui-tui/src/app/useMainApp.ts

@@ -30,6 +30,7 @@ import type { Msg, PanelSection, SlashCatalog } from '../types.js'
 
 import { createGatewayEventHandler } from './createGatewayEventHandler.js'
 import { createSlashHandler } from './createSlashHandler.js'
+import { planGatewayRecovery } from './gatewayRecovery.js'
 import { getInputSelection } from './inputSelectionStore.js'
 import { type GatewayRpc, type TranscriptRow } from './interfaces.js'
 import { $overlayState, patchOverlayState } from './overlayStore.js'
@@ -200,6 +201,8 @@ export function useMainApp(gw: GatewayClient) {
   const terminalHintsShownRef = useRef(new Set<string>())
   const historyItemsRef = useRef(historyItems)
   const lastUserMsgRef = useRef(lastUserMsg)
+  const recoverSidRef = useRef<null | string>(null)
+  const recoveryAtRef = useRef<number[]>([])
   const msgIdsRef = useRef(new WeakMap<Msg, string>())
   const msgIdSeqRef = useRef(0)
   const heightCachesRef = useRef(new Map<string, Map<string, number>>())
@@ -694,6 +697,7 @@ export function useMainApp(gw: GatewayClient) {
           STARTUP_RESUME_ID,
           colsRef,
           newSession: session.newSession,
+          recoverSidRef,
           resetSession: session.resetSession,
           resumeById: session.resumeById,
           setCatalog
@@ -734,7 +738,34 @@ export function useMainApp(gw: GatewayClient) {
 
     const exitHandler = () => {
       turnController.reset()
+
+      // A still-owned child dying while the TUI is alive is an *unexpected*
+      // death — a user /quit exits Node before this fires, and a replaced child
+      // is identity-skipped in GatewayClient. Rather than stranding a long
+      // session (the user's complaint), respawn the gateway and resume the
+      // persisted session via the next gateway.ready, so a single crash / OOM /
+      // signal doesn't lose their work. planGatewayRecovery bounds the attempts
+      // so a gateway that crash-loops on startup can't spawn-storm, and falls
+      // back to recoverSidRef when sid was already cleared by a prior exit.
+      const plan = planGatewayRecovery(getUiState().sid, recoverSidRef.current, recoveryAtRef.current, Date.now())
+
+      // Clear sid immediately: while the gateway is down, sid-guarded effects
+      // (session.active_list poll, queue drain) would otherwise fire RPCs at a
+      // dead/respawning gateway. recoverSidRef carries the session forward, and
+      // resumeById restores sid once the fresh gateway is ready.
+      recoveryAtRef.current = plan.attempts
       patchUiState({ busy: false, sid: null, status: 'gateway exited' })
+
+      if (plan.recover && plan.sid) {
+        recoverSidRef.current = plan.sid
+        turnController.pushActivity('gateway exited · recovering session…', 'warn')
+        sys('gateway exited — recovering your session (any in-flight reply was lost)')
+        gw.start()
+
+        return
+      }
+
+      recoverSidRef.current = null
       turnController.pushActivity('gateway exited · /logs to inspect', 'error')
       sys('error: gateway exited')
     }

ui-tui/src/entry.tsx

@@ -11,6 +11,7 @@ import { setupGracefulExit } from './lib/gracefulExit.js'
 import { formatBytes, type HeapDumpResult, performHeapDump } from './lib/memory.js'
 import { type MemorySnapshot, startMemoryMonitor } from './lib/memoryMonitor.js'
 import { openExternalUrl } from './lib/openExternalUrl.js'
+import { recordParentLifecycle } from './lib/parentLog.js'
 import { clampStdoutDimensions } from './lib/terminalDimensions.js'
 import { resetTerminalModes } from './lib/terminalModes.js'
 
@@ -56,16 +57,25 @@ setupGracefulExit({
   onError: (scope, err) => {
     const message = err instanceof Error ? `${err.name}: ${err.message}\n${err.stack ?? ''}` : String(err)
 
+    recordParentLifecycle(`${scope}: ${message.split('\n')[0]?.slice(0, 400) ?? ''}`)
     process.stderr.write(`hermes-tui lifecycle ${scope}: ${message.slice(0, 2000)}\n`)
   },
   onSignal: signal => {
+    // The next line in the crash log is the child's `=== SIGTERM received ===`
+    // (gw.kill forwards SIGTERM regardless of which signal hit us) — this is
+    // what tells SIGHUP (terminal/SSH dropped) apart from a real SIGTERM.
+    recordParentLifecycle(`graceful-exit received signal=${signal} → killing gateway`)
     resetTerminalModes()
     process.stderr.write(`hermes-tui lifecycle: received ${signal}\n`)
   }
 })
 
 const stopMemoryMonitor = startMemoryMonitor({
   onCritical: (snap, dump) => {
+    // process.exit(137) closes the child's stdin → the gateway logs a clean
+    // EOF, NOT SIGTERM. Recording it here is the only way a crash report can
+    // attribute a death to Node OOM rather than a signal-driven kill.
+    recordParentLifecycle(`memory-critical process.exit(137) heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)} dump=${dump?.heapPath ?? 'failed'}`)
     resetTerminalModes()
     process.stderr.write(`hermes-tui lifecycle: memory critical exit heap=${formatBytes(snap.heapUsed)} rss=${formatBytes(snap.rss)}\n`)
     process.stderr.write(dumpNotice(snap, dump))