fix(state): in-product recovery after deleted-WAL guard fires (#110054)

agent/turn_explainers.py

@@ -120,19 +120,23 @@
         "pending_messages/pending-*.json."
     ),
     "deleted_wal": (
-        "the turn was stopped because a live Hermes process held a retired "
-        "state.db-wal generation after its pathname was deleted or "
+        "Hermes paused saving this chat because another Hermes process replaced "
+        "its session database. Nothing is lost. Click Recover / run "
+        "`hermes {profile_arg}doctor --fix`.\n\n"
+        "Operator runbook: the turn was stopped because a live Hermes process held a "
+        "retired state.db-wal generation after its pathname was deleted or "
         "replaced. Stop the gateway, dashboard, and cron writers; "
         "do not overwrite the current state.db or delete its sidecars. "
         "Check the logs for whether Hermes captured the retired generation, "
         "then read the adjacent state.db.retired-wal-*/manifest.json. If "
         "manifest.main.mode is `copied`, inspect that artifact with `hermes "
-        "sessions recover --source <state.db.retired-wal-*/state.db> "
+        "{profile_arg}sessions recover --source <state.db.retired-wal-*/state.db> "
         "--inspect-only` before deciding whether its committed frames belong "
         "on the current database. A `header_only` artifact is forensic and "
         "does not contain a copied state.db to inspect. Unwritten messages "
         "were diverted to sessions/<session_id>.jsonl and, on the gateway, "
-        "pending_messages/pending-*.json."
+        "pending_messages/pending-*.json.\n"
+        "Recovery guide: https://hermes.nousresearch.com/docs/user-guide/session-storage-recovery"
     ),
     "corrupt": (
         "the turn was stopped because the state database "
@@ -328,7 +332,7 @@ def _format_turn_completion_explanation(
             body = _PERSISTENCE_CAUSE_EXPLANATIONS.get(
                 persistence_cause or "unknown", _PERSISTENCE_DEFAULT_EXPLANATION
             )
-            if persistence_cause in ("corrupt", "fts_index"):
+            if persistence_cause in ("corrupt", "fts_index", "deleted_wal"):
                 # Copy-pasteable, so name the store that actually failed and pin the profile:
                 # a multi-profile backend (Desktop serve) hosts sessions whose state.db is NOT
                 # the process default, and a bare `hermes` follows active_profile (#105887).

apps/desktop/src/api/system.ts

@@ -241,8 +241,8 @@ export function getGhAuthStatus(refresh = false): Promise<{ available: boolean;
 // getActionStatus().
 // ---------------------------------------------------------------------------
 
-export function runDoctor(): Promise<ActionResponse> {
-  return hermesApi<ActionResponse>({ path: '/api/ops/doctor', method: 'POST', body: {} })
+export function runDoctor(fix = false): Promise<ActionResponse> {
+  return hermesApi<ActionResponse>({ path: '/api/ops/doctor', method: 'POST', body: { fix } })
 }
 
 export function runSecurityAudit(): Promise<ActionResponse> {

apps/desktop/src/app/command-center/maintenance.tsx

@@ -206,6 +206,12 @@ export function MaintenancePanel() {
           label={mm.doctor}
           onRun={() => void launch(mm.doctor, runDoctor)}
         />
+        <OpRow
+          description="Health-check and auto-fix session storage conflicts, retired WAL holders, and repairable issues"
+          disabled={actionStatus?.running === true}
+          label="Run doctor --fix"
+          onRun={() => void launch('Doctor (--fix)', () => runDoctor(true))}
+        />
         <OpRow
           description={mm.securityAuditDesc}
           disabled={actionStatus?.running === true}

apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/message-stream.ts

@@ -1,5 +1,6 @@
 import type { BillingBlock } from '@hermes/shared'
 
+import { runDoctor } from '@/api/system'
 import { burstVibeHearts } from '@/components/chat/vibe-hearts'
 import { reportFirstBuildTurnComplete } from '@/components/onboarding-chat/first-build'
 import { translateNow } from '@/i18n'
@@ -367,6 +368,35 @@ export function handleMessageStreamEvent(ctx: GatewayEventContext): boolean {
       notify({ kind: 'warning', message: payload.warning })
     }
 
+    // In-product recovery action for deleted-WAL persistence failure (#110054)
+    if (typeof payload?.failure_reason === 'string' && payload.failure_reason.includes('deleted_wal')) {
+      notify({
+        kind: 'error',
+        title: translateNow('common.error'),
+        message: 'Hermes paused saving this chat because another process replaced its session database.',
+        action: {
+          label: 'Recover',
+          onClick: () => {
+            void runDoctor(true)
+              .then(() => {
+                notify({
+                  kind: 'success',
+                  title: 'Recovery started',
+                  message: 'Hermes doctor is repairing database access in the background.'
+                })
+              })
+              .catch((err: unknown) => {
+                notify({
+                  kind: 'error',
+                  title: 'Recovery failed to start',
+                  message: err instanceof Error ? err.message : String(err)
+                })
+              })
+          }
+        }
+      })
+    }
+
     if (isActiveEvent) {
       setTurnStartedAt(null)
 

apps/desktop/src/app/session/hooks/use-message-stream/terminal-error-frame.test.tsx

@@ -112,4 +112,20 @@ describe('terminal error message.complete frames', () => {
     expect(bubble?.error).toBe('kaput')
     expect(bubble?.errorSurface).toBeUndefined()
   })
+
+  it('handles terminal error with deleted_wal failure_reason and preserves failure state', async () => {
+    mountStream()
+    await start()
+    await delta('…')
+
+    await completeWithError({
+      text: 'Hermes paused saving this chat',
+      error: 'session storage could not be written',
+      failure_reason: 'session_persistence_failed:deleted_wal'
+    })
+
+    const bubble = lastAssistant()
+    expect(bubble?.error).toBe('session storage could not be written')
+    expect(getState().busy).toBe(false)
+  })
 })