refactor(onboard): make FSM resume recovery a single explicit path (#6227)

src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts

@@ -10,7 +10,7 @@ import * as agentDefs from "../../agent/defs";
 import * as agentRuntime from "../../agent/runtime";
 import * as gatewayRuntime from "../../gateway-runtime-action";
 import * as nim from "../../inference/nim";
-import * as resumeRepair from "../../onboard/resume-machine-repair";
+import * as sessionRecovery from "../../onboard/session-recovery";
 import * as sandboxList from "../../openshell-sandbox-list";
 import * as sandboxVersion from "../../sandbox/version";
 import type { Session } from "../../state/onboard-session";
@@ -180,7 +180,7 @@ describe("rebuild resume snapshot repair", () => {
           observed.preRepairGatewayStatus = reopened.steps.gateway.status;
           observed.preRepairStatus = reopened.status;
           observed.preRepairResumable = reopened.resumable;
-          resumeRepair.repairResumeMachineSnapshot(reopened, "2026-06-01T00:01:00.000Z");
+          sessionRecovery.applySessionRecovery(reopened, "2026-06-01T00:01:00.000Z");
           observed.repairedMachineState = reopened.machine.state;
           observed.sandboxEnvInsideOnboard = process.env.NEMOCLAW_SANDBOX_NAME ?? null;
           throw new Error("stop-after-resume-repair-probe");

src/lib/onboard.ts

@@ -141,8 +141,8 @@ const {
   MessagingHostStateApplier,
 } = require("./onboard/messaging-channel-setup") as typeof import("./onboard/messaging-channel-setup");
 const {
-  repairResumeMachineSnapshot,
-}: typeof import("./onboard/resume-machine-repair") = require("./onboard/resume-machine-repair");
+  applySessionRecovery,
+}: typeof import("./onboard/session-recovery") = require("./onboard/session-recovery");
 const bedrockRuntimeOnboard: typeof import("./onboard/bedrock-runtime") =
   require("./onboard/bedrock-runtime");
 const {
@@ -4222,7 +4222,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise<void> {
         createSession: onboardSession.createSession,
         saveSession: onboardSession.saveSession,
         updateSession: onboardSession.updateSession,
-        repairResumeMachineSnapshot,
+        applySessionRecovery,
         setOnboardBrandingAgent,
         getResumeConfigConflicts,
         recordResumeConflict: (conflict) => onboardRuntimeBoundary.recordResumeConflict(conflict),

src/lib/onboard/exit-step-failure.test.ts

@@ -27,17 +27,21 @@ const restoreOriginalHome =
       };
 let tmpDir: string;
 let session: typeof sessionModule;
+let machineEvents: typeof import("./machine/events");
 
 beforeEach(async () => {
   tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-exit-step-failure-"));
   process.env.HOME = tmpDir;
   vi.resetModules();
   session = await import("../state/onboard-session");
+  machineEvents = await import("./machine/events");
+  machineEvents.clearOnboardMachineEventListeners();
   session.clearSession();
   resetOnboardResumeHintForTests();
 });
 
 afterEach(() => {
+  machineEvents.clearOnboardMachineEventListeners();
   session.clearSession();
   fs.rmSync(tmpDir, { recursive: true, force: true });
   restoreOriginalHome();
@@ -119,15 +123,34 @@ describe("terminal step failure helper", () => {
   });
 
   it("leaves sessions without a started step untouched", () => {
-    const markStepFailed = vi.fn(() => session.createSession());
+    const finalizeIncompleteOnboardStep = vi.fn(() => session.createSession());
 
     expect(
       markLastStartedStepFailed(
-        { loadSession: () => session.createSession(), markStepFailed },
+        { loadSession: () => session.createSession(), finalizeIncompleteOnboardStep },
         "boom",
       ),
     ).toBeNull();
-    expect(markStepFailed).not.toHaveBeenCalled();
+    expect(finalizeIncompleteOnboardStep).not.toHaveBeenCalled();
+  });
+
+  it("records exactly one failed transition even if the exit backstop fires twice", () => {
+    session.saveSession(session.createSession({ lastStepStarted: "inference" }));
+    const emitted: string[] = [];
+    machineEvents.addOnboardMachineEventListener((event) => emitted.push(event.type));
+
+    markLastStartedStepFailed(session, "Onboarding exited before the step completed.");
+    const afterFirst = requireLoadedSession();
+    const revisionAfterFirst = afterFirst.machine.revision;
+    expect(afterFirst.machine.state).toBe("failed");
+
+    // A second backstop invocation must not re-transition an already-terminal
+    // machine or bump the revision again.
+    markLastStartedStepFailed(session, "Onboarding exited before the step completed.");
+    const afterSecond = requireLoadedSession();
+    expect(afterSecond.machine.state).toBe("failed");
+    expect(afterSecond.machine.revision).toBe(revisionAfterFirst);
+    expect(emitted).toEqual(["state.failed", "onboard.failed"]);
   });
 });
 
@@ -226,7 +249,7 @@ const resumableSession = { lastStepStarted: "inference" };
 registerIncompleteOnboardExitFailureHandler(
   {
     loadSession: () => resumableSession,
-    markStepFailed: () => resumableSession,
+    finalizeIncompleteOnboardStep: () => resumableSession,
   },
   () => false,
   "Onboarding exited before the step completed.",

src/lib/onboard/exit-step-failure.ts

@@ -2,15 +2,11 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import type { Session } from "../state/onboard-session";
-import {
-  LEGACY_MACHINE_STEP_MUTATION_OPTIONS,
-  type StepMutationOptions,
-} from "../state/onboard-step-mutation";
 import { printOnboardResumeHint } from "./resume-hint";
 
 export interface ExitStepFailureSessionDeps {
   loadSession(): Pick<Session, "lastStepStarted"> | null;
-  markStepFailed(stepName: string, message?: string | null, options?: StepMutationOptions): Session;
+  finalizeIncompleteOnboardStep(stepName: string, message?: string | null): Session | null;
 }
 
 export interface OnboardExitFailureProcessLike {
@@ -28,13 +24,14 @@ export function markLastStartedStepFailed(
   message: string,
 ): Session | null {
   // Repairs the invalid state where onboard/rebuild exits nonzero after a step
-  // starts but before normal completion handlers can run. Keep the explicit
-  // legacy machine mutation until those process-exit paths have a single
-  // terminal lifecycle owner; covered by exit-step-failure, rebuild-flow, and
-  // onboard-exit-handler tests.
+  // starts but before normal completion handlers can run. Routes through the
+  // single terminal-failure owner (finalizeIncompleteOnboardStep), which
+  // validates the failed transition and is idempotent against an already
+  // terminal machine, rather than the legacy step-mutation escape hatch.
+  // Covered by exit-step-failure, rebuild-flow, and onboard-exit-handler tests.
   const failedStep = deps.loadSession()?.lastStepStarted;
   if (!failedStep) return null;
-  return deps.markStepFailed(failedStep, message, LEGACY_MACHINE_STEP_MUTATION_OPTIONS);
+  return deps.finalizeIncompleteOnboardStep(failedStep, message);
 }
 
 export function registerIncompleteOnboardExitFailureHandler(

src/lib/onboard/machine/runtime.ts

@@ -129,6 +129,30 @@ export class OnboardRuntime {
     return session;
   }
 
+  /**
+   * Attempts observer dispatch for a durable recovery receipt.
+   *
+   * The receipt stays on the snapshot until the next machine transition, so a
+   * process restart before that transition retries the same deterministic ID.
+   * Observer delivery remains best-effort by design.
+   */
+  async emitPendingSessionRecovery(): Promise<Session> {
+    const session = this.ensureSession();
+    const receipt = session.machine.recoveryReceipt;
+    if (!receipt) return session;
+    this.emit("state.repair.completed", session, {
+      state: receipt.entry,
+      metadata: {
+        reason: receipt.reason,
+        entry: receipt.entry,
+        receiptId: receipt.id,
+        appliedAt: receipt.appliedAt,
+        revision: receipt.revision,
+      },
+    });
+    return session;
+  }
+
   async markStepStarted(
     stepName: string,
     options: StepMutationOptions = RECORD_ONLY_STEP_MUTATION_OPTIONS,

src/lib/onboard/machine/transitions.test.ts

@@ -147,6 +147,19 @@ describe("onboard machine transitions", () => {
     );
   });
 
+  it("never allows a terminal failed state to re-enter an agent or flow state (#6179)", () => {
+    for (const to of ["agent_setup", "openclaw", "sandbox", "policies", "init"] as const) {
+      expect(canTransitionOnboardMachineState("failed", to)).toBe(false);
+      expect(getOnboardMachineTransition("failed", to)).toBeNull();
+      expect(() => assertValidOnboardMachineTransition("failed", to)).toThrow(`failed -> ${to}`);
+    }
+    // Failure edges only ever point *into* the terminal failed state.
+    for (const transition of ONBOARD_MACHINE_TRANSITIONS) {
+      expect(transition.from).not.toBe("failed");
+      expect(transition.from).not.toBe("complete");
+    }
+  });
+
   it("keeps the next-state map aligned with the transition list", () => {
     for (const state of ONBOARD_MACHINE_STATES) {
       expect(

src/lib/onboard/machine/transitions.ts

@@ -8,6 +8,33 @@ import {
   ONBOARD_TERMINAL_MACHINE_STATES,
 } from "./types";
 
+/**
+ * The legal onboarding transition graph.
+ *
+ * There are exactly two families of edges and no others:
+ *
+ * 1. Direct edges (`ONBOARD_MACHINE_DIRECT_TRANSITIONS`) — the forward flow
+ *    plus the `inference -> provider_selection` retry and the
+ *    `sandbox -> {openclaw,agent_setup}` branch. `kind` is `advance`, `retry`,
+ *    or `branch`.
+ * 2. Failure edges (`ONBOARD_MACHINE_FAILURE_TRANSITIONS`) — every non-terminal
+ *    state may transition to `failed`. `kind` is `failure`.
+ *
+ * Terminality invariant: `complete` and `failed` are terminal and have no
+ * outgoing edges. In particular there is deliberately **no** edge out of
+ * `failed` into any agent/flow state, so a completed-then-reopened session can
+ * never take an invalid `failed -> <agent>` transition (#6179).
+ *
+ * Recovery model: resuming an interrupted run does not transition out of a
+ * terminal state. Instead a single, side-effect-free recovery pass
+ * (`applySessionRecovery`) validates and re-seats the durable snapshot at a
+ * legal non-terminal entry state before any flow handler runs and writes a
+ * deterministic recovery receipt. After `onboard.resumed`, the runtime makes a
+ * best-effort `state.repair.completed` dispatch attempt for that receipt. A
+ * restart before the next transition retries the same receipt ID. Terminal
+ * states therefore stay terminal within the graph while recovery remains
+ * explicit and observable.
+ */
 export const ONBOARD_MACHINE_DIRECT_TRANSITIONS = [
   { from: "init", to: "preflight", kind: "advance" },
   { from: "preflight", to: "gateway", kind: "advance" },

src/lib/onboard/resume-machine-repair.test.ts

@@ -13,9 +13,10 @@ import {
 } from "../state/onboard-session";
 import { advanceTo, branchTo } from "./machine/result";
 import { OnboardRuntime, type OnboardRuntimeDeps } from "./machine/runtime";
-import { repairResumeMachineSnapshot, resumeMachineState } from "./resume-machine-repair";
+import { resumeMachineState } from "./resume-machine-repair";
 import { classifyResumeMachineRepair } from "./resume-repair-policy";
 import { OnboardRuntimeBoundary } from "./runtime-boundary";
+import { applySessionRecovery } from "./session-recovery";
 
 /**
  * Builds a failed durable session while letting each test set the interrupted step.
@@ -96,7 +97,7 @@ function createBoundaryHarness(initial: Session) {
  * Replays the live resume sequence from failed snapshot repair through completion.
  */
 async function runRecordOnlyResumeSequence(initial: Session): Promise<Session> {
-  repairResumeMachineSnapshot(initial, "2026-06-01T00:01:00.000Z");
+  applySessionRecovery(initial, "2026-06-01T00:01:00.000Z");
   initial.failure = null;
   initial.status = "in_progress";
   const { boundary, getSession } = createBoundaryHarness(initial);
@@ -187,13 +188,18 @@ describe("resume machine repair", () => {
     });
 
     expect(resumeMachineState(session)).toBe("preflight");
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
-    expect(session.machine).toEqual({
+    expect(session.machine).toMatchObject({
       version: MACHINE_SNAPSHOT_VERSION,
       state: "preflight",
       stateEnteredAt: "2026-06-01T00:01:00.000Z",
       revision: 8,
+      recoveryReceipt: {
+        reason: "failed_terminal_snapshot",
+        entry: "preflight",
+        revision: 8,
+      },
     });
   });
 
@@ -234,7 +240,7 @@ describe("resume machine repair", () => {
       },
     });
 
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
     expect(session.machine).toEqual({
       version: MACHINE_SNAPSHOT_VERSION,
@@ -259,13 +265,18 @@ describe("resume machine repair", () => {
     session.steps.preflight.status = "complete";
     session.steps.gateway.status = "complete";
 
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
-    expect(session.machine).toEqual({
+    expect(session.machine).toMatchObject({
       version: MACHINE_SNAPSHOT_VERSION,
       state: "provider_selection",
       stateEnteredAt: "2026-06-01T00:01:00.000Z",
       revision: 10,
+      recoveryReceipt: {
+        reason: "reopened_complete_snapshot",
+        entry: "provider_selection",
+        revision: 10,
+      },
     });
   });
 
@@ -282,7 +293,7 @@ describe("resume machine repair", () => {
     session.resumable = false;
     session.status = "complete";
 
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
     expect(session.machine).toEqual({
       version: MACHINE_SNAPSHOT_VERSION,

src/lib/onboard/resume-machine-repair.ts

@@ -1,11 +1,10 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { MACHINE_SNAPSHOT_VERSION, type Session } from "../state/onboard-session";
+import type { Session } from "../state/onboard-session";
 import { nextMachineStateAfterCompletedStep } from "../state/onboard-step-state";
 import { machineStateFromOnboardSessionStep } from "./machine/events";
 import type { OnboardMachineState } from "./machine/types";
-import { classifyResumeMachineRepair } from "./resume-repair-policy";
 
 /**
  * Reads the legacy step-level source of truth for interrupted sessions whose
@@ -31,6 +30,12 @@ function activeStepMachineState(session: Session): OnboardMachineState | null {
 
 /**
  * Computes the nonterminal state where a failed durable session should resume.
+ *
+ * This derives the resume entry from the legacy step-level source of truth and
+ * is one of the building blocks for the single recovery pass in
+ * `session-recovery.ts` (`planSessionRecovery` / `applySessionRecovery`), which
+ * classifies, validates, and applies the recovery. Remove this bridge once step
+ * fields stop being used to derive resume state (#6227).
  */
 export function resumeMachineState(session: Session): OnboardMachineState {
   return (
@@ -39,27 +44,3 @@ export function resumeMachineState(session: Session): OnboardMachineState {
     "init"
   );
 }
-
-/**
- * Repairs legacy terminal-session/FSM boundaries during --resume.
- *
- * Source fix constraint: terminal -> resume is not a modeled FSM transition
- * yet, and legacy step fields still act as the secondary durable source for
- * resume. Remove this bridge once terminal-session recovery is represented by
- * explicit FSM recovery results or step fields stop being used to derive resume
- * state.
- */
-export function repairResumeMachineSnapshot(
-  session: Session,
-  stateEnteredAt = new Date().toISOString(),
-): Session {
-  if (classifyResumeMachineRepair(session).action !== "repair") return session;
-  const state = resumeMachineState(session);
-  session.machine = {
-    version: MACHINE_SNAPSHOT_VERSION,
-    state,
-    stateEnteredAt,
-    revision: session.machine.revision + 1,
-  };
-  return session;
-}