refactor(onboard): make FSM resume recovery a single explicit path (#6227)

src/lib/actions/sandbox/rebuild-flow.test.ts

@@ -70,7 +70,7 @@ type RebuildFlowHarness = {
   executeSandboxCommandSpy: MockInstance;
   ensureMessagingHostForwardAfterRebuildSpy: MockInstance;
   logSpy: MockInstance;
-  markStepFailedSpy: MockInstance;
+  finalizeIncompleteOnboardStepSpy: MockInstance;
   onboardSpy: MockInstance;
   registryUpdateSpy: MockInstance;
   releaseOnboardLockSpy: MockInstance;
@@ -138,12 +138,17 @@ function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSe
 }
 
 function installTerminalStepFailureMock(
-  onboardSession: { markStepFailed: (...args: unknown[]) => unknown },
+  onboardSession: { finalizeIncompleteOnboardStep: (...args: unknown[]) => unknown },
   session: RebuildFlowSession,
 ): MockInstance {
   return vi
-    .spyOn(onboardSession, "markStepFailed")
-    .mockImplementation((stepName: unknown, message: unknown, options: unknown) => {
+    .spyOn(onboardSession, "finalizeIncompleteOnboardStep")
+    .mockImplementation((stepName: unknown, message: unknown) => {
+      // Idempotent terminal owner: never re-transition an already-terminal
+      // machine.
+      if (session.machine.state === "failed" || session.machine.state === "complete") {
+        return session;
+      }
       const stepKey = String(stepName);
       const step = session.steps[stepKey] ?? createStep("pending");
       session.steps[stepKey] = step;
@@ -155,10 +160,8 @@ function installTerminalStepFailureMock(
         message: typeof message === "string" ? message : null,
         recordedAt: "2026-06-01T00:02:00.000Z",
       };
-      const updateMachine =
-        (options as { updateMachine?: boolean } | undefined)?.updateMachine === true;
-      session.machine.state = updateMachine ? "failed" : session.machine.state;
-      session.machine.revision += updateMachine ? 1 : 0;
+      session.machine.state = "failed";
+      session.machine.revision += 1;
       return session;
     });
 }
@@ -218,7 +221,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild
   const releaseOnboardLockSpy = vi
     .spyOn(onboardSession, "releaseOnboardLock")
     .mockImplementation(() => undefined);
-  const markStepFailedSpy = installTerminalStepFailureMock(onboardSession, session);
+  const finalizeIncompleteOnboardStepSpy = installTerminalStepFailureMock(onboardSession, session);
   session.sandboxName = overrides.sessionSandboxName ?? session.sandboxName;
   const sandboxEntry = {
     name: "alpha",
@@ -347,7 +350,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild
     executeSandboxCommandSpy,
     ensureMessagingHostForwardAfterRebuildSpy,
     logSpy,
-    markStepFailedSpy,
+    finalizeIncompleteOnboardStepSpy,
     onboardSpy,
     registryUpdateSpy,
     releaseOnboardLockSpy,
@@ -1025,10 +1028,9 @@ describe("rebuildSandbox flow", () => {
     ).rejects.toThrow("Recreate failed");
 
     expect(harness.releaseOnboardLockSpy).toHaveBeenCalled();
-    expect(harness.markStepFailedSpy).toHaveBeenCalledWith(
+    expect(harness.finalizeIncompleteOnboardStepSpy).toHaveBeenCalledWith(
       "sandbox",
       "Rebuild recreate failed",
-      expect.objectContaining({ updateMachine: true }),
     );
     expect(harness.session).toMatchObject({
       status: "failed",

src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts

@@ -50,7 +50,7 @@ describe("rebuild resume snapshot repair", () => {
     const agentDefs = requireDist("../../agent/defs.js");
     const agentRuntime = requireDist("../../agent/runtime.js");
     const onboardMod = requireDist("../../onboard.js");
-    const resumeRepair = requireDist("../../onboard/resume-machine-repair.js");
+    const sessionRecovery = requireDist("../../onboard/session-recovery.js");
     const onboardSession = requireDist("../../state/onboard-session.js");
     const registry = requireDist("../../state/registry.js");
     const sandboxSession = requireDist("../../state/sandbox-session.js");
@@ -147,7 +147,7 @@ describe("rebuild resume snapshot repair", () => {
         observed.preRepairMachineState = reopened.machine.state;
         observed.preRepairStatus = reopened.status;
         observed.preRepairResumable = reopened.resumable;
-        resumeRepair.repairResumeMachineSnapshot(reopened, "2026-06-01T00:01:00.000Z");
+        sessionRecovery.applySessionRecovery(reopened, "2026-06-01T00:01:00.000Z");
         observed.repairedMachineState = reopened.machine.state;
         throw new Error("stop-after-resume-repair-probe");
       }),

src/lib/onboard.ts

@@ -136,8 +136,8 @@ const {
   clearAgentScopedResumeState,
 }: typeof import("./onboard/agent-resume-state") = require("./onboard/agent-resume-state");
 const {
-  repairResumeMachineSnapshot,
-}: typeof import("./onboard/resume-machine-repair") = require("./onboard/resume-machine-repair");
+  applySessionRecovery,
+}: typeof import("./onboard/session-recovery") = require("./onboard/session-recovery");
 const {
   stopTrackedModelRouterForAgentChange,
 }: typeof import("./onboard/model-router-process") = require("./onboard/model-router-process");
@@ -4766,7 +4766,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
   try {
     onboardTrace = onboardTracing.startOnboardTrace(opts, process.env);
     let selectedMessagingChannels: string[] = [];
-    let { session, fromDockerfile } = await onboardSessionBootstrap.prepareOnboardSession(
+    let { session, fromDockerfile, recovery } = await onboardSessionBootstrap.prepareOnboardSession(
       {
         resume,
         fresh,
@@ -4783,7 +4783,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
         createSession: onboardSession.createSession,
         saveSession: onboardSession.saveSession,
         updateSession: onboardSession.updateSession,
-        repairResumeMachineSnapshot,
+        applySessionRecovery,
         setOnboardBrandingAgent,
         getResumeConfigConflicts,
         recordResumeConflict: (conflict) => onboardRuntimeBoundary.recordResumeConflict(conflict),
@@ -4794,6 +4794,14 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
       },
     );
     await onboardRuntimeBoundary.recordOnboardStarted(resume);
+    if (recovery?.action === "recover") {
+      // Exactly one explicit recovery event for the single, already-applied
+      // deterministic re-seat of a terminal snapshot to its non-terminal entry.
+      await recordRepairEvent("state.repair.completed", {
+        state: recovery.entry,
+        metadata: { reason: recovery.reason, entry: recovery.entry },
+      });
+    }
     await (resume ? recordCompatibleStateResult : recordStateResult)(
       advanceTo("preflight", { metadata: { state: "init" } }),
     );

src/lib/onboard/exit-step-failure.test.ts

@@ -112,14 +112,30 @@ describe("terminal step failure helper", () => {
   });
 
   it("leaves sessions without a started step untouched", () => {
-    const markStepFailed = vi.fn(() => session.createSession());
+    const finalizeIncompleteOnboardStep = vi.fn(() => session.createSession());
 
     expect(
       markLastStartedStepFailed(
-        { loadSession: () => session.createSession(), markStepFailed },
+        { loadSession: () => session.createSession(), finalizeIncompleteOnboardStep },
         "boom",
       ),
     ).toBeNull();
-    expect(markStepFailed).not.toHaveBeenCalled();
+    expect(finalizeIncompleteOnboardStep).not.toHaveBeenCalled();
+  });
+
+  it("records exactly one failed transition even if the exit backstop fires twice", () => {
+    session.saveSession(session.createSession({ lastStepStarted: "inference" }));
+
+    markLastStartedStepFailed(session, "Onboarding exited before the step completed.");
+    const afterFirst = requireLoadedSession();
+    const revisionAfterFirst = afterFirst.machine.revision;
+    expect(afterFirst.machine.state).toBe("failed");
+
+    // A second backstop invocation must not re-transition an already-terminal
+    // machine or bump the revision again.
+    markLastStartedStepFailed(session, "Onboarding exited before the step completed.");
+    const afterSecond = requireLoadedSession();
+    expect(afterSecond.machine.state).toBe("failed");
+    expect(afterSecond.machine.revision).toBe(revisionAfterFirst);
   });
 });

src/lib/onboard/exit-step-failure.ts

@@ -2,14 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import type { Session } from "../state/onboard-session";
-import {
-  LEGACY_MACHINE_STEP_MUTATION_OPTIONS,
-  type StepMutationOptions,
-} from "../state/onboard-step-mutation";
 
 export interface ExitStepFailureSessionDeps {
   loadSession(): Pick<Session, "lastStepStarted"> | null;
-  markStepFailed(stepName: string, message?: string | null, options?: StepMutationOptions): Session;
+  finalizeIncompleteOnboardStep(stepName: string, message?: string | null): Session | null;
 }
 
 export interface OnboardExitFailureProcessLike {
@@ -21,13 +17,14 @@ export function markLastStartedStepFailed(
   message: string,
 ): Session | null {
   // Repairs the invalid state where onboard/rebuild exits nonzero after a step
-  // starts but before normal completion handlers can run. Keep the explicit
-  // legacy machine mutation until those process-exit paths have a single
-  // terminal lifecycle owner; covered by exit-step-failure, rebuild-flow, and
-  // onboard-exit-handler tests.
+  // starts but before normal completion handlers can run. Routes through the
+  // single terminal-failure owner (finalizeIncompleteOnboardStep), which
+  // validates the failed transition and is idempotent against an already
+  // terminal machine, rather than the legacy step-mutation escape hatch.
+  // Covered by exit-step-failure, rebuild-flow, and onboard-exit-handler tests.
   const failedStep = deps.loadSession()?.lastStepStarted;
   if (!failedStep) return null;
-  return deps.markStepFailed(failedStep, message, LEGACY_MACHINE_STEP_MUTATION_OPTIONS);
+  return deps.finalizeIncompleteOnboardStep(failedStep, message);
 }
 
 export function registerIncompleteOnboardExitFailureHandler(

src/lib/onboard/machine/transitions.test.ts

@@ -147,6 +147,19 @@ describe("onboard machine transitions", () => {
     );
   });
 
+  it("never allows a terminal failed state to re-enter an agent or flow state (#6179)", () => {
+    for (const to of ["agent_setup", "openclaw", "sandbox", "policies", "init"] as const) {
+      expect(canTransitionOnboardMachineState("failed", to)).toBe(false);
+      expect(getOnboardMachineTransition("failed", to)).toBeNull();
+      expect(() => assertValidOnboardMachineTransition("failed", to)).toThrow(`failed -> ${to}`);
+    }
+    // Failure edges only ever point *into* the terminal failed state.
+    for (const transition of ONBOARD_MACHINE_TRANSITIONS) {
+      expect(transition.from).not.toBe("failed");
+      expect(transition.from).not.toBe("complete");
+    }
+  });
+
   it("keeps the next-state map aligned with the transition list", () => {
     for (const state of ONBOARD_MACHINE_STATES) {
       expect(

src/lib/onboard/machine/transitions.ts

@@ -8,6 +8,30 @@ import {
   ONBOARD_TERMINAL_MACHINE_STATES,
 } from "./types";
 
+/**
+ * The legal onboarding transition graph.
+ *
+ * There are exactly two families of edges and no others:
+ *
+ * 1. Direct edges (`ONBOARD_MACHINE_DIRECT_TRANSITIONS`) — the forward flow
+ *    plus the `inference -> provider_selection` retry and the
+ *    `sandbox -> {openclaw,agent_setup}` branch. `kind` is `advance`, `retry`,
+ *    or `branch`.
+ * 2. Failure edges (`ONBOARD_MACHINE_FAILURE_TRANSITIONS`) — every non-terminal
+ *    state may transition to `failed`. `kind` is `failure`.
+ *
+ * Terminality invariant: `complete` and `failed` are terminal and have no
+ * outgoing edges. In particular there is deliberately **no** edge out of
+ * `failed` into any agent/flow state, so a completed-then-reopened session can
+ * never take an invalid `failed -> <agent>` transition (#6179).
+ *
+ * Recovery model: resuming an interrupted run does not transition out of a
+ * terminal state. Instead a single, side-effect-free recovery pass
+ * (`applySessionRecovery`) validates and re-seats the durable snapshot at a
+ * legal non-terminal entry state before any flow handler runs, emitting one
+ * explicit `state.repair.*` event. Terminal states therefore stay terminal
+ * within the graph while recovery remains explicit and observable.
+ */
 export const ONBOARD_MACHINE_DIRECT_TRANSITIONS = [
   { from: "init", to: "preflight", kind: "advance" },
   { from: "preflight", to: "gateway", kind: "advance" },

src/lib/onboard/resume-machine-repair.test.ts

@@ -13,9 +13,10 @@ import {
 } from "../state/onboard-session";
 import { advanceTo, branchTo } from "./machine/result";
 import { OnboardRuntime, type OnboardRuntimeDeps } from "./machine/runtime";
-import { repairResumeMachineSnapshot, resumeMachineState } from "./resume-machine-repair";
+import { resumeMachineState } from "./resume-machine-repair";
 import { classifyResumeMachineRepair } from "./resume-repair-policy";
 import { OnboardRuntimeBoundary } from "./runtime-boundary";
+import { applySessionRecovery } from "./session-recovery";
 
 /**
  * Builds a failed durable session while letting each test set the interrupted step.
@@ -96,7 +97,7 @@ function createBoundaryHarness(initial: Session) {
  * Replays the live resume sequence from failed snapshot repair through completion.
  */
 async function runRecordOnlyResumeSequence(initial: Session): Promise<Session> {
-  repairResumeMachineSnapshot(initial, "2026-06-01T00:01:00.000Z");
+  applySessionRecovery(initial, "2026-06-01T00:01:00.000Z");
   initial.failure = null;
   initial.status = "in_progress";
   const { boundary, getSession } = createBoundaryHarness(initial);
@@ -187,7 +188,7 @@ describe("resume machine repair", () => {
     });
 
     expect(resumeMachineState(session)).toBe("preflight");
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
     expect(session.machine).toEqual({
       version: MACHINE_SNAPSHOT_VERSION,
@@ -234,7 +235,7 @@ describe("resume machine repair", () => {
       },
     });
 
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
     expect(session.machine).toEqual({
       version: MACHINE_SNAPSHOT_VERSION,
@@ -259,7 +260,7 @@ describe("resume machine repair", () => {
     session.steps.preflight.status = "complete";
     session.steps.gateway.status = "complete";
 
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
     expect(session.machine).toEqual({
       version: MACHINE_SNAPSHOT_VERSION,
@@ -282,7 +283,7 @@ describe("resume machine repair", () => {
     session.resumable = false;
     session.status = "complete";
 
-    repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z");
+    applySessionRecovery(session, "2026-06-01T00:01:00.000Z");
 
     expect(session.machine).toEqual({
       version: MACHINE_SNAPSHOT_VERSION,

src/lib/onboard/resume-machine-repair.ts

@@ -1,11 +1,10 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { MACHINE_SNAPSHOT_VERSION, type Session } from "../state/onboard-session";
+import type { Session } from "../state/onboard-session";
 import { nextMachineStateAfterCompletedStep } from "../state/onboard-step-state";
 import { machineStateFromOnboardSessionStep } from "./machine/events";
 import type { OnboardMachineState } from "./machine/types";
-import { classifyResumeMachineRepair } from "./resume-repair-policy";
 
 /**
  * Reads the legacy step-level source of truth for interrupted sessions whose
@@ -31,6 +30,12 @@ function activeStepMachineState(session: Session): OnboardMachineState | null {
 
 /**
  * Computes the nonterminal state where a failed durable session should resume.
+ *
+ * This derives the resume entry from the legacy step-level source of truth and
+ * is one of the building blocks for the single recovery pass in
+ * `session-recovery.ts` (`planSessionRecovery` / `applySessionRecovery`), which
+ * classifies, validates, and applies the recovery. Remove this bridge once step
+ * fields stop being used to derive resume state.
  */
 export function resumeMachineState(session: Session): OnboardMachineState {
   return (
@@ -39,27 +44,3 @@ export function resumeMachineState(session: Session): OnboardMachineState {
     "init"
   );
 }
-
-/**
- * Repairs legacy terminal-session/FSM boundaries during --resume.
- *
- * Source fix constraint: terminal -> resume is not a modeled FSM transition
- * yet, and legacy step fields still act as the secondary durable source for
- * resume. Remove this bridge once terminal-session recovery is represented by
- * explicit FSM recovery results or step fields stop being used to derive resume
- * state.
- */
-export function repairResumeMachineSnapshot(
-  session: Session,
-  stateEnteredAt = new Date().toISOString(),
-): Session {
-  if (classifyResumeMachineRepair(session).action !== "repair") return session;
-  const state = resumeMachineState(session);
-  session.machine = {
-    version: MACHINE_SNAPSHOT_VERSION,
-    state,
-    stateEnteredAt,
-    revision: session.machine.revision + 1,
-  };
-  return session;
-}

src/lib/onboard/session-bootstrap.test.ts

@@ -4,8 +4,8 @@
 import { describe, expect, it, vi } from "vitest";
 
 import { createSession, type Session } from "../state/onboard-session";
-import { prepareOnboardSession, type OnboardSessionBootstrapDeps } from "./session-bootstrap";
 import type { ResumeConfigConflict } from "./resume-config";
+import { type OnboardSessionBootstrapDeps, prepareOnboardSession } from "./session-bootstrap";
 
 class ExitError extends Error {
   constructor(readonly code: number) {
@@ -43,7 +43,9 @@ function createDeps(
       session = next;
       return next;
     }),
-    repairResumeMachineSnapshot: vi.fn((current: Session) => current),
+    applySessionRecovery: vi.fn(
+      () => ({ action: "keep", reason: "nonterminal_snapshot" }) as const,
+    ),
     setOnboardBrandingAgent: vi.fn(),
     getResumeConfigConflicts: vi.fn(() => []),
     recordResumeConflict: vi.fn(async () => undefined),
@@ -117,7 +119,8 @@ describe("prepareOnboardSession", () => {
     expect(result.session?.mode).toBe("non-interactive");
     expect(result.session?.failure).toBeNull();
     expect(result.session?.status).toBe("in_progress");
-    expect(deps.repairResumeMachineSnapshot).toHaveBeenCalledWith(initial);
+    expect(deps.applySessionRecovery).toHaveBeenCalledWith(initial);
+    expect(result.recovery).toEqual({ action: "keep", reason: "nonterminal_snapshot" });
     expect(deps.setOnboardBrandingAgent).toHaveBeenCalledWith("hermes");
   });