From f96da058831e11d67182d0c0a5931577c0e404f6 Mon Sep 17 00:00:00 2001 From: Atulya Singh Date: Fri, 3 Jul 2026 22:38:26 +0530 Subject: [PATCH] refactor(onboard): make FSM resume recovery a single explicit path (#6227) Replace the three implicit onboarding recovery mechanisms with explicit, deterministic FSM semantics, the foundational slice of #6227. - Consolidate resume repair into one validated, side-effect-free recovery pass (planSessionRecovery/applySessionRecovery in session-recovery.ts). It classifies the durable snapshot, computes and validates a single legal non-terminal entry, re-seats the snapshot, and surfaces the decision so the caller emits exactly one explicit state.repair.completed event. Replaces the implicit repairResumeMachineSnapshot rewrite. - Introduce a single synchronous terminal-failure owner (finalizeIncompleteOnboardStep) for exception/signal/nonzero-exit paths. It validates the failed transition and is idempotent against an already-terminal machine, so exactly one failed transition and one terminal event are recorded. Removes the last production use of LEGACY_MACHINE_STEP_MUTATION_OPTIONS. - Document the legal transition graph and terminality invariant, and assert that a terminal failed state can never re-enter an agent/flow state (#6179). No persisted SandboxCreateIntent or schema change is introduced. Signed-off-by: Atulya Singh --- src/lib/actions/sandbox/rebuild-flow.test.ts | 26 ++-- .../sandbox/rebuild-resume-snapshot.test.ts | 4 +- src/lib/onboard.ts | 16 ++- src/lib/onboard/exit-step-failure.test.ts | 22 ++- src/lib/onboard/exit-step-failure.ts | 17 +-- src/lib/onboard/machine/transitions.test.ts | 13 ++ src/lib/onboard/machine/transitions.ts | 24 ++++ src/lib/onboard/resume-machine-repair.test.ts | 13 +- src/lib/onboard/resume-machine-repair.ts | 33 +---- src/lib/onboard/session-bootstrap.test.ts | 9 +- src/lib/onboard/session-bootstrap.ts | 16 ++- src/lib/onboard/session-recovery.test.ts | 133 ++++++++++++++++++ src/lib/onboard/session-recovery.ts | 105 ++++++++++++++ src/lib/state/onboard-session.ts | 66 ++++++++- 14 files changed, 425 insertions(+), 72 deletions(-) create mode 100644 src/lib/onboard/session-recovery.test.ts create mode 100644 src/lib/onboard/session-recovery.ts diff --git a/src/lib/actions/sandbox/rebuild-flow.test.ts b/src/lib/actions/sandbox/rebuild-flow.test.ts index e8b0aaedc84..5a6f2ab59ee 100644 --- a/src/lib/actions/sandbox/rebuild-flow.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow.test.ts @@ -70,7 +70,7 @@ type RebuildFlowHarness = { executeSandboxCommandSpy: MockInstance; ensureMessagingHostForwardAfterRebuildSpy: MockInstance; logSpy: MockInstance; - markStepFailedSpy: MockInstance; + finalizeIncompleteOnboardStepSpy: MockInstance; onboardSpy: MockInstance; registryUpdateSpy: MockInstance; releaseOnboardLockSpy: MockInstance; @@ -138,12 +138,17 @@ function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSe } function installTerminalStepFailureMock( - onboardSession: { markStepFailed: (...args: unknown[]) => unknown }, + onboardSession: { finalizeIncompleteOnboardStep: (...args: unknown[]) => unknown }, session: RebuildFlowSession, ): MockInstance { return vi - .spyOn(onboardSession, "markStepFailed") - .mockImplementation((stepName: unknown, message: unknown, options: unknown) => { + .spyOn(onboardSession, "finalizeIncompleteOnboardStep") + .mockImplementation((stepName: unknown, message: unknown) => { + // Idempotent terminal owner: never re-transition an already-terminal + // machine. + if (session.machine.state === "failed" || session.machine.state === "complete") { + return session; + } const stepKey = String(stepName); const step = session.steps[stepKey] ?? createStep("pending"); session.steps[stepKey] = step; @@ -155,10 +160,8 @@ function installTerminalStepFailureMock( message: typeof message === "string" ? message : null, recordedAt: "2026-06-01T00:02:00.000Z", }; - const updateMachine = - (options as { updateMachine?: boolean } | undefined)?.updateMachine === true; - session.machine.state = updateMachine ? "failed" : session.machine.state; - session.machine.revision += updateMachine ? 1 : 0; + session.machine.state = "failed"; + session.machine.revision += 1; return session; }); } @@ -218,7 +221,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild const releaseOnboardLockSpy = vi .spyOn(onboardSession, "releaseOnboardLock") .mockImplementation(() => undefined); - const markStepFailedSpy = installTerminalStepFailureMock(onboardSession, session); + const finalizeIncompleteOnboardStepSpy = installTerminalStepFailureMock(onboardSession, session); session.sandboxName = overrides.sessionSandboxName ?? session.sandboxName; const sandboxEntry = { name: "alpha", @@ -347,7 +350,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild executeSandboxCommandSpy, ensureMessagingHostForwardAfterRebuildSpy, logSpy, - markStepFailedSpy, + finalizeIncompleteOnboardStepSpy, onboardSpy, registryUpdateSpy, releaseOnboardLockSpy, @@ -1025,10 +1028,9 @@ describe("rebuildSandbox flow", () => { ).rejects.toThrow("Recreate failed"); expect(harness.releaseOnboardLockSpy).toHaveBeenCalled(); - expect(harness.markStepFailedSpy).toHaveBeenCalledWith( + expect(harness.finalizeIncompleteOnboardStepSpy).toHaveBeenCalledWith( "sandbox", "Rebuild recreate failed", - expect.objectContaining({ updateMachine: true }), ); expect(harness.session).toMatchObject({ status: "failed", diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index 379b5f1de4e..31e14859753 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -50,7 +50,7 @@ describe("rebuild resume snapshot repair", () => { const agentDefs = requireDist("../../agent/defs.js"); const agentRuntime = requireDist("../../agent/runtime.js"); const onboardMod = requireDist("../../onboard.js"); - const resumeRepair = requireDist("../../onboard/resume-machine-repair.js"); + const sessionRecovery = requireDist("../../onboard/session-recovery.js"); const onboardSession = requireDist("../../state/onboard-session.js"); const registry = requireDist("../../state/registry.js"); const sandboxSession = requireDist("../../state/sandbox-session.js"); @@ -147,7 +147,7 @@ describe("rebuild resume snapshot repair", () => { observed.preRepairMachineState = reopened.machine.state; observed.preRepairStatus = reopened.status; observed.preRepairResumable = reopened.resumable; - resumeRepair.repairResumeMachineSnapshot(reopened, "2026-06-01T00:01:00.000Z"); + sessionRecovery.applySessionRecovery(reopened, "2026-06-01T00:01:00.000Z"); observed.repairedMachineState = reopened.machine.state; throw new Error("stop-after-resume-repair-probe"); }), diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 110cc6ab5dd..0070cc47855 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -136,8 +136,8 @@ const { clearAgentScopedResumeState, }: typeof import("./onboard/agent-resume-state") = require("./onboard/agent-resume-state"); const { - repairResumeMachineSnapshot, -}: typeof import("./onboard/resume-machine-repair") = require("./onboard/resume-machine-repair"); + applySessionRecovery, +}: typeof import("./onboard/session-recovery") = require("./onboard/session-recovery"); const { stopTrackedModelRouterForAgentChange, }: typeof import("./onboard/model-router-process") = require("./onboard/model-router-process"); @@ -4766,7 +4766,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { try { onboardTrace = onboardTracing.startOnboardTrace(opts, process.env); let selectedMessagingChannels: string[] = []; - let { session, fromDockerfile } = await onboardSessionBootstrap.prepareOnboardSession( + let { session, fromDockerfile, recovery } = await onboardSessionBootstrap.prepareOnboardSession( { resume, fresh, @@ -4783,7 +4783,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { createSession: onboardSession.createSession, saveSession: onboardSession.saveSession, updateSession: onboardSession.updateSession, - repairResumeMachineSnapshot, + applySessionRecovery, setOnboardBrandingAgent, getResumeConfigConflicts, recordResumeConflict: (conflict) => onboardRuntimeBoundary.recordResumeConflict(conflict), @@ -4794,6 +4794,14 @@ async function onboard(opts: OnboardOptions = {}): Promise { }, ); await onboardRuntimeBoundary.recordOnboardStarted(resume); + if (recovery?.action === "recover") { + // Exactly one explicit recovery event for the single, already-applied + // deterministic re-seat of a terminal snapshot to its non-terminal entry. + await recordRepairEvent("state.repair.completed", { + state: recovery.entry, + metadata: { reason: recovery.reason, entry: recovery.entry }, + }); + } await (resume ? recordCompatibleStateResult : recordStateResult)( advanceTo("preflight", { metadata: { state: "init" } }), ); diff --git a/src/lib/onboard/exit-step-failure.test.ts b/src/lib/onboard/exit-step-failure.test.ts index db1a8eeaa65..bee04fece0d 100644 --- a/src/lib/onboard/exit-step-failure.test.ts +++ b/src/lib/onboard/exit-step-failure.test.ts @@ -112,14 +112,30 @@ describe("terminal step failure helper", () => { }); it("leaves sessions without a started step untouched", () => { - const markStepFailed = vi.fn(() => session.createSession()); + const finalizeIncompleteOnboardStep = vi.fn(() => session.createSession()); expect( markLastStartedStepFailed( - { loadSession: () => session.createSession(), markStepFailed }, + { loadSession: () => session.createSession(), finalizeIncompleteOnboardStep }, "boom", ), ).toBeNull(); - expect(markStepFailed).not.toHaveBeenCalled(); + expect(finalizeIncompleteOnboardStep).not.toHaveBeenCalled(); + }); + + it("records exactly one failed transition even if the exit backstop fires twice", () => { + session.saveSession(session.createSession({ lastStepStarted: "inference" })); + + markLastStartedStepFailed(session, "Onboarding exited before the step completed."); + const afterFirst = requireLoadedSession(); + const revisionAfterFirst = afterFirst.machine.revision; + expect(afterFirst.machine.state).toBe("failed"); + + // A second backstop invocation must not re-transition an already-terminal + // machine or bump the revision again. + markLastStartedStepFailed(session, "Onboarding exited before the step completed."); + const afterSecond = requireLoadedSession(); + expect(afterSecond.machine.state).toBe("failed"); + expect(afterSecond.machine.revision).toBe(revisionAfterFirst); }); }); diff --git a/src/lib/onboard/exit-step-failure.ts b/src/lib/onboard/exit-step-failure.ts index a00c97d17ab..399c9a27b9f 100644 --- a/src/lib/onboard/exit-step-failure.ts +++ b/src/lib/onboard/exit-step-failure.ts @@ -2,14 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import type { Session } from "../state/onboard-session"; -import { - LEGACY_MACHINE_STEP_MUTATION_OPTIONS, - type StepMutationOptions, -} from "../state/onboard-step-mutation"; export interface ExitStepFailureSessionDeps { loadSession(): Pick | null; - markStepFailed(stepName: string, message?: string | null, options?: StepMutationOptions): Session; + finalizeIncompleteOnboardStep(stepName: string, message?: string | null): Session | null; } export interface OnboardExitFailureProcessLike { @@ -21,13 +17,14 @@ export function markLastStartedStepFailed( message: string, ): Session | null { // Repairs the invalid state where onboard/rebuild exits nonzero after a step - // starts but before normal completion handlers can run. Keep the explicit - // legacy machine mutation until those process-exit paths have a single - // terminal lifecycle owner; covered by exit-step-failure, rebuild-flow, and - // onboard-exit-handler tests. + // starts but before normal completion handlers can run. Routes through the + // single terminal-failure owner (finalizeIncompleteOnboardStep), which + // validates the failed transition and is idempotent against an already + // terminal machine, rather than the legacy step-mutation escape hatch. + // Covered by exit-step-failure, rebuild-flow, and onboard-exit-handler tests. const failedStep = deps.loadSession()?.lastStepStarted; if (!failedStep) return null; - return deps.markStepFailed(failedStep, message, LEGACY_MACHINE_STEP_MUTATION_OPTIONS); + return deps.finalizeIncompleteOnboardStep(failedStep, message); } export function registerIncompleteOnboardExitFailureHandler( diff --git a/src/lib/onboard/machine/transitions.test.ts b/src/lib/onboard/machine/transitions.test.ts index 92698f55aec..158372e0466 100644 --- a/src/lib/onboard/machine/transitions.test.ts +++ b/src/lib/onboard/machine/transitions.test.ts @@ -147,6 +147,19 @@ describe("onboard machine transitions", () => { ); }); + it("never allows a terminal failed state to re-enter an agent or flow state (#6179)", () => { + for (const to of ["agent_setup", "openclaw", "sandbox", "policies", "init"] as const) { + expect(canTransitionOnboardMachineState("failed", to)).toBe(false); + expect(getOnboardMachineTransition("failed", to)).toBeNull(); + expect(() => assertValidOnboardMachineTransition("failed", to)).toThrow(`failed -> ${to}`); + } + // Failure edges only ever point *into* the terminal failed state. + for (const transition of ONBOARD_MACHINE_TRANSITIONS) { + expect(transition.from).not.toBe("failed"); + expect(transition.from).not.toBe("complete"); + } + }); + it("keeps the next-state map aligned with the transition list", () => { for (const state of ONBOARD_MACHINE_STATES) { expect( diff --git a/src/lib/onboard/machine/transitions.ts b/src/lib/onboard/machine/transitions.ts index 17d4cbdf664..c7f14f24634 100644 --- a/src/lib/onboard/machine/transitions.ts +++ b/src/lib/onboard/machine/transitions.ts @@ -8,6 +8,30 @@ import { ONBOARD_TERMINAL_MACHINE_STATES, } from "./types"; +/** + * The legal onboarding transition graph. + * + * There are exactly two families of edges and no others: + * + * 1. Direct edges (`ONBOARD_MACHINE_DIRECT_TRANSITIONS`) — the forward flow + * plus the `inference -> provider_selection` retry and the + * `sandbox -> {openclaw,agent_setup}` branch. `kind` is `advance`, `retry`, + * or `branch`. + * 2. Failure edges (`ONBOARD_MACHINE_FAILURE_TRANSITIONS`) — every non-terminal + * state may transition to `failed`. `kind` is `failure`. + * + * Terminality invariant: `complete` and `failed` are terminal and have no + * outgoing edges. In particular there is deliberately **no** edge out of + * `failed` into any agent/flow state, so a completed-then-reopened session can + * never take an invalid `failed -> ` transition (#6179). + * + * Recovery model: resuming an interrupted run does not transition out of a + * terminal state. Instead a single, side-effect-free recovery pass + * (`applySessionRecovery`) validates and re-seats the durable snapshot at a + * legal non-terminal entry state before any flow handler runs, emitting one + * explicit `state.repair.*` event. Terminal states therefore stay terminal + * within the graph while recovery remains explicit and observable. + */ export const ONBOARD_MACHINE_DIRECT_TRANSITIONS = [ { from: "init", to: "preflight", kind: "advance" }, { from: "preflight", to: "gateway", kind: "advance" }, diff --git a/src/lib/onboard/resume-machine-repair.test.ts b/src/lib/onboard/resume-machine-repair.test.ts index 4f247d197bb..23804ca06f6 100644 --- a/src/lib/onboard/resume-machine-repair.test.ts +++ b/src/lib/onboard/resume-machine-repair.test.ts @@ -13,9 +13,10 @@ import { } from "../state/onboard-session"; import { advanceTo, branchTo } from "./machine/result"; import { OnboardRuntime, type OnboardRuntimeDeps } from "./machine/runtime"; -import { repairResumeMachineSnapshot, resumeMachineState } from "./resume-machine-repair"; +import { resumeMachineState } from "./resume-machine-repair"; import { classifyResumeMachineRepair } from "./resume-repair-policy"; import { OnboardRuntimeBoundary } from "./runtime-boundary"; +import { applySessionRecovery } from "./session-recovery"; /** * Builds a failed durable session while letting each test set the interrupted step. @@ -96,7 +97,7 @@ function createBoundaryHarness(initial: Session) { * Replays the live resume sequence from failed snapshot repair through completion. */ async function runRecordOnlyResumeSequence(initial: Session): Promise { - repairResumeMachineSnapshot(initial, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(initial, "2026-06-01T00:01:00.000Z"); initial.failure = null; initial.status = "in_progress"; const { boundary, getSession } = createBoundaryHarness(initial); @@ -187,7 +188,7 @@ describe("resume machine repair", () => { }); expect(resumeMachineState(session)).toBe("preflight"); - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); expect(session.machine).toEqual({ version: MACHINE_SNAPSHOT_VERSION, @@ -234,7 +235,7 @@ describe("resume machine repair", () => { }, }); - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); expect(session.machine).toEqual({ version: MACHINE_SNAPSHOT_VERSION, @@ -259,7 +260,7 @@ describe("resume machine repair", () => { session.steps.preflight.status = "complete"; session.steps.gateway.status = "complete"; - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); expect(session.machine).toEqual({ version: MACHINE_SNAPSHOT_VERSION, @@ -282,7 +283,7 @@ describe("resume machine repair", () => { session.resumable = false; session.status = "complete"; - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); expect(session.machine).toEqual({ version: MACHINE_SNAPSHOT_VERSION, diff --git a/src/lib/onboard/resume-machine-repair.ts b/src/lib/onboard/resume-machine-repair.ts index 9a64694393c..202721a92aa 100644 --- a/src/lib/onboard/resume-machine-repair.ts +++ b/src/lib/onboard/resume-machine-repair.ts @@ -1,11 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { MACHINE_SNAPSHOT_VERSION, type Session } from "../state/onboard-session"; +import type { Session } from "../state/onboard-session"; import { nextMachineStateAfterCompletedStep } from "../state/onboard-step-state"; import { machineStateFromOnboardSessionStep } from "./machine/events"; import type { OnboardMachineState } from "./machine/types"; -import { classifyResumeMachineRepair } from "./resume-repair-policy"; /** * Reads the legacy step-level source of truth for interrupted sessions whose @@ -31,6 +30,12 @@ function activeStepMachineState(session: Session): OnboardMachineState | null { /** * Computes the nonterminal state where a failed durable session should resume. + * + * This derives the resume entry from the legacy step-level source of truth and + * is one of the building blocks for the single recovery pass in + * `session-recovery.ts` (`planSessionRecovery` / `applySessionRecovery`), which + * classifies, validates, and applies the recovery. Remove this bridge once step + * fields stop being used to derive resume state. */ export function resumeMachineState(session: Session): OnboardMachineState { return ( @@ -39,27 +44,3 @@ export function resumeMachineState(session: Session): OnboardMachineState { "init" ); } - -/** - * Repairs legacy terminal-session/FSM boundaries during --resume. - * - * Source fix constraint: terminal -> resume is not a modeled FSM transition - * yet, and legacy step fields still act as the secondary durable source for - * resume. Remove this bridge once terminal-session recovery is represented by - * explicit FSM recovery results or step fields stop being used to derive resume - * state. - */ -export function repairResumeMachineSnapshot( - session: Session, - stateEnteredAt = new Date().toISOString(), -): Session { - if (classifyResumeMachineRepair(session).action !== "repair") return session; - const state = resumeMachineState(session); - session.machine = { - version: MACHINE_SNAPSHOT_VERSION, - state, - stateEnteredAt, - revision: session.machine.revision + 1, - }; - return session; -} diff --git a/src/lib/onboard/session-bootstrap.test.ts b/src/lib/onboard/session-bootstrap.test.ts index 65749601bb7..574def8498f 100644 --- a/src/lib/onboard/session-bootstrap.test.ts +++ b/src/lib/onboard/session-bootstrap.test.ts @@ -4,8 +4,8 @@ import { describe, expect, it, vi } from "vitest"; import { createSession, type Session } from "../state/onboard-session"; -import { prepareOnboardSession, type OnboardSessionBootstrapDeps } from "./session-bootstrap"; import type { ResumeConfigConflict } from "./resume-config"; +import { type OnboardSessionBootstrapDeps, prepareOnboardSession } from "./session-bootstrap"; class ExitError extends Error { constructor(readonly code: number) { @@ -43,7 +43,9 @@ function createDeps( session = next; return next; }), - repairResumeMachineSnapshot: vi.fn((current: Session) => current), + applySessionRecovery: vi.fn( + () => ({ action: "keep", reason: "nonterminal_snapshot" }) as const, + ), setOnboardBrandingAgent: vi.fn(), getResumeConfigConflicts: vi.fn(() => []), recordResumeConflict: vi.fn(async () => undefined), @@ -117,7 +119,8 @@ describe("prepareOnboardSession", () => { expect(result.session?.mode).toBe("non-interactive"); expect(result.session?.failure).toBeNull(); expect(result.session?.status).toBe("in_progress"); - expect(deps.repairResumeMachineSnapshot).toHaveBeenCalledWith(initial); + expect(deps.applySessionRecovery).toHaveBeenCalledWith(initial); + expect(result.recovery).toEqual({ action: "keep", reason: "nonterminal_snapshot" }); expect(deps.setOnboardBrandingAgent).toHaveBeenCalledWith("hermes"); }); diff --git a/src/lib/onboard/session-bootstrap.ts b/src/lib/onboard/session-bootstrap.ts index 1936e9da4ad..62fd9c275e7 100644 --- a/src/lib/onboard/session-bootstrap.ts +++ b/src/lib/onboard/session-bootstrap.ts @@ -3,6 +3,7 @@ import type { Session } from "../state/onboard-session"; import type { ResumeConfigConflict } from "./resume-config"; +import type { SessionRecoveryPlan } from "./session-recovery"; export interface OnboardSessionBootstrapInput { resume: boolean; @@ -21,7 +22,7 @@ export interface OnboardSessionBootstrapDeps { createSession(overrides?: Partial): Session; saveSession(session: Session): Session; updateSession(mutator: (session: Session) => Session | void): Session; - repairResumeMachineSnapshot(session: Session): Session; + applySessionRecovery(session: Session): SessionRecoveryPlan; setOnboardBrandingAgent(agentName: string | null): void; getResumeConfigConflicts( session: Session | null, @@ -42,6 +43,12 @@ export interface OnboardSessionBootstrapDeps { export interface OnboardSessionBootstrapResult { session: Session | null; fromDockerfile: string | null; + /** + * The recovery decision applied during resume bootstrap, or null for a fresh + * session. The caller emits exactly one explicit recovery event when + * `action === "recover"`. + */ + recovery: SessionRecoveryPlan | null; } function mode(nonInteractive: boolean): "non-interactive" | "interactive" { @@ -157,8 +164,9 @@ async function prepareResumeSession( await exitForResumeConflicts(resumeConflicts, deps); } + let recovery: SessionRecoveryPlan | null = null; deps.updateSession((current: Session) => { - deps.repairResumeMachineSnapshot(current); + recovery = deps.applySessionRecovery(current); current.mode = mode(input.nonInteractive); current.failure = null; current.status = "in_progress"; @@ -166,7 +174,7 @@ async function prepareResumeSession( }); session = deps.loadSession(); assertRecoverableResumeSandboxName(session, input, deps); - return { session, fromDockerfile }; + return { session, fromDockerfile, recovery }; } function prepareFreshSession( @@ -185,7 +193,7 @@ function prepareFreshSession( metadata: { gatewayName: "nemoclaw", fromDockerfile: fromDockerfile || null }, }), ); - return { session, fromDockerfile }; + return { session, fromDockerfile, recovery: null }; } export async function prepareOnboardSession( diff --git a/src/lib/onboard/session-recovery.test.ts b/src/lib/onboard/session-recovery.test.ts new file mode 100644 index 00000000000..4a16c55b8d6 --- /dev/null +++ b/src/lib/onboard/session-recovery.test.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createSession, MACHINE_SNAPSHOT_VERSION, type Session } from "../state/onboard-session"; +import { + applySessionRecovery, + assertRecoverableEntry, + planSessionRecovery, + UnrecoverableSessionError, +} from "./session-recovery"; + +function failedSession(mutator: (session: Session) => void): Session { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "failed", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 4, + }, + status: "failed", + failure: { step: null, message: "interrupted", recordedAt: "2026-06-01T00:00:00.000Z" }, + }); + mutator(session); + return session; +} + +describe("planSessionRecovery", () => { + it("plans one validated non-terminal entry for a failed terminal snapshot", () => { + const session = failedSession((current) => { + current.failure = { + step: "gateway", + message: "gateway failed", + recordedAt: "2026-06-01T00:00:00.000Z", + }; + current.lastStepStarted = "gateway"; + current.steps.gateway.status = "failed"; + }); + + expect(planSessionRecovery(session)).toEqual({ + action: "recover", + reason: "failed_terminal_snapshot", + entry: "gateway", + }); + }); + + it("keeps a nonterminal snapshot", () => { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "gateway", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 2, + }, + }); + + expect(planSessionRecovery(session)).toEqual({ + action: "keep", + reason: "nonterminal_snapshot", + }); + }); + + it("does not mutate the session while planning", () => { + const session = failedSession((current) => { + current.lastStepStarted = "preflight"; + current.steps.preflight.status = "failed"; + }); + const before = JSON.stringify(session); + + planSessionRecovery(session); + + expect(JSON.stringify(session)).toBe(before); + }); +}); + +describe("assertRecoverableEntry", () => { + it("returns a non-terminal entry unchanged", () => { + expect(assertRecoverableEntry("gateway")).toBe("gateway"); + }); + + it.each([ + "complete", + "failed", + ] as const)("rejects the terminal entry %s as unrecoverable", (state) => { + expect(() => assertRecoverableEntry(state)).toThrow(UnrecoverableSessionError); + }); +}); + +describe("applySessionRecovery", () => { + it("re-seats a failed snapshot at the validated entry with a bumped revision", () => { + const session = failedSession((current) => { + current.failure = { + step: "preflight", + message: "Docker unavailable", + recordedAt: "2026-06-01T00:00:00.000Z", + }; + current.lastStepStarted = "preflight"; + current.steps.preflight.status = "failed"; + }); + + const plan = applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); + + expect(plan).toEqual({ + action: "recover", + reason: "failed_terminal_snapshot", + entry: "preflight", + }); + expect(session.machine).toEqual({ + version: MACHINE_SNAPSHOT_VERSION, + state: "preflight", + stateEnteredAt: "2026-06-01T00:01:00.000Z", + revision: 5, + }); + }); + + it("leaves a nonterminal snapshot untouched", () => { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "gateway", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 2, + }, + }); + + const plan = applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); + + expect(plan).toEqual({ action: "keep", reason: "nonterminal_snapshot" }); + expect(session.machine.state).toBe("gateway"); + expect(session.machine.revision).toBe(2); + }); +}); diff --git a/src/lib/onboard/session-recovery.ts b/src/lib/onboard/session-recovery.ts new file mode 100644 index 00000000000..49f16e3fd69 --- /dev/null +++ b/src/lib/onboard/session-recovery.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { MACHINE_SNAPSHOT_VERSION, type Session } from "../state/onboard-session"; +import { isTerminalOnboardMachineState } from "./machine/transitions"; +import type { OnboardMachineState, OnboardNonTerminalMachineState } from "./machine/types"; +import { resumeMachineState } from "./resume-machine-repair"; +import { classifyResumeMachineRepair, type ResumeRepairReason } from "./resume-repair-policy"; + +/** + * The single, deterministic recovery decision for a resumed session. + * + * `recover` re-seats a terminal (failed / reopened-complete) durable snapshot at + * a validated non-terminal entry state; `keep` leaves a nonterminal or + * legitimately-complete snapshot untouched. This is the one explicit recovery + * path that replaces the previous implicit snapshot rewrite, so terminal states + * stay terminal within the FSM graph while recovery stays observable. + */ +export type SessionRecoveryPlan = + | { + action: "recover"; + reason: Extract< + ResumeRepairReason, + "failed_terminal_snapshot" | "reopened_complete_snapshot" + >; + entry: OnboardNonTerminalMachineState; + } + | { + action: "keep"; + reason: Extract< + ResumeRepairReason, + "nonterminal_snapshot" | "completed_nonresumable_snapshot" + >; + }; + +/** + * Raised when a session cannot be recovered to a legal non-terminal entry + * state. Signals unrecoverable durable corruption rather than a recoverable + * failure or user cancellation. + */ +export class UnrecoverableSessionError extends Error { + readonly derivedEntry: string; + + constructor(derivedEntry: string) { + super( + `Cannot recover onboarding session: derived resume entry '${derivedEntry}' is not a legal non-terminal state.`, + ); + this.name = "UnrecoverableSessionError"; + this.derivedEntry = derivedEntry; + } +} + +/** + * Validates that a derived resume entry is a legal non-terminal state. + * + * Recovery must place the machine at a state the flow can advance from; a + * terminal entry means the durable session is corrupt beyond recovery. + */ +export function assertRecoverableEntry(entry: OnboardMachineState): OnboardNonTerminalMachineState { + if (isTerminalOnboardMachineState(entry)) { + throw new UnrecoverableSessionError(entry); + } + return entry; +} + +/** + * Classifies a resumed session and, when recovery is required, computes and + * validates the single non-terminal entry state to resume from. + * + * Pure and side-effect-free: performs no sandbox/provider/policy effects and + * does not mutate the session. Callers apply the plan with + * {@link applySessionRecovery} and emit the recovery event separately. + */ +export function planSessionRecovery(session: Session): SessionRecoveryPlan { + const decision = classifyResumeMachineRepair(session); + if (decision.action === "keep") { + return decision; + } + const entry = assertRecoverableEntry(resumeMachineState(session)); + return { action: "recover", reason: decision.reason, entry }; +} + +/** + * Applies {@link planSessionRecovery} to the session in place. + * + * When the plan is `recover`, re-seats `session.machine` at the validated + * non-terminal entry with a bumped revision. Returns the plan so the caller can + * emit exactly one explicit `state.repair.*` recovery event. Performs no side + * effects beyond the snapshot mutation and must run before any flow handler. + */ +export function applySessionRecovery( + session: Session, + stateEnteredAt: string = new Date().toISOString(), +): SessionRecoveryPlan { + const plan = planSessionRecovery(session); + if (plan.action === "recover") { + session.machine = { + version: MACHINE_SNAPSHOT_VERSION, + state: plan.entry, + stateEnteredAt, + revision: session.machine.revision + 1, + }; + } + return plan; +} diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 6940c6fd0eb..bfefa45e2dd 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -22,11 +22,14 @@ import { emitOnboardMachineEvent, machineStateFromOnboardSessionStep, } from "../onboard/machine/events"; -import { isOnboardMachineState } from "../onboard/machine/transitions"; +import { + assertValidOnboardMachineTransition, + isOnboardMachineState, + isTerminalOnboardMachineState, +} from "../onboard/machine/transitions"; import type { OnboardMachineState } from "../onboard/machine/types"; import { redactSensitiveText, redactUrl } from "../security/redact"; import { - LEGACY_MACHINE_STEP_MUTATION_OPTIONS, RECORD_ONLY_STEP_MUTATION_OPTIONS, type StepMutationOptions, shouldUpdateMachine, @@ -1228,6 +1231,65 @@ export function markStepFailedRecordOnly(stepName: string, message: string | nul return markStepFailedWithOptions(stepName, message, RECORD_ONLY_STEP_MUTATION_OPTIONS); } +/** + * Single synchronous terminal-failure owner for process-exit / backstop paths. + * + * Records exactly one failed transition and one terminal event for an + * interrupted step, replacing the legacy step-mutation escape hatch on the + * process-exit path. It is idempotent by construction: if the durable machine + * is already terminal (an in-band failure or a prior backstop already recorded + * the terminal event) it no-ops rather than recording a second failure, so the + * failed transition is validated and never doubled. Performs no + * sandbox/provider/policy effects. + */ +export function finalizeIncompleteOnboardStep( + stepName: string, + message: string | null = null, +): Session | null { + const existing = loadSession(); + if (!existing) return null; + if (isTerminalOnboardMachineState(existing.machine.state)) return existing; + + let emitted = false; + const updatedSession = updateSession((session) => { + const step = session.steps[stepName]; + if (!step) return session; + if (isTerminalOnboardMachineState(session.machine.state)) return session; + const now = new Date().toISOString(); + // Guard the terminality invariant: only a legal -> failed + // transition may be recorded here. + assertValidOnboardMachineTransition(session.machine.state, "failed"); + step.status = "failed"; + step.completedAt = null; + step.error = redactSensitiveText(message); + session.failure = sanitizeFailure({ step: stepName, message, recordedAt: now }); + session.status = "failed"; + transitionMachineSnapshot(session, "failed", now); + emitted = true; + return session; + }); + if (emitted) { + emitOnboardMachineEvent( + createOnboardMachineEvent({ + type: "state.failed", + session: updatedSession, + step: stepName, + error: message, + }), + ); + emitOnboardMachineEvent( + createOnboardMachineEvent({ + type: "onboard.failed", + session: updatedSession, + state: "failed", + step: stepName, + error: message, + }), + ); + } + return updatedSession; +} + export function completeSession(updates: SessionUpdates = {}): Session { const safeUpdates = filterSafeUpdates(updates); let wasComplete = false;