diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index d36f1e45d32..d62699c6fac 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -10,7 +10,7 @@ import * as agentDefs from "../../agent/defs"; import * as agentRuntime from "../../agent/runtime"; import * as gatewayRuntime from "../../gateway-runtime-action"; import * as nim from "../../inference/nim"; -import * as resumeRepair from "../../onboard/resume-machine-repair"; +import * as sessionRecovery from "../../onboard/session-recovery"; import * as sandboxList from "../../openshell-sandbox-list"; import * as sandboxVersion from "../../sandbox/version"; import type { Session } from "../../state/onboard-session"; @@ -180,7 +180,7 @@ describe("rebuild resume snapshot repair", () => { observed.preRepairGatewayStatus = reopened.steps.gateway.status; observed.preRepairStatus = reopened.status; observed.preRepairResumable = reopened.resumable; - resumeRepair.repairResumeMachineSnapshot(reopened, "2026-06-01T00:01:00.000Z"); + sessionRecovery.applySessionRecovery(reopened, "2026-06-01T00:01:00.000Z"); observed.repairedMachineState = reopened.machine.state; observed.sandboxEnvInsideOnboard = process.env.NEMOCLAW_SANDBOX_NAME ?? null; throw new Error("stop-after-resume-repair-probe"); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 9c348d06862..c0197a798bf 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -141,8 +141,8 @@ const { MessagingHostStateApplier, } = require("./onboard/messaging-channel-setup") as typeof import("./onboard/messaging-channel-setup"); const { - repairResumeMachineSnapshot, -}: typeof import("./onboard/resume-machine-repair") = require("./onboard/resume-machine-repair"); + applySessionRecovery, +}: typeof import("./onboard/session-recovery") = require("./onboard/session-recovery"); const bedrockRuntimeOnboard: typeof import("./onboard/bedrock-runtime") = require("./onboard/bedrock-runtime"); const { @@ -4222,7 +4222,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { createSession: onboardSession.createSession, saveSession: onboardSession.saveSession, updateSession: onboardSession.updateSession, - repairResumeMachineSnapshot, + applySessionRecovery, setOnboardBrandingAgent, getResumeConfigConflicts, recordResumeConflict: (conflict) => onboardRuntimeBoundary.recordResumeConflict(conflict), diff --git a/src/lib/onboard/exit-step-failure.test.ts b/src/lib/onboard/exit-step-failure.test.ts index a8464eeebbb..0f1076025e6 100644 --- a/src/lib/onboard/exit-step-failure.test.ts +++ b/src/lib/onboard/exit-step-failure.test.ts @@ -27,17 +27,21 @@ const restoreOriginalHome = }; let tmpDir: string; let session: typeof sessionModule; +let machineEvents: typeof import("./machine/events"); beforeEach(async () => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-exit-step-failure-")); process.env.HOME = tmpDir; vi.resetModules(); session = await import("../state/onboard-session"); + machineEvents = await import("./machine/events"); + machineEvents.clearOnboardMachineEventListeners(); session.clearSession(); resetOnboardResumeHintForTests(); }); afterEach(() => { + machineEvents.clearOnboardMachineEventListeners(); session.clearSession(); fs.rmSync(tmpDir, { recursive: true, force: true }); restoreOriginalHome(); @@ -119,15 +123,34 @@ describe("terminal step failure helper", () => { }); it("leaves sessions without a started step untouched", () => { - const markStepFailed = vi.fn(() => session.createSession()); + const finalizeIncompleteOnboardStep = vi.fn(() => session.createSession()); expect( markLastStartedStepFailed( - { loadSession: () => session.createSession(), markStepFailed }, + { loadSession: () => session.createSession(), finalizeIncompleteOnboardStep }, "boom", ), ).toBeNull(); - expect(markStepFailed).not.toHaveBeenCalled(); + expect(finalizeIncompleteOnboardStep).not.toHaveBeenCalled(); + }); + + it("records exactly one failed transition even if the exit backstop fires twice", () => { + session.saveSession(session.createSession({ lastStepStarted: "inference" })); + const emitted: string[] = []; + machineEvents.addOnboardMachineEventListener((event) => emitted.push(event.type)); + + markLastStartedStepFailed(session, "Onboarding exited before the step completed."); + const afterFirst = requireLoadedSession(); + const revisionAfterFirst = afterFirst.machine.revision; + expect(afterFirst.machine.state).toBe("failed"); + + // A second backstop invocation must not re-transition an already-terminal + // machine or bump the revision again. + markLastStartedStepFailed(session, "Onboarding exited before the step completed."); + const afterSecond = requireLoadedSession(); + expect(afterSecond.machine.state).toBe("failed"); + expect(afterSecond.machine.revision).toBe(revisionAfterFirst); + expect(emitted).toEqual(["state.failed", "onboard.failed"]); }); }); @@ -226,7 +249,7 @@ const resumableSession = { lastStepStarted: "inference" }; registerIncompleteOnboardExitFailureHandler( { loadSession: () => resumableSession, - markStepFailed: () => resumableSession, + finalizeIncompleteOnboardStep: () => resumableSession, }, () => false, "Onboarding exited before the step completed.", diff --git a/src/lib/onboard/exit-step-failure.ts b/src/lib/onboard/exit-step-failure.ts index 9609b886f94..3fc144889f4 100644 --- a/src/lib/onboard/exit-step-failure.ts +++ b/src/lib/onboard/exit-step-failure.ts @@ -2,15 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import type { Session } from "../state/onboard-session"; -import { - LEGACY_MACHINE_STEP_MUTATION_OPTIONS, - type StepMutationOptions, -} from "../state/onboard-step-mutation"; import { printOnboardResumeHint } from "./resume-hint"; export interface ExitStepFailureSessionDeps { loadSession(): Pick | null; - markStepFailed(stepName: string, message?: string | null, options?: StepMutationOptions): Session; + finalizeIncompleteOnboardStep(stepName: string, message?: string | null): Session | null; } export interface OnboardExitFailureProcessLike { @@ -28,13 +24,14 @@ export function markLastStartedStepFailed( message: string, ): Session | null { // Repairs the invalid state where onboard/rebuild exits nonzero after a step - // starts but before normal completion handlers can run. Keep the explicit - // legacy machine mutation until those process-exit paths have a single - // terminal lifecycle owner; covered by exit-step-failure, rebuild-flow, and - // onboard-exit-handler tests. + // starts but before normal completion handlers can run. Routes through the + // single terminal-failure owner (finalizeIncompleteOnboardStep), which + // validates the failed transition and is idempotent against an already + // terminal machine, rather than the legacy step-mutation escape hatch. + // Covered by exit-step-failure, rebuild-flow, and onboard-exit-handler tests. const failedStep = deps.loadSession()?.lastStepStarted; if (!failedStep) return null; - return deps.markStepFailed(failedStep, message, LEGACY_MACHINE_STEP_MUTATION_OPTIONS); + return deps.finalizeIncompleteOnboardStep(failedStep, message); } export function registerIncompleteOnboardExitFailureHandler( diff --git a/src/lib/onboard/machine/runtime.ts b/src/lib/onboard/machine/runtime.ts index a1507b5e326..462c2904d51 100644 --- a/src/lib/onboard/machine/runtime.ts +++ b/src/lib/onboard/machine/runtime.ts @@ -129,6 +129,30 @@ export class OnboardRuntime { return session; } + /** + * Attempts observer dispatch for a durable recovery receipt. + * + * The receipt stays on the snapshot until the next machine transition, so a + * process restart before that transition retries the same deterministic ID. + * Observer delivery remains best-effort by design. + */ + async emitPendingSessionRecovery(): Promise { + const session = this.ensureSession(); + const receipt = session.machine.recoveryReceipt; + if (!receipt) return session; + this.emit("state.repair.completed", session, { + state: receipt.entry, + metadata: { + reason: receipt.reason, + entry: receipt.entry, + receiptId: receipt.id, + appliedAt: receipt.appliedAt, + revision: receipt.revision, + }, + }); + return session; + } + async markStepStarted( stepName: string, options: StepMutationOptions = RECORD_ONLY_STEP_MUTATION_OPTIONS, diff --git a/src/lib/onboard/machine/transitions.test.ts b/src/lib/onboard/machine/transitions.test.ts index 92698f55aec..158372e0466 100644 --- a/src/lib/onboard/machine/transitions.test.ts +++ b/src/lib/onboard/machine/transitions.test.ts @@ -147,6 +147,19 @@ describe("onboard machine transitions", () => { ); }); + it("never allows a terminal failed state to re-enter an agent or flow state (#6179)", () => { + for (const to of ["agent_setup", "openclaw", "sandbox", "policies", "init"] as const) { + expect(canTransitionOnboardMachineState("failed", to)).toBe(false); + expect(getOnboardMachineTransition("failed", to)).toBeNull(); + expect(() => assertValidOnboardMachineTransition("failed", to)).toThrow(`failed -> ${to}`); + } + // Failure edges only ever point *into* the terminal failed state. + for (const transition of ONBOARD_MACHINE_TRANSITIONS) { + expect(transition.from).not.toBe("failed"); + expect(transition.from).not.toBe("complete"); + } + }); + it("keeps the next-state map aligned with the transition list", () => { for (const state of ONBOARD_MACHINE_STATES) { expect( diff --git a/src/lib/onboard/machine/transitions.ts b/src/lib/onboard/machine/transitions.ts index 17d4cbdf664..e4f169257ce 100644 --- a/src/lib/onboard/machine/transitions.ts +++ b/src/lib/onboard/machine/transitions.ts @@ -8,6 +8,33 @@ import { ONBOARD_TERMINAL_MACHINE_STATES, } from "./types"; +/** + * The legal onboarding transition graph. + * + * There are exactly two families of edges and no others: + * + * 1. Direct edges (`ONBOARD_MACHINE_DIRECT_TRANSITIONS`) — the forward flow + * plus the `inference -> provider_selection` retry and the + * `sandbox -> {openclaw,agent_setup}` branch. `kind` is `advance`, `retry`, + * or `branch`. + * 2. Failure edges (`ONBOARD_MACHINE_FAILURE_TRANSITIONS`) — every non-terminal + * state may transition to `failed`. `kind` is `failure`. + * + * Terminality invariant: `complete` and `failed` are terminal and have no + * outgoing edges. In particular there is deliberately **no** edge out of + * `failed` into any agent/flow state, so a completed-then-reopened session can + * never take an invalid `failed -> ` transition (#6179). + * + * Recovery model: resuming an interrupted run does not transition out of a + * terminal state. Instead a single, side-effect-free recovery pass + * (`applySessionRecovery`) validates and re-seats the durable snapshot at a + * legal non-terminal entry state before any flow handler runs and writes a + * deterministic recovery receipt. After `onboard.resumed`, the runtime makes a + * best-effort `state.repair.completed` dispatch attempt for that receipt. A + * restart before the next transition retries the same receipt ID. Terminal + * states therefore stay terminal within the graph while recovery remains + * explicit and observable. + */ export const ONBOARD_MACHINE_DIRECT_TRANSITIONS = [ { from: "init", to: "preflight", kind: "advance" }, { from: "preflight", to: "gateway", kind: "advance" }, diff --git a/src/lib/onboard/resume-machine-repair.test.ts b/src/lib/onboard/resume-machine-repair.test.ts index 4f247d197bb..a121aa24ccd 100644 --- a/src/lib/onboard/resume-machine-repair.test.ts +++ b/src/lib/onboard/resume-machine-repair.test.ts @@ -13,9 +13,10 @@ import { } from "../state/onboard-session"; import { advanceTo, branchTo } from "./machine/result"; import { OnboardRuntime, type OnboardRuntimeDeps } from "./machine/runtime"; -import { repairResumeMachineSnapshot, resumeMachineState } from "./resume-machine-repair"; +import { resumeMachineState } from "./resume-machine-repair"; import { classifyResumeMachineRepair } from "./resume-repair-policy"; import { OnboardRuntimeBoundary } from "./runtime-boundary"; +import { applySessionRecovery } from "./session-recovery"; /** * Builds a failed durable session while letting each test set the interrupted step. @@ -96,7 +97,7 @@ function createBoundaryHarness(initial: Session) { * Replays the live resume sequence from failed snapshot repair through completion. */ async function runRecordOnlyResumeSequence(initial: Session): Promise { - repairResumeMachineSnapshot(initial, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(initial, "2026-06-01T00:01:00.000Z"); initial.failure = null; initial.status = "in_progress"; const { boundary, getSession } = createBoundaryHarness(initial); @@ -187,13 +188,18 @@ describe("resume machine repair", () => { }); expect(resumeMachineState(session)).toBe("preflight"); - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); - expect(session.machine).toEqual({ + expect(session.machine).toMatchObject({ version: MACHINE_SNAPSHOT_VERSION, state: "preflight", stateEnteredAt: "2026-06-01T00:01:00.000Z", revision: 8, + recoveryReceipt: { + reason: "failed_terminal_snapshot", + entry: "preflight", + revision: 8, + }, }); }); @@ -234,7 +240,7 @@ describe("resume machine repair", () => { }, }); - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); expect(session.machine).toEqual({ version: MACHINE_SNAPSHOT_VERSION, @@ -259,13 +265,18 @@ describe("resume machine repair", () => { session.steps.preflight.status = "complete"; session.steps.gateway.status = "complete"; - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); - expect(session.machine).toEqual({ + expect(session.machine).toMatchObject({ version: MACHINE_SNAPSHOT_VERSION, state: "provider_selection", stateEnteredAt: "2026-06-01T00:01:00.000Z", revision: 10, + recoveryReceipt: { + reason: "reopened_complete_snapshot", + entry: "provider_selection", + revision: 10, + }, }); }); @@ -282,7 +293,7 @@ describe("resume machine repair", () => { session.resumable = false; session.status = "complete"; - repairResumeMachineSnapshot(session, "2026-06-01T00:01:00.000Z"); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); expect(session.machine).toEqual({ version: MACHINE_SNAPSHOT_VERSION, diff --git a/src/lib/onboard/resume-machine-repair.ts b/src/lib/onboard/resume-machine-repair.ts index 9a64694393c..32cf2322c1f 100644 --- a/src/lib/onboard/resume-machine-repair.ts +++ b/src/lib/onboard/resume-machine-repair.ts @@ -1,11 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { MACHINE_SNAPSHOT_VERSION, type Session } from "../state/onboard-session"; +import type { Session } from "../state/onboard-session"; import { nextMachineStateAfterCompletedStep } from "../state/onboard-step-state"; import { machineStateFromOnboardSessionStep } from "./machine/events"; import type { OnboardMachineState } from "./machine/types"; -import { classifyResumeMachineRepair } from "./resume-repair-policy"; /** * Reads the legacy step-level source of truth for interrupted sessions whose @@ -31,6 +30,12 @@ function activeStepMachineState(session: Session): OnboardMachineState | null { /** * Computes the nonterminal state where a failed durable session should resume. + * + * This derives the resume entry from the legacy step-level source of truth and + * is one of the building blocks for the single recovery pass in + * `session-recovery.ts` (`planSessionRecovery` / `applySessionRecovery`), which + * classifies, validates, and applies the recovery. Remove this bridge once step + * fields stop being used to derive resume state (#6227). */ export function resumeMachineState(session: Session): OnboardMachineState { return ( @@ -39,27 +44,3 @@ export function resumeMachineState(session: Session): OnboardMachineState { "init" ); } - -/** - * Repairs legacy terminal-session/FSM boundaries during --resume. - * - * Source fix constraint: terminal -> resume is not a modeled FSM transition - * yet, and legacy step fields still act as the secondary durable source for - * resume. Remove this bridge once terminal-session recovery is represented by - * explicit FSM recovery results or step fields stop being used to derive resume - * state. - */ -export function repairResumeMachineSnapshot( - session: Session, - stateEnteredAt = new Date().toISOString(), -): Session { - if (classifyResumeMachineRepair(session).action !== "repair") return session; - const state = resumeMachineState(session); - session.machine = { - version: MACHINE_SNAPSHOT_VERSION, - state, - stateEnteredAt, - revision: session.machine.revision + 1, - }; - return session; -} diff --git a/src/lib/onboard/runtime-boundary.test.ts b/src/lib/onboard/runtime-boundary.test.ts index b2d8f97feff..87bac50d760 100644 --- a/src/lib/onboard/runtime-boundary.test.ts +++ b/src/lib/onboard/runtime-boundary.test.ts @@ -16,6 +16,7 @@ import { advanceTo, branchTo, completeOnboardMachine, retryTo } from "./machine/ import { OnboardRuntime, type OnboardRuntimeDeps } from "./machine/runtime"; import type { OnboardMachineState } from "./machine/types"; import { OnboardRuntimeBoundary } from "./runtime-boundary"; +import { applySessionRecovery } from "./session-recovery"; function cloneSession(session: Session): Session { return normalizeSession(JSON.parse(JSON.stringify(session))) ?? session; @@ -176,6 +177,56 @@ describe("OnboardRuntimeBoundary", () => { expect(harness.events[1]).toMatchObject({ state: "init" }); }); + it("dispatches a durable recovery receipt after resume and clears it on transition (#6227)", async () => { + const recovered = createSession({ + resumable: true, + status: "in_progress", + lastCompletedStep: "gateway", + machine: { + version: 1, + state: "complete", + stateEnteredAt: "2026-05-27T00:00:00.000Z", + revision: 9, + }, + }); + recovered.steps.preflight.status = "complete"; + recovered.steps.gateway.status = "complete"; + applySessionRecovery(recovered, "2026-05-27T00:01:00.000Z"); + const receiptId = recovered.machine.recoveryReceipt?.id; + const harness = createRuntimeHarness(recovered); + const boundary = new OnboardRuntimeBoundary({ + toSessionUpdates: (updates) => filterSafeUpdates(updates as SessionUpdates) as SessionUpdates, + maybeForceE2eStepFailure: () => undefined, + createRuntime: harness.createRuntime, + }); + + await boundary.recordOnboardStarted(true); + + expect(harness.events.map((event) => event.type)).toEqual([ + "onboard.resumed", + "state.repair.completed", + ]); + expect(harness.events[1]).toMatchObject({ + state: "provider_selection", + metadata: { + reason: "reopened_complete_snapshot", + entry: "provider_selection", + receiptId, + revision: 10, + }, + }); + + await boundary.recordStateResult( + advanceTo("inference", { metadata: { state: "provider_selection" } }), + ); + expect(harness.getSession().machine.recoveryReceipt).toBeUndefined(); + + await boundary.recordOnboardStarted(true); + expect(harness.events.filter((event) => event.type === "state.repair.completed")).toHaveLength( + 1, + ); + }); + it("defaults boundary step recorders to record-only machine mutations", async () => { const harness = createRuntimeHarness(); const boundary = new OnboardRuntimeBoundary({ diff --git a/src/lib/onboard/runtime-boundary.ts b/src/lib/onboard/runtime-boundary.ts index 88e006fc7cd..cf4428a0841 100644 --- a/src/lib/onboard/runtime-boundary.ts +++ b/src/lib/onboard/runtime-boundary.ts @@ -82,7 +82,10 @@ export class OnboardRuntimeBoundary { } async recordOnboardStarted(resumed: boolean): Promise { - return this.getRuntime().start({ resumed }); + const runtime = this.getRuntime(); + const session = await runtime.start({ resumed }); + await runtime.emitPendingSessionRecovery(); + return session; } async startRecordedStep( diff --git a/src/lib/onboard/session-bootstrap.test.ts b/src/lib/onboard/session-bootstrap.test.ts index 13b3a3c4591..248caec8fdb 100644 --- a/src/lib/onboard/session-bootstrap.test.ts +++ b/src/lib/onboard/session-bootstrap.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it, vi } from "vitest"; -import { createSession, type Session } from "../state/onboard-session"; +import { createSession, type Session, type SessionRecoveryReceipt } from "../state/onboard-session"; import type { ResumeConfigConflict } from "./resume-config"; import { type OnboardSessionBootstrapDeps, prepareOnboardSession } from "./session-bootstrap"; @@ -43,7 +43,7 @@ function createDeps( session = next; return next; }), - repairResumeMachineSnapshot: vi.fn((current: Session) => current), + applySessionRecovery: vi.fn(), setOnboardBrandingAgent: vi.fn(), getResumeConfigConflicts: vi.fn(() => []), recordResumeConflict: vi.fn(async () => undefined), @@ -142,12 +142,47 @@ describe("prepareOnboardSession", () => { expect(result.session?.mode).toBe("non-interactive"); expect(result.session?.failure).toBeNull(); expect(result.session?.status).toBe("in_progress"); + expect(deps.applySessionRecovery).toHaveBeenCalledWith(initial); expect(result.session?.observabilityEnabled).toBe(true); expect(result.session?.observabilityRequestedExplicitly).toBe(true); - expect(deps.repairResumeMachineSnapshot).toHaveBeenCalledWith(initial); expect(deps.setOnboardBrandingAgent).toHaveBeenCalledWith("hermes"); }); + it("persists a recovered terminal snapshot receipt (#6227)", async () => { + const initial = createSession({ sandboxName: "demo", status: "failed" }); + const receipt: SessionRecoveryReceipt = { + id: "a".repeat(64), + reason: "failed_terminal_snapshot", + entry: "gateway", + appliedAt: "2026-06-10T00:01:00.000Z", + revision: initial.machine.revision + 1, + }; + const applySessionRecovery = vi.fn((current: Session) => { + current.machine = { + version: current.machine.version, + state: receipt.entry, + stateEnteredAt: receipt.appliedAt, + revision: receipt.revision, + recoveryReceipt: receipt, + }; + }); + const { deps } = createDeps(initial, { applySessionRecovery }); + + const result = await prepareOnboardSession( + { + resume: true, + fresh: false, + requestedFromDockerfile: null, + requestedSandboxName: null, + cannotPrompt: false, + nonInteractive: false, + }, + deps, + ); + + expect(result.session?.machine.recoveryReceipt).toEqual(receipt); + }); + it.each([ { recorded: true, requested: false }, { recorded: false, requested: true }, diff --git a/src/lib/onboard/session-bootstrap.ts b/src/lib/onboard/session-bootstrap.ts index ebe062358af..899cb0d60e1 100644 --- a/src/lib/onboard/session-bootstrap.ts +++ b/src/lib/onboard/session-bootstrap.ts @@ -25,7 +25,7 @@ export interface OnboardSessionBootstrapDeps { createSession(overrides?: Partial): Session; saveSession(session: Session): Session; updateSession(mutator: (session: Session) => Session | void): Session; - repairResumeMachineSnapshot(session: Session): Session; + applySessionRecovery(session: Session): void; setOnboardBrandingAgent(agentName: string | null): void; getResumeConfigConflicts( session: Session | null, @@ -168,7 +168,7 @@ async function prepareResumeSession( } deps.updateSession((current: Session) => { - deps.repairResumeMachineSnapshot(current); + deps.applySessionRecovery(current); if (typeof input.requestedObservabilityEnabled === "boolean") { current.observabilityEnabled = input.requestedObservabilityEnabled; current.observabilityRequestedExplicitly = true; diff --git a/src/lib/onboard/session-recovery.test.ts b/src/lib/onboard/session-recovery.test.ts new file mode 100644 index 00000000000..fdbbd805729 --- /dev/null +++ b/src/lib/onboard/session-recovery.test.ts @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + createSession, + MACHINE_SNAPSHOT_VERSION, + normalizeSession, + type Session, +} from "../state/onboard-session"; +import { + applySessionRecovery, + assertRecoverableEntry, + planSessionRecovery, + UnrecoverableSessionError, +} from "./session-recovery"; + +function failedSession(mutator: (session: Session) => void): Session { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "failed", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 4, + }, + status: "failed", + failure: { step: null, message: "interrupted", recordedAt: "2026-06-01T00:00:00.000Z" }, + }); + mutator(session); + return session; +} + +function reopenedCompleteSession(): Session { + const session = createSession({ + resumable: true, + status: "in_progress", + lastCompletedStep: "gateway", + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "complete", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 9, + }, + }); + session.steps.preflight.status = "complete"; + session.steps.gateway.status = "complete"; + return session; +} + +describe("planSessionRecovery", () => { + it("plans one validated non-terminal entry for a failed terminal snapshot", () => { + const session = failedSession((current) => { + current.failure = { + step: "gateway", + message: "gateway failed", + recordedAt: "2026-06-01T00:00:00.000Z", + }; + current.lastStepStarted = "gateway"; + current.steps.gateway.status = "failed"; + }); + + expect(planSessionRecovery(session)).toEqual({ + action: "recover", + reason: "failed_terminal_snapshot", + entry: "gateway", + }); + }); + + it("keeps a nonterminal snapshot", () => { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "gateway", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 2, + }, + }); + + expect(planSessionRecovery(session)).toEqual({ + action: "keep", + reason: "nonterminal_snapshot", + }); + }); + + it("plans the next entry for a reopened complete snapshot (#6227)", () => { + expect(planSessionRecovery(reopenedCompleteSession())).toEqual({ + action: "recover", + reason: "reopened_complete_snapshot", + entry: "provider_selection", + }); + }); + + it("does not mutate the session while planning", () => { + const session = failedSession((current) => { + current.lastStepStarted = "preflight"; + current.steps.preflight.status = "failed"; + }); + const before = JSON.stringify(session); + + planSessionRecovery(session); + + expect(JSON.stringify(session)).toBe(before); + }); +}); + +describe("assertRecoverableEntry", () => { + it("returns a non-terminal entry unchanged", () => { + expect(assertRecoverableEntry("gateway")).toBe("gateway"); + }); + + it.each([ + "complete", + "failed", + ] as const)("rejects the terminal entry %s as unrecoverable", (state) => { + expect(() => assertRecoverableEntry(state)).toThrow(UnrecoverableSessionError); + }); +}); + +describe("applySessionRecovery", () => { + it("re-seats a failed snapshot at the validated entry with a bumped revision", () => { + const session = failedSession((current) => { + current.failure = { + step: "preflight", + message: "Docker unavailable", + recordedAt: "2026-06-01T00:00:00.000Z", + }; + current.lastStepStarted = "preflight"; + current.steps.preflight.status = "failed"; + }); + + const plan = applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); + + expect(plan).toEqual({ + action: "recover", + reason: "failed_terminal_snapshot", + entry: "preflight", + }); + expect(session.machine).toMatchObject({ + version: MACHINE_SNAPSHOT_VERSION, + state: "preflight", + stateEnteredAt: "2026-06-01T00:01:00.000Z", + revision: 5, + recoveryReceipt: { + reason: "failed_terminal_snapshot", + entry: "preflight", + appliedAt: "2026-06-01T00:01:00.000Z", + revision: 5, + }, + }); + expect(session.machine.recoveryReceipt?.id).toMatch(/^[a-f0-9]{64}$/); + }); + + it("re-seats a reopened complete snapshot and records its recovery reason (#6227)", () => { + const session = reopenedCompleteSession(); + + const plan = applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); + + expect(plan).toEqual({ + action: "recover", + reason: "reopened_complete_snapshot", + entry: "provider_selection", + }); + expect(session.machine).toMatchObject({ + state: "provider_selection", + stateEnteredAt: "2026-06-01T00:01:00.000Z", + revision: 10, + recoveryReceipt: { + reason: "reopened_complete_snapshot", + entry: "provider_selection", + appliedAt: "2026-06-01T00:01:00.000Z", + revision: 10, + }, + }); + }); + + it("leaves a nonterminal snapshot untouched", () => { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "gateway", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 2, + }, + }); + + const plan = applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); + + expect(plan).toEqual({ action: "keep", reason: "nonterminal_snapshot" }); + expect(session.machine.state).toBe("gateway"); + expect(session.machine.revision).toBe(2); + }); + + it("rejects a noncanonical recovery timestamp", () => { + expect(() => applySessionRecovery(reopenedCompleteSession(), "yesterday")).toThrow( + "canonical ISO timestamp", + ); + }); +}); + +describe("session recovery receipt persistence", () => { + it("round-trips one stable receipt ID without another recovery revision (#6227)", () => { + const original = reopenedCompleteSession(); + applySessionRecovery(original, "2026-06-01T00:01:00.000Z"); + const persisted = normalizeSession(JSON.parse(JSON.stringify(original))) as Session; + const receipt = persisted.machine.recoveryReceipt; + expect(receipt).toBeDefined(); + const revision = persisted.machine.revision; + + expect(planSessionRecovery(persisted)).toEqual({ + action: "keep", + reason: "nonterminal_snapshot", + }); + + const restarted = normalizeSession(JSON.parse(JSON.stringify(persisted))) as Session; + expect(restarted.machine.recoveryReceipt).toEqual(receipt); + expect(restarted.machine.revision).toBe(revision); + }); + + it.each([ + ["unknown reason", { reason: "unknown" }], + ["terminal entry", { entry: "complete" }], + ["mismatched revision", { revision: 99 }], + ["mismatched ID", { id: "b".repeat(64) }], + ["mismatched timestamp", { appliedAt: "2026-06-01T00:02:00.000Z" }], + ["noncanonical timestamp", { appliedAt: "yesterday" }], + ])("drops a malformed recovery receipt with %s", (_label, mutation) => { + const session = reopenedCompleteSession(); + applySessionRecovery(session, "2026-06-01T00:01:00.000Z"); + const serialized = JSON.parse(JSON.stringify(session)) as { + machine: { recoveryReceipt: Record }; + }; + Object.assign(serialized.machine.recoveryReceipt, mutation); + + expect(normalizeSession(serialized as never)?.machine.recoveryReceipt).toBeUndefined(); + }); +}); diff --git a/src/lib/onboard/session-recovery.ts b/src/lib/onboard/session-recovery.ts new file mode 100644 index 00000000000..713a5089874 --- /dev/null +++ b/src/lib/onboard/session-recovery.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + createSessionRecoveryReceiptId, + MACHINE_SNAPSHOT_VERSION, + type Session, +} from "../state/onboard-session"; +import { isTerminalOnboardMachineState } from "./machine/transitions"; +import type { OnboardMachineState, OnboardNonTerminalMachineState } from "./machine/types"; +import { resumeMachineState } from "./resume-machine-repair"; +import { classifyResumeMachineRepair, type ResumeRepairReason } from "./resume-repair-policy"; + +/** + * The single, deterministic recovery decision for a resumed session. + * + * `recover` re-seats a terminal (failed / reopened-complete) durable snapshot at + * a validated non-terminal entry state; `keep` leaves a nonterminal or + * legitimately-complete snapshot untouched. This is the one explicit recovery + * path that replaces the previous implicit snapshot rewrite, so terminal states + * stay terminal within the FSM graph while recovery stays observable. + */ +export type SessionRecoveryPlan = + | { + action: "recover"; + reason: Extract< + ResumeRepairReason, + "failed_terminal_snapshot" | "reopened_complete_snapshot" + >; + entry: OnboardNonTerminalMachineState; + } + | { + action: "keep"; + reason: Extract< + ResumeRepairReason, + "nonterminal_snapshot" | "completed_nonresumable_snapshot" + >; + }; + +/** + * Raised when a session cannot be recovered to a legal non-terminal entry + * state. Signals unrecoverable durable corruption rather than a recoverable + * failure or user cancellation. + */ +export class UnrecoverableSessionError extends Error { + readonly derivedEntry: string; + + constructor(derivedEntry: string) { + super( + `Cannot recover onboarding session: derived resume entry '${derivedEntry}' is not a legal non-terminal state.`, + ); + this.name = "UnrecoverableSessionError"; + this.derivedEntry = derivedEntry; + } +} + +/** + * Validates that a derived resume entry is a legal non-terminal state. + * + * Recovery must place the machine at a state the flow can advance from; a + * terminal entry means the durable session is corrupt beyond recovery. + */ +export function assertRecoverableEntry(entry: OnboardMachineState): OnboardNonTerminalMachineState { + if (isTerminalOnboardMachineState(entry)) { + throw new UnrecoverableSessionError(entry); + } + return entry; +} + +function assertCanonicalRecoveryTimestamp(value: string): string { + try { + if (new Date(value).toISOString() === value) return value; + } catch { + // Fall through to the stable caller-facing error below. + } + throw new TypeError("Session recovery requires a canonical ISO timestamp."); +} + +/** + * Classifies a resumed session and, when recovery is required, computes and + * validates the single non-terminal entry state to resume from. + * + * Pure and side-effect-free: performs no sandbox/provider/policy effects and + * does not mutate the session. Callers apply the plan with + * {@link applySessionRecovery}. + */ +export function planSessionRecovery(session: Session): SessionRecoveryPlan { + const decision = classifyResumeMachineRepair(session); + if (decision.action === "keep") { + return decision; + } + const entry = assertRecoverableEntry(resumeMachineState(session)); + return { action: "recover", reason: decision.reason, entry }; +} + +/** + * Applies {@link planSessionRecovery} to the session in place. + * + * When the plan is `recover`, re-seats `session.machine` at the validated + * non-terminal entry with a bumped revision and a deterministic durable + * receipt. The receipt lets the next process retry the same completion-event + * dispatch ID if this process stops after the repaired snapshot is persisted + * but before the next transition. Observer delivery remains best-effort. + * Performs no side effects beyond the snapshot mutation and must run before + * any flow handler. + */ +export function applySessionRecovery( + session: Session, + stateEnteredAt: string = new Date().toISOString(), +): SessionRecoveryPlan { + const plan = planSessionRecovery(session); + if (plan.action === "recover") { + const appliedAt = assertCanonicalRecoveryTimestamp(stateEnteredAt); + const revision = session.machine.revision + 1; + const id = createSessionRecoveryReceiptId(session.sessionId, revision, plan.reason, plan.entry); + session.machine = { + version: MACHINE_SNAPSHOT_VERSION, + state: plan.entry, + stateEnteredAt: appliedAt, + revision, + recoveryReceipt: { + id, + reason: plan.reason, + entry: plan.entry, + appliedAt, + revision, + }, + }; + } + return plan; +} diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index a1dac16985f..86be6ef498a 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -7,7 +7,7 @@ * step-level progress tracking and file-based locking. */ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; @@ -22,8 +22,12 @@ import { emitOnboardMachineEvent, machineStateFromOnboardSessionStep, } from "../onboard/machine/events"; -import { isOnboardMachineState } from "../onboard/machine/transitions"; -import type { OnboardMachineState } from "../onboard/machine/types"; +import { + assertValidOnboardMachineTransition, + isOnboardMachineState, + isTerminalOnboardMachineState, +} from "../onboard/machine/transitions"; +import type { OnboardMachineState, OnboardNonTerminalMachineState } from "../onboard/machine/types"; import { redactSensitiveText, redactUrl } from "../security/redact"; import { assignSafeToolDisclosureUpdate, @@ -32,7 +36,6 @@ import { type ToolDisclosure, } from "./onboard-session-tool-disclosure"; import { - LEGACY_MACHINE_STEP_MUTATION_OPTIONS, RECORD_ONLY_STEP_MUTATION_OPTIONS, type StepMutationOptions, shouldUpdateMachine, @@ -82,11 +85,42 @@ export interface SessionMetadata { fromDockerfile: string | null; } +export type SessionRecoveryReceiptReason = + | "failed_terminal_snapshot" + | "reopened_complete_snapshot"; + +/** + * Durable, secret-free receipt for a terminal snapshot recovery. + * + * The receipt remains attached until the next machine snapshot replaces it. + * If the process stops after the repaired snapshot is saved but before the + * next transition, the next resume retries the same observer-dispatch ID. + */ +export interface SessionRecoveryReceipt { + id: string; + reason: SessionRecoveryReceiptReason; + entry: OnboardNonTerminalMachineState; + appliedAt: string; + revision: number; +} + +export function createSessionRecoveryReceiptId( + sessionId: string, + revision: number, + reason: SessionRecoveryReceiptReason, + entry: OnboardNonTerminalMachineState, +): string { + return createHash("sha256") + .update(JSON.stringify([sessionId, revision, reason, entry])) + .digest("hex"); +} + export interface OnboardMachineSnapshot { version: typeof MACHINE_SNAPSHOT_VERSION; state: OnboardMachineState; stateEnteredAt: string | null; revision: number; + recoveryReceipt?: SessionRecoveryReceipt; } export interface Session { @@ -274,6 +308,15 @@ function readNonNegativeInteger(value: SessionJsonValue | undefined): number | n return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : null; } +function readCanonicalIsoTimestamp(value: SessionJsonValue | undefined): string | null { + if (typeof value !== "string") return null; + try { + return new Date(value).toISOString() === value ? value : null; + } catch { + return null; + } +} + function readStringArray(value: SessionJsonValue | undefined): string[] | null { if (!Array.isArray(value)) return null; return value.filter((entry): entry is string => typeof entry === "string"); @@ -341,14 +384,57 @@ function parseStepState(value: SessionJsonValue | undefined): StepState | null { }; } -function parseMachineSnapshot(value: SessionJsonValue | undefined): OnboardMachineSnapshot | null { +function parseSessionRecoveryReceipt( + value: SessionJsonValue | undefined, + snapshotState: OnboardMachineState, + snapshotStateEnteredAt: string | null, + snapshotRevision: number, + sessionId: string, +): SessionRecoveryReceipt | null { + if (!isObject(value)) return null; + const id = readString(value.id); + const reason = readString(value.reason); + const entry = readString(value.entry); + const appliedAt = readCanonicalIsoTimestamp(value.appliedAt); + const revision = readNonNegativeInteger(value.revision); + if (!id || !/^[a-f0-9]{64}$/.test(id)) return null; + if (reason !== "failed_terminal_snapshot" && reason !== "reopened_complete_snapshot") { + return null; + } + if (!entry || !isOnboardMachineState(entry) || isTerminalOnboardMachineState(entry)) return null; + if ( + entry !== snapshotState || + !appliedAt || + appliedAt !== snapshotStateEnteredAt || + revision !== snapshotRevision || + id !== createSessionRecoveryReceiptId(sessionId, revision, reason, entry) + ) { + return null; + } + return { id, reason, entry, appliedAt, revision }; +} + +function parseMachineSnapshot( + value: SessionJsonValue | undefined, + sessionId: string, +): OnboardMachineSnapshot | null { if (!isObject(value) || value.version !== MACHINE_SNAPSHOT_VERSION) return null; if (!isOnboardMachineState(value.state)) return null; + const stateEnteredAt = readString(value.stateEnteredAt); + const revision = readNonNegativeInteger(value.revision) ?? 0; + const recoveryReceipt = parseSessionRecoveryReceipt( + value.recoveryReceipt, + value.state, + stateEnteredAt, + revision, + sessionId, + ); return { version: MACHINE_SNAPSHOT_VERSION, state: value.state, - stateEnteredAt: readString(value.stateEnteredAt), - revision: readNonNegativeInteger(value.revision) ?? 0, + stateEnteredAt, + revision, + ...(recoveryReceipt ? { recoveryReceipt } : {}), }; } @@ -447,13 +533,14 @@ function transitionMachineSnapshot( export function createSession(overrides: Partial = {}): Session { const now = new Date().toISOString(); const startedAt = overrides.startedAt ?? now; + const sessionId = overrides.sessionId ?? `${Date.now()}-${randomUUID()}`; const steps = { ...defaultSteps(), ...(overrides.steps ?? {}), }; const session: Session = { version: SESSION_VERSION, - sessionId: overrides.sessionId ?? `${Date.now()}-${randomUUID()}`, + sessionId, resumable: true, status: "in_progress", mode: overrides.mode ?? "interactive", @@ -492,7 +579,7 @@ export function createSession(overrides: Partial = {}): Session { fromDockerfile: overrides.metadata?.fromDockerfile ?? null, }, machine: - parseMachineSnapshot(overrides.machine as SessionJsonValue | undefined) ?? + parseMachineSnapshot(overrides.machine as SessionJsonValue | undefined, sessionId) ?? createMachineSnapshot("init", startedAt), steps, }; @@ -548,7 +635,8 @@ export function normalizeSession(data: Session | SessionJsonValue | undefined): } } - normalized.machine = parseMachineSnapshot(data.machine) ?? inferMachineSnapshot(normalized); + normalized.machine = + parseMachineSnapshot(data.machine, normalized.sessionId) ?? inferMachineSnapshot(normalized); preserveInvalidSessionToolDisclosure(data, normalized); return normalized; @@ -1261,6 +1349,65 @@ export function markStepFailedRecordOnly(stepName: string, message: string | nul return markStepFailedWithOptions(stepName, message, RECORD_ONLY_STEP_MUTATION_OPTIONS); } +/** + * Single synchronous terminal-failure owner for process-exit / backstop paths. + * + * Records exactly one failed transition and one terminal event pair for an + * interrupted step, replacing the legacy step-mutation escape hatch on the + * process-exit path. It is idempotent by construction: if the durable machine + * is already terminal (an in-band failure or a prior backstop already recorded + * the terminal event pair) it no-ops rather than recording a second failure, so the + * failed transition is validated and never doubled. Performs no + * sandbox/provider/policy effects. + */ +export function finalizeIncompleteOnboardStep( + stepName: string, + message: string | null = null, +): Session | null { + const existing = loadSession(); + if (!existing) return null; + if (isTerminalOnboardMachineState(existing.machine.state)) return existing; + + let emitted = false; + const updatedSession = updateSession((session) => { + const step = session.steps[stepName]; + if (!step) return session; + if (isTerminalOnboardMachineState(session.machine.state)) return session; + const now = new Date().toISOString(); + // Guard the terminality invariant: only a legal -> failed + // transition may be recorded here. + assertValidOnboardMachineTransition(session.machine.state, "failed"); + step.status = "failed"; + step.completedAt = null; + step.error = redactSensitiveText(message); + session.failure = sanitizeFailure({ step: stepName, message, recordedAt: now }); + session.status = "failed"; + transitionMachineSnapshot(session, "failed", now); + emitted = true; + return session; + }); + if (emitted) { + emitOnboardMachineEvent( + createOnboardMachineEvent({ + type: "state.failed", + session: updatedSession, + step: stepName, + error: message, + }), + ); + emitOnboardMachineEvent( + createOnboardMachineEvent({ + type: "onboard.failed", + session: updatedSession, + state: "failed", + step: stepName, + error: message, + }), + ); + } + return updatedSession; +} + export function completeSession(updates: SessionUpdates = {}): Session { const safeUpdates = filterSafeUpdates(updates); let wasComplete = false; diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 0cd51cd46d2..1f225534c74 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -134,7 +134,7 @@ export type RebuildFlowHarness = { executeSandboxCommandSpy: MockInstance; ensureMessagingHostForwardAfterRebuildSpy: MockInstance; logSpy: MockInstance; - markStepFailedSpy: MockInstance; + finalizeIncompleteOnboardStepSpy: MockInstance; openShieldsSpy: MockInstance; onboardSpy: MockInstance; preflightAuthoritativeRebuildTargetSpy: MockInstance; @@ -229,15 +229,18 @@ function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSe } function installTerminalStepFailureMock( - onboardSession: { markStepFailed: (...args: unknown[]) => unknown }, + onboardSession: { finalizeIncompleteOnboardStep: (...args: unknown[]) => unknown }, session: RebuildFlowSession, ): MockInstance { return vi - .spyOn(onboardSession, "markStepFailed") - .mockImplementation((stepName: unknown, message: unknown, options: unknown) => { + .spyOn(onboardSession, "finalizeIncompleteOnboardStep") + .mockImplementation((stepName: unknown, message: unknown) => { + if (session.machine.state === "failed" || session.machine.state === "complete") { + return session; + } const stepKey = String(stepName); - const step = session.steps[stepKey] ?? createStep("pending"); - session.steps[stepKey] = step; + const step = session.steps[stepKey]; + if (!step) return session; step.status = "failed"; step.error = typeof message === "string" ? message : null; session.status = "failed"; @@ -246,10 +249,8 @@ function installTerminalStepFailureMock( message: typeof message === "string" ? message : null, recordedAt: "2026-06-01T00:02:00.000Z", }; - const updateMachine = - (options as { updateMachine?: boolean } | undefined)?.updateMachine === true; - session.machine.state = updateMachine ? "failed" : session.machine.state; - session.machine.revision += updateMachine ? 1 : 0; + session.machine.state = "failed"; + session.machine.revision += 1; return session; }); } @@ -329,7 +330,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): const releaseOnboardLockSpy = vi .spyOn(onboardSession, "releaseOnboardLock") .mockImplementation(() => undefined); - const markStepFailedSpy = installTerminalStepFailureMock(onboardSession, session); + const finalizeIncompleteOnboardStepSpy = installTerminalStepFailureMock(onboardSession, session); session.sandboxName = overrides.sessionSandboxName ?? session.sandboxName; const sandboxEntry = { name: "alpha", @@ -630,7 +631,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): executeSandboxCommandSpy, ensureMessagingHostForwardAfterRebuildSpy, logSpy, - markStepFailedSpy, + finalizeIncompleteOnboardStepSpy, openShieldsSpy, onboardSpy, preflightAuthoritativeRebuildTargetSpy, diff --git a/test/helpers/rebuild-flow-target-image-cases.ts b/test/helpers/rebuild-flow-target-image-cases.ts index 0829855e8a8..ecbb2aa7d3e 100644 --- a/test/helpers/rebuild-flow-target-image-cases.ts +++ b/test/helpers/rebuild-flow-target-image-cases.ts @@ -378,10 +378,9 @@ export function registerRebuildFlowTargetImageTests(): void { expect(process.env[overrideEnvVar]).toBe("nemoclaw-hermes-sandbox-base-local:image-caller"); expect(harness.releaseOnboardLockSpy).toHaveBeenCalled(); - expect(harness.markStepFailedSpy).toHaveBeenCalledWith( + expect(harness.finalizeIncompleteOnboardStepSpy).toHaveBeenCalledWith( "sandbox", "Rebuild recreate failed", - expect.objectContaining({ updateMachine: true }), ); expect(harness.session).toMatchObject({ status: "failed", diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index 1f7d57e788d..d9164c137d6 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -179,7 +179,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .spyOn(onboardSession, "releaseOnboardLock") .mockImplementation(() => undefined); vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({ acquired: true }); - const markStepFailedSpy = installTerminalStepFailureMock(onboardSession, session); + const finalizeIncompleteOnboardStepSpy = installTerminalStepFailureMock(onboardSession, session); session.sandboxName = overrides.sessionSandboxName ?? session.sandboxName; const modelsCustomOpenClawImage = typeof overrides.sandboxEntry?.fromDockerfile === "string" && @@ -502,7 +502,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): ensureValidatedBraveSearchCredentialSpy, hydrateCredentialEnvSpy, logSpy, - markStepFailedSpy, + finalizeIncompleteOnboardStepSpy, onboardSpy, registryUpdateSpy, setDefaultSpy, diff --git a/test/helpers/rebuild-flow-test-support.ts b/test/helpers/rebuild-flow-test-support.ts index 1611b80735f..bd1b57993a1 100644 --- a/test/helpers/rebuild-flow-test-support.ts +++ b/test/helpers/rebuild-flow-test-support.ts @@ -105,7 +105,7 @@ export type RebuildFlowHarness = { ensureValidatedBraveSearchCredentialSpy: MockInstance; hydrateCredentialEnvSpy: MockInstance; logSpy: MockInstance; - markStepFailedSpy: MockInstance; + finalizeIncompleteOnboardStepSpy: MockInstance; onboardSpy: MockInstance; registryUpdateSpy: MockInstance; setDefaultSpy: MockInstance; @@ -178,15 +178,18 @@ export function createRebuildFlowSession(machineSnapshotVersion: number): Rebuil }; } export function installTerminalStepFailureMock( - onboardSession: { markStepFailed: (...args: unknown[]) => unknown }, + onboardSession: { finalizeIncompleteOnboardStep: (...args: unknown[]) => unknown }, session: RebuildFlowSession, ): MockInstance { return vi - .spyOn(onboardSession, "markStepFailed") - .mockImplementation((stepName: unknown, message: unknown, options: unknown) => { + .spyOn(onboardSession, "finalizeIncompleteOnboardStep") + .mockImplementation((stepName: unknown, message: unknown) => { + if (session.machine.state === "failed" || session.machine.state === "complete") { + return session; + } const stepKey = String(stepName); - const step = session.steps[stepKey] ?? createStep("pending"); - session.steps[stepKey] = step; + const step = session.steps[stepKey]; + if (!step) return session; step.status = "failed"; step.error = typeof message === "string" ? message : null; session.status = "failed"; @@ -195,10 +198,8 @@ export function installTerminalStepFailureMock( message: typeof message === "string" ? message : null, recordedAt: "2026-06-01T00:02:00.000Z", }; - const updateMachine = - (options as { updateMachine?: boolean } | undefined)?.updateMachine === true; - session.machine.state = updateMachine ? "failed" : session.machine.state; - session.machine.revision += updateMachine ? 1 : 0; + session.machine.state = "failed"; + session.machine.revision += 1; return session; }); } diff --git a/test/onboard-lifecycle.test.ts b/test/onboard-lifecycle.test.ts index 1b22a3726cf..e6c29a9eba5 100644 --- a/test/onboard-lifecycle.test.ts +++ b/test/onboard-lifecycle.test.ts @@ -53,7 +53,7 @@ function runOnboardEntrypoint( return JSON.parse(line) as T; } -function runLifecycleEntrypoint(mode: "fresh" | "resume"): LifecyclePayload { +function runLifecycleEntrypoint(mode: "fresh" | "resume" | "recovery"): LifecyclePayload { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-lifecycle-")); const scriptPath = path.join(tmpDir, `onboard-lifecycle-${mode}.cjs`); @@ -99,9 +99,31 @@ if (${JSON.stringify(mode)} === "resume") { }), ); } +if (${JSON.stringify(mode)} === "recovery") { + const session = onboardModule.onboardSession.createSession({ + mode: "non-interactive", + sandboxName: "resume-lifecycle", + status: "failed", + lastStepStarted: "gateway", + failure: { + step: "gateway", + message: "gateway failed", + recordedAt: "2026-05-27T00:00:00.000Z", + }, + machine: { + version: 1, + state: "failed", + stateEnteredAt: "2026-05-27T00:00:00.000Z", + revision: 4, + }, + metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, + }); + session.steps.gateway.status = "failed"; + onboardModule.onboardSession.saveSession(session); +} const options = { - resume: ${JSON.stringify(mode)} === "resume", + resume: ${JSON.stringify(mode)} !== "fresh", nonInteractive: true, acceptThirdPartySoftware: true, sandboxName: "fresh-lifecycle", @@ -265,6 +287,23 @@ describe("onboard entrypoint lifecycle events", () => { assert.deepEqual(payload.events, [{ type: "onboard.resumed", state: "init", step: null }]); }); + it("emits recovery completion after onboard.resumed (#6227)", () => { + const payload = runLifecycleEntrypoint("recovery"); + + assert.deepEqual(payload.calls, [ + { + resumed: true, + sessionBeforeExists: true, + mode: "non-interactive", + sandboxName: "resume-lifecycle", + }, + ]); + assert.deepEqual(payload.events, [ + { type: "onboard.resumed", state: "gateway", step: null }, + { type: "state.repair.completed", state: "gateway", step: null }, + ]); + }); + it("emits one resume.conflict event for each resume mismatch before exiting", () => { const payload = runResumeConflictEntrypoint();