From c3984520d3ce540aea62201795deb0604771986a Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 03:35:48 -0700 Subject: [PATCH 01/42] fix(security): bind sandbox mutation authority Signed-off-by: Apurv Kumaria --- src/lib/onboard.ts | 6 +- src/lib/onboard/cancel-rollback.test.ts | 344 +++++------------- src/lib/onboard/cancel-rollback.ts | 147 ++------ .../onboard/created-sandbox-finalization.ts | 30 +- ...naged-workload-rebuild-transaction.test.ts | 209 ++++++++++- .../managed-workload/rebuild/commit.ts | 59 ++- .../managed-workload/rebuild/contract.ts | 4 + .../onboard/managed-workload/rebuild/plan.ts | 1 + .../managed-workload/rebuild/recovery.ts | 5 + .../rebuild/replacement-authority.ts | 98 +++++ .../managed-workload/rebuild/transaction.ts | 21 +- .../sandbox-create/orchestration.test.ts | 108 ++++-- .../onboard/sandbox-create/orchestration.ts | 42 +-- .../provider-publication.test.ts | 56 +-- .../sandbox-create/provider-publication.ts | 31 +- .../state/registry-rebuild-authority.test.ts | 49 ++- src/lib/state/registry/rebuild-authority.ts | 48 ++- .../policy/managed-policy-receipt-fixture.ts | 4 - .../policy/policies-permissive-policy.test.ts | 2 +- test/runtime/policy/policies.test.ts | 2 +- .../runtime/policy/policy-explain-cli.test.ts | 2 +- .../policy-mutation-read-failure.test.ts | 2 +- 22 files changed, 708 insertions(+), 562 deletions(-) create mode 100644 src/lib/onboard/managed-workload/rebuild/replacement-authority.ts delete mode 100644 test/runtime/policy/managed-policy-receipt-fixture.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 874a9c89d9f..588b72240be 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2635,11 +2635,7 @@ const onboardRuntimeBoundary = new OnboardRuntimeBoundary({ maybeForceE2eStepFailure, }); -const sandboxCancelRollback = installSandboxCancelRollback({ - runOpenshell, - registry, - clearOnboardSession: onboardSession.clearSession, -}); // #4614 +const sandboxCancelRollback = installSandboxCancelRollback({}); // #4614 const { arePolicyPresetsApplied, diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 64c83eef5ad..c8deaa32896 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -8,339 +8,162 @@ import { createSandboxCancelRollback, installSandboxCancelRollback, makeOnboardCancelExit, - type SandboxCancelRollbackDeps, } from "./cancel-rollback"; -import type { SandboxEntry } from "../state/registry"; const SANDBOX_FINGERPRINT = "a".repeat(64); -type ExternalPendingPolicyVerification = Extract< - NonNullable, - { policyAuthority: "externally-managed" } ->; - -function pendingSandboxEntry( - overrides: Partial = {}, -): SandboxEntry { - return { - name: "new-sb", - gatewayName: "nemoclaw", - gatewayPort: 8080, - lifecycleGeneration: "generation-1", - lifecycleLiveIdentityFingerprint: SANDBOX_FINGERPRINT, - pendingPolicyVerification: { - schemaVersion: 1, - state: "verified-create", - gatewayName: "nemoclaw", - gatewayPort: 8080, - sandboxName: "new-sb", - lifecycleGeneration: "generation-1", - sandboxIdentityFingerprint: SANDBOX_FINGERPRINT, - route: "none", - policyHash: "policy-hash", - policyVersion: 1, - policyAuthority: "externally-managed", - observedPolicyAuthority: "externally-managed", - ...overrides, - }, - }; -} -function createDeps(overrides: Partial = {}) { - const calls = { - deleteContainer: vi.fn((_name: string) => true), - removeFromRegistry: vi.fn(), - clearSession: vi.fn(), - log: vi.fn(), - }; - const deps: SandboxCancelRollbackDeps = { - deleteSandboxContainer: calls.deleteContainer, - removeSandboxFromRegistry: calls.removeFromRegistry, - clearOnboardSession: calls.clearSession, - log: calls.log, - ...overrides, - }; - return { calls, deps }; +function createHarness() { + const log = vi.fn(); + return { log, rollback: createSandboxCancelRollback({ log }) }; } describe("createSandboxCancelRollback", () => { - it("rolls back (delete + unregister) when armed and cancelled", () => { - const { deps, calls } = createDeps(); - const rollback = createSandboxCancelRollback(deps); + it("preserves an armed cancelled sandbox and reports its captured identity (#9833)", () => { + const { rollback, log } = createHarness(); - rollback.arm("new-sb"); + rollback.arm("new-sb", SANDBOX_FINGERPRINT); rollback.markCancelled(); rollback.runIfArmed(); - expect(calls.deleteContainer).toHaveBeenCalledWith("new-sb"); - expect(calls.removeFromRegistry).toHaveBeenCalledWith("new-sb"); - // also discards the aborted session so `nemoclaw list` recovery can't resurrect it - expect(calls.clearSession).toHaveBeenCalledOnce(); - // delete is attempted before the registry entry is removed - expect(calls.deleteContainer.mock.invocationCallOrder[0]).toBeLessThan( - calls.removeFromRegistry.mock.invocationCallOrder[0], - ); - expect(calls.log).toHaveBeenCalledWith( - expect.stringContaining("removed incomplete sandbox 'new-sb'"), - ); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain("preserved incomplete sandbox 'new-sb'"); + expect(guidance).toContain(SANDBOX_FINGERPRINT); + expect(guidance).toContain("OpenShell administrator"); + expect(guidance).toContain("Do not delete the sandbox by mutable sandbox name"); }); - it("preserves recovery state when container deletion is not confirmed", () => { - const { deps, calls } = createDeps({ deleteSandboxContainer: vi.fn(() => false) }); - const rollback = createSandboxCancelRollback(deps); + it.each([ + ["missing", undefined], + ["invalid", "not-a-fingerprint"], + ])("preserves registry and session recovery guidance when identity is %s", (_case, identity) => { + const { rollback, log } = createHarness(); - rollback.arm("new-sb"); + rollback.arm("new-sb", identity); rollback.markCancelled(); rollback.runIfArmed(); - expect(calls.removeFromRegistry).not.toHaveBeenCalled(); - expect(calls.clearSession).not.toHaveBeenCalled(); - expect(calls.log).toHaveBeenCalledWith( - expect.stringContaining("preserved incomplete sandbox 'new-sb'"), - ); - expect(calls.log.mock.calls.flat().join("\n")).not.toContain("openshell sandbox delete"); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain("preserved incomplete sandbox 'new-sb'"); + expect(guidance).toContain("identity fingerprint is unavailable"); + expect(guidance).toContain("preserve the registry and onboarding recovery state"); }); - it("does NOT roll back on a non-cancel exit (armed but not cancelled)", () => { - const { deps, calls } = createDeps(); - const rollback = createSandboxCancelRollback(deps); + it("keeps the captured identity when the mutable name is replaced before exit (#9833)", () => { + const { rollback, log } = createHarness(); + const replacementFingerprint = "b".repeat(64); - rollback.arm("new-sb"); - // no markCancelled() — this is an ordinary failure-path process.exit + rollback.arm("new-sb", SANDBOX_FINGERPRINT); + // A replacement can take the same mutable name, but cannot alter the identity + // captured by the completed create boundary. + const sameNameReplacement = { name: "new-sb", fingerprint: replacementFingerprint }; + expect(sameNameReplacement.fingerprint).not.toBe(SANDBOX_FINGERPRINT); + rollback.markCancelled(); rollback.runIfArmed(); - expect(calls.deleteContainer).not.toHaveBeenCalled(); - expect(calls.removeFromRegistry).not.toHaveBeenCalled(); - expect(calls.clearSession).not.toHaveBeenCalled(); - expect(calls.log).not.toHaveBeenCalled(); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain(SANDBOX_FINGERPRINT); + expect(guidance).not.toContain(replacementFingerprint); }); - it("does NOT roll back when cancelled before any sandbox was armed", () => { - const { deps, calls } = createDeps(); - const rollback = createSandboxCancelRollback(deps); + it("does not run on a non-cancel exit", () => { + const { rollback, log } = createHarness(); + + rollback.arm("new-sb", SANDBOX_FINGERPRINT); + rollback.runIfArmed(); + + expect(log).not.toHaveBeenCalled(); + }); + + it("does not run when cancelled before any sandbox was armed", () => { + const { rollback, log } = createHarness(); rollback.markCancelled(); rollback.runIfArmed(); - expect(calls.deleteContainer).not.toHaveBeenCalled(); - expect(calls.removeFromRegistry).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); }); - it("does NOT roll back after disarm (policies confirmed), even if later cancelled", () => { - const { deps, calls } = createDeps(); - const rollback = createSandboxCancelRollback(deps); + it("does not run after disarm", () => { + const { rollback, log } = createHarness(); - rollback.arm("new-sb"); + rollback.arm("new-sb", SANDBOX_FINGERPRINT); rollback.disarm(); rollback.markCancelled(); rollback.runIfArmed(); - expect(calls.deleteContainer).not.toHaveBeenCalled(); - expect(calls.removeFromRegistry).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); }); - it("is idempotent — runs the rollback at most once", () => { - const { deps, calls } = createDeps(); - const rollback = createSandboxCancelRollback(deps); + it("runs at most once", () => { + const { rollback, log } = createHarness(); - rollback.arm("new-sb"); + rollback.arm("new-sb", SANDBOX_FINGERPRINT); rollback.markCancelled(); rollback.runIfArmed(); - rollback.runIfArmed(); + const callCount = log.mock.calls.length; rollback.runIfArmed(); - expect(calls.deleteContainer).toHaveBeenCalledTimes(1); - expect(calls.removeFromRegistry).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledTimes(callCount); }); - it("reports armed state via isArmed()", () => { - const { deps } = createDeps(); - const rollback = createSandboxCancelRollback(deps); + it("tracks the latest armed sandbox and identity", () => { + const { rollback, log } = createHarness(); expect(rollback.isArmed()).toBe(false); - rollback.arm("new-sb"); + rollback.arm("first", "b".repeat(64)); + rollback.arm("second", SANDBOX_FINGERPRINT); expect(rollback.isArmed()).toBe(true); - rollback.disarm(); - expect(rollback.isArmed()).toBe(false); - }); - - it("re-arming after a previous sandbox tracks the latest name", () => { - const { deps, calls } = createDeps(); - const rollback = createSandboxCancelRollback(deps); - - rollback.arm("first"); - rollback.arm("second"); rollback.markCancelled(); rollback.runIfArmed(); - expect(calls.deleteContainer).toHaveBeenCalledWith("second"); - expect(calls.deleteContainer).not.toHaveBeenCalledWith("first"); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain("second"); + expect(guidance).toContain(SANDBOX_FINGERPRINT); + expect(guidance).not.toContain("b".repeat(64)); + expect(rollback.isArmed()).toBe(false); }); }); describe("installSandboxCancelRollback", () => { - it("wires delete to openshell and unregister to the registry, and registers an exit hook", () => { - const runOpenshell = vi.fn(() => ({ status: 0 })); - const removeSandbox = vi.fn(); - const exitHandlers: Array<() => void> = []; - - const rollback = installSandboxCancelRollback({ - runOpenshell, - registry: { getSandbox: () => null, removeSandbox }, - clearOnboardSession: () => {}, - registerExitHandler: (h) => exitHandlers.push(h), - }); - - expect(exitHandlers).toHaveLength(1); - - rollback.arm("new-sb"); - rollback.markCancelled(); - exitHandlers[0](); - - expect(runOpenshell).toHaveBeenCalledWith(["sandbox", "delete", "new-sb"], { - ignoreError: true, - }); - expect(removeSandbox).toHaveBeenCalledWith("new-sb"); - }); - - it("does not fire the rollback on a non-cancel exit", () => { - const runOpenshell = vi.fn(() => ({ status: 0 })); - const removeSandbox = vi.fn(); - const exitHandlers: Array<() => void> = []; - - const rollback = installSandboxCancelRollback({ - runOpenshell, - registry: { getSandbox: () => null, removeSandbox }, - clearOnboardSession: () => {}, - registerExitHandler: (h) => exitHandlers.push(h), - }); - rollback.arm("new-sb"); // armed, but never cancelled - exitHandlers[0](); - - expect(runOpenshell).not.toHaveBeenCalled(); - expect(removeSandbox).not.toHaveBeenCalled(); - }); - - it("deletes a pending create only after its exact identity and checkpoint are re-read", () => { - const entry = pendingSandboxEntry(); - const getSandbox = vi.fn(() => entry); - const runOpenshell = vi.fn(() => ({ status: 0 })); - const removeSandbox = vi.fn(); - const clearOnboardSession = vi.fn(); - const inspectIdentity = vi.fn(() => SANDBOX_FINGERPRINT); - const exitHandlers: Array<() => void> = []; - - const rollback = installSandboxCancelRollback({ - runOpenshell, - registry: { getSandbox, removeSandbox }, - clearOnboardSession, - inspectOpenShellSandboxIdentityFingerprint: inspectIdentity, - registerExitHandler: (handler) => exitHandlers.push(handler), - }); - rollback.arm("new-sb"); - rollback.markCancelled(); - exitHandlers[0](); - - expect(getSandbox).toHaveBeenCalledTimes(2); - expect(inspectIdentity).toHaveBeenCalledWith({ - sandboxName: "new-sb", - gatewayName: "nemoclaw", - }); - expect(runOpenshell).toHaveBeenCalledWith(["sandbox", "delete", "-g", "nemoclaw", "new-sb"], { - ignoreError: true, - }); - expect(removeSandbox).toHaveBeenCalledWith("new-sb"); - expect(clearOnboardSession).toHaveBeenCalledOnce(); - }); - - it("preserves a pending create and its fingerprint when deletion is not confirmed", () => { - const entry = pendingSandboxEntry(); - const runOpenshell = vi.fn(() => ({ status: 1 })); + it("registers a non-destructive exit handler that retains external recovery state (#9833)", () => { + const runOpenshell = vi.fn(); const removeSandbox = vi.fn(); const clearOnboardSession = vi.fn(); const log = vi.fn(); const exitHandlers: Array<() => void> = []; - const rollback = installSandboxCancelRollback({ - runOpenshell, - registry: { getSandbox: () => entry, removeSandbox }, - clearOnboardSession, - inspectOpenShellSandboxIdentityFingerprint: () => SANDBOX_FINGERPRINT, log, registerExitHandler: (handler) => exitHandlers.push(handler), }); - rollback.arm("new-sb"); + + expect(exitHandlers).toHaveLength(1); + rollback.arm("new-sb", SANDBOX_FINGERPRINT); rollback.markCancelled(); exitHandlers[0](); - expect(runOpenshell).toHaveBeenCalledWith(["sandbox", "delete", "-g", "nemoclaw", "new-sb"], { - ignoreError: true, - }); + expect(runOpenshell).not.toHaveBeenCalled(); expect(removeSandbox).not.toHaveBeenCalled(); expect(clearOnboardSession).not.toHaveBeenCalled(); - const guidance = log.mock.calls.flat().join("\n"); - expect(guidance).toContain(SANDBOX_FINGERPRINT); - expect(guidance).toContain("identity-bound recovery"); - expect(guidance).not.toContain("openshell sandbox delete"); + expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); }); - it.each([ - ["does not match", () => "b".repeat(64)], - [ - "cannot be inspected", - () => { - throw new Error("identity unavailable"); - }, - ], - ])("preserves a pending create when its exact identity %s", (_case, inspect) => { - const entry = pendingSandboxEntry(); - const runOpenshell = vi.fn(() => ({ status: 0 })); - const removeSandbox = vi.fn(); - const clearOnboardSession = vi.fn(); + it("preserves missing-checkpoint recovery state without a mutable-name fallback (#9833)", () => { + const runOpenshell = vi.fn(); const log = vi.fn(); const exitHandlers: Array<() => void> = []; - const rollback = installSandboxCancelRollback({ - runOpenshell, - registry: { getSandbox: () => entry, removeSandbox }, - clearOnboardSession, - inspectOpenShellSandboxIdentityFingerprint: vi.fn(inspect), log, registerExitHandler: (handler) => exitHandlers.push(handler), }); - rollback.arm("new-sb"); - rollback.markCancelled(); - exitHandlers[0](); - expect(runOpenshell).not.toHaveBeenCalled(); - expect(removeSandbox).not.toHaveBeenCalled(); - expect(clearOnboardSession).not.toHaveBeenCalled(); - expect(log.mock.calls.flat().join("\n")).toContain("preserved incomplete sandbox 'new-sb'"); - }); - - it("preserves a pending create when its durable checkpoint changes during inspection", () => { - const entry = pendingSandboxEntry(); - const changed = pendingSandboxEntry({ policyVersion: 2 }); - const getSandbox = vi.fn().mockReturnValueOnce(entry).mockReturnValueOnce(changed); - const runOpenshell = vi.fn(() => ({ status: 0 })); - const removeSandbox = vi.fn(); - const clearOnboardSession = vi.fn(); - const exitHandlers: Array<() => void> = []; - - const rollback = installSandboxCancelRollback({ - runOpenshell, - registry: { getSandbox, removeSandbox }, - clearOnboardSession, - inspectOpenShellSandboxIdentityFingerprint: () => SANDBOX_FINGERPRINT, - registerExitHandler: (handler) => exitHandlers.push(handler), - }); rollback.arm("new-sb"); rollback.markCancelled(); exitHandlers[0](); - expect(getSandbox).toHaveBeenCalledTimes(2); expect(runOpenshell).not.toHaveBeenCalled(); - expect(removeSandbox).not.toHaveBeenCalled(); - expect(clearOnboardSession).not.toHaveBeenCalled(); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain("identity fingerprint is unavailable"); + expect(guidance).toContain("OpenShell administrator"); }); }); @@ -361,17 +184,12 @@ describe("makeOnboardCancelExit", () => { }); describe("buildCancelRollbackMessage", () => { - it("reports a clean removal when the delete succeeded", () => { - const lines = buildCancelRollbackMessage("sb", true); - expect(lines.join("\n")).toContain("removed incomplete sandbox 'sb'"); - expect(lines.join("\n")).not.toContain("openshell sandbox delete"); - }); + it("preserves identity-bound recovery guidance", () => { + const message = buildCancelRollbackMessage("sb", SANDBOX_FINGERPRINT).join("\n"); - it("preserves identity-bound recovery guidance when the delete failed", () => { - const lines = buildCancelRollbackMessage("sb", false, SANDBOX_FINGERPRINT); - expect(lines.join("\n")).toContain("preserved incomplete sandbox 'sb'"); - expect(lines.join("\n")).toContain(SANDBOX_FINGERPRINT); - expect(lines.join("\n")).toContain("identity-bound recovery"); - expect(lines.join("\n")).not.toContain("openshell sandbox delete"); + expect(message).toContain("preserved incomplete sandbox 'sb'"); + expect(message).toContain(SANDBOX_FINGERPRINT); + expect(message).toContain("identity-bound recovery or removal"); + expect(message).not.toContain("openshell sandbox delete"); }); }); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index d3f76541ce1..be0ed4dfd3d 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -1,11 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { isDeepStrictEqual } from "node:util"; - -import { inspectOpenShellSandboxIdentityFingerprint } from "../adapters/openshell/policy-authority"; -import type { SandboxEntry } from "../state/registry"; - // Re-exported so the onboard entrypoint imports its sandbox default/cancel // lifecycle helpers from a single module. export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preservation"; @@ -28,28 +23,13 @@ export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preser * behind is preserved exactly as before. Only an explicit cancel rolls back. */ export interface SandboxCancelRollbackDeps { - /** Delete the OpenShell sandbox container. Returns true when the delete succeeded. */ - deleteSandboxContainer(sandboxName: string, gatewayName?: string): boolean; - /** Prove a pending create checkpoint immediately before its name-based delete. */ - prepareSandboxContainerDeletion?(sandboxName: string): { - readonly gatewayName: string; - readonly sandboxIdentityFingerprint: string; - } | null; - /** Remove the sandbox entry from the NemoClaw registry (clears default). */ - removeSandboxFromRegistry(sandboxName: string): void; - /** - * Discard the onboard session for the aborted run. Without this, the session - * still records the sandbox step as "complete", and `nemoclaw list`'s - * session-recovery resurrects the just-removed sandbox as a phantom entry. - */ - clearOnboardSession(): void; /** Emit an operator-facing line (stderr). */ log(message: string): void; } export interface SandboxCancelRollback { /** Arm rollback for a just-created sandbox. */ - arm(sandboxName: string): void; + arm(sandboxName: string, sandboxIdentityFingerprint?: string): void; /** Disarm once the sandbox is past the cancellable window (policies confirmed). */ disarm(): void; /** Record that the operator cancelled at a cancellable step. */ @@ -62,36 +42,25 @@ export interface SandboxCancelRollback { export function buildCancelRollbackMessage( sandboxName: string, - deleteSucceeded: boolean, sandboxIdentityFingerprint?: string, ): string[] { - if (deleteSucceeded) { - return [ - "", - ` Onboarding cancelled — removed incomplete sandbox '${sandboxName}' (no policy presets were applied).`, - ]; - } return [ "", - ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}' because OpenShell did not confirm its deletion.`, + ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}' because OpenShell cannot delete it by immutable identity.`, ...(sandboxIdentityFingerprint ? [ ` Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}`, - " Preserve this fingerprint and give it to an OpenShell administrator for identity-bound recovery.", + " Preserve this fingerprint and give it to an OpenShell administrator for identity-bound recovery or removal.", ] - : [" Preserve its registry and onboarding recovery state for identity-bound recovery."]), - " Do not delete this sandbox by mutable sandbox name.", + : [ + " Its durable identity fingerprint is unavailable; preserve the registry and onboarding recovery state.", + " Ask an OpenShell administrator to establish the immutable sandbox identity before recovery or removal.", + ]), + " Do not delete the sandbox by mutable sandbox name.", ]; } export interface InstallSandboxCancelRollbackOptions { - runOpenshell: (args: string[], opts: { ignoreError: boolean }) => { status: number | null }; - registry: { - getSandbox(name: string): SandboxEntry | null; - removeSandbox(name: string): void; - }; - inspectOpenShellSandboxIdentityFingerprint?: typeof inspectOpenShellSandboxIdentityFingerprint; - clearOnboardSession: () => void; log?: (message: string) => void; /** Override for tests; defaults to `process.on("exit", ...)`. */ registerExitHandler?: (handler: () => void) => void; @@ -103,42 +72,13 @@ export interface InstallSandboxCancelRollbackOptions { * orchestration lives in a focused module rather than the onboard entrypoint. * * `process.exit()` — how the policy-step prompts terminate on Ctrl+C — - * synchronously emits 'exit', and runOpenshell/removeSandbox are synchronous, - * so the rollback completes inside the handler. No-op unless armed AND cancelled. + * synchronously emits 'exit', so the recovery notice completes inside the + * handler. No-op unless armed AND cancelled. */ export function installSandboxCancelRollback( opts: InstallSandboxCancelRollbackOptions, ): SandboxCancelRollback { const rollback = createSandboxCancelRollback({ - prepareSandboxContainerDeletion: (name) => { - const checkpoint = opts.registry.getSandbox(name)?.pendingPolicyVerification; - if (!checkpoint) return null; - const inspectIdentity = - opts.inspectOpenShellSandboxIdentityFingerprint ?? - inspectOpenShellSandboxIdentityFingerprint; - const liveFingerprint = inspectIdentity({ - sandboxName: name, - gatewayName: checkpoint.gatewayName, - }); - const currentCheckpoint = opts.registry.getSandbox(name)?.pendingPolicyVerification; - if ( - liveFingerprint !== checkpoint.sandboxIdentityFingerprint || - !isDeepStrictEqual(currentCheckpoint, checkpoint) - ) { - throw new Error("pending sandbox policy verification identity changed"); - } - return { - gatewayName: checkpoint.gatewayName, - sandboxIdentityFingerprint: checkpoint.sandboxIdentityFingerprint, - }; - }, - deleteSandboxContainer: (name, gatewayName) => - opts.runOpenshell( - gatewayName ? ["sandbox", "delete", "-g", gatewayName, name] : ["sandbox", "delete", name], - { ignoreError: true }, - ).status === 0, - removeSandboxFromRegistry: (name) => opts.registry.removeSandbox(name), - clearOnboardSession: opts.clearOnboardSession, log: opts.log ?? ((message) => console.error(message)), }); const register = @@ -170,65 +110,42 @@ export function makeOnboardCancelExit( export function createSandboxCancelRollback( deps: SandboxCancelRollbackDeps, ): SandboxCancelRollback { - let armedSandboxName: string | null = null; + let armedSandbox: { + readonly name: string; + readonly identityFingerprint: string | null; + } | null = null; let cancelRequested = false; let done = false; return { - arm(sandboxName: string): void { - armedSandboxName = sandboxName; + arm(sandboxName: string, sandboxIdentityFingerprint?: string): void { + armedSandbox = { + name: sandboxName, + identityFingerprint: + typeof sandboxIdentityFingerprint === "string" && + /^[0-9a-f]{64}$/u.test(sandboxIdentityFingerprint) + ? sandboxIdentityFingerprint + : null, + }; }, disarm(): void { - armedSandboxName = null; + armedSandbox = null; }, markCancelled(): void { cancelRequested = true; }, isArmed(): boolean { - return armedSandboxName !== null; + return armedSandbox !== null; }, runIfArmed(): void { - if (done || !cancelRequested || armedSandboxName === null) return; + if (done || !cancelRequested || armedSandbox === null) return; done = true; - const sandboxName = armedSandboxName; - armedSandboxName = null; - - let deletionAuthority: { - readonly gatewayName: string; - readonly sandboxIdentityFingerprint: string; - } | null = null; - try { - deletionAuthority = deps.prepareSandboxContainerDeletion?.(sandboxName) ?? null; - } catch { - deps.log(""); - deps.log( - ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}' because its durable creation identity could not be proved immediately before deletion.`, - ); - return; - } - - let deleteSucceeded = false; - try { - deleteSucceeded = deletionAuthority - ? deps.deleteSandboxContainer(sandboxName, deletionAuthority.gatewayName) - : deps.deleteSandboxContainer(sandboxName); - } catch { - deleteSucceeded = false; - } - if (!deleteSucceeded) { - for (const line of buildCancelRollbackMessage( - sandboxName, - false, - deletionAuthority?.sandboxIdentityFingerprint, - )) { - deps.log(line); - } - return; - } - deps.removeSandboxFromRegistry(sandboxName); - // Discard the aborted session so `nemoclaw list` recovery doesn't resurrect it. - deps.clearOnboardSession(); - for (const line of buildCancelRollbackMessage(sandboxName, true)) { + const { name: sandboxName, identityFingerprint } = armedSandbox; + armedSandbox = null; + for (const line of buildCancelRollbackMessage( + sandboxName, + identityFingerprint ?? undefined, + )) { deps.log(line); } }, diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index 1116ab3a93d..22a0356fe88 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -29,14 +29,9 @@ import type { HermesPortableConfiguredReceipt } from "./experimental/hermes-port import { warnIfLandlockUnsupported } from "./landlock-warning"; import * as managedWorkloadOnboard from "./managed-workload/onboard-orchestration"; import { printMessagingProviderMissing } from "./preflight-messages"; -import { - pendingSandboxPolicyVerificationForBoundary, -} from "./sandbox-create/policy-creation-receipt"; +import { pendingSandboxPolicyVerificationForBoundary } from "./sandbox-create/policy-creation-receipt"; import type { SandboxGpuCreateFlowResult } from "./sandbox-gpu-create-flow"; -import type { - VerifiedSandboxPolicyBoundary, - VerifiedSandboxPolicyRegistration, -} from "./types"; +import type { VerifiedSandboxPolicyBoundary, VerifiedSandboxPolicyRegistration } from "./types"; import type { SelectionDrift } from "./selection-drift"; import { applyOnboardVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; import { @@ -250,6 +245,7 @@ export function completeOrdinaryOnboardSandboxCreation( readonly runtimeFields: RegistrationSeed["runtimeFields"]; readonly messagingProviders: readonly string[]; readonly liveExists: boolean; + readonly lifecycleLiveIdentityFingerprint?: string; }, deps: { readonly setDefault: (sandboxName: string) => void; @@ -257,7 +253,7 @@ export function completeOrdinaryOnboardSandboxCreation( readonly scriptsDir: string; readonly gatewayName: string; readonly providerExistsInGateway: (providerName: string) => boolean; - readonly armCancelRollback: (sandboxName: string) => void; + readonly armCancelRollback: (sandboxName: string, sandboxIdentityFingerprint: string) => void; readonly dockerInfoFormat: Parameters[0]["dockerInfoFormat"]; readonly runCapture: Parameters[0]["runCapture"]; readonly revalidatePolicyAuthority: (operation: string) => void; @@ -287,7 +283,16 @@ export function completeOrdinaryOnboardSandboxCreation( deps.revalidatePolicyAuthority(`reporting sandbox '${input.sandboxName}' creation success`); console.log(` ✓ Sandbox '${input.sandboxName}' created`); warnIfLandlockUnsupported(deps); - if (!input.liveExists) deps.armCancelRollback(input.sandboxName); + if (!input.liveExists) { + const lifecycleLiveIdentityFingerprint = input.lifecycleLiveIdentityFingerprint; + if ( + !lifecycleLiveIdentityFingerprint || + !/^[0-9a-f]{64}$/u.test(lifecycleLiveIdentityFingerprint) + ) { + throw new Error(`Sandbox '${input.sandboxName}' has no exact identity for cancel recovery.`); + } + deps.armCancelRollback(input.sandboxName, lifecycleLiveIdentityFingerprint); + } return input.sandboxName; } @@ -474,8 +479,7 @@ export function createCreatedSandboxCompletionActions( policyAuthority: verifiedPolicyRegistration.policyAuthority, ...(verifiedPolicyRegistration.policyAuthority === "nemoclaw-managed" ? { - policyCreationReceipt: - verifiedPolicyRegistration.policyCreationReceipt, + policyCreationReceipt: verifiedPolicyRegistration.policyCreationReceipt, } : {}), inferenceRouteReservation, @@ -566,8 +570,8 @@ type OnboardGatewayBinding = { }; type OnboardPreparedPolicy = Omit< Pick< - managedWorkloadOnboard.PreparedOnboardSandboxWorkloadLaunch, - "initialSandboxPolicy" | "policyTier" | "policyAuthority" | "dashboardRemoteBindPrepared" + managedWorkloadOnboard.PreparedOnboardSandboxWorkloadLaunch, + "initialSandboxPolicy" | "policyTier" | "policyAuthority" | "dashboardRemoteBindPrepared" >, "policyAuthority" > & { diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index 72ee82fe656..5fd983dd25c 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -36,7 +36,10 @@ import { type StagedManagedWorkloadReplacement, } from "./managed-workload/rebuild/contract"; import { createManagedWorkloadReplacementRollback } from "./managed-workload/rebuild/rollback"; -import { runManagedWorkloadRebuildTransaction } from "./managed-workload/rebuild/transaction"; +import { + type ManagedWorkloadRebuildTransactionDependencies, + runManagedWorkloadRebuildTransaction, +} from "./managed-workload/rebuild/transaction"; import type { RuntimeProviderBundle } from "./runtime-provider/contract"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION } from "./runtime-provider/contract"; import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; @@ -47,6 +50,47 @@ const PROVIDERS = ["docker", "mxc"] as const; const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; const OLD_RELEASE = "v0.0.99"; const NEW_RELEASE = "v0.0.100"; +const OLD_GENERATION = "00000000-0000-4000-8000-000000000001"; +const NEW_GENERATION = "00000000-0000-4000-8000-000000000002"; +const OLD_FINGERPRINT = "a".repeat(64); +const NEW_FINGERPRINT = "b".repeat(64); + +const replacementPolicyAuthority = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + policySourcePath: "/tmp/replacement-policy.yaml", + route: "none" as const, + plannedAuthority: "nemoclaw-managed" as const, +}; + +function replacementAuthorityDependencies( + overrides: Partial< + NonNullable + > = {}, +): NonNullable { + return { + inspectSandboxIdentity: vi.fn(() => NEW_FINGERPRINT), + verifyCreatedPolicy: vi.fn( + (input) => + ({ + policyAuthority: "nemoclaw-managed", + observedPolicyAuthority: "owner-unknown", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + sandboxName: input.sandboxName, + lifecycleGeneration: input.lifecycleGeneration, + sandboxIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, + policyHash: "policy-new", + policyVersion: 2, + }, + }) as const, + ), + ...overrides, + }; +} function raiseInjectedFailure(message: string): never { throw new Error(message); @@ -131,8 +175,8 @@ function previousEntry( model: "nvidia/nemotron", imageTag: workload.reference, workload, - lifecycleGeneration: "generation-old", - lifecycleLiveIdentityFingerprint: "fingerprint-old", + lifecycleGeneration: OLD_GENERATION, + lifecycleLiveIdentityFingerprint: OLD_FINGERPRINT, gatewayName: "nemoclaw", gatewayPort: 8080, }; @@ -266,7 +310,7 @@ function operationsHarness( providerId: string, events: string[], failAt: FailurePhase = null, - previousLiveIdentityFingerprint = "fingerprint-old", + previousLiveIdentityFingerprint = OLD_FINGERPRINT, ): ManagedWorkloadRebuildProviderOperations { const bound = { schemaVersion: 1 as const, @@ -283,8 +327,8 @@ function operationsHarness( ...bound, previousRuntimeHandle: prepared.previousRuntimeHandle, stagingHandle: "runtime-new-staged-exact", - lifecycleGeneration: "generation-new", - liveIdentityFingerprint: "fingerprint-new", + lifecycleGeneration: NEW_GENERATION, + liveIdentityFingerprint: NEW_FINGERPRINT, }; const ready: ReadyManagedWorkloadReplacement = { ...staged, @@ -347,6 +391,14 @@ function transactionHarness( failAt: FailurePhase = null, platform: (typeof PLATFORMS)[number] = "linux/amd64", previousEntryOverrides: Partial = {}, + replacementOptions: { + readonly policy?: Parameters< + typeof runManagedWorkloadRebuildTransaction + >[0]["replacementPolicyAuthority"]; + readonly dependencies?: NonNullable< + ManagedWorkloadRebuildTransactionDependencies["replacementAuthority"] + >; + } = {}, ) { const events: string[] = []; const oldEntry = { ...previousEntry(agent, providerId, platform), ...previousEntryOverrides }; @@ -410,6 +462,7 @@ function transactionHarness( provider: bundle(providerId), handoff: handoff(agent, providerId, platform), operations, + replacementPolicyAuthority: replacementOptions.policy ?? replacementPolicyAuthority, replacementMetadata: { model: "nvidia/nemotron-new" }, transactionId: "transaction-1", }, @@ -428,6 +481,8 @@ function transactionHarness( return structuredClone(currentEntry); }, commitAuthority, + replacementAuthority: + replacementOptions.dependencies ?? replacementAuthorityDependencies(), }, ), }; @@ -653,8 +708,8 @@ describe("managed workload rebuild transaction", () => { openshellDriver: provider, model: "nvidia/nemotron-new", fromDockerfile: null, - lifecycleGeneration: "generation-new", - lifecycleLiveIdentityFingerprint: "fingerprint-new", + lifecycleGeneration: NEW_GENERATION, + lifecycleLiveIdentityFingerprint: NEW_FINGERPRINT, workload: { kind: "managed-image", platform, @@ -699,7 +754,7 @@ describe("managed workload rebuild transaction", () => { ); }); - it("publishes a replacement without carrying the previous policy receipt (#9833)", async () => { + it("publishes a replacement-bound receipt without carrying the previous receipt (#9833)", async () => { const lifecycleGeneration = "00000000-0000-4000-8000-000000000001"; const sandboxIdentityFingerprint = "a".repeat(64); const harness = transactionHarness("openclaw", "mxc", null, "linux/amd64", { @@ -721,9 +776,111 @@ describe("managed workload rebuild transaction", () => { const result = await harness.run(); - expect(result.entry.lifecycleGeneration).toBe("generation-new"); - expect(result.entry).not.toHaveProperty("policyAuthority"); + expect(result.entry.lifecycleGeneration).toBe(NEW_GENERATION); + expect(result.entry).toMatchObject({ + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: NEW_GENERATION, + sandboxIdentityFingerprint: NEW_FINGERPRINT, + policyHash: "policy-new", + policyVersion: 2, + }, + }); + expect(result.entry.policyCreationReceipt).not.toEqual(harness.oldEntry.policyCreationReceipt); + }); + + it("publishes verified global authority without a NemoClaw receipt (#9833)", async () => { + const verifyCreatedPolicy = vi.fn(() => ({ + policyAuthority: "externally-managed" as const, + policyCreationReceipt: null, + observedPolicyAuthority: "externally-managed" as const, + policyIdentity: { hash: "global-policy", activeVersion: 4 }, + })); + const harness = transactionHarness( + "openclaw", + "mxc", + null, + "linux/amd64", + {}, + { + policy: { + ...replacementPolicyAuthority, + plannedAuthority: "externally-managed", + }, + dependencies: replacementAuthorityDependencies({ verifyCreatedPolicy }), + }, + ); + + const result = await harness.run(); + + expect(result.entry).toMatchObject({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: NEW_GENERATION, + lifecycleLiveIdentityFingerprint: NEW_FINGERPRINT, + policyAuthority: "externally-managed", + }); expect(result.entry).not.toHaveProperty("policyCreationReceipt"); + expect(verifyCreatedPolicy).toHaveBeenCalledWith( + expect.objectContaining({ + plannedAuthority: "externally-managed", + lifecycleGeneration: NEW_GENERATION, + lifecycleLiveIdentityFingerprint: NEW_FINGERPRINT, + }), + ); + }); + + it("rolls back and preserves old authority when replacement identity changes during policy proof (#9833)", async () => { + const inspectSandboxIdentity = vi + .fn() + .mockReturnValueOnce(NEW_FINGERPRINT) + .mockReturnValueOnce("c".repeat(64)); + const harness = transactionHarness( + "openclaw", + "mxc", + null, + "linux/amd64", + {}, + { + dependencies: replacementAuthorityDependencies({ inspectSandboxIdentity }), + }, + ); + + await expect(harness.run()).rejects.toMatchObject({ + phase: "registry-commit", + message: expect.stringContaining("identity changed after policy verification"), + }); + + expect(inspectSandboxIdentity).toHaveBeenCalledTimes(2); + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.operations.rollback).toHaveBeenCalledOnce(); + expect(harness.events).not.toContain("registry-commit"); + expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); + }); + + it("rolls back and preserves old authority when replacement policy proof fails (#9833)", async () => { + const harness = transactionHarness( + "openclaw", + "mxc", + null, + "linux/amd64", + {}, + { + dependencies: replacementAuthorityDependencies({ + verifyCreatedPolicy: vi.fn(() => { + throw new Error("replacement policy changed"); + }), + }), + }, + ); + + await expect(harness.run()).rejects.toMatchObject({ phase: "registry-commit" }); + + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.operations.rollback).toHaveBeenCalledOnce(); + expect(harness.events).not.toContain("registry-commit"); }); it("rolls back a not-ready replacement by exact staged handle", async () => { @@ -739,7 +896,7 @@ describe("managed workload rebuild transaction", () => { "readiness", "rollback:runtime-new-staged-exact", ]); - expect(harness.currentEntry().lifecycleGeneration).toBe("generation-old"); + expect(harness.currentEntry().lifecycleGeneration).toBe(OLD_GENERATION); }); it.each(INVALID_PROVIDER_ARTIFACT_CASES)( @@ -782,6 +939,7 @@ describe("managed workload rebuild transaction", () => { provider: bundle("mxc"), handoff: handoff("openclaw", "mxc"), operations, + replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(currentEntry) }, @@ -846,9 +1004,20 @@ describe("managed workload rebuild transaction", () => { operation: "retire-previous", previousRuntimeHandle: "runtime-old-exact", stagingHandle: "runtime-new-staged-exact", + replacement: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + policyRegistration: { + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + lifecycleGeneration: NEW_GENERATION, + sandboxIdentityFingerprint: NEW_FINGERPRINT, + }, + }, + }, }, }); - expect(harness.currentEntry().lifecycleGeneration).toBe("generation-new"); + expect(harness.currentEntry().lifecycleGeneration).toBe(NEW_GENERATION); expect(harness.events.at(-1)).toBe("retire:runtime-old-exact"); expect(harness.operations.rollback).not.toHaveBeenCalled(); }); @@ -866,7 +1035,7 @@ describe("managed workload rebuild transaction", () => { expect(result).toMatchObject({ status: "committed", entry: { - lifecycleGeneration: "generation-new", + lifecycleGeneration: NEW_GENERATION, workload: { platform: "linux/arm64" }, }, }); @@ -893,7 +1062,7 @@ describe("managed workload rebuild transaction", () => { }, }); - expect(harness.currentEntry().lifecycleGeneration).toBe("generation-new"); + expect(harness.currentEntry().lifecycleGeneration).toBe(NEW_GENERATION); expect(harness.operations.rollback).not.toHaveBeenCalled(); expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); }); @@ -927,8 +1096,8 @@ describe("managed workload rebuild transaction", () => { transactionId: "transaction-1", previousRuntimeHandle: "runtime-old-exact", stagingHandle: "runtime-new-staged-exact", - lifecycleGeneration: "generation-new", - liveIdentityFingerprint: "fingerprint-new", + lifecycleGeneration: NEW_GENERATION, + liveIdentityFingerprint: NEW_FINGERPRINT, }; const rollback = createManagedWorkloadReplacementRollback(plan, staged, providerOperations); @@ -964,6 +1133,7 @@ describe("managed workload rebuild transaction", () => { }, }, operations, + replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, @@ -993,6 +1163,7 @@ describe("managed workload rebuild transaction", () => { }, }, operations, + replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, @@ -1018,6 +1189,7 @@ describe("managed workload rebuild transaction", () => { replacementProfile: hermesHandoff.replacementProfile, }, operations, + replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, @@ -1049,6 +1221,7 @@ describe("managed workload rebuild transaction", () => { provider: bundle("mxc"), handoff: handoff("openclaw", "mxc"), operations, + replacementPolicyAuthority, transactionId: "transaction-1", }, { @@ -1057,6 +1230,7 @@ describe("managed workload rebuild transaction", () => { status: "committed", entry: structuredClone(replacement), }), + replacementAuthority: replacementAuthorityDependencies(), }, ); @@ -1074,6 +1248,7 @@ describe("managed workload rebuild transaction", () => { provider: bundle("mxc"), handoff: handoff("openclaw", "mxc"), operations, + replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, diff --git a/src/lib/onboard/managed-workload/rebuild/commit.ts b/src/lib/onboard/managed-workload/rebuild/commit.ts index c555dadffa7..c758cf60591 100644 --- a/src/lib/onboard/managed-workload/rebuild/commit.ts +++ b/src/lib/onboard/managed-workload/rebuild/commit.ts @@ -8,6 +8,7 @@ import { sandboxRebuildReplacementMatchesEntry, } from "../../../state/registry/rebuild-authority"; import type { SandboxEntry } from "../../../state/registry/types"; +import type { VerifiedSandboxPolicyBoundary } from "../../types"; import type { ManagedWorkloadRebuildPlan, ReboundManagedWorkloadReplacement } from "./contract"; import { ManagedWorkloadRebuildIndeterminatePublicationError, @@ -25,6 +26,7 @@ export type ReadSandboxRebuildEntry = (sandboxName: string) => SandboxEntry | nu function reconcileAmbiguousPublication( plan: ManagedWorkloadRebuildPlan, replacement: ReboundManagedWorkloadReplacement, + policyBoundary: VerifiedSandboxPolicyBoundary, candidate: SandboxEntry, publicationError: unknown, readSandbox?: ReadSandboxRebuildEntry, @@ -32,7 +34,12 @@ function reconcileAmbiguousPublication( if (!readSandbox) { throw new ManagedWorkloadRebuildIndeterminatePublicationError( "publication failed without an authoritative reconciliation read", - createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), + createManagedWorkloadRebuildRecoveryTask( + plan, + replacement, + policyBoundary, + "reconcile-publication", + ), { cause: publicationError }, ); } @@ -42,7 +49,12 @@ function reconcileAmbiguousPublication( } catch (reconciliationError) { throw new ManagedWorkloadRebuildIndeterminatePublicationError( "publication and authoritative reconciliation both failed", - createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), + createManagedWorkloadRebuildRecoveryTask( + plan, + replacement, + policyBoundary, + "reconcile-publication", + ), { cause: new AggregateError( [publicationError, reconciliationError], @@ -63,7 +75,12 @@ function reconcileAmbiguousPublication( } throw new ManagedWorkloadRebuildIndeterminatePublicationError( "publication could not be reconciled to the replacement or exact old authority", - createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), + createManagedWorkloadRebuildRecoveryTask( + plan, + replacement, + policyBoundary, + "reconcile-publication", + ), { cause: publicationError }, ); } @@ -72,7 +89,18 @@ export function materializeManagedWorkloadReplacementEntry( previousEntry: SandboxEntry, plan: ManagedWorkloadRebuildPlan, replacement: ReboundManagedWorkloadReplacement, + policyBoundary: VerifiedSandboxPolicyBoundary, ): SandboxEntry { + if ( + policyBoundary.sandboxName !== plan.sandboxName || + policyBoundary.lifecycleGeneration !== replacement.lifecycleGeneration || + policyBoundary.lifecycleLiveIdentityFingerprint !== replacement.liveIdentityFingerprint + ) { + throw new ManagedWorkloadRebuildTransactionError( + "registry-commit", + "the verified replacement policy boundary does not match the replacement lifecycle", + ); + } const { policyAuthority: _previousPolicyAuthority, policyCreationReceipt: _previousPolicyCreationReceipt, @@ -84,6 +112,7 @@ export function materializeManagedWorkloadReplacementEntry( name: plan.sandboxName, pendingRouteReservation: undefined, reservationSessionId: undefined, + pendingPolicyVerification: undefined, openshellDriver: plan.providerId, agent: plan.agent, fromDockerfile: null, @@ -91,6 +120,12 @@ export function materializeManagedWorkloadReplacementEntry( workload: plan.replacementReceipt, lifecycleGeneration: replacement.lifecycleGeneration, lifecycleLiveIdentityFingerprint: replacement.liveIdentityFingerprint, + gatewayName: policyBoundary.gatewayName, + gatewayPort: policyBoundary.gatewayPort, + policyAuthority: policyBoundary.registration.policyAuthority, + ...(policyBoundary.registration.policyAuthority === "nemoclaw-managed" + ? { policyCreationReceipt: policyBoundary.registration.policyCreationReceipt } + : {}), }); } @@ -98,15 +133,28 @@ export function commitManagedWorkloadReplacement( previousEntry: SandboxEntry, plan: ManagedWorkloadRebuildPlan, replacement: ReboundManagedWorkloadReplacement, + policyBoundary: VerifiedSandboxPolicyBoundary, commit: CommitSandboxRebuildAuthority = compareAndSwapSandboxRebuildAuthority, readSandbox?: ReadSandboxRebuildEntry, ): SandboxEntry { - const candidate = materializeManagedWorkloadReplacementEntry(previousEntry, plan, replacement); + const candidate = materializeManagedWorkloadReplacementEntry( + previousEntry, + plan, + replacement, + policyBoundary, + ); let result: SandboxRebuildAuthoritySwapResult; try { result = commit(plan.previousAuthority, candidate); } catch (error) { - return reconcileAmbiguousPublication(plan, replacement, candidate, error, readSandbox); + return reconcileAmbiguousPublication( + plan, + replacement, + policyBoundary, + candidate, + error, + readSandbox, + ); } if (result.status !== "committed") { throw new ManagedWorkloadRebuildTransactionError( @@ -118,6 +166,7 @@ export function commitManagedWorkloadReplacement( return reconcileAmbiguousPublication( plan, replacement, + policyBoundary, candidate, new Error("the commit adapter returned a mismatched committed entry"), readSandbox, diff --git a/src/lib/onboard/managed-workload/rebuild/contract.ts b/src/lib/onboard/managed-workload/rebuild/contract.ts index fc977453daf..8a669832cae 100644 --- a/src/lib/onboard/managed-workload/rebuild/contract.ts +++ b/src/lib/onboard/managed-workload/rebuild/contract.ts @@ -4,6 +4,7 @@ import type { SandboxRebuildAuthority } from "../../../state/registry/rebuild-authority"; import type { SandboxEntry } from "../../../state/registry/types"; import type { ManagedImageAgent } from "../../managed-image/contract"; +import type { VerifiedSandboxPolicyRegistration } from "../../types"; import type { ManagedWorkloadRebuildHandoff, ManagedWorkloadReceipt } from "../../workload/rebuild"; export type ManagedWorkloadRebuildPhase = @@ -156,6 +157,9 @@ export interface ManagedWorkloadRebuildRecoveryTask { readonly receipt: ManagedWorkloadReceipt; readonly lifecycleGeneration: string; readonly liveIdentityFingerprint: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly policyRegistration: VerifiedSandboxPolicyRegistration; }; } diff --git a/src/lib/onboard/managed-workload/rebuild/plan.ts b/src/lib/onboard/managed-workload/rebuild/plan.ts index 836d2e7d19e..b66547fba9b 100644 --- a/src/lib/onboard/managed-workload/rebuild/plan.ts +++ b/src/lib/onboard/managed-workload/rebuild/plan.ts @@ -22,6 +22,7 @@ import { ManagedWorkloadRebuildTransactionError } from "./contract"; const PROTECTED_REBUILD_METADATA_FIELDS = new Set([ "name", "pendingRouteReservation", + "pendingPolicyVerification", "reservationSessionId", "openshellDriver", "fromDockerfile", diff --git a/src/lib/onboard/managed-workload/rebuild/recovery.ts b/src/lib/onboard/managed-workload/rebuild/recovery.ts index 950597fb451..3d0eea68b53 100644 --- a/src/lib/onboard/managed-workload/rebuild/recovery.ts +++ b/src/lib/onboard/managed-workload/rebuild/recovery.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { cloneAndDeepFreeze } from "../../../core/immutable"; +import type { VerifiedSandboxPolicyBoundary } from "../../types"; import type { ManagedWorkloadRebuildPlan, ManagedWorkloadRebuildRecoveryTask, @@ -11,6 +12,7 @@ import type { export function createManagedWorkloadRebuildRecoveryTask( plan: ManagedWorkloadRebuildPlan, replacement: StagedManagedWorkloadReplacement, + policyBoundary: VerifiedSandboxPolicyBoundary, operation: ManagedWorkloadRebuildRecoveryTask["operation"], ): ManagedWorkloadRebuildRecoveryTask { return cloneAndDeepFreeze({ @@ -28,6 +30,9 @@ export function createManagedWorkloadRebuildRecoveryTask( receipt: plan.replacementReceipt, lifecycleGeneration: replacement.lifecycleGeneration, liveIdentityFingerprint: replacement.liveIdentityFingerprint, + gatewayName: policyBoundary.gatewayName, + gatewayPort: policyBoundary.gatewayPort, + policyRegistration: policyBoundary.registration, }, }); } diff --git a/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts b/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts new file mode 100644 index 00000000000..f5e73bc5dff --- /dev/null +++ b/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxPolicyAuthority } from "../../../adapters/openshell/policy-authority"; +import { inspectOpenShellSandboxIdentityFingerprint } from "../../../adapters/openshell/policy-authority"; +import { cloneAndDeepFreeze } from "../../../core/immutable"; +import type { SelectedDockerGpuRoute } from "../../docker-gpu-route"; +import { + verifyCreatedSandboxPolicyRegistration, + type CreatedSandboxPolicyRegistrationInput, +} from "../../sandbox-create/policy-creation-receipt"; +import type { VerifiedSandboxPolicyBoundary } from "../../types"; +import type { ReboundManagedWorkloadReplacement } from "./contract"; +import { ManagedWorkloadRebuildTransactionError } from "./contract"; + +export interface ManagedWorkloadReplacementPolicyAuthorityInput { + readonly gatewayName: string; + readonly gatewayPort: number; + readonly policySourcePath: string; + readonly route: SelectedDockerGpuRoute; + readonly plannedAuthority: Exclude; +} + +export interface ManagedWorkloadReplacementAuthorityDependencies { + readonly inspectSandboxIdentity?: typeof inspectOpenShellSandboxIdentityFingerprint; + readonly verifyCreatedPolicy?: typeof verifyCreatedSandboxPolicyRegistration; +} + +function verificationFailure(message: string, cause?: unknown): never { + throw new ManagedWorkloadRebuildTransactionError("registry-commit", message, { + ...(cause === undefined ? {} : { cause }), + }); +} + +/** + * Bind replacement publication to one live gateway, sandbox identity, and + * effective policy. The identity-policy-identity sequence runs immediately + * before the registry CAS and never derives authority from the previous row. + */ +export function verifyManagedWorkloadReplacementAuthority(input: { + readonly sandboxName: string; + readonly replacement: ReboundManagedWorkloadReplacement; + readonly policy: ManagedWorkloadReplacementPolicyAuthorityInput; + readonly dependencies?: ManagedWorkloadReplacementAuthorityDependencies; +}): VerifiedSandboxPolicyBoundary { + const inspectIdentity = + input.dependencies?.inspectSandboxIdentity ?? inspectOpenShellSandboxIdentityFingerprint; + const verifyPolicy = + input.dependencies?.verifyCreatedPolicy ?? verifyCreatedSandboxPolicyRegistration; + const identityInput = { + sandboxName: input.sandboxName, + gatewayName: input.policy.gatewayName, + }; + const requireExactIdentity = (timing: "before" | "after"): void => { + let observed: string; + try { + observed = inspectIdentity(identityInput); + } catch (error) { + verificationFailure( + `the replacement sandbox identity could not be verified ${timing} policy verification`, + error, + ); + } + if (observed !== input.replacement.liveIdentityFingerprint) { + verificationFailure(`the replacement sandbox identity changed ${timing} policy verification`); + } + }; + + requireExactIdentity("before"); + let registration: ReturnType; + const policyInput: CreatedSandboxPolicyRegistrationInput = { + sandboxName: input.sandboxName, + gatewayName: input.policy.gatewayName, + gatewayPort: input.policy.gatewayPort, + lifecycleGeneration: input.replacement.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: input.replacement.liveIdentityFingerprint, + policySourcePath: input.policy.policySourcePath, + route: input.policy.route, + plannedAuthority: input.policy.plannedAuthority, + operation: `publish replacement sandbox '${input.sandboxName}'`, + }; + try { + registration = verifyPolicy(policyInput); + } catch (error) { + verificationFailure("the replacement sandbox policy authority could not be verified", error); + } + requireExactIdentity("after"); + + return cloneAndDeepFreeze({ + registration, + sandboxName: input.sandboxName, + gatewayName: input.policy.gatewayName, + gatewayPort: input.policy.gatewayPort, + lifecycleGeneration: input.replacement.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: input.replacement.liveIdentityFingerprint, + route: input.policy.route, + }); +} diff --git a/src/lib/onboard/managed-workload/rebuild/transaction.ts b/src/lib/onboard/managed-workload/rebuild/transaction.ts index 96302e24755..2b9b0486c7e 100644 --- a/src/lib/onboard/managed-workload/rebuild/transaction.ts +++ b/src/lib/onboard/managed-workload/rebuild/transaction.ts @@ -28,12 +28,18 @@ import { createManagedWorkloadPreparationAbort, createManagedWorkloadReplacementRollback, } from "./rollback"; +import { + type ManagedWorkloadReplacementAuthorityDependencies, + type ManagedWorkloadReplacementPolicyAuthorityInput, + verifyManagedWorkloadReplacementAuthority, +} from "./replacement-authority"; export interface RunManagedWorkloadRebuildTransactionInput { readonly previousEntry: SandboxEntry; readonly provider: RuntimeProviderBundle; readonly handoff: ManagedWorkloadRebuildHandoff; readonly operations: ManagedWorkloadRebuildProviderOperations; + readonly replacementPolicyAuthority: ManagedWorkloadReplacementPolicyAuthorityInput; readonly replacementMetadata?: Readonly>; readonly transactionId?: string; } @@ -41,6 +47,7 @@ export interface RunManagedWorkloadRebuildTransactionInput { export interface ManagedWorkloadRebuildTransactionDependencies { readonly getSandbox?: (sandboxName: string) => SandboxEntry | null; readonly commitAuthority?: CommitSandboxRebuildAuthority; + readonly replacementAuthority?: ManagedWorkloadReplacementAuthorityDependencies; } function readSandboxFromRegistry(sandboxName: string): SandboxEntry | null { @@ -138,10 +145,17 @@ export async function runManagedWorkloadRebuildTransaction( const ready = await requireReadyManagedWorkloadReplacement(plan, staged, input.operations); const restored = await restoreStagedManagedWorkloadState(plan, ready, input.operations); const rebound = await rebindStagedManagedWorkloadProviders(plan, restored, input.operations); + const policyBoundary = verifyManagedWorkloadReplacementAuthority({ + sandboxName: plan.sandboxName, + replacement: rebound, + policy: input.replacementPolicyAuthority, + dependencies: dependencies.replacementAuthority, + }); const entry = commitManagedWorkloadReplacement( input.previousEntry, plan, rebound, + policyBoundary, dependencies.commitAuthority, readSandbox, ); @@ -154,7 +168,12 @@ export async function runManagedWorkloadRebuildTransaction( entry, previousCleanup: "pending", cleanupError, - recoveryTask: createManagedWorkloadRebuildRecoveryTask(plan, rebound, "retire-previous"), + recoveryTask: createManagedWorkloadRebuildRecoveryTask( + plan, + rebound, + policyBoundary, + "retire-previous", + ), }; } } catch (error) { diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 51631b86325..2d49d608315 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -16,25 +16,9 @@ import { } from "./orchestration"; describe("deferred provider effect authority", () => { - it("refuses a second provider attachment after policy authority changes (#9833)", async () => { - const events: string[] = []; - const recordPolicyCheck = (operation: string) => { - events.push(`policy: ${operation}`); - }; - const revalidatePolicyRequirements = vi.fn(recordPolicyCheck); - const runOpenshell = vi.fn((args: string[]) => { - events.push(args.join(" ")); - revalidatePolicyRequirements - .mockImplementationOnce(recordPolicyCheck) - .mockImplementationOnce((operation) => { - recordPolicyCheck(operation); - throw new Error("policy authority changed after the first provider attachment"); - }); - return { status: 0 }; - }); - const revalidateSandboxIdentity = vi.fn((_exactIdentity: string, operation: string) => { - events.push(`identity: ${operation}`); - }); + it("refuses every deferred provider attachment before a same-name replacement can receive credentials (#9833)", async () => { + const revalidatePolicyRequirements = vi.fn(); + const runOpenshell = vi.fn(() => ({ status: 0 })); const boundary = createProviderEffectBoundary({ deferred: true, sandboxName: "alpha", @@ -55,8 +39,6 @@ describe("deferred provider effect authority", () => { runVerifiedSandboxCreateEffects: null, activateDeferredProviderEffects: () => ["first", "second"], revalidatePolicyAuthorityBeforeCreate: vi.fn(), - runOpenshell: runOpenshell as never, - revalidateSandboxIdentity, }); const runAfterVerifiedCreate = boundary.runAfterVerifiedCreate; expect(runAfterVerifiedCreate).toBeTypeOf("function"); @@ -77,18 +59,15 @@ describe("deferred provider effect authority", () => { route: "direct" as never, revalidatePolicyRequirements, }), - ).rejects.toThrow("policy authority changed after the first provider attachment"); + ).rejects.toThrow("OpenShell cannot attach providers to the immutable identity"); - expect(runOpenshell).toHaveBeenCalledExactlyOnceWith( - ["sandbox", "provider", "attach", "-g", "nemoclaw", "alpha", "first"], - { ignoreError: true, suppressOutput: true }, + expect(runOpenshell).not.toHaveBeenCalledWith( + expect.arrayContaining(["sandbox", "provider", "attach"]), + expect.anything(), ); - expect(revalidateSandboxIdentity).toHaveBeenCalledWith( - "a".repeat(64), - "attaching provider 'second' to sandbox 'alpha'", + expect(revalidatePolicyRequirements).toHaveBeenCalledWith( + "attaching deferred providers to sandbox 'alpha'", ); - expect(events).toContain("policy: attaching provider 'second' to sandbox 'alpha'"); - expect(events).not.toContain("sandbox provider attach -g nemoclaw alpha second"); }); }); @@ -491,6 +470,75 @@ describe("sandbox create policy authority checks", () => { expect(sandboxIdentity).toBe("replacement"); }); + it("retains the durable checkpoint when identity-bound provider attachment is unavailable (#9833)", async () => { + const runOpenshell = vi.fn(() => ({ status: 0 })); + const checkpoint = { state: "absent" }; + const providerBoundary = createProviderEffectBoundary({ + deferred: true, + sandboxName: "alpha", + gatewayName: "nemoclaw", + preparationInput: { + openshellDriver: "kubernetes", + inferenceProvider: null, + messagingProviders: [], + messagingProviderRequests: [], + extraProviders: [], + gatewayName: "nemoclaw", + }, + preparationDeps: { + providerExistsInGateway: vi.fn(() => true), + runOpenshell: runOpenshell as never, + cleanupCreateSources: vi.fn(), + }, + runVerifiedSandboxCreateEffects: null, + activateDeferredProviderEffects: () => ["credential-provider"], + revalidatePolicyAuthorityBeforeCreate: vi.fn(), + }); + const error = await runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + ...exactIdentityBoundary(), + persistVerifiedPolicy: () => { + checkpoint.state = "verified-create"; + }, + runVerifiedCreateEffects: async () => { + await providerBoundary.runAfterVerifiedCreate?.({ + registration: { + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "alpha", + lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + sandboxIdentityFingerprint: exactIdentity, + policyHash: "policy-alpha", + policyVersion: 1, + }, + observedPolicyAuthority: "owner-unknown", + }, + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + lifecycleLiveIdentityFingerprint: exactIdentity, + route: "none", + revalidatePolicyRequirements: vi.fn(), + }); + }, + cleanupTemporarySources: vi.fn(), + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(AggregateError); + expect(checkpoint.state).toBe("verified-create"); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + it("reports temporary source cleanup failure with sandbox preservation (#9833)", async () => { const revalidate = vi.fn(); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 0d44010322c..07b6e35655c 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -41,6 +41,16 @@ import { export const createOnboardPolicyAuthorityBindings = policyAuthorityPreflight.createOnboardPolicyAuthorityBindings; +function cancelRecoveryIdentity( + liveExists: boolean, + requireVerifiedPolicyGate: () => VerifiedSandboxPolicyBoundary, +): { readonly lifecycleLiveIdentityFingerprint?: string } { + if (liveExists) return {}; + return { + lifecycleLiveIdentityFingerprint: requireVerifiedPolicyGate().lifecycleLiveIdentityFingerprint, + }; +} + type SandboxRecreateReasonInput = { sandboxName: string; recreateForAgentDrift: boolean; @@ -404,8 +414,6 @@ export function createProviderEffectBoundary(input: { readonly runVerifiedSandboxCreateEffects: import("../types").VerifiedSandboxCreateEffects | null; readonly activateDeferredProviderEffects: (() => readonly string[]) | null; readonly revalidatePolicyAuthorityBeforeCreate: () => void; - readonly runOpenshell: SandboxCreateOrchestrationRuntime["runOpenshell"]; - readonly revalidateSandboxIdentity: (exactIdentity: string, operation: string) => void; }): ProviderEffectBoundary { const validate = () => validateAttachedMessagingProvidersBeforeSandboxCreation( @@ -447,20 +455,11 @@ export function createProviderEffectBoundary(input: { context.revalidatePolicyRequirements( `attaching deferred providers to sandbox '${input.sandboxName}'`, ); - attachProvidersAfterSandboxCreation( - { - sandboxName: input.sandboxName, - gatewayName: input.gatewayName, - providerNames, - }, - { - runOpenshell: input.runOpenshell, - revalidateSandboxIdentity: (operation) => { - input.revalidateSandboxIdentity(context.lifecycleLiveIdentityFingerprint, operation); - context.revalidatePolicyRequirements(operation); - }, - }, - ); + attachProvidersAfterSandboxCreation({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + providerNames, + }); }, }; } @@ -1865,16 +1864,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche false, `publishing providers before creating sandbox gateway '${GATEWAY_NAME}'`, ), - runOpenshell, - revalidateSandboxIdentity: (exactIdentity, operation) => - sandboxRecreateTransaction.revalidateCreatedSandboxLifecycleRegistration( - { sandboxName, gatewayName: GATEWAY_NAME }, - { - lifecycleGeneration: createdSandboxLifecycle.generation, - lifecycleLiveIdentityFingerprint: exactIdentity, - }, - getSandboxRecreateObservation, - ), }); providerEffectBoundary.validateBeforeCreate(); @@ -1979,6 +1968,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche runtimeFields: sandboxRuntimeFields, messagingProviders, liveExists, + ...cancelRecoveryIdentity(liveExists, requireVerifiedPolicyGate), }, { setDefault: registry.setDefault, diff --git a/src/lib/onboard/sandbox-create/provider-publication.test.ts b/src/lib/onboard/sandbox-create/provider-publication.test.ts index b7dbdd0aa10..d194600dfbc 100644 --- a/src/lib/onboard/sandbox-create/provider-publication.test.ts +++ b/src/lib/onboard/sandbox-create/provider-publication.test.ts @@ -117,56 +117,24 @@ function prepareProviders( } describe("sandbox provider preparation", () => { - it("identity-fences every deferred provider attachment (#9833)", () => { - const events: string[] = []; - const runOpenshell = vi.fn((args: string[]) => { - events.push(args.join(" ")); - return { status: 0 }; - }); - - attachProvidersAfterSandboxCreation( - { + it("refuses name-addressed deferred provider attachment before mutation (#9833)", () => { + expect(() => + attachProvidersAfterSandboxCreation({ sandboxName: "alpha", gatewayName: "nemoclaw", providerNames: ["inference", "alpha-telegram"], - }, - { - runOpenshell: runOpenshell as never, - revalidateSandboxIdentity: (operation) => events.push(operation), - }, - ); - - expect(events).toEqual([ - "attaching provider 'inference' to sandbox 'alpha'", - "sandbox provider attach -g nemoclaw alpha inference", - "confirming provider 'inference' on sandbox 'alpha'", - "attaching provider 'alpha-telegram' to sandbox 'alpha'", - "sandbox provider attach -g nemoclaw alpha alpha-telegram", - "confirming provider 'alpha-telegram' on sandbox 'alpha'", - ]); + }), + ).toThrow("OpenShell cannot attach providers to the immutable identity of sandbox 'alpha'"); }); - it("redacts deferred attachment command output on failure (#9833)", () => { - const revalidateSandboxIdentity = vi.fn(); - + it("allows an empty deferred attachment set without a mutable-name operation (#9833)", () => { expect(() => - attachProvidersAfterSandboxCreation( - { - sandboxName: "alpha", - gatewayName: "nemoclaw", - providerNames: ["alpha-telegram"], - }, - { - runOpenshell: vi.fn(() => ({ - status: 1, - stdout: "secret-stdout", - stderr: "secret-stderr", - })) as never, - revalidateSandboxIdentity, - }, - ), - ).toThrow("OpenShell did not attach provider 'alpha-telegram' to the verified sandbox."); - expect(revalidateSandboxIdentity).toHaveBeenCalledOnce(); + attachProvidersAfterSandboxCreation({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + providerNames: [], + }), + ).not.toThrow(); }); it("confirms an exact messaging binding before and after publication (#9875)", () => { diff --git a/src/lib/onboard/sandbox-create/provider-publication.ts b/src/lib/onboard/sandbox-create/provider-publication.ts index e50249c13d3..bf1d203d22a 100644 --- a/src/lib/onboard/sandbox-create/provider-publication.ts +++ b/src/lib/onboard/sandbox-create/provider-publication.ts @@ -33,10 +33,6 @@ type DeferredProviderAttachmentInput = { readonly providerNames: readonly string[]; }; -type DeferredProviderAttachmentDeps = Pick & { - readonly revalidateSandboxIdentity: (operation: string) => void; -}; - function expectedMessagingBindings(input: ProviderPreparationInput) { return new Map( input.messagingProviderRequests @@ -149,25 +145,10 @@ export function publishAttachedProvidersBeforeDockerSandboxCreation( } /** Attach the planned providers only after the created sandbox passed its exact policy gate. */ -export function attachProvidersAfterSandboxCreation( - input: DeferredProviderAttachmentInput, - deps: DeferredProviderAttachmentDeps, -): void { - for (const providerName of input.providerNames) { - deps.revalidateSandboxIdentity( - `attaching provider '${providerName}' to sandbox '${input.sandboxName}'`, - ); - const attached = deps.runOpenshell( - ["sandbox", "provider", "attach", "-g", input.gatewayName, input.sandboxName, providerName], - { ignoreError: true, suppressOutput: true }, - ); - if (attached.status !== 0) { - throw new Error( - `OpenShell did not attach provider '${providerName}' to the verified sandbox.`, - ); - } - deps.revalidateSandboxIdentity( - `confirming provider '${providerName}' on sandbox '${input.sandboxName}'`, - ); - } +export function attachProvidersAfterSandboxCreation(input: DeferredProviderAttachmentInput): void { + if (input.providerNames.length === 0) return; + throw new Error( + `OpenShell cannot attach providers to the immutable identity of sandbox '${input.sandboxName}'. ` + + `The sandbox remains incomplete on gateway '${input.gatewayName}'; preserve its verified create checkpoint for administrator recovery.`, + ); } diff --git a/src/lib/state/registry-rebuild-authority.test.ts b/src/lib/state/registry-rebuild-authority.test.ts index 1c4c78d3204..7012c36b713 100644 --- a/src/lib/state/registry-rebuild-authority.test.ts +++ b/src/lib/state/registry-rebuild-authority.test.ts @@ -29,6 +29,8 @@ vi.mock("./registry/lock", () => ({ const ENCODED_PROFILE = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); const PROFILE_SHA256 = createHash("sha256").update(ENCODED_PROFILE, "utf8").digest("hex"); +const REPLACEMENT_GENERATION = "00000000-0000-4000-8000-000000000002"; +const REPLACEMENT_FINGERPRINT = "b".repeat(64); function receipt(digest: string): Extract { return { @@ -82,9 +84,21 @@ function registry(current: SandboxEntry = entry()): SandboxRegistry { function replacement(): SandboxEntry { const workload = receipt("b"); return { - ...entry("generation-new", "fingerprint-new"), + ...entry(REPLACEMENT_GENERATION, REPLACEMENT_FINGERPRINT), imageTag: workload.reference, workload, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "alpha", + lifecycleGeneration: REPLACEMENT_GENERATION, + sandboxIdentityFingerprint: REPLACEMENT_FINGERPRINT, + policyHash: "replacement-policy", + policyVersion: 2, + }, }; } @@ -149,8 +163,8 @@ describe("sandbox rebuild authority", () => { expect(swapped.result).toMatchObject({ status: "committed", entry: { - lifecycleGeneration: "generation-new", - lifecycleLiveIdentityFingerprint: "fingerprint-new", + lifecycleGeneration: REPLACEMENT_GENERATION, + lifecycleLiveIdentityFingerprint: REPLACEMENT_FINGERPRINT, workload: { reference: receipt("b").reference }, }, }); @@ -202,8 +216,8 @@ describe("sandbox rebuild authority", () => { expect(result).toMatchObject({ status: "committed", entry: { - lifecycleGeneration: "generation-new", - lifecycleLiveIdentityFingerprint: "fingerprint-new", + lifecycleGeneration: REPLACEMENT_GENERATION, + lifecycleLiveIdentityFingerprint: REPLACEMENT_FINGERPRINT, }, }); expect(registryPersistence.load).toHaveBeenCalledTimes(2); @@ -243,9 +257,14 @@ describe("sandbox rebuild authority", () => { sandboxRebuildReplacementMatchesEntry(expected, { ...expected, model: "updated-after-publication", - gatewayPort: 9090, }), ).toBe(true); + expect( + sandboxRebuildReplacementMatchesEntry(expected, { + ...expected, + gatewayPort: 9090, + }), + ).toBe(false); expect( sandboxRebuildReplacementMatchesEntry(expected, { ...expected, @@ -279,6 +298,24 @@ describe("sandbox rebuild authority", () => { imageTag: receipt("a").reference, }), ], + ["gateway", (candidate: SandboxEntry) => ({ ...candidate, gatewayPort: 9090 })], + [ + "policy authority", + (candidate: SandboxEntry) => ({ + ...candidate, + policyAuthority: "externally-managed" as const, + }), + ], + [ + "policy receipt", + (candidate: SandboxEntry) => ({ + ...candidate, + policyCreationReceipt: { + ...candidate.policyCreationReceipt!, + sandboxIdentityFingerprint: "c".repeat(64), + }, + }), + ], ] as const)("rejects replacement %s drift before CAS", (_label, mutate) => { const before = registry(); const authority = captureSandboxRebuildAuthority(before.sandboxes.alpha!, "docker"); diff --git a/src/lib/state/registry/rebuild-authority.ts b/src/lib/state/registry/rebuild-authority.ts index 5a63f903e76..d1124abbf51 100644 --- a/src/lib/state/registry/rebuild-authority.ts +++ b/src/lib/state/registry/rebuild-authority.ts @@ -12,6 +12,7 @@ import { import { withLock } from "./lock"; import { load, save } from "./persistence"; import type { SandboxEntry, SandboxRegistry, SandboxWorkloadReceipt } from "./types"; +import { cloneSandboxPolicyCreationReceipt } from "../registry-normalization"; import { cloneSandboxWorkloadReceipt } from "./workload"; type ManagedWorkloadReceipt = Extract; @@ -157,10 +158,8 @@ export function captureSandboxRebuildAuthority( /** * Reconcile an ambiguous persistence result using the replacement's exact new - * runtime identity. This deliberately ignores mutable non-authority fields: - * after publication another writer may update those fields, but no different - * rebuild may claim the same generation, live fingerprint, provider, and - * workload receipt. + * runtime, gateway, and policy identity. Mutable metadata may change after + * publication, but replacement authority itself must remain exact. */ export function sandboxRebuildReplacementMatchesEntry( replacement: SandboxEntry, @@ -172,6 +171,10 @@ export function sandboxRebuildReplacementMatchesEntry( (entry.openshellDriver ?? null) === (replacement.openshellDriver ?? null) && entry.lifecycleGeneration === replacement.lifecycleGeneration && entry.lifecycleLiveIdentityFingerprint === replacement.lifecycleLiveIdentityFingerprint && + entry.gatewayName === replacement.gatewayName && + entry.gatewayPort === replacement.gatewayPort && + entry.policyAuthority === replacement.policyAuthority && + isDeepStrictEqual(entry.policyCreationReceipt, replacement.policyCreationReceipt) && entry.imageTag === replacement.imageTag && entry.agent === replacement.agent && isDeepStrictEqual( @@ -224,6 +227,43 @@ function validateReplacement( "replacement must have a distinct live identity fingerprint", ); } + if ( + !boundedIdentity(replacement.gatewayName) || + !Number.isSafeInteger(replacement.gatewayPort) || + Number(replacement.gatewayPort) < 1 || + Number(replacement.gatewayPort) > 65_535 + ) { + throw new SandboxRebuildAuthorityError("replacement gateway authority is missing or invalid"); + } + if (replacement.policyAuthority === "nemoclaw-managed") { + let receipt: ReturnType; + try { + receipt = cloneSandboxPolicyCreationReceipt(replacement.policyCreationReceipt); + } catch (error) { + throw new SandboxRebuildAuthorityError( + `replacement policy creation receipt is invalid: ${error instanceof Error ? error.message : "unknown error"}`, + ); + } + if ( + !receipt || + receipt.sandboxName !== replacement.name || + receipt.gatewayName !== replacement.gatewayName || + receipt.gatewayPort !== replacement.gatewayPort || + receipt.lifecycleGeneration !== replacement.lifecycleGeneration || + receipt.sandboxIdentityFingerprint !== replacement.lifecycleLiveIdentityFingerprint + ) { + throw new SandboxRebuildAuthorityError( + "replacement policy receipt does not match its gateway and sandbox identity", + ); + } + } else if ( + replacement.policyAuthority !== "externally-managed" || + replacement.policyCreationReceipt !== undefined + ) { + throw new SandboxRebuildAuthorityError( + "replacement policy authority is missing or inconsistent", + ); + } const workload = clonedManagedReceipt(replacement.workload); requireReceiptAgent(replacement.agent, workload, "replacement"); if (replacement.imageTag !== workload.reference) { diff --git a/test/runtime/policy/managed-policy-receipt-fixture.ts b/test/runtime/policy/managed-policy-receipt-fixture.ts deleted file mode 100644 index 11e9ed20c4b..00000000000 --- a/test/runtime/policy/managed-policy-receipt-fixture.ts +++ /dev/null @@ -1,4 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -export * from "../../helpers/managed-policy-receipt-fixture"; diff --git a/test/runtime/policy/policies-permissive-policy.test.ts b/test/runtime/policy/policies-permissive-policy.test.ts index 42a759660f0..138ca28ee76 100644 --- a/test/runtime/policy/policies-permissive-policy.test.ts +++ b/test/runtime/policy/policies-permissive-policy.test.ts @@ -13,7 +13,7 @@ import { managedPolicyMetadata, managedRegistrationSource, SANDBOX_ID, -} from "./managed-policy-receipt-fixture"; +} from "../../helpers/managed-policy-receipt-fixture"; const REPO_ROOT = path.join(import.meta.dirname, "../../.."); const POLICIES_PATH = JSON.stringify(path.join(REPO_ROOT, "src", "lib", "policy", "index.ts")); diff --git a/test/runtime/policy/policies.test.ts b/test/runtime/policy/policies.test.ts index 6fca5c4b412..9f59bc7ec9e 100644 --- a/test/runtime/policy/policies.test.ts +++ b/test/runtime/policy/policies.test.ts @@ -17,7 +17,7 @@ import { POLICY_VERSION, SANDBOX_ID, SANDBOX_IDENTITY, -} from "./managed-policy-receipt-fixture"; +} from "../../helpers/managed-policy-receipt-fixture"; const requireForTest = createRequire(import.meta.url); const YAML = requireForTest("yaml"); diff --git a/test/runtime/policy/policy-explain-cli.test.ts b/test/runtime/policy/policy-explain-cli.test.ts index aac129948bc..7c55c148850 100644 --- a/test/runtime/policy/policy-explain-cli.test.ts +++ b/test/runtime/policy/policy-explain-cli.test.ts @@ -12,7 +12,7 @@ import { POLICY_HASH, POLICY_VERSION, SANDBOX_ID, -} from "./managed-policy-receipt-fixture"; +} from "../../helpers/managed-policy-receipt-fixture"; const CLI = path.join(import.meta.dirname, "../../..", "bin", "nemoclaw.js"); diff --git a/test/runtime/policy/policy-mutation-read-failure.test.ts b/test/runtime/policy/policy-mutation-read-failure.test.ts index 59b2b8004f3..81560f7761a 100644 --- a/test/runtime/policy/policy-mutation-read-failure.test.ts +++ b/test/runtime/policy/policy-mutation-read-failure.test.ts @@ -12,7 +12,7 @@ import { POLICY_HASH, POLICY_VERSION, SANDBOX_IDENTITY, -} from "./managed-policy-receipt-fixture"; +} from "../../helpers/managed-policy-receipt-fixture"; const requireForTest = createRequire(import.meta.url); const policies = requireForTest( From f9286b66e343c7fc3c54981e3fe377c741cd454b Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 03:48:47 -0700 Subject: [PATCH 02/42] test(onboard): remove disconnected rollback assertions Signed-off-by: Apurv Kumaria --- src/lib/onboard/cancel-rollback.test.ts | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index c8deaa32896..6c25f665093 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -48,23 +48,6 @@ describe("createSandboxCancelRollback", () => { expect(guidance).toContain("preserve the registry and onboarding recovery state"); }); - it("keeps the captured identity when the mutable name is replaced before exit (#9833)", () => { - const { rollback, log } = createHarness(); - const replacementFingerprint = "b".repeat(64); - - rollback.arm("new-sb", SANDBOX_FINGERPRINT); - // A replacement can take the same mutable name, but cannot alter the identity - // captured by the completed create boundary. - const sameNameReplacement = { name: "new-sb", fingerprint: replacementFingerprint }; - expect(sameNameReplacement.fingerprint).not.toBe(SANDBOX_FINGERPRINT); - rollback.markCancelled(); - rollback.runIfArmed(); - - const guidance = log.mock.calls.flat().join("\n"); - expect(guidance).toContain(SANDBOX_FINGERPRINT); - expect(guidance).not.toContain(replacementFingerprint); - }); - it("does not run on a non-cancel exit", () => { const { rollback, log } = createHarness(); @@ -126,9 +109,6 @@ describe("createSandboxCancelRollback", () => { describe("installSandboxCancelRollback", () => { it("registers a non-destructive exit handler that retains external recovery state (#9833)", () => { - const runOpenshell = vi.fn(); - const removeSandbox = vi.fn(); - const clearOnboardSession = vi.fn(); const log = vi.fn(); const exitHandlers: Array<() => void> = []; const rollback = installSandboxCancelRollback({ @@ -141,14 +121,10 @@ describe("installSandboxCancelRollback", () => { rollback.markCancelled(); exitHandlers[0](); - expect(runOpenshell).not.toHaveBeenCalled(); - expect(removeSandbox).not.toHaveBeenCalled(); - expect(clearOnboardSession).not.toHaveBeenCalled(); expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); }); it("preserves missing-checkpoint recovery state without a mutable-name fallback (#9833)", () => { - const runOpenshell = vi.fn(); const log = vi.fn(); const exitHandlers: Array<() => void> = []; const rollback = installSandboxCancelRollback({ @@ -160,7 +136,6 @@ describe("installSandboxCancelRollback", () => { rollback.markCancelled(); exitHandlers[0](); - expect(runOpenshell).not.toHaveBeenCalled(); const guidance = log.mock.calls.flat().join("\n"); expect(guidance).toContain("identity fingerprint is unavailable"); expect(guidance).toContain("OpenShell administrator"); From 9bf87dabb770b1fca2a22d41952dc041ab84a638 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 04:17:49 -0700 Subject: [PATCH 03/42] fix(security): serialize rebuild authority publication Signed-off-by: Apurv Kumaria --- ...naged-workload-rebuild-transaction.test.ts | 148 ++++++++++++++++++ .../rebuild/replacement-authority.ts | 3 +- .../managed-workload/rebuild/transaction.ts | 89 +++++++++-- 3 files changed, 225 insertions(+), 15 deletions(-) diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index 5fd983dd25c..75c0288e435 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -55,6 +55,30 @@ const NEW_GENERATION = "00000000-0000-4000-8000-000000000002"; const OLD_FINGERPRINT = "a".repeat(64); const NEW_FINGERPRINT = "b".repeat(64); +type SandboxMutationLock = NonNullable< + ManagedWorkloadRebuildTransactionDependencies["withSandboxMutationLock"] +>; + +const immediateSandboxMutationLock: SandboxMutationLock = async (_sandboxName, operation) => + await operation(); + +function serializedSandboxMutationLock(): SandboxMutationLock { + let tail = Promise.resolve(); + return async (_sandboxName, operation) => { + const previous = tail; + let release = (): void => {}; + tail = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await operation(); + } finally { + release(); + } + }; +} + const replacementPolicyAuthority = { gatewayName: "nemoclaw", gatewayPort: 8080, @@ -398,6 +422,7 @@ function transactionHarness( readonly dependencies?: NonNullable< ManagedWorkloadRebuildTransactionDependencies["replacementAuthority"] >; + readonly withSandboxMutationLock?: SandboxMutationLock | null; } = {}, ) { const events: string[] = []; @@ -483,6 +508,10 @@ function transactionHarness( commitAuthority, replacementAuthority: replacementOptions.dependencies ?? replacementAuthorityDependencies(), + withSandboxMutationLock: + replacementOptions.withSandboxMutationLock === null + ? undefined + : (replacementOptions.withSandboxMutationLock ?? immediateSandboxMutationLock), }, ), }; @@ -883,6 +912,124 @@ describe("managed workload rebuild transaction", () => { expect(harness.events).not.toContain("registry-commit"); }); + it("serializes a same-name replacement requested after final identity verification (#9833)", async () => { + const withSandboxMutationLock = serializedSandboxMutationLock(); + let liveIdentity = NEW_FINGERPRINT; + let sameNameReplacement = Promise.resolve(); + let harness!: ReturnType; + const inspectSandboxIdentity = vi + .fn() + .mockImplementationOnce(() => { + harness.events.push("identity-read:1"); + return liveIdentity; + }) + .mockImplementationOnce(() => { + const observed = liveIdentity; + harness.events.push("identity-read:2"); + harness.events.push("same-name-replacement-requested"); + sameNameReplacement = withSandboxMutationLock("rebuild-openclaw", () => { + liveIdentity = "c".repeat(64); + harness.events.push("same-name-replacement-ran"); + }); + return observed; + }); + harness = transactionHarness( + "openclaw", + "mxc", + null, + "linux/amd64", + {}, + { + dependencies: replacementAuthorityDependencies({ inspectSandboxIdentity }), + withSandboxMutationLock, + }, + ); + + const result = await harness.run(); + await sameNameReplacement; + + expect(result.status).toBe("committed"); + expect(inspectSandboxIdentity).toHaveBeenCalledTimes(2); + expect(harness.events.indexOf("registry-commit")).toBeLessThan( + harness.events.indexOf("same-name-replacement-ran"), + ); + expect(harness.events).toContain("same-name-replacement-requested"); + expect(liveIdentity).toBe("c".repeat(64)); + }); + + it("keeps old authority when the replacement publication lock fails (#9833)", async () => { + const harness = transactionHarness( + "openclaw", + "mxc", + null, + "linux/amd64", + {}, + { + withSandboxMutationLock: async () => { + throw new Error("sandbox mutation lock unavailable"); + }, + }, + ); + + await expect(harness.run()).rejects.toMatchObject({ + phase: "registry-commit", + message: expect.stringContaining("sandbox mutation lock could not protect"), + }); + + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.operations.rollback).toHaveBeenCalledOnce(); + expect(harness.events).not.toContain("registry-commit"); + expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); + }); + + it("keeps old authority when replacement publication has no sandbox lock (#9833)", async () => { + const harness = transactionHarness( + "openclaw", + "mxc", + null, + "linux/amd64", + {}, + { + withSandboxMutationLock: null, + }, + ); + + await expect(harness.run()).rejects.toMatchObject({ + phase: "registry-commit", + message: expect.stringContaining("requires the sandbox mutation lock"), + }); + + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.operations.rollback).toHaveBeenCalledOnce(); + expect(harness.events).not.toContain("registry-commit"); + expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); + }); + + it("does not roll back publication when the sandbox lock release fails (#9833)", async () => { + const harness = transactionHarness( + "openclaw", + "mxc", + null, + "linux/amd64", + {}, + { + withSandboxMutationLock: async (_sandboxName, operation) => { + await operation(); + throw new Error("sandbox mutation lock release failed"); + }, + }, + ); + + await expect(harness.run()).rejects.toMatchObject({ + name: "ManagedWorkloadRebuildIndeterminatePublicationError", + recoveryTask: { operation: "reconcile-publication" }, + }); + + expect(harness.currentEntry().lifecycleGeneration).toBe(NEW_GENERATION); + expect(harness.operations.rollback).not.toHaveBeenCalled(); + expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); + }); + it("rolls back a not-ready replacement by exact staged handle", async () => { const harness = transactionHarness("hermes", "docker", "readiness"); @@ -1231,6 +1378,7 @@ describe("managed workload rebuild transaction", () => { entry: structuredClone(replacement), }), replacementAuthority: replacementAuthorityDependencies(), + withSandboxMutationLock: immediateSandboxMutationLock, }, ); diff --git a/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts b/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts index f5e73bc5dff..434a6246b17 100644 --- a/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts +++ b/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts @@ -4,7 +4,6 @@ import type { SandboxPolicyAuthority } from "../../../adapters/openshell/policy-authority"; import { inspectOpenShellSandboxIdentityFingerprint } from "../../../adapters/openshell/policy-authority"; import { cloneAndDeepFreeze } from "../../../core/immutable"; -import type { SelectedDockerGpuRoute } from "../../docker-gpu-route"; import { verifyCreatedSandboxPolicyRegistration, type CreatedSandboxPolicyRegistrationInput, @@ -17,7 +16,7 @@ export interface ManagedWorkloadReplacementPolicyAuthorityInput { readonly gatewayName: string; readonly gatewayPort: number; readonly policySourcePath: string; - readonly route: SelectedDockerGpuRoute; + readonly route: CreatedSandboxPolicyRegistrationInput["route"]; readonly plannedAuthority: Exclude; } diff --git a/src/lib/onboard/managed-workload/rebuild/transaction.ts b/src/lib/onboard/managed-workload/rebuild/transaction.ts index 2b9b0486c7e..3151b48ff98 100644 --- a/src/lib/onboard/managed-workload/rebuild/transaction.ts +++ b/src/lib/onboard/managed-workload/rebuild/transaction.ts @@ -48,6 +48,10 @@ export interface ManagedWorkloadRebuildTransactionDependencies { readonly getSandbox?: (sandboxName: string) => SandboxEntry | null; readonly commitAuthority?: CommitSandboxRebuildAuthority; readonly replacementAuthority?: ManagedWorkloadReplacementAuthorityDependencies; + readonly withSandboxMutationLock?: ( + sandboxName: string, + operation: () => Promise | T, + ) => Promise; } function readSandboxFromRegistry(sandboxName: string): SandboxEntry | null { @@ -77,14 +81,80 @@ async function failAfterCleanup(error: unknown, cleanup: () => Promise): P throw rethrowWithRollback(error, rollbackError); } +async function rejectUnprotectedReplacementPublication(): Promise { + throw new ManagedWorkloadRebuildTransactionError( + "registry-commit", + "replacement publication requires the sandbox mutation lock", + ); +} + +async function publishManagedWorkloadReplacement( + input: RunManagedWorkloadRebuildTransactionInput, + plan: ReturnType, + replacement: Parameters[0]["replacement"], + dependencies: ManagedWorkloadRebuildTransactionDependencies, + readSandbox: (sandboxName: string) => SandboxEntry | null, +): Promise<{ + readonly entry: SandboxEntry; + readonly policyBoundary: ReturnType; +}> { + const withMutationLock = + dependencies.withSandboxMutationLock ?? rejectUnprotectedReplacementPublication; + let published: + | { + readonly entry: SandboxEntry; + readonly policyBoundary: ReturnType; + } + | undefined; + try { + return await withMutationLock(plan.sandboxName, () => { + const policyBoundary = verifyManagedWorkloadReplacementAuthority({ + sandboxName: plan.sandboxName, + replacement, + policy: input.replacementPolicyAuthority, + dependencies: dependencies.replacementAuthority, + }); + const entry = commitManagedWorkloadReplacement( + input.previousEntry, + plan, + replacement, + policyBoundary, + dependencies.commitAuthority, + readSandbox, + ); + published = { entry, policyBoundary }; + return published; + }); + } catch (error) { + if (error instanceof ManagedWorkloadRebuildTransactionError) throw error; + if (published) { + throw new ManagedWorkloadRebuildIndeterminatePublicationError( + "replacement publication completed but sandbox mutation lock release failed", + createManagedWorkloadRebuildRecoveryTask( + plan, + replacement, + published.policyBoundary, + "reconcile-publication", + ), + { cause: error }, + ); + } + throw new ManagedWorkloadRebuildTransactionError( + "registry-commit", + "the sandbox mutation lock could not protect replacement publication", + { cause: error }, + ); + } +} + /** * Execute a dormant, provider-neutral managed rebuild transaction. * * The durable row and provider-owned old runtime remain authoritative through * prepare, create, readiness, state restore, and provider rebind. Only the - * exact final CAS publishes the replacement. The exact old runtime handle is - * retired afterward, so no failure can turn a same-name lookup into deletion - * authority. + * exact final CAS publishes the replacement. The final live verification and + * CAS share the sandbox mutation lock. The exact old runtime handle is retired + * afterward, so no failure can turn a same-name lookup into deletion authority. */ export async function runManagedWorkloadRebuildTransaction( input: RunManagedWorkloadRebuildTransactionInput, @@ -145,18 +215,11 @@ export async function runManagedWorkloadRebuildTransaction( const ready = await requireReadyManagedWorkloadReplacement(plan, staged, input.operations); const restored = await restoreStagedManagedWorkloadState(plan, ready, input.operations); const rebound = await rebindStagedManagedWorkloadProviders(plan, restored, input.operations); - const policyBoundary = verifyManagedWorkloadReplacementAuthority({ - sandboxName: plan.sandboxName, - replacement: rebound, - policy: input.replacementPolicyAuthority, - dependencies: dependencies.replacementAuthority, - }); - const entry = commitManagedWorkloadReplacement( - input.previousEntry, + const { entry, policyBoundary } = await publishManagedWorkloadReplacement( + input, plan, rebound, - policyBoundary, - dependencies.commitAuthority, + dependencies, readSandbox, ); try { From ba6a21fa3e39274bd8677aeafe0fc680b6e0d02e Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 04:32:56 -0700 Subject: [PATCH 04/42] fix(onboard): reject unsafe deferred provider plans Signed-off-by: Apurv Kumaria --- .../sandbox-create-plan-materialization.ts | 20 +++++ src/lib/onboard/sandbox-create-plan.test.ts | 75 +++++++++---------- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index 3b59c81f988..4f640dc8d2e 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -261,6 +261,23 @@ function buildCreateProviderSet( ); } +function assertDeferredProviderPlanSupported( + intent: SandboxCreateIntent, + messagingProviders: readonly string[], + initialSandboxPolicy: InitialSandboxPolicy, +): void { + const requiresProviderAttachment = + Boolean(intent.inferenceProvider) || + messagingProviders.length > 0 || + intent.extraProviders.length > 0 || + intent.hermesToolGateways.length > 0; + if (!requiresProviderAttachment) return; + initialSandboxPolicy.cleanup?.(); + throw new Error( + `Cannot create sandbox '${intent.sandboxName}' with deferred providers because OpenShell cannot bind provider attachment to a verified immutable sandbox identity. No sandbox was created; use an OpenShell release with identity-bound provider attachment before retrying.`, + ); +} + /** Materialize policy, route metadata, resources, and providers from a secretless intent. */ export function materializeSandboxCreatePlan({ intent, @@ -327,6 +344,9 @@ export function materializeSandboxCreatePlan({ intent.policy.activeMessagingChannels, intent.disabledChannelNames, ); + if (deferSandboxEffectsUntilPolicyVerification) { + assertDeferredProviderPlanSupported(intent, plannedMessagingProviders, initialSandboxPolicy); + } if (policyAuthority === "nemoclaw-managed") { assertCredentialBindingProvidersAttached( initialSandboxPolicy, diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 1ed1c5c62d9..720c8301460 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -588,7 +588,7 @@ describe("resolveSandboxCreateIntent", () => { expect(JSON.stringify(intent)).toBe(serializedIntent); }); - it("defers every provider effect and create attachment until activation (#9833)", () => { + it("rejects deferred provider plans before provider effects or sandbox creation (#9833)", () => { const tokenDefs = [ { name: "sandbox-telegram-bridge", @@ -620,49 +620,50 @@ describe("resolveSandboxCreateIntent", () => { policyTier: null, }); const events: string[] = []; - const plan = materializeSandboxCreatePlan({ - intent, - fromRef: "example.invalid/image@sha256:abc", - policyAuthority: "externally-managed", - deferSandboxEffectsUntilPolicyVerification: true, - messagingTokenDefs: tokenDefs, - prepareInitialSandboxCreatePolicy: () => ({ - policyPath: "/tmp/policy.yaml", - appliedPresets: ["telegram"], - }), - runProviderPreDeleteCleanup: () => events.push("cleanup"), - upsertMessagingProviders: vi.fn(() => { - events.push("upsert"); - return ["sandbox-telegram-bridge"]; - }), - getHermesToolGatewayProviderName: () => { - events.push("hermes"); - return "sandbox-hermes-tools"; - }, + const cleanupPolicy = vi.fn(() => { + events.push("policy-cleanup"); + return true; + }); + const runProviderPreDeleteCleanup = vi.fn(() => events.push("provider-cleanup")); + const upsertMessagingProviders = vi.fn(() => { + events.push("upsert"); + return ["sandbox-telegram-bridge"]; + }); + const getHermesToolGatewayProviderName = vi.fn(() => { + events.push("hermes"); + return "sandbox-hermes-tools"; }); - expect(events).toEqual([]); - expect(plan.createArgs).not.toContain("--provider"); - expect(plan.messagingProviders).toEqual([ - "sandbox-telegram-bridge", - "sandbox-existing-discord", - ]); + expect(() => + materializeSandboxCreatePlan({ + intent, + fromRef: "example.invalid/image@sha256:abc", + policyAuthority: "externally-managed", + deferSandboxEffectsUntilPolicyVerification: true, + messagingTokenDefs: tokenDefs, + prepareInitialSandboxCreatePolicy: () => ({ + policyPath: "/tmp/policy.yaml", + appliedPresets: ["telegram"], + cleanup: cleanupPolicy, + }), + runProviderPreDeleteCleanup, + upsertMessagingProviders, + getHermesToolGatewayProviderName, + }), + ).toThrow("No sandbox was created"); - expect(plan.activateDeferredProviderEffects?.()).toEqual([ - "nvidia-prod", - "sandbox-telegram-bridge", - "sandbox-existing-discord", - "sandbox-hermes-tools", - "custom-provider", - ]); - expect(events).toEqual(["cleanup", "upsert", "hermes"]); + expect(events).toEqual(["policy-cleanup"]); + expect(cleanupPolicy).toHaveBeenCalledOnce(); + expect(runProviderPreDeleteCleanup).not.toHaveBeenCalled(); + expect(upsertMessagingProviders).not.toHaveBeenCalled(); + expect(getHermesToolGatewayProviderName).not.toHaveBeenCalled(); }); it("keeps the NemoClaw policy on a managed create when effects are deferred (#9833)", () => { const intent = resolveSandboxCreateIntent({ basePolicyPath: "/repo/policy.yaml", sandboxName: "sandbox", - inferenceProvider: "nvidia-prod", + inferenceProvider: null, channels, enabledChannels: [], disabledChannelNames: new Set(), @@ -694,9 +695,7 @@ describe("resolveSandboxCreateIntent", () => { getHermesToolGatewayProviderName: vi.fn(), }); - expect(plan.createArgs).toEqual( - expect.arrayContaining(["--policy", "/tmp/policy.yaml"]), - ); + expect(plan.createArgs).toEqual(expect.arrayContaining(["--policy", "/tmp/policy.yaml"])); expect(plan.createArgs).not.toContain("--provider"); }); From 32c74ca7a03accf7d0e65c9b689c91b6a83e4046 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 11:14:01 -0700 Subject: [PATCH 05/42] test(onboard): align APF provider refusal Signed-off-by: Apurv Kumaria --- .../onboard-fresh-create-identity.test.ts | 167 ++++++++++-------- 1 file changed, 96 insertions(+), 71 deletions(-) diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index adb631e980f..a0bbdcfd3e0 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -19,10 +19,17 @@ beforeEach(() => { describe("fresh create identity", () => { it.each([ - { label: "managed creation", apfInterceptorRequested: false }, - { label: "APF-selected creation", apfInterceptorRequested: true }, + { + title: + "registers managed creation only after owner-scoped identity and policy confirmation (#9833)", + apfInterceptorRequested: false, + }, + { + title: "rejects provider-backed APF creation before sandbox or provider effects (#9833)", + apfInterceptorRequested: true, + }, ])( - "registers $label only after owner-scoped identity and policy confirmation (#9833)", + "$title", { timeout: 45000, }, @@ -83,10 +90,10 @@ const keepAlive = setInterval(() => {}, 1000); const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; runner.run = (command, opts = {}) => { const cmd = _n(command); - const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); - if (profileResult !== null) return profileResult; _deleted = _deleted || cmd.includes("sandbox delete"); commands.push({ command: cmd, env: opts.env || null }); + const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); + if (profileResult !== null) return profileResult; if (cmd.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } @@ -187,6 +194,25 @@ childProcess.spawn = (...args) => { const { createSandbox } = require(${onboardPath}); +const writePayload = (sandboxName, creationError) => { + const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); + fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ + sandboxName, + creationError, + sandboxCreated, + sandboxListCalls, + killCalls: createCommand?.child?.killCalls ?? [], + groupKillCalls, + unrefCalls: createCommand?.child?.unrefCalls ?? 0, + stdoutDestroyCalls: createCommand?.child?.stdout.destroyCalls ?? 0, + stderrDestroyCalls: createCommand?.child?.stderr.destroyCalls ?? 0, + lifecycleObservationCommands, + registeredSandbox, + createCommand: createCommand?.command ?? null, + commandNames: commands.map((entry) => entry.command), + })); +}; + (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const createArgs = fixtureMocks.sandboxCreateArgsWithVerifiedReservation( @@ -202,21 +228,13 @@ const { createSandbox } = require(${onboardPath}); observabilityEnabled: false, }; } - const sandboxName = await createSandbox(...createArgs); - const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); - fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ - sandboxName, - sandboxListCalls, - killCalls: createCommand.child.killCalls, - groupKillCalls, - unrefCalls: createCommand.child.unrefCalls, - stdoutDestroyCalls: createCommand.child.stdout.destroyCalls, - stderrDestroyCalls: createCommand.child.stderr.destroyCalls, - lifecycleObservationCommands, - registeredSandbox, - createCommand: createCommand.command, - commandNames: commands.map((entry) => entry.command), - })); + try { + const sandboxName = await createSandbox(...createArgs); + writePayload(sandboxName, null); + } catch (error) { + if (!apfInterceptorRequested) throw error; + writePayload(null, error instanceof Error ? error.message : String(error)); + } clearInterval(keepAlive); })().catch((error) => { clearInterval(keepAlive); @@ -241,59 +259,66 @@ const { createSandbox } = require(${onboardPath}); assert.equal(result.status, 0, result.stderr); const payload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); - assert.equal(payload.sandboxName, "my-assistant"); - assert.ok(payload.sandboxListCalls >= 2); - assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); - assert.deepEqual(payload.killCalls, []); - assert.equal(payload.unrefCalls, 1); - assert.equal(payload.stdoutDestroyCalls, 1); - assert.equal(payload.stderrDestroyCalls, 1); - assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); - assert.equal( - payload.registeredSandbox.lifecycleLiveIdentityFingerprint, - createHash("sha256").update("sbx-fresh-create").digest("hex"), - ); - const assertPolicyMode = apfInterceptorRequested - ? () => { - assert.equal(payload.registeredSandbox.policyAuthority, "externally-managed"); - assert.equal(payload.registeredSandbox.policyCreationReceipt, undefined); - assert.deepEqual(payload.registeredSandbox.appliedPolicies ?? [], []); - assert.doesNotMatch(payload.createCommand, /(?:^|\s)--policy(?:=|\s)/u); - const createIndex = payload.commandNames.findIndex((command: string) => - command.includes("sandbox create"), - ); - const deferredEffectIndexes = payload.commandNames - .map((command: string, index: number) => ({ command, index })) - .filter(({ command }: { command: string }) => - /provider (?:profile import|create)|sandbox provider attach/u.test(command), - ) - .map(({ index }: { index: number }) => index); - assert.ok(deferredEffectIndexes.every((index: number) => index > createIndex)); - } - : () => { - assert.equal(payload.registeredSandbox.policyAuthority, "nemoclaw-managed"); - assert.ok(payload.registeredSandbox.policyCreationReceipt); - }; - assertPolicyMode(); - assert.match( - payload.createCommand, - /--label ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62}/u, - ); - const ownerScopedObservations = payload.lifecycleObservationCommands.filter( - (command: string) => command.includes("-g nemoclaw"), - ); - assert.ok( - ownerScopedObservations.length >= 6, - "expected owner-scoped sandbox identity observations", - ); - assert.ok( - ownerScopedObservations.every( - (command: string) => - command.includes("sandbox get -g nemoclaw my-assistant") || - command.includes("sandbox list -g nemoclaw"), + const providerEffectCommands = payload.commandNames.filter((command: string) => + /(?:^|\s)provider (?:create|update|delete|profile import)\b|(?:^|\s)sandbox provider (?:attach|detach)\b/u.test( + command, ), - `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, ); + const assertProviderBackedApfRefusal = () => { + assert.match( + payload.creationError, + /Cannot create sandbox 'my-assistant' with deferred providers .* No sandbox was created/u, + ); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, false); + assert.equal(payload.createCommand, null); + assert.equal(payload.registeredSandbox, null); + assert.deepEqual(providerEffectCommands, []); + assert.equal( + payload.commandNames.some((command: string) => command.includes("sandbox create")), + false, + ); + }; + const assertManagedCreation = () => { + assert.equal(payload.creationError, null); + assert.equal(payload.sandboxName, "my-assistant"); + assert.ok(payload.sandboxListCalls >= 2); + assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); + assert.deepEqual(payload.killCalls, []); + assert.equal(payload.unrefCalls, 1); + assert.equal(payload.stdoutDestroyCalls, 1); + assert.equal(payload.stderrDestroyCalls, 1); + assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); + assert.equal( + payload.registeredSandbox.lifecycleLiveIdentityFingerprint, + createHash("sha256").update("sbx-fresh-create").digest("hex"), + ); + assert.equal(payload.registeredSandbox.policyAuthority, "nemoclaw-managed"); + assert.ok(payload.registeredSandbox.policyCreationReceipt); + assert.match( + payload.createCommand, + /--label ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62}/u, + ); + const ownerScopedObservations = payload.lifecycleObservationCommands.filter( + (command: string) => command.includes("-g nemoclaw"), + ); + assert.ok( + ownerScopedObservations.length >= 6, + "expected owner-scoped sandbox identity observations", + ); + assert.ok( + ownerScopedObservations.every( + (command: string) => + command.includes("sandbox get -g nemoclaw my-assistant") || + command.includes("sandbox list -g nemoclaw"), + ), + `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, + ); + }; + const assertOutcome = apfInterceptorRequested + ? assertProviderBackedApfRefusal + : assertManagedCreation; + assertOutcome(); }, ); }); From cd15b07a525e035bf959939ae3c3ba72f7f12509 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 15:09:29 -0700 Subject: [PATCH 06/42] fix(onboard): limit APF creation to providerless plans Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 14 +- src/lib/onboard.ts | 22 +- src/lib/onboard/cancel-rollback.ts | 37 +- src/lib/onboard/command-support.test.ts | 1 + src/lib/onboard/command-support.ts | 2 +- .../onboard/machine/core-flow-composition.ts | 1 + .../onboard/machine/core-flow-phases.test.ts | 917 ++++++++++++------ src/lib/onboard/machine/core-flow-phases.ts | 109 ++- src/lib/onboard/machine/flow-context.ts | 6 +- .../machine/flow-phases/provider-sandbox.ts | 20 +- .../machine/handlers/provider-inference.ts | 5 +- .../machine/handlers/sandbox-apf.test.ts | 46 +- src/lib/onboard/machine/handlers/sandbox.ts | 106 +- ...naged-workload-rebuild-transaction.test.ts | 357 +------ .../onboard-orchestration.test.ts | 11 + .../managed-workload/onboard-orchestration.ts | 2 + .../managed-workload/rebuild/commit.ts | 59 +- .../managed-workload/rebuild/contract.ts | 4 - .../onboard/managed-workload/rebuild/plan.ts | 1 - .../managed-workload/rebuild/recovery.ts | 5 - .../rebuild/replacement-authority.ts | 97 -- .../managed-workload/rebuild/transaction.ts | 96 +- .../onboard/sandbox-create/orchestration.ts | 21 +- .../registry-create-only-reservation.test.ts | 107 ++ .../state/registry-rebuild-authority.test.ts | 49 +- .../state/registry-route-reservation.test.ts | 4 +- src/lib/state/registry.ts | 10 +- src/lib/state/registry/rebuild-authority.ts | 48 +- .../onboard-fresh-create-identity.test.ts | 75 +- 29 files changed, 1123 insertions(+), 1109 deletions(-) delete mode 100644 src/lib/onboard/managed-workload/rebuild/replacement-authority.ts create mode 100644 src/lib/state/registry-create-only-reservation.test.ts diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 9a6bc8c912f..ce3e2146a4a 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -463,6 +463,16 @@ Use `$$nemoclaw rebuild` when you want NemoClaw to recreate the s #### `--apf-interceptor` Use this option to request a policyless sandbox creation for an APF-interceptor flow. +This option currently supports providerless sandbox creation only. +APF onboarding with an inference provider and model is not yet supported. +APF onboarding with an OpenShell provider for a Model Context Protocol (MCP) server is also not yet supported. +OpenShell cannot bind provider attachment to the new sandbox's verified immutable ID. +If the prepared plan contains any provider, NemoClaw exits before it: + +- Creates the sandbox. +- Registers or changes credentials. +- Creates, updates, or deletes providers. + The option requires these conditions: - Start a new onboarding session. @@ -885,8 +895,8 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. -If you cancel a brand-new onboarding run at the policy preset step, NemoClaw rolls back the sandbox, registry entry, and onboarding session instead of leaving a default sandbox with unfinished policy state. -Existing live sandboxes are not deleted by this cancel rollback path. +If you cancel a brand-new onboarding run at the policy preset step, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. +NemoClaw reports the durable sandbox identity fingerprint when it is available and never deletes the sandbox by mutable name. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. In interactive mode, the wizard asks for confirmation before delete and recreate. diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 431741d7f7d..1f1ed9a3604 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -458,6 +458,7 @@ const { }: typeof import("./onboard/cancel-rollback") = require("./onboard/cancel-rollback"); const { createCoreOnboardFlowPhases, + isCoreFlowCompleteBeforeFinalization, prepareCoreOnboardFlowContext, prepareFinalOnboardFlowContext, runCoreOnboardFlowSlice, @@ -2965,13 +2966,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { onboardSessionBootstrap.reportReadOnlyHostMounts(effectiveHostMounts, note); const explicitSandboxGpuFlag = resolveSandboxGpuFlagFromOptions(opts); const recordedGpuPassthroughBeforePreflight = session?.gpuPassthrough === true; - type InitialOnboardFlowContext = - import("./onboard/machine/initial-flow-composition").InitialOnboardFlowContext< - typeof agent, - ReturnType, - ReturnType - >; - const initialFlowContext: InitialOnboardFlowContext = { + const initialFlowContext = { resume, fresh, session, @@ -2992,12 +2987,13 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { webSearchConfig: session?.webSearchConfig || null, webSearchSupported: false, selectedMessagingChannels, - gpu: null, - sandboxGpuConfig: null, + gpu: null as ReturnType | null, + sandboxGpuConfig: null as ReturnType | null, gpuPassthrough: false, resumeHasResolvedGpuIntent: false, requestedGpuPassthrough: opts.gpu === true, }; + type InitialOnboardFlowContext = typeof initialFlowContext; const policyAuthorityBindings = sandboxCreateOrchestration.createOnboardPolicyAuthorityBindings( sandboxCreateOrchestrationRuntime, @@ -3123,6 +3119,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, providerInference: { gatewayName: GATEWAY_NAME, + inspectSandboxForCreate, forceProviderSelection: forceProviderSelectionForAgentChange, ...authoritativeRebuildTarget.rebuildProviderFlowOptions(opts, coreFlowContext), endpointProvenance, @@ -3317,6 +3314,13 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { resume, recordRepairEvent, }); + if (isCoreFlowCompleteBeforeFinalization(coreFlowResult)) { + sandboxCancelRollback.disarm(); + await portableRetirementEntry.supersede(lockedRuntime.checkpointProfile); + completed = true; + process.exitCode = 0; + return; + } setupInferenceFactory.selectGatewayForFollowupOrExit(GATEWAY_NAME, runOpenshell); const finalFlowContext = prepareFinalOnboardFlowContext(coreFlowResult); let liveFinalFlowContext: InitialOnboardFlowContext = finalFlowContext; diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index be0ed4dfd3d..4d3084f2a80 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -6,21 +6,19 @@ export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preservation"; /** - * Rollback guard for a sandbox that was created during onboarding but whose - * onboarding was cancelled before the policy-preset step was confirmed. + * Preservation guard for a sandbox whose onboarding is cancelled before the + * policy-preset step is confirmed. * - * Without this, pressing Ctrl+C at the `[8/8] Policy presets` screen leaves a - * fully created OpenShell container registered as the default sandbox even - * though no policies were ever applied (#4614). + * Cancellation preserves the incomplete sandbox, registry entry, and onboarding + * session. The guard emits identity-bound recovery guidance and never deletes a + * sandbox by mutable name (#4614). * - * The guard is deliberately a two-key gate — it only fires when BOTH: - * - a freshly-created sandbox is `arm()`ed (set after createSandbox succeeds), AND - * - the operator actually cancelled via `markCancelled()` (the policy-step - * prompts call this on Ctrl+C / SIGTERM before exiting). + * The guard activates only when both conditions are true: + * - `arm()` records a newly created sandbox after `createSandbox` succeeds. + * - `markCancelled()` records Ctrl+C or SIGTERM at a policy-step prompt. * - * This keeps every other `process.exit(1)` failure path untouched: a genuine - * build/verify failure exits without `markCancelled()`, so the sandbox it left - * behind is preserved exactly as before. Only an explicit cancel rolls back. + * Other `process.exit(1)` failure paths do not call `markCancelled()`. Their + * existing preservation behavior remains unchanged. */ export interface SandboxCancelRollbackDeps { /** Emit an operator-facing line (stderr). */ @@ -28,13 +26,13 @@ export interface SandboxCancelRollbackDeps { } export interface SandboxCancelRollback { - /** Arm rollback for a just-created sandbox. */ + /** Arm cancellation recovery guidance for a just-created sandbox. */ arm(sandboxName: string, sandboxIdentityFingerprint?: string): void; /** Disarm once the sandbox is past the cancellable window (policies confirmed). */ disarm(): void; /** Record that the operator cancelled at a cancellable step. */ markCancelled(): void; - /** Run the rollback iff armed AND cancelled. Idempotent. */ + /** Report preservation guidance iff armed AND cancelled. Idempotent. */ runIfArmed(): void; /** Test/introspection helper. */ isArmed(): boolean; @@ -67,13 +65,12 @@ export interface InstallSandboxCancelRollbackOptions { } /** - * Wire a sandbox cancel-rollback to OpenShell + the registry and register the - * process-exit hook that fires it. Kept here (not in onboard.ts) so the - * orchestration lives in a focused module rather than the onboard entrypoint. + * Register the process-exit hook that emits cancellation recovery guidance. + * Keep this orchestration outside the onboard entrypoint. * - * `process.exit()` — how the policy-step prompts terminate on Ctrl+C — - * synchronously emits 'exit', so the recovery notice completes inside the - * handler. No-op unless armed AND cancelled. + * Policy-step prompts use `process.exit()` for Ctrl+C. It synchronously emits + * `exit`, so the handler reports the recovery guidance before exit completes. + * The handler does nothing unless it is armed and the operator cancels. */ export function installSandboxCancelRollback( opts: InstallSandboxCancelRollbackOptions, diff --git a/src/lib/onboard/command-support.test.ts b/src/lib/onboard/command-support.test.ts index 36a7ddd7c8f..fcef7cae4b8 100644 --- a/src/lib/onboard/command-support.test.ts +++ b/src/lib/onboard/command-support.test.ts @@ -48,6 +48,7 @@ describe("buildOnboardFlags --apf-interceptor help", () => { expect(flags["apf-interceptor"].hidden).not.toBe(true); expect(flags["apf-interceptor"].allowNo).not.toBe(true); + expect(flags["apf-interceptor"].description).toContain("providerless sandbox"); expect(flags["apf-interceptor"].description).toContain("contained sandbox-scoped policy"); expect(flags["apf-interceptor"].description).toContain("without claiming its provenance"); expect(flags["apf-interceptor"].exclusive).toEqual(["resume", "recreate-sandbox"]); diff --git a/src/lib/onboard/command-support.ts b/src/lib/onboard/command-support.ts index be031cf81ee..806ea45d057 100644 --- a/src/lib/onboard/command-support.ts +++ b/src/lib/onboard/command-support.ts @@ -109,7 +109,7 @@ export function buildOnboardFlags(options: { includeEvents?: boolean } = {}): Re "recreate-sandbox": Flags.boolean({ description: "Delete and recreate an existing sandbox" }), "apf-interceptor": Flags.boolean({ description: - "Create without a caller policy and require a contained sandbox-scoped policy without claiming its provenance", + "Create a providerless sandbox without a caller policy and require a contained sandbox-scoped policy without claiming its provenance", exclusive: ["resume", "recreate-sandbox"], }), gpu: Flags.boolean({ diff --git a/src/lib/onboard/machine/core-flow-composition.ts b/src/lib/onboard/machine/core-flow-composition.ts index c3223d9e215..d371f2d4da8 100644 --- a/src/lib/onboard/machine/core-flow-composition.ts +++ b/src/lib/onboard/machine/core-flow-composition.ts @@ -13,6 +13,7 @@ import type { OnboardFlowContext } from "./flow-context"; import { createResumeProviderShim, type ResumeProviderShimDeps } from "./resume-provider-shim"; export { + isCoreFlowCompleteBeforeFinalization, prepareCoreOnboardFlowContext, prepareFinalOnboardFlowContext, runCoreOnboardFlowSlice, diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 64380d08d20..251d53f76cc 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -95,6 +95,8 @@ function repairRecorder(events: string[] = []): OnboardPrerequisiteRepairEventRe function createPhases( overrides: { endpointProvenance?: Partial; + inspectSandboxForCreate?: ProviderOptions["inspectSandboxForCreate"]; + providerEnv?: NodeJS.ProcessEnv; providerDeps?: Partial; sandboxOptions?: Partial>; sandboxDeps?: Partial; @@ -118,8 +120,12 @@ function createPhases( const providerInference = createProviderInferenceOnboardFlowPhase({ gatewayName: "nemoclaw", forceProviderSelection: false, + inspectSandboxForCreate: + overrides.inspectSandboxForCreate ?? + (() => ({ existingEntry: null, preservedMcpState: undefined, liveExists: false })), + apfInterceptorRequested: overrides.sandboxOptions?.apfInterceptorRequested === true, endpointProvenance, - env: {}, + env: overrides.providerEnv ?? {}, constants: { hermesProviderName: "hermes", hermesApiKeyAuthMethod: "api_key", @@ -138,10 +144,8 @@ function createPhases( _gatewayName: string, operation: () => Promise | T, ) => await operation(), - withModelRouterPortLifecycleLock: async ( - _port: number, - operation: () => Promise | T, - ) => await operation(), + withModelRouterPortLifecycleLock: async (_port: number, operation: () => Promise | T) => + await operation(), getModelRouterPort: () => 4000, normalizeHermesAuthMethod: (value) => value === "oauth" || value === "api_key" ? value : null, @@ -335,8 +339,8 @@ describe("core onboard flow phases", () => { return durableSession; }); const createSandbox = vi.fn(async (...args: unknown[]) => { - const authority = args.at(-3) as { sessionId?: unknown } | null; - const createIntent = args.at(-2) as { + const authority = args[14] as { sessionId?: unknown } | null; + const createIntent = args[15] as { endpointSource?: InferenceEndpointSource | null; }; const reservation = getSandbox(sandboxName); @@ -511,15 +515,16 @@ describe("core onboard flow phases", () => { credentialEnv: "NVIDIA_INFERENCE_API_KEY", }), ); - expect(createSandbox.mock.calls[0]?.at(-2)).toMatchObject({ + const createIntent = (createSandbox.mock.calls[0] as unknown[] | undefined)?.[15]; + expect(createIntent).toMatchObject({ compatibleEndpointReasoning: "true", - deferSandboxEffectsUntilPolicyVerification: true, resolved: { inferenceProvider: "compatible-endpoint", extraProviders: ["current-provider"], staleExtraProviders: ["stale-provider"], }, }); + expect(createIntent).not.toHaveProperty("deferSandboxEffectsUntilPolicyVerification"); }); it("carries authoritative rebuild state into sandbox creation (#7803)", async () => { @@ -539,38 +544,35 @@ describe("core onboard flow phases", () => { const providerResult = await providerPhase.run(context({ agent: { name: "hermes" } })); await sandboxPhase.run(providerResult.context); - expect(createSandbox.mock.calls[0]?.at(-2)).toMatchObject({ + expect((createSandbox.mock.calls[0] as unknown[] | undefined)?.[15]).toMatchObject({ rebuildPreservedEnv, rebuildPolicyPresets, }); }); - it("defers normal-entry credential provider effects until policy verification and checkpoint publication", async () => { + it("keeps ordinary provider effects on the create-time path", async () => { const events: string[] = []; const stageSandboxCredentialProviders = vi.fn(async () => { events.push("credential-provider-effect"); return []; }); const createSandbox = vi.fn(async (...args: unknown[]) => { - const createIntent = args.at(-2) as { + const createIntent = args[15] as { deferSandboxEffectsUntilPolicyVerification?: boolean; resolved?: { policy?: { basePolicyPath?: string } }; }; - const runVerifiedEffects = args.at(-1) as + const runVerifiedEffects = args[16] as | ((context: { revalidatePolicyRequirements: (operation: string) => void; }) => Promise) | undefined; expect(createIntent).toMatchObject({ - deferSandboxEffectsUntilPolicyVerification: true, resolved: { policy: { basePolicyPath: "/repo/policy.yaml" } }, }); - expect(runVerifiedEffects).toEqual(expect.any(Function)); - expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); - events.push("policy-verified", "checkpoint-published"); - await runVerifiedEffects?.({ - revalidatePolicyRequirements: () => events.push("policy-checkpoint-revalidated"), - }); + expect(createIntent.deferSandboxEffectsUntilPolicyVerification).toBeUndefined(); + expect(runVerifiedEffects).toBeUndefined(); + expect(stageSandboxCredentialProviders).toHaveBeenCalledOnce(); + events.push("sandbox-create"); return "created-sandbox"; }); const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ @@ -580,34 +582,292 @@ describe("core onboard flow phases", () => { const providerResult = await providerPhase.run(context()); await sandboxPhase.run(providerResult.context); - expect(events).toEqual([ - "policy-verified", - "checkpoint-published", - "policy-checkpoint-revalidated", - "credential-provider-effect", + expect(events).toEqual(["credential-provider-effect", "sandbox-create"]); + }); + + it("rejects provider-backed APF before provider inference or sandbox effects", async () => { + const setupInference = vi.fn(async () => ({ ok: true as const })); + const createSandbox = vi.fn(async () => "created-sandbox"); + const { providerInference: providerPhase } = createPhases({ + providerDeps: { setupInference }, + sandboxOptions: { apfInterceptorRequested: true }, + sandboxDeps: { createSandbox }, + }); + + await expect( + providerPhase.run( + context({ + fresh: true, + model: "gpt-5.4", + provider: "nvidia-prod", + selectedMessagingChannels: [], + }), + ), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(setupInference).not.toHaveBeenCalled(); + expect(createSandbox).not.toHaveBeenCalled(); + }); + + it.each([ + "NEMOCLAW_PROVIDER", + "NEMOCLAW_MODEL", + "NEMOCLAW_PROVIDER_MODEL", + "NEMOCLAW_SERVING_PRESET", + ])("rejects APF when %s requests a provider plan", async (key) => { + const reserveSandboxInferenceRoute = vi.fn(() => true); + const checkpointSandboxIdentity = vi.fn(async () => undefined); + const { providerInference: providerPhase } = createPhases({ + providerEnv: { [key]: "requested" }, + providerDeps: { checkpointSandboxIdentity, reserveSandboxInferenceRoute }, + sandboxOptions: { apfInterceptorRequested: true }, + }); + + await expect( + providerPhase.run( + context({ fresh: true, model: null, provider: null, selectedMessagingChannels: [] }), + ), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(checkpointSandboxIdentity).not.toHaveBeenCalled(); + expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); + }); + + it("rejects APF when a serving profile requests an inference plan", async () => { + const session = createSession({ apfInterceptorRequested: true }); + session.servingProfileProvenance = {} as never; + const reserveSandboxInferenceRoute = vi.fn(() => true); + const checkpointSandboxIdentity = vi.fn(async () => undefined); + const { providerInference: providerPhase } = createPhases({ + providerDeps: { checkpointSandboxIdentity, reserveSandboxInferenceRoute }, + sandboxOptions: { apfInterceptorRequested: true }, + }); + + await expect( + providerPhase.run( + context({ + fresh: true, + session, + model: null, + provider: null, + selectedMessagingChannels: [], + }), + ), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(checkpointSandboxIdentity).not.toHaveBeenCalled(); + expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); + }); + + it("completes providerless APF after the verified sandbox-create boundary", async () => { + const setupNim = vi.fn(); + const setupInference = vi.fn(async () => ({ ok: true as const })); + let providerlessReservation: { + name: string; + gatewayName: string; + pendingRouteReservation: true; + reservationSessionId?: string; + provider: string | null; + model: string | null; + endpointUrl: string | null; + endpointSource: InferenceEndpointSource | null; + credentialEnv: string | null; + preferredInferenceApi: string | null; + } | null = null; + const reserveRoute = vi.fn( + ( + name: string, + route: Parameters[1], + ) => { + providerlessReservation = { + name, + pendingRouteReservation: true, + ...route, + }; + return true; + }, + ); + const updateSandboxRegistry = vi.fn(); + const recordStepComplete = vi.fn(async (_stepName: string, updates: SessionUpdates = {}) => { + Object.assign(session, updates); + return session; + }); + const createSandbox = vi.fn(async (...args: unknown[]) => { + expect(args[1]).toBe(""); + expect(args[2]).toBe(""); + expect(args[15]).toMatchObject({ + apfInterceptorRequested: true, + deferSandboxEffectsUntilPolicyVerification: true, + }); + return "created-sandbox"; + }); + const session = createSession({ apfInterceptorRequested: true }); + const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ + providerDeps: { + reserveSandboxInferenceRoute: reserveRoute, + setupInference, + setupNim, + }, + sandboxOptions: { apfInterceptorRequested: true }, + sandboxDeps: { + createSandbox, + getSandboxRegistryEntry: () => providerlessReservation, + recordStepComplete, + setupMessagingChannels: vi.fn(async () => []), + updateSandboxRegistry, + }, + }); + const initial = context({ + fresh: true, + session, + model: null, + provider: null, + selectedMessagingChannels: [], + }); + + const providerResult = await providerPhase.run(initial); + expect(providerResult.result).toEqual([ + advanceTo("inference", { + metadata: { state: "provider_selection", providerlessApf: true }, + }), + advanceTo("sandbox", { + metadata: { state: "inference", providerlessApf: true }, + }), ]); + expect(reserveRoute).toHaveBeenCalledWith( + "my-sandbox", + expect.objectContaining({ + provider: null, + model: null, + gatewayName: "nemoclaw", + reservationSessionId: session.sessionId, + }), + { requireAbsent: true }, + ); + const sandboxResult = await sandboxPhase.run(providerResult.context); + + expect(sandboxResult.result).toMatchObject({ type: "complete" }); + expect(setupNim).not.toHaveBeenCalled(); + expect(setupInference).not.toHaveBeenCalled(); + expect(createSandbox).toHaveBeenCalledOnce(); + expect(updateSandboxRegistry).toHaveBeenCalledWith( + "created-sandbox", + expect.not.objectContaining({ model: expect.anything(), provider: expect.anything() }), + ); + const sandboxCompletion = recordStepComplete.mock.calls.find(([step]) => step === "sandbox"); + expect(sandboxCompletion?.[1]).not.toHaveProperty("model"); + expect(sandboxCompletion?.[1]).not.toHaveProperty("provider"); + }); + + it.each([ + ["registered", true, false], + ["live", false, true], + ] as const)( + "rejects a %s APF sandbox-name collision without changing registry or external state", + async (_label, registered, liveExists) => { + const existingEntry = registered + ? { + name: "my-sandbox", + provider: "nim", + model: "nvidia/test", + gatewayName: "nemoclaw", + } + : null; + const originalEntry = structuredClone(existingEntry); + const reserveRoute = vi.fn(() => true); + const checkpointSandboxIdentity = vi.fn(async () => undefined); + const createSandbox = vi.fn(async () => "created-sandbox"); + const stageSandboxCredentialProviders = vi.fn(async () => []); + const { providerInference: providerPhase } = createPhases({ + inspectSandboxForCreate: () => ({ + existingEntry, + preservedMcpState: undefined, + liveExists, + }), + providerDeps: { checkpointSandboxIdentity, reserveSandboxInferenceRoute: reserveRoute }, + sandboxOptions: { apfInterceptorRequested: true }, + sandboxDeps: { createSandbox, stageSandboxCredentialProviders }, + }); + + await expect( + providerPhase.run( + context({ + fresh: true, + session: createSession({ apfInterceptorRequested: true }), + selectedMessagingChannels: [], + }), + ), + ).rejects.toThrow(/cannot adopt existing sandbox/u); + + expect(existingEntry).toEqual(originalEntry); + expect(checkpointSandboxIdentity).not.toHaveBeenCalled(); + expect(reserveRoute).not.toHaveBeenCalled(); + expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); + expect(createSandbox).not.toHaveBeenCalled(); + }, + ); + + it("refuses an APF reservation race without sandbox, credential, or provider effects", async () => { + const reserveRoute = vi.fn(() => false); + const createSandbox = vi.fn(async () => "created-sandbox"); + const stageSandboxCredentialProviders = vi.fn(async () => []); + const { providerInference: providerPhase } = createPhases({ + providerDeps: { reserveSandboxInferenceRoute: reserveRoute }, + sandboxOptions: { apfInterceptorRequested: true }, + sandboxDeps: { createSandbox, stageSandboxCredentialProviders }, + }); + + await expect( + providerPhase.run( + context({ + fresh: true, + session: createSession({ apfInterceptorRequested: true }), + selectedMessagingChannels: [], + }), + ), + ).rejects.toThrow(/could not reserve sandbox/u); + + expect(reserveRoute).toHaveBeenCalledWith("my-sandbox", expect.any(Object), { + requireAbsent: true, + }); + expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); + expect(createSandbox).not.toHaveBeenCalled(); }); it.each([ ["post-create policy verification", "policy verification refused"], ["durable checkpoint publication", "checkpoint publication refused"], - ])("withholds normal-entry provider effects when %s fails", async (_boundary, failure) => { + ])("withholds APF provider effects when %s fails", async (_boundary, failure) => { const stageSandboxCredentialProviders = vi.fn(async () => []); const createSandbox = vi.fn(async (...args: unknown[]) => { - const createIntent = args.at(-2) as { + const createIntent = args[15] as { deferSandboxEffectsUntilPolicyVerification?: boolean; }; - const runVerifiedEffects = args.at(-1); + const runVerifiedEffects = args[16]; expect(createIntent.deferSandboxEffectsUntilPolicyVerification).toBe(true); expect(runVerifiedEffects).toEqual(expect.any(Function)); expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); throw new Error(failure); }); + const session = createSession({ apfInterceptorRequested: true }); const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ - sandboxDeps: { createSandbox, stageSandboxCredentialProviders }, + sandboxOptions: { apfInterceptorRequested: true }, + sandboxDeps: { + createSandbox, + getSandboxRegistryEntry: () => ({ + name: "my-sandbox", + gatewayName: "nemoclaw", + pendingRouteReservation: true, + reservationSessionId: session.sessionId, + }), + setupMessagingChannels: vi.fn(async () => []), + stageSandboxCredentialProviders, + }, }); - const providerResult = await providerPhase.run(context()); + const providerResult = await providerPhase.run( + context({ fresh: true, session, selectedMessagingChannels: [] }), + ); await expect(sandboxPhase.run(providerResult.context)).rejects.toThrow(failure); expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); @@ -740,155 +1000,172 @@ describe("core onboard flow phases", () => { false, ], ["provider-mismatched", "nvidia-prod", "https://persisted.example.test/v1", null, null, false], - ] as const)("binds %s persisted onboard provenance to its exact provider endpoint", async (_label, registeredProvider, registeredEndpointUrl, expectedSource, expectedOnboardEndpointUrl, expectTrustedUrl) => { - const setupInference = vi.fn(async () => ({ ok: true as const })); - const updateSandboxRegistry = vi.fn(); - const getSandboxRegistryEntry = vi.fn((_sandboxName: string) => ({ - name: "my-sandbox", - provider: registeredProvider, - model: "custom/model", - endpointUrl: registeredEndpointUrl, - endpointSource: "onboard" as const, - credentialEnv: "COMPATIBLE_API_KEY", - preferredInferenceApi: "openai-completions", - gatewayName: "nemoclaw", - gpuEnabled: false, - policies: [], - })); - const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ - providerDeps: { - setupInference, - hydrateCredentialEnv: vi.fn(() => "host-key"), - }, - endpointProvenance: { - getSandboxRegistryEntry, - }, - sandboxDeps: { updateSandboxRegistry }, - }); - const session = createSession({ - provider: "compatible-endpoint", - model: "custom/model", - endpointUrl: "https://persisted.example.test/v1", - credentialEnv: "COMPATIBLE_API_KEY", - preferredInferenceApi: "openai-completions", - steps: { provider_selection: completeStep() }, - }); - - const result = await providerPhase.run( - context({ - resume: true, - session, + ] as const)( + "binds %s persisted onboard provenance to its exact provider endpoint", + async ( + _label, + registeredProvider, + registeredEndpointUrl, + expectedSource, + expectedOnboardEndpointUrl, + expectTrustedUrl, + ) => { + const setupInference = vi.fn(async () => ({ ok: true as const })); + const updateSandboxRegistry = vi.fn(); + const getSandboxRegistryEntry = vi.fn((_sandboxName: string) => ({ + name: "my-sandbox", + provider: registeredProvider, + model: "custom/model", + endpointUrl: registeredEndpointUrl, + endpointSource: "onboard" as const, + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + gatewayName: "nemoclaw", + gpuEnabled: false, + policies: [], + })); + const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ + providerDeps: { + setupInference, + hydrateCredentialEnv: vi.fn(() => "host-key"), + }, + endpointProvenance: { + getSandboxRegistryEntry, + }, + sandboxDeps: { updateSandboxRegistry }, + }); + const session = createSession({ provider: "compatible-endpoint", model: "custom/model", endpointUrl: "https://persisted.example.test/v1", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", - }), - ); + steps: { provider_selection: completeStep() }, + }); - const inferenceOptions = setupInference.mock.calls[0]?.at(-1) as - | { endpointSource?: string | null; onboardEndpointUrl?: string } - | undefined; - expect(inferenceOptions).toMatchObject({ endpointSource: expectedSource }); - expect(inferenceOptions?.onboardEndpointUrl ?? null).toBe(expectedOnboardEndpointUrl); - expect(Object.hasOwn(inferenceOptions ?? {}, "onboardEndpointUrl")).toBe(expectTrustedUrl); - expect(result.context.endpointSource).toBe(expectedSource); - expect(result.context.onboardEndpointUrl ?? null).toBe(expectedOnboardEndpointUrl); - expect(getSandboxRegistryEntry).toHaveBeenCalledWith("my-sandbox"); + const result = await providerPhase.run( + context({ + resume: true, + session, + provider: "compatible-endpoint", + model: "custom/model", + endpointUrl: "https://persisted.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }), + ); - await sandboxPhase.run(result.context); + const inferenceOptions = setupInference.mock.calls[0]?.at(-1) as + | { endpointSource?: string | null; onboardEndpointUrl?: string } + | undefined; + expect(inferenceOptions).toMatchObject({ endpointSource: expectedSource }); + expect(inferenceOptions?.onboardEndpointUrl ?? null).toBe(expectedOnboardEndpointUrl); + expect(Object.hasOwn(inferenceOptions ?? {}, "onboardEndpointUrl")).toBe(expectTrustedUrl); + expect(result.context.endpointSource).toBe(expectedSource); + expect(result.context.onboardEndpointUrl ?? null).toBe(expectedOnboardEndpointUrl); + expect(getSandboxRegistryEntry).toHaveBeenCalledWith("my-sandbox"); - expect(updateSandboxRegistry).toHaveBeenCalledWith( - "created-sandbox", - expect.objectContaining({ endpointSource: expectedSource }), - ); - }); + await sandboxPhase.run(result.context); + + expect(updateSandboxRegistry).toHaveBeenCalledWith( + "created-sandbox", + expect.objectContaining({ endpointSource: expectedSource }), + ); + }, + ); it.each([ ["fresh", false], ["resumed", true], - ] as const)("uses the strict runner for %s provider selection sessions", async (_label, resume) => { - const phaseCalls: string[] = []; - const appliedTransitions: string[] = []; - const sandboxEffect = vi.fn(); - let runtimeSession = createSession({ - machine: { - version: 1, - state: "provider_selection", - stateEnteredAt: "2026-06-09T00:00:00.000Z", - revision: 1, - }, - }); - const runProviderInference = vi.fn((ctx: CoreContext) => { - phaseCalls.push("provider_selection"); - return { - context: { ...ctx, endpointUrl: "https://example.test/v1" }, - result: [ - advanceTo("inference", { metadata: { state: "provider_selection" } }), - advanceTo("sandbox", { metadata: { state: "inference" } }), - ], - }; - }); - const runSandbox = vi.fn((ctx: CoreContext) => { - phaseCalls.push("sandbox"); - sandboxEffect(ctx); - return { - context: { ...ctx, sandboxName: "created-sandbox" }, - result: branchTo("openclaw", { metadata: { state: "sandbox" } }), + ] as const)( + "uses the strict runner for %s provider selection sessions", + async (_label, resume) => { + const phaseCalls: string[] = []; + const appliedTransitions: string[] = []; + const sandboxEffect = vi.fn(); + let runtimeSession = createSession({ + machine: { + version: 1, + state: "provider_selection", + stateEnteredAt: "2026-06-09T00:00:00.000Z", + revision: 1, + }, + }); + const runProviderInference = vi.fn((ctx: CoreContext) => { + phaseCalls.push("provider_selection"); + return { + context: { ...ctx, endpointUrl: "https://example.test/v1" }, + result: [ + advanceTo("inference", { metadata: { state: "provider_selection" } }), + advanceTo("sandbox", { metadata: { state: "inference" } }), + ], + }; + }); + const runSandbox = vi.fn((ctx: CoreContext) => { + phaseCalls.push("sandbox"); + sandboxEffect(ctx); + return { + context: { ...ctx, sandboxName: "created-sandbox" }, + result: branchTo("openclaw", { metadata: { state: "sandbox" } }), + }; + }); + const phases: CoreOnboardFlowPhases = { + providerInference: { + state: "provider_selection", + run: runProviderInference, + }, + sandbox: { + state: "sandbox", + run: runSandbox, + }, }; - }); - const phases: CoreOnboardFlowPhases = { - providerInference: { - state: "provider_selection", - run: runProviderInference, - }, - sandbox: { - state: "sandbox", - run: runSandbox, - }, - }; - const result = await runCoreOnboardFlowSlice({ - context: context({ + const result = await runCoreOnboardFlowSlice({ + context: context({ + resume, + fresh: !resume, + model: "nvidia/test", + provider: "nim", + }), + runtime: { + session: async () => runtimeSession, + applyResult: async (stateResult) => { + const transition = stateResult as ReturnType; + appliedTransitions.push(`${transition.transitionKind}:${transition.next}`); + runtimeSession = createSession({ + machine: { + version: 1, + state: transition.next, + stateEnteredAt: "2026-06-09T00:03:00.000Z", + revision: runtimeSession.machine.revision + 1, + }, + }); + return runtimeSession; + }, + }, + phases, resume, - fresh: !resume, - model: "nvidia/test", - provider: "nim", - }), - runtime: { - session: async () => runtimeSession, - applyResult: async (stateResult) => { - const transition = stateResult as ReturnType; - appliedTransitions.push(`${transition.transitionKind}:${transition.next}`); - runtimeSession = createSession({ - machine: { - version: 1, - state: transition.next, - stateEnteredAt: "2026-06-09T00:03:00.000Z", - revision: runtimeSession.machine.revision + 1, - }, - }); - return runtimeSession; + recordRepairEvent: async () => { + throw new Error("repair recorder should not run on the exact-entry path"); }, - }, - phases, - resume, - recordRepairEvent: async () => { - throw new Error("repair recorder should not run on the exact-entry path"); - }, - }); + }); - expect(phaseCalls).toEqual(["provider_selection", "sandbox"]); - expect(runProviderInference).toHaveBeenCalledOnce(); - expect(runSandbox).toHaveBeenCalledOnce(); - expect(sandboxEffect).toHaveBeenCalledOnce(); - expect(sandboxEffect).toHaveBeenCalledWith( - expect.objectContaining({ endpointUrl: "https://example.test/v1" }), - ); - expect(appliedTransitions).toEqual(["advance:inference", "advance:sandbox", "branch:openclaw"]); - expect(result.context.endpointUrl).toBe("https://example.test/v1"); - expect(result.context.sandboxName).toBe("created-sandbox"); - expect(result.session.machine.state).toBe("openclaw"); - }); + expect(phaseCalls).toEqual(["provider_selection", "sandbox"]); + expect(runProviderInference).toHaveBeenCalledOnce(); + expect(runSandbox).toHaveBeenCalledOnce(); + expect(sandboxEffect).toHaveBeenCalledOnce(); + expect(sandboxEffect).toHaveBeenCalledWith( + expect.objectContaining({ endpointUrl: "https://example.test/v1" }), + ); + expect(appliedTransitions).toEqual([ + "advance:inference", + "advance:sandbox", + "branch:openclaw", + ]); + expect(result.context.endpointUrl).toBe("https://example.test/v1"); + expect(result.context.sandboxName).toBe("created-sandbox"); + expect(result.session.machine.state).toBe("openclaw"); + }, + ); it("runs the sandbox effect once at exact sandbox entry", async () => { const createSandbox = vi.fn(async () => "created-sandbox"); @@ -947,176 +1224,176 @@ describe("core onboard flow phases", () => { ["inference", true, ["advance:sandbox", "branch:openclaw"]], ["sandbox", true, ["branch:openclaw"]], ["sandbox", false, ["branch:openclaw"]], - ] as const)("repairs provider context before strict %s entry", async (state, resume, expected) => { - const calls: string[] = []; - const applied: string[] = []; - const repairEvents: string[] = []; - let runtimeSession = createSession({ - machine: { - version: 1, - state, - stateEnteredAt: "2026-06-09T00:02:00.000Z", - revision: 7, - }, - }); - const phases: CoreOnboardFlowPhases = { - providerInference: { - state: "provider_selection", - run: (ctx) => { - calls.push("provider_selection"); - return { + ] as const)( + "repairs provider context before strict %s entry", + async (state, resume, expected) => { + const calls: string[] = []; + const applied: string[] = []; + const repairEvents: string[] = []; + let runtimeSession = createSession({ + machine: { + version: 1, + state, + stateEnteredAt: "2026-06-09T00:02:00.000Z", + revision: 7, + }, + }); + const phases: CoreOnboardFlowPhases = { + providerInference: { + state: "provider_selection", + run: (ctx) => { + calls.push("provider_selection"); + return { + context: { ...ctx, endpointUrl: "https://example.test/v1" }, + result: [ + advanceTo("inference", { metadata: { state: "provider_selection" } }), + advanceTo("sandbox", { metadata: { state: "inference" } }), + ], + }; + }, + }, + sandbox: { + state: "sandbox", + run: (ctx) => { + calls.push("sandbox"); + return { + context: { ...ctx, sandboxName: "created-sandbox" }, + result: branchTo("openclaw", { metadata: { state: "sandbox" } }), + }; + }, + }, + }; + + const result = await runCoreOnboardFlowSlice({ + context: context({ resume }), + runtime: { + session: async () => runtimeSession, + applyResult: async (stateResult) => { + if (stateResult.type === "transition") { + applied.push(`${stateResult.transitionKind}:${stateResult.next}`); + runtimeSession.machine = { + ...runtimeSession.machine, + state: stateResult.next, + revision: runtimeSession.machine.revision + 1, + }; + } + return runtimeSession; + }, + }, + phases, + resume, + recordRepairEvent: repairRecorder(repairEvents), + }); + + expect(calls).toEqual(["provider_selection", "sandbox"]); + expect(applied).toEqual(expected); + expect(repairEvents).toEqual([ + "state.repair.started:provider_selection", + "state.repair.completed:provider_selection", + ]); + expect(result.context.endpointUrl).toBe("https://example.test/v1"); + expect(result.context.sandboxName).toBe("created-sandbox"); + expect(result.session.machine.state).toBe("openclaw"); + }, + ); + + it.each(["openclaw", "agent_setup", "policies", "finalizing", "post_verify"] as const)( + "repairs core prerequisites without changing resumed %s entry", + async (state) => { + const repairEvents: string[] = []; + const branchState = state === "agent_setup" ? "agent_setup" : "openclaw"; + const session = createSession({ + machine: { + version: 1, + state, + stateEnteredAt: "2026-06-09T00:02:00.000Z", + revision: 7, + }, + }); + const phases: CoreOnboardFlowPhases = { + providerInference: { + state: "provider_selection", + run: (ctx) => ({ context: { ...ctx, endpointUrl: "https://example.test/v1" }, result: [ advanceTo("inference", { metadata: { state: "provider_selection" } }), advanceTo("sandbox", { metadata: { state: "inference" } }), ], - }; + }), }, - }, - sandbox: { - state: "sandbox", - run: (ctx) => { - calls.push("sandbox"); - return { + sandbox: { + state: "sandbox", + run: (ctx) => ({ context: { ...ctx, sandboxName: "created-sandbox" }, - result: branchTo("openclaw", { metadata: { state: "sandbox" } }), - }; + result: branchTo(branchState, { metadata: { state: "sandbox" } }), + }), }, - }, - }; + }; - const result = await runCoreOnboardFlowSlice({ - context: context({ resume }), - runtime: { - session: async () => runtimeSession, - applyResult: async (stateResult) => { - if (stateResult.type === "transition") { - applied.push(`${stateResult.transitionKind}:${stateResult.next}`); - runtimeSession.machine = { - ...runtimeSession.machine, - state: stateResult.next, - revision: runtimeSession.machine.revision + 1, - }; - } - return runtimeSession; + const result = await runCoreOnboardFlowSlice({ + context: context({ resume: true }), + runtime: { + session: async () => session, + applyResult: async () => { + throw new Error("prerequisite repair must not apply a machine result"); + }, }, - }, - phases, - resume, - recordRepairEvent: repairRecorder(repairEvents), - }); + phases, + resume: true, + recordRepairEvent: repairRecorder(repairEvents), + }); - expect(calls).toEqual(["provider_selection", "sandbox"]); - expect(applied).toEqual(expected); - expect(repairEvents).toEqual([ - "state.repair.started:provider_selection", - "state.repair.completed:provider_selection", - ]); - expect(result.context.endpointUrl).toBe("https://example.test/v1"); - expect(result.context.sandboxName).toBe("created-sandbox"); - expect(result.session.machine.state).toBe("openclaw"); - }); + expect(repairEvents).toEqual([ + "state.repair.started:provider_selection", + "state.repair.completed:provider_selection", + "state.repair.started:sandbox", + "state.repair.completed:sandbox", + ]); + expect(result.session.machine.state).toBe(state); + expect(result.context.sandboxName).toBe("created-sandbox"); + }, + ); - it.each([ - "openclaw", - "agent_setup", - "policies", - "finalizing", - "post_verify", - ] as const)("repairs core prerequisites without changing resumed %s entry", async (state) => { - const repairEvents: string[] = []; - const branchState = state === "agent_setup" ? "agent_setup" : "openclaw"; - const session = createSession({ - machine: { - version: 1, - state, - stateEnteredAt: "2026-06-09T00:02:00.000Z", - revision: 7, - }, - }); - const phases: CoreOnboardFlowPhases = { - providerInference: { + it.each(["complete", "failed"] as const)( + "rejects terminal %s sessions before core repair effects", + async (state) => { + const providerInference: OnboardSequencePhase = { state: "provider_selection", - run: (ctx) => ({ - context: { ...ctx, endpointUrl: "https://example.test/v1" }, - result: [ - advanceTo("inference", { metadata: { state: "provider_selection" } }), - advanceTo("sandbox", { metadata: { state: "inference" } }), - ], - }), - }, - sandbox: { + run: vi.fn((ctx) => ({ + context: ctx, + result: advanceTo("sandbox", { metadata: { state: "inference" } }), + })), + }; + const sandbox: OnboardSequencePhase = { state: "sandbox", - run: (ctx) => ({ - context: { ...ctx, sandboxName: "created-sandbox" }, - result: branchTo(branchState, { metadata: { state: "sandbox" } }), - }), - }, - }; - - const result = await runCoreOnboardFlowSlice({ - context: context({ resume: true }), - runtime: { - session: async () => session, - applyResult: async () => { - throw new Error("prerequisite repair must not apply a machine result"); - }, - }, - phases, - resume: true, - recordRepairEvent: repairRecorder(repairEvents), - }); - - expect(repairEvents).toEqual([ - "state.repair.started:provider_selection", - "state.repair.completed:provider_selection", - "state.repair.started:sandbox", - "state.repair.completed:sandbox", - ]); - expect(result.session.machine.state).toBe(state); - expect(result.context.sandboxName).toBe("created-sandbox"); - }); - - it.each([ - "complete", - "failed", - ] as const)("rejects terminal %s sessions before core repair effects", async (state) => { - const providerInference: OnboardSequencePhase = { - state: "provider_selection", - run: vi.fn((ctx) => ({ - context: ctx, - result: advanceTo("sandbox", { metadata: { state: "inference" } }), - })), - }; - const sandbox: OnboardSequencePhase = { - state: "sandbox", - run: vi.fn((ctx) => ({ - context: ctx, - result: branchTo("openclaw", { metadata: { state: "sandbox" } }), - })), - }; + run: vi.fn((ctx) => ({ + context: ctx, + result: branchTo("openclaw", { metadata: { state: "sandbox" } }), + })), + }; - await expect( - runCoreOnboardFlowSlice({ - context: context({ resume: true }), - runtime: { - session: async () => - createSession({ - machine: { - version: 1, - state, - stateEnteredAt: "2026-06-09T00:00:00.000Z", - revision: 7, - }, - }), - applyResult: async () => createSession(), - }, - phases: { providerInference, sandbox }, - resume: true, - recordRepairEvent: repairRecorder(), - }), - ).rejects.toThrow("Unexpected onboarding flow state before slice entry"); - expect(providerInference.run).not.toHaveBeenCalled(); - expect(sandbox.run).not.toHaveBeenCalled(); - }); + await expect( + runCoreOnboardFlowSlice({ + context: context({ resume: true }), + runtime: { + session: async () => + createSession({ + machine: { + version: 1, + state, + stateEnteredAt: "2026-06-09T00:00:00.000Z", + revision: 7, + }, + }), + applyResult: async () => createSession(), + }, + phases: { providerInference, sandbox }, + resume: true, + recordRepairEvent: repairRecorder(), + }), + ).rejects.toThrow("Unexpected onboarding flow state before slice entry"); + expect(providerInference.run).not.toHaveBeenCalled(); + expect(sandbox.run).not.toHaveBeenCalled(); + }, + ); }); diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index c6f94e514e6..ab12f292613 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -19,6 +19,7 @@ import { import { createProviderInferencePhase, createSandboxPhase } from "./flow-phases/provider-sandbox"; import { UnexpectedOnboardFlowSliceStateError } from "./flow-slice-error"; import { runCoreOnboardFlowSequence } from "./flow-slices"; +import { advanceTo } from "./result"; import { handleProviderInferenceState, type ProviderInferenceStateOptions, @@ -52,9 +53,11 @@ export interface ProviderInferenceOnboardFlowPhaseOptions< gatewayName: string; forceProviderSelection: boolean; forceInferenceSetup?: boolean; + apfInterceptorRequested?: boolean; authoritativeResumeConfig?: boolean; providerRecoveryReceipt?: ProviderRecoveryReceipt | null; providerRecoveryReceiptLedger?: ReturnType; + inspectSandboxForCreate: import("../sandbox-lifecycle").SandboxLifecycleHelpers["inspectSandboxForCreate"]; endpointProvenance: EndpointProvenanceOptions; env: NodeJS.ProcessEnv; constants: ProviderInferenceStateOptions["constants"]; @@ -105,6 +108,49 @@ interface EndpointProvenance { onboardEndpointUrl: string | null; } +export function isCoreFlowCompleteBeforeFinalization(result: { + readonly context: Pick; + readonly session: { readonly machine: { readonly state: string } }; +}): boolean { + return ( + result.context.providerlessApf === true && + result.session.machine.state === "complete" && + Boolean(result.context.sandboxName) + ); +} + +const APF_PROVIDER_INTENT_ENV_KEYS = [ + "NEMOCLAW_PROVIDER", + "NEMOCLAW_MODEL", + "NEMOCLAW_PROVIDER_MODEL", + "NEMOCLAW_SERVING_PRESET", +] as const; + +function hasProviderBackedApfIntent(context: OnboardFlowContext, env: NodeJS.ProcessEnv): boolean { + const routeValues = [ + context.provider, + context.model, + context.endpointUrl, + context.onboardEndpointUrl, + context.credentialEnv, + context.preferredInferenceApi, + context.compatibleEndpointReasoning, + context.compatibleEndpointReasoningEffort, + context.nimContainer, + ]; + return ( + routeValues.some((value) => typeof value === "string" && value.trim().length > 0) || + context.endpointSource != null || + context.selectedMessagingChannels.length > 0 || + context.hermesToolGateways.length > 0 || + context.webSearchConfig !== null || + context.hostLocalInferenceRouteOnly === true || + context.hostLocalInferenceSandboxProofAuthority != null || + context.session?.servingProfileProvenance != null || + APF_PROVIDER_INTENT_ENV_KEYS.some((key) => String(env[key] ?? "").trim().length > 0) + ); +} + function endpointProvenanceForPhase( context: Pick, options: EndpointProvenanceOptions, @@ -143,6 +189,63 @@ export function createProviderInferenceOnboardFlowPhase< Host = unknown, >(options: ProviderInferenceOnboardFlowPhaseOptions): OnboardSequencePhase { return createProviderInferencePhase(async (context) => { + if ( + options.apfInterceptorRequested === true || + context.session?.apfInterceptorRequested === true + ) { + if (hasProviderBackedApfIntent(context, options.env)) { + throw new Error( + "APF interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + ); + } + const sandboxName = + context.sandboxName ?? (await options.deps.promptValidatedSandboxName(context.agent)); + const reservationSessionId = context.session?.sessionId; + if (!reservationSessionId) { + throw new Error( + "APF interceptor onboarding requires a durable session before providerless sandbox creation.", + ); + } + const observed = options.inspectSandboxForCreate(sandboxName); + if (observed.existingEntry || observed.liveExists) { + throw new Error( + `APF interceptor selection cannot adopt existing sandbox '${sandboxName}'. Choose a new sandbox name.`, + ); + } + await options.deps.checkpointSandboxIdentity(sandboxName, context.agent); + const reserved = await options.deps.withGatewayRouteMutationLock(options.gatewayName, () => + options.deps.reserveSandboxInferenceRoute( + sandboxName, + { + provider: null, + model: null, + endpointUrl: null, + endpointSource: null, + credentialEnv: null, + preferredInferenceApi: null, + gatewayName: options.gatewayName, + reservationSessionId, + }, + { requireAbsent: true }, + ), + ); + if (!reserved) { + throw new Error( + `APF interceptor onboarding could not reserve sandbox '${sandboxName}' for verified providerless creation.`, + ); + } + return { + context: { ...context, sandboxName, providerlessApf: true }, + result: [ + advanceTo("inference", { + metadata: { state: "provider_selection", providerlessApf: true }, + }), + advanceTo("sandbox", { + metadata: { state: "inference", providerlessApf: true }, + }), + ], + }; + } const endpointProvenance = endpointProvenanceForPhase(context, options.endpointProvenance); const providerInferenceResult = await handleProviderInferenceState({ gatewayName: options.gatewayName, @@ -232,7 +335,7 @@ export function createSandboxOnboardFlowPhase< apfInterceptorRequested: options.apfInterceptorRequested === true, authoritativeResumeConfig: options.authoritativeResumeConfig, authoritativePolicyTier: options.authoritativePolicyTier, - deferSandboxEffectsUntilPolicyVerification: true, + deferSandboxEffectsUntilPolicyVerification: options.apfInterceptorRequested === true, recreateJournalTargetIntentFingerprint: options.recreateJournalTargetIntentFingerprint, endpointSource: endpointProvenance.endpointSource, @@ -245,8 +348,8 @@ export function createSandboxOnboardFlowPhase< recreateSandbox: options.recreateSandbox, session: context.session, sandboxName: context.sandboxName, - model: context.model, - provider: context.provider, + model: context.model ?? "", + provider: context.provider ?? "", endpointUrl: context.endpointUrl, compatibleEndpointReasoning: context.compatibleEndpointReasoning, credentialEnv: context.credentialEnv, diff --git a/src/lib/onboard/machine/flow-context.ts b/src/lib/onboard/machine/flow-context.ts index a1fab15c506..73df46ed8ae 100644 --- a/src/lib/onboard/machine/flow-context.ts +++ b/src/lib/onboard/machine/flow-context.ts @@ -33,6 +33,8 @@ export interface OnboardFlowContext( - context: ProviderModelSelectedOnboardFlowContext, + context: Context, patch: SandboxCreatedContextUpdate, -): SandboxCreatedOnboardFlowContext { +): Context & { sandboxName: string } { return { ...context, ...patch }; } diff --git a/src/lib/onboard/machine/flow-phases/provider-sandbox.ts b/src/lib/onboard/machine/flow-phases/provider-sandbox.ts index 739241adcbc..852cbb338fb 100644 --- a/src/lib/onboard/machine/flow-phases/provider-sandbox.ts +++ b/src/lib/onboard/machine/flow-phases/provider-sandbox.ts @@ -1,27 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { - OnboardFlowContext, - OnboardFlowPhaseResult, - ProviderModelSelectedOnboardFlowContext, - ProviderSelectedOnboardFlowContext, - SandboxCreatedOnboardFlowContext, -} from "../flow-context"; +import type { OnboardFlowContext, OnboardFlowPhaseResult } from "../flow-context"; import { assertProviderSelectedContext, onboardFlowPhaseResult } from "../flow-context"; import type { OnboardSequencePhase } from "../sequence-runner"; type ProviderInferencePhaseHandler = ( context: Context, ) => Promise<{ - context: ProviderModelSelectedOnboardFlowContext; + context: Context; result: OnboardFlowPhaseResult["result"]; }>; -type SandboxPhaseHandler = ( - context: ProviderSelectedOnboardFlowContext, -) => Promise<{ - context: SandboxCreatedOnboardFlowContext; +type SandboxPhaseHandler = (context: Context) => Promise<{ + context: Context & { sandboxName: string }; result: OnboardFlowPhaseResult["result"]; }>; @@ -43,7 +35,9 @@ export function createSandboxPhase( return { state: "sandbox", async run(context) { - assertProviderSelectedContext(context, "sandbox setup"); + if (context.providerlessApf !== true) { + assertProviderSelectedContext(context, "sandbox setup"); + } const result = await runSandbox(context); return onboardFlowPhaseResult(result.context, result.result); }, diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index f141f0c9ef7..850fe27238f 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -278,8 +278,8 @@ export interface ProviderInferenceStateOptions { reserveSandboxInferenceRoute( sandboxName: string, route: { - provider: string; - model: string; + provider: string | null; + model: string | null; endpointUrl: string | null; endpointSource: InferenceEndpointSource | null; credentialEnv: string | null; @@ -287,6 +287,7 @@ export interface ProviderInferenceStateOptions { gatewayName: string; reservationSessionId?: string; }, + options?: { requireAbsent?: boolean }, ): boolean; registryUpdateSandbox(sandboxName: string, updates: { nimContainer?: string | null }): void; checkpointSandboxIdentity(sandboxName: string, agent: Agent): Promise; diff --git a/src/lib/onboard/machine/handlers/sandbox-apf.test.ts b/src/lib/onboard/machine/handlers/sandbox-apf.test.ts index d23ac034f94..7814fb2b638 100644 --- a/src/lib/onboard/machine/handlers/sandbox-apf.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-apf.test.ts @@ -21,7 +21,7 @@ describe("APF sandbox create selection", () => { expect(apfCreateFingerprintFields(true)).toEqual(["apf-interceptor"]); }); - it("defers fresh-create credentials behind the verified APF effect callback (#9833)", async () => { + it("defers providerless creation effects behind the verified APF callback (#9833)", async () => { const session = createSession({ apfInterceptorRequested: true }); const { deps, calls } = createDeps( { @@ -30,10 +30,6 @@ describe("APF sandbox create selection", () => { gatewayName: "nemoclaw", pendingRouteReservation: true, reservationSessionId: session.sessionId, - provider: "provider", - model: "model", - endpointUrl: null, - preferredInferenceApi: "openai-completions", webSearchEnabled: false, toolDisclosure: "progressive", fromDockerfile: null, @@ -51,6 +47,9 @@ describe("APF sandbox create selection", () => { ...baseOptions(deps, session), fresh: true, apfInterceptorRequested: true, + model: "", + provider: "", + preferredInferenceApi: null, }); expect(calls.stageCredentialProviders).not.toHaveBeenCalled(); @@ -72,6 +71,43 @@ describe("APF sandbox create selection", () => { ); }); + it("rejects a resolved APF provider plan before sandbox or provider effects (#9833)", async () => { + const session = createSession({ apfInterceptorRequested: true }); + const { deps, calls } = createDeps( + { + getSandboxRegistryEntry: (name) => ({ + name, + gatewayName: "nemoclaw", + pendingRouteReservation: true, + reservationSessionId: session.sessionId, + }), + getSandboxRecreateObservation: () => ({ + state: "missing" as const, + liveIdentityFingerprint: null, + }), + planRegisteredExtraProviders: () => ({ + extraProviders: [], + staleExtraProviders: ["stale-provider"], + }), + }, + session, + ); + + await expect( + handleSandboxState({ + ...baseOptions(deps, session), + fresh: true, + apfInterceptorRequested: true, + model: "", + provider: "", + preferredInferenceApi: null, + }), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(calls.stageCredentialProviders).not.toHaveBeenCalled(); + expect(calls.createSandbox).not.toHaveBeenCalled(); + }); + it("rejects registered sandbox adoption before credential staging (#9833)", async () => { const session = createSession({ apfInterceptorRequested: true }); const { deps, calls } = createDeps({}, session); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index f570136ce2a..425ca8e1897 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -104,7 +104,7 @@ import { import { withSandboxPhaseTrace } from "../../tracing"; import type { InferenceRouteReservationAuthority, SandboxCreateIntent } from "../../types"; -import { branchTo, type OnboardStateTransitionResult } from "../result"; +import { branchTo, completeOnboardMachine, type OnboardStateResult } from "../result"; import * as dcodeResume from "./sandbox-dcode-resume"; import { hasMessagingCredentialDrift, @@ -290,7 +290,7 @@ export interface SandboxStateOptions< ): Promise; startRecordedStep( stepName: string, - updates: { sandboxName?: string | null; provider: string; model: string }, + updates: { sandboxName?: string | null; provider?: string | null; model?: string | null }, ): Promise; getRecordedMessagingChannelsForResume( resume: boolean, @@ -408,7 +408,7 @@ export interface SandboxStateResult { selectedMessagingChannels: string[]; webSearchSupported: boolean; session: Session | null; - stateResult: OnboardStateTransitionResult; + stateResult: OnboardStateResult; } interface SandboxStepState { @@ -1147,6 +1147,28 @@ class SandboxStateFlow< ` Error: sandbox route reservation '${sandboxName ?? "unknown"}' disappeared while onboarding was in progress. Retry onboarding.`, ); } + if (this.options.apfInterceptorRequested === true) { + const reservationSessionId = this.options.session?.sessionId; + const isExactProviderlessReservation = + typeof reservationSessionId === "string" && + reservationSessionId.length > 0 && + targetEntry.pendingRouteReservation === true && + targetEntry.reservationSessionId === reservationSessionId && + resolveSandboxGatewayName(targetEntry) === this.options.gatewayName && + (targetEntry.provider ?? null) === null && + (targetEntry.model ?? null) === null && + (targetEntry.endpointUrl ?? null) === null && + (targetEntry.endpointSource ?? null) === null && + (targetEntry.credentialEnv ?? null) === null && + (targetEntry.preferredInferenceApi ?? null) === null && + (targetEntry.compatibleEndpointReasoning ?? null) === null && + (targetEntry.compatibleEndpointReasoningEffort ?? null) === null && + (targetEntry.nimContainer ?? null) === null; + if (isExactProviderlessReservation) return; + this.failGatewayRouteCheck( + ` Error: providerless APF sandbox '${sandboxName}' lost its exact route reservation while onboarding was in progress. Retry onboarding.`, + ); + } if (getSandboxEntryInference(targetEntry).kind !== "configured") { this.failGatewayRouteCheck( ` Error: sandbox '${sandboxName}' has incomplete route metadata, so its shared-gateway compatibility cannot be proven. Remove and re-onboard that sandbox.`, @@ -1864,6 +1886,23 @@ class SandboxStateFlow< }; } + private assertProviderlessApfCreatePlan(createIntent: CompleteSandboxCreateIntent): void { + if (this.options.apfInterceptorRequested !== true) return; + const resolved = createIntent.resolved; + const hasProviderPlan = + Boolean(resolved.inferenceProvider?.trim()) || + resolved.activeMessagingChannels.length > 0 || + resolved.messagingProviderRequests.length > 0 || + resolved.reusableMessagingProviders.length > 0 || + resolved.extraProviders.length > 0 || + resolved.staleExtraProviders.length > 0 || + resolved.hermesToolGateways.length > 0; + if (!hasProviderPlan) return; + throw new Error( + "APF interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + ); + } + private assertApfFreshCreate(sandboxName: string, decision: SandboxCreationDecision): void { if (this.options.apfInterceptorRequested !== true) return; if (this.options.resume || this.options.recreateSandbox(false) || decision.kind !== "create") { @@ -1872,7 +1911,19 @@ class SandboxStateFlow< ); } const registered = this.deps.getSandboxRegistryEntry(sandboxName); - if (registered && registered.pendingRouteReservation !== true) { + const sessionId = this.options.session?.sessionId; + const ownsProviderlessReservation = + registered?.pendingRouteReservation === true && + typeof sessionId === "string" && + registered.reservationSessionId === sessionId && + registered.gatewayName === this.options.gatewayName && + registered.provider == null && + registered.model == null && + registered.endpointUrl == null && + registered.endpointSource == null && + registered.credentialEnv == null && + registered.preferredInferenceApi == null; + if (registered && !ownsProviderlessReservation) { throw new Error( `APF interceptor selection cannot adopt registered sandbox '${sandboxName}'. Choose a new sandbox name.`, ); @@ -2153,6 +2204,11 @@ class SandboxStateFlow< resourceProfile, effectiveHermesToolGateways, ); + this.assertProviderlessApfCreatePlan(createIntent); + const providerlessApf = + this.options.apfInterceptorRequested === true && + this.options.provider.trim().length === 0 && + this.options.model.trim().length === 0; this.assertGatewayRouteCompatible(requestedSandboxName); this.assertCheckpointBindingsStillLive(state); this.assertCheckpointCreateInputsStillMatch( @@ -2162,8 +2218,7 @@ class SandboxStateFlow< ); await this.deps.startRecordedStep("sandbox", { sandboxName: requestedSandboxName, - provider: this.options.provider, - model: this.options.model, + ...(providerlessApf ? {} : { provider: this.options.provider, model: this.options.model }), }); this.deps.updateSession((current) => { current.messagingPlan = messagingPlan; @@ -2257,13 +2312,17 @@ class SandboxStateFlow< // Preserve the validated route and credential env-var name, never a credential value. revalidatePolicyRequirements("register the created sandbox"); this.deps.updateSandboxRegistry(sandboxName, { - model: this.options.model, - provider: this.options.provider, - endpointUrl: this.options.endpointUrl, - endpointSource: createIntent.endpointSource ?? null, - credentialEnv: this.options.credentialEnv, - nimContainer: this.options.nimContainer, - preferredInferenceApi: this.options.preferredInferenceApi, + ...(providerlessApf + ? {} + : { + model: this.options.model, + provider: this.options.provider, + endpointUrl: this.options.endpointUrl, + endpointSource: createIntent.endpointSource ?? null, + credentialEnv: this.options.credentialEnv, + nimContainer: this.options.nimContainer, + preferredInferenceApi: this.options.preferredInferenceApi, + }), ...agentRegistryFields, }); // Finalization marks the default so a cancelled onboarding cannot leave a @@ -2273,8 +2332,9 @@ class SandboxStateFlow< "sandbox", this.deps.toSessionUpdates({ sandboxName, - provider: this.options.provider, - model: this.options.model, + ...(providerlessApf + ? {} + : { provider: this.options.provider, model: this.options.model }), nimContainer: this.options.nimContainer, webSearchConfig: state.webSearchConfig, messagingPlan, @@ -2479,6 +2539,11 @@ class SandboxStateFlow< " Tavily Search replaces Hermes managed Web search/extract and removes the conflicting nous-web selection.", ); } + const metadata = { + state: "sandbox", + sandboxName: state.sandboxName, + agent: (this.options.agent as { name?: string } | null)?.name ?? "openclaw", + }; return { sandboxName: state.sandboxName, webSearchConfig: state.webSearchConfig, @@ -2487,13 +2552,10 @@ class SandboxStateFlow< selectedMessagingChannels: state.selectedMessagingChannels, webSearchSupported: state.webSearchSupported, session: state.session, - stateResult: branchTo(this.options.agent ? "agent_setup" : "openclaw", { - metadata: { - state: "sandbox", - sandboxName: state.sandboxName, - agent: (this.options.agent as { name?: string } | null)?.name ?? "openclaw", - }, - }), + stateResult: + this.options.apfInterceptorRequested === true + ? completeOnboardMachine({}, metadata) + : branchTo(this.options.agent ? "agent_setup" : "openclaw", { metadata }), }; } diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index 75c0288e435..72ee82fe656 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -36,10 +36,7 @@ import { type StagedManagedWorkloadReplacement, } from "./managed-workload/rebuild/contract"; import { createManagedWorkloadReplacementRollback } from "./managed-workload/rebuild/rollback"; -import { - type ManagedWorkloadRebuildTransactionDependencies, - runManagedWorkloadRebuildTransaction, -} from "./managed-workload/rebuild/transaction"; +import { runManagedWorkloadRebuildTransaction } from "./managed-workload/rebuild/transaction"; import type { RuntimeProviderBundle } from "./runtime-provider/contract"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION } from "./runtime-provider/contract"; import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; @@ -50,71 +47,6 @@ const PROVIDERS = ["docker", "mxc"] as const; const PLATFORMS = ["linux/amd64", "linux/arm64"] as const; const OLD_RELEASE = "v0.0.99"; const NEW_RELEASE = "v0.0.100"; -const OLD_GENERATION = "00000000-0000-4000-8000-000000000001"; -const NEW_GENERATION = "00000000-0000-4000-8000-000000000002"; -const OLD_FINGERPRINT = "a".repeat(64); -const NEW_FINGERPRINT = "b".repeat(64); - -type SandboxMutationLock = NonNullable< - ManagedWorkloadRebuildTransactionDependencies["withSandboxMutationLock"] ->; - -const immediateSandboxMutationLock: SandboxMutationLock = async (_sandboxName, operation) => - await operation(); - -function serializedSandboxMutationLock(): SandboxMutationLock { - let tail = Promise.resolve(); - return async (_sandboxName, operation) => { - const previous = tail; - let release = (): void => {}; - tail = new Promise((resolve) => { - release = resolve; - }); - await previous; - try { - return await operation(); - } finally { - release(); - } - }; -} - -const replacementPolicyAuthority = { - gatewayName: "nemoclaw", - gatewayPort: 8080, - policySourcePath: "/tmp/replacement-policy.yaml", - route: "none" as const, - plannedAuthority: "nemoclaw-managed" as const, -}; - -function replacementAuthorityDependencies( - overrides: Partial< - NonNullable - > = {}, -): NonNullable { - return { - inspectSandboxIdentity: vi.fn(() => NEW_FINGERPRINT), - verifyCreatedPolicy: vi.fn( - (input) => - ({ - policyAuthority: "nemoclaw-managed", - observedPolicyAuthority: "owner-unknown", - policyCreationReceipt: { - schemaVersion: 1, - origin: "sandbox-create", - gatewayName: input.gatewayName, - gatewayPort: input.gatewayPort, - sandboxName: input.sandboxName, - lifecycleGeneration: input.lifecycleGeneration, - sandboxIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, - policyHash: "policy-new", - policyVersion: 2, - }, - }) as const, - ), - ...overrides, - }; -} function raiseInjectedFailure(message: string): never { throw new Error(message); @@ -199,8 +131,8 @@ function previousEntry( model: "nvidia/nemotron", imageTag: workload.reference, workload, - lifecycleGeneration: OLD_GENERATION, - lifecycleLiveIdentityFingerprint: OLD_FINGERPRINT, + lifecycleGeneration: "generation-old", + lifecycleLiveIdentityFingerprint: "fingerprint-old", gatewayName: "nemoclaw", gatewayPort: 8080, }; @@ -334,7 +266,7 @@ function operationsHarness( providerId: string, events: string[], failAt: FailurePhase = null, - previousLiveIdentityFingerprint = OLD_FINGERPRINT, + previousLiveIdentityFingerprint = "fingerprint-old", ): ManagedWorkloadRebuildProviderOperations { const bound = { schemaVersion: 1 as const, @@ -351,8 +283,8 @@ function operationsHarness( ...bound, previousRuntimeHandle: prepared.previousRuntimeHandle, stagingHandle: "runtime-new-staged-exact", - lifecycleGeneration: NEW_GENERATION, - liveIdentityFingerprint: NEW_FINGERPRINT, + lifecycleGeneration: "generation-new", + liveIdentityFingerprint: "fingerprint-new", }; const ready: ReadyManagedWorkloadReplacement = { ...staged, @@ -415,15 +347,6 @@ function transactionHarness( failAt: FailurePhase = null, platform: (typeof PLATFORMS)[number] = "linux/amd64", previousEntryOverrides: Partial = {}, - replacementOptions: { - readonly policy?: Parameters< - typeof runManagedWorkloadRebuildTransaction - >[0]["replacementPolicyAuthority"]; - readonly dependencies?: NonNullable< - ManagedWorkloadRebuildTransactionDependencies["replacementAuthority"] - >; - readonly withSandboxMutationLock?: SandboxMutationLock | null; - } = {}, ) { const events: string[] = []; const oldEntry = { ...previousEntry(agent, providerId, platform), ...previousEntryOverrides }; @@ -487,7 +410,6 @@ function transactionHarness( provider: bundle(providerId), handoff: handoff(agent, providerId, platform), operations, - replacementPolicyAuthority: replacementOptions.policy ?? replacementPolicyAuthority, replacementMetadata: { model: "nvidia/nemotron-new" }, transactionId: "transaction-1", }, @@ -506,12 +428,6 @@ function transactionHarness( return structuredClone(currentEntry); }, commitAuthority, - replacementAuthority: - replacementOptions.dependencies ?? replacementAuthorityDependencies(), - withSandboxMutationLock: - replacementOptions.withSandboxMutationLock === null - ? undefined - : (replacementOptions.withSandboxMutationLock ?? immediateSandboxMutationLock), }, ), }; @@ -737,8 +653,8 @@ describe("managed workload rebuild transaction", () => { openshellDriver: provider, model: "nvidia/nemotron-new", fromDockerfile: null, - lifecycleGeneration: NEW_GENERATION, - lifecycleLiveIdentityFingerprint: NEW_FINGERPRINT, + lifecycleGeneration: "generation-new", + lifecycleLiveIdentityFingerprint: "fingerprint-new", workload: { kind: "managed-image", platform, @@ -783,7 +699,7 @@ describe("managed workload rebuild transaction", () => { ); }); - it("publishes a replacement-bound receipt without carrying the previous receipt (#9833)", async () => { + it("publishes a replacement without carrying the previous policy receipt (#9833)", async () => { const lifecycleGeneration = "00000000-0000-4000-8000-000000000001"; const sandboxIdentityFingerprint = "a".repeat(64); const harness = transactionHarness("openclaw", "mxc", null, "linux/amd64", { @@ -805,229 +721,9 @@ describe("managed workload rebuild transaction", () => { const result = await harness.run(); - expect(result.entry.lifecycleGeneration).toBe(NEW_GENERATION); - expect(result.entry).toMatchObject({ - policyAuthority: "nemoclaw-managed", - policyCreationReceipt: { - gatewayName: "nemoclaw", - gatewayPort: 8080, - lifecycleGeneration: NEW_GENERATION, - sandboxIdentityFingerprint: NEW_FINGERPRINT, - policyHash: "policy-new", - policyVersion: 2, - }, - }); - expect(result.entry.policyCreationReceipt).not.toEqual(harness.oldEntry.policyCreationReceipt); - }); - - it("publishes verified global authority without a NemoClaw receipt (#9833)", async () => { - const verifyCreatedPolicy = vi.fn(() => ({ - policyAuthority: "externally-managed" as const, - policyCreationReceipt: null, - observedPolicyAuthority: "externally-managed" as const, - policyIdentity: { hash: "global-policy", activeVersion: 4 }, - })); - const harness = transactionHarness( - "openclaw", - "mxc", - null, - "linux/amd64", - {}, - { - policy: { - ...replacementPolicyAuthority, - plannedAuthority: "externally-managed", - }, - dependencies: replacementAuthorityDependencies({ verifyCreatedPolicy }), - }, - ); - - const result = await harness.run(); - - expect(result.entry).toMatchObject({ - gatewayName: "nemoclaw", - gatewayPort: 8080, - lifecycleGeneration: NEW_GENERATION, - lifecycleLiveIdentityFingerprint: NEW_FINGERPRINT, - policyAuthority: "externally-managed", - }); + expect(result.entry.lifecycleGeneration).toBe("generation-new"); + expect(result.entry).not.toHaveProperty("policyAuthority"); expect(result.entry).not.toHaveProperty("policyCreationReceipt"); - expect(verifyCreatedPolicy).toHaveBeenCalledWith( - expect.objectContaining({ - plannedAuthority: "externally-managed", - lifecycleGeneration: NEW_GENERATION, - lifecycleLiveIdentityFingerprint: NEW_FINGERPRINT, - }), - ); - }); - - it("rolls back and preserves old authority when replacement identity changes during policy proof (#9833)", async () => { - const inspectSandboxIdentity = vi - .fn() - .mockReturnValueOnce(NEW_FINGERPRINT) - .mockReturnValueOnce("c".repeat(64)); - const harness = transactionHarness( - "openclaw", - "mxc", - null, - "linux/amd64", - {}, - { - dependencies: replacementAuthorityDependencies({ inspectSandboxIdentity }), - }, - ); - - await expect(harness.run()).rejects.toMatchObject({ - phase: "registry-commit", - message: expect.stringContaining("identity changed after policy verification"), - }); - - expect(inspectSandboxIdentity).toHaveBeenCalledTimes(2); - expect(harness.currentEntry()).toEqual(harness.oldEntry); - expect(harness.operations.rollback).toHaveBeenCalledOnce(); - expect(harness.events).not.toContain("registry-commit"); - expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); - }); - - it("rolls back and preserves old authority when replacement policy proof fails (#9833)", async () => { - const harness = transactionHarness( - "openclaw", - "mxc", - null, - "linux/amd64", - {}, - { - dependencies: replacementAuthorityDependencies({ - verifyCreatedPolicy: vi.fn(() => { - throw new Error("replacement policy changed"); - }), - }), - }, - ); - - await expect(harness.run()).rejects.toMatchObject({ phase: "registry-commit" }); - - expect(harness.currentEntry()).toEqual(harness.oldEntry); - expect(harness.operations.rollback).toHaveBeenCalledOnce(); - expect(harness.events).not.toContain("registry-commit"); - }); - - it("serializes a same-name replacement requested after final identity verification (#9833)", async () => { - const withSandboxMutationLock = serializedSandboxMutationLock(); - let liveIdentity = NEW_FINGERPRINT; - let sameNameReplacement = Promise.resolve(); - let harness!: ReturnType; - const inspectSandboxIdentity = vi - .fn() - .mockImplementationOnce(() => { - harness.events.push("identity-read:1"); - return liveIdentity; - }) - .mockImplementationOnce(() => { - const observed = liveIdentity; - harness.events.push("identity-read:2"); - harness.events.push("same-name-replacement-requested"); - sameNameReplacement = withSandboxMutationLock("rebuild-openclaw", () => { - liveIdentity = "c".repeat(64); - harness.events.push("same-name-replacement-ran"); - }); - return observed; - }); - harness = transactionHarness( - "openclaw", - "mxc", - null, - "linux/amd64", - {}, - { - dependencies: replacementAuthorityDependencies({ inspectSandboxIdentity }), - withSandboxMutationLock, - }, - ); - - const result = await harness.run(); - await sameNameReplacement; - - expect(result.status).toBe("committed"); - expect(inspectSandboxIdentity).toHaveBeenCalledTimes(2); - expect(harness.events.indexOf("registry-commit")).toBeLessThan( - harness.events.indexOf("same-name-replacement-ran"), - ); - expect(harness.events).toContain("same-name-replacement-requested"); - expect(liveIdentity).toBe("c".repeat(64)); - }); - - it("keeps old authority when the replacement publication lock fails (#9833)", async () => { - const harness = transactionHarness( - "openclaw", - "mxc", - null, - "linux/amd64", - {}, - { - withSandboxMutationLock: async () => { - throw new Error("sandbox mutation lock unavailable"); - }, - }, - ); - - await expect(harness.run()).rejects.toMatchObject({ - phase: "registry-commit", - message: expect.stringContaining("sandbox mutation lock could not protect"), - }); - - expect(harness.currentEntry()).toEqual(harness.oldEntry); - expect(harness.operations.rollback).toHaveBeenCalledOnce(); - expect(harness.events).not.toContain("registry-commit"); - expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); - }); - - it("keeps old authority when replacement publication has no sandbox lock (#9833)", async () => { - const harness = transactionHarness( - "openclaw", - "mxc", - null, - "linux/amd64", - {}, - { - withSandboxMutationLock: null, - }, - ); - - await expect(harness.run()).rejects.toMatchObject({ - phase: "registry-commit", - message: expect.stringContaining("requires the sandbox mutation lock"), - }); - - expect(harness.currentEntry()).toEqual(harness.oldEntry); - expect(harness.operations.rollback).toHaveBeenCalledOnce(); - expect(harness.events).not.toContain("registry-commit"); - expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); - }); - - it("does not roll back publication when the sandbox lock release fails (#9833)", async () => { - const harness = transactionHarness( - "openclaw", - "mxc", - null, - "linux/amd64", - {}, - { - withSandboxMutationLock: async (_sandboxName, operation) => { - await operation(); - throw new Error("sandbox mutation lock release failed"); - }, - }, - ); - - await expect(harness.run()).rejects.toMatchObject({ - name: "ManagedWorkloadRebuildIndeterminatePublicationError", - recoveryTask: { operation: "reconcile-publication" }, - }); - - expect(harness.currentEntry().lifecycleGeneration).toBe(NEW_GENERATION); - expect(harness.operations.rollback).not.toHaveBeenCalled(); - expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); }); it("rolls back a not-ready replacement by exact staged handle", async () => { @@ -1043,7 +739,7 @@ describe("managed workload rebuild transaction", () => { "readiness", "rollback:runtime-new-staged-exact", ]); - expect(harness.currentEntry().lifecycleGeneration).toBe(OLD_GENERATION); + expect(harness.currentEntry().lifecycleGeneration).toBe("generation-old"); }); it.each(INVALID_PROVIDER_ARTIFACT_CASES)( @@ -1086,7 +782,6 @@ describe("managed workload rebuild transaction", () => { provider: bundle("mxc"), handoff: handoff("openclaw", "mxc"), operations, - replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(currentEntry) }, @@ -1151,20 +846,9 @@ describe("managed workload rebuild transaction", () => { operation: "retire-previous", previousRuntimeHandle: "runtime-old-exact", stagingHandle: "runtime-new-staged-exact", - replacement: { - gatewayName: "nemoclaw", - gatewayPort: 8080, - policyRegistration: { - policyAuthority: "nemoclaw-managed", - policyCreationReceipt: { - lifecycleGeneration: NEW_GENERATION, - sandboxIdentityFingerprint: NEW_FINGERPRINT, - }, - }, - }, }, }); - expect(harness.currentEntry().lifecycleGeneration).toBe(NEW_GENERATION); + expect(harness.currentEntry().lifecycleGeneration).toBe("generation-new"); expect(harness.events.at(-1)).toBe("retire:runtime-old-exact"); expect(harness.operations.rollback).not.toHaveBeenCalled(); }); @@ -1182,7 +866,7 @@ describe("managed workload rebuild transaction", () => { expect(result).toMatchObject({ status: "committed", entry: { - lifecycleGeneration: NEW_GENERATION, + lifecycleGeneration: "generation-new", workload: { platform: "linux/arm64" }, }, }); @@ -1209,7 +893,7 @@ describe("managed workload rebuild transaction", () => { }, }); - expect(harness.currentEntry().lifecycleGeneration).toBe(NEW_GENERATION); + expect(harness.currentEntry().lifecycleGeneration).toBe("generation-new"); expect(harness.operations.rollback).not.toHaveBeenCalled(); expect(harness.operations.retirePrevious).not.toHaveBeenCalled(); }); @@ -1243,8 +927,8 @@ describe("managed workload rebuild transaction", () => { transactionId: "transaction-1", previousRuntimeHandle: "runtime-old-exact", stagingHandle: "runtime-new-staged-exact", - lifecycleGeneration: NEW_GENERATION, - liveIdentityFingerprint: NEW_FINGERPRINT, + lifecycleGeneration: "generation-new", + liveIdentityFingerprint: "fingerprint-new", }; const rollback = createManagedWorkloadReplacementRollback(plan, staged, providerOperations); @@ -1280,7 +964,6 @@ describe("managed workload rebuild transaction", () => { }, }, operations, - replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, @@ -1310,7 +993,6 @@ describe("managed workload rebuild transaction", () => { }, }, operations, - replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, @@ -1336,7 +1018,6 @@ describe("managed workload rebuild transaction", () => { replacementProfile: hermesHandoff.replacementProfile, }, operations, - replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, @@ -1368,7 +1049,6 @@ describe("managed workload rebuild transaction", () => { provider: bundle("mxc"), handoff: handoff("openclaw", "mxc"), operations, - replacementPolicyAuthority, transactionId: "transaction-1", }, { @@ -1377,8 +1057,6 @@ describe("managed workload rebuild transaction", () => { status: "committed", entry: structuredClone(replacement), }), - replacementAuthority: replacementAuthorityDependencies(), - withSandboxMutationLock: immediateSandboxMutationLock, }, ); @@ -1396,7 +1074,6 @@ describe("managed workload rebuild transaction", () => { provider: bundle("mxc"), handoff: handoff("openclaw", "mxc"), operations, - replacementPolicyAuthority, transactionId: "transaction-1", }, { getSandbox: () => structuredClone(oldEntry) }, diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts index 73ddb90eacc..1d6082dc89a 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -173,6 +173,17 @@ describe("managed workload onboard orchestration", () => { ).toBe(false); }); + it("does not activate stock managed images for providerless APF creation (#9833)", () => { + expect( + shouldActivateStockManagedRuntime({ + portableLifecycle: false, + hermesPortableLifecycle: false, + apfInterceptorRequested: true, + agentName: "openclaw", + }), + ).toBe(false); + }); + it("rejects an unavailable catalog for stock managed-image onboarding", async () => { const { runtime } = createFreshOnboardingRuntime( {}, diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index 07f71433d58..0be8bcbcfbc 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -148,9 +148,11 @@ export interface ManagedWorkloadOnboardRuntime { export function shouldActivateStockManagedRuntime(input: { readonly portableLifecycle: boolean; readonly hermesPortableLifecycle: boolean; + readonly apfInterceptorRequested?: boolean; readonly agentName: string; }): boolean { return ( + input.apfInterceptorRequested !== true && !input.portableLifecycle && !input.hermesPortableLifecycle && isShippedManagedImageAgent(input.agentName) diff --git a/src/lib/onboard/managed-workload/rebuild/commit.ts b/src/lib/onboard/managed-workload/rebuild/commit.ts index c758cf60591..c555dadffa7 100644 --- a/src/lib/onboard/managed-workload/rebuild/commit.ts +++ b/src/lib/onboard/managed-workload/rebuild/commit.ts @@ -8,7 +8,6 @@ import { sandboxRebuildReplacementMatchesEntry, } from "../../../state/registry/rebuild-authority"; import type { SandboxEntry } from "../../../state/registry/types"; -import type { VerifiedSandboxPolicyBoundary } from "../../types"; import type { ManagedWorkloadRebuildPlan, ReboundManagedWorkloadReplacement } from "./contract"; import { ManagedWorkloadRebuildIndeterminatePublicationError, @@ -26,7 +25,6 @@ export type ReadSandboxRebuildEntry = (sandboxName: string) => SandboxEntry | nu function reconcileAmbiguousPublication( plan: ManagedWorkloadRebuildPlan, replacement: ReboundManagedWorkloadReplacement, - policyBoundary: VerifiedSandboxPolicyBoundary, candidate: SandboxEntry, publicationError: unknown, readSandbox?: ReadSandboxRebuildEntry, @@ -34,12 +32,7 @@ function reconcileAmbiguousPublication( if (!readSandbox) { throw new ManagedWorkloadRebuildIndeterminatePublicationError( "publication failed without an authoritative reconciliation read", - createManagedWorkloadRebuildRecoveryTask( - plan, - replacement, - policyBoundary, - "reconcile-publication", - ), + createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), { cause: publicationError }, ); } @@ -49,12 +42,7 @@ function reconcileAmbiguousPublication( } catch (reconciliationError) { throw new ManagedWorkloadRebuildIndeterminatePublicationError( "publication and authoritative reconciliation both failed", - createManagedWorkloadRebuildRecoveryTask( - plan, - replacement, - policyBoundary, - "reconcile-publication", - ), + createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), { cause: new AggregateError( [publicationError, reconciliationError], @@ -75,12 +63,7 @@ function reconcileAmbiguousPublication( } throw new ManagedWorkloadRebuildIndeterminatePublicationError( "publication could not be reconciled to the replacement or exact old authority", - createManagedWorkloadRebuildRecoveryTask( - plan, - replacement, - policyBoundary, - "reconcile-publication", - ), + createManagedWorkloadRebuildRecoveryTask(plan, replacement, "reconcile-publication"), { cause: publicationError }, ); } @@ -89,18 +72,7 @@ export function materializeManagedWorkloadReplacementEntry( previousEntry: SandboxEntry, plan: ManagedWorkloadRebuildPlan, replacement: ReboundManagedWorkloadReplacement, - policyBoundary: VerifiedSandboxPolicyBoundary, ): SandboxEntry { - if ( - policyBoundary.sandboxName !== plan.sandboxName || - policyBoundary.lifecycleGeneration !== replacement.lifecycleGeneration || - policyBoundary.lifecycleLiveIdentityFingerprint !== replacement.liveIdentityFingerprint - ) { - throw new ManagedWorkloadRebuildTransactionError( - "registry-commit", - "the verified replacement policy boundary does not match the replacement lifecycle", - ); - } const { policyAuthority: _previousPolicyAuthority, policyCreationReceipt: _previousPolicyCreationReceipt, @@ -112,7 +84,6 @@ export function materializeManagedWorkloadReplacementEntry( name: plan.sandboxName, pendingRouteReservation: undefined, reservationSessionId: undefined, - pendingPolicyVerification: undefined, openshellDriver: plan.providerId, agent: plan.agent, fromDockerfile: null, @@ -120,12 +91,6 @@ export function materializeManagedWorkloadReplacementEntry( workload: plan.replacementReceipt, lifecycleGeneration: replacement.lifecycleGeneration, lifecycleLiveIdentityFingerprint: replacement.liveIdentityFingerprint, - gatewayName: policyBoundary.gatewayName, - gatewayPort: policyBoundary.gatewayPort, - policyAuthority: policyBoundary.registration.policyAuthority, - ...(policyBoundary.registration.policyAuthority === "nemoclaw-managed" - ? { policyCreationReceipt: policyBoundary.registration.policyCreationReceipt } - : {}), }); } @@ -133,28 +98,15 @@ export function commitManagedWorkloadReplacement( previousEntry: SandboxEntry, plan: ManagedWorkloadRebuildPlan, replacement: ReboundManagedWorkloadReplacement, - policyBoundary: VerifiedSandboxPolicyBoundary, commit: CommitSandboxRebuildAuthority = compareAndSwapSandboxRebuildAuthority, readSandbox?: ReadSandboxRebuildEntry, ): SandboxEntry { - const candidate = materializeManagedWorkloadReplacementEntry( - previousEntry, - plan, - replacement, - policyBoundary, - ); + const candidate = materializeManagedWorkloadReplacementEntry(previousEntry, plan, replacement); let result: SandboxRebuildAuthoritySwapResult; try { result = commit(plan.previousAuthority, candidate); } catch (error) { - return reconcileAmbiguousPublication( - plan, - replacement, - policyBoundary, - candidate, - error, - readSandbox, - ); + return reconcileAmbiguousPublication(plan, replacement, candidate, error, readSandbox); } if (result.status !== "committed") { throw new ManagedWorkloadRebuildTransactionError( @@ -166,7 +118,6 @@ export function commitManagedWorkloadReplacement( return reconcileAmbiguousPublication( plan, replacement, - policyBoundary, candidate, new Error("the commit adapter returned a mismatched committed entry"), readSandbox, diff --git a/src/lib/onboard/managed-workload/rebuild/contract.ts b/src/lib/onboard/managed-workload/rebuild/contract.ts index 8a669832cae..fc977453daf 100644 --- a/src/lib/onboard/managed-workload/rebuild/contract.ts +++ b/src/lib/onboard/managed-workload/rebuild/contract.ts @@ -4,7 +4,6 @@ import type { SandboxRebuildAuthority } from "../../../state/registry/rebuild-authority"; import type { SandboxEntry } from "../../../state/registry/types"; import type { ManagedImageAgent } from "../../managed-image/contract"; -import type { VerifiedSandboxPolicyRegistration } from "../../types"; import type { ManagedWorkloadRebuildHandoff, ManagedWorkloadReceipt } from "../../workload/rebuild"; export type ManagedWorkloadRebuildPhase = @@ -157,9 +156,6 @@ export interface ManagedWorkloadRebuildRecoveryTask { readonly receipt: ManagedWorkloadReceipt; readonly lifecycleGeneration: string; readonly liveIdentityFingerprint: string; - readonly gatewayName: string; - readonly gatewayPort: number; - readonly policyRegistration: VerifiedSandboxPolicyRegistration; }; } diff --git a/src/lib/onboard/managed-workload/rebuild/plan.ts b/src/lib/onboard/managed-workload/rebuild/plan.ts index b66547fba9b..836d2e7d19e 100644 --- a/src/lib/onboard/managed-workload/rebuild/plan.ts +++ b/src/lib/onboard/managed-workload/rebuild/plan.ts @@ -22,7 +22,6 @@ import { ManagedWorkloadRebuildTransactionError } from "./contract"; const PROTECTED_REBUILD_METADATA_FIELDS = new Set([ "name", "pendingRouteReservation", - "pendingPolicyVerification", "reservationSessionId", "openshellDriver", "fromDockerfile", diff --git a/src/lib/onboard/managed-workload/rebuild/recovery.ts b/src/lib/onboard/managed-workload/rebuild/recovery.ts index 3d0eea68b53..950597fb451 100644 --- a/src/lib/onboard/managed-workload/rebuild/recovery.ts +++ b/src/lib/onboard/managed-workload/rebuild/recovery.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { cloneAndDeepFreeze } from "../../../core/immutable"; -import type { VerifiedSandboxPolicyBoundary } from "../../types"; import type { ManagedWorkloadRebuildPlan, ManagedWorkloadRebuildRecoveryTask, @@ -12,7 +11,6 @@ import type { export function createManagedWorkloadRebuildRecoveryTask( plan: ManagedWorkloadRebuildPlan, replacement: StagedManagedWorkloadReplacement, - policyBoundary: VerifiedSandboxPolicyBoundary, operation: ManagedWorkloadRebuildRecoveryTask["operation"], ): ManagedWorkloadRebuildRecoveryTask { return cloneAndDeepFreeze({ @@ -30,9 +28,6 @@ export function createManagedWorkloadRebuildRecoveryTask( receipt: plan.replacementReceipt, lifecycleGeneration: replacement.lifecycleGeneration, liveIdentityFingerprint: replacement.liveIdentityFingerprint, - gatewayName: policyBoundary.gatewayName, - gatewayPort: policyBoundary.gatewayPort, - policyRegistration: policyBoundary.registration, }, }); } diff --git a/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts b/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts deleted file mode 100644 index 434a6246b17..00000000000 --- a/src/lib/onboard/managed-workload/rebuild/replacement-authority.ts +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { SandboxPolicyAuthority } from "../../../adapters/openshell/policy-authority"; -import { inspectOpenShellSandboxIdentityFingerprint } from "../../../adapters/openshell/policy-authority"; -import { cloneAndDeepFreeze } from "../../../core/immutable"; -import { - verifyCreatedSandboxPolicyRegistration, - type CreatedSandboxPolicyRegistrationInput, -} from "../../sandbox-create/policy-creation-receipt"; -import type { VerifiedSandboxPolicyBoundary } from "../../types"; -import type { ReboundManagedWorkloadReplacement } from "./contract"; -import { ManagedWorkloadRebuildTransactionError } from "./contract"; - -export interface ManagedWorkloadReplacementPolicyAuthorityInput { - readonly gatewayName: string; - readonly gatewayPort: number; - readonly policySourcePath: string; - readonly route: CreatedSandboxPolicyRegistrationInput["route"]; - readonly plannedAuthority: Exclude; -} - -export interface ManagedWorkloadReplacementAuthorityDependencies { - readonly inspectSandboxIdentity?: typeof inspectOpenShellSandboxIdentityFingerprint; - readonly verifyCreatedPolicy?: typeof verifyCreatedSandboxPolicyRegistration; -} - -function verificationFailure(message: string, cause?: unknown): never { - throw new ManagedWorkloadRebuildTransactionError("registry-commit", message, { - ...(cause === undefined ? {} : { cause }), - }); -} - -/** - * Bind replacement publication to one live gateway, sandbox identity, and - * effective policy. The identity-policy-identity sequence runs immediately - * before the registry CAS and never derives authority from the previous row. - */ -export function verifyManagedWorkloadReplacementAuthority(input: { - readonly sandboxName: string; - readonly replacement: ReboundManagedWorkloadReplacement; - readonly policy: ManagedWorkloadReplacementPolicyAuthorityInput; - readonly dependencies?: ManagedWorkloadReplacementAuthorityDependencies; -}): VerifiedSandboxPolicyBoundary { - const inspectIdentity = - input.dependencies?.inspectSandboxIdentity ?? inspectOpenShellSandboxIdentityFingerprint; - const verifyPolicy = - input.dependencies?.verifyCreatedPolicy ?? verifyCreatedSandboxPolicyRegistration; - const identityInput = { - sandboxName: input.sandboxName, - gatewayName: input.policy.gatewayName, - }; - const requireExactIdentity = (timing: "before" | "after"): void => { - let observed: string; - try { - observed = inspectIdentity(identityInput); - } catch (error) { - verificationFailure( - `the replacement sandbox identity could not be verified ${timing} policy verification`, - error, - ); - } - if (observed !== input.replacement.liveIdentityFingerprint) { - verificationFailure(`the replacement sandbox identity changed ${timing} policy verification`); - } - }; - - requireExactIdentity("before"); - let registration: ReturnType; - const policyInput: CreatedSandboxPolicyRegistrationInput = { - sandboxName: input.sandboxName, - gatewayName: input.policy.gatewayName, - gatewayPort: input.policy.gatewayPort, - lifecycleGeneration: input.replacement.lifecycleGeneration, - lifecycleLiveIdentityFingerprint: input.replacement.liveIdentityFingerprint, - policySourcePath: input.policy.policySourcePath, - route: input.policy.route, - plannedAuthority: input.policy.plannedAuthority, - operation: `publish replacement sandbox '${input.sandboxName}'`, - }; - try { - registration = verifyPolicy(policyInput); - } catch (error) { - verificationFailure("the replacement sandbox policy authority could not be verified", error); - } - requireExactIdentity("after"); - - return cloneAndDeepFreeze({ - registration, - sandboxName: input.sandboxName, - gatewayName: input.policy.gatewayName, - gatewayPort: input.policy.gatewayPort, - lifecycleGeneration: input.replacement.lifecycleGeneration, - lifecycleLiveIdentityFingerprint: input.replacement.liveIdentityFingerprint, - route: input.policy.route, - }); -} diff --git a/src/lib/onboard/managed-workload/rebuild/transaction.ts b/src/lib/onboard/managed-workload/rebuild/transaction.ts index 3151b48ff98..96302e24755 100644 --- a/src/lib/onboard/managed-workload/rebuild/transaction.ts +++ b/src/lib/onboard/managed-workload/rebuild/transaction.ts @@ -28,18 +28,12 @@ import { createManagedWorkloadPreparationAbort, createManagedWorkloadReplacementRollback, } from "./rollback"; -import { - type ManagedWorkloadReplacementAuthorityDependencies, - type ManagedWorkloadReplacementPolicyAuthorityInput, - verifyManagedWorkloadReplacementAuthority, -} from "./replacement-authority"; export interface RunManagedWorkloadRebuildTransactionInput { readonly previousEntry: SandboxEntry; readonly provider: RuntimeProviderBundle; readonly handoff: ManagedWorkloadRebuildHandoff; readonly operations: ManagedWorkloadRebuildProviderOperations; - readonly replacementPolicyAuthority: ManagedWorkloadReplacementPolicyAuthorityInput; readonly replacementMetadata?: Readonly>; readonly transactionId?: string; } @@ -47,11 +41,6 @@ export interface RunManagedWorkloadRebuildTransactionInput { export interface ManagedWorkloadRebuildTransactionDependencies { readonly getSandbox?: (sandboxName: string) => SandboxEntry | null; readonly commitAuthority?: CommitSandboxRebuildAuthority; - readonly replacementAuthority?: ManagedWorkloadReplacementAuthorityDependencies; - readonly withSandboxMutationLock?: ( - sandboxName: string, - operation: () => Promise | T, - ) => Promise; } function readSandboxFromRegistry(sandboxName: string): SandboxEntry | null { @@ -81,80 +70,14 @@ async function failAfterCleanup(error: unknown, cleanup: () => Promise): P throw rethrowWithRollback(error, rollbackError); } -async function rejectUnprotectedReplacementPublication(): Promise { - throw new ManagedWorkloadRebuildTransactionError( - "registry-commit", - "replacement publication requires the sandbox mutation lock", - ); -} - -async function publishManagedWorkloadReplacement( - input: RunManagedWorkloadRebuildTransactionInput, - plan: ReturnType, - replacement: Parameters[0]["replacement"], - dependencies: ManagedWorkloadRebuildTransactionDependencies, - readSandbox: (sandboxName: string) => SandboxEntry | null, -): Promise<{ - readonly entry: SandboxEntry; - readonly policyBoundary: ReturnType; -}> { - const withMutationLock = - dependencies.withSandboxMutationLock ?? rejectUnprotectedReplacementPublication; - let published: - | { - readonly entry: SandboxEntry; - readonly policyBoundary: ReturnType; - } - | undefined; - try { - return await withMutationLock(plan.sandboxName, () => { - const policyBoundary = verifyManagedWorkloadReplacementAuthority({ - sandboxName: plan.sandboxName, - replacement, - policy: input.replacementPolicyAuthority, - dependencies: dependencies.replacementAuthority, - }); - const entry = commitManagedWorkloadReplacement( - input.previousEntry, - plan, - replacement, - policyBoundary, - dependencies.commitAuthority, - readSandbox, - ); - published = { entry, policyBoundary }; - return published; - }); - } catch (error) { - if (error instanceof ManagedWorkloadRebuildTransactionError) throw error; - if (published) { - throw new ManagedWorkloadRebuildIndeterminatePublicationError( - "replacement publication completed but sandbox mutation lock release failed", - createManagedWorkloadRebuildRecoveryTask( - plan, - replacement, - published.policyBoundary, - "reconcile-publication", - ), - { cause: error }, - ); - } - throw new ManagedWorkloadRebuildTransactionError( - "registry-commit", - "the sandbox mutation lock could not protect replacement publication", - { cause: error }, - ); - } -} - /** * Execute a dormant, provider-neutral managed rebuild transaction. * * The durable row and provider-owned old runtime remain authoritative through * prepare, create, readiness, state restore, and provider rebind. Only the - * exact final CAS publishes the replacement. The final live verification and - * CAS share the sandbox mutation lock. The exact old runtime handle is retired - * afterward, so no failure can turn a same-name lookup into deletion authority. + * exact final CAS publishes the replacement. The exact old runtime handle is + * retired afterward, so no failure can turn a same-name lookup into deletion + * authority. */ export async function runManagedWorkloadRebuildTransaction( input: RunManagedWorkloadRebuildTransactionInput, @@ -215,11 +138,11 @@ export async function runManagedWorkloadRebuildTransaction( const ready = await requireReadyManagedWorkloadReplacement(plan, staged, input.operations); const restored = await restoreStagedManagedWorkloadState(plan, ready, input.operations); const rebound = await rebindStagedManagedWorkloadProviders(plan, restored, input.operations); - const { entry, policyBoundary } = await publishManagedWorkloadReplacement( - input, + const entry = commitManagedWorkloadReplacement( + input.previousEntry, plan, rebound, - dependencies, + dependencies.commitAuthority, readSandbox, ); try { @@ -231,12 +154,7 @@ export async function runManagedWorkloadRebuildTransaction( entry, previousCleanup: "pending", cleanupError, - recoveryTask: createManagedWorkloadRebuildRecoveryTask( - plan, - rebound, - policyBoundary, - "retire-previous", - ), + recoveryTask: createManagedWorkloadRebuildRecoveryTask(plan, rebound, "retire-previous"), }; } } catch (error) { diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index fd586bdcaef..e39350144f3 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -230,8 +230,8 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< const compensationErrors = cleanupTemporarySources(); const validationDetail = validationError instanceof Error && isPolicyAuthorityRefusalError(validationError) - ? validationError.message - : null; + ? validationError.message + : null; const identityGuidance = exactIdentity ? ` Durable sandbox identity fingerprint: ${exactIdentity}. Use it only to compare the surviving sandbox with the failed create. Do not delete the sandbox by name, even after this comparison. Contact the OpenShell administrator for an identity-bound recovery or removal procedure.` : " OpenShell did not return a durable identity for comparison. Do not delete the sandbox by name. Contact the OpenShell administrator for an identity-bound recovery or removal procedure."; @@ -819,6 +819,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche stockManagedRuntime: managedWorkloadOnboard.shouldActivateStockManagedRuntime({ portableLifecycle: sandboxGpuCreateFlow.resolvePortableLifecycleMode(agent), hermesPortableLifecycle: agentCreateInput.hermesPortableLifecycle, + apfInterceptorRequested: createIntent?.apfInterceptorRequested === true, agentName: requestedAgentName, }), tempManagedRuntimeCatalog, @@ -1484,10 +1485,10 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ).messagingTokenDefs; }, runProviderPreDeleteCleanup: (verifiedPolicyRevalidation) => { - (verifiedPolicyRevalidation ?? - ((operation) => revalidatePolicyAuthority(false, operation)))( - `cleaning up providers for sandbox '${sandboxName}'`, - ); + ( + verifiedPolicyRevalidation ?? + ((operation) => revalidatePolicyAuthority(false, operation)) + )(`cleaning up providers for sandbox '${sandboxName}'`); runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact, @@ -1498,10 +1499,10 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche upsertMessagingProviders(tokenDefs, { ...options, revalidatePolicyRequirements: (operation) => - (options.revalidatePolicyRequirements ?? - ((targetOperation) => revalidatePolicyAuthority(false, targetOperation)))( - operation, - ), + ( + options.revalidatePolicyRequirements ?? + ((targetOperation) => revalidatePolicyAuthority(false, targetOperation)) + )(operation), }), getHermesToolGatewayProviderName: (targetSandbox) => getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), diff --git a/src/lib/state/registry-create-only-reservation.test.ts b/src/lib/state/registry-create-only-reservation.test.ts new file mode 100644 index 00000000000..f5baf8a187b --- /dev/null +++ b/src/lib/state/registry-create-only-reservation.test.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const PROVIDERLESS_ROUTE = { + provider: null, + model: null, + endpointUrl: null, + endpointSource: null, + credentialEnv: null, + preferredInferenceApi: null, + gatewayName: "nemoclaw", + reservationSessionId: "session-apf", +} as const; + +describe("create-only sandbox route reservation", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + it("atomically refuses a reservation when any registry row exists", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-create-reservation-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + registry.registerSandbox({ + name: "alpha", + provider: "nim", + model: "model-a", + endpointUrl: null, + endpointSource: null, + credentialEnv: null, + preferredInferenceApi: null, + agent: "openclaw", + openshellDriver: "docker", + gatewayName: "nemoclaw", + }); + const before = registry.getSandbox("alpha"); + + expect( + registry.reserveSandboxInferenceRoute("alpha", PROVIDERLESS_ROUTE, { + requireAbsent: true, + }), + ).toBe(false); + expect(registry.getSandbox("alpha")).toEqual(before); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it("creates an absent providerless reservation without claiming the default", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-create-reservation-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + + expect( + registry.reserveSandboxInferenceRoute("alpha", PROVIDERLESS_ROUTE, { + requireAbsent: true, + }), + ).toBe(true); + expect(registry.getSandbox("alpha")).toMatchObject({ + name: "alpha", + pendingRouteReservation: true, + reservationSessionId: "session-apf", + provider: null, + model: null, + }); + expect(registry.getDefault()).toBeNull(); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it.each(["session-apf", "session-foreign"])( + "does not reuse an existing %s pending row", + async (existingSessionId) => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-create-reservation-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + registry.reserveSandboxInferenceRoute("alpha", { + ...PROVIDERLESS_ROUTE, + reservationSessionId: existingSessionId, + }); + const before = registry.getSandbox("alpha"); + + expect( + registry.reserveSandboxInferenceRoute("alpha", PROVIDERLESS_ROUTE, { + requireAbsent: true, + }), + ).toBe(false); + expect(registry.getSandbox("alpha")).toEqual(before); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/lib/state/registry-rebuild-authority.test.ts b/src/lib/state/registry-rebuild-authority.test.ts index 7012c36b713..1c4c78d3204 100644 --- a/src/lib/state/registry-rebuild-authority.test.ts +++ b/src/lib/state/registry-rebuild-authority.test.ts @@ -29,8 +29,6 @@ vi.mock("./registry/lock", () => ({ const ENCODED_PROFILE = encodeManagedStartupProfile(managedStartupE2eProfile("openclaw")); const PROFILE_SHA256 = createHash("sha256").update(ENCODED_PROFILE, "utf8").digest("hex"); -const REPLACEMENT_GENERATION = "00000000-0000-4000-8000-000000000002"; -const REPLACEMENT_FINGERPRINT = "b".repeat(64); function receipt(digest: string): Extract { return { @@ -84,21 +82,9 @@ function registry(current: SandboxEntry = entry()): SandboxRegistry { function replacement(): SandboxEntry { const workload = receipt("b"); return { - ...entry(REPLACEMENT_GENERATION, REPLACEMENT_FINGERPRINT), + ...entry("generation-new", "fingerprint-new"), imageTag: workload.reference, workload, - policyAuthority: "nemoclaw-managed", - policyCreationReceipt: { - schemaVersion: 1, - origin: "sandbox-create", - gatewayName: "nemoclaw", - gatewayPort: 8080, - sandboxName: "alpha", - lifecycleGeneration: REPLACEMENT_GENERATION, - sandboxIdentityFingerprint: REPLACEMENT_FINGERPRINT, - policyHash: "replacement-policy", - policyVersion: 2, - }, }; } @@ -163,8 +149,8 @@ describe("sandbox rebuild authority", () => { expect(swapped.result).toMatchObject({ status: "committed", entry: { - lifecycleGeneration: REPLACEMENT_GENERATION, - lifecycleLiveIdentityFingerprint: REPLACEMENT_FINGERPRINT, + lifecycleGeneration: "generation-new", + lifecycleLiveIdentityFingerprint: "fingerprint-new", workload: { reference: receipt("b").reference }, }, }); @@ -216,8 +202,8 @@ describe("sandbox rebuild authority", () => { expect(result).toMatchObject({ status: "committed", entry: { - lifecycleGeneration: REPLACEMENT_GENERATION, - lifecycleLiveIdentityFingerprint: REPLACEMENT_FINGERPRINT, + lifecycleGeneration: "generation-new", + lifecycleLiveIdentityFingerprint: "fingerprint-new", }, }); expect(registryPersistence.load).toHaveBeenCalledTimes(2); @@ -257,14 +243,9 @@ describe("sandbox rebuild authority", () => { sandboxRebuildReplacementMatchesEntry(expected, { ...expected, model: "updated-after-publication", - }), - ).toBe(true); - expect( - sandboxRebuildReplacementMatchesEntry(expected, { - ...expected, gatewayPort: 9090, }), - ).toBe(false); + ).toBe(true); expect( sandboxRebuildReplacementMatchesEntry(expected, { ...expected, @@ -298,24 +279,6 @@ describe("sandbox rebuild authority", () => { imageTag: receipt("a").reference, }), ], - ["gateway", (candidate: SandboxEntry) => ({ ...candidate, gatewayPort: 9090 })], - [ - "policy authority", - (candidate: SandboxEntry) => ({ - ...candidate, - policyAuthority: "externally-managed" as const, - }), - ], - [ - "policy receipt", - (candidate: SandboxEntry) => ({ - ...candidate, - policyCreationReceipt: { - ...candidate.policyCreationReceipt!, - sandboxIdentityFingerprint: "c".repeat(64), - }, - }), - ], ] as const)("rejects replacement %s drift before CAS", (_label, mutate) => { const before = registry(); const authority = captureSandboxRebuildAuthority(before.sandboxes.alpha!, "docker"); diff --git a/src/lib/state/registry-route-reservation.test.ts b/src/lib/state/registry-route-reservation.test.ts index 3ab5e63053c..6017a7e9779 100644 --- a/src/lib/state/registry-route-reservation.test.ts +++ b/src/lib/state/registry-route-reservation.test.ts @@ -336,9 +336,7 @@ describe("sandbox inference route reservation", () => { "preferredInferenceApi", ] as const; expect( - routeKeys.filter( - (key) => reconstructedSelection[key] !== reservedSelection[key], - ), + routeKeys.filter((key) => reconstructedSelection[key] !== reservedSelection[key]), ).toEqual(["endpointSource"]); const registered = registerCreatedSandbox({ diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index e935960527c..da1d5eb6c94 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -219,8 +219,7 @@ function assertPendingPolicyVerificationMatchesRegistration( checkpoint.gatewayName === requestedEntry.gatewayName && checkpoint.gatewayPort === requestedEntry.gatewayPort && checkpoint.lifecycleGeneration === requestedEntry.lifecycleGeneration && - checkpoint.sandboxIdentityFingerprint === - requestedEntry.lifecycleLiveIdentityFingerprint && + checkpoint.sandboxIdentityFingerprint === requestedEntry.lifecycleLiveIdentityFingerprint && checkpoint.policyAuthority === requestedEntry.policyAuthority && reservation.authority.sandboxName === requestedEntry.name && reservation.authority.gatewayName === requestedEntry.gatewayName && @@ -647,6 +646,11 @@ type SandboxInferenceRouteReservation = Pick< hostLocalInferenceProvenance?: SandboxEntry["hostLocalInferenceProvenance"]; }; +interface SandboxInferenceRouteReservationOptions { + /** Refuse instead of changing any existing registry row. */ + requireAbsent?: boolean; +} + /** * Persist a route dependency before releasing the shared-gateway mutation * lock. A newly reserved row deliberately does not claim the default sandbox; @@ -655,10 +659,12 @@ type SandboxInferenceRouteReservation = Pick< export function reserveSandboxInferenceRoute( name: string, route: SandboxInferenceRouteReservation, + options: SandboxInferenceRouteReservationOptions = {}, ): boolean { return withLock(() => { const data = load(); const existing = data.sandboxes[name]; + if (options.requireAbsent === true && existing !== undefined) return false; const normalized = normalizeInferenceSelection(route); if (existing?.pendingPolicyVerification) { const sameReservation = diff --git a/src/lib/state/registry/rebuild-authority.ts b/src/lib/state/registry/rebuild-authority.ts index d1124abbf51..5a63f903e76 100644 --- a/src/lib/state/registry/rebuild-authority.ts +++ b/src/lib/state/registry/rebuild-authority.ts @@ -12,7 +12,6 @@ import { import { withLock } from "./lock"; import { load, save } from "./persistence"; import type { SandboxEntry, SandboxRegistry, SandboxWorkloadReceipt } from "./types"; -import { cloneSandboxPolicyCreationReceipt } from "../registry-normalization"; import { cloneSandboxWorkloadReceipt } from "./workload"; type ManagedWorkloadReceipt = Extract; @@ -158,8 +157,10 @@ export function captureSandboxRebuildAuthority( /** * Reconcile an ambiguous persistence result using the replacement's exact new - * runtime, gateway, and policy identity. Mutable metadata may change after - * publication, but replacement authority itself must remain exact. + * runtime identity. This deliberately ignores mutable non-authority fields: + * after publication another writer may update those fields, but no different + * rebuild may claim the same generation, live fingerprint, provider, and + * workload receipt. */ export function sandboxRebuildReplacementMatchesEntry( replacement: SandboxEntry, @@ -171,10 +172,6 @@ export function sandboxRebuildReplacementMatchesEntry( (entry.openshellDriver ?? null) === (replacement.openshellDriver ?? null) && entry.lifecycleGeneration === replacement.lifecycleGeneration && entry.lifecycleLiveIdentityFingerprint === replacement.lifecycleLiveIdentityFingerprint && - entry.gatewayName === replacement.gatewayName && - entry.gatewayPort === replacement.gatewayPort && - entry.policyAuthority === replacement.policyAuthority && - isDeepStrictEqual(entry.policyCreationReceipt, replacement.policyCreationReceipt) && entry.imageTag === replacement.imageTag && entry.agent === replacement.agent && isDeepStrictEqual( @@ -227,43 +224,6 @@ function validateReplacement( "replacement must have a distinct live identity fingerprint", ); } - if ( - !boundedIdentity(replacement.gatewayName) || - !Number.isSafeInteger(replacement.gatewayPort) || - Number(replacement.gatewayPort) < 1 || - Number(replacement.gatewayPort) > 65_535 - ) { - throw new SandboxRebuildAuthorityError("replacement gateway authority is missing or invalid"); - } - if (replacement.policyAuthority === "nemoclaw-managed") { - let receipt: ReturnType; - try { - receipt = cloneSandboxPolicyCreationReceipt(replacement.policyCreationReceipt); - } catch (error) { - throw new SandboxRebuildAuthorityError( - `replacement policy creation receipt is invalid: ${error instanceof Error ? error.message : "unknown error"}`, - ); - } - if ( - !receipt || - receipt.sandboxName !== replacement.name || - receipt.gatewayName !== replacement.gatewayName || - receipt.gatewayPort !== replacement.gatewayPort || - receipt.lifecycleGeneration !== replacement.lifecycleGeneration || - receipt.sandboxIdentityFingerprint !== replacement.lifecycleLiveIdentityFingerprint - ) { - throw new SandboxRebuildAuthorityError( - "replacement policy receipt does not match its gateway and sandbox identity", - ); - } - } else if ( - replacement.policyAuthority !== "externally-managed" || - replacement.policyCreationReceipt !== undefined - ) { - throw new SandboxRebuildAuthorityError( - "replacement policy authority is missing or inconsistent", - ); - } const workload = clonedManagedReceipt(replacement.workload); requireReceiptAgent(replacement.agent, workload, "replacement"); if (replacement.imageTag !== workload.reference) { diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index a0bbdcfd3e0..b9ca2fe780d 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -20,20 +20,36 @@ beforeEach(() => { describe("fresh create identity", () => { it.each([ { - title: - "registers managed creation only after owner-scoped identity and policy confirmation (#9833)", + title: "binds ordinary providers at create time before managed registration (#9833)", apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "managed-provider" as const, }, { title: "rejects provider-backed APF creation before sandbox or provider effects (#9833)", apfInterceptorRequested: true, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "provider-refusal" as const, + }, + { + title: + "registers providerless APF only after identity, policy, and checkpoint verification (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "providerless-apf" as const, }, ])( "$title", { timeout: 45000, }, - async ({ apfInterceptorRequested }) => { + async ({ agent, apfInterceptorRequested, expectedOutcome, model, provider }) => { const repoRoot = path.join(import.meta.dirname, "../.."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-ready-")); const fakeBin = path.join(tmpDir, "bin"); @@ -88,6 +104,9 @@ let registeredSandbox = null; let effectivePolicy = {}; const keepAlive = setInterval(() => {}, 1000); const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; +const agent = ${JSON.stringify(agent)}; +const model = ${JSON.stringify(model)}; +const provider = ${JSON.stringify(provider)}; runner.run = (command, opts = {}) => { const cmd = _n(command); _deleted = _deleted || cmd.includes("sandbox delete"); @@ -133,8 +152,8 @@ runner.run = (command, opts = {}) => { }; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", + provider, + model, apfInterceptorRequested, onVerifyCreatedPolicy: (input) => { effectivePolicy = require(${policyMergePath}).parseOpenShellPolicy( @@ -216,7 +235,7 @@ const writePayload = (sandboxName, creationError) => { (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const createArgs = fixtureMocks.sandboxCreateArgsWithVerifiedReservation( - [null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], + [null, model, provider, null, null, null, null, null, agent, null, null, null, []], createFixture, ); if (apfInterceptorRequested) { @@ -264,6 +283,11 @@ const writePayload = (sandboxName, creationError) => { command, ), ); + const providerExposureCommands = payload.commandNames.filter((command: string) => + /(?:^|\s)provider (?:create|update|profile import)\b|(?:^|\s)sandbox provider attach\b/u.test( + command, + ), + ); const assertProviderBackedApfRefusal = () => { assert.match( payload.creationError, @@ -279,22 +303,15 @@ const writePayload = (sandboxName, creationError) => { false, ); }; - const assertManagedCreation = () => { + const assertSuccessfulCreation = () => { assert.equal(payload.creationError, null); assert.equal(payload.sandboxName, "my-assistant"); assert.ok(payload.sandboxListCalls >= 2); - assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); - assert.deepEqual(payload.killCalls, []); - assert.equal(payload.unrefCalls, 1); - assert.equal(payload.stdoutDestroyCalls, 1); - assert.equal(payload.stderrDestroyCalls, 1); assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); assert.equal( payload.registeredSandbox.lifecycleLiveIdentityFingerprint, createHash("sha256").update("sbx-fresh-create").digest("hex"), ); - assert.equal(payload.registeredSandbox.policyAuthority, "nemoclaw-managed"); - assert.ok(payload.registeredSandbox.policyCreationReceipt); assert.match( payload.createCommand, /--label ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62}/u, @@ -315,10 +332,32 @@ const writePayload = (sandboxName, creationError) => { `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, ); }; - const assertOutcome = apfInterceptorRequested - ? assertProviderBackedApfRefusal - : assertManagedCreation; - assertOutcome(); + const assertManagedProviderCreation = () => { + assertSuccessfulCreation(); + assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); + assert.deepEqual(payload.killCalls, []); + assert.equal(payload.unrefCalls, 1); + assert.equal(payload.stdoutDestroyCalls, 1); + assert.equal(payload.stderrDestroyCalls, 1); + assert.equal(payload.registeredSandbox.policyAuthority, "nemoclaw-managed"); + assert.ok(payload.registeredSandbox.policyCreationReceipt); + assert.match(payload.createCommand, /--policy \S+/u); + assert.match(payload.createCommand, /--provider nvidia-prod/u); + }; + const assertProviderlessApfCreation = () => { + assertSuccessfulCreation(); + assert.equal(payload.registeredSandbox.policyAuthority, "externally-managed"); + assert.equal(payload.registeredSandbox.policyCreationReceipt, undefined); + assert.doesNotMatch(payload.createCommand, /(?:^|\s)--policy(?:\s|$)/u); + assert.doesNotMatch(payload.createCommand, /(?:^|\s)--provider(?:\s|$)/u); + assert.deepEqual(providerExposureCommands, []); + }; + const assertions = { + "managed-provider": assertManagedProviderCreation, + "provider-refusal": assertProviderBackedApfRefusal, + "providerless-apf": assertProviderlessApfCreation, + }; + assertions[expectedOutcome](); }, ); }); From ec88d7c581e66aeb392850e0891f5ed834087429 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 16:56:19 -0700 Subject: [PATCH 07/42] fix(onboard): refuse interceptor provider effects early Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 9 ++-- src/lib/onboard/cancel-rollback.ts | 9 ++-- src/lib/onboard/lifecycle-contracts.md | 2 +- .../onboard/machine/core-flow-phases.test.ts | 49 +++++++++++++++++++ src/lib/onboard/machine/core-flow-phases.ts | 12 ++++- .../machine/handlers/sandbox-apf.test.ts | 32 ++++++++++++ src/lib/onboard/machine/handlers/sandbox.ts | 41 +++++++++++++++- 7 files changed, 143 insertions(+), 11 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index ce3e2146a4a..58b2c040866 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -494,12 +494,13 @@ $$nemoclaw onboard --fresh --apf-interceptor --name my-apf-sandbox If post-create verification or native GPU fallback fails after OpenShell may have created the sandbox, NemoClaw preserves the incomplete sandbox because automatic deletion would use its mutable name. +The same preservation applies if you cancel at the policy-tier selector or either policy-preset selector after sandbox creation. Do not destroy that sandbox by name. Retain the reported sandbox name, create-attempt label, and durable identity fingerprint for comparison only. If OpenShell did not return the fingerprint, recovery remains blocked until an administrator resolves the create-attempt label to one exact sandbox. Ask an OpenShell administrator to obtain the exact live durable ID, verify it against the fingerprint, and use an identity-bound removal procedure. -After the administrator confirms removal, start a fresh onboarding attempt. -`--resume` and `--recreate-sandbox` cannot recover the incomplete attempt. +This onboarding mode does not support `--resume` or `--recreate-sandbox`, regardless of whether sandbox creation began. +After the administrator confirms identity-bound removal, repeat the original command with `--fresh` and a new name. #### `--tool-disclosure ` @@ -895,8 +896,10 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. -If you cancel a brand-new onboarding run at the policy preset step, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. +If you cancel a brand-new onboarding run at the policy-tier selector or either policy-preset selector after sandbox creation, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. NemoClaw reports the durable sandbox identity fingerprint when it is available and never deletes the sandbox by mutable name. +Do not resume or recreate that incomplete sandbox. +After an OpenShell administrator verifies the live durable ID, removes that exact sandbox through an identity-bound procedure, and confirms removal, repeat the original onboarding intent with `$$nemoclaw onboard --fresh --name `. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. In interactive mode, the wizard asks for confirmation before delete and recreate. diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index 4d3084f2a80..c71ae982bc8 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -7,7 +7,7 @@ export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preser /** * Preservation guard for a sandbox whose onboarding is cancelled before the - * policy-preset step is confirmed. + * policy tier and preset selection window is confirmed. * * Cancellation preserves the incomplete sandbox, registry entry, and onboarding * session. The guard emits identity-bound recovery guidance and never deletes a @@ -15,7 +15,8 @@ export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preser * * The guard activates only when both conditions are true: * - `arm()` records a newly created sandbox after `createSandbox` succeeds. - * - `markCancelled()` records Ctrl+C or SIGTERM at a policy-step prompt. + * - `markCancelled()` records Ctrl+C or SIGTERM at the policy-tier or either + * policy-preset selector. * * Other `process.exit(1)` failure paths do not call `markCancelled()`. Their * existing preservation behavior remains unchanged. @@ -28,7 +29,7 @@ export interface SandboxCancelRollbackDeps { export interface SandboxCancelRollback { /** Arm cancellation recovery guidance for a just-created sandbox. */ arm(sandboxName: string, sandboxIdentityFingerprint?: string): void; - /** Disarm once the sandbox is past the cancellable window (policies confirmed). */ + /** Disarm once the sandbox is past the cancellable policy-selection window. */ disarm(): void; /** Record that the operator cancelled at a cancellable step. */ markCancelled(): void; @@ -88,7 +89,7 @@ export function installSandboxCancelRollback( } /** - * Build the cancel handler the policy-step prompts run on Ctrl+C / SIGTERM: + * Build the cancel handler the policy-selection prompts run on Ctrl+C / SIGTERM: * restore the terminal (`cleanup`), record the cancel, then exit non-zero. * Shared so both the tier and preset selectors stay in sync. */ diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 00ca0efcf7c..86fd5a09d35 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because OpenShell cannot condition deletion on its durable identity. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation clears only the aborted session and leaves the registry row unchanged. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because OpenShell cannot condition deletion on its durable identity. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Cancellation after creation preserves the incomplete session, registry row, and identity evidence for administrator recovery and never deletes by mutable name. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 251d53f76cc..b44559592c2 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -633,6 +633,54 @@ describe("core onboard flow phases", () => { expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); }); + it("rejects an explicit APF web-search provider before route reservation", async () => { + const reserveSandboxInferenceRoute = vi.fn(() => true); + const checkpointSandboxIdentity = vi.fn(async () => undefined); + const { providerInference: providerPhase } = createPhases({ + providerEnv: { NEMOCLAW_WEB_SEARCH_PROVIDER: "brave", BRAVE_API_KEY: "secret-value" }, + providerDeps: { checkpointSandboxIdentity, reserveSandboxInferenceRoute }, + sandboxOptions: { apfInterceptorRequested: true }, + }); + + await expect( + providerPhase.run( + context({ fresh: true, model: null, provider: null, selectedMessagingChannels: [] }), + ), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(checkpointSandboxIdentity).not.toHaveBeenCalled(); + expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); + }); + + it("rejects APF messaging intent before route reservation or external effects", async () => { + const setupInference = vi.fn(async () => ({ ok: true as const })); + const reserveSandboxInferenceRoute = vi.fn(() => true); + const checkpointSandboxIdentity = vi.fn(async () => undefined); + const { providerInference: providerPhase } = createPhases({ + providerDeps: { + checkpointSandboxIdentity, + reserveSandboxInferenceRoute, + setupInference, + }, + sandboxOptions: { apfInterceptorRequested: true }, + }); + + await expect( + providerPhase.run( + context({ + fresh: true, + model: null, + provider: null, + selectedMessagingChannels: ["telegram"], + }), + ), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(checkpointSandboxIdentity).not.toHaveBeenCalled(); + expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); + expect(setupInference).not.toHaveBeenCalled(); + }); + it("rejects APF when a serving profile requests an inference plan", async () => { const session = createSession({ apfInterceptorRequested: true }); session.servingProfileProvenance = {} as never; @@ -703,6 +751,7 @@ describe("core onboard flow phases", () => { }); const session = createSession({ apfInterceptorRequested: true }); const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ + providerEnv: { NEMOCLAW_WEB_SEARCH_PROVIDER: "none" }, providerDeps: { reserveSandboxInferenceRoute: reserveRoute, setupInference, diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index ab12f292613..2ec02a9c0aa 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -126,6 +126,8 @@ const APF_PROVIDER_INTENT_ENV_KEYS = [ "NEMOCLAW_SERVING_PRESET", ] as const; +const APF_PROVIDERLESS_WEB_SEARCH_ENV_VALUES = new Set(["", "none", "off", "disabled", "no", "0"]); + function hasProviderBackedApfIntent(context: OnboardFlowContext, env: NodeJS.ProcessEnv): boolean { const routeValues = [ context.provider, @@ -144,10 +146,16 @@ function hasProviderBackedApfIntent(context: OnboardFlowContext, env: NodeJS.Pro context.selectedMessagingChannels.length > 0 || context.hermesToolGateways.length > 0 || context.webSearchConfig !== null || + Boolean(context.session?.messagingPlan) || context.hostLocalInferenceRouteOnly === true || context.hostLocalInferenceSandboxProofAuthority != null || context.session?.servingProfileProvenance != null || - APF_PROVIDER_INTENT_ENV_KEYS.some((key) => String(env[key] ?? "").trim().length > 0) + APF_PROVIDER_INTENT_ENV_KEYS.some((key) => String(env[key] ?? "").trim().length > 0) || + !APF_PROVIDERLESS_WEB_SEARCH_ENV_VALUES.has( + String(env.NEMOCLAW_WEB_SEARCH_PROVIDER ?? "") + .trim() + .toLowerCase(), + ) ); } @@ -195,7 +203,7 @@ export function createProviderInferenceOnboardFlowPhase< ) { if (hasProviderBackedApfIntent(context, options.env)) { throw new Error( - "APF interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", ); } const sandboxName = diff --git a/src/lib/onboard/machine/handlers/sandbox-apf.test.ts b/src/lib/onboard/machine/handlers/sandbox-apf.test.ts index 7814fb2b638..29fa411ef9d 100644 --- a/src/lib/onboard/machine/handlers/sandbox-apf.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-apf.test.ts @@ -53,6 +53,9 @@ describe("APF sandbox create selection", () => { }); expect(calls.stageCredentialProviders).not.toHaveBeenCalled(); + expect(calls.configureWebSearch).not.toHaveBeenCalled(); + expect(calls.validateBrave).not.toHaveBeenCalled(); + expect(calls.setupMessaging).not.toHaveBeenCalled(); expect(calls.createSandbox).toHaveBeenCalledOnce(); const createCall = calls.createSandbox.mock.calls[0] ?? []; expect(createCall.at(-2)).toMatchObject({ @@ -71,6 +74,35 @@ describe("APF sandbox create selection", () => { ); }); + it.each([ + [ + "an explicit web-search environment selection", + { env: { NEMOCLAW_WEB_SEARCH_PROVIDER: "brave", BRAVE_API_KEY: "secret-value" } }, + ], + ["a selected messaging channel", { selectedMessagingChannels: ["telegram"] }], + ])("rejects %s before credential, provider, or sandbox effects", async (_label, intent) => { + const session = createSession({ apfInterceptorRequested: true }); + const { deps, calls } = createDeps({}, session); + + await expect( + handleSandboxState({ + ...baseOptions(deps, session), + ...intent, + fresh: true, + apfInterceptorRequested: true, + model: "", + provider: "", + preferredInferenceApi: null, + }), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(calls.configureWebSearch).not.toHaveBeenCalled(); + expect(calls.validateBrave).not.toHaveBeenCalled(); + expect(calls.setupMessaging).not.toHaveBeenCalled(); + expect(calls.stageCredentialProviders).not.toHaveBeenCalled(); + expect(calls.createSandbox).not.toHaveBeenCalled(); + }); + it("rejects a resolved APF provider plan before sandbox or provider effects (#9833)", async () => { const session = createSession({ apfInterceptorRequested: true }); const { deps, calls } = createDeps( diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 425ca8e1897..aaf5fa368f8 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -696,6 +696,23 @@ class SandboxStateFlow< return !agentName || agentName === "openclaw"; } + private assertProviderlessApfInput(): void { + if (this.options.apfInterceptorRequested !== true) return; + const explicitWebSearch = parseExplicitWebSearchProvider( + this.options.env[WEB_SEARCH_PROVIDER_ENV], + ).provider; + const hasProviderIntent = + this.options.webSearchConfig !== null || + explicitWebSearch !== null || + this.options.selectedMessagingChannels.length > 0 || + this.options.hermesToolGateways.length > 0 || + Boolean(this.options.session?.messagingPlan); + if (!hasProviderIntent) return; + throw new Error( + "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + ); + } + private prepareWebSearchSupport(): SandboxStepState { const probePath = this.options.fromDockerfile ? this.deps.resolvePath(this.options.fromDockerfile) @@ -1899,7 +1916,7 @@ class SandboxStateFlow< resolved.hermesToolGateways.length > 0; if (!hasProviderPlan) return; throw new Error( - "APF interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", ); } @@ -2443,6 +2460,27 @@ class SandboxStateFlow< nextState = this.checkpointSandboxName(nextState, requestedSandboxName); } nextState = this.recordSandboxIdentityForCreate(nextState, requestedSandboxName); + if (this.options.apfInterceptorRequested === true) { + const registryMessagingAuthority = + this.deps.getRegistrySandboxMessagingAuthority(requestedSandboxName); + if (registryMessagingAuthority.plan !== null) { + throw new Error( + "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + ); + } + return this.createAndRecordSandbox( + nextState, + requestedSandboxName, + null, + registryMessagingAuthority, + decision, + async (state) => + this.checkpointMessaging(this.checkpointWebSearch(state, null), { + plan: null, + selectedChannels: [], + }), + ); + } const webSearchConfig = await this.resolveWebSearchForCreation(nextState); const webSearchConfigChanged = nextState.webSearchConfigChanged || @@ -2560,6 +2598,7 @@ class SandboxStateFlow< } async run(): Promise> { + this.assertProviderlessApfInput(); if (this.options.session?.checkpoint) { this.replayableCheckpointProviderBindings(this.options.session.checkpoint); } From 665012edf51b89183a5617296d8c5738dcfed026 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 18:07:01 -0700 Subject: [PATCH 08/42] fix(onboard): preserve non-tty cancellation recovery Signed-off-by: Apurv Kumaria --- .../onboard/policy-selection-prompts.test.ts | 52 ++++++++++++++++++ src/lib/onboard/policy-selection-prompts.ts | 54 +++++++++++++++++-- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard/policy-selection-prompts.test.ts b/src/lib/onboard/policy-selection-prompts.test.ts index d5d863daca9..f07d2969460 100644 --- a/src/lib/onboard/policy-selection-prompts.test.ts +++ b/src/lib/onboard/policy-selection-prompts.test.ts @@ -156,6 +156,58 @@ describe("createPolicySelectionPromptHelpers", () => { expect(errorSpy).toHaveBeenCalledWith(" Unknown preset name ignored: missing"); }); + it.each([ + [ + "policy tier", + (helpers: ReturnType) => + helpers.selectPolicyTier(), + ], + [ + "tier presets", + (helpers: ReturnType) => + helpers.selectTierPresetsAndAccess("balanced", [{ name: "npm" }]), + ], + [ + "preset checkbox", + (helpers: ReturnType) => + helpers.presetsCheckboxSelector([{ name: "npm", description: "npm registry" }], []), + ], + ])( + "marks cancellation when SIGTERM interrupts the non-TTY %s selector", + async (_label, select) => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const { helpers, markCancelled, processEvents, prompt } = createHarness({ + stdinTTY: false, + stdoutTTY: false, + }); + prompt.mockImplementation(() => new Promise(() => undefined)); + + const selection = select(helpers); + expect(prompt).toHaveBeenCalledOnce(); + processEvents.emit("SIGTERM"); + + await expect(selection).rejects.toMatchObject({ code: 1 }); + expect(markCancelled).toHaveBeenCalledOnce(); + expect(processEvents.listenerCount("SIGINT")).toBe(0); + expect(processEvents.listenerCount("SIGTERM")).toBe(0); + }, + ); + + it("marks cancellation when a non-TTY prompt reports Ctrl-C", async () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const { helpers, markCancelled, processEvents, prompt } = createHarness({ + stdinTTY: false, + stdoutTTY: false, + }); + prompt.mockRejectedValue(Object.assign(new Error("Prompt interrupted"), { code: "SIGINT" })); + + await expect(helpers.selectPolicyTier()).rejects.toMatchObject({ code: 1 }); + + expect(markCancelled).toHaveBeenCalledOnce(); + expect(processEvents.listenerCount("SIGINT")).toBe(0); + expect(processEvents.listenerCount("SIGTERM")).toBe(0); + }); + it("selectTierPresetsAndAccess returns raw-mode access toggles on Enter", async () => { const { helpers, markCancelled, stdin } = createHarness(); const result = helpers.selectTierPresetsAndAccess("balanced", [ diff --git a/src/lib/onboard/policy-selection-prompts.ts b/src/lib/onboard/policy-selection-prompts.ts index b6d06873a7b..bf9ff28d0be 100644 --- a/src/lib/onboard/policy-selection-prompts.ts +++ b/src/lib/onboard/policy-selection-prompts.ts @@ -23,8 +23,8 @@ type PolicyPromptOutput = { write(chunk: string): unknown; }; type PolicyPromptProcessEvents = { - once(event: "SIGTERM", listener: () => void): unknown; - removeListener(event: "SIGTERM", listener: () => void): unknown; + once(event: "SIGINT" | "SIGTERM", listener: () => void): unknown; + removeListener(event: "SIGINT" | "SIGTERM", listener: () => void): unknown; }; export interface PolicySelectionPromptDeps { @@ -78,6 +78,48 @@ export function createPolicySelectionPromptHelpers(deps: PolicySelectionPromptDe const stdout = deps.stdout ?? process.stdout; const processEvents = deps.processEvents ?? process; + function promptWithOnboardCancel(question: string): Promise { + return new Promise((resolve, reject) => { + let cancelled = false; + const cleanup = () => { + processEvents.removeListener("SIGINT", onCancel); + processEvents.removeListener("SIGTERM", onCancel); + }; + const cancelExit = makeOnboardCancelExit(sandboxCancelRollback, cleanup, (code) => + reject(new OnboardDeferredExitError(code)), + ); + const onCancel = () => { + if (cancelled) return; + cancelled = true; + cancelExit(); + }; + processEvents.once("SIGINT", onCancel); + processEvents.once("SIGTERM", onCancel); + let answer: Promise; + try { + answer = prompt(question); + } catch (error) { + cleanup(); + reject(error); + return; + } + void answer.then( + (value) => { + cleanup(); + resolve(value); + }, + (error: unknown) => { + if ((error as NodeJS.ErrnoException)?.code === "SIGINT") { + onCancel(); + return; + } + cleanup(); + reject(error); + }, + ); + }); + } + /** * Prompt the user to select a policy tier (restricted / balanced / open / personal). * Uses the same radio-style TUI as presetsCheckboxSelector (single-select). @@ -110,7 +152,7 @@ export function createPolicySelectionPromptHelpers(deps: PolicySelectionPromptDe console.log(` ${marker} ${tier.label}`); }); console.log(""); - const answer = await prompt( + const answer = await promptWithOnboardCancel( ` Select tier [1-${allTiers.length}] (default: ${allTiers.indexOf(defaultTier) + 1} ${defaultTier.name}): `, ); const chosen = selectFromNumberedMenuOrExit( @@ -283,7 +325,7 @@ export function createPolicySelectionPromptHelpers(deps: PolicySelectionPromptDe console.log(` ${check} ${badge} ${preset.name}`); }); console.log(""); - const rawInclude = await prompt( + const rawInclude = await promptWithOnboardCancel( " Include presets (comma-separated names, Enter to keep defaults): ", ); if (rawInclude.trim()) { @@ -433,7 +475,9 @@ export function createPolicySelectionPromptHelpers(deps: PolicySelectionPromptDe console.log(` ${marker} ${preset.name.padEnd(14)} — ${preset.description}`); }); console.log(""); - const raw = await prompt(" Select presets (comma-separated names, Enter to skip): "); + const raw = await promptWithOnboardCancel( + " Select presets (comma-separated names, Enter to skip): ", + ); if (!raw.trim()) { console.log(" Skipping policy presets."); return []; From 6da68ad5ffed692adce5d7904f9e1cdb61944977 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 18:41:21 -0700 Subject: [PATCH 09/42] test(onboard): cover cancellation recovery Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 4 +- .../onboard/machine/core-flow-phases.test.ts | 7 ++ .../onboard-fresh-create-identity.test.ts | 81 ++++++++++++++++--- 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 58b2c040866..5559d2535bf 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -494,7 +494,6 @@ $$nemoclaw onboard --fresh --apf-interceptor --name my-apf-sandbox If post-create verification or native GPU fallback fails after OpenShell may have created the sandbox, NemoClaw preserves the incomplete sandbox because automatic deletion would use its mutable name. -The same preservation applies if you cancel at the policy-tier selector or either policy-preset selector after sandbox creation. Do not destroy that sandbox by name. Retain the reported sandbox name, create-attempt label, and durable identity fingerprint for comparison only. If OpenShell did not return the fingerprint, recovery remains blocked until an administrator resolves the create-attempt label to one exact sandbox. @@ -899,7 +898,8 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. If you cancel a brand-new onboarding run at the policy-tier selector or either policy-preset selector after sandbox creation, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. NemoClaw reports the durable sandbox identity fingerprint when it is available and never deletes the sandbox by mutable name. Do not resume or recreate that incomplete sandbox. -After an OpenShell administrator verifies the live durable ID, removes that exact sandbox through an identity-bound procedure, and confirms removal, repeat the original onboarding intent with `$$nemoclaw onboard --fresh --name `. +After an OpenShell administrator verifies the live durable ID, removes that exact sandbox through an identity-bound procedure, and confirms removal, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and change only the sandbox name. +`--fresh` starts a new session and does not retain those selections. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. In interactive mode, the wizard asks for confirmation before delete and recreate. diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index b44559592c2..e29726c82eb 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -20,6 +20,7 @@ import { createProviderInferenceOnboardFlowPhase, createSandboxOnboardFlowPhase, type EndpointProvenanceOptions, + isCoreFlowCompleteBeforeFinalization, type ProviderInferenceOnboardFlowPhaseOptions, runCoreOnboardFlowSlice, type SandboxOnboardFlowPhaseOptions, @@ -796,6 +797,12 @@ describe("core onboard flow phases", () => { const sandboxResult = await sandboxPhase.run(providerResult.context); expect(sandboxResult.result).toMatchObject({ type: "complete" }); + expect( + isCoreFlowCompleteBeforeFinalization({ + context: sandboxResult.context, + session: { machine: { state: "complete" } }, + }), + ).toBe(true); expect(setupNim).not.toHaveBeenCalled(); expect(setupInference).not.toHaveBeenCalled(); expect(createSandbox).toHaveBeenCalledOnce(); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index b9ca2fe780d..9024d1ae9a7 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -44,6 +44,14 @@ describe("fresh create identity", () => { agent: null, expectedOutcome: "providerless-apf" as const, }, + { + title: "preserves a newly created sandbox when non-TTY policy selection is cancelled (#9833)", + apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "cancel-after-create" as const, + }, ])( "$title", { @@ -107,6 +115,8 @@ const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; const agent = ${JSON.stringify(agent)}; const model = ${JSON.stringify(model)}; const provider = ${JSON.stringify(provider)}; +const cancelAfterCreate = ${JSON.stringify(expectedOutcome === "cancel-after-create")}; +let cancelPrompt = false; runner.run = (command, opts = {}) => { const cmd = _n(command); _deleted = _deleted || cmd.includes("sandbox delete"); @@ -163,7 +173,12 @@ runner.run = (command, opts = {}) => { registerSandbox: (entry) => { registeredSandbox = entry; }, }); preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; +credentials.prompt = async () => { + if (cancelPrompt) { + throw Object.assign(new Error("Prompt interrupted"), { code: "SIGINT" }); + } + return ""; +}; const groupKillCalls = []; const realProcessKill = process.kill.bind(process); @@ -211,13 +226,24 @@ childProcess.spawn = (...args) => { return child; }; -const { createSandbox } = require(${onboardPath}); +const onboardModule = require(${onboardPath}); +const { createSandbox } = onboardModule; +if (cancelAfterCreate) { + const session = onboardModule.onboardSession.createSession({ + mode: "interactive", + sandboxName: "my-assistant", + metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, + }); + onboardModule.onboardSession.saveSession(session); +} -const writePayload = (sandboxName, creationError) => { +const writePayload = (sandboxName, creationError, exitCode = 0) => { const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ sandboxName, creationError, + exitCode, + deleted: _deleted, sandboxCreated, sandboxListCalls, killCalls: createCommand?.child?.killCalls ?? [], @@ -227,6 +253,8 @@ const writePayload = (sandboxName, creationError) => { stderrDestroyCalls: createCommand?.child?.stderr.destroyCalls ?? 0, lifecycleObservationCommands, registeredSandbox, + currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, + savedSession: cancelAfterCreate ? onboardModule.onboardSession.loadSession() : null, createCommand: createCommand?.command ?? null, commandNames: commands.map((entry) => entry.command), })); @@ -249,8 +277,15 @@ const writePayload = (sandboxName, creationError) => { } try { const sandboxName = await createSandbox(...createArgs); + if (cancelAfterCreate) { + process.on("exit", (code) => writePayload(sandboxName, null, code)); + cancelPrompt = true; + await onboardModule.selectPolicyTier(); + throw new Error("expected policy selection cancellation"); + } writePayload(sandboxName, null); } catch (error) { + if (cancelAfterCreate) throw error; if (!apfInterceptorRequested) throw error; writePayload(null, error instanceof Error ? error.message : String(error)); } @@ -263,20 +298,22 @@ const writePayload = (sandboxName, creationError) => { `; fs.writeFileSync(scriptPath, script); + const childEnv = { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: expectedOutcome === "cancel-after-create" ? "" : "1", + OPENSHELL_DRIVERS: "docker", + }; const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - OPENSHELL_DRIVERS: "docker", - }, + env: childEnv, timeout: 30000, }); - assert.equal(result.status, 0, result.stderr); + assert.equal(result.status, expectedOutcome === "cancel-after-create" ? 1 : 0, result.stderr); + assert.ok(fs.existsSync(payloadPath), result.stderr); const payload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); const providerEffectCommands = payload.commandNames.filter((command: string) => /(?:^|\s)provider (?:create|update|delete|profile import)\b|(?:^|\s)sandbox provider (?:attach|detach)\b/u.test( @@ -352,10 +389,32 @@ const writePayload = (sandboxName, creationError) => { assert.doesNotMatch(payload.createCommand, /(?:^|\s)--provider(?:\s|$)/u); assert.deepEqual(providerExposureCommands, []); }; + const assertCancellationRecovery = () => { + const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); + assert.equal(payload.exitCode, 1); + assert.equal(payload.sandboxName, "my-assistant"); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox.name, "my-assistant"); + assert.equal( + payload.currentRegistryEntry.lifecycleLiveIdentityFingerprint, + identityFingerprint, + ); + assert.equal(payload.currentRegistryEntry.name, "my-assistant"); + assert.equal(payload.savedSession.status, "in_progress"); + assert.equal(payload.savedSession.sandboxName, "my-assistant"); + assert.equal( + payload.commandNames.some((command: string) => command.includes("sandbox delete")), + false, + ); + assert.match(result.stderr, /preserved incomplete sandbox 'my-assistant'/u); + assert.match(result.stderr, new RegExp(identityFingerprint, "u")); + assert.match(result.stderr, /Do not delete the sandbox by mutable sandbox name/u); + }; const assertions = { "managed-provider": assertManagedProviderCreation, "provider-refusal": assertProviderBackedApfRefusal, "providerless-apf": assertProviderlessApfCreation, + "cancel-after-create": assertCancellationRecovery, }; assertions[expectedOutcome](); }, From 5696a0a136e50f16226aa636dccb11a6b1904584 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 18:58:43 -0700 Subject: [PATCH 10/42] fix(onboard): report missing identity recovery Signed-off-by: Apurv Kumaria --- .../created-sandbox-finalization.test.ts | 42 +++++++++++++++++++ .../onboard/created-sandbox-finalization.ts | 10 +++++ 2 files changed, 52 insertions(+) diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index 7ed71dfee61..93921e7b6cd 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -11,6 +11,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { SandboxEntry } from "../state/registry"; import * as sandboxState from "../state/sandbox"; import { + completeOrdinaryOnboardSandboxCreation, createCreatedSandboxCompletionActions, createOnboardCreatedSandboxCompletion, createOnboardCreatedSandboxRegistration, @@ -57,6 +58,47 @@ describe("created sandbox registration authority", () => { }); }); +describe("new sandbox cancellation recovery", () => { + it("preserves recovery guidance when the durable identity is unavailable (#9833)", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const runFile = vi.fn(); + const armCancelRollback = vi.fn(); + + expect(() => + completeOrdinaryOnboardSandboxCreation( + { + sandboxName: "new-sandbox", + sandboxWasLiveDefault: false, + runtimeFields: { openshellDriver: "docker" } as never, + messagingProviders: [], + liveExists: false, + }, + { + setDefault: vi.fn(), + runFile, + scriptsDir: "/repo/scripts", + gatewayName: "nemoclaw", + providerExistsInGateway: () => true, + armCancelRollback, + dockerInfoFormat: () => "", + runCapture: () => "", + revalidatePolicyAuthority: vi.fn(), + applyVmDnsMonkeypatch: vi.fn(), + }, + ), + ).toThrow("Sandbox 'new-sandbox' has no exact identity for cancel recovery."); + + const guidance = error.mock.calls.flat().join("\n"); + expect(guidance).toContain("Sandbox 'new-sandbox' was created on gateway 'nemoclaw'"); + expect(guidance).toContain("registry entry and onboarding session were preserved"); + expect(guidance).toContain("Do not delete the sandbox by mutable sandbox name"); + expect(guidance).toContain("establish the exact live durable identity before removal"); + expect(guidance).toContain("add --fresh, and use a new sandbox name"); + expect(runFile).not.toHaveBeenCalled(); + expect(armCancelRollback).not.toHaveBeenCalled(); + }); +}); + function executable(file: string, contents: string): void { fs.writeFileSync(file, contents, { mode: 0o755 }); } diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index 22a0356fe88..e3d25089eb8 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -289,6 +289,16 @@ export function completeOrdinaryOnboardSandboxCreation( !lifecycleLiveIdentityFingerprint || !/^[0-9a-f]{64}$/u.test(lifecycleLiveIdentityFingerprint) ) { + for (const line of [ + "", + ` Sandbox '${input.sandboxName}' was created on gateway '${deps.gatewayName}', but NemoClaw could not verify its durable identity.`, + " The sandbox registry entry and onboarding session were preserved for recovery.", + " Do not delete the sandbox by mutable sandbox name.", + " Ask an OpenShell administrator to establish the exact live durable identity before removal.", + " After confirmed identity-bound removal, rerun the original onboarding command with the same required inputs, add --fresh, and use a new sandbox name.", + ]) { + console.error(line); + } throw new Error(`Sandbox '${input.sandboxName}' has no exact identity for cancel recovery.`); } deps.armCancelRollback(input.sandboxName, lifecycleLiveIdentityFingerprint); From 568ead2049abb949d70db1ba7c1b383e0aee7d2e Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 19:06:01 -0700 Subject: [PATCH 11/42] fix(onboard): reject deferred provider intent early Signed-off-by: Apurv Kumaria --- .../onboard/machine/core-flow-phases.test.ts | 40 +++++++++++++++++++ src/lib/onboard/machine/core-flow-phases.ts | 7 ++++ 2 files changed, 47 insertions(+) diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index e29726c82eb..5ddbfcdaa7e 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; +import * as credentialStore from "../../credentials/store"; import { type InferenceEndpointSource, normalizeInferenceSelection, @@ -653,6 +654,45 @@ describe("core onboard flow phases", () => { expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); }); + it("rejects an extra-placeholder provider before route, credential, provider, or sandbox effects", async () => { + const credentialRead = vi.spyOn(credentialStore, "getCredential").mockReturnValue(null); + const setupInference = vi.fn(async () => ({ ok: true as const })); + const reserveSandboxInferenceRoute = vi.fn(() => true); + const checkpointSandboxIdentity = vi.fn(async () => undefined); + const stageSandboxCredentialProviders = vi.fn(); + const createSandbox = vi.fn(async () => "created-sandbox"); + const { providerInference: providerPhase } = createPhases({ + providerEnv: { + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: "TELEGRAM_BOT_TOKEN_AGENT_A", + TELEGRAM_BOT_TOKEN_AGENT_A: "secret-canary", + }, + providerDeps: { + checkpointSandboxIdentity, + reserveSandboxInferenceRoute, + setupInference, + }, + sandboxOptions: { apfInterceptorRequested: true }, + sandboxDeps: { createSandbox, stageSandboxCredentialProviders }, + }); + + try { + await expect( + providerPhase.run( + context({ fresh: true, model: null, provider: null, selectedMessagingChannels: [] }), + ), + ).rejects.toThrow(/supports providerless sandbox creation only/u); + + expect(checkpointSandboxIdentity).not.toHaveBeenCalled(); + expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); + expect(credentialRead).not.toHaveBeenCalled(); + expect(setupInference).not.toHaveBeenCalled(); + expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); + expect(createSandbox).not.toHaveBeenCalled(); + } finally { + credentialRead.mockRestore(); + } + }); + it("rejects APF messaging intent before route reservation or external effects", async () => { const setupInference = vi.fn(async () => ({ ok: true as const })); const reserveSandboxInferenceRoute = vi.fn(() => true); diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 2ec02a9c0aa..053654418a0 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -7,6 +7,11 @@ import { } from "../../inference/selection"; import type { WebSearchConfig } from "../../inference/web-search"; import type { DcodeAutoApprovalMode } from "../dcode-auto-approval"; +import { + canonicalPlaceholderKeys, + EXTRA_PLACEHOLDER_KEYS_ENV, + parseExtraPlaceholderKeys, +} from "../extra-placeholder-keys"; import type { createProviderRecoveryReceiptLedger, ProviderRecoveryReceipt, @@ -151,6 +156,8 @@ function hasProviderBackedApfIntent(context: OnboardFlowContext, env: NodeJS.Pro context.hostLocalInferenceSandboxProofAuthority != null || context.session?.servingProfileProvenance != null || APF_PROVIDER_INTENT_ENV_KEYS.some((key) => String(env[key] ?? "").trim().length > 0) || + parseExtraPlaceholderKeys(env[EXTRA_PLACEHOLDER_KEYS_ENV], canonicalPlaceholderKeys()).keys + .length > 0 || !APF_PROVIDERLESS_WEB_SEARCH_ENV_VALUES.has( String(env.NEMOCLAW_WEB_SEARCH_PROVIDER ?? "") .trim() From 2c87fe5fb086d9a8be4bd3a6e74db47a7eac7c78 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 20:36:08 -0700 Subject: [PATCH 12/42] fix(onboard): retain managed image authority Signed-off-by: Apurv Kumaria --- .../onboard/managed-workload/onboard-orchestration.test.ts | 5 ++--- src/lib/onboard/managed-workload/onboard-orchestration.ts | 2 -- src/lib/onboard/sandbox-create/orchestration.ts | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts index c8b17a956e9..ecf2ffd0970 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -183,15 +183,14 @@ describe("managed workload onboard orchestration", () => { ).toBe(false); }); - it("does not activate stock managed images for providerless APF creation (#9833)", () => { + it("keeps stock managed images required during providerless interceptor creation (#9833)", () => { expect( shouldActivateStockManagedRuntime({ portableLifecycle: false, hermesPortableLifecycle: false, - apfInterceptorRequested: true, agentName: "openclaw", }), - ).toBe(false); + ).toBe(true); }); it("rejects an unavailable catalog for stock managed-image onboarding", async () => { diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index 6d6c1817fa5..dc090945816 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -153,11 +153,9 @@ export interface ManagedWorkloadOnboardRuntime { export function shouldActivateStockManagedRuntime(input: { readonly portableLifecycle: boolean; readonly hermesPortableLifecycle: boolean; - readonly apfInterceptorRequested?: boolean; readonly agentName: string; }): boolean { return ( - input.apfInterceptorRequested !== true && !input.portableLifecycle && !input.hermesPortableLifecycle && isShippedManagedImageAgent(input.agentName) diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 321891d9e72..283b284f0d0 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -883,7 +883,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche stockManagedRuntime: managedWorkloadOnboard.shouldActivateStockManagedRuntime({ portableLifecycle: sandboxGpuCreateFlow.resolvePortableLifecycleMode(agent), hermesPortableLifecycle: agentCreateInput.hermesPortableLifecycle, - apfInterceptorRequested: createIntent?.apfInterceptorRequested === true, agentName: requestedAgentName, }), tempManagedRuntimeCatalog, From 9c9674870352ec3a291bf85c8ae982611d31282d Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 21:03:50 -0700 Subject: [PATCH 13/42] fix(onboard): reject staged provider intent early Signed-off-by: Apurv Kumaria --- src/lib/onboard/entry-options.ts | 51 +++++++++++++++- .../onboard/machine/core-flow-phases.test.ts | 1 + src/lib/onboard/machine/core-flow-phases.ts | 30 ++-------- .../managed-workload/onboard-orchestration.ts | 16 ++--- .../onboard/sandbox-create/orchestration.ts | 5 ++ .../onboard-fresh-create-identity.test.ts | 60 ++++++++++++++++++- .../onboard-fsm-live-slices.test.ts | 26 +++++++- 7 files changed, 152 insertions(+), 37 deletions(-) diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index c3b5e1b8ed4..fa38d468eb6 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -4,6 +4,11 @@ import { isNonInteractiveEnv } from "../core/non-interactive"; import { getNameValidationGuidance } from "../name-validation"; import { cliDisplayName } from "./branding"; +import { + canonicalPlaceholderKeys, + EXTRA_PLACEHOLDER_KEYS_ENV, + parseExtraPlaceholderKeys, +} from "./extra-placeholder-keys"; import { RESERVED_SANDBOX_NAMES } from "./sandbox-agent"; import { requireStationExpressResumeIntent, @@ -54,7 +59,42 @@ export interface ResolvedOnboardEntryOptions { } type NonInteractiveEntryOptions = { nonInteractive?: boolean }; -type ResumableEntryOptions = NonInteractiveEntryOptions & { resume?: boolean; fresh?: boolean }; +type ResumableEntryOptions = NonInteractiveEntryOptions & { + resume?: boolean; + fresh?: boolean; + apfInterceptorRequested?: boolean | null; +}; + +const PROVIDER_INTENT_ENV_KEYS = [ + "NEMOCLAW_PROVIDER", + "NEMOCLAW_MODEL", + "NEMOCLAW_PROVIDER_MODEL", + "NEMOCLAW_SERVING_PRESET", + "NEMOCLAW_MESSAGING_PLAN_B64", +] as const; + +const PROVIDERLESS_WEB_SEARCH_ENV_VALUES = new Set(["", "none", "off", "disabled", "no", "0"]); + +/** Reject ambient provider intent before onboarding records or external effects. */ +export function assertProviderlessInterceptorEnvironment( + interceptorRequested: boolean, + env: NodeJS.ProcessEnv, +): void { + if (!interceptorRequested) return; + const hasProviderIntent = + PROVIDER_INTENT_ENV_KEYS.some((key) => String(env[key] ?? "").trim().length > 0) || + parseExtraPlaceholderKeys(env[EXTRA_PLACEHOLDER_KEYS_ENV], canonicalPlaceholderKeys()).keys + .length > 0 || + !PROVIDERLESS_WEB_SEARCH_ENV_VALUES.has( + String(env.NEMOCLAW_WEB_SEARCH_PROVIDER ?? "") + .trim() + .toLowerCase(), + ); + if (!hasProviderIntent) return; + throw new Error( + "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + ); +} export function resolveOnboardRunOptions( options: OnboardEntryOptionsInput["opts"] & { autoYes?: boolean; nonInteractive?: boolean }, @@ -153,8 +193,15 @@ export function wrapOnboard( run: (options?: Options) => Promise, session: StationExpressSessionLifecycle, ): (options?: Options) => Promise { + const guardProviderlessInput = async (options?: Options): Promise => { + assertProviderlessInterceptorEnvironment( + options?.apfInterceptorRequested === true, + process.env, + ); + await run(options); + }; return wrapStationExpressOnboard( - withNonInteractiveEnvironment(run), + withNonInteractiveEnvironment(guardProviderlessInput), session.loadSession, session.reconcileStationExpressReceiptRetirement, ); diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 5ddbfcdaa7e..689abbf7f91 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -616,6 +616,7 @@ describe("core onboard flow phases", () => { "NEMOCLAW_MODEL", "NEMOCLAW_PROVIDER_MODEL", "NEMOCLAW_SERVING_PRESET", + "NEMOCLAW_MESSAGING_PLAN_B64", ])("rejects APF when %s requests a provider plan", async (key) => { const reserveSandboxInferenceRoute = vi.fn(() => true); const checkpointSandboxIdentity = vi.fn(async () => undefined); diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 053654418a0..c733dabcf86 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -7,11 +7,7 @@ import { } from "../../inference/selection"; import type { WebSearchConfig } from "../../inference/web-search"; import type { DcodeAutoApprovalMode } from "../dcode-auto-approval"; -import { - canonicalPlaceholderKeys, - EXTRA_PLACEHOLDER_KEYS_ENV, - parseExtraPlaceholderKeys, -} from "../extra-placeholder-keys"; +import { assertProviderlessInterceptorEnvironment } from "../entry-options"; import type { createProviderRecoveryReceiptLedger, ProviderRecoveryReceipt, @@ -124,16 +120,7 @@ export function isCoreFlowCompleteBeforeFinalization(result: { ); } -const APF_PROVIDER_INTENT_ENV_KEYS = [ - "NEMOCLAW_PROVIDER", - "NEMOCLAW_MODEL", - "NEMOCLAW_PROVIDER_MODEL", - "NEMOCLAW_SERVING_PRESET", -] as const; - -const APF_PROVIDERLESS_WEB_SEARCH_ENV_VALUES = new Set(["", "none", "off", "disabled", "no", "0"]); - -function hasProviderBackedApfIntent(context: OnboardFlowContext, env: NodeJS.ProcessEnv): boolean { +function hasProviderBackedApfIntent(context: OnboardFlowContext): boolean { const routeValues = [ context.provider, context.model, @@ -154,15 +141,7 @@ function hasProviderBackedApfIntent(context: OnboardFlowContext, env: NodeJS.Pro Boolean(context.session?.messagingPlan) || context.hostLocalInferenceRouteOnly === true || context.hostLocalInferenceSandboxProofAuthority != null || - context.session?.servingProfileProvenance != null || - APF_PROVIDER_INTENT_ENV_KEYS.some((key) => String(env[key] ?? "").trim().length > 0) || - parseExtraPlaceholderKeys(env[EXTRA_PLACEHOLDER_KEYS_ENV], canonicalPlaceholderKeys()).keys - .length > 0 || - !APF_PROVIDERLESS_WEB_SEARCH_ENV_VALUES.has( - String(env.NEMOCLAW_WEB_SEARCH_PROVIDER ?? "") - .trim() - .toLowerCase(), - ) + context.session?.servingProfileProvenance != null ); } @@ -208,7 +187,8 @@ export function createProviderInferenceOnboardFlowPhase< options.apfInterceptorRequested === true || context.session?.apfInterceptorRequested === true ) { - if (hasProviderBackedApfIntent(context, options.env)) { + assertProviderlessInterceptorEnvironment(true, options.env); + if (hasProviderBackedApfIntent(context)) { throw new Error( "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", ); diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index dc090945816..a45b7173268 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -133,8 +133,8 @@ export interface CreateManagedWorkloadOnboardRuntimeInput { readonly legacyDockerfilePath: string; readonly customDockerfilePath: string | null; readonly rootDir: string; - readonly model: string; - readonly provider: string; + readonly model: string | null; + readonly provider: string | null; readonly preferredInferenceApi: string | null; readonly endpointUrl: string | null; readonly startupProfile: ManagedProfileInput; @@ -301,25 +301,27 @@ export function createManagedWorkloadOnboardRuntime( return input.managedWorkloadRebuild.replacementProfile; } if (preparedProfile) return preparedProfile; + const selectedModel = input.model?.trim() || "unconfigured"; + const selectedProvider = input.provider?.trim() || null; const inferenceApi = input.agentName === "langchain-deepagents-code" ? "openai-completions" : dependencies.resolveAgentInferenceApi( input.agentName, - input.provider, + selectedProvider, input.preferredInferenceApi, ); const inference: SandboxInferenceConfig = dependencies.getSandboxInferenceConfig( - input.model, - input.provider, + selectedModel, + selectedProvider, inferenceApi, ); preparedProfile = buildManagedStartupOnboardProfile({ agentName: input.agentName, inference: { routeProvider: inference.providerKey, - upstreamProvider: input.provider.trim() ? input.provider : inference.providerKey, - model: input.model, + upstreamProvider: selectedProvider ?? inference.providerKey, + model: selectedModel, routedBaseUrl: inference.inferenceBaseUrl, upstreamEndpointUrl: input.agentName === "langchain-deepagents-code" ? input.endpointUrl : null, diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 283b284f0d0..60877047b0a 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -19,6 +19,7 @@ import type { HermesAuthMethod } from "../hermes-auth"; import * as policyAuthorityPreflight from "../policy-authority/preflight"; import type { PreparedSandboxBuildContext } from "../build-context-stage"; import type { DcodeSelectionDriftReader } from "../dcode-selection-drift"; +import { assertProviderlessInterceptorEnvironment } from "../entry-options"; import type { ManagedHermesStateVolumeCleanupResult, ManagedHermesStateVolumeContext, @@ -739,6 +740,10 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche "sandbox name", ); assertApfCreateIntent(createIntent); + assertProviderlessInterceptorEnvironment( + createIntent?.apfInterceptorRequested === true, + process.env, + ); preparedDcodeRebuild.assertPreparedDcodeTarget(preparedBuildContext, agent, fromDockerfile); const effectiveAgent = sandboxAgent.getEffectiveSandboxAgent(agent); const requestedAgentName = getRequestedSandboxAgentName(effectiveAgent); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 9024d1ae9a7..4dfb5bc6fba 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -11,6 +11,7 @@ import path from "node:path"; import { beforeEach, describe, it, vi } from "vitest"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; import { type CommandEntry, onboardScriptMocksPath } from "../helpers/onboard-split-context"; +import { encodeMessagingPlan, makeMessagingPlan } from "../helpers/messaging-plan-fixtures"; beforeEach(() => { vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); @@ -44,6 +45,14 @@ describe("fresh create identity", () => { agent: null, expectedOutcome: "providerless-apf" as const, }, + { + title: "rejects staged messaging intent before any onboarding side effect (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "staged-messaging-refusal" as const, + }, { title: "preserves a newly created sandbox when non-TTY policy selection is cancelled (#9833)", apfInterceptorRequested: false, @@ -110,13 +119,31 @@ let dockerPsCalls = 0; let sandboxCreated = false; let registeredSandbox = null; let effectivePolicy = {}; +let credentialReadCalls = 0; +let routeReservationCalls = 0; const keepAlive = setInterval(() => {}, 1000); const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; const agent = ${JSON.stringify(agent)}; const model = ${JSON.stringify(model)}; const provider = ${JSON.stringify(provider)}; const cancelAfterCreate = ${JSON.stringify(expectedOutcome === "cancel-after-create")}; +const stagedMessagingRefusal = ${JSON.stringify(expectedOutcome === "staged-messaging-refusal")}; let cancelPrompt = false; +const originalGetCredential = credentials.getCredential; +if (stagedMessagingRefusal) { + credentials.getCredential = (...args) => { + credentialReadCalls += 1; + if (typeof args[0] !== "string") return null; + return originalGetCredential(...args); + }; +} +const originalReserveSandboxInferenceRoute = registry.reserveSandboxInferenceRoute; +if (stagedMessagingRefusal) { + registry.reserveSandboxInferenceRoute = (...args) => { + routeReservationCalls += 1; + return originalReserveSandboxInferenceRoute(...args); + }; +} runner.run = (command, opts = {}) => { const cmd = _n(command); _deleted = _deleted || cmd.includes("sandbox delete"); @@ -253,6 +280,8 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { stderrDestroyCalls: createCommand?.child?.stderr.destroyCalls ?? 0, lifecycleObservationCommands, registeredSandbox, + credentialReadCalls, + routeReservationCalls, currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, savedSession: cancelAfterCreate ? onboardModule.onboardSession.loadSession() : null, createCommand: createCommand?.command ?? null, @@ -287,7 +316,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { } catch (error) { if (cancelAfterCreate) throw error; if (!apfInterceptorRequested) throw error; - writePayload(null, error instanceof Error ? error.message : String(error)); + writePayload(null, error instanceof Error ? error.message : String(error)); } clearInterval(keepAlive); })().catch((error) => { @@ -304,6 +333,12 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: expectedOutcome === "cancel-after-create" ? "" : "1", OPENSHELL_DRIVERS: "docker", + NEMOCLAW_MESSAGING_PLAN_B64: + expectedOutcome === "staged-messaging-refusal" + ? encodeMessagingPlan( + makeMessagingPlan({ sandboxName: "my-assistant", channels: ["telegram"] }), + ) + : "", }; const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, @@ -340,10 +375,30 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { false, ); }; + const assertStagedMessagingRefusal = () => { + assert.match( + payload.creationError, + /supports providerless sandbox creation only.*No sandbox or provider was created/u, + ); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, false); + assert.equal(payload.createCommand, null); + assert.equal(payload.registeredSandbox, null); + assert.equal(payload.credentialReadCalls, 0); + assert.equal(payload.routeReservationCalls, 0); + assert.deepEqual(providerEffectCommands, []); + assert.equal( + payload.commandNames.some((command: string) => + /(?:^|\s)(?:docker build|policy (?:set|apply)|sandbox create)(?:\s|$)/u.test(command), + ), + false, + ); + }; const assertSuccessfulCreation = () => { - assert.equal(payload.creationError, null); + assert.equal(payload.creationError, null, result.stderr); assert.equal(payload.sandboxName, "my-assistant"); assert.ok(payload.sandboxListCalls >= 2); + assert.equal(payload.registeredSandbox.workload.kind, "managed-image"); assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); assert.equal( payload.registeredSandbox.lifecycleLiveIdentityFingerprint, @@ -414,6 +469,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { "managed-provider": assertManagedProviderCreation, "provider-refusal": assertProviderBackedApfRefusal, "providerless-apf": assertProviderlessApfCreation, + "staged-messaging-refusal": assertStagedMessagingRefusal, "cancel-after-create": assertCancellationRecovery, }; assertions[expectedOutcome](); diff --git a/test/onboarding/onboard-fsm-live-slices.test.ts b/test/onboarding/onboard-fsm-live-slices.test.ts index f09760f8f59..8252432bb74 100644 --- a/test/onboarding/onboard-fsm-live-slices.test.ts +++ b/test/onboarding/onboard-fsm-live-slices.test.ts @@ -23,6 +23,7 @@ type ProbeMode = | "authoritative-core-gateway-policy-tier" | "dashboard-port-composition" | "ordinary-policy-tier" + | "providerless-staged-messaging" | "ahead-core"; interface ProbeOptions { @@ -456,6 +457,7 @@ const { onboard } = require(${onboardPath}); acceptThirdPartySoftware: true, noGpu: true, sandboxName: "fsm-sandbox", + apfInterceptorRequested: scenario.mode === "providerless-staged-messaging", resume: scenario.mode === "resume-initial" || scenario.mode.includes("core-gateway"), ...(scenario.mode.startsWith("authoritative-") ? { @@ -473,7 +475,9 @@ const { onboard } = require(${onboardPath}); error === sentinel || error?.message === sentinel.message || (scenario.mode === "endpoint-override" && - error?.name === "OpenShellGatewayEndpointOverrideError") + error?.name === "OpenShellGatewayEndpointOverrideError") || + (scenario.mode === "providerless-staged-messaging" && + /supports providerless sandbox creation only/.test(String(error?.message))) ) { const payload = JSON.stringify({ called }); if (scenario.mode === "dashboard-port-composition") { @@ -502,6 +506,19 @@ const { onboard } = require(${onboardPath}); ? { OPENSHELL_GATEWAY_ENDPOINT: "http://127.0.0.1:65535" } : {}), ...(options.policyTier ? { NEMOCLAW_POLICY_TIER: options.policyTier } : {}), + ...(scenario.mode === "providerless-staged-messaging" + ? { + NEMOCLAW_MESSAGING_PLAN_B64: Buffer.from( + JSON.stringify({ + schemaVersion: 1, + sandboxName: "fsm-sandbox", + agent: "openclaw", + workflow: "onboard", + channels: [{ channelId: "telegram", active: true }], + }), + ).toString("base64"), + } + : {}), }, timeout: scenario.mode === "dashboard-port-composition" ? 60_000 : probeTimeoutMs, }, @@ -541,6 +558,13 @@ describe("live onboard FSM slice boundaries", () => { assert.deepEqual(runSliceProbe({ slice: "initial", mode: "endpoint-override" }), []); }); + it("rejects staged messaging before entering the onboarding state machine (#9833)", () => { + assert.deepEqual( + runSliceProbe({ slice: "initial", mode: "providerless-staged-messaging" }), + [], + ); + }); + it("enters the core slice after the initial slice reaches provider selection", () => { assert.deepEqual(runSliceProbe({ slice: "core" }), ["initial:init", "core"]); }); From 06256df262cf2a692007f00f1b5056f517b4b185 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 21:10:35 -0700 Subject: [PATCH 14/42] fix(onboard): surface retained sandbox recovery Signed-off-by: Apurv Kumaria --- .../sandbox-create/orchestration.test.ts | 13 ++++++-- .../onboard/sandbox-create/orchestration.ts | 16 +++++----- .../onboard-fresh-create-identity.test.ts | 32 +++++++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 32013ba1248..e15ca54711c 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -446,12 +446,18 @@ describe("sandbox create policy authority checks", () => { }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).message).toMatch( + new RegExp( + `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*did not run OpenShell's mutable-name deletion command.*Do not delete the sandbox by mutable sandbox name.*OpenShell administrator.*identity-bound recovery or removal procedure`, + "u", + ), + ); expect((error as AggregateError).errors).toEqual( expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching( new RegExp( - `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*Do not delete the sandbox by name, even after this comparison.*Contact the OpenShell administrator for an identity-bound recovery or removal procedure`, + `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*did not run OpenShell's mutable-name deletion command.*Do not delete the sandbox by mutable sandbox name.*OpenShell administrator.*identity-bound recovery or removal procedure`, "u", ), ), @@ -489,7 +495,7 @@ describe("sandbox create policy authority checks", () => { expect.objectContaining({ message: expect.stringMatching( new RegExp( - `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*Do not delete the sandbox by name, even after this comparison`, + `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*Do not delete the sandbox by mutable sandbox name`, "u", ), ), @@ -804,6 +810,9 @@ describe("sandbox create policy authority checks", () => { }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).message).toMatch( + /left sandbox 'alpha' in place.*did not return a durable sandbox identity fingerprint.*Do not delete the sandbox by mutable sandbox name.*identity-bound recovery or removal procedure/u, + ); expect((error as AggregateError).errors).toEqual( expect.arrayContaining([ expect.objectContaining({ message: expect.stringContaining("post-create verification") }), diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 60877047b0a..b662fdbb02b 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -298,16 +298,16 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< ? validationError.message : null; const identityGuidance = exactIdentity - ? ` Durable sandbox identity fingerprint: ${exactIdentity}. Use it only to compare the surviving sandbox with the failed create. Do not delete the sandbox by name, even after this comparison. Contact the OpenShell administrator for an identity-bound recovery or removal procedure.` - : " OpenShell did not return a durable identity for comparison. Do not delete the sandbox by name. Contact the OpenShell administrator for an identity-bound recovery or removal procedure."; - compensationErrors.push( - new Error( - `NemoClaw left sandbox '${input.sandboxName}' in place after policy authority validation failed because OpenShell can delete it only by mutable name.${identityGuidance}`, - ), - ); + ? `Durable sandbox identity fingerprint: ${exactIdentity}. Use it only to compare the surviving sandbox with the failed create.` + : "OpenShell did not return a durable sandbox identity fingerprint for comparison."; + const recoveryGuidance = + `NemoClaw left sandbox '${input.sandboxName}' in place after policy authority validation failed. ` + + `${identityGuidance} NemoClaw did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. ` + + "Do not delete the sandbox by mutable sandbox name. Ask the OpenShell administrator to inspect the surviving sandbox and use an identity-bound recovery or removal procedure."; + compensationErrors.push(new Error(recoveryGuidance)); throw new AggregateError( [validationError, ...compensationErrors], - `Sandbox policy authority validation failed after creation${validationDetail ? `: ${validationDetail}` : ""}; automatic sandbox cleanup was not safe.`, + `Sandbox policy authority validation failed after creation${validationDetail ? `: ${validationDetail}` : ""}; automatic sandbox cleanup was not safe. ${recoveryGuidance}`, ); }; const verifyCreatedSandbox = async (created: Created): Promise => { diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 4dfb5bc6fba..2058023473e 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -45,6 +45,14 @@ describe("fresh create identity", () => { agent: null, expectedOutcome: "providerless-apf" as const, }, + { + title: "surfaces retained sandbox recovery through the public error message (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-authority-refusal" as const, + }, { title: "rejects staged messaging intent before any onboarding side effect (#9833)", apfInterceptorRequested: true, @@ -128,6 +136,9 @@ const model = ${JSON.stringify(model)}; const provider = ${JSON.stringify(provider)}; const cancelAfterCreate = ${JSON.stringify(expectedOutcome === "cancel-after-create")}; const stagedMessagingRefusal = ${JSON.stringify(expectedOutcome === "staged-messaging-refusal")}; +const postCreateAuthorityRefusal = ${JSON.stringify( + expectedOutcome === "post-create-authority-refusal", + )}; let cancelPrompt = false; const originalGetCredential = credentials.getCredential; if (stagedMessagingRefusal) { @@ -193,6 +204,9 @@ runner.run = (command, opts = {}) => { model, apfInterceptorRequested, onVerifyCreatedPolicy: (input) => { + if (postCreateAuthorityRefusal) { + throw new Error("external policy authority changed"); + } effectivePolicy = require(${policyMergePath}).parseOpenShellPolicy( fs.readFileSync(input.policySourcePath, "utf8"), ).policy; @@ -444,6 +458,23 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assert.doesNotMatch(payload.createCommand, /(?:^|\s)--provider(?:\s|$)/u); assert.deepEqual(providerExposureCommands, []); }; + const assertPostCreateAuthorityRefusal = () => { + const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.match(payload.creationError, /left sandbox 'my-assistant' in place/u); + assert.match(payload.creationError, new RegExp(identityFingerprint, "u")); + assert.match( + payload.creationError, + /did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox/u, + ); + assert.match(payload.creationError, /Do not delete the sandbox by mutable sandbox name/u); + assert.match( + payload.creationError, + /Ask the OpenShell administrator.*identity-bound recovery or removal procedure/u, + ); + }; const assertCancellationRecovery = () => { const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.exitCode, 1); @@ -469,6 +500,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { "managed-provider": assertManagedProviderCreation, "provider-refusal": assertProviderBackedApfRefusal, "providerless-apf": assertProviderlessApfCreation, + "post-create-authority-refusal": assertPostCreateAuthorityRefusal, "staged-messaging-refusal": assertStagedMessagingRefusal, "cancel-after-create": assertCancellationRecovery, }; From cd0d674b1a2f9a68afcf480b6da13fa6a1b2f8eb Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 21:15:24 -0700 Subject: [PATCH 15/42] docs(onboard): complete cancellation recovery guidance Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 7 ++++++- src/lib/onboard/cancel-rollback.test.ts | 8 +++++++- src/lib/onboard/cancel-rollback.ts | 10 +++++++--- src/lib/onboard/lifecycle-contracts.md | 2 +- test/onboarding/onboard-fresh-create-identity.test.ts | 4 ++++ 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index cb33ec5122f..bb54235ef36 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -896,7 +896,12 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. If you cancel a brand-new onboarding run at the policy-tier selector or either policy-preset selector after sandbox creation, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. -NemoClaw reports the durable sandbox identity fingerprint when it is available and never deletes the sandbox by mutable name. +NemoClaw reports the durable sandbox identity fingerprint when it is available. +It does not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. +Do not delete the sandbox by mutable name. +Provider registrations and gateway-bound credentials created before cancellation may remain. +Ask an OpenShell administrator to inspect the exact sandbox identity, provider registrations, and gateway-bound credentials, then retain them or remove only resources whose ownership is confirmed. +Before reusing the sandbox name, confirm that the exact sandbox is absent and rotate any credential that may have been configured or exposed before cancellation. Do not resume or recreate that incomplete sandbox. After an OpenShell administrator verifies the live durable ID, removes that exact sandbox through an identity-bound procedure, and confirms removal, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and change only the sandbox name. `--fresh` starts a new session and does not retain those selections. diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 6c25f665093..7880bd6be3a 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -29,7 +29,12 @@ describe("createSandboxCancelRollback", () => { expect(guidance).toContain("preserved incomplete sandbox 'new-sb'"); expect(guidance).toContain(SANDBOX_FINGERPRINT); expect(guidance).toContain("OpenShell administrator"); + expect(guidance).toContain("did not run OpenShell's mutable-name deletion command"); expect(guidance).toContain("Do not delete the sandbox by mutable sandbox name"); + expect(guidance).toContain("Provider registrations and gateway-bound credentials"); + expect(guidance).toContain("remove only resources whose ownership is confirmed"); + expect(guidance).toContain("confirm that the exact sandbox is absent"); + expect(guidance).toContain("rotate any credential"); }); it.each([ @@ -164,7 +169,8 @@ describe("buildCancelRollbackMessage", () => { expect(message).toContain("preserved incomplete sandbox 'sb'"); expect(message).toContain(SANDBOX_FINGERPRINT); - expect(message).toContain("identity-bound recovery or removal"); + expect(message).toContain("identity-bound inspection, recovery, or removal"); expect(message).not.toContain("openshell sandbox delete"); + expect(message).not.toContain("cannot delete it by immutable identity"); }); }); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index c71ae982bc8..56941d35eaa 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -45,17 +45,21 @@ export function buildCancelRollbackMessage( ): string[] { return [ "", - ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}' because OpenShell cannot delete it by immutable identity.`, + ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}'.`, ...(sandboxIdentityFingerprint ? [ ` Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}`, - " Preserve this fingerprint and give it to an OpenShell administrator for identity-bound recovery or removal.", + " Preserve this fingerprint for identity-bound inspection, recovery, or removal.", ] : [ " Its durable identity fingerprint is unavailable; preserve the registry and onboarding recovery state.", - " Ask an OpenShell administrator to establish the immutable sandbox identity before recovery or removal.", + " Ask an OpenShell administrator to establish the exact sandbox identity before recovery or removal.", ]), + " NemoClaw did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox.", " Do not delete the sandbox by mutable sandbox name.", + " Provider registrations and gateway-bound credentials created before cancellation may remain.", + " Ask an OpenShell administrator to inspect the exact sandbox identity, provider registrations, and gateway-bound credentials. Retain them or remove only resources whose ownership is confirmed.", + " Before reusing the sandbox name, confirm that the exact sandbox is absent and rotate any credential that may have been configured or exposed before cancellation.", ]; } diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 86fd5a09d35..19944b76ba6 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because OpenShell cannot condition deletion on its durable identity. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Cancellation after creation preserves the incomplete session, registry row, and identity evidence for administrator recovery and never deletes by mutable name. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because NemoClaw refuses the available mutable-name deletion command when it could target a replacement. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Cancellation after creation preserves the incomplete session, registry row, and identity evidence for administrator recovery. Provider registrations and gateway-bound credentials may remain; recovery inspects their exact ownership, removes only confirmed resources, verifies sandbox absence, and rotates affected credentials. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 2058023473e..0e8e344e2bd 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -495,6 +495,10 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assert.match(result.stderr, /preserved incomplete sandbox 'my-assistant'/u); assert.match(result.stderr, new RegExp(identityFingerprint, "u")); assert.match(result.stderr, /Do not delete the sandbox by mutable sandbox name/u); + assert.match(result.stderr, /Provider registrations and gateway-bound credentials/u); + assert.match(result.stderr, /remove only resources whose ownership is confirmed/u); + assert.match(result.stderr, /confirm that the exact sandbox is absent/u); + assert.match(result.stderr, /rotate any credential/u); }; const assertions = { "managed-provider": assertManagedProviderCreation, From e861eaf015df7f615d3b9a350c27d1ceb0ef43c5 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 23:30:02 -0700 Subject: [PATCH 16/42] fix(onboard): block cancelled sandbox reentry Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 2 +- src/lib/onboard.ts | 8 +- src/lib/onboard/cancel-rollback.test.ts | 27 +++ src/lib/onboard/cancel-rollback.ts | 16 ++ src/lib/onboard/entry-options.test.ts | 74 ++++++++ src/lib/onboard/entry-options.ts | 71 ++++++-- src/lib/onboard/exit-step-failure.test.ts | 14 ++ src/lib/onboard/exit-step-failure.ts | 1 + .../onboard-session-normalization.test.ts | 30 +++- src/lib/state/onboard-session.test.ts | 36 ++-- src/lib/state/onboard-session.ts | 99 ++++++++++ .../onboard-fresh-create-identity.test.ts | 169 +++++++++++++++--- 12 files changed, 490 insertions(+), 57 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index bb54235ef36..d87250fc2ce 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -902,7 +902,7 @@ Do not delete the sandbox by mutable name. Provider registrations and gateway-bound credentials created before cancellation may remain. Ask an OpenShell administrator to inspect the exact sandbox identity, provider registrations, and gateway-bound credentials, then retain them or remove only resources whose ownership is confirmed. Before reusing the sandbox name, confirm that the exact sandbox is absent and rotate any credential that may have been configured or exposed before cancellation. -Do not resume or recreate that incomplete sandbox. +NemoClaw records the incomplete sandbox as recovery-only and rejects automatic resume, explicit `--resume`, reuse, and recreation. After an OpenShell administrator verifies the live durable ID, removes that exact sandbox through an identity-bound procedure, and confirms removal, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and change only the sandbox name. `--fresh` starts a new session and does not retain those selections. diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1f1ed9a3604..829d4f3f0ad 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2635,9 +2635,9 @@ const onboardRuntimeBoundary = new OnboardRuntimeBoundary({ toSessionUpdates(updates as Parameters[0]), maybeForceE2eStepFailure, }); - -const sandboxCancelRollback = installSandboxCancelRollback({}); // #4614 - +const sandboxCancelRollback = installSandboxCancelRollback({ + recordRecovery: onboardSession.markCancellationRecovery, +}); // #4614 const { arePolicyPresetsApplied, computeSetupPresetSuggestions, @@ -2752,7 +2752,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { AUTO_YES = opts.autoYes === true || process.env.NEMOCLAW_YES === "1"; const entryOptions = onboardEntryOptions.resolveDefaultRunEntryOptions( opts, - onboardSession.loadSession()?.status ?? null, + onboardSession.loadSession(), validateName, ); const { fresh, nonInteractive, cannotPrompt, resume } = entryOptions; diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 7880bd6be3a..cb55fac2afc 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -115,9 +115,11 @@ describe("createSandboxCancelRollback", () => { describe("installSandboxCancelRollback", () => { it("registers a non-destructive exit handler that retains external recovery state (#9833)", () => { const log = vi.fn(); + const recordRecovery = vi.fn(); const exitHandlers: Array<() => void> = []; const rollback = installSandboxCancelRollback({ log, + recordRecovery, registerExitHandler: (handler) => exitHandlers.push(handler), }); @@ -126,6 +128,31 @@ describe("installSandboxCancelRollback", () => { rollback.markCancelled(); exitHandlers[0](); + expect(recordRecovery).toHaveBeenCalledWith("new-sb", SANDBOX_FINGERPRINT); + expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); + }); + + it("persists recovery before a deferred process exit and records it once (#9833)", () => { + const log = vi.fn(); + const recordRecovery = vi.fn(); + const exitHandlers: Array<() => void> = []; + const rollback = installSandboxCancelRollback({ + log, + recordRecovery, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + const deferredExit = new Error("deferred exit"); + const cancel = makeOnboardCancelExit(rollback, vi.fn(), () => { + throw deferredExit; + }); + + rollback.arm("new-sb", SANDBOX_FINGERPRINT); + expect(() => cancel()).toThrow(deferredExit); + expect(recordRecovery).toHaveBeenCalledOnce(); + expect(recordRecovery).toHaveBeenCalledWith("new-sb", SANDBOX_FINGERPRINT); + + exitHandlers[0](); + expect(recordRecovery).toHaveBeenCalledOnce(); expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); }); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index 56941d35eaa..2b1f5f1e2bb 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -24,6 +24,8 @@ export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preser export interface SandboxCancelRollbackDeps { /** Emit an operator-facing line (stderr). */ log(message: string): void; + /** Persist the recovery-only session before process exit completes. */ + recordRecovery?(sandboxName: string, sandboxIdentityFingerprint?: string): void; } export interface SandboxCancelRollback { @@ -65,6 +67,7 @@ export function buildCancelRollbackMessage( export interface InstallSandboxCancelRollbackOptions { log?: (message: string) => void; + recordRecovery?: SandboxCancelRollbackDeps["recordRecovery"]; /** Override for tests; defaults to `process.on("exit", ...)`. */ registerExitHandler?: (handler: () => void) => void; } @@ -82,6 +85,7 @@ export function installSandboxCancelRollback( ): SandboxCancelRollback { const rollback = createSandboxCancelRollback({ log: opts.log ?? ((message) => console.error(message)), + recordRecovery: opts.recordRecovery, }); const register = opts.registerExitHandler ?? @@ -117,8 +121,15 @@ export function createSandboxCancelRollback( readonly identityFingerprint: string | null; } | null = null; let cancelRequested = false; + let recoveryRecorded = false; let done = false; + const recordArmedRecovery = (): void => { + if (recoveryRecorded || armedSandbox === null) return; + recoveryRecorded = true; + deps.recordRecovery?.(armedSandbox.name, armedSandbox.identityFingerprint ?? undefined); + }; + return { arm(sandboxName: string, sandboxIdentityFingerprint?: string): void { armedSandbox = { @@ -135,6 +146,10 @@ export function createSandboxCancelRollback( }, markCancelled(): void { cancelRequested = true; + // Persist before requesting process exit. This also covers callers that + // defer the real exit and prevents later exit handlers from replacing + // the recovery-only marker with an ordinary resumable failure. + recordArmedRecovery(); }, isArmed(): boolean { return armedSandbox !== null; @@ -143,6 +158,7 @@ export function createSandboxCancelRollback( if (done || !cancelRequested || armedSandbox === null) return; done = true; const { name: sandboxName, identityFingerprint } = armedSandbox; + recordArmedRecovery(); armedSandbox = null; for (const line of buildCancelRollbackMessage( sandboxName, diff --git a/src/lib/onboard/entry-options.test.ts b/src/lib/onboard/entry-options.test.ts index 93292f2882f..8c55365e926 100644 --- a/src/lib/onboard/entry-options.test.ts +++ b/src/lib/onboard/entry-options.test.ts @@ -213,6 +213,80 @@ describe("resolveOnboardEntryOptions", () => { expect(result.resume).toBe(false); }); + it.each([ + ["automatic", {}], + ["explicit", { resume: true }], + ])("rejects %s recovery-only continuation before onboarding effects", (_label, opts) => { + const deps = createDeps(); + + expect(() => + resolveOnboardEntryOptions( + { + opts, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "recovery_required", + persistedRecoverySandboxName: "retained-sb", + }, + deps, + ), + ).toThrow(ExitError); + expect(deps.error).toHaveBeenCalledWith( + expect.stringContaining("preserved sandbox 'retained-sb' in recovery-only state"), + ); + expect(deps.error).toHaveBeenCalledWith( + expect.stringContaining("resume, reuse, and recreation are disabled"), + ); + }); + + it.each([ + ["a missing name", null], + ["the retained name", "retained-sb"], + ])("rejects recovery-only --fresh with %s", (_label, sandboxName) => { + const deps = createDeps(); + + expect(() => + resolveOnboardEntryOptions( + { + opts: { fresh: true, sandboxName }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "recovery_required", + persistedRecoverySandboxName: "retained-sb", + }, + deps, + ), + ).toThrow(ExitError); + expect(deps.error).toHaveBeenCalledWith( + expect.stringContaining("--fresh with an explicit sandbox name different"), + ); + }); + + it("allows recovery-only --fresh with a different explicit name", () => { + const deps = createDeps(); + + const result = resolveOnboardEntryOptions( + { + opts: { fresh: true, sandboxName: "replacement-sb" }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "recovery_required", + persistedRecoverySandboxName: "retained-sb", + }, + deps, + ); + + expect(result).toMatchObject({ + fresh: true, + resume: false, + requestedSandboxName: "replacement-sb", + }); + expect(deps.error).not.toHaveBeenCalled(); + }); + it("does not auto-resume when --fresh is set even with an in_progress session (#5470)", () => { const deps = createDeps(); diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index fa38d468eb6..cde4ec3700d 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -34,6 +34,7 @@ export interface OnboardEntryOptionsInput { * flag-only behavior for callers that don't load the session. */ persistedSessionStatus?: string | null; + persistedRecoverySandboxName?: string | null; } export interface OnboardEntryOptionsDeps { @@ -58,6 +59,11 @@ export interface ResolvedOnboardEntryOptions { cannotPrompt: boolean; } +type PersistedOnboardEntrySession = { + readonly status: string; + readonly cancellationRecovery?: { readonly sandboxName: string } | null; +}; + type NonInteractiveEntryOptions = { nonInteractive?: boolean }; type ResumableEntryOptions = NonInteractiveEntryOptions & { resume?: boolean; @@ -105,6 +111,7 @@ export function resolveOnboardRunOptions( stdinIsTty: Boolean(process.stdin?.isTTY), stdoutIsTty: Boolean(process.stdout?.isTTY), }, + persistedRecoverySandboxName: string | null = null, ) { const resume = options.resume === true || (options.fresh !== true && persistedSessionStatus === "in_progress"); @@ -115,7 +122,13 @@ export function resolveOnboardRunOptions( return { resume, nonInteractive, - entryOptionsInput: { opts: options, env, ...terminal, persistedSessionStatus }, + entryOptionsInput: { + opts: options, + env, + ...terminal, + persistedSessionStatus, + persistedRecoverySandboxName, + }, }; } @@ -125,12 +138,15 @@ export function resolveOnboardRunEntryOptions( persistedSessionStatus: string | null, isNonInteractiveEnv: () => boolean, deps: Omit, + persistedRecoverySandboxName: string | null = null, ) { const context = resolveOnboardRunOptions( options, env, persistedSessionStatus, isNonInteractiveEnv, + undefined, + persistedRecoverySandboxName, ); return { ...context, @@ -143,18 +159,25 @@ export function resolveOnboardRunEntryOptions( export function resolveDefaultRunEntryOptions( options: OnboardEntryOptionsInput["opts"] & { autoYes?: boolean; nonInteractive?: boolean }, - persistedSessionStatus: string | null, + persistedSession: PersistedOnboardEntrySession | null, validateSandboxName: OnboardEntryOptionsDeps["validateName"], env: NodeJS.ProcessEnv = process.env, ) { - return resolveOnboardRunEntryOptions(options, env, persistedSessionStatus, isNonInteractiveEnv, { - validateName: validateSandboxName, - reservedSandboxNames: RESERVED_SANDBOX_NAMES, - cliDisplayName, - getNameValidationGuidance, - error: (message) => console.error(message), - exitProcess: (code) => process.exit(code), - }); + return resolveOnboardRunEntryOptions( + options, + env, + persistedSession?.status ?? null, + isNonInteractiveEnv, + { + validateName: validateSandboxName, + reservedSandboxNames: RESERVED_SANDBOX_NAMES, + cliDisplayName, + getNameValidationGuidance, + error: (message) => console.error(message), + exitProcess: (code) => process.exit(code), + }, + persistedSession?.cancellationRecovery?.sandboxName ?? null, + ); } export function assertDefaultSandboxNameAllowed(sandboxName: string): void { @@ -280,6 +303,34 @@ export function resolveOnboardEntryOptions( } requestedSandboxName = validated; } + if (input.persistedSessionStatus === "recovery_required") { + const recoverySandboxName = input.persistedRecoverySandboxName?.trim() || null; + if (!fresh) { + deps.error( + ` Onboarding cannot continue because cancellation preserved sandbox '${recoverySandboxName ?? "unknown"}' in recovery-only state.`, + ); + deps.error( + " Automatic and explicit resume, reuse, and recreation are disabled to protect the retained sandbox.", + ); + deps.error( + " After an OpenShell administrator completes identity-bound recovery or removal, start fresh with --fresh --name .", + ); + deps.exitProcess(1); + } + if ( + !recoverySandboxName || + !requestedSandboxName || + requestedSandboxName === recoverySandboxName + ) { + deps.error( + " Recovery-only onboarding state requires --fresh with an explicit sandbox name different from the retained sandbox.", + ); + deps.error( + " Verify identity-bound removal with an OpenShell administrator before starting the new onboarding session.", + ); + deps.exitProcess(1); + } + } if (cannotPrompt && !resume && requestedFromDockerfile && !requestedSandboxName) { deps.error( " --from requires --name (or NEMOCLAW_SANDBOX_NAME) when running without a TTY or with --non-interactive.", diff --git a/src/lib/onboard/exit-step-failure.test.ts b/src/lib/onboard/exit-step-failure.test.ts index 0f09145602f..bae8e2b353e 100644 --- a/src/lib/onboard/exit-step-failure.test.ts +++ b/src/lib/onboard/exit-step-failure.test.ts @@ -250,6 +250,20 @@ describe("incomplete-onboard --resume backstop (#6003)", () => { expect(runExitHandler(1)).not.toContain("--resume"); }); + it("preserves recovery-only cancellation through the later exit backstop (#9833)", () => { + const fingerprint = "a".repeat(64); + session.saveSession( + session.createSession({ lastStepStarted: "sandbox", sandboxName: "retained-sb" }), + ); + session.markCancellationRecovery("retained-sb", fingerprint); + const beforeExit = requireLoadedSession(); + + const output = runExitHandler(1); + + expect(output).not.toContain("--resume"); + expect(requireLoadedSession()).toEqual(beforeExit); + }); + it("stays silent when cancel cleanup clears the session before a signal is re-raised", async () => { const signalListeners = new Map<"SIGINT" | "SIGTERM", () => void>(); const kill = vi.fn(); diff --git a/src/lib/onboard/exit-step-failure.ts b/src/lib/onboard/exit-step-failure.ts index 19a18972a95..0977dae42fd 100644 --- a/src/lib/onboard/exit-step-failure.ts +++ b/src/lib/onboard/exit-step-failure.ts @@ -56,6 +56,7 @@ export function registerIncompleteOnboardExitFailureHandler( // printOnboardResumeHint also self-dedupes against tailored hints. const interrupted = markLastStartedStepFailed(deps, message, true); if (!interrupted) return; + if (interrupted.status === "recovery_required") return; printOnboardResumeHint(portable, undefined, interrupted.sandboxName); }; diff --git a/src/lib/state/onboard-session-normalization.test.ts b/src/lib/state/onboard-session-normalization.test.ts index 4a02572817d..7e04d971ec0 100644 --- a/src/lib/state/onboard-session-normalization.test.ts +++ b/src/lib/state/onboard-session-normalization.test.ts @@ -3,7 +3,12 @@ import { describe, expect, it } from "vitest"; -import { createSession, filterSafeUpdates, normalizeSession } from "./onboard-session"; +import { + createSession, + filterSafeUpdates, + normalizeSession, + summarizeForDebug, +} from "./onboard-session"; type LegacySession = Omit, "machine"> & { machine?: unknown; @@ -16,6 +21,29 @@ function requireNormalizedSession(legacy: LegacySession) { } describe("onboard session normalization", () => { + it("preserves valid recovery-only cancellation state (#9833)", () => { + const cancellationRecovery = { + reason: "cancelled_after_sandbox_creation" as const, + sandboxName: "retained-sb", + sandboxIdentityFingerprint: "a".repeat(64), + recordedAt: "2026-08-27T00:00:00.000Z", + }; + const normalized = normalizeSession({ + ...createSession({ sandboxName: "retained-sb" }), + resumable: false, + status: "recovery_required", + cancellationRecovery, + }); + + expect(normalized).toMatchObject({ + sandboxName: "retained-sb", + resumable: false, + status: "recovery_required", + cancellationRecovery, + }); + expect(summarizeForDebug(normalized)?.cancellationRecovery).toEqual(cancellationRecovery); + }); + it("keeps APF create intent and defaults legacy sessions to false (#9833)", () => { const selected = createSession({ apfInterceptorRequested: true }); expect(normalizeSession(selected)?.apfInterceptorRequested).toBe(true); diff --git a/src/lib/state/onboard-session.test.ts b/src/lib/state/onboard-session.test.ts index 7f76bd6b119..2414e1b5a68 100644 --- a/src/lib/state/onboard-session.test.ts +++ b/src/lib/state/onboard-session.test.ts @@ -129,24 +129,24 @@ describe("onboard session", () => { ); }); - it.each([ - true, - false, - ])("persists explicit observability intent when enabled=$enabled", (observabilityEnabled) => { - session.saveSession( - session.createSession({ - observabilityEnabled, - observabilityRequestedExplicitly: true, - }), - ); - const loaded = requireLoadedSession(session.loadSession()); - const summary = requireDebugSummary(session.summarizeForDebug()); - - expect(loaded.observabilityEnabled).toBe(observabilityEnabled); - expect(loaded.observabilityRequestedExplicitly).toBe(true); - expect(summary.observabilityEnabled).toBe(observabilityEnabled); - expect(summary.observabilityRequestedExplicitly).toBe(true); - }); + it.each([true, false])( + "persists explicit observability intent when enabled=$enabled", + (observabilityEnabled) => { + session.saveSession( + session.createSession({ + observabilityEnabled, + observabilityRequestedExplicitly: true, + }), + ); + const loaded = requireLoadedSession(session.loadSession()); + const summary = requireDebugSummary(session.summarizeForDebug()); + + expect(loaded.observabilityEnabled).toBe(observabilityEnabled); + expect(loaded.observabilityRequestedExplicitly).toBe(true); + expect(summary.observabilityEnabled).toBe(observabilityEnabled); + expect(summary.observabilityRequestedExplicitly).toBe(true); + }, + ); it("defaults legacy observability intent and provenance off", () => { const legacy = session.createSession() as unknown as Record; diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index f62a5f0ed3e..adb63ac75ba 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -65,6 +65,7 @@ export { normalizePersistedSandboxHostMounts } from "./registry/host-mount"; export const SESSION_VERSION = 1; export const MACHINE_SNAPSHOT_VERSION = 1; +export const CANCELLATION_RECOVERY_STATUS = "recovery_required"; const INVALID_HOST_MOUNT_SESSIONS = new WeakSet(); export const SESSION_DIR = nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY_PORT); export const SESSION_FILE = path.join(SESSION_DIR, "onboard-session.json"); @@ -107,6 +108,13 @@ export interface SessionFailure { interrupted?: boolean; } +export interface SessionCancellationRecovery { + readonly reason: "cancelled_after_sandbox_creation"; + readonly sandboxName: string; + readonly sandboxIdentityFingerprint: string | null; + readonly recordedAt: string; +} + export interface SessionMetadata { gatewayName: string; fromDockerfile: string | null; @@ -205,6 +213,7 @@ export interface Session { lastStepStarted: string | null; lastCompletedStep: string | null; failure: SessionFailure | null; + cancellationRecovery: SessionCancellationRecovery | null; agent: string | null; sandboxName: string | null; provider: string | null; @@ -365,6 +374,7 @@ export interface DebugSessionSummary { lastStepStarted: string | null; lastCompletedStep: string | null; failure: SessionFailure | null; + cancellationRecovery: SessionCancellationRecovery | null; gatewayAuthority: GatewayOwnerDescription | null; machine: OnboardMachineSnapshot; steps: Record; @@ -678,6 +688,31 @@ export function sanitizeFailure( return step || message ? { step, message, recordedAt, interrupted } : null; } +function parseSessionCancellationRecovery( + value: SessionJsonValue | undefined, +): SessionCancellationRecovery | null { + if (!isObject(value) || value.reason !== "cancelled_after_sandbox_creation") return null; + const sandboxName = readString(value.sandboxName); + const recordedAt = readCanonicalIsoTimestamp(value.recordedAt); + const fingerprint = + value.sandboxIdentityFingerprint === null ? null : readString(value.sandboxIdentityFingerprint); + if ( + !sandboxName || + sandboxName.length > NAME_MAX_LENGTH || + !NAME_VALID_PATTERN.test(sandboxName) || + !recordedAt || + (fingerprint !== null && !/^[0-9a-f]{64}$/u.test(fingerprint)) + ) { + return null; + } + return { + reason: "cancelled_after_sandbox_creation", + sandboxName, + sandboxIdentityFingerprint: fingerprint, + recordedAt, + }; +} + // ── Session CRUD ───────────────────────────────────────────────── function createMachineSnapshot( @@ -775,6 +810,9 @@ export function createSession(overrides: Partial = {}): Session { lastStepStarted: overrides.lastStepStarted ?? null, lastCompletedStep: overrides.lastCompletedStep ?? null, failure: overrides.failure ?? null, + cancellationRecovery: parseSessionCancellationRecovery( + overrides.cancellationRecovery as SessionJsonValue | undefined, + ), agent: overrides.agent ?? null, sandboxName: overrides.sandboxName ?? null, provider: overrides.provider ?? null, @@ -900,6 +938,14 @@ export function normalizeSession(data: Session | SessionJsonValue | undefined): ) { return null; } + const cancellationRecovery = parseSessionCancellationRecovery(data.cancellationRecovery); + if ( + hasOwn(data, "cancellationRecovery") && + data.cancellationRecovery !== null && + !cancellationRecovery + ) { + return null; + } const normalized = createSession({ sessionId: readString(data.sessionId) ?? undefined, @@ -943,11 +989,20 @@ export function normalizeSession(data: Session | SessionJsonValue | undefined): lastStepStarted: readString(data.lastStepStarted), lastCompletedStep: readString(data.lastCompletedStep), failure: sanitizeFailure(isObject(data.failure) ? data.failure : null), + cancellationRecovery, metadata: parseSessionMetadata(data.metadata), checkpoint: data.checkpoint as unknown as OnboardCheckpoint | null, }); normalized.resumable = data.resumable !== false; normalized.status = readString(data.status) ?? normalized.status; + if ( + (normalized.status === CANCELLATION_RECOVERY_STATUS) !== Boolean(cancellationRecovery) || + (cancellationRecovery !== null && + (normalized.resumable !== false || + normalized.sandboxName !== cancellationRecovery.sandboxName)) + ) { + return null; + } if ( normalized.stationExpressIntent && (data.resumable !== true || @@ -1577,6 +1632,43 @@ export function updateSession(mutator: (session: Session) => Session | void): Se return saveSession(next); } +export function markCancellationRecovery( + sandboxName: string, + sandboxIdentityFingerprint?: string, +): Session { + if ( + sandboxName.length > NAME_MAX_LENGTH || + !NAME_VALID_PATTERN.test(sandboxName) || + (sandboxIdentityFingerprint !== undefined && + !/^[0-9a-f]{64}$/u.test(sandboxIdentityFingerprint)) + ) { + throw new Error("Cannot record cancellation recovery with invalid sandbox identity data."); + } + return updateSession((session) => { + if (session.sandboxName !== null && session.sandboxName !== sandboxName) { + throw new Error("Cannot record cancellation recovery for a different onboarding sandbox."); + } + const recordedAt = new Date().toISOString(); + session.sandboxName = sandboxName; + session.resumable = false; + session.status = CANCELLATION_RECOVERY_STATUS; + session.cancellationRecovery = { + reason: "cancelled_after_sandbox_creation", + sandboxName, + sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? null, + recordedAt, + }; + session.failure = { + step: session.lastStepStarted, + message: + "Onboarding was cancelled after sandbox creation; administrator recovery is required.", + recordedAt, + interrupted: true, + }; + return session; + }); +} + export type CompareAndSwapSessionResult = "updated" | "busy" | "mismatch"; /** @@ -1754,6 +1846,12 @@ export function finalizeIncompleteOnboardStep( ): Session | null { const existing = loadSession(); if (!existing) return null; + // A cancellation after sandbox creation has its own fail-closed lifecycle. + // Preserve that durable marker when the ordinary process-exit backstop runs + // later in the same exit sequence. + if (existing.status === CANCELLATION_RECOVERY_STATUS && existing.cancellationRecovery !== null) { + return existing; + } if (isTerminalOnboardMachineState(existing.machine.state)) return existing; let emitted = false; @@ -1940,6 +2038,7 @@ export function summarizeForDebug( lastStepStarted: session.lastStepStarted, lastCompletedStep: session.lastCompletedStep, failure: sanitizeFailure(session.failure), + cancellationRecovery: session.cancellationRecovery, gatewayAuthority, machine: session.machine, steps: Object.fromEntries( diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 0e8e344e2bd..3319666a5d1 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -62,12 +62,28 @@ describe("fresh create identity", () => { expectedOutcome: "staged-messaging-refusal" as const, }, { - title: "preserves a newly created sandbox when non-TTY policy selection is cancelled (#9833)", + title: "makes a tier-cancelled created sandbox recovery-only (#9833)", apfInterceptorRequested: false, provider: "nvidia-prod", model: "gpt-5.4", agent: null, - expectedOutcome: "cancel-after-create" as const, + expectedOutcome: "cancel-after-create-tier" as const, + }, + { + title: "makes a tier-preset-cancelled created sandbox recovery-only (#9833)", + apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "cancel-after-create-tier-presets" as const, + }, + { + title: "makes a custom-preset-cancelled created sandbox recovery-only (#9833)", + apfInterceptorRequested: false, + provider: "nvidia-prod", + model: "gpt-5.4", + agent: null, + expectedOutcome: "cancel-after-create-custom-presets" as const, }, ])( "$title", @@ -134,27 +150,29 @@ const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; const agent = ${JSON.stringify(agent)}; const model = ${JSON.stringify(model)}; const provider = ${JSON.stringify(provider)}; -const cancelAfterCreate = ${JSON.stringify(expectedOutcome === "cancel-after-create")}; +const cancellationSelector = ${JSON.stringify( + expectedOutcome.startsWith("cancel-after-create-") + ? expectedOutcome.slice("cancel-after-create-".length) + : null, + )}; +const cancelAfterCreate = cancellationSelector !== null; +const recoveryReentry = process.env.NEMOCLAW_RECOVERY_REENTRY || ""; const stagedMessagingRefusal = ${JSON.stringify(expectedOutcome === "staged-messaging-refusal")}; const postCreateAuthorityRefusal = ${JSON.stringify( expectedOutcome === "post-create-authority-refusal", )}; let cancelPrompt = false; const originalGetCredential = credentials.getCredential; -if (stagedMessagingRefusal) { - credentials.getCredential = (...args) => { - credentialReadCalls += 1; - if (typeof args[0] !== "string") return null; - return originalGetCredential(...args); - }; -} +credentials.getCredential = (...args) => { + credentialReadCalls += 1; + if (typeof args[0] !== "string") return null; + return originalGetCredential(...args); +}; const originalReserveSandboxInferenceRoute = registry.reserveSandboxInferenceRoute; -if (stagedMessagingRefusal) { - registry.reserveSandboxInferenceRoute = (...args) => { - routeReservationCalls += 1; - return originalReserveSandboxInferenceRoute(...args); - }; -} +registry.reserveSandboxInferenceRoute = (...args) => { + routeReservationCalls += 1; + return originalReserveSandboxInferenceRoute(...args); +}; runner.run = (command, opts = {}) => { const cmd = _n(command); _deleted = _deleted || cmd.includes("sandbox delete"); @@ -198,11 +216,16 @@ runner.run = (command, opts = {}) => { if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; + const retainedRegistryEntry = recoveryReentry && fs.existsSync(${JSON.stringify(payloadPath)}) + ? JSON.parse(fs.readFileSync(${JSON.stringify(payloadPath)}, "utf8")).currentRegistryEntry + : null; + const registryMutationCalls = []; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", provider, model, apfInterceptorRequested, + getSandbox: () => retainedRegistryEntry, onVerifyCreatedPolicy: (input) => { if (postCreateAuthorityRefusal) { throw new Error("external policy authority changed"); @@ -211,7 +234,13 @@ runner.run = (command, opts = {}) => { fs.readFileSync(input.policySourcePath, "utf8"), ).policy; }, - registerSandbox: (entry) => { registeredSandbox = entry; }, + registerSandbox: (entry) => { + registeredSandbox = entry; + registryMutationCalls.push({ operation: "register", name: entry.name }); + }, + updateSandbox: (name) => { registryMutationCalls.push({ operation: "update", name }); }, + setDefault: (name) => { registryMutationCalls.push({ operation: "set-default", name }); }, + removeSandbox: (name) => { registryMutationCalls.push({ operation: "remove", name }); }, }); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => { @@ -269,13 +298,17 @@ childProcess.spawn = (...args) => { const onboardModule = require(${onboardPath}); const { createSandbox } = onboardModule; -if (cancelAfterCreate) { +if (cancelAfterCreate && !recoveryReentry) { const session = onboardModule.onboardSession.createSession({ mode: "interactive", sandboxName: "my-assistant", metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, }); onboardModule.onboardSession.saveSession(session); + onboardModule.registerIncompleteOnboardExitHandlerForSession( + onboardModule.onboardSession, + () => false, + ); } const writePayload = (sandboxName, creationError, exitCode = 0) => { @@ -296,6 +329,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { registeredSandbox, credentialReadCalls, routeReservationCalls, + registryMutationCalls, currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, savedSession: cancelAfterCreate ? onboardModule.onboardSession.loadSession() : null, createCommand: createCommand?.command ?? null, @@ -305,6 +339,25 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; + if (recoveryReentry) { + try { + await onboardModule.onboard({ + resume: recoveryReentry === "explicit", + fresh: recoveryReentry === "fresh-same", + sandboxName: recoveryReentry === "fresh-same" ? "my-assistant" : undefined, + deferProcessExit: true, + }); + writePayload(null, "recovery-only onboarding unexpectedly continued", 0); + } catch (error) { + writePayload( + null, + error instanceof Error ? error.message : String(error), + typeof error?.code === "number" ? error.code : 1, + ); + } + clearInterval(keepAlive); + return; + } const createArgs = fixtureMocks.sandboxCreateArgsWithVerifiedReservation( [null, model, provider, null, null, null, null, null, agent, null, null, null, []], createFixture, @@ -323,7 +376,20 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { if (cancelAfterCreate) { process.on("exit", (code) => writePayload(sandboxName, null, code)); cancelPrompt = true; - await onboardModule.selectPolicyTier(); + if (cancellationSelector === "tier") { + await onboardModule.selectPolicyTier(); + } else if (cancellationSelector === "tier-presets") { + await onboardModule.selectTierPresetsAndAccess( + "balanced", + [{ name: "github", description: "GitHub" }], + ["github"], + ); + } else { + await onboardModule.presetsCheckboxSelector( + [{ name: "github", description: "GitHub" }], + ["github"], + ); + } throw new Error("expected policy selection cancellation"); } writePayload(sandboxName, null); @@ -345,7 +411,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { ...process.env, HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: expectedOutcome === "cancel-after-create" ? "" : "1", + NEMOCLAW_NON_INTERACTIVE: expectedOutcome.startsWith("cancel-after-create-") ? "" : "1", OPENSHELL_DRIVERS: "docker", NEMOCLAW_MESSAGING_PLAN_B64: expectedOutcome === "staged-messaging-refusal" @@ -361,7 +427,8 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { timeout: 30000, }); - assert.equal(result.status, expectedOutcome === "cancel-after-create" ? 1 : 0, result.stderr); + const cancellationOutcome = expectedOutcome.startsWith("cancel-after-create-"); + assert.equal(result.status, cancellationOutcome ? 1 : 0, result.stderr); assert.ok(fs.existsSync(payloadPath), result.stderr); const payload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); const providerEffectCommands = payload.commandNames.filter((command: string) => @@ -486,8 +553,18 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { identityFingerprint, ); assert.equal(payload.currentRegistryEntry.name, "my-assistant"); - assert.equal(payload.savedSession.status, "in_progress"); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); assert.equal(payload.savedSession.sandboxName, "my-assistant"); + assert.equal( + payload.savedSession.cancellationRecovery.reason, + "cancelled_after_sandbox_creation", + ); + assert.equal(payload.savedSession.cancellationRecovery.sandboxName, "my-assistant"); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); assert.equal( payload.commandNames.some((command: string) => command.includes("sandbox delete")), false, @@ -499,6 +576,50 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assert.match(result.stderr, /remove only resources whose ownership is confirmed/u); assert.match(result.stderr, /confirm that the exact sandbox is absent/u); assert.match(result.stderr, /rotate any credential/u); + + const reentryCases = [ + { + mode: "automatic", + messages: [ + /preserved sandbox 'my-assistant' in recovery-only state/u, + /resume, reuse, and recreation are disabled/u, + ], + }, + { + mode: "explicit", + messages: [ + /preserved sandbox 'my-assistant' in recovery-only state/u, + /resume, reuse, and recreation are disabled/u, + ], + }, + { + mode: "fresh-same", + messages: [ + /requires --fresh with an explicit sandbox name different from the retained sandbox/u, + ], + }, + ] as const; + for (const { messages, mode: reentryMode } of reentryCases) { + const reentry = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...childEnv, + NEMOCLAW_RECOVERY_REENTRY: reentryMode, + }, + timeout: 30000, + }); + assert.equal(reentry.status, 0, reentry.stderr); + const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + assert.equal(reentryPayload.exitCode, 1); + messages.forEach((message) => assert.match(reentry.stderr, message)); + assert.deepEqual(reentryPayload.commandNames, []); + assert.equal(reentryPayload.credentialReadCalls, 0); + assert.equal(reentryPayload.routeReservationCalls, 0); + assert.deepEqual(reentryPayload.registryMutationCalls, []); + assert.deepEqual(reentryPayload.currentRegistryEntry, payload.currentRegistryEntry); + assert.deepEqual(reentryPayload.savedSession, payload.savedSession); + } }; const assertions = { "managed-provider": assertManagedProviderCreation, @@ -506,7 +627,9 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { "providerless-apf": assertProviderlessApfCreation, "post-create-authority-refusal": assertPostCreateAuthorityRefusal, "staged-messaging-refusal": assertStagedMessagingRefusal, - "cancel-after-create": assertCancellationRecovery, + "cancel-after-create-tier": assertCancellationRecovery, + "cancel-after-create-tier-presets": assertCancellationRecovery, + "cancel-after-create-custom-presets": assertCancellationRecovery, }; assertions[expectedOutcome](); }, From ac6717e4b12722998e954f8310e40896ce8596d8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 23:36:49 -0700 Subject: [PATCH 17/42] fix(onboard): preserve deferred create integration Signed-off-by: Apurv Kumaria --- src/lib/onboard/machine/handlers/sandbox.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 6363a7ebd0d..ef67b7da4e1 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -2477,6 +2477,7 @@ class SandboxStateFlow< null, registryMessagingAuthority, decision, + true, async (state) => this.checkpointMessaging(this.checkpointWebSearch(state, null), { plan: null, @@ -2557,8 +2558,7 @@ class SandboxStateFlow< const hasCreateTimeCredentialBindings = webSearchProviderBindings.length > 0 || messagingProviderBindings.length > 0; const deferCredentialProviderEffects = - this.deferSandboxEffectsUntilPolicyVerification() && - (this.options.apfInterceptorRequested === true || !hasCreateTimeCredentialBindings); + this.deferSandboxEffectsUntilPolicyVerification() && !hasCreateTimeCredentialBindings; if (!deferCredentialProviderEffects) { nextState = await activateCredentialProviders(nextState); } From 518c7ce22c38b718ff56c3cc56600940f6f3c0ca Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 26 Aug 2026 23:50:11 -0700 Subject: [PATCH 18/42] fix(onboard): persist recovery without sandbox identity Signed-off-by: Apurv Kumaria --- src/lib/onboard/created-sandbox-finalization.test.ts | 4 ++++ src/lib/onboard/created-sandbox-finalization.ts | 2 ++ src/lib/onboard/sandbox-create/orchestration.ts | 1 + 3 files changed, 7 insertions(+) diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index 93921e7b6cd..a4e08310e90 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -63,6 +63,7 @@ describe("new sandbox cancellation recovery", () => { const error = vi.spyOn(console, "error").mockImplementation(() => undefined); const runFile = vi.fn(); const armCancelRollback = vi.fn(); + const markCancellationRecovery = vi.fn(); expect(() => completeOrdinaryOnboardSandboxCreation( @@ -80,6 +81,7 @@ describe("new sandbox cancellation recovery", () => { gatewayName: "nemoclaw", providerExistsInGateway: () => true, armCancelRollback, + markCancellationRecovery, dockerInfoFormat: () => "", runCapture: () => "", revalidatePolicyAuthority: vi.fn(), @@ -96,6 +98,8 @@ describe("new sandbox cancellation recovery", () => { expect(guidance).toContain("add --fresh, and use a new sandbox name"); expect(runFile).not.toHaveBeenCalled(); expect(armCancelRollback).not.toHaveBeenCalled(); + expect(markCancellationRecovery).toHaveBeenCalledOnce(); + expect(markCancellationRecovery).toHaveBeenCalledWith("new-sandbox"); }); }); diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index e3d25089eb8..0133f1d071e 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -254,6 +254,7 @@ export function completeOrdinaryOnboardSandboxCreation( readonly gatewayName: string; readonly providerExistsInGateway: (providerName: string) => boolean; readonly armCancelRollback: (sandboxName: string, sandboxIdentityFingerprint: string) => void; + readonly markCancellationRecovery: (sandboxName: string) => unknown; readonly dockerInfoFormat: Parameters[0]["dockerInfoFormat"]; readonly runCapture: Parameters[0]["runCapture"]; readonly revalidatePolicyAuthority: (operation: string) => void; @@ -289,6 +290,7 @@ export function completeOrdinaryOnboardSandboxCreation( !lifecycleLiveIdentityFingerprint || !/^[0-9a-f]{64}$/u.test(lifecycleLiveIdentityFingerprint) ) { + deps.markCancellationRecovery(input.sandboxName); for (const line of [ "", ` Sandbox '${input.sandboxName}' was created on gateway '${deps.gatewayName}', but NemoClaw could not verify its durable identity.`, diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 87dbc377e51..e60d8b7f5a3 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -2164,6 +2164,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche gatewayName: GATEWAY_NAME, providerExistsInGateway, armCancelRollback: sandboxCancelRollback.arm, + markCancellationRecovery: onboardSession.markCancellationRecovery, dockerInfoFormat, runCapture, revalidatePolicyAuthority: (operation) => revalidatePolicyAuthority(true, operation), From 56fa746b79e4ab0ca1923ad48cf7b82f949e8d21 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 00:25:07 -0700 Subject: [PATCH 19/42] test(onboard): align created sandbox identity fixtures Signed-off-by: Apurv Kumaria --- src/lib/onboard/sandbox-gpu-create-flow.test.ts | 2 +- test/helpers/onboard-openshell-fixture.ts | 2 +- test/helpers/onboard-script-mocks.cjs | 4 ++-- test/onboarding/onboard-fresh-create-identity.test.ts | 6 +++--- test/onboarding/onboard-installer-restore-intent.test.ts | 6 +++--- test/onboarding/onboard-messaging.test.ts | 6 +++--- test/onboarding/onboard-prepared-build-context.test.ts | 2 +- test/onboarding/onboard-reservation-recreate.test.ts | 2 +- test/onboarding/onboard-sandbox-build.test.ts | 6 +++--- test/onboarding/onboard-terminal-dashboard.test.ts | 2 +- test/onboarding/onboard.test.ts | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 08ef87b805a..09bdbcdfa9f 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -354,7 +354,7 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { const adapterOverride = {} as never; deps.createManagedBootstrapAdapter = vi.fn(() => adapterOverride); vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => - args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", + args[1] === "get" ? "ID: alpha-sandbox-id\n" : "alpha Ready", ); recoverUnfinished.mockRejectedValueOnce(new Error("unfinished recovery failed")); diff --git a/test/helpers/onboard-openshell-fixture.ts b/test/helpers/onboard-openshell-fixture.ts index 5888d5beff3..510f3e1472d 100644 --- a/test/helpers/onboard-openshell-fixture.ts +++ b/test/helpers/onboard-openshell-fixture.ts @@ -14,7 +14,7 @@ export function writeOkOpenshell( ): void { const gatewayPort = options.gatewayPort ?? 8080; const sandboxGet = options.readySandboxGet - ? 'if [ "${1:-}" = sandbox ] && [ "${2:-}" = get ]; then printf "Sandbox:\\n\\n Id: fixture-created-sandbox\\n Name: %s\\n Phase: Ready\\n" "${!#}"; fi\n' + ? 'if [ "${1:-}" = sandbox ] && [ "${2:-}" = get ]; then printf "Sandbox:\\n\\n Id: sbx-4f2a91c0d7\\n Name: %s\\n Phase: Ready\\n" "${!#}"; fi\n' : ""; writeExecutable( path.join(fakeBin, "openshell"), diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 1d30af8ca2f..1c9523bd1ef 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -351,7 +351,7 @@ function mockCreatedSandboxIdentityList(command, options = {}) { const nonce = selector.slice(prefix.length); return JSON.stringify([ { - id: options.sandboxId || "fixture-created-sandbox", + id: options.sandboxId || "sbx-4f2a91c0d7", name: options.sandboxName || "my-assistant", labels: { "ai.nvidia.nemoclaw.create-attempt": nonce }, resource_version: 1, @@ -550,7 +550,7 @@ function managedSandboxPolicyReceiptFixture(entry, options = {}) { const gatewayName = options.gatewayName || "nemoclaw"; const gatewayPort = options.gatewayPort || 8080; const lifecycleGeneration = options.lifecycleGeneration || "123e4567-e89b-42d3-a456-426614174983"; - const sandboxId = options.sandboxId || "fixture-created-sandbox"; + const sandboxId = options.sandboxId || "sbx-4f2a91c0d7"; const sandboxIdentityFingerprint = require("node:crypto") .createHash("sha256") .update(sandboxId) diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 3319666a5d1..c431b11489e 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -509,9 +509,9 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assertSuccessfulCreation(); assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); assert.deepEqual(payload.killCalls, []); - assert.equal(payload.unrefCalls, 1); - assert.equal(payload.stdoutDestroyCalls, 1); - assert.equal(payload.stderrDestroyCalls, 1); + assert.equal(payload.unrefCalls, 0); + assert.equal(payload.stdoutDestroyCalls, 0); + assert.equal(payload.stderrDestroyCalls, 0); assert.equal(payload.registeredSandbox.policyAuthority, "nemoclaw-managed"); assert.ok(payload.registeredSandbox.policyCreationReceipt); assert.match(payload.createCommand, /--policy \S+/u); diff --git a/test/onboarding/onboard-installer-restore-intent.test.ts b/test/onboarding/onboard-installer-restore-intent.test.ts index bf0bf2f39de..1fb1cc2b8ce 100644 --- a/test/onboarding/onboard-installer-restore-intent.test.ts +++ b/test/onboarding/onboard-installer-restore-intent.test.ts @@ -78,7 +78,7 @@ runner.run = (command) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: fixture-created-sandbox\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { @@ -89,7 +89,7 @@ runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); if (createdIdentity !== null) return createdIdentity; } - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } @@ -441,7 +441,7 @@ runner.runCapture = (command) => { const normalized = _n(command); if (normalized.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; if (normalized.includes("policy get") && normalized.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); - if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (normalized.includes("sandbox list")) return "my-assistant NotReady"; // Keep dashboard allocation inside this restore-intent fixture; host port // occupancy is unrelated to the not-ready decision under test. diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index a1bcd67f2cb..e08ae148c99 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -159,7 +159,6 @@ const { createSandbox, setupMessagingChannels } = require(${onboardPath}); }); `; fs.writeFileSync(scriptPath, script); - const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", @@ -790,7 +789,8 @@ const { createSandbox } = require(${onboardPath}); const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, - encoding: "utf-8", timeout: 30_000, + encoding: "utf-8", + timeout: 30_000, env: { ...process.env, HOME: tmpDir, @@ -1283,7 +1283,7 @@ runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderR runner.runCapture = (command) => { // Existing sandbox that is ready if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) { - return "Name: my-assistant\nId: fixture-created-sandbox\n"; + return "Name: my-assistant\nId: sbx-4f2a91c0d7\n"; } if (_n(command).includes("sandbox list")) return "my-assistant Ready"; // All messaging providers already exist in gateway diff --git a/test/onboarding/onboard-prepared-build-context.test.ts b/test/onboarding/onboard-prepared-build-context.test.ts index 4b63a1d18b8..72cd35b41c5 100644 --- a/test/onboarding/onboard-prepared-build-context.test.ts +++ b/test/onboarding/onboard-prepared-build-context.test.ts @@ -179,7 +179,7 @@ runner.runCapture = (command) => { ].join("\n"); } if (normalized.includes("sandbox get")) { - return sandboxCreated ? sandboxName + "\nId: fixture-created-sandbox\n" : ""; + return sandboxCreated ? sandboxName + "\nId: sbx-4f2a91c0d7\n" : ""; } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; return ""; diff --git a/test/onboarding/onboard-reservation-recreate.test.ts b/test/onboarding/onboard-reservation-recreate.test.ts index 9653ac812f9..6c59dfff948 100644 --- a/test/onboarding/onboard-reservation-recreate.test.ts +++ b/test/onboarding/onboard-reservation-recreate.test.ts @@ -84,7 +84,7 @@ runner.runCapture = (command) => { const cmd = _n(command); const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); if (createdIdentity !== null) return createdIdentity; - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } diff --git a/test/onboarding/onboard-sandbox-build.test.ts b/test/onboarding/onboard-sandbox-build.test.ts index 8d2e3405ae4..0fc4436edbc 100644 --- a/test/onboarding/onboard-sandbox-build.test.ts +++ b/test/onboarding/onboard-sandbox-build.test.ts @@ -296,7 +296,7 @@ runner.run = (command, opts = {}) => { if (normalized.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } - return normalized.includes("sandbox get hermes-sandbox") ? { status: 0, stdout: Buffer.from("Name: hermes-sandbox\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; + return normalized.includes("sandbox get") && normalized.includes("hermes-sandbox") ? { status: 0, stdout: Buffer.from("Name: hermes-sandbox\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { commands.push({ command: _n([file, ...args]), env: opts.env || null }); @@ -306,7 +306,7 @@ runner.runCapture = (command) => { const normalized = _n(command); const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "hermes-sandbox" }); if (createdIdentity !== null) return createdIdentity; - if (normalized.includes("sandbox get hermes-sandbox")) return ""; + if (normalized.includes("sandbox get") && normalized.includes("hermes-sandbox")) return ""; if (normalized.includes("sandbox list")) return "hermes-sandbox Ready"; { const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); @@ -528,7 +528,7 @@ runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); if (createdIdentity !== null) return createdIdentity; if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) { - return sandboxCreated ? "Name: my-assistant\\nId: fixture-created-sandbox\\nPhase: Ready\\n" : ""; + return sandboxCreated ? "Name: my-assistant\\nId: sbx-4f2a91c0d7\\nPhase: Ready\\n" : ""; } if (normalized.includes("sandbox list")) return "my-assistant Ready"; { diff --git a/test/onboarding/onboard-terminal-dashboard.test.ts b/test/onboarding/onboard-terminal-dashboard.test.ts index 4b145aaa767..a0acdd3f902 100644 --- a/test/onboarding/onboard-terminal-dashboard.test.ts +++ b/test/onboarding/onboard-terminal-dashboard.test.ts @@ -129,7 +129,7 @@ runner.runCapture = (command) => { } if (normalized.includes("sandbox get") && normalized.includes(sandboxName)) { return scenario === "reuse" - ? [sandboxName, "Id: fixture-created-sandbox"].join(String.fromCharCode(10)) + ? [sandboxName, "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) : ""; } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; diff --git a/test/onboarding/onboard.test.ts b/test/onboarding/onboard.test.ts index 384782507ab..ce0a00afc49 100644 --- a/test/onboarding/onboard.test.ts +++ b/test/onboarding/onboard.test.ts @@ -696,7 +696,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; From 8b8903bf37d728dcde636c18495623c3b25d7687 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 01:07:59 -0700 Subject: [PATCH 20/42] fix(onboard): retry recovery persistence on exit Signed-off-by: Apurv Kumaria --- src/lib/onboard/cancel-rollback.test.ts | 25 +++++++++++++++++++++++++ src/lib/onboard/cancel-rollback.ts | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index cb55fac2afc..41c517e3498 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -156,6 +156,31 @@ describe("installSandboxCancelRollback", () => { expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); }); + it("retries recovery from the exit handler after the immediate durable write fails (#9833)", () => { + const log = vi.fn(); + const recordRecovery = vi + .fn() + .mockImplementationOnce(() => { + throw new Error("recovery write failed"); + }) + .mockImplementationOnce(() => undefined); + const exitHandlers: Array<() => void> = []; + const rollback = installSandboxCancelRollback({ + log, + recordRecovery, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + + rollback.arm("new-sb", SANDBOX_FINGERPRINT); + expect(() => rollback.markCancelled()).toThrow("recovery write failed"); + expect(recordRecovery).toHaveBeenCalledOnce(); + + exitHandlers[0](); + expect(recordRecovery).toHaveBeenCalledTimes(2); + expect(recordRecovery).toHaveBeenLastCalledWith("new-sb", SANDBOX_FINGERPRINT); + expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); + }); + it("preserves missing-checkpoint recovery state without a mutable-name fallback (#9833)", () => { const log = vi.fn(); const exitHandlers: Array<() => void> = []; diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index 2b1f5f1e2bb..b2ef6ad5efd 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -126,8 +126,8 @@ export function createSandboxCancelRollback( const recordArmedRecovery = (): void => { if (recoveryRecorded || armedSandbox === null) return; - recoveryRecorded = true; deps.recordRecovery?.(armedSandbox.name, armedSandbox.identityFingerprint ?? undefined); + recoveryRecorded = true; }; return { @@ -156,9 +156,9 @@ export function createSandboxCancelRollback( }, runIfArmed(): void { if (done || !cancelRequested || armedSandbox === null) return; - done = true; const { name: sandboxName, identityFingerprint } = armedSandbox; recordArmedRecovery(); + done = true; armedSandbox = null; for (const line of buildCancelRollbackMessage( sandboxName, From 62b102153ded4230dd42b8450fe4938710256b84 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 01:18:14 -0700 Subject: [PATCH 21/42] fix(onboard): reject provider plans before credential reads Signed-off-by: Apurv Kumaria --- .../onboard/machine/handlers/sandbox-apf.test.ts | 7 +++++++ src/lib/onboard/machine/handlers/sandbox.ts | 12 ++++++++++++ src/lib/onboard/messaging-prep.test.ts | 14 ++++++++++++++ src/lib/onboard/messaging-prep.ts | 7 +++++-- .../onboard-fresh-create-identity.test.ts | 1 + 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/machine/handlers/sandbox-apf.test.ts b/src/lib/onboard/machine/handlers/sandbox-apf.test.ts index 29fa411ef9d..523f0ded467 100644 --- a/src/lib/onboard/machine/handlers/sandbox-apf.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-apf.test.ts @@ -136,6 +136,7 @@ describe("APF sandbox create selection", () => { }), ).rejects.toThrow(/supports providerless sandbox creation only/u); + expect(calls.resolveCreateIntent).not.toHaveBeenCalled(); expect(calls.stageCredentialProviders).not.toHaveBeenCalled(); expect(calls.createSandbox).not.toHaveBeenCalled(); }); @@ -149,6 +150,9 @@ describe("APF sandbox create selection", () => { ...baseOptions(deps, session), fresh: true, apfInterceptorRequested: true, + model: "", + provider: "", + preferredInferenceApi: null, }), ).rejects.toThrow(/cannot adopt registered sandbox/u); @@ -174,6 +178,9 @@ describe("APF sandbox create selection", () => { ...baseOptions(deps, session), fresh: true, apfInterceptorRequested: true, + model: "", + provider: "", + preferredInferenceApi: null, }), ).rejects.toThrow(/cannot adopt live sandbox/u); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index ef67b7da4e1..b686c26808c 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -702,6 +702,8 @@ class SandboxStateFlow< this.options.env[WEB_SEARCH_PROVIDER_ENV], ).provider; const hasProviderIntent = + this.options.provider.trim().length > 0 || + this.options.model.trim().length > 0 || this.options.webSearchConfig !== null || explicitWebSearch !== null || this.options.selectedMessagingChannels.length > 0 || @@ -2211,6 +2213,16 @@ class SandboxStateFlow< requestedSandboxName, registryMessagingAuthoritySnapshot, ); + if ( + this.options.apfInterceptorRequested === true && + (extraProviderPlan.extraProviders.length > 0 || + extraProviderPlan.staleExtraProviders.length > 0 || + effectiveHermesToolGateways.length > 0) + ) { + throw new Error( + "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + ); + } // Build the complete create plan after acquiring the sandbox lock. A // baseline transaction may have started while onboarding waited, and a // pre-lock snapshot must never survive a destructive recreate. diff --git a/src/lib/onboard/messaging-prep.test.ts b/src/lib/onboard/messaging-prep.test.ts index 875e5303072..28f7b860508 100644 --- a/src/lib/onboard/messaging-prep.test.ts +++ b/src/lib/onboard/messaging-prep.test.ts @@ -43,6 +43,20 @@ function createInput( } describe("prepareCreateSandboxMessaging", () => { + it("does not read messaging credentials when no channel is enabled (#9833)", () => { + const getValidatedMessagingTokenByEnvKey = vi.fn(() => "secret-value"); + + const result = prepareCreateSandboxMessaging( + createInput({ + enabledChannels: [], + getValidatedMessagingTokenByEnvKey, + }), + ); + + expect(getValidatedMessagingTokenByEnvKey).not.toHaveBeenCalled(); + expect(result.messagingTokenDefs).toEqual([]); + }); + it("filters token definitions and reuses missing-token providers with matching bindings", () => { const registerExtraPlaceholderProviders = vi.fn(() => ["SLACK_BOT_TOKEN_AGENT_A"]); const providerMatchesGatewayCredential = vi.fn( diff --git a/src/lib/onboard/messaging-prep.ts b/src/lib/onboard/messaging-prep.ts index cdf40a51b00..cdd99d0530c 100644 --- a/src/lib/onboard/messaging-prep.ts +++ b/src/lib/onboard/messaging-prep.ts @@ -97,7 +97,6 @@ export function prepareCreateSandboxMessaging( return { name: credential.providerNameTemplate.replaceAll("{sandboxName}", input.sandboxName), envKey: credential.providerEnvKey, - token: input.getValidatedMessagingTokenByEnvKey(input.channels, credential.providerEnvKey), providerType: staticProviderType ?? MESSAGING_CREDENTIAL_PROVIDER_TYPE, retainWhileDisabled: staticProviderType !== null, }; @@ -107,7 +106,11 @@ export function prepareCreateSandboxMessaging( !enabledEnvKeys || enabledEnvKeys.has(envKey) || (retainWhileDisabled && disabledEnvKeys.has(envKey)), - ); + ) + .map((definition) => ({ + ...definition, + token: input.getValidatedMessagingTokenByEnvKey(input.channels, definition.envKey), + })); const messagingTokenDefs: MessagingTokenDef[] = messagingCredentialDefs .filter(({ envKey }) => !disabledEnvKeys.has(envKey)) .map(({ retainWhileDisabled: _retainWhileDisabled, ...definition }) => definition); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index c431b11489e..5c33f8a7f08 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -523,6 +523,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assert.equal(payload.registeredSandbox.policyCreationReceipt, undefined); assert.doesNotMatch(payload.createCommand, /(?:^|\s)--policy(?:\s|$)/u); assert.doesNotMatch(payload.createCommand, /(?:^|\s)--provider(?:\s|$)/u); + assert.equal(payload.credentialReadCalls, 0); assert.deepEqual(providerExposureCommands, []); }; const assertPostCreateAuthorityRefusal = () => { From 646240923b6c5d655628d0b17acf3a00e3928bc4 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 01:30:18 -0700 Subject: [PATCH 22/42] fix(onboard): preserve cancellation recovery guidance Signed-off-by: Apurv Kumaria --- src/lib/onboard/cancel-rollback.test.ts | 30 +++++++++++++++++++++++-- src/lib/onboard/cancel-rollback.ts | 22 ++++++++++++++---- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 41c517e3498..df39d9fa44e 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -172,13 +172,39 @@ describe("installSandboxCancelRollback", () => { }); rollback.arm("new-sb", SANDBOX_FINGERPRINT); - expect(() => rollback.markCancelled()).toThrow("recovery write failed"); + expect(() => rollback.markCancelled()).not.toThrow(); expect(recordRecovery).toHaveBeenCalledOnce(); exitHandlers[0](); expect(recordRecovery).toHaveBeenCalledTimes(2); expect(recordRecovery).toHaveBeenLastCalledWith("new-sb", SANDBOX_FINGERPRINT); - expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain(SANDBOX_FINGERPRINT); + expect(guidance).not.toContain("could not save the onboarding recovery record"); + }); + + it("exits and reports identity recovery when both durable writes fail (#9833)", () => { + const log = vi.fn(); + const recordRecovery = vi.fn(() => { + throw new Error("recovery write failed"); + }); + const exitHandlers: Array<() => void> = []; + const rollback = installSandboxCancelRollback({ + log, + recordRecovery, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + const exit = vi.fn(); + + rollback.arm("new-sb", SANDBOX_FINGERPRINT); + expect(() => makeOnboardCancelExit(rollback, vi.fn(), exit)()).not.toThrow(); + expect(exit).toHaveBeenCalledWith(1); + + expect(() => exitHandlers[0]()).not.toThrow(); + expect(recordRecovery).toHaveBeenCalledTimes(2); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain(SANDBOX_FINGERPRINT); + expect(guidance).toContain("could not save the onboarding recovery record"); }); it("preserves missing-checkpoint recovery state without a mutable-name fallback (#9833)", () => { diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index b2ef6ad5efd..e33ed08c493 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -122,12 +122,21 @@ export function createSandboxCancelRollback( } | null = null; let cancelRequested = false; let recoveryRecorded = false; + let recoveryPersistenceFailed = false; let done = false; - const recordArmedRecovery = (): void => { - if (recoveryRecorded || armedSandbox === null) return; - deps.recordRecovery?.(armedSandbox.name, armedSandbox.identityFingerprint ?? undefined); - recoveryRecorded = true; + const recordArmedRecovery = (): boolean => { + if (recoveryRecorded) return true; + if (armedSandbox === null) return false; + try { + deps.recordRecovery?.(armedSandbox.name, armedSandbox.identityFingerprint ?? undefined); + recoveryRecorded = true; + recoveryPersistenceFailed = false; + return true; + } catch { + recoveryPersistenceFailed = true; + return false; + } }; return { @@ -160,6 +169,11 @@ export function createSandboxCancelRollback( recordArmedRecovery(); done = true; armedSandbox = null; + if (recoveryPersistenceFailed) { + deps.log( + " NemoClaw could not save the onboarding recovery record; preserve the registry entry and exact sandbox identity for administrator recovery.", + ); + } for (const line of buildCancelRollbackMessage( sandboxName, identityFingerprint ?? undefined, From c7eea16106923841c583c7df90b7ea46190259d8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 02:22:22 -0700 Subject: [PATCH 23/42] fix(onboard): persist retained create recovery Signed-off-by: Apurv Kumaria --- .../sandbox-create/orchestration.test.ts | 41 +++++++++--- .../onboard/sandbox-create/orchestration.ts | 28 ++++++-- src/lib/state/onboard-session.ts | 66 ++++++++++++++++++- 3 files changed, 117 insertions(+), 18 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 5f7c732a7d4..a0bf2915798 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -46,12 +46,25 @@ describe("retained create recovery persistence", () => { session.saveSession(session.createSession({ sandboxName: "alpha" })); expect( - persistRetainedSandboxRecoveryMessage(message, session.finalizeIncompleteOnboardStep), + persistRetainedSandboxRecoveryMessage( + { + sandboxName: "alpha", + message, + ...(fingerprint ? { sandboxIdentityFingerprint: fingerprint } : {}), + }, + session.markRetainedSandboxRecovery, + ), ).toBe(true); const stored = session.loadSession(); - expect(stored?.machine.state).toBe("failed"); - expect(stored?.steps.sandbox?.status).toBe("failed"); + expect(stored?.status).toBe("recovery_required"); + expect(stored?.resumable).toBe(false); + expect(stored?.cancellationRecovery?.reason).toBe( + "retained_after_sandbox_creation_failure", + ); + expect(stored?.cancellationRecovery?.sandboxName).toBe("alpha"); + expect(stored?.machine.state).not.toBe("failed"); + expect(stored?.steps.sandbox?.status).not.toBe("failed"); expect(stored?.failure?.message).toContain(createAttemptLabel); expect(stored?.steps.sandbox?.error).toContain(createAttemptLabel); const fingerprintExpectation = fingerprint @@ -72,13 +85,17 @@ describe("retained create recovery persistence", () => { expect( persistRetainedSandboxRecoveryMessage( - "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=authority", + { + sandboxName: "alpha", + message: "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=authority", + }, finalizeIncompleteOnboardStep, ), ).toBe(false); expect(finalizeIncompleteOnboardStep).toHaveBeenCalledExactlyOnceWith( - "sandbox", + "alpha", "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=authority", + undefined, ); }); @@ -94,14 +111,18 @@ describe("retained create recovery persistence", () => { expect( persistRetainedSandboxRecoveryMessage( - "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=unpersisted", - session.finalizeIncompleteOnboardStep, + { + sandboxName: "alpha", + message: "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=unpersisted", + }, + session.markRetainedSandboxRecovery, ), - ).toBe(false); + ).toBe(true); const stored = session.loadSession(); - expect(stored?.failure?.message).toBe("Earlier sandbox failure"); - expect(stored?.steps.sandbox?.error).toBe("Earlier sandbox failure"); + expect(stored?.status).toBe("recovery_required"); + expect(stored?.failure?.message).toContain("create-attempt=unpersisted"); + expect(stored?.steps.sandbox?.error).toContain("create-attempt=unpersisted"); } finally { vi.resetModules(); fs.rmSync(tempHome, { force: true, recursive: true }); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index e60d8b7f5a3..5ff60cd6c86 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -62,10 +62,28 @@ function cancelRecoveryIdentity( /** Persist one create-attempt recovery message through the onboard session owner. */ export function persistRetainedSandboxRecoveryMessage( - message: string, - finalizeIncompleteOnboardStep: (stepName: string, message: string) => unknown | null, + input: { + readonly sandboxName: string; + readonly message: string; + readonly sandboxIdentityFingerprint?: string; + }, + markRetainedSandboxRecovery: ( + sandboxName: string, + message: string, + sandboxIdentityFingerprint?: string, + ) => unknown | null, ): boolean { - return finalizeIncompleteOnboardStep("sandbox", message) !== null; + try { + return ( + markRetainedSandboxRecovery( + input.sandboxName, + input.message, + input.sandboxIdentityFingerprint, + ) !== null + ); + } catch { + return false; + } } /** Select the policyless APF create plan only when no active global policy exists. */ @@ -1887,8 +1905,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche : {}), persistRetainedSandboxRecovery: (message) => persistRetainedSandboxRecoveryMessage( - message, - onboardSession.finalizeIncompleteOnboardStep, + { sandboxName, message }, + onboardSession.markRetainedSandboxRecovery, ), provider, sandboxGpuConfig: effectiveSandboxGpuConfig, diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 2f333165283..add4ee946a4 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -109,7 +109,7 @@ export interface SessionFailure { } export interface SessionCancellationRecovery { - readonly reason: "cancelled_after_sandbox_creation"; + readonly reason: "cancelled_after_sandbox_creation" | "retained_after_sandbox_creation_failure"; readonly sandboxName: string; readonly sandboxIdentityFingerprint: string | null; readonly recordedAt: string; @@ -691,7 +691,13 @@ export function sanitizeFailure( function parseSessionCancellationRecovery( value: SessionJsonValue | undefined, ): SessionCancellationRecovery | null { - if (!isObject(value) || value.reason !== "cancelled_after_sandbox_creation") return null; + if ( + !isObject(value) || + (value.reason !== "cancelled_after_sandbox_creation" && + value.reason !== "retained_after_sandbox_creation_failure") + ) { + return null; + } const sandboxName = readString(value.sandboxName); const recordedAt = readCanonicalIsoTimestamp(value.recordedAt); const fingerprint = @@ -706,7 +712,7 @@ function parseSessionCancellationRecovery( return null; } return { - reason: "cancelled_after_sandbox_creation", + reason: value.reason, sandboxName, sandboxIdentityFingerprint: fingerprint, recordedAt, @@ -1669,6 +1675,60 @@ export function markCancellationRecovery( }); } +export function markRetainedSandboxRecovery( + sandboxName: string, + message: string, + sandboxIdentityFingerprint?: string, +): Session { + if ( + sandboxName.length > NAME_MAX_LENGTH || + !NAME_VALID_PATTERN.test(sandboxName) || + (sandboxIdentityFingerprint !== undefined && + !/^[0-9a-f]{64}$/u.test(sandboxIdentityFingerprint)) + ) { + throw new Error("Cannot record retained sandbox recovery with invalid identity data."); + } + const saved = updateSession((session) => { + if (session.sandboxName !== null && session.sandboxName !== sandboxName) { + throw new Error( + "Cannot record retained sandbox recovery for a different onboarding sandbox.", + ); + } + const recordedAt = new Date().toISOString(); + const sanitizedMessage = redactSensitiveText(message); + session.sandboxName = sandboxName; + session.resumable = false; + session.status = CANCELLATION_RECOVERY_STATUS; + session.cancellationRecovery = { + reason: "retained_after_sandbox_creation_failure", + sandboxName, + sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? null, + recordedAt, + }; + session.failure = { + step: session.lastStepStarted, + message: sanitizedMessage, + recordedAt, + interrupted: true, + }; + const sandboxStep = session.steps.sandbox; + if (sandboxStep) sandboxStep.error = sanitizedMessage; + return session; + }); + const reread = loadSession(); + if ( + reread?.sessionId !== saved.sessionId || + reread.status !== CANCELLATION_RECOVERY_STATUS || + reread.resumable !== false || + reread.cancellationRecovery?.reason !== "retained_after_sandbox_creation_failure" || + reread.cancellationRecovery.sandboxName !== sandboxName || + reread.cancellationRecovery.sandboxIdentityFingerprint !== (sandboxIdentityFingerprint ?? null) + ) { + throw new Error("Retained sandbox recovery did not survive durable readback."); + } + return reread; +} + export type CompareAndSwapSessionResult = "updated" | "busy" | "mismatch"; /** From b30e1002bd0c1445e290b9be867d02755822e341 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 02:28:01 -0700 Subject: [PATCH 24/42] fix(onboard): record post-create recovery Signed-off-by: Apurv Kumaria --- .../sandbox-create/orchestration.test.ts | 26 ++++++++++++++++++ .../onboard/sandbox-create/orchestration.ts | 27 ++++++++++++++++++- .../onboard-fresh-create-identity.test.ts | 15 ++++++++++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index a0bf2915798..9dcb3823376 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -580,6 +580,32 @@ describe("sandbox create policy authority checks", () => { expect(events).toEqual(["create-check", "create", "ready-check", "cleanup-sources"]); }); + it("records recovery before returning a post-create authority failure (#9833)", async () => { + const persistRetainedSandboxRecovery = vi.fn(() => true); + + await expect( + runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + ...exactIdentityBoundary(), + revalidateVerifiedPolicy: () => { + throw new Error("external policy authority changed"); + }, + persistRetainedSandboxRecovery, + cleanupTemporarySources: vi.fn(), + }), + ).rejects.toThrow("automatic sandbox cleanup was not safe"); + + expect(persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("left sandbox 'alpha' in place"), + exactIdentity, + ); + }); + it("does not delete a same-name replacement after final authority failure (#9833)", async () => { let sandboxIdentity = "created"; const revalidate = vi.fn(); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 5ff60cd6c86..06ff37a5b5b 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -302,6 +302,10 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< exactIdentity: string, evidence: Evidence, ) => Promise; + readonly persistRetainedSandboxRecovery?: ( + message: string, + exactIdentity: string | null, + ) => boolean; readonly cleanupTemporarySources: () => void; }): Promise { input.revalidate(false, `creating sandbox '${input.sandboxName}'`); @@ -318,7 +322,6 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< } }; const refuseAfterCreate = (validationError: unknown): never => { - const compensationErrors = cleanupTemporarySources(); const validationDetail = validationError instanceof Error && isPolicyAuthorityRefusalError(validationError) ? validationError.message @@ -330,6 +333,19 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< `NemoClaw left sandbox '${input.sandboxName}' in place after policy authority validation failed. ` + `${identityGuidance} NemoClaw did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. ` + "Do not delete the sandbox by mutable sandbox name. Ask the OpenShell administrator to inspect the surviving sandbox and use an identity-bound recovery or removal procedure."; + const compensationErrors: unknown[] = []; + if (input.persistRetainedSandboxRecovery) { + try { + if (!input.persistRetainedSandboxRecovery(recoveryGuidance, exactIdentity)) { + compensationErrors.push( + new Error("NemoClaw could not save the retained sandbox recovery record."), + ); + } + } catch (error) { + compensationErrors.push(error); + } + } + compensationErrors.push(...cleanupTemporarySources()); compensationErrors.push(new Error(recoveryGuidance)); throw new AggregateError( [validationError, ...compensationErrors], @@ -1883,6 +1899,15 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche revalidateVerifiedPolicy: (_identity, _exactIdentity, boundary, operation) => { revalidateVerifiedPolicyRegistration(boundary, operation); }, + persistRetainedSandboxRecovery: (message, exactIdentity) => + persistRetainedSandboxRecoveryMessage( + { + sandboxName, + message, + ...(exactIdentity ? { sandboxIdentityFingerprint: exactIdentity } : {}), + }, + onboardSession.markRetainedSandboxRecovery, + ), cleanupTemporarySources: cleanupSandboxCreateSources, runVerifiedCreateEffects: runDeferredProviderEffects ? async (_identity, _exactIdentity, boundary) => { diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 5c33f8a7f08..51daa5fe831 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -331,7 +331,10 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { routeReservationCalls, registryMutationCalls, currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, - savedSession: cancelAfterCreate ? onboardModule.onboardSession.loadSession() : null, + savedSession: + cancelAfterCreate || postCreateAuthorityRefusal + ? onboardModule.onboardSession.loadSession() + : null, createCommand: createCommand?.command ?? null, commandNames: commands.map((entry) => entry.command), })); @@ -542,6 +545,16 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { payload.creationError, /Ask the OpenShell administrator.*identity-bound recovery or removal procedure/u, ); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.reason, + "retained_after_sandbox_creation_failure", + ); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); }; const assertCancellationRecovery = () => { const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); From 96a2f4ba963c777b98958247dd63e8ba93b4f9fe Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 02:36:40 -0700 Subject: [PATCH 25/42] fix(onboard): retain recovery through finalization Signed-off-by: Apurv Kumaria --- .../onboard/sandbox-create/orchestration.ts | 126 ++++++++++++++---- .../onboard-fresh-create-identity.test.ts | 70 +++++++++- 2 files changed, 170 insertions(+), 26 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 06ff37a5b5b..e921d26d56e 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -86,6 +86,63 @@ export function persistRetainedSandboxRecoveryMessage( } } +function persistPostCreateRecovery(input: { + readonly stage: "registry publication" | "onboarding finalization"; + readonly sandboxName: string; + readonly gatewayName: string; + readonly lifecycleGeneration: string; + readonly exactIdentity?: string; + readonly markRetainedSandboxRecovery: ( + sandboxName: string, + message: string, + sandboxIdentityFingerprint?: string, + ) => unknown | null; +}): void { + const message = + `Sandbox '${input.sandboxName}' was retained after ${input.stage} failed. ` + + `Gateway '${input.gatewayName}'. Lifecycle generation '${input.lifecycleGeneration}'. ` + + "Do not delete the sandbox by mutable name; preserve it for identity-bound administrator recovery."; + const persisted = persistRetainedSandboxRecoveryMessage( + { + sandboxName: input.sandboxName, + message, + ...(input.exactIdentity + ? { sandboxIdentityFingerprint: input.exactIdentity } + : {}), + }, + input.markRetainedSandboxRecovery, + ); + if (!persisted) { + console.error( + " NemoClaw could not save the retained sandbox recovery record. Preserve the terminal output and registry state for an OpenShell administrator.", + ); + } +} + +async function runAsyncWithPostCreateRecovery( + operation: () => Promise, + recordRecovery: () => void, +): Promise { + try { + return await operation(); + } catch (error) { + recordRecovery(); + throw error; + } +} + +function runWithPostCreateRecovery( + operation: () => Result, + recordRecovery: () => void, +): Result { + try { + return operation(); + } catch (error) { + recordRecovery(); + throw error; + } +} + /** Select the policyless APF create plan only when no active global policy exists. */ export function resolveSandboxCreatePolicyAuthority( observedAuthority: "nemoclaw-managed" | "externally-managed", @@ -1834,6 +1891,19 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche pendingSandboxPolicyVerificationForBoundary(boundary), ); }; + const recordPostCreateRecovery = ( + stage: "registry publication" | "onboarding finalization", + ): void => + persistPostCreateRecovery({ + stage, + sandboxName, + gatewayName: GATEWAY_NAME, + lifecycleGeneration: createdSandboxLifecycle.generation, + ...(verifiedPolicyGate + ? { exactIdentity: verifiedPolicyGate.lifecycleLiveIdentityFingerprint } + : {}), + markRetainedSandboxRecovery: onboardSession.markRetainedSandboxRecovery, + }); const runCreateFlow = async ( attemptCreateArgv: string[], hermesPortableReadyCapture?: import("../sandbox-gpu-create-flow").HermesPortableReadyCapture, @@ -2182,36 +2252,44 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche providerEffectBoundary.runAfterVerifiedCreate, ); try { - await completeCreatedSandboxRegistration(created, null); + await runAsyncWithPostCreateRecovery( + () => completeCreatedSandboxRegistration(created, null), + () => recordPostCreateRecovery("registry publication"), + ); apfPolicyRegistrationFinalized = apfInterceptorRequested; } finally { cleanupInitialCreateSource(); } } - hermesStateVolumeLifecycle?.commit(); - if ("complete" in recreateRuntime) recreateRuntime.complete(); - if (agentCreateInput.hermesPortableLifecycle) return sandboxName; - return completeOrdinaryOnboardSandboxCreation( - { - sandboxName, - sandboxWasLiveDefault, - runtimeFields: sandboxRuntimeFields, - messagingProviders, - liveExists, - ...cancelRecoveryIdentity(liveExists, requireVerifiedPolicyGate), - }, - { - setDefault: registry.setDefault, - runFile, - scriptsDir: SCRIPTS, - gatewayName: GATEWAY_NAME, - providerExistsInGateway, - armCancelRollback: sandboxCancelRollback.arm, - markCancellationRecovery: onboardSession.markCancellationRecovery, - dockerInfoFormat, - runCapture, - revalidatePolicyAuthority: (operation) => revalidatePolicyAuthority(true, operation), + return runWithPostCreateRecovery( + () => { + hermesStateVolumeLifecycle?.commit(); + if ("complete" in recreateRuntime) recreateRuntime.complete(); + if (agentCreateInput.hermesPortableLifecycle) return sandboxName; + return completeOrdinaryOnboardSandboxCreation( + { + sandboxName, + sandboxWasLiveDefault, + runtimeFields: sandboxRuntimeFields, + messagingProviders, + liveExists, + ...cancelRecoveryIdentity(liveExists, requireVerifiedPolicyGate), + }, + { + setDefault: registry.setDefault, + runFile, + scriptsDir: SCRIPTS, + gatewayName: GATEWAY_NAME, + providerExistsInGateway, + armCancelRollback: sandboxCancelRollback.arm, + markCancellationRecovery: onboardSession.markCancellationRecovery, + dockerInfoFormat, + runCapture, + revalidatePolicyAuthority: (operation) => revalidatePolicyAuthority(true, operation), + }, + ); }, + () => recordPostCreateRecovery("onboarding finalization"), ); }; } diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 51daa5fe831..b1958977ad1 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -53,6 +53,22 @@ describe("fresh create identity", () => { agent: null, expectedOutcome: "post-create-authority-refusal" as const, }, + { + title: "retains recovery state when registry publication fails after create (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-registration-refusal" as const, + }, + { + title: "retains recovery state when final checks fail after registration (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-finalization-refusal" as const, + }, { title: "rejects staged messaging intent before any onboarding side effect (#9833)", apfInterceptorRequested: true, @@ -161,6 +177,12 @@ const stagedMessagingRefusal = ${JSON.stringify(expectedOutcome === "staged-mess const postCreateAuthorityRefusal = ${JSON.stringify( expectedOutcome === "post-create-authority-refusal", )}; +const postCreateRegistrationRefusal = ${JSON.stringify( + expectedOutcome === "post-create-registration-refusal", + )}; +const postCreateFinalizationRefusal = ${JSON.stringify( + expectedOutcome === "post-create-finalization-refusal", + )}; let cancelPrompt = false; const originalGetCredential = credentials.getCredential; credentials.getCredential = (...args) => { @@ -189,7 +211,12 @@ runner.run = (command, opts = {}) => { runner.runCapture = (command) => { const cmd = _n(command); if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; - if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: effectivePolicy }); + if (cmd.includes("policy get") && cmd.includes("--output json")) { + if (postCreateFinalizationRefusal && registeredSandbox) { + throw new Error("final onboarding policy check failed"); + } + return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: effectivePolicy }); + } if (cmd.includes("sandbox get") || cmd.includes("sandbox list")) { lifecycleObservationCommands.push(cmd); } @@ -235,6 +262,9 @@ runner.run = (command, opts = {}) => { ).policy; }, registerSandbox: (entry) => { + if (postCreateRegistrationRefusal) { + throw new Error("registry publication failed"); + } registeredSandbox = entry; registryMutationCalls.push({ operation: "register", name: entry.name }); }, @@ -332,7 +362,10 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { registryMutationCalls, currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, savedSession: - cancelAfterCreate || postCreateAuthorityRefusal + cancelAfterCreate || + postCreateAuthorityRefusal || + postCreateRegistrationRefusal || + postCreateFinalizationRefusal ? onboardModule.onboardSession.loadSession() : null, createCommand: createCommand?.command ?? null, @@ -556,6 +589,37 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { identityFingerprint, ); }; + const assertPostCreateRegistrationRefusal = () => { + const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /registry publication failed/u); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + }; + const assertPostCreateFinalizationRefusal = () => { + const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox.name, "my-assistant"); + assert.match( + payload.creationError, + /OpenShell sandbox policy authority inspection failed/u, + ); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + }; const assertCancellationRecovery = () => { const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.exitCode, 1); @@ -640,6 +704,8 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { "provider-refusal": assertProviderBackedApfRefusal, "providerless-apf": assertProviderlessApfCreation, "post-create-authority-refusal": assertPostCreateAuthorityRefusal, + "post-create-registration-refusal": assertPostCreateRegistrationRefusal, + "post-create-finalization-refusal": assertPostCreateFinalizationRefusal, "staged-messaging-refusal": assertStagedMessagingRefusal, "cancel-after-create-tier": assertCancellationRecovery, "cancel-after-create-tier-presets": assertCancellationRecovery, From 7cfed07228679680a5e708d6dfead18913a3f2ef Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 03:03:05 -0700 Subject: [PATCH 26/42] fix(onboard): preserve retained recovery records Signed-off-by: Apurv Kumaria --- src/lib/onboard.ts | 4 +- src/lib/onboard/entry-options.test.ts | 42 ++ src/lib/onboard/entry-options.ts | 39 ++ src/lib/state/onboard-session.ts | 91 +++- .../retained-sandbox-recovery.ts | 403 ++++++++++++++++++ .../state/retained-sandbox-recovery.test.ts | 120 ++++++ .../onboard-fresh-create-identity.test.ts | 98 ++++- 7 files changed, 784 insertions(+), 13 deletions(-) create mode 100644 src/lib/state/onboard-session/retained-sandbox-recovery.ts create mode 100644 src/lib/state/retained-sandbox-recovery.test.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 829d4f3f0ad..2df60c25a74 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2754,6 +2754,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { opts, onboardSession.loadSession(), validateName, + process.env, + onboardSession.listRetainedSandboxRecoveryRecords().map((record) => record.sandboxName), ); const { fresh, nonInteractive, cannotPrompt, resume } = entryOptions; const { requestedFromDockerfile, requestedSandboxName } = entryOptions; @@ -2780,14 +2782,12 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { sessionFile: onboardSession.SESSION_FILE, withLifecycleLock: sandboxMutationLock.withMcpLifecycleLock, }); - let portableEnvScope: | import("./onboard/session-bootstrap").PortableOnboardEnvironmentScope | null = null; const restorePortableEnvScope = () => portableEnvScope?.restore(); // Secure removal remains gated on successful migration of every staged legacy credential. let stagedLegacyKeys: string[] = []; - let onboardTrace: ReturnType = { collector: null, span: null, diff --git a/src/lib/onboard/entry-options.test.ts b/src/lib/onboard/entry-options.test.ts index 8c55365e926..cb59b108950 100644 --- a/src/lib/onboard/entry-options.test.ts +++ b/src/lib/onboard/entry-options.test.ts @@ -287,6 +287,48 @@ describe("resolveOnboardEntryOptions", () => { expect(deps.error).not.toHaveBeenCalled(); }); + it("blocks a retained name after a different fresh session replaced the active session", () => { + const deps = createDeps(); + + expect(() => + resolveOnboardEntryOptions( + { + opts: { fresh: true, sandboxName: "retained-sb" }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "in_progress", + persistedRecoverySandboxName: null, + retainedRecoverySandboxNames: ["retained-sb"], + }, + deps, + ), + ).toThrow(ExitError); + expect(deps.error).toHaveBeenCalledWith( + expect.stringContaining("retained sandbox 'retained-sb'"), + ); + }); + + it("allows a different fresh name without clearing an independent recovery record", () => { + const deps = createDeps(); + + const result = resolveOnboardEntryOptions( + { + opts: { fresh: true, sandboxName: "replacement-sb" }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "recovery_required", + persistedRecoverySandboxName: "retained-sb", + retainedRecoverySandboxNames: ["retained-sb"], + }, + deps, + ); + + expect(result.requestedSandboxName).toBe("replacement-sb"); + expect(deps.error).not.toHaveBeenCalled(); + }); + it("does not auto-resume when --fresh is set even with an in_progress session (#5470)", () => { const deps = createDeps(); diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index cde4ec3700d..2afa3b97c52 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -35,6 +35,8 @@ export interface OnboardEntryOptionsInput { */ persistedSessionStatus?: string | null; persistedRecoverySandboxName?: string | null; + persistedSessionSandboxName?: string | null; + retainedRecoverySandboxNames?: readonly string[]; } export interface OnboardEntryOptionsDeps { @@ -61,6 +63,7 @@ export interface ResolvedOnboardEntryOptions { type PersistedOnboardEntrySession = { readonly status: string; + readonly sandboxName?: string | null; readonly cancellationRecovery?: { readonly sandboxName: string } | null; }; @@ -112,6 +115,8 @@ export function resolveOnboardRunOptions( stdoutIsTty: Boolean(process.stdout?.isTTY), }, persistedRecoverySandboxName: string | null = null, + persistedSessionSandboxName: string | null = null, + retainedRecoverySandboxNames: readonly string[] = [], ) { const resume = options.resume === true || (options.fresh !== true && persistedSessionStatus === "in_progress"); @@ -128,6 +133,8 @@ export function resolveOnboardRunOptions( ...terminal, persistedSessionStatus, persistedRecoverySandboxName, + persistedSessionSandboxName, + retainedRecoverySandboxNames, }, }; } @@ -139,6 +146,8 @@ export function resolveOnboardRunEntryOptions( isNonInteractiveEnv: () => boolean, deps: Omit, persistedRecoverySandboxName: string | null = null, + persistedSessionSandboxName: string | null = null, + retainedRecoverySandboxNames: readonly string[] = [], ) { const context = resolveOnboardRunOptions( options, @@ -147,6 +156,8 @@ export function resolveOnboardRunEntryOptions( isNonInteractiveEnv, undefined, persistedRecoverySandboxName, + persistedSessionSandboxName, + retainedRecoverySandboxNames, ); return { ...context, @@ -162,6 +173,7 @@ export function resolveDefaultRunEntryOptions( persistedSession: PersistedOnboardEntrySession | null, validateSandboxName: OnboardEntryOptionsDeps["validateName"], env: NodeJS.ProcessEnv = process.env, + retainedRecoverySandboxNames: readonly string[] = [], ) { return resolveOnboardRunEntryOptions( options, @@ -177,6 +189,8 @@ export function resolveDefaultRunEntryOptions( exitProcess: (code) => process.exit(code), }, persistedSession?.cancellationRecovery?.sandboxName ?? null, + persistedSession?.sandboxName ?? null, + retainedRecoverySandboxNames, ); } @@ -303,6 +317,31 @@ export function resolveOnboardEntryOptions( } requestedSandboxName = validated; } + const retainedRecoverySandboxNames = new Set( + (input.retainedRecoverySandboxNames ?? []).map((name) => name.trim()).filter(Boolean), + ); + const recoveryEntryName = + requestedSandboxName ?? input.persistedSessionSandboxName?.trim() ?? null; + if (retainedRecoverySandboxNames.size > 0) { + if (!recoveryEntryName) { + deps.error( + " Onboarding cannot continue while a retained sandbox recovery record is unresolved without an explicit different sandbox name.", + ); + deps.error( + " Use --fresh --name , or complete identity-bound administrator recovery for the retained sandbox first.", + ); + deps.exitProcess(1); + } + if (retainedRecoverySandboxNames.has(recoveryEntryName)) { + deps.error( + ` Onboarding cannot use retained sandbox '${recoveryEntryName}' while its identity-bound recovery record is unresolved.`, + ); + deps.error( + " Automatic and explicit resume, reuse, recreation, and same-name fresh onboarding remain disabled until administrator resolution is recorded.", + ); + deps.exitProcess(1); + } + } if (input.persistedSessionStatus === "recovery_required") { const recoverySandboxName = input.persistedRecoverySandboxName?.trim() || null; if (!fresh) { diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index add4ee946a4..7003722f6ee 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -19,7 +19,12 @@ import { parseServingProfileProvenance, type ServingProfileProvenance, } from "../inference/serving/profile-provenance"; -import { normalizeWebSearchConfig, type WebSearchConfig } from "../inference/web-search"; +import { + normalizeWebSearchConfig, + webSearchEnvFor, + webSearchProviderForConfig, + type WebSearchConfig, +} from "../inference/web-search"; import type { SandboxMessagingPlan } from "../messaging/manifest"; import { compactSandboxMessagingPlanForPersistence } from "../messaging/persistence"; import { parseSandboxMessagingPlan } from "../messaging/plan-validation"; @@ -48,7 +53,6 @@ import { } from "../onboard/station-express-resume"; import { redactSensitiveText, redactUrl } from "../security/redact"; import { inspectCheckpoint, serializeCheckpoint } from "./onboard-checkpoint"; -import { decisionUnset } from "./onboard-checkpoint-decision"; import type { OnboardCheckpoint } from "./onboard-checkpoint-types"; import { assignSafeToolDisclosureUpdate, @@ -57,6 +61,17 @@ import { type ToolDisclosure, } from "./onboard-session-tool-disclosure"; import { nextMachineStateAfterCompletedStep } from "./onboard-step-state"; +import { + listRetainedSandboxRecoveryRecords as readRetainedSandboxRecoveryRecords, + recordRetainedSandboxRecovery as writeRetainedSandboxRecovery, + resolveRetainedSandboxRecovery as writeRetainedSandboxResolution, + retainedSandboxRecoveryFile, + type RecordRetainedSandboxRecoveryInput, + type ResolveRetainedSandboxRecoveryInput, + type RetainedSandboxAdministratorResolutionReceipt, + type RetainedSandboxRecoveryRecord, + type RetainedSandboxRecoveryReason, +} from "./onboard-session/retained-sandbox-recovery"; import type { SandboxHostMount } from "./registry/types"; import { hasUnsafeHostMountTerminalText } from "./registry/host-mount"; import { nemoclawStateRoot } from "./state-root"; @@ -70,6 +85,7 @@ const INVALID_HOST_MOUNT_SESSIONS = new WeakSet(); export const SESSION_DIR = nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY_PORT); export const SESSION_FILE = path.join(SESSION_DIR, "onboard-session.json"); export const LOCK_FILE = path.join(SESSION_DIR, "onboard.lock"); +export const RETAINED_SANDBOX_RECOVERY_FILE = retainedSandboxRecoveryFile(SESSION_DIR); const SAFE_VLLM_INSTALL_MODEL = /^[A-Za-z0-9._:/-]+$/; export class InvalidPersistedPolicyAuthorityError extends Error {} @@ -1638,9 +1654,62 @@ export function updateSession(mutator: (session: Session) => Session | void): Se return saveSession(next); } +export interface RetainedSandboxRecoveryContext { + readonly gatewayName?: string; + readonly gatewayPort?: number; + readonly lifecycleGeneration?: string | null; +} + +function persistIndependentRetainedSandboxRecovery( + session: Session, + reason: RetainedSandboxRecoveryReason, + sandboxIdentityFingerprint: string | null, + context: RetainedSandboxRecoveryContext, +): void { + const messagingCredentialEnvironmentVariables = + session.messagingPlan?.credentialBindings.map((binding) => binding.providerEnvKey) ?? []; + const credentialEnvironmentVariables = [ + ...(session.credentialEnv ? [session.credentialEnv] : []), + ...(session.webSearchConfig + ? [webSearchEnvFor(webSearchProviderForConfig(session.webSearchConfig))] + : []), + ...messagingCredentialEnvironmentVariables, + ]; + writeRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, { + sandboxName: session.sandboxName!, + sandboxIdentityFingerprint, + gatewayName: context.gatewayName ?? session.metadata.gatewayName, + gatewayPort: context.gatewayPort ?? GATEWAY_PORT, + lifecycleGeneration: context.lifecycleGeneration ?? null, + resources: { + sharedInferenceProviders: session.provider ? [session.provider] : [], + sandboxScopedProviders: session.stagedCredentialProviders, + credentialEnvironmentVariables, + }, + reason, + }); +} + +export function listRetainedSandboxRecoveryRecords(): readonly RetainedSandboxRecoveryRecord[] { + return readRetainedSandboxRecoveryRecords(RETAINED_SANDBOX_RECOVERY_FILE); +} + +export function recordRetainedSandboxRecovery( + input: RecordRetainedSandboxRecoveryInput, +): RetainedSandboxRecoveryRecord { + return writeRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, input); +} + +export function resolveRetainedSandboxRecovery( + input: ResolveRetainedSandboxRecoveryInput, +): RetainedSandboxAdministratorResolutionReceipt { + return writeRetainedSandboxResolution(RETAINED_SANDBOX_RECOVERY_FILE, input); +} + export function markCancellationRecovery( sandboxName: string, sandboxIdentityFingerprint?: string, + context: RetainedSandboxRecoveryContext = {}, ): Session { if ( sandboxName.length > NAME_MAX_LENGTH || @@ -1650,7 +1719,7 @@ export function markCancellationRecovery( ) { throw new Error("Cannot record cancellation recovery with invalid sandbox identity data."); } - return updateSession((session) => { + const saved = updateSession((session) => { if (session.sandboxName !== null && session.sandboxName !== sandboxName) { throw new Error("Cannot record cancellation recovery for a different onboarding sandbox."); } @@ -1673,12 +1742,20 @@ export function markCancellationRecovery( }; return session; }); + persistIndependentRetainedSandboxRecovery( + saved, + "cancelled_after_sandbox_creation", + sandboxIdentityFingerprint ?? null, + context, + ); + return saved; } export function markRetainedSandboxRecovery( sandboxName: string, message: string, sandboxIdentityFingerprint?: string, + context: RetainedSandboxRecoveryContext = {}, ): Session { if ( sandboxName.length > NAME_MAX_LENGTH || @@ -1726,6 +1803,12 @@ export function markRetainedSandboxRecovery( ) { throw new Error("Retained sandbox recovery did not survive durable readback."); } + persistIndependentRetainedSandboxRecovery( + reread, + "retained_after_sandbox_creation_failure", + sandboxIdentityFingerprint ?? null, + context, + ); return reread; } @@ -1853,7 +1936,7 @@ export function markStepRejected(stepName: string): Session { if (session.checkpoint) { session.checkpoint = { ...session.checkpoint, - sandboxIdentity: decisionUnset(), + sandboxIdentity: { kind: "unset" }, updatedAt: new Date().toISOString(), }; } diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts new file mode 100644 index 00000000000..67e95eae45e --- /dev/null +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -0,0 +1,403 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const SCHEMA_VERSION = 1; +const FINGERPRINT_PATTERN = /^[0-9a-f]{64}$/u; +const SAFE_EVIDENCE_PATTERN = /^[A-Za-z0-9._:@/-]{1,256}$/u; +const NAME_MAX_LENGTH = 63; +const NAME_VALID_PATTERN = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u; + +export function retainedSandboxRecoveryFile(sessionDirectory: string): string { + return path.join(sessionDirectory, "retained-sandbox-recovery.json"); +} + +export type RetainedSandboxRecoveryReason = + | "cancelled_after_sandbox_creation" + | "retained_after_sandbox_creation_failure"; + +export interface RetainedSandboxResourceEvidence { + readonly sharedInferenceProviders: readonly string[]; + readonly sandboxScopedProviders: readonly string[]; + readonly credentialEnvironmentVariables: readonly string[]; +} + +export interface RetainedSandboxRecoveryRecord { + readonly schemaVersion: typeof SCHEMA_VERSION; + readonly recordId: string; + readonly sandboxName: string; + readonly sandboxIdentityFingerprint: string | null; + readonly identityWasUnavailable: boolean; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly lifecycleGeneration: string | null; + readonly resources: RetainedSandboxResourceEvidence; + readonly reason: RetainedSandboxRecoveryReason; + readonly recordedAt: string; +} + +export interface RetainedSandboxAdministratorResolutionReceipt { + readonly schemaVersion: typeof SCHEMA_VERSION; + readonly receiptId: string; + readonly recordId: string; + readonly sandboxName: string; + readonly sandboxIdentityFingerprint: string | null; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly outcome: "removed_verified_identity" | "confirmed_absent_without_identity"; + readonly resolvedAt: string; +} + +interface RetainedSandboxRecoveryState { + readonly schemaVersion: typeof SCHEMA_VERSION; + readonly unresolved: readonly RetainedSandboxRecoveryRecord[]; + readonly resolutions: readonly RetainedSandboxAdministratorResolutionReceipt[]; +} + +export interface RecordRetainedSandboxRecoveryInput { + readonly sandboxName: string; + readonly sandboxIdentityFingerprint: string | null; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly lifecycleGeneration: string | null; + readonly resources: RetainedSandboxResourceEvidence; + readonly reason: RetainedSandboxRecoveryReason; + readonly recordedAt?: string; +} + +export interface ResolveRetainedSandboxRecoveryInput { + readonly recordId: string; + readonly receiptId: string; + readonly sandboxName: string; + readonly sandboxIdentityFingerprint: string | null; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly outcome: RetainedSandboxAdministratorResolutionReceipt["outcome"]; + readonly resolvedAt?: string; +} + +const emptyState = (): RetainedSandboxRecoveryState => ({ + schemaVersion: SCHEMA_VERSION, + unresolved: [], + resolutions: [], +}); + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readStateFile(filePath: string): unknown { + try { + const stat = fs.lstatSync(filePath); + if (stat.isSymbolicLink()) { + throw new Error("Retained sandbox recovery state cannot be a symbolic link."); + } + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch (error) { + if ( + error instanceof Error && + "code" in error && + (error as NodeJS.ErrnoException).code === "ENOENT" + ) { + return emptyState(); + } + throw error; + } +} + +function writeStateFile(filePath: string, state: RetainedSandboxRecoveryState): void { + const directory = path.dirname(filePath); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + try { + if (fs.lstatSync(filePath).isSymbolicLink()) { + throw new Error("Retained sandbox recovery state cannot be a symbolic link."); + } + } catch (error) { + if ( + !( + error instanceof Error && + "code" in error && + (error as NodeJS.ErrnoException).code === "ENOENT" + ) + ) { + throw error; + } + } + const temporary = path.join( + directory, + `.retained-sandbox-recovery.${String(process.pid)}.${randomUUID()}.tmp`, + ); + try { + fs.writeFileSync(temporary, JSON.stringify(state, null, 2), { mode: 0o600 }); + const descriptor = fs.openSync(temporary, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + fs.renameSync(temporary, filePath); + const directoryDescriptor = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(directoryDescriptor); + } finally { + fs.closeSync(directoryDescriptor); + } + } finally { + fs.rmSync(temporary, { force: true }); + } +} + +function validSandboxName(value: unknown): value is string { + return ( + typeof value === "string" && value.length <= NAME_MAX_LENGTH && NAME_VALID_PATTERN.test(value) + ); +} + +function validSafeEvidence(value: unknown): value is string { + return typeof value === "string" && SAFE_EVIDENCE_PATTERN.test(value); +} + +function validTimestamp(value: unknown): value is string { + return typeof value === "string" && Number.isFinite(Date.parse(value)); +} + +function validGatewayPort(value: unknown): value is number { + return Number.isInteger(value) && Number(value) >= 1024 && Number(value) <= 65535; +} + +function parseEvidence(value: unknown): RetainedSandboxResourceEvidence | null { + if (!isObjectRecord(value)) return null; + const parse = (candidate: unknown): string[] | null => + Array.isArray(candidate) && candidate.every(validSafeEvidence) + ? [...new Set(candidate)].sort() + : null; + const sharedInferenceProviders = parse(value.sharedInferenceProviders); + const sandboxScopedProviders = parse(value.sandboxScopedProviders); + const credentialEnvironmentVariables = parse(value.credentialEnvironmentVariables); + return sharedInferenceProviders && sandboxScopedProviders && credentialEnvironmentVariables + ? { sharedInferenceProviders, sandboxScopedProviders, credentialEnvironmentVariables } + : null; +} + +function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { + if (!isObjectRecord(value)) return null; + const resources = parseEvidence(value.resources); + const fingerprint = value.sandboxIdentityFingerprint; + const reason = value.reason; + if ( + value.schemaVersion !== SCHEMA_VERSION || + typeof value.recordId !== "string" || + !FINGERPRINT_PATTERN.test(value.recordId) || + !validSandboxName(value.sandboxName) || + (fingerprint !== null && + (typeof fingerprint !== "string" || !FINGERPRINT_PATTERN.test(fingerprint))) || + value.identityWasUnavailable !== (fingerprint === null) || + !validSafeEvidence(value.gatewayName) || + !validGatewayPort(value.gatewayPort) || + (value.lifecycleGeneration !== null && !validSafeEvidence(value.lifecycleGeneration)) || + !resources || + !["cancelled_after_sandbox_creation", "retained_after_sandbox_creation_failure"].includes( + String(reason), + ) || + !validTimestamp(value.recordedAt) + ) { + return null; + } + return { + schemaVersion: SCHEMA_VERSION, + recordId: value.recordId, + sandboxName: value.sandboxName, + sandboxIdentityFingerprint: fingerprint, + identityWasUnavailable: fingerprint === null, + gatewayName: value.gatewayName, + gatewayPort: value.gatewayPort, + lifecycleGeneration: value.lifecycleGeneration, + resources, + reason: reason as RetainedSandboxRecoveryReason, + recordedAt: value.recordedAt, + }; +} + +function parseReceipt(value: unknown): RetainedSandboxAdministratorResolutionReceipt | null { + if (!isObjectRecord(value)) return null; + const fingerprint = value.sandboxIdentityFingerprint; + const outcome = value.outcome; + if ( + value.schemaVersion !== SCHEMA_VERSION || + typeof value.receiptId !== "string" || + !FINGERPRINT_PATTERN.test(value.receiptId) || + typeof value.recordId !== "string" || + !FINGERPRINT_PATTERN.test(value.recordId) || + !validSandboxName(value.sandboxName) || + (fingerprint !== null && + (typeof fingerprint !== "string" || !FINGERPRINT_PATTERN.test(fingerprint))) || + !validSafeEvidence(value.gatewayName) || + !validGatewayPort(value.gatewayPort) || + !["removed_verified_identity", "confirmed_absent_without_identity"].includes(String(outcome)) || + !validTimestamp(value.resolvedAt) + ) { + return null; + } + return { + schemaVersion: SCHEMA_VERSION, + receiptId: value.receiptId, + recordId: value.recordId, + sandboxName: value.sandboxName, + sandboxIdentityFingerprint: fingerprint, + gatewayName: value.gatewayName, + gatewayPort: value.gatewayPort, + outcome: outcome as RetainedSandboxAdministratorResolutionReceipt["outcome"], + resolvedAt: value.resolvedAt, + }; +} + +function loadState(filePath: string): RetainedSandboxRecoveryState { + const value = readStateFile(filePath); + if (!isObjectRecord(value) || value.schemaVersion !== SCHEMA_VERSION) { + throw new Error("Retained sandbox recovery state has an unsupported schema."); + } + const unresolved = Array.isArray(value.unresolved) ? value.unresolved.map(parseRecord) : null; + const resolutions = Array.isArray(value.resolutions) ? value.resolutions.map(parseReceipt) : null; + if (!unresolved || unresolved.includes(null) || !resolutions || resolutions.includes(null)) { + throw new Error("Retained sandbox recovery state is invalid; onboarding remains blocked."); + } + return { + schemaVersion: SCHEMA_VERSION, + unresolved: unresolved as RetainedSandboxRecoveryRecord[], + resolutions: resolutions as RetainedSandboxAdministratorResolutionReceipt[], + }; +} + +function recoveryRecordId(input: RecordRetainedSandboxRecoveryInput): string { + return createHash("sha256") + .update( + JSON.stringify([ + input.gatewayName, + input.gatewayPort, + input.sandboxName, + input.sandboxIdentityFingerprint, + ]), + ) + .digest("hex"); +} + +function assertRecordInput(input: RecordRetainedSandboxRecoveryInput): void { + if ( + !validSandboxName(input.sandboxName) || + (input.sandboxIdentityFingerprint !== null && + !FINGERPRINT_PATTERN.test(input.sandboxIdentityFingerprint)) || + !validSafeEvidence(input.gatewayName) || + !validGatewayPort(input.gatewayPort) || + (input.lifecycleGeneration !== null && !validSafeEvidence(input.lifecycleGeneration)) || + !parseEvidence(input.resources) + ) { + throw new Error("Cannot persist invalid retained sandbox recovery evidence."); + } +} + +export function listRetainedSandboxRecoveryRecords( + filePath: string, +): readonly RetainedSandboxRecoveryRecord[] { + return loadState(filePath).unresolved; +} + +export function recordRetainedSandboxRecovery( + filePath: string, + input: RecordRetainedSandboxRecoveryInput, +): RetainedSandboxRecoveryRecord { + assertRecordInput(input); + const record: RetainedSandboxRecoveryRecord = { + schemaVersion: SCHEMA_VERSION, + recordId: recoveryRecordId(input), + sandboxName: input.sandboxName, + sandboxIdentityFingerprint: input.sandboxIdentityFingerprint, + identityWasUnavailable: input.sandboxIdentityFingerprint === null, + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + lifecycleGeneration: input.lifecycleGeneration, + resources: parseEvidence(input.resources)!, + reason: input.reason, + recordedAt: input.recordedAt ?? new Date().toISOString(), + }; + if (!validTimestamp(record.recordedAt)) { + throw new Error("Cannot persist retained sandbox recovery with an invalid timestamp."); + } + const current = loadState(filePath); + const next: RetainedSandboxRecoveryState = { + ...current, + unresolved: [ + ...current.unresolved.filter( + (candidate) => + candidate.gatewayName !== record.gatewayName || + candidate.gatewayPort !== record.gatewayPort || + candidate.sandboxName !== record.sandboxName, + ), + record, + ], + }; + writeStateFile(filePath, next); + const reread = loadState(filePath).unresolved.find( + (candidate) => candidate.recordId === record.recordId, + ); + if (!reread || JSON.stringify(reread) !== JSON.stringify(record)) { + throw new Error("Retained sandbox recovery record did not survive durable readback."); + } + return reread; +} + +export function resolveRetainedSandboxRecovery( + filePath: string, + input: ResolveRetainedSandboxRecoveryInput, +): RetainedSandboxAdministratorResolutionReceipt { + const current = loadState(filePath); + const record = current.unresolved.find((candidate) => candidate.recordId === input.recordId); + if ( + !record || + !FINGERPRINT_PATTERN.test(input.receiptId) || + record.sandboxName !== input.sandboxName || + record.gatewayName !== input.gatewayName || + record.gatewayPort !== input.gatewayPort || + record.sandboxIdentityFingerprint !== input.sandboxIdentityFingerprint || + (record.sandboxIdentityFingerprint === null) !== + (input.outcome === "confirmed_absent_without_identity") + ) { + throw new Error("Administrator resolution receipt does not match retained sandbox identity."); + } + const receipt: RetainedSandboxAdministratorResolutionReceipt = { + schemaVersion: SCHEMA_VERSION, + receiptId: input.receiptId, + recordId: record.recordId, + sandboxName: record.sandboxName, + sandboxIdentityFingerprint: record.sandboxIdentityFingerprint, + gatewayName: record.gatewayName, + gatewayPort: record.gatewayPort, + outcome: input.outcome, + resolvedAt: input.resolvedAt ?? new Date().toISOString(), + }; + if (!validTimestamp(receipt.resolvedAt)) { + throw new Error("Administrator resolution receipt has an invalid timestamp."); + } + writeStateFile(filePath, { + ...current, + unresolved: current.unresolved.filter((candidate) => candidate.recordId !== record.recordId), + resolutions: [ + ...current.resolutions.filter((candidate) => candidate.recordId !== record.recordId), + receipt, + ], + }); + const reread = loadState(filePath); + const durableReceipt = reread.resolutions.find( + (candidate) => candidate.receiptId === receipt.receiptId, + ); + if ( + reread.unresolved.some((candidate) => candidate.recordId === record.recordId) || + !durableReceipt || + JSON.stringify(durableReceipt) !== JSON.stringify(receipt) + ) { + throw new Error("Administrator resolution receipt did not survive durable readback."); + } + return durableReceipt; +} diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts new file mode 100644 index 00000000000..01e8ae2501a --- /dev/null +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +let home: string; + +beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-retained-recovery-")); + vi.stubEnv("HOME", home); + vi.resetModules(); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(home, { recursive: true, force: true }); +}); + +const evidence = { + sharedInferenceProviders: ["nvidia"], + sandboxScopedProviders: ["sandbox-telegram"], + credentialEnvironmentVariables: ["NVIDIA_API_KEY", "TELEGRAM_BOT_TOKEN"], +} as const; + +describe("retained sandbox recovery state", () => { + it("persists verified identity and secret-free resource evidence independently", async () => { + const recovery = await import("./onboard-session"); + const fingerprint = "a".repeat(64); + + const recorded = recovery.recordRetainedSandboxRecovery({ + sandboxName: "retained-sb", + sandboxIdentityFingerprint: fingerprint, + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + resources: evidence, + reason: "cancelled_after_sandbox_creation", + recordedAt: "2026-08-27T00:00:00.000Z", + }); + + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([recorded]); + expect(recorded).toMatchObject({ + sandboxName: "retained-sb", + sandboxIdentityFingerprint: fingerprint, + identityWasUnavailable: false, + resources: evidence, + }); + expect(fs.readFileSync(recovery.RETAINED_SANDBOX_RECOVERY_FILE, "utf8")).not.toContain( + "secret-value", + ); + }); + + it("records an explicit missing identity", async () => { + const recovery = await import("./onboard-session"); + + const recorded = recovery.recordRetainedSandboxRecovery({ + sandboxName: "missing-id", + sandboxIdentityFingerprint: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: null, + resources: { + sharedInferenceProviders: [], + sandboxScopedProviders: [], + credentialEnvironmentVariables: [], + }, + reason: "retained_after_sandbox_creation_failure", + }); + + expect(recorded).toMatchObject({ + sandboxIdentityFingerprint: null, + identityWasUnavailable: true, + lifecycleGeneration: null, + }); + }); + + it("clears only after a durable exact-identity administrator receipt", async () => { + const recovery = await import("./onboard-session"); + const fingerprint = "b".repeat(64); + const recorded = recovery.recordRetainedSandboxRecovery({ + sandboxName: "retained-sb", + sandboxIdentityFingerprint: fingerprint, + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + resources: evidence, + reason: "cancelled_after_sandbox_creation", + }); + + expect(() => + recovery.resolveRetainedSandboxRecovery({ + recordId: recorded.recordId, + receiptId: "c".repeat(64), + sandboxName: recorded.sandboxName, + sandboxIdentityFingerprint: "d".repeat(64), + gatewayName: recorded.gatewayName, + gatewayPort: recorded.gatewayPort, + outcome: "removed_verified_identity", + }), + ).toThrow(/does not match retained sandbox identity/u); + expect(recovery.listRetainedSandboxRecoveryRecords()).toHaveLength(1); + + const receipt = recovery.resolveRetainedSandboxRecovery({ + recordId: recorded.recordId, + receiptId: "c".repeat(64), + sandboxName: recorded.sandboxName, + sandboxIdentityFingerprint: fingerprint, + gatewayName: recorded.gatewayName, + gatewayPort: recorded.gatewayPort, + outcome: "removed_verified_identity", + }); + + expect(receipt.recordId).toBe(recorded.recordId); + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([]); + }); +}); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index b1958977ad1..45ce5d614b9 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -123,6 +123,12 @@ describe("fresh create identity", () => { const credentialsPath = JSON.stringify( path.join(repoRoot, "src", "lib", "credentials", "store.ts"), ); + const entryOptionsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "entry-options.ts"), + ); + const retainedRecoveryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), + ); const dockerExecPath = JSON.stringify( path.join(repoRoot, "src", "lib", "adapters", "docker", "exec.ts"), ); @@ -142,6 +148,8 @@ let _deleted = false; const registry = require(${registryPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); +const entryOptions = require(${entryOptionsPath}); +const retainedRecovery = require(${retainedRecoveryPath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); const dockerExec = require(${dockerExecPath}); @@ -368,6 +376,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { postCreateFinalizationRefusal ? onboardModule.onboardSession.loadSession() : null, + retainedRecoveryRecords: retainedRecovery.listRetainedSandboxRecoveryRecords(), createCommand: createCommand?.command ?? null, commandNames: commands.map((entry) => entry.command), })); @@ -376,11 +385,35 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; if (recoveryReentry) { + if (recoveryReentry === "fresh-different") { + const retainedNames = retainedRecovery + .listRetainedSandboxRecoveryRecords() + .map((record) => record.sandboxName); + const resolved = entryOptions.resolveDefaultRunEntryOptions( + { fresh: true, sandboxName: "replacement-sb" }, + onboardModule.onboardSession.loadSession(), + runner.validateName, + process.env, + retainedNames, + ); + onboardModule.onboardSession.clearSession(); + onboardModule.onboardSession.saveSession( + onboardModule.onboardSession.createSession({ + mode: resolved.nonInteractive ? "non-interactive" : "interactive", + sandboxName: resolved.requestedSandboxName, + metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, + }), + ); + writePayload("replacement-sb", null, 0); + clearInterval(keepAlive); + return; + } try { await onboardModule.onboard({ resume: recoveryReentry === "explicit", fresh: recoveryReentry === "fresh-same", - sandboxName: recoveryReentry === "fresh-same" ? "my-assistant" : undefined, + recreateSandbox: recoveryReentry === "recreate", + sandboxName: "my-assistant", deferProcessExit: true, }); writePayload(null, "recovery-only onboarding unexpectedly continued", 0); @@ -588,6 +621,16 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, identityFingerprint, ); + assert.equal(payload.retainedRecoveryRecords.length, 1); + assert.deepEqual(payload.retainedRecoveryRecords[0], { + ...payload.retainedRecoveryRecords[0], + sandboxName: "my-assistant", + sandboxIdentityFingerprint: identityFingerprint, + identityWasUnavailable: false, + gatewayName: "nemoclaw", + gatewayPort: 8080, + reason: "retained_after_sandbox_creation_failure", + }); }; const assertPostCreateRegistrationRefusal = () => { const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); @@ -602,6 +645,16 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, identityFingerprint, ); + assert.equal(payload.retainedRecoveryRecords.length, 1); + assert.deepEqual(payload.retainedRecoveryRecords[0], { + ...payload.retainedRecoveryRecords[0], + sandboxName: "my-assistant", + sandboxIdentityFingerprint: identityFingerprint, + identityWasUnavailable: false, + gatewayName: "nemoclaw", + gatewayPort: 8080, + reason: "cancelled_after_sandbox_creation", + }); }; const assertPostCreateFinalizationRefusal = () => { const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); @@ -655,25 +708,55 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assert.match(result.stderr, /confirm that the exact sandbox is absent/u); assert.match(result.stderr, /rotate any credential/u); + const differentName = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...childEnv, + NEMOCLAW_RECOVERY_REENTRY: "fresh-different", + }, + timeout: 30000, + }); + assert.equal(differentName.status, 0, differentName.stderr); + const differentNamePayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + assert.equal(differentNamePayload.exitCode, 0); + assert.equal(differentNamePayload.savedSession.sandboxName, "replacement-sb"); + assert.deepEqual( + differentNamePayload.retainedRecoveryRecords, + payload.retainedRecoveryRecords, + ); + assert.deepEqual(differentNamePayload.commandNames, []); + assert.equal(differentNamePayload.credentialReadCalls, 0); + assert.equal(differentNamePayload.routeReservationCalls, 0); + assert.deepEqual(differentNamePayload.registryMutationCalls, []); + const reentryCases = [ { mode: "automatic", messages: [ - /preserved sandbox 'my-assistant' in recovery-only state/u, - /resume, reuse, and recreation are disabled/u, + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, ], }, { mode: "explicit", messages: [ - /preserved sandbox 'my-assistant' in recovery-only state/u, - /resume, reuse, and recreation are disabled/u, + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, + ], + }, + { + mode: "recreate", + messages: [ + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, ], }, { mode: "fresh-same", messages: [ - /requires --fresh with an explicit sandbox name different from the retained sandbox/u, + /cannot use retained sandbox 'my-assistant'/u, + /same-name fresh onboarding remain disabled/u, ], }, ] as const; @@ -696,7 +779,8 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assert.equal(reentryPayload.routeReservationCalls, 0); assert.deepEqual(reentryPayload.registryMutationCalls, []); assert.deepEqual(reentryPayload.currentRegistryEntry, payload.currentRegistryEntry); - assert.deepEqual(reentryPayload.savedSession, payload.savedSession); + assert.deepEqual(reentryPayload.savedSession, differentNamePayload.savedSession); + assert.deepEqual(reentryPayload.retainedRecoveryRecords, payload.retainedRecoveryRecords); } }; const assertions = { From 1641b1561f48f212641c39301a17c729f14322a0 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 03:10:23 -0700 Subject: [PATCH 27/42] docs(onboard): scope retained recovery guidance Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 13 ++++++++----- src/lib/onboard/cancel-rollback.test.ts | 9 ++++++--- src/lib/onboard/cancel-rollback.ts | 8 +++++--- src/lib/onboard/lifecycle-contracts.md | 2 +- .../onboard-fresh-create-identity.test.ts | 12 +++++++++--- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 6f89cbaa0a0..2d46b8b34bb 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -900,11 +900,14 @@ If you cancel a brand-new onboarding run at the policy-tier selector or either p NemoClaw reports the durable sandbox identity fingerprint when it is available. It does not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. Do not delete the sandbox by mutable name. -Provider registrations and gateway-bound credentials created before cancellation may remain. -Ask an OpenShell administrator to inspect the exact sandbox identity, provider registrations, and gateway-bound credentials, then retain them or remove only resources whose ownership is confirmed. -Before reusing the sandbox name, confirm that the exact sandbox is absent and rotate any credential that may have been configured or exposed before cancellation. -NemoClaw records the incomplete sandbox as recovery-only and rejects automatic resume, explicit `--resume`, reuse, and recreation. -After an OpenShell administrator verifies the live durable ID, removes that exact sandbox through an identity-bound procedure, and confirms removal, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and change only the sandbox name. +Shared inference providers are gateway configuration, not sandbox cleanup targets. +Sandbox-scoped provider registrations or gateway-bound credentials may remain when the durable recovery record lists them. +Ask an OpenShell administrator to inspect the exact sandbox identity and remove only sandbox-scoped resources whose ownership is confirmed for the retained sandbox. +A credential environment name in the recovery record does not prove that its value was exposed. +Rotate a credential only when identity-bound inspection proves that it was exposed or attached to a retained sandbox-scoped resource. +NemoClaw stores the recovery record independently from the active onboarding session. +A fresh run with a different name can proceed without clearing that record, but automatic resume, explicit `--resume`, reuse, recreation, and fresh onboarding with the retained name remain blocked. +After an OpenShell administrator verifies the live durable ID, resolves or removes that exact sandbox through an identity-bound procedure, and durably records the resolution, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and use an available sandbox name. `--fresh` starts a new session and does not retain those selections. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index df39d9fa44e..e3f02aefea3 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -31,10 +31,13 @@ describe("createSandboxCancelRollback", () => { expect(guidance).toContain("OpenShell administrator"); expect(guidance).toContain("did not run OpenShell's mutable-name deletion command"); expect(guidance).toContain("Do not delete the sandbox by mutable sandbox name"); - expect(guidance).toContain("Provider registrations and gateway-bound credentials"); - expect(guidance).toContain("remove only resources whose ownership is confirmed"); + expect(guidance).toContain("Shared inference providers are gateway configuration"); + expect(guidance).toContain("not sandbox cleanup targets"); + expect(guidance).toContain("sandbox-scoped resources whose ownership is confirmed"); expect(guidance).toContain("confirm that the exact sandbox is absent"); - expect(guidance).toContain("rotate any credential"); + expect(guidance).toContain("credential environment name alone does not prove exposure"); + expect(guidance).toContain("rotate a credential only when identity-bound inspection proves"); + expect(guidance).not.toContain("rotate any credential"); }); it.each([ diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index e33ed08c493..dfce5764597 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -59,9 +59,11 @@ export function buildCancelRollbackMessage( ]), " NemoClaw did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox.", " Do not delete the sandbox by mutable sandbox name.", - " Provider registrations and gateway-bound credentials created before cancellation may remain.", - " Ask an OpenShell administrator to inspect the exact sandbox identity, provider registrations, and gateway-bound credentials. Retain them or remove only resources whose ownership is confirmed.", - " Before reusing the sandbox name, confirm that the exact sandbox is absent and rotate any credential that may have been configured or exposed before cancellation.", + " Shared inference providers are gateway configuration and are not sandbox cleanup targets.", + " Sandbox-scoped provider registrations or gateway-bound credentials may remain when the durable recovery record lists them.", + " Ask an OpenShell administrator to inspect the exact sandbox identity and remove only sandbox-scoped resources whose ownership is confirmed for this retained sandbox.", + " A recorded credential environment name alone does not prove exposure; rotate a credential only when identity-bound inspection proves that it was exposed or attached to a retained sandbox-scoped resource.", + " Before reusing the sandbox name, confirm that the exact sandbox is absent and record the identity-bound administrator resolution.", ]; } diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 19944b76ba6..8fa6669f487 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because NemoClaw refuses the available mutable-name deletion command when it could target a replacement. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Cancellation after creation preserves the incomplete session, registry row, and identity evidence for administrator recovery. Provider registrations and gateway-bound credentials may remain; recovery inspects their exact ownership, removes only confirmed resources, verifies sandbox absence, and rotates affected credentials. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because NemoClaw refuses the available mutable-name deletion command when it could target a replacement. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Cancellation after creation preserves the incomplete session, registry row, and an independent identity-bound recovery record. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Recovery removes only confirmed sandbox-scoped resources and rotates credentials only when inspection proves exposure or attachment to a retained resource; a recorded environment-variable name alone is not exposure evidence. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 65d24b2ebb8..feed3b4fda4 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -703,10 +703,16 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { assert.match(result.stderr, /preserved incomplete sandbox 'my-assistant'/u); assert.match(result.stderr, new RegExp(identityFingerprint, "u")); assert.match(result.stderr, /Do not delete the sandbox by mutable sandbox name/u); - assert.match(result.stderr, /Provider registrations and gateway-bound credentials/u); - assert.match(result.stderr, /remove only resources whose ownership is confirmed/u); + assert.match(result.stderr, /Shared inference providers are gateway configuration/u); + assert.match(result.stderr, /not sandbox cleanup targets/u); + assert.match(result.stderr, /sandbox-scoped resources whose ownership is confirmed/u); assert.match(result.stderr, /confirm that the exact sandbox is absent/u); - assert.match(result.stderr, /rotate any credential/u); + assert.match(result.stderr, /credential environment name alone does not prove exposure/u); + assert.match( + result.stderr, + /rotate a credential only when identity-bound inspection proves/u, + ); + assert.doesNotMatch(result.stderr, /rotate any credential/u); const differentName = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, From c508d66b98eb1dda10c6737d84283bdad95adb93 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 03:42:17 -0700 Subject: [PATCH 28/42] fix(onboard): retain recovery persistence retries Signed-off-by: Apurv Kumaria --- src/lib/onboard/cancel-rollback.test.ts | 34 +++++ src/lib/onboard/cancel-rollback.ts | 29 ++-- src/lib/onboard/entry-options.test.ts | 29 +++- src/lib/onboard/entry-options.ts | 9 ++ .../sandbox-create/orchestration.test.ts | 116 +++++++++++++++ .../onboard/sandbox-create/orchestration.ts | 135 +++++++++++++----- .../onboard-fresh-create-identity.test.ts | 134 ++++++++++++++++- 7 files changed, 431 insertions(+), 55 deletions(-) diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index e3f02aefea3..b088c309f25 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -210,6 +210,40 @@ describe("installSandboxCancelRollback", () => { expect(guidance).toContain("could not save the onboarding recovery record"); }); + it("retries recovery on a repeated exit callback after two durable writer failures (#9833)", () => { + const log = vi.fn(); + const recordRecovery = vi + .fn() + .mockImplementationOnce(() => { + throw new Error("immediate recovery write failed"); + }) + .mockImplementationOnce(() => { + throw new Error("first exit recovery write failed"); + }) + .mockImplementationOnce(() => undefined); + const exitHandlers: Array<() => void> = []; + const rollback = installSandboxCancelRollback({ + log, + recordRecovery, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + + rollback.arm("new-sb", SANDBOX_FINGERPRINT); + rollback.markCancelled(); + exitHandlers[0](); + exitHandlers[0](); + + expect(recordRecovery).toHaveBeenCalledTimes(3); + expect(recordRecovery).toHaveBeenNthCalledWith(3, "new-sb", SANDBOX_FINGERPRINT); + const guidanceCalls = log.mock.calls.filter(([message]) => + String(message).includes("preserved incomplete sandbox"), + ); + expect(guidanceCalls).toHaveLength(1); + + exitHandlers[0](); + expect(recordRecovery).toHaveBeenCalledTimes(3); + }); + it("preserves missing-checkpoint recovery state without a mutable-name fallback (#9833)", () => { const log = vi.fn(); const exitHandlers: Array<() => void> = []; diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index dfce5764597..77655ff4e06 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -125,6 +125,7 @@ export function createSandboxCancelRollback( let cancelRequested = false; let recoveryRecorded = false; let recoveryPersistenceFailed = false; + let guidanceReported = false; let done = false; const recordArmedRecovery = (): boolean => { @@ -168,20 +169,24 @@ export function createSandboxCancelRollback( runIfArmed(): void { if (done || !cancelRequested || armedSandbox === null) return; const { name: sandboxName, identityFingerprint } = armedSandbox; - recordArmedRecovery(); + const persisted = recordArmedRecovery(); + if (!guidanceReported) { + guidanceReported = true; + if (recoveryPersistenceFailed) { + deps.log( + " NemoClaw could not save the onboarding recovery record; preserve the registry entry and exact sandbox identity for administrator recovery.", + ); + } + for (const line of buildCancelRollbackMessage( + sandboxName, + identityFingerprint ?? undefined, + )) { + deps.log(line); + } + } + if (!persisted) return; done = true; armedSandbox = null; - if (recoveryPersistenceFailed) { - deps.log( - " NemoClaw could not save the onboarding recovery record; preserve the registry entry and exact sandbox identity for administrator recovery.", - ); - } - for (const line of buildCancelRollbackMessage( - sandboxName, - identityFingerprint ?? undefined, - )) { - deps.log(line); - } }, }; } diff --git a/src/lib/onboard/entry-options.test.ts b/src/lib/onboard/entry-options.test.ts index cb59b108950..ae40e6f3bc9 100644 --- a/src/lib/onboard/entry-options.test.ts +++ b/src/lib/onboard/entry-options.test.ts @@ -228,15 +228,17 @@ describe("resolveOnboardEntryOptions", () => { stdoutIsTty: true, persistedSessionStatus: "recovery_required", persistedRecoverySandboxName: "retained-sb", + persistedSessionSandboxName: "retained-sb", + retainedRecoverySandboxNames: ["retained-sb"], }, deps, ), ).toThrow(ExitError); expect(deps.error).toHaveBeenCalledWith( - expect.stringContaining("preserved sandbox 'retained-sb' in recovery-only state"), + expect.stringContaining("cannot use retained sandbox 'retained-sb'"), ); expect(deps.error).toHaveBeenCalledWith( - expect.stringContaining("resume, reuse, and recreation are disabled"), + expect.stringContaining("resume, reuse, recreation, and same-name fresh onboarding"), ); }); @@ -275,6 +277,7 @@ describe("resolveOnboardEntryOptions", () => { stdoutIsTty: true, persistedSessionStatus: "recovery_required", persistedRecoverySandboxName: "retained-sb", + retainedRecoverySandboxNames: ["retained-sb"], }, deps, ); @@ -287,6 +290,28 @@ describe("resolveOnboardEntryOptions", () => { expect(deps.error).not.toHaveBeenCalled(); }); + it("rejects a different fresh name when recovery has no independent durable record", () => { + const deps = createDeps(); + + expect(() => + resolveOnboardEntryOptions( + { + opts: { fresh: true, sandboxName: "replacement-sb" }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "recovery_required", + persistedRecoverySandboxName: "retained-sb", + retainedRecoverySandboxNames: [], + }, + deps, + ), + ).toThrow(ExitError); + expect(deps.error).toHaveBeenCalledWith( + expect.stringContaining("independent retained sandbox recovery record"), + ); + }); + it("blocks a retained name after a different fresh session replaced the active session", () => { const deps = createDeps(); diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index 2afa3b97c52..4ce3d17c7a0 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -369,6 +369,15 @@ export function resolveOnboardEntryOptions( ); deps.exitProcess(1); } + if (!retainedRecoverySandboxNames.has(recoverySandboxName)) { + deps.error( + " Onboarding cannot replace the recovery-only session because its independent retained sandbox recovery record is unavailable.", + ); + deps.error( + " Preserve the session and registry state for identity-bound administrator recovery.", + ); + deps.exitProcess(1); + } } if (cannotPrompt && !resume && requestedFromDockerfile && !requestedSandboxName) { deps.error( diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 9dcb3823376..d1436bb0353 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -16,11 +16,15 @@ import { completeHermesPortableSandboxRegistration, createProviderEffectBoundary, hasManagedMcpRebuildHandoff, + installPostCreateRecoveryRetryOwner, + persistPostCreateRecovery, persistRetainedSandboxRecoveryMessage, readManagedDcodeCreateSelectionDrift, readSandboxRecreateRegistryEntry, resolveSandboxCreatePolicyAuthority, + runAsyncWithPostCreateRecovery, runSandboxCreateWithPolicyAuthorityChecks, + runWithPostCreateRecovery, } from "./orchestration"; describe("retained create recovery persistence", () => { @@ -129,6 +133,118 @@ describe("retained create recovery persistence", () => { vi.unstubAllEnvs(); } }); + + it.each([ + ["registry publication", "false"], + ["registry publication", "throw"], + ["registry publication", "journal readback mismatch"], + ["onboarding finalization", "false"], + ["onboarding finalization", "throw"], + ["onboarding finalization", "journal readback mismatch"], + ] as const)( + "keeps the original %s error when recovery persistence returns %s (#9833)", + async (stage, failureMode) => { + const operationError = new Error(`${stage} failed`); + const recoveryFailures = { + false: () => false, + throw: () => { + throw new Error("retained sandbox recovery writer threw"); + }, + "journal readback mismatch": () => { + throw new Error("Retained sandbox recovery record did not survive durable readback."); + }, + } satisfies Record false | never>; + const markRetainedSandboxRecovery = vi.fn(recoveryFailures[failureMode]); + const recordRecovery = () => + persistPostCreateRecovery({ + stage, + sandboxName: "alpha", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-1", + exactIdentity: "f".repeat(64), + markRetainedSandboxRecovery, + }); + + const caught = + stage === "registry publication" + ? await runAsyncWithPostCreateRecovery( + async () => Promise.reject(operationError), + recordRecovery, + ).catch((error: unknown) => error) + : (() => { + try { + return runWithPostCreateRecovery(() => { + throw operationError; + }, recordRecovery); + } catch (error) { + return error; + } + })(); + + expect(caught).toBeInstanceOf(AggregateError); + expect((caught as AggregateError).errors).toEqual( + expect.arrayContaining([ + operationError, + expect.objectContaining({ + message: expect.stringContaining("could not save the retained sandbox recovery"), + }), + ]), + ); + expect(((caught as AggregateError).errors[1] as Error).cause).toEqual( + failureMode === "false" + ? undefined + : expect.objectContaining({ + message: expect.stringMatching(/writer threw|did not survive durable readback/u), + }), + ); + }, + ); + + it.each(["registry publication", "onboarding finalization"] as const)( + "retries %s recovery at exit without rerunning the failed operation (#9833)", + async (stage) => { + const exitHandlers: Array<() => void> = []; + const owner = installPostCreateRecoveryRetryOwner({ + log: vi.fn(), + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + const operationError = new Error(`${stage} failed`); + const operation = vi.fn(() => { + throw operationError; + }); + const recordRecovery = vi + .fn() + .mockImplementationOnce(() => { + throw new Error("retained recovery write failed"); + }) + .mockImplementationOnce(() => undefined); + const recordWithOwner = () => owner.record(recordRecovery); + + const caught = + stage === "registry publication" + ? await runAsyncWithPostCreateRecovery(async () => operation(), recordWithOwner).catch( + (error: unknown) => error, + ) + : (() => { + try { + return runWithPostCreateRecovery(operation, recordWithOwner); + } catch (error) { + return error; + } + })(); + + expect(caught).toBeInstanceOf(AggregateError); + expect(operation).toHaveBeenCalledOnce(); + expect(recordRecovery).toHaveBeenCalledOnce(); + + exitHandlers[0](); + expect(recordRecovery).toHaveBeenCalledTimes(2); + expect(operation).toHaveBeenCalledOnce(); + + exitHandlers[0](); + expect(recordRecovery).toHaveBeenCalledTimes(2); + }, + ); }); describe("APF create policy selection", () => { diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index e921d26d56e..0c3bb75dbef 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -73,20 +73,63 @@ export function persistRetainedSandboxRecoveryMessage( sandboxIdentityFingerprint?: string, ) => unknown | null, ): boolean { - try { - return ( - markRetainedSandboxRecovery( - input.sandboxName, - input.message, - input.sandboxIdentityFingerprint, - ) !== null - ); - } catch { - return false; + return Boolean( + markRetainedSandboxRecovery(input.sandboxName, input.message, input.sandboxIdentityFingerprint), + ); +} + +export class RetainedSandboxRecoveryPersistenceError extends Error { + constructor( + readonly stage: "registry publication" | "onboarding finalization", + options?: ErrorOptions, + ) { + super(`NemoClaw could not save the retained sandbox recovery record after ${stage}.`, options); + this.name = "RetainedSandboxRecoveryPersistenceError"; } } -function persistPostCreateRecovery(input: { +export interface PostCreateRecoveryRetryOwner { + record(recordRecovery: () => void): void; +} + +export function installPostCreateRecoveryRetryOwner( + options: { + readonly log?: (message: string) => void; + readonly registerExitHandler?: (handler: () => void) => void; + } = {}, +): PostCreateRecoveryRetryOwner { + let pending: (() => void) | null = null; + const log = options.log ?? ((message: string) => console.error(message)); + const attemptPending = (propagateFailure: boolean): void => { + if (pending === null) return; + const attempt = pending; + try { + attempt(); + if (pending === attempt) pending = null; + } catch (error) { + if (propagateFailure) throw error; + log( + " NemoClaw still could not save the retained sandbox recovery record; the recovery-only session remains blocked for administrator recovery.", + ); + } + }; + const owner: PostCreateRecoveryRetryOwner = { + record(recordRecovery): void { + attemptPending(true); + pending = recordRecovery; + attemptPending(true); + }, + }; + const register = + options.registerExitHandler ?? + ((handler: () => void) => { + process.on("exit", handler); + }); + register(() => attemptPending(false)); + return owner; +} + +export function persistPostCreateRecovery(input: { readonly stage: "registry publication" | "onboarding finalization"; readonly sandboxName: string; readonly gatewayName: string; @@ -102,44 +145,55 @@ function persistPostCreateRecovery(input: { `Sandbox '${input.sandboxName}' was retained after ${input.stage} failed. ` + `Gateway '${input.gatewayName}'. Lifecycle generation '${input.lifecycleGeneration}'. ` + "Do not delete the sandbox by mutable name; preserve it for identity-bound administrator recovery."; - const persisted = persistRetainedSandboxRecoveryMessage( - { - sandboxName: input.sandboxName, - message, - ...(input.exactIdentity - ? { sandboxIdentityFingerprint: input.exactIdentity } - : {}), - }, - input.markRetainedSandboxRecovery, - ); + let persisted = false; + try { + persisted = persistRetainedSandboxRecoveryMessage( + { + sandboxName: input.sandboxName, + message, + ...(input.exactIdentity ? { sandboxIdentityFingerprint: input.exactIdentity } : {}), + }, + input.markRetainedSandboxRecovery, + ); + } catch (cause) { + throw new RetainedSandboxRecoveryPersistenceError(input.stage, { cause }); + } if (!persisted) { - console.error( - " NemoClaw could not save the retained sandbox recovery record. Preserve the terminal output and registry state for an OpenShell administrator.", + throw new RetainedSandboxRecoveryPersistenceError(input.stage); + } +} + +function throwPostCreateFailure(error: unknown, recordRecovery: () => void): never { + try { + recordRecovery(); + } catch (recoveryError) { + throw new AggregateError( + [error, recoveryError], + "The sandbox operation failed, and its retained recovery record could not be persisted.", ); } + throw error; } -async function runAsyncWithPostCreateRecovery( +export async function runAsyncWithPostCreateRecovery( operation: () => Promise, recordRecovery: () => void, ): Promise { try { return await operation(); } catch (error) { - recordRecovery(); - throw error; + return throwPostCreateFailure(error, recordRecovery); } } -function runWithPostCreateRecovery( +export function runWithPostCreateRecovery( operation: () => Result, recordRecovery: () => void, ): Result { try { return operation(); } catch (error) { - recordRecovery(); - throw error; + return throwPostCreateFailure(error, recordRecovery); } } @@ -700,6 +754,7 @@ function readHermesPortableLifecycleGeneration(input: { } export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrchestrationRuntime) { + const postCreateRecoveryRetryOwner = installPostCreateRecoveryRetryOwner(); return async function createSandboxWithBaseImageResolution( baseImageResolutionContext: import("../base-image-resolution-flow").BaseImageResolutionContext, portableRuntimeContext: PortableOnboardRuntimeContext | null, @@ -1894,16 +1949,18 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche const recordPostCreateRecovery = ( stage: "registry publication" | "onboarding finalization", ): void => - persistPostCreateRecovery({ - stage, - sandboxName, - gatewayName: GATEWAY_NAME, - lifecycleGeneration: createdSandboxLifecycle.generation, - ...(verifiedPolicyGate - ? { exactIdentity: verifiedPolicyGate.lifecycleLiveIdentityFingerprint } - : {}), - markRetainedSandboxRecovery: onboardSession.markRetainedSandboxRecovery, - }); + postCreateRecoveryRetryOwner.record(() => + persistPostCreateRecovery({ + stage, + sandboxName, + gatewayName: GATEWAY_NAME, + lifecycleGeneration: createdSandboxLifecycle.generation, + ...(verifiedPolicyGate + ? { exactIdentity: verifiedPolicyGate.lifecycleLiveIdentityFingerprint } + : {}), + markRetainedSandboxRecovery: onboardSession.markRetainedSandboxRecovery, + }), + ); const runCreateFlow = async ( attemptCreateArgv: string[], hermesPortableReadyCapture?: import("../sandbox-gpu-create-flow").HermesPortableReadyCapture, diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index feed3b4fda4..5e1b3504f88 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -61,6 +61,22 @@ describe("fresh create identity", () => { agent: null, expectedOutcome: "post-create-registration-refusal" as const, }, + { + title: "blocks every reentry when registry-failure recovery has no durable journal (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-registration-recovery-readback-failure" as const, + }, + { + title: "retries registry-failure recovery from the process-exit owner (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-registration-recovery-retry" as const, + }, { title: "retains recovery state when final checks fail after registration (#9833)", apfInterceptorRequested: true, @@ -186,7 +202,16 @@ const postCreateAuthorityRefusal = ${JSON.stringify( expectedOutcome === "post-create-authority-refusal", )}; const postCreateRegistrationRefusal = ${JSON.stringify( - expectedOutcome === "post-create-registration-refusal", + expectedOutcome === "post-create-registration-refusal" || + expectedOutcome === "post-create-registration-recovery-readback-failure" || + expectedOutcome === "post-create-registration-recovery-retry", + )}; +let recoveryJournalReadbackFailuresRemaining = ${JSON.stringify( + expectedOutcome === "post-create-registration-recovery-readback-failure" + ? 100 + : expectedOutcome === "post-create-registration-recovery-retry" + ? 1 + : 0, )}; const postCreateFinalizationRefusal = ${JSON.stringify( expectedOutcome === "post-create-finalization-refusal", @@ -336,6 +361,22 @@ childProcess.spawn = (...args) => { const onboardModule = require(${onboardPath}); const { createSandbox } = onboardModule; +if (recoveryJournalReadbackFailuresRemaining > 0) { + const renameSync = fs.renameSync.bind(fs); + fs.renameSync = (source, destination) => { + renameSync(source, destination); + if ( + recoveryJournalReadbackFailuresRemaining > 0 && + String(destination).endsWith("retained-sandbox-recovery.json") + ) { + recoveryJournalReadbackFailuresRemaining -= 1; + fs.writeFileSync( + destination, + JSON.stringify({ schemaVersion: 1, unresolved: [], resolutions: [] }), + ); + } + }; +} if (cancelAfterCreate && !recoveryReentry) { const session = onboardModule.onboardSession.createSession({ mode: "interactive", @@ -381,10 +422,32 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { commandNames: commands.map((entry) => entry.command), })); }; +let finalCreationError = null; +if (${JSON.stringify(expectedOutcome === "post-create-registration-recovery-retry")}) { + process.on("exit", (code) => writePayload(null, finalCreationError, code)); +} (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; if (recoveryReentry) { + if (recoveryReentry === "fresh-different-no-journal") { + try { + await onboardModule.onboard({ + fresh: true, + sandboxName: "replacement-sb", + deferProcessExit: true, + }); + writePayload(null, "recovery-only onboarding unexpectedly continued", 0); + } catch (error) { + writePayload( + null, + error instanceof Error ? error.message : String(error), + typeof error?.code === "number" ? error.code : 1, + ); + } + clearInterval(keepAlive); + return; + } if (recoveryReentry === "fresh-different") { const retainedNames = retainedRecovery .listRetainedSandboxRecoveryRecords() @@ -465,7 +528,8 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { } catch (error) { if (cancelAfterCreate) throw error; if (!apfInterceptorRequested) throw error; - writePayload(null, error instanceof Error ? error.message : String(error)); + finalCreationError = error instanceof Error ? error.message : String(error); + writePayload(null, finalCreationError); } clearInterval(keepAlive); })().catch((error) => { @@ -656,6 +720,69 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { reason: "cancelled_after_sandbox_creation", }); }; + const assertPostCreateRegistrationRecoveryReadbackFailure = () => { + const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /recovery record could not be persisted/u); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + assert.deepEqual(payload.retainedRecoveryRecords, []); + + const reentryCases = [ + { + mode: "fresh-different-no-journal", + message: /independent retained sandbox recovery record is unavailable/u, + }, + { + mode: "fresh-same", + message: /explicit sandbox name different from the retained sandbox/u, + }, + ] as const; + for (const { message, mode } of reentryCases) { + const reentry = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...childEnv, + NEMOCLAW_RECOVERY_REENTRY: mode, + }, + timeout: 30000, + }); + assert.equal(reentry.status, 0, reentry.stderr); + const reentryPayload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + assert.equal(reentryPayload.exitCode, 1); + assert.match(reentry.stderr, message); + assert.deepEqual(reentryPayload.commandNames, []); + assert.equal(reentryPayload.credentialReadCalls, 0); + assert.equal(reentryPayload.routeReservationCalls, 0); + assert.deepEqual(reentryPayload.registryMutationCalls, []); + assert.equal(reentryPayload.savedSession.sandboxName, "my-assistant"); + assert.deepEqual(reentryPayload.retainedRecoveryRecords, []); + } + }; + const assertPostCreateRegistrationRecoveryRetry = () => { + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /recovery record could not be persisted/u); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal(payload.retainedRecoveryRecords.length, 1); + assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); + assert.equal( + payload.commandNames.filter((command: string) => command.includes("sandbox create")) + .length, + 1, + ); + }; const assertPostCreateFinalizationRefusal = () => { const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.sandboxName, null); @@ -795,6 +922,9 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { "providerless-apf": assertProviderlessApfCreation, "post-create-authority-refusal": assertPostCreateAuthorityRefusal, "post-create-registration-refusal": assertPostCreateRegistrationRefusal, + "post-create-registration-recovery-readback-failure": + assertPostCreateRegistrationRecoveryReadbackFailure, + "post-create-registration-recovery-retry": assertPostCreateRegistrationRecoveryRetry, "post-create-finalization-refusal": assertPostCreateFinalizationRefusal, "staged-messaging-refusal": assertStagedMessagingRefusal, "cancel-after-create-tier": assertCancellationRecovery, From 2aeff9c35dfab81b96d4ff32f2c30d2b5aa030a3 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 17:48:17 +0700 Subject: [PATCH 29/42] fix(security): close retained recovery state race --- .../retained-sandbox-recovery.ts | 19 +++++++++++++++---- .../state/retained-sandbox-recovery.test.ts | 12 ++++++++++++ .../onboard-fresh-create-identity.test.ts | 2 +- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index 67e95eae45e..bbe3a88728f 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -5,6 +5,8 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { openRegularFileNoFollow } from "../../adapters/fs/regular-file"; + const SCHEMA_VERSION = 1; const FINGERPRINT_PATTERN = /^[0-9a-f]{64}$/u; const SAFE_EVIDENCE_PATTERN = /^[A-Za-z0-9._:@/-]{1,256}$/u; @@ -91,11 +93,12 @@ function isObjectRecord(value: unknown): value is Record { function readStateFile(filePath: string): unknown { try { - const stat = fs.lstatSync(filePath); - if (stat.isSymbolicLink()) { - throw new Error("Retained sandbox recovery state cannot be a symbolic link."); + const file = openRegularFileNoFollow(filePath); + try { + return JSON.parse(file.readUtf8()); + } finally { + file.close(); } - return JSON.parse(fs.readFileSync(filePath, "utf8")); } catch (error) { if ( error instanceof Error && @@ -104,6 +107,14 @@ function readStateFile(filePath: string): unknown { ) { return emptyState(); } + if ( + error instanceof Error && + "code" in error && + ((error as NodeJS.ErrnoException).code === "ELOOP" || + (error as NodeJS.ErrnoException).code === "EMLINK") + ) { + throw new Error("Retained sandbox recovery state cannot be a symbolic link."); + } throw error; } } diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index 01e8ae2501a..d234052074a 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -78,6 +78,18 @@ describe("retained sandbox recovery state", () => { }); }); + it("refuses a symbolic-link recovery state without reading its target", async () => { + const recovery = await import("./onboard-session"); + const externalState = path.join(home, "external-recovery.json"); + const externalContents = "{not recovery json"; + fs.writeFileSync(externalState, externalContents); + fs.mkdirSync(path.dirname(recovery.RETAINED_SANDBOX_RECOVERY_FILE), { recursive: true }); + fs.symlinkSync(externalState, recovery.RETAINED_SANDBOX_RECOVERY_FILE); + + expect(() => recovery.listRetainedSandboxRecoveryRecords()).toThrow(/symbolic link/u); + expect(fs.readFileSync(externalState, "utf8")).toBe(externalContents); + }); + it("clears only after a durable exact-identity administrator receipt", async () => { const recovery = await import("./onboard-session"); const fingerprint = "b".repeat(64); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 5e1b3504f88..6776e32a163 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -717,7 +717,7 @@ if (${JSON.stringify(expectedOutcome === "post-create-registration-recovery-retr identityWasUnavailable: false, gatewayName: "nemoclaw", gatewayPort: 8080, - reason: "cancelled_after_sandbox_creation", + reason: "retained_after_sandbox_creation_failure", }); }; const assertPostCreateRegistrationRecoveryReadbackFailure = () => { From 10cfe1349b8b44f7a17acbc3ce0d16028ff7ee1c Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 04:08:00 -0700 Subject: [PATCH 30/42] fix(onboard): reject provider effects before create resolution Signed-off-by: Apurv Kumaria --- .../onboard/sandbox-create/orchestration.ts | 58 +++++++++++++++++-- .../onboard-fresh-create-identity.test.ts | 8 ++- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 0c3bb75dbef..288a3bf44b3 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -226,6 +226,45 @@ export function assertApfCreateIntent( } } +function assertProviderlessApfCreateInput(input: { + readonly createIntent: SandboxCreateIntent | null; + readonly model: string; + readonly provider: string; + readonly preferredInferenceApi: string | null; + readonly webSearchConfig: WebSearchConfig | null; + readonly enabledChannels: readonly string[] | null; + readonly hermesToolGateways: readonly string[]; +}): void { + if (input.createIntent?.apfInterceptorRequested !== true) return; + const resolved = input.createIntent.resolved; + const hasProviderIntent = + input.webSearchConfig !== null || + input.createIntent.reuseRegisteredCredentials === true || + [ + input.provider, + input.model, + input.preferredInferenceApi, + input.createIntent.endpointUrl, + resolved?.inferenceProvider, + ].some((value) => Boolean(value?.trim())) || + [ + input.enabledChannels, + input.hermesToolGateways, + input.createIntent.extraProviders, + resolved?.activeMessagingChannels, + resolved?.messagingProviderRequests, + resolved?.reusableMessagingProviders, + resolved?.extraProviders, + resolved?.staleExtraProviders, + resolved?.hermesToolGateways, + resolved?.extraPlaceholderKeys, + ].some((values) => (values?.length ?? 0) > 0); + if (!hasProviderIntent) return; + throw new Error( + "Interceptor onboarding supports providerless sandbox creation only. No sandbox or provider was created.", + ); +} + type SandboxRecreateReasonInput = { sandboxName: string; recreateForAgentDrift: boolean; @@ -888,16 +927,25 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche runCapture, } = runtime; - step(6, 8, "Creating sandbox"); - const sandboxName = validateName( - sandboxNameOverride ?? (await promptValidatedSandboxName(agent)), - "sandbox name", - ); assertApfCreateIntent(createIntent); + assertProviderlessApfCreateInput({ + createIntent, + model, + provider, + preferredInferenceApi, + webSearchConfig, + enabledChannels, + hermesToolGateways, + }); assertProviderlessInterceptorEnvironment( createIntent?.apfInterceptorRequested === true, process.env, ); + step(6, 8, "Creating sandbox"); + const sandboxName = validateName( + sandboxNameOverride ?? (await promptValidatedSandboxName(agent)), + "sandbox name", + ); preparedDcodeRebuild.assertPreparedDcodeTarget(preparedBuildContext, agent, fromDockerfile); const effectiveAgent = sandboxAgent.getEffectiveSandboxAgent(agent); const requestedAgentName = getRequestedSandboxAgentName(effectiveAgent); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 6776e32a163..2d60ebdf100 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -190,6 +190,7 @@ const apfInterceptorRequested = ${JSON.stringify(apfInterceptorRequested)}; const agent = ${JSON.stringify(agent)}; const model = ${JSON.stringify(model)}; const provider = ${JSON.stringify(provider)}; +const selectedChannels = ${JSON.stringify(expectedOutcome === "provider-refusal" ? ["telegram"] : null)}; const cancellationSelector = ${JSON.stringify( expectedOutcome.startsWith("cancel-after-create-") ? expectedOutcome.slice("cancel-after-create-".length) @@ -491,7 +492,7 @@ if (${JSON.stringify(expectedOutcome === "post-create-registration-recovery-retr return; } const createArgs = fixtureMocks.sandboxCreateArgsWithVerifiedReservation( - [null, model, provider, null, null, null, null, null, agent, null, null, null, []], + [null, model, provider, null, null, null, selectedChannels, null, agent, null, null, null, []], createFixture, ); if (apfInterceptorRequested) { @@ -577,12 +578,15 @@ if (${JSON.stringify(expectedOutcome === "post-create-registration-recovery-retr const assertProviderBackedApfRefusal = () => { assert.match( payload.creationError, - /Cannot create sandbox 'my-assistant' with deferred providers .* No sandbox was created/u, + /supports providerless sandbox creation only.*No sandbox or provider was created/u, ); assert.equal(payload.sandboxName, null); assert.equal(payload.sandboxCreated, false); assert.equal(payload.createCommand, null); assert.equal(payload.registeredSandbox, null); + assert.equal(payload.credentialReadCalls, 0); + assert.equal(payload.routeReservationCalls, 0); + assert.deepEqual(payload.registryMutationCalls, []); assert.deepEqual(providerEffectCommands, []); assert.equal( payload.commandNames.some((command: string) => command.includes("sandbox create")), From ffe9fa54509d34aa99225168a96703c9294cba1c Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 05:25:49 -0700 Subject: [PATCH 31/42] fix(onboard): retain recovery after create failures Signed-off-by: Apurv Kumaria --- .../sandbox-create/orchestration.test.ts | 74 +++++++++++++++++++ .../onboard/sandbox-create/orchestration.ts | 50 ++++++++++--- src/lib/onboard/sandbox-gpu-create-flow.ts | 5 +- .../sandbox-gpu-create-identity-gate.test.ts | 44 +++++++++++ .../onboard/sandbox-gpu-create-run-attempt.ts | 48 +++++++++--- .../onboard-fresh-create-identity.test.ts | 53 ++++++++++++- 6 files changed, 251 insertions(+), 23 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index d1436bb0353..55e03296086 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -722,6 +722,80 @@ describe("sandbox create policy authority checks", () => { ); }); + it("records recovery when the create runner fails after verification (#9833)", async () => { + const createFailure = new Error("runtime patch failed after verification"); + const persistRetainedSandboxRecovery = vi.fn(() => true); + + const error = await runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + throw createFailure; + }, + ...exactIdentityBoundary(), + persistRetainedSandboxRecovery, + cleanupTemporarySources: vi.fn(), + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).errors).toContain(createFailure); + expect(persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("left sandbox 'alpha' in place"), + exactIdentity, + ); + }); + + it.each(["false", "throw", "journal readback mismatch"] as const)( + "retries create-runner recovery when its durable writer returns %s (#9833)", + async (failureMode) => { + const exitHandlers: Array<() => void> = []; + const retryOwner = installPostCreateRecoveryRetryOwner({ + log: vi.fn(), + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + const createFailure = new Error("runtime patch failed after verification"); + const writerFailures = { + false: () => false, + throw: () => { + throw new Error("retained recovery writer threw"); + }, + "journal readback mismatch": () => { + throw new Error("Retained sandbox recovery record did not survive durable readback."); + }, + } satisfies Record boolean>; + const writer = vi + .fn() + .mockImplementationOnce(writerFailures[failureMode]) + .mockReturnValue(true); + const create = vi.fn(async (verifyCreatedSandbox: (created: string) => Promise) => { + await verifyCreatedSandbox("created"); + throw createFailure; + }); + + const error = await runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create, + ...exactIdentityBoundary(), + persistRetainedSandboxRecovery: writer, + retainedSandboxRecoveryRetryOwner: retryOwner, + cleanupTemporarySources: vi.fn(), + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(AggregateError); + expect(writer).toHaveBeenCalledOnce(); + expect(create).toHaveBeenCalledOnce(); + + exitHandlers[0](); + expect(writer).toHaveBeenCalledTimes(2); + expect(create).toHaveBeenCalledOnce(); + + exitHandlers[0](); + expect(writer).toHaveBeenCalledTimes(2); + }, + ); + it("does not delete a same-name replacement after final authority failure (#9833)", async () => { let sandboxIdentity = "created"; const revalidate = vi.fn(); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 288a3bf44b3..416638f0862 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -129,6 +129,22 @@ export function installPostCreateRecoveryRetryOwner( return owner; } +function persistRetainedSandboxRecoveryWithRetry( + retryOwner: PostCreateRecoveryRetryOwner | undefined, + persist: () => boolean, +): boolean { + let persisted = false; + const attempt = (): void => { + persisted = persist(); + if (!persisted) { + throw new Error("NemoClaw could not save the retained sandbox recovery record."); + } + }; + if (retryOwner) retryOwner.record(attempt); + else attempt(); + return persisted; +} + export function persistPostCreateRecovery(input: { readonly stage: "registry publication" | "onboarding finalization"; readonly sandboxName: string; @@ -456,11 +472,13 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< message: string, exactIdentity: string | null, ) => boolean; + readonly retainedSandboxRecoveryRetryOwner?: PostCreateRecoveryRetryOwner; readonly cleanupTemporarySources: () => void; }): Promise { input.revalidate(false, `creating sandbox '${input.sandboxName}'`); let exactIdentity: string | null = null; let cleanupAttempted = false; + let recoveryAttempted = false; const cleanupTemporarySources = (): unknown[] => { if (cleanupAttempted) return []; cleanupAttempted = true; @@ -472,6 +490,7 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< } }; const refuseAfterCreate = (validationError: unknown): never => { + recoveryAttempted = true; const validationDetail = validationError instanceof Error && isPolicyAuthorityRefusalError(validationError) ? validationError.message @@ -486,11 +505,9 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< const compensationErrors: unknown[] = []; if (input.persistRetainedSandboxRecovery) { try { - if (!input.persistRetainedSandboxRecovery(recoveryGuidance, exactIdentity)) { - compensationErrors.push( - new Error("NemoClaw could not save the retained sandbox recovery record."), - ); - } + persistRetainedSandboxRecoveryWithRetry(input.retainedSandboxRecoveryRetryOwner, () => + input.persistRetainedSandboxRecovery!(recoveryGuidance, exactIdentity), + ); } catch (error) { compensationErrors.push(error); } @@ -541,6 +558,7 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< try { result = await input.create(verifyCreatedSandbox); } catch (error) { + if (exactIdentity !== null && !recoveryAttempted) return refuseAfterCreate(error); const cleanupErrors = cleanupTemporarySources(); if (cleanupErrors.length > 0) { throw new AggregateError( @@ -2009,6 +2027,20 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche markRetainedSandboxRecovery: onboardSession.markRetainedSandboxRecovery, }), ); + const persistCreateFlowRecovery = ( + message: string, + exactIdentity: string | null = null, + ): boolean => + persistRetainedSandboxRecoveryWithRetry(postCreateRecoveryRetryOwner, () => + persistRetainedSandboxRecoveryMessage( + { + sandboxName, + message, + ...(exactIdentity ? { sandboxIdentityFingerprint: exactIdentity } : {}), + }, + onboardSession.markRetainedSandboxRecovery, + ), + ); const runCreateFlow = async ( attemptCreateArgv: string[], hermesPortableReadyCapture?: import("../sandbox-gpu-create-flow").HermesPortableReadyCapture, @@ -2083,6 +2115,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }, onboardSession.markRetainedSandboxRecovery, ), + retainedSandboxRecoveryRetryOwner: postCreateRecoveryRetryOwner, cleanupTemporarySources: cleanupSandboxCreateSources, runVerifiedCreateEffects: runDeferredProviderEffects ? async (_identity, _exactIdentity, boundary) => { @@ -2103,11 +2136,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche requirePolicylessCreate: true as const, } : {}), - persistRetainedSandboxRecovery: (message) => - persistRetainedSandboxRecoveryMessage( - { sandboxName, message }, - onboardSession.markRetainedSandboxRecovery, - ), + persistRetainedSandboxRecovery: (message, sandboxIdentityFingerprint) => + persistCreateFlowRecovery(message, sandboxIdentityFingerprint ?? null), provider, sandboxGpuConfig: effectiveSandboxGpuConfig, gpuRoutePlan, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index d05db31f531..3702bfd3571 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -213,7 +213,10 @@ export interface SandboxGpuCreateFlowInput { /** Reject every initial or fallback create attempt that carries a caller policy. */ requirePolicylessCreate?: true; /** Durably retain exact create-attempt recovery evidence before identity-bound recovery stops. */ - persistRetainedSandboxRecovery?: (message: string) => boolean; + persistRetainedSandboxRecovery?: ( + message: string, + sandboxIdentityFingerprint?: string, + ) => boolean; provider: string; sandboxGpuConfig: SandboxGpuConfig; gpuRoutePlan: import("./docker-gpu-route").DockerGpuRoutePlan; diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index a3c67d9e915..f0f63e9d1ef 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -316,6 +316,12 @@ describe("created sandbox identity gate", () => { ); expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining( + `Durable sandbox identity fingerprint: ${fingerprintSandboxRecreateValue("alpha-sandbox-id")}`, + ), + fingerprintSandboxRecreateValue("alpha-sandbox-id"), + ); expect(patch.exitOnPatchError).not.toHaveBeenCalled(); expect(patch.ensureApplied).not.toHaveBeenCalled(); expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); @@ -349,6 +355,12 @@ describe("created sandbox identity gate", () => { ); expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining( + `Durable sandbox identity fingerprint: ${fingerprintSandboxRecreateValue("alpha-sandbox-id")}`, + ), + fingerprintSandboxRecreateValue("alpha-sandbox-id"), + ); expect(patch.exitOnPatchError).not.toHaveBeenCalled(); expect(patch.ensureApplied).not.toHaveBeenCalled(); expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); @@ -458,6 +470,38 @@ describe("created sandbox identity gate", () => { expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); }); + it("returns a post-verification readiness failure to the recovery owner (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: false, + reason: "timeout", + failurePhase: null, + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + const exit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("direct process exit bypassed the recovery owner"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "Sandbox 'alpha' did not become ready after verified creation", + ); + + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); + expect(exit).not.toHaveBeenCalled(); + }); + it("uses a distinct identity label for each create attempt (#9833)", async () => { const input = createGpuFlowInput(); input.verifyCreatedSandboxBeforeEffects = vi.fn(); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index c0e426a9a9a..b063d5834e6 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -393,7 +393,9 @@ export function createSandboxGpuCreateAttemptRunner( const createAttemptNonce = deferPostCreateEffects ? randomBytes(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH / 2).toString("hex") : null; - const persistIdentitySettlementRecovery = (): void => { + const persistIdentitySettlementRecovery = ( + sandboxIdentityFingerprint: string | null = null, + ): void => { if (!createAttemptNonce) { throw new Error("Sandbox create-attempt identity was not generated."); } @@ -401,15 +403,18 @@ export function createSandboxGpuCreateAttemptRunner( if (!persist) { throw new Error("Verified sandbox creation has no durable recovery evidence owner."); } + const identityEvidence = sandboxIdentityFingerprint + ? `Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}. Sandbox '${input.sandboxName}' did not remain visible through owning gateway '${input.gatewayName}' before policy verification. ` + : `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. Gateway '${input.gatewayName}'. OpenShell did not return one exact durable sandbox identity for this create attempt. `; const message = `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}. ` + - `Sandbox '${input.sandboxName}' reached Ready before OpenShell returned one exact durable create identity. ` + - `Gateway '${input.gatewayName}'. ` + - "OpenShell did not return one exact durable sandbox identity for this create attempt. " + + identityEvidence + "Do not delete a sandbox by mutable name; preserve it until an OpenShell administrator resolves the create-attempt label to one sandbox."; let persisted = false; try { - persisted = persist(message); + persisted = sandboxIdentityFingerprint + ? persist(message, sandboxIdentityFingerprint) + : persist(message); } catch { persisted = false; } @@ -420,6 +425,14 @@ export function createSandboxGpuCreateAttemptRunner( ); } }; + const waitForCreatedSandboxPublication = (sandboxId: string): void => { + try { + waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); + } catch (error) { + persistIdentitySettlementRecovery(fingerprintSandboxRecreateValue(sandboxId)); + throw error; + } + }; const captureRetainedSandboxRecovery = () => { if (!input.requirePolicylessCreate || !createAttemptNonce) return {}; let liveIdentityFingerprint: string | null = null; @@ -564,6 +577,10 @@ export function createSandboxGpuCreateAttemptRunner( let createResult: Awaited>; let managedIncompleteCreateRecovered = false; let createdSandboxVerified = false; + const failAfterCreatedSandboxVerification = (message: string, status: number): never => { + if (createdSandboxVerified) throw new Error(message); + return process.exit(status); + }; if (managedBootstrap && managedLifecycle) { try { createResult = await managedLifecycle.runCreate( @@ -594,6 +611,7 @@ export function createSandboxGpuCreateAttemptRunner( sleep: deps.sleep, }); if (!readiness.ready) { + if (createAttemptNonce) persistIdentitySettlementRecovery(); throw new Error( sandboxReadinessTracing .formatCreatedSandboxReadinessFailureMessage( @@ -610,6 +628,7 @@ export function createSandboxGpuCreateAttemptRunner( timeout: SANDBOX_READY_PROBE_TIMEOUT_MS, }); if (!isSandboxReady(list, input.sandboxName)) { + if (createAttemptNonce) persistIdentitySettlementRecovery(); throw new Error( "Managed bootstrap create completed without an authoritative Ready sandbox.", ); @@ -635,7 +654,7 @@ export function createSandboxGpuCreateAttemptRunner( { cause: error }, ); } - waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); + waitForCreatedSandboxPublication(sandboxId); await verifyCreatedSandboxBeforeEffects(sandboxId, route, input); createdSandboxVerified = true; if (deferPostCreateEffects) { @@ -763,7 +782,7 @@ export function createSandboxGpuCreateAttemptRunner( { cause: error }, ); } - waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); + waitForCreatedSandboxPublication(sandboxId); await verifyCreatedSandboxBeforeEffects(sandboxId, route, input); createdSandboxVerified = true; } @@ -784,7 +803,10 @@ export function createSandboxGpuCreateAttemptRunner( printCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, }); - process.exit(createResult.status === 0 ? 1 : createResult.status); + failAfterCreatedSandboxVerification( + `Sandbox '${input.sandboxName}' did not return one exact durable sandbox ID before runtime recreation after verified creation.`, + createResult.status === 0 ? 1 : createResult.status, + ); } if (!portableLifecycle || managedLifecycle) { revalidatePostCreateEffect(`apply runtime patch for sandbox '${input.sandboxName}'`); @@ -874,7 +896,10 @@ export function createSandboxGpuCreateAttemptRunner( ); console.error(" Verify the sandbox identity before manual cleanup."); } - process.exit(createResult.status === 0 ? 1 : createResult.status); + failAfterCreatedSandboxVerification( + `Sandbox '${input.sandboxName}' did not become ready after verified creation.`, + createResult.status === 0 ? 1 : createResult.status, + ); } if (input.sandboxGpuConfig.sandboxGpuEnabled) { revalidatePostCreateEffect(`verify GPU access for sandbox '${input.sandboxName}'`); @@ -933,7 +958,10 @@ export function createSandboxGpuCreateAttemptRunner( console.error( " To explicitly select the compatibility route, clean up the sandbox and retry with NEMOCLAW_DOCKER_GPU_PATCH=1.", ); - process.exit(1); + failAfterCreatedSandboxVerification( + `Sandbox '${input.sandboxName}' failed GPU proof after verified creation.`, + 1, + ); } if (proof.status === "failed") { await runtimePatch.rollbackManagedStartupAfterCreateFailure(); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 2d60ebdf100..d5737382465 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -53,6 +53,14 @@ describe("fresh create identity", () => { agent: null, expectedOutcome: "post-create-authority-refusal" as const, }, + { + title: "retains recovery state when the create runner fails after verification (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: null, + expectedOutcome: "post-create-runner-refusal" as const, + }, { title: "retains recovery state when registry publication fails after create (#9833)", apfInterceptorRequested: true, @@ -202,6 +210,7 @@ const stagedMessagingRefusal = ${JSON.stringify(expectedOutcome === "staged-mess const postCreateAuthorityRefusal = ${JSON.stringify( expectedOutcome === "post-create-authority-refusal", )}; +const postCreateRunnerRefusal = ${JSON.stringify(expectedOutcome === "post-create-runner-refusal")}; const postCreateRegistrationRefusal = ${JSON.stringify( expectedOutcome === "post-create-registration-refusal" || expectedOutcome === "post-create-registration-recovery-readback-failure" || @@ -210,7 +219,8 @@ const postCreateRegistrationRefusal = ${JSON.stringify( let recoveryJournalReadbackFailuresRemaining = ${JSON.stringify( expectedOutcome === "post-create-registration-recovery-readback-failure" ? 100 - : expectedOutcome === "post-create-registration-recovery-retry" + : expectedOutcome === "post-create-registration-recovery-retry" || + expectedOutcome === "post-create-runner-refusal" ? 1 : 0, )}; @@ -281,6 +291,7 @@ runner.run = (command, opts = {}) => { ? JSON.parse(fs.readFileSync(${JSON.stringify(payloadPath)}, "utf8")).currentRegistryEntry : null; const registryMutationCalls = []; + let checkpointReadCalls = 0; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", provider, @@ -306,6 +317,16 @@ runner.run = (command, opts = {}) => { setDefault: (name) => { registryMutationCalls.push({ operation: "set-default", name }); }, removeSandbox: (name) => { registryMutationCalls.push({ operation: "remove", name }); }, }); +if (postCreateRunnerRefusal) { + const requireCurrentCheckpoint = registry.requireCurrentPendingSandboxPolicyVerification; + registry.requireCurrentPendingSandboxPolicyVerification = (...args) => { + checkpointReadCalls += 1; + if (checkpointReadCalls === 6) { + throw new Error("post-verification create runner checkpoint failed"); + } + return requireCurrentCheckpoint(...args); + }; +} preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => { if (cancelPrompt) { @@ -409,11 +430,13 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { registeredSandbox, credentialReadCalls, routeReservationCalls, + checkpointReadCalls, registryMutationCalls, currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, savedSession: cancelAfterCreate || postCreateAuthorityRefusal || + postCreateRunnerRefusal || postCreateRegistrationRefusal || postCreateFinalizationRefusal ? onboardModule.onboardSession.loadSession() @@ -424,7 +447,10 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { })); }; let finalCreationError = null; -if (${JSON.stringify(expectedOutcome === "post-create-registration-recovery-retry")}) { +if (${JSON.stringify( + expectedOutcome === "post-create-registration-recovery-retry" || + expectedOutcome === "post-create-runner-refusal", + )}) { process.on("exit", (code) => writePayload(null, finalCreationError, code)); } @@ -700,6 +726,28 @@ if (${JSON.stringify(expectedOutcome === "post-create-registration-recovery-retr reason: "retained_after_sandbox_creation_failure", }); }; + const assertPostCreateRunnerRefusal = () => { + const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); + assert.equal(payload.sandboxName, null); + assert.equal(payload.sandboxCreated, true); + assert.equal(payload.deleted, false); + assert.equal(payload.registeredSandbox, null); + assert.match(payload.creationError, /automatic sandbox cleanup was not safe/u); + assert.equal(payload.savedSession.status, "recovery_required"); + assert.equal(payload.savedSession.resumable, false); + assert.equal( + payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, + identityFingerprint, + ); + assert.equal(payload.retainedRecoveryRecords.length, 1); + assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); + assert.ok(payload.checkpointReadCalls >= 6); + assert.equal( + payload.commandNames.filter((command: string) => command.includes("sandbox create")) + .length, + 1, + ); + }; const assertPostCreateRegistrationRefusal = () => { const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.sandboxName, null); @@ -925,6 +973,7 @@ if (${JSON.stringify(expectedOutcome === "post-create-registration-recovery-retr "provider-refusal": assertProviderBackedApfRefusal, "providerless-apf": assertProviderlessApfCreation, "post-create-authority-refusal": assertPostCreateAuthorityRefusal, + "post-create-runner-refusal": assertPostCreateRunnerRefusal, "post-create-registration-refusal": assertPostCreateRegistrationRefusal, "post-create-registration-recovery-readback-failure": assertPostCreateRegistrationRecoveryReadbackFailure, From a09264d90ce5b04c07563750c65ddb2b0367adb9 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 05:40:25 -0700 Subject: [PATCH 32/42] fix(onboard): keep retained recovery records unresolved Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 3 +- src/lib/onboard/cancel-rollback.test.ts | 2 +- src/lib/onboard/cancel-rollback.ts | 2 +- src/lib/onboard/entry-options.ts | 8 +- src/lib/state/onboard-session.ts | 9 --- .../retained-sandbox-recovery.ts | 67 +---------------- .../state/retained-sandbox-recovery.test.ts | 74 ++++++++++++++----- .../onboard-fresh-create-identity.test.ts | 2 +- 8 files changed, 66 insertions(+), 101 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 2d46b8b34bb..27ea50f7c32 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -907,7 +907,8 @@ A credential environment name in the recovery record does not prove that its val Rotate a credential only when identity-bound inspection proves that it was exposed or attached to a retained sandbox-scoped resource. NemoClaw stores the recovery record independently from the active onboarding session. A fresh run with a different name can proceed without clearing that record, but automatic resume, explicit `--resume`, reuse, recreation, and fresh onboarding with the retained name remain blocked. -After an OpenShell administrator verifies the live durable ID, resolves or removes that exact sandbox through an identity-bound procedure, and durably records the resolution, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and use an available sandbox name. +NemoClaw has no supported operation in this release to clear the recovery record, so the retained name remains unavailable even after external recovery or removal. +Preserve the record as evidence, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and use another available sandbox name. `--fresh` starts a new session and does not retain those selections. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index b088c309f25..66d6573d2b4 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -34,7 +34,7 @@ describe("createSandboxCancelRollback", () => { expect(guidance).toContain("Shared inference providers are gateway configuration"); expect(guidance).toContain("not sandbox cleanup targets"); expect(guidance).toContain("sandbox-scoped resources whose ownership is confirmed"); - expect(guidance).toContain("confirm that the exact sandbox is absent"); + expect(guidance).toContain("no supported operation to clear this recovery record"); expect(guidance).toContain("credential environment name alone does not prove exposure"); expect(guidance).toContain("rotate a credential only when identity-bound inspection proves"); expect(guidance).not.toContain("rotate any credential"); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index 77655ff4e06..d41fd39c909 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -63,7 +63,7 @@ export function buildCancelRollbackMessage( " Sandbox-scoped provider registrations or gateway-bound credentials may remain when the durable recovery record lists them.", " Ask an OpenShell administrator to inspect the exact sandbox identity and remove only sandbox-scoped resources whose ownership is confirmed for this retained sandbox.", " A recorded credential environment name alone does not prove exposure; rotate a credential only when identity-bound inspection proves that it was exposed or attached to a retained sandbox-scoped resource.", - " Before reusing the sandbox name, confirm that the exact sandbox is absent and record the identity-bound administrator resolution.", + " NemoClaw has no supported operation to clear this recovery record; use a different sandbox name for later onboarding.", ]; } diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index 4ce3d17c7a0..41fe67f56d2 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -328,7 +328,7 @@ export function resolveOnboardEntryOptions( " Onboarding cannot continue while a retained sandbox recovery record is unresolved without an explicit different sandbox name.", ); deps.error( - " Use --fresh --name , or complete identity-bound administrator recovery for the retained sandbox first.", + " Use --fresh --name ; the retained sandbox recovery record stays unresolved.", ); deps.exitProcess(1); } @@ -337,7 +337,7 @@ export function resolveOnboardEntryOptions( ` Onboarding cannot use retained sandbox '${recoveryEntryName}' while its identity-bound recovery record is unresolved.`, ); deps.error( - " Automatic and explicit resume, reuse, recreation, and same-name fresh onboarding remain disabled until administrator resolution is recorded.", + " Automatic and explicit resume, reuse, recreation, and same-name fresh onboarding remain disabled; NemoClaw has no supported operation to clear this recovery record.", ); deps.exitProcess(1); } @@ -352,7 +352,7 @@ export function resolveOnboardEntryOptions( " Automatic and explicit resume, reuse, and recreation are disabled to protect the retained sandbox.", ); deps.error( - " After an OpenShell administrator completes identity-bound recovery or removal, start fresh with --fresh --name .", + " Use --fresh --name ; the retained sandbox recovery record stays unresolved.", ); deps.exitProcess(1); } @@ -365,7 +365,7 @@ export function resolveOnboardEntryOptions( " Recovery-only onboarding state requires --fresh with an explicit sandbox name different from the retained sandbox.", ); deps.error( - " Verify identity-bound removal with an OpenShell administrator before starting the new onboarding session.", + " The retained sandbox recovery record stays unresolved when onboarding starts with another name.", ); deps.exitProcess(1); } diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 7003722f6ee..0534316c0d3 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -64,11 +64,8 @@ import { nextMachineStateAfterCompletedStep } from "./onboard-step-state"; import { listRetainedSandboxRecoveryRecords as readRetainedSandboxRecoveryRecords, recordRetainedSandboxRecovery as writeRetainedSandboxRecovery, - resolveRetainedSandboxRecovery as writeRetainedSandboxResolution, retainedSandboxRecoveryFile, type RecordRetainedSandboxRecoveryInput, - type ResolveRetainedSandboxRecoveryInput, - type RetainedSandboxAdministratorResolutionReceipt, type RetainedSandboxRecoveryRecord, type RetainedSandboxRecoveryReason, } from "./onboard-session/retained-sandbox-recovery"; @@ -1700,12 +1697,6 @@ export function recordRetainedSandboxRecovery( return writeRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, input); } -export function resolveRetainedSandboxRecovery( - input: ResolveRetainedSandboxRecoveryInput, -): RetainedSandboxAdministratorResolutionReceipt { - return writeRetainedSandboxResolution(RETAINED_SANDBOX_RECOVERY_FILE, input); -} - export function markCancellationRecovery( sandboxName: string, sandboxIdentityFingerprint?: string, diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index bbe3a88728f..33fcb5be1dc 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -41,7 +41,7 @@ export interface RetainedSandboxRecoveryRecord { readonly recordedAt: string; } -export interface RetainedSandboxAdministratorResolutionReceipt { +interface RetainedSandboxAdministratorResolutionReceipt { readonly schemaVersion: typeof SCHEMA_VERSION; readonly receiptId: string; readonly recordId: string; @@ -70,17 +70,6 @@ export interface RecordRetainedSandboxRecoveryInput { readonly recordedAt?: string; } -export interface ResolveRetainedSandboxRecoveryInput { - readonly recordId: string; - readonly receiptId: string; - readonly sandboxName: string; - readonly sandboxIdentityFingerprint: string | null; - readonly gatewayName: string; - readonly gatewayPort: number; - readonly outcome: RetainedSandboxAdministratorResolutionReceipt["outcome"]; - readonly resolvedAt?: string; -} - const emptyState = (): RetainedSandboxRecoveryState => ({ schemaVersion: SCHEMA_VERSION, unresolved: [], @@ -358,57 +347,3 @@ export function recordRetainedSandboxRecovery( } return reread; } - -export function resolveRetainedSandboxRecovery( - filePath: string, - input: ResolveRetainedSandboxRecoveryInput, -): RetainedSandboxAdministratorResolutionReceipt { - const current = loadState(filePath); - const record = current.unresolved.find((candidate) => candidate.recordId === input.recordId); - if ( - !record || - !FINGERPRINT_PATTERN.test(input.receiptId) || - record.sandboxName !== input.sandboxName || - record.gatewayName !== input.gatewayName || - record.gatewayPort !== input.gatewayPort || - record.sandboxIdentityFingerprint !== input.sandboxIdentityFingerprint || - (record.sandboxIdentityFingerprint === null) !== - (input.outcome === "confirmed_absent_without_identity") - ) { - throw new Error("Administrator resolution receipt does not match retained sandbox identity."); - } - const receipt: RetainedSandboxAdministratorResolutionReceipt = { - schemaVersion: SCHEMA_VERSION, - receiptId: input.receiptId, - recordId: record.recordId, - sandboxName: record.sandboxName, - sandboxIdentityFingerprint: record.sandboxIdentityFingerprint, - gatewayName: record.gatewayName, - gatewayPort: record.gatewayPort, - outcome: input.outcome, - resolvedAt: input.resolvedAt ?? new Date().toISOString(), - }; - if (!validTimestamp(receipt.resolvedAt)) { - throw new Error("Administrator resolution receipt has an invalid timestamp."); - } - writeStateFile(filePath, { - ...current, - unresolved: current.unresolved.filter((candidate) => candidate.recordId !== record.recordId), - resolutions: [ - ...current.resolutions.filter((candidate) => candidate.recordId !== record.recordId), - receipt, - ], - }); - const reread = loadState(filePath); - const durableReceipt = reread.resolutions.find( - (candidate) => candidate.receiptId === receipt.receiptId, - ); - if ( - reread.unresolved.some((candidate) => candidate.recordId === record.recordId) || - !durableReceipt || - JSON.stringify(durableReceipt) !== JSON.stringify(receipt) - ) { - throw new Error("Administrator resolution receipt did not survive durable readback."); - } - return durableReceipt; -} diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index d234052074a..da47f282fcb 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -90,8 +90,9 @@ describe("retained sandbox recovery state", () => { expect(fs.readFileSync(externalState, "utf8")).toBe(externalContents); }); - it("clears only after a durable exact-identity administrator receipt", async () => { + it("does not expose a caller-supplied recovery resolution path (#9833)", async () => { const recovery = await import("./onboard-session"); + const recoveryStore = await import("./onboard-session/retained-sandbox-recovery"); const fingerprint = "b".repeat(64); const recorded = recovery.recordRetainedSandboxRecovery({ sandboxName: "retained-sb", @@ -102,21 +103,10 @@ describe("retained sandbox recovery state", () => { resources: evidence, reason: "cancelled_after_sandbox_creation", }); - - expect(() => - recovery.resolveRetainedSandboxRecovery({ - recordId: recorded.recordId, - receiptId: "c".repeat(64), - sandboxName: recorded.sandboxName, - sandboxIdentityFingerprint: "d".repeat(64), - gatewayName: recorded.gatewayName, - gatewayPort: recorded.gatewayPort, - outcome: "removed_verified_identity", - }), - ).toThrow(/does not match retained sandbox identity/u); - expect(recovery.listRetainedSandboxRecoveryRecords()).toHaveLength(1); - - const receipt = recovery.resolveRetainedSandboxRecovery({ + const unsupportedClear = (recovery as unknown as Record)[ + "resolveRetainedSandboxRecovery" + ]; + (unsupportedClear as undefined | ((input: Record) => unknown))?.({ recordId: recorded.recordId, receiptId: "c".repeat(64), sandboxName: recorded.sandboxName, @@ -126,7 +116,55 @@ describe("retained sandbox recovery state", () => { outcome: "removed_verified_identity", }); - expect(receipt.recordId).toBe(recorded.recordId); - expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([]); + expect(unsupportedClear).toBeUndefined(); + expect( + (recoveryStore as unknown as Record)["resolveRetainedSandboxRecovery"], + ).toBeUndefined(); + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([recorded]); + }); + + it("preserves legacy resolution evidence while recording new recovery state (#9833)", async () => { + const recovery = await import("./onboard-session"); + const recorded = recovery.recordRetainedSandboxRecovery({ + sandboxName: "legacy-sb", + sandboxIdentityFingerprint: "d".repeat(64), + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "legacy-generation", + resources: evidence, + reason: "cancelled_after_sandbox_creation", + }); + const legacyResolution = { + schemaVersion: 1, + receiptId: "e".repeat(64), + recordId: recorded.recordId, + sandboxName: recorded.sandboxName, + sandboxIdentityFingerprint: recorded.sandboxIdentityFingerprint, + gatewayName: recorded.gatewayName, + gatewayPort: recorded.gatewayPort, + outcome: "removed_verified_identity", + resolvedAt: "2026-08-27T00:00:00.000Z", + }; + const legacyState = JSON.parse( + fs.readFileSync(recovery.RETAINED_SANDBOX_RECOVERY_FILE, "utf8"), + ); + legacyState.unresolved = []; + legacyState.resolutions = [legacyResolution]; + fs.writeFileSync(recovery.RETAINED_SANDBOX_RECOVERY_FILE, JSON.stringify(legacyState)); + + recovery.recordRetainedSandboxRecovery({ + sandboxName: "new-sb", + sandboxIdentityFingerprint: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: null, + resources: evidence, + reason: "retained_after_sandbox_creation_failure", + }); + + const durableState = JSON.parse( + fs.readFileSync(recovery.RETAINED_SANDBOX_RECOVERY_FILE, "utf8"), + ); + expect(durableState.resolutions).toEqual([legacyResolution]); }); }); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index d5737382465..ce4f093ed2a 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -885,7 +885,7 @@ if (${JSON.stringify( assert.match(result.stderr, /Shared inference providers are gateway configuration/u); assert.match(result.stderr, /not sandbox cleanup targets/u); assert.match(result.stderr, /sandbox-scoped resources whose ownership is confirmed/u); - assert.match(result.stderr, /confirm that the exact sandbox is absent/u); + assert.match(result.stderr, /no supported operation to clear this recovery record/u); assert.match(result.stderr, /credential environment name alone does not prove exposure/u); assert.match( result.stderr, From 06e60709060a5c245be8cf771b951abd764975a2 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 06:20:45 -0700 Subject: [PATCH 33/42] fix(onboard): bind retained recovery evidence Signed-off-by: Apurv Kumaria --- src/lib/onboard/cancel-rollback.test.ts | 29 ++++++-- src/lib/onboard/cancel-rollback.ts | 28 ++++++-- .../sandbox-create/orchestration.test.ts | 71 +++++++++++++++++++ .../onboard/sandbox-create/orchestration.ts | 62 +++++++++++++--- src/lib/state/onboard-session.ts | 3 + .../retained-sandbox-recovery.ts | 40 +++++++++-- .../state/retained-sandbox-recovery.test.ts | 39 +++++++++- .../onboard-fresh-create-identity.test.ts | 70 ++++++++++-------- 8 files changed, 285 insertions(+), 57 deletions(-) diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 66d6573d2b4..0a11d9ebba2 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -131,10 +131,31 @@ describe("installSandboxCancelRollback", () => { rollback.markCancelled(); exitHandlers[0](); - expect(recordRecovery).toHaveBeenCalledWith("new-sb", SANDBOX_FINGERPRINT); + expect(recordRecovery).toHaveBeenCalledWith("new-sb", SANDBOX_FINGERPRINT, undefined); expect(log.mock.calls.flat().join("\n")).toContain(SANDBOX_FINGERPRINT); }); + it("forwards the full verified tuple from cancellation to durable state (#9833)", () => { + const recordRecovery = vi.fn(); + const recoveryContext = { + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000004", + verifiedEffectivePolicyIdentity: { hash: "sha256:policy-4", activeVersion: 4 }, + } as const; + const rollback = createSandboxCancelRollback({ log: vi.fn(), recordRecovery }); + const armWithContext = rollback.arm as ( + sandboxName: string, + sandboxIdentityFingerprint: string, + context: typeof recoveryContext, + ) => void; + + armWithContext("new-sb", SANDBOX_FINGERPRINT, recoveryContext); + rollback.markCancelled(); + + expect(recordRecovery).toHaveBeenCalledWith("new-sb", SANDBOX_FINGERPRINT, recoveryContext); + }); + it("persists recovery before a deferred process exit and records it once (#9833)", () => { const log = vi.fn(); const recordRecovery = vi.fn(); @@ -152,7 +173,7 @@ describe("installSandboxCancelRollback", () => { rollback.arm("new-sb", SANDBOX_FINGERPRINT); expect(() => cancel()).toThrow(deferredExit); expect(recordRecovery).toHaveBeenCalledOnce(); - expect(recordRecovery).toHaveBeenCalledWith("new-sb", SANDBOX_FINGERPRINT); + expect(recordRecovery).toHaveBeenCalledWith("new-sb", SANDBOX_FINGERPRINT, undefined); exitHandlers[0](); expect(recordRecovery).toHaveBeenCalledOnce(); @@ -180,7 +201,7 @@ describe("installSandboxCancelRollback", () => { exitHandlers[0](); expect(recordRecovery).toHaveBeenCalledTimes(2); - expect(recordRecovery).toHaveBeenLastCalledWith("new-sb", SANDBOX_FINGERPRINT); + expect(recordRecovery).toHaveBeenLastCalledWith("new-sb", SANDBOX_FINGERPRINT, undefined); const guidance = log.mock.calls.flat().join("\n"); expect(guidance).toContain(SANDBOX_FINGERPRINT); expect(guidance).not.toContain("could not save the onboarding recovery record"); @@ -234,7 +255,7 @@ describe("installSandboxCancelRollback", () => { exitHandlers[0](); expect(recordRecovery).toHaveBeenCalledTimes(3); - expect(recordRecovery).toHaveBeenNthCalledWith(3, "new-sb", SANDBOX_FINGERPRINT); + expect(recordRecovery).toHaveBeenNthCalledWith(3, "new-sb", SANDBOX_FINGERPRINT, undefined); const guidanceCalls = log.mock.calls.filter(([message]) => String(message).includes("preserved incomplete sandbox"), ); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index d41fd39c909..4decc3ed335 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { RetainedSandboxRecoveryContext } from "../state/onboard-session"; + // Re-exported so the onboard entrypoint imports its sandbox default/cancel // lifecycle helpers from a single module. export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preservation"; @@ -25,12 +27,20 @@ export interface SandboxCancelRollbackDeps { /** Emit an operator-facing line (stderr). */ log(message: string): void; /** Persist the recovery-only session before process exit completes. */ - recordRecovery?(sandboxName: string, sandboxIdentityFingerprint?: string): void; + recordRecovery?( + sandboxName: string, + sandboxIdentityFingerprint?: string, + context?: RetainedSandboxRecoveryContext, + ): void; } export interface SandboxCancelRollback { /** Arm cancellation recovery guidance for a just-created sandbox. */ - arm(sandboxName: string, sandboxIdentityFingerprint?: string): void; + arm( + sandboxName: string, + sandboxIdentityFingerprint?: string, + context?: RetainedSandboxRecoveryContext, + ): void; /** Disarm once the sandbox is past the cancellable policy-selection window. */ disarm(): void; /** Record that the operator cancelled at a cancellable step. */ @@ -121,6 +131,7 @@ export function createSandboxCancelRollback( let armedSandbox: { readonly name: string; readonly identityFingerprint: string | null; + readonly context: RetainedSandboxRecoveryContext | undefined; } | null = null; let cancelRequested = false; let recoveryRecorded = false; @@ -132,7 +143,11 @@ export function createSandboxCancelRollback( if (recoveryRecorded) return true; if (armedSandbox === null) return false; try { - deps.recordRecovery?.(armedSandbox.name, armedSandbox.identityFingerprint ?? undefined); + deps.recordRecovery?.( + armedSandbox.name, + armedSandbox.identityFingerprint ?? undefined, + armedSandbox.context, + ); recoveryRecorded = true; recoveryPersistenceFailed = false; return true; @@ -143,7 +158,11 @@ export function createSandboxCancelRollback( }; return { - arm(sandboxName: string, sandboxIdentityFingerprint?: string): void { + arm( + sandboxName: string, + sandboxIdentityFingerprint?: string, + context?: RetainedSandboxRecoveryContext, + ): void { armedSandbox = { name: sandboxName, identityFingerprint: @@ -151,6 +170,7 @@ export function createSandboxCancelRollback( /^[0-9a-f]{64}$/u.test(sandboxIdentityFingerprint) ? sandboxIdentityFingerprint : null, + context, }; }, disarm(): void { diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 55e03296086..9a959f2e57c 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -27,6 +27,13 @@ import { runWithPostCreateRecovery, } from "./orchestration"; +const UNVERIFIED_RECOVERY_CONTEXT = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, +} as const; + describe("retained create recovery persistence", () => { it.each([ ["available fingerprint", "f".repeat(64)], @@ -55,6 +62,7 @@ describe("retained create recovery persistence", () => { sandboxName: "alpha", message, ...(fingerprint ? { sandboxIdentityFingerprint: fingerprint } : {}), + recoveryContext: UNVERIFIED_RECOVERY_CONTEXT, }, session.markRetainedSandboxRecovery, ), @@ -84,6 +92,34 @@ describe("retained create recovery persistence", () => { }, ); + it("forwards the full verified recovery tuple to durable state (#9833)", () => { + const recoveryContext = { + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000004", + verifiedEffectivePolicyIdentity: { hash: "sha256:policy-4", activeVersion: 4 }, + } as const; + const markRetainedSandboxRecovery = vi.fn(() => true); + const input = { + stage: "registry publication" as const, + sandboxName: "alpha", + gatewayName: recoveryContext.gatewayName, + lifecycleGeneration: recoveryContext.lifecycleGeneration, + exactIdentity: "f".repeat(64), + recoveryContext, + markRetainedSandboxRecovery, + }; + + persistPostCreateRecovery(input); + + expect(markRetainedSandboxRecovery).toHaveBeenCalledWith( + "alpha", + expect.stringContaining(recoveryContext.lifecycleGeneration), + "f".repeat(64), + recoveryContext, + ); + }); + it("reports persistence failure when no onboard session owns the recovery (#9211)", () => { const finalizeIncompleteOnboardStep = vi.fn(() => null); @@ -92,6 +128,7 @@ describe("retained create recovery persistence", () => { { sandboxName: "alpha", message: "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=authority", + recoveryContext: UNVERIFIED_RECOVERY_CONTEXT, }, finalizeIncompleteOnboardStep, ), @@ -100,6 +137,7 @@ describe("retained create recovery persistence", () => { "alpha", "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=authority", undefined, + UNVERIFIED_RECOVERY_CONTEXT, ); }); @@ -118,6 +156,7 @@ describe("retained create recovery persistence", () => { { sandboxName: "alpha", message: "Create-attempt label: ai.nvidia.nemoclaw.create-attempt=unpersisted", + recoveryContext: UNVERIFIED_RECOVERY_CONTEXT, }, session.markRetainedSandboxRecovery, ), @@ -162,6 +201,7 @@ describe("retained create recovery persistence", () => { gatewayName: "nemoclaw", lifecycleGeneration: "generation-1", exactIdentity: "f".repeat(64), + recoveryContext: UNVERIFIED_RECOVERY_CONTEXT, markRetainedSandboxRecovery, }); @@ -719,6 +759,36 @@ describe("sandbox create policy authority checks", () => { expect(persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringContaining("left sandbox 'alpha' in place"), exactIdentity, + "verified", + ); + }); + + it("retains verified policy evidence when checkpoint persistence fails (#9833)", async () => { + const verifiedEvidence = { policyHash: "sha256:policy-4", policyVersion: 4 } as const; + const persistRetainedSandboxRecovery = vi.fn(() => true); + + await expect( + runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + ...exactIdentityBoundary(), + verifyCreatedPolicy: () => verifiedEvidence, + persistVerifiedPolicy: () => { + throw new Error("checkpoint write failed"); + }, + persistRetainedSandboxRecovery, + cleanupTemporarySources: vi.fn(), + }), + ).rejects.toThrow("automatic sandbox cleanup was not safe"); + + expect(persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("left sandbox 'alpha' in place"), + exactIdentity, + verifiedEvidence, ); }); @@ -743,6 +813,7 @@ describe("sandbox create policy authority checks", () => { expect(persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( expect.stringContaining("left sandbox 'alpha' in place"), exactIdentity, + "verified", ); }); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 416638f0862..506ea99a0c6 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -10,6 +10,7 @@ import { HERMES_PORTABLE_OPENSHELL_VERSION } from "../../adapters/openshell/reso import type { AgentDefinition } from "../../agent/defs"; import type { WebSearchConfig } from "../../inference/web-search"; import type { BackupResult } from "../../state/sandbox"; +import type { RetainedSandboxRecoveryContext } from "../../state/onboard-session"; import type { SandboxEntry } from "../../state/registry"; import type { PendingSandboxPolicyVerification, @@ -66,15 +67,22 @@ export function persistRetainedSandboxRecoveryMessage( readonly sandboxName: string; readonly message: string; readonly sandboxIdentityFingerprint?: string; + readonly recoveryContext: RetainedSandboxRecoveryContext; }, markRetainedSandboxRecovery: ( sandboxName: string, message: string, sandboxIdentityFingerprint?: string, + context?: RetainedSandboxRecoveryContext, ) => unknown | null, ): boolean { return Boolean( - markRetainedSandboxRecovery(input.sandboxName, input.message, input.sandboxIdentityFingerprint), + markRetainedSandboxRecovery( + input.sandboxName, + input.message, + input.sandboxIdentityFingerprint, + input.recoveryContext, + ), ); } @@ -151,10 +159,12 @@ export function persistPostCreateRecovery(input: { readonly gatewayName: string; readonly lifecycleGeneration: string; readonly exactIdentity?: string; + readonly recoveryContext: RetainedSandboxRecoveryContext; readonly markRetainedSandboxRecovery: ( sandboxName: string, message: string, sandboxIdentityFingerprint?: string, + context?: RetainedSandboxRecoveryContext, ) => unknown | null; }): void { const message = @@ -168,6 +178,7 @@ export function persistPostCreateRecovery(input: { sandboxName: input.sandboxName, message, ...(input.exactIdentity ? { sandboxIdentityFingerprint: input.exactIdentity } : {}), + recoveryContext: input.recoveryContext, }, input.markRetainedSandboxRecovery, ); @@ -471,12 +482,14 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< readonly persistRetainedSandboxRecovery?: ( message: string, exactIdentity: string | null, + evidence: Evidence | null, ) => boolean; readonly retainedSandboxRecoveryRetryOwner?: PostCreateRecoveryRetryOwner; readonly cleanupTemporarySources: () => void; }): Promise { input.revalidate(false, `creating sandbox '${input.sandboxName}'`); let exactIdentity: string | null = null; + let observedPolicyEvidence: Evidence | null = null; let cleanupAttempted = false; let recoveryAttempted = false; const cleanupTemporarySources = (): unknown[] => { @@ -506,7 +519,11 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< if (input.persistRetainedSandboxRecovery) { try { persistRetainedSandboxRecoveryWithRetry(input.retainedSandboxRecoveryRetryOwner, () => - input.persistRetainedSandboxRecovery!(recoveryGuidance, exactIdentity), + input.persistRetainedSandboxRecovery!( + recoveryGuidance, + exactIdentity, + observedPolicyEvidence, + ), ); } catch (error) { compensationErrors.push(error); @@ -533,6 +550,7 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< `verifying effective policy for sandbox '${input.sandboxName}'`, ); const evidence = input.verifyCreatedPolicy(created, capturedIdentity); + observedPolicyEvidence = evidence; input.revalidateCreatedSandboxIdentity( capturedIdentity, `recording verified policy for sandbox '${input.sandboxName}'`, @@ -2012,21 +2030,35 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche pendingSandboxPolicyVerificationForBoundary(boundary), ); }; + const retainedSandboxRecoveryContext = ( + boundary: VerifiedSandboxPolicyBoundary | null, + ): RetainedSandboxRecoveryContext => { + const checkpoint = boundary ? pendingSandboxPolicyVerificationForBoundary(boundary) : null; + return { + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + lifecycleGeneration: createdSandboxLifecycle.generation, + verifiedEffectivePolicyIdentity: checkpoint + ? { hash: checkpoint.policyHash, activeVersion: checkpoint.policyVersion } + : null, + }; + }; const recordPostCreateRecovery = ( stage: "registry publication" | "onboarding finalization", - ): void => + ): void => { + const boundary = requireVerifiedPolicyGate(); postCreateRecoveryRetryOwner.record(() => persistPostCreateRecovery({ stage, sandboxName, gatewayName: GATEWAY_NAME, lifecycleGeneration: createdSandboxLifecycle.generation, - ...(verifiedPolicyGate - ? { exactIdentity: verifiedPolicyGate.lifecycleLiveIdentityFingerprint } - : {}), + exactIdentity: boundary.lifecycleLiveIdentityFingerprint, + recoveryContext: retainedSandboxRecoveryContext(boundary), markRetainedSandboxRecovery: onboardSession.markRetainedSandboxRecovery, }), ); + }; const persistCreateFlowRecovery = ( message: string, exactIdentity: string | null = null, @@ -2037,6 +2069,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche sandboxName, message, ...(exactIdentity ? { sandboxIdentityFingerprint: exactIdentity } : {}), + recoveryContext: retainedSandboxRecoveryContext(verifiedPolicyGate), }, onboardSession.markRetainedSandboxRecovery, ), @@ -2106,12 +2139,13 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche revalidateVerifiedPolicy: (_identity, _exactIdentity, boundary, operation) => { revalidateVerifiedPolicyRegistration(boundary, operation); }, - persistRetainedSandboxRecovery: (message, exactIdentity) => + persistRetainedSandboxRecovery: (message, exactIdentity, boundary) => persistRetainedSandboxRecoveryMessage( { sandboxName, message, ...(exactIdentity ? { sandboxIdentityFingerprint: exactIdentity } : {}), + recoveryContext: retainedSandboxRecoveryContext(boundary), }, onboardSession.markRetainedSandboxRecovery, ), @@ -2416,8 +2450,18 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche scriptsDir: SCRIPTS, gatewayName: GATEWAY_NAME, providerExistsInGateway, - armCancelRollback: sandboxCancelRollback.arm, - markCancellationRecovery: onboardSession.markCancellationRecovery, + armCancelRollback: (name, identity) => + sandboxCancelRollback.arm( + name, + identity, + retainedSandboxRecoveryContext(requireVerifiedPolicyGate()), + ), + markCancellationRecovery: (name) => + onboardSession.markCancellationRecovery( + name, + undefined, + retainedSandboxRecoveryContext(requireVerifiedPolicyGate()), + ), dockerInfoFormat, runCapture, revalidatePolicyAuthority: (operation) => revalidatePolicyAuthority(true, operation), diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 0534316c0d3..90216f7ddbe 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -68,6 +68,7 @@ import { type RecordRetainedSandboxRecoveryInput, type RetainedSandboxRecoveryRecord, type RetainedSandboxRecoveryReason, + type RetainedSandboxVerifiedEffectivePolicyIdentity, } from "./onboard-session/retained-sandbox-recovery"; import type { SandboxHostMount } from "./registry/types"; import { hasUnsafeHostMountTerminalText } from "./registry/host-mount"; @@ -1655,6 +1656,7 @@ export interface RetainedSandboxRecoveryContext { readonly gatewayName?: string; readonly gatewayPort?: number; readonly lifecycleGeneration?: string | null; + readonly verifiedEffectivePolicyIdentity?: RetainedSandboxVerifiedEffectivePolicyIdentity | null; } function persistIndependentRetainedSandboxRecovery( @@ -1678,6 +1680,7 @@ function persistIndependentRetainedSandboxRecovery( gatewayName: context.gatewayName ?? session.metadata.gatewayName, gatewayPort: context.gatewayPort ?? GATEWAY_PORT, lifecycleGeneration: context.lifecycleGeneration ?? null, + verifiedEffectivePolicyIdentity: context.verifiedEffectivePolicyIdentity ?? null, resources: { sharedInferenceProviders: session.provider ? [session.provider] : [], sandboxScopedProviders: session.stagedCredentialProviders, diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index 33fcb5be1dc..3e037c2cef5 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -27,6 +27,11 @@ export interface RetainedSandboxResourceEvidence { readonly credentialEnvironmentVariables: readonly string[]; } +export interface RetainedSandboxVerifiedEffectivePolicyIdentity { + readonly hash: string; + readonly activeVersion: number; +} + export interface RetainedSandboxRecoveryRecord { readonly schemaVersion: typeof SCHEMA_VERSION; readonly recordId: string; @@ -36,6 +41,7 @@ export interface RetainedSandboxRecoveryRecord { readonly gatewayName: string; readonly gatewayPort: number; readonly lifecycleGeneration: string | null; + readonly verifiedEffectivePolicyIdentity: RetainedSandboxVerifiedEffectivePolicyIdentity | null; readonly resources: RetainedSandboxResourceEvidence; readonly reason: RetainedSandboxRecoveryReason; readonly recordedAt: string; @@ -65,6 +71,7 @@ export interface RecordRetainedSandboxRecoveryInput { readonly gatewayName: string; readonly gatewayPort: number; readonly lifecycleGeneration: string | null; + readonly verifiedEffectivePolicyIdentity: RetainedSandboxVerifiedEffectivePolicyIdentity | null; readonly resources: RetainedSandboxResourceEvidence; readonly reason: RetainedSandboxRecoveryReason; readonly recordedAt?: string; @@ -182,10 +189,28 @@ function parseEvidence(value: unknown): RetainedSandboxResourceEvidence | null { : null; } +function parseVerifiedEffectivePolicyIdentity( + value: unknown, +): RetainedSandboxVerifiedEffectivePolicyIdentity | null | undefined { + if (value === null || value === undefined) return null; + if ( + !isObjectRecord(value) || + !validSafeEvidence(value.hash) || + !Number.isSafeInteger(value.activeVersion) || + Number(value.activeVersion) < 1 + ) { + return undefined; + } + return { hash: value.hash, activeVersion: Number(value.activeVersion) }; +} + function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { if (!isObjectRecord(value)) return null; const resources = parseEvidence(value.resources); const fingerprint = value.sandboxIdentityFingerprint; + const verifiedEffectivePolicyIdentity = parseVerifiedEffectivePolicyIdentity( + value.verifiedEffectivePolicyIdentity, + ); const reason = value.reason; if ( value.schemaVersion !== SCHEMA_VERSION || @@ -198,6 +223,7 @@ function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { !validSafeEvidence(value.gatewayName) || !validGatewayPort(value.gatewayPort) || (value.lifecycleGeneration !== null && !validSafeEvidence(value.lifecycleGeneration)) || + verifiedEffectivePolicyIdentity === undefined || !resources || !["cancelled_after_sandbox_creation", "retained_after_sandbox_creation_failure"].includes( String(reason), @@ -215,6 +241,7 @@ function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { gatewayName: value.gatewayName, gatewayPort: value.gatewayPort, lifecycleGeneration: value.lifecycleGeneration, + verifiedEffectivePolicyIdentity, resources, reason: reason as RetainedSandboxRecoveryReason, recordedAt: value.recordedAt, @@ -279,6 +306,8 @@ function recoveryRecordId(input: RecordRetainedSandboxRecoveryInput): string { input.gatewayPort, input.sandboxName, input.sandboxIdentityFingerprint, + input.lifecycleGeneration, + input.verifiedEffectivePolicyIdentity, ]), ) .digest("hex"); @@ -292,6 +321,7 @@ function assertRecordInput(input: RecordRetainedSandboxRecoveryInput): void { !validSafeEvidence(input.gatewayName) || !validGatewayPort(input.gatewayPort) || (input.lifecycleGeneration !== null && !validSafeEvidence(input.lifecycleGeneration)) || + parseVerifiedEffectivePolicyIdentity(input.verifiedEffectivePolicyIdentity) === undefined || !parseEvidence(input.resources) ) { throw new Error("Cannot persist invalid retained sandbox recovery evidence."); @@ -318,6 +348,9 @@ export function recordRetainedSandboxRecovery( gatewayName: input.gatewayName, gatewayPort: input.gatewayPort, lifecycleGeneration: input.lifecycleGeneration, + verifiedEffectivePolicyIdentity: input.verifiedEffectivePolicyIdentity + ? { ...input.verifiedEffectivePolicyIdentity } + : null, resources: parseEvidence(input.resources)!, reason: input.reason, recordedAt: input.recordedAt ?? new Date().toISOString(), @@ -329,12 +362,7 @@ export function recordRetainedSandboxRecovery( const next: RetainedSandboxRecoveryState = { ...current, unresolved: [ - ...current.unresolved.filter( - (candidate) => - candidate.gatewayName !== record.gatewayName || - candidate.gatewayPort !== record.gatewayPort || - candidate.sandboxName !== record.sandboxName, - ), + ...current.unresolved.filter((candidate) => candidate.recordId !== record.recordId), record, ], }; diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index da47f282fcb..92cbfa76cea 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -30,23 +30,26 @@ describe("retained sandbox recovery state", () => { it("persists verified identity and secret-free resource evidence independently", async () => { const recovery = await import("./onboard-session"); const fingerprint = "a".repeat(64); - - const recorded = recovery.recordRetainedSandboxRecovery({ + const input = { sandboxName: "retained-sb", sandboxIdentityFingerprint: fingerprint, gatewayName: "nemoclaw", gatewayPort: 8080, lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + verifiedEffectivePolicyIdentity: { hash: "sha256:policy-1", activeVersion: 1 }, resources: evidence, reason: "cancelled_after_sandbox_creation", recordedAt: "2026-08-27T00:00:00.000Z", - }); + } as const; + + const recorded = recovery.recordRetainedSandboxRecovery(input); expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([recorded]); expect(recorded).toMatchObject({ sandboxName: "retained-sb", sandboxIdentityFingerprint: fingerprint, identityWasUnavailable: false, + verifiedEffectivePolicyIdentity: input.verifiedEffectivePolicyIdentity, resources: evidence, }); expect(fs.readFileSync(recovery.RETAINED_SANDBOX_RECOVERY_FILE, "utf8")).not.toContain( @@ -63,6 +66,7 @@ describe("retained sandbox recovery state", () => { gatewayName: "nemoclaw", gatewayPort: 8080, lifecycleGeneration: null, + verifiedEffectivePolicyIdentity: null, resources: { sharedInferenceProviders: [], sandboxScopedProviders: [], @@ -78,6 +82,32 @@ describe("retained sandbox recovery state", () => { }); }); + it("preserves distinct unresolved lifecycle tuples for one sandbox name (#9833)", async () => { + const recovery = await import("./onboard-session"); + const first = recovery.recordRetainedSandboxRecovery({ + sandboxName: "retained-sb", + sandboxIdentityFingerprint: "1".repeat(64), + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + verifiedEffectivePolicyIdentity: { hash: "sha256:policy-1", activeVersion: 1 }, + resources: evidence, + reason: "cancelled_after_sandbox_creation", + }); + const second = recovery.recordRetainedSandboxRecovery({ + sandboxName: "retained-sb", + sandboxIdentityFingerprint: "2".repeat(64), + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000002", + verifiedEffectivePolicyIdentity: { hash: "sha256:policy-2", activeVersion: 2 }, + resources: evidence, + reason: "retained_after_sandbox_creation_failure", + }); + + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([first, second]); + }); + it("refuses a symbolic-link recovery state without reading its target", async () => { const recovery = await import("./onboard-session"); const externalState = path.join(home, "external-recovery.json"); @@ -100,6 +130,7 @@ describe("retained sandbox recovery state", () => { gatewayName: "nemoclaw", gatewayPort: 8080, lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, resources: evidence, reason: "cancelled_after_sandbox_creation", }); @@ -131,6 +162,7 @@ describe("retained sandbox recovery state", () => { gatewayName: "nemoclaw", gatewayPort: 8080, lifecycleGeneration: "legacy-generation", + verifiedEffectivePolicyIdentity: null, resources: evidence, reason: "cancelled_after_sandbox_creation", }); @@ -158,6 +190,7 @@ describe("retained sandbox recovery state", () => { gatewayName: "nemoclaw", gatewayPort: 8080, lifecycleGeneration: null, + verifiedEffectivePolicyIdentity: null, resources: evidence, reason: "retained_after_sandbox_creation_failure", }); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index ce4f093ed2a..318c1ece693 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -254,7 +254,7 @@ runner.run = (command, opts = {}) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:18080"; if (cmd.includes("policy get") && cmd.includes("--output json")) { if (postCreateFinalizationRefusal && registeredSandbox) { throw new Error("final onboarding policy check failed"); @@ -294,6 +294,7 @@ runner.run = (command, opts = {}) => { let checkpointReadCalls = 0; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", + gatewayName: "nemoclaw-18080", provider, model, apfInterceptorRequested, @@ -403,7 +404,7 @@ if (cancelAfterCreate && !recoveryReentry) { const session = onboardModule.onboardSession.createSession({ mode: "interactive", sandboxName: "my-assistant", - metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, + metadata: { gatewayName: "nemoclaw-18080", fromDockerfile: null }, }); onboardModule.onboardSession.saveSession(session); onboardModule.registerIncompleteOnboardExitHandlerForSession( @@ -433,6 +434,7 @@ const writePayload = (sandboxName, creationError, exitCode = 0) => { checkpointReadCalls, registryMutationCalls, currentRegistryEntry: cancelAfterCreate ? registry.getSandbox("my-assistant") : null, + recoveryRegistryEntry: registry.getSandbox("my-assistant"), savedSession: cancelAfterCreate || postCreateAuthorityRefusal || @@ -455,7 +457,7 @@ if (${JSON.stringify( } (async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; + process.env.OPENSHELL_GATEWAY = "nemoclaw-18080"; if (recoveryReentry) { if (recoveryReentry === "fresh-different-no-journal") { try { @@ -491,7 +493,7 @@ if (${JSON.stringify( onboardModule.onboardSession.createSession({ mode: resolved.nonInteractive ? "non-interactive" : "interactive", sandboxName: resolved.requestedSandboxName, - metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, + metadata: { gatewayName: "nemoclaw-18080", fromDockerfile: null }, }), ); writePayload("replacement-sb", null, 0); @@ -572,6 +574,7 @@ if (${JSON.stringify( HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: expectedOutcome.startsWith("cancel-after-create-") ? "" : "1", + NEMOCLAW_GATEWAY_PORT: "18080", OPENSHELL_DRIVERS: "docker", NEMOCLAW_MESSAGING_PLAN_B64: expectedOutcome === "staged-messaging-refusal" @@ -601,6 +604,17 @@ if (${JSON.stringify( command, ), ); + const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); + const assertRecoveryTuple = (record: Record) => { + assert.equal(record.gatewayName, "nemoclaw-18080"); + assert.equal(record.gatewayPort, 18080); + assert.equal(record.sandboxIdentityFingerprint, identityFingerprint); + assert.equal(record.lifecycleGeneration, payload.recoveryRegistryEntry.lifecycleGeneration); + assert.deepEqual(record.verifiedEffectivePolicyIdentity, { + hash: "fixture-policy", + activeVersion: 1, + }); + }; const assertProviderBackedApfRefusal = () => { assert.match( payload.creationError, @@ -658,7 +672,7 @@ if (${JSON.stringify( /--label ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62}/u, ); const ownerScopedObservations = payload.lifecycleObservationCommands.filter( - (command: string) => command.includes("-g nemoclaw"), + (command: string) => command.includes("-g nemoclaw-18080"), ); assert.ok( ownerScopedObservations.length >= 6, @@ -667,8 +681,8 @@ if (${JSON.stringify( assert.ok( ownerScopedObservations.every( (command: string) => - command.includes("sandbox get -g nemoclaw my-assistant") || - command.includes("sandbox list -g nemoclaw"), + command.includes("sandbox get -g nemoclaw-18080 my-assistant") || + command.includes("sandbox list -g nemoclaw-18080"), ), `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, ); @@ -690,7 +704,6 @@ if (${JSON.stringify( assert.deepEqual(providerExposureCommands, []); }; const assertPostCreateAuthorityRefusal = () => { - const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.sandboxName, null); assert.equal(payload.sandboxCreated, true); assert.equal(payload.deleted, false); @@ -716,18 +729,17 @@ if (${JSON.stringify( identityFingerprint, ); assert.equal(payload.retainedRecoveryRecords.length, 1); - assert.deepEqual(payload.retainedRecoveryRecords[0], { - ...payload.retainedRecoveryRecords[0], - sandboxName: "my-assistant", - sandboxIdentityFingerprint: identityFingerprint, - identityWasUnavailable: false, - gatewayName: "nemoclaw", - gatewayPort: 8080, - reason: "retained_after_sandbox_creation_failure", - }); + const record = payload.retainedRecoveryRecords[0]; + assert.equal(record.sandboxName, "my-assistant"); + assert.equal(record.sandboxIdentityFingerprint, identityFingerprint); + assert.equal(record.identityWasUnavailable, false); + assert.equal(record.gatewayName, "nemoclaw-18080"); + assert.equal(record.gatewayPort, 18080); + assert.match(record.lifecycleGeneration, /^[0-9a-f-]{36}$/u); + assert.equal(record.verifiedEffectivePolicyIdentity, null); + assert.equal(record.reason, "retained_after_sandbox_creation_failure"); }; const assertPostCreateRunnerRefusal = () => { - const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.sandboxName, null); assert.equal(payload.sandboxCreated, true); assert.equal(payload.deleted, false); @@ -741,6 +753,7 @@ if (${JSON.stringify( ); assert.equal(payload.retainedRecoveryRecords.length, 1); assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); + assertRecoveryTuple(payload.retainedRecoveryRecords[0]); assert.ok(payload.checkpointReadCalls >= 6); assert.equal( payload.commandNames.filter((command: string) => command.includes("sandbox create")) @@ -749,7 +762,6 @@ if (${JSON.stringify( ); }; const assertPostCreateRegistrationRefusal = () => { - const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.sandboxName, null); assert.equal(payload.sandboxCreated, true); assert.equal(payload.deleted, false); @@ -762,18 +774,13 @@ if (${JSON.stringify( identityFingerprint, ); assert.equal(payload.retainedRecoveryRecords.length, 1); - assert.deepEqual(payload.retainedRecoveryRecords[0], { - ...payload.retainedRecoveryRecords[0], - sandboxName: "my-assistant", - sandboxIdentityFingerprint: identityFingerprint, - identityWasUnavailable: false, - gatewayName: "nemoclaw", - gatewayPort: 8080, - reason: "retained_after_sandbox_creation_failure", - }); + const record = payload.retainedRecoveryRecords[0]; + assert.equal(record.sandboxName, "my-assistant"); + assert.equal(record.identityWasUnavailable, false); + assert.equal(record.reason, "retained_after_sandbox_creation_failure"); + assertRecoveryTuple(record); }; const assertPostCreateRegistrationRecoveryReadbackFailure = () => { - const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.sandboxName, null); assert.equal(payload.sandboxCreated, true); assert.equal(payload.deleted, false); @@ -829,6 +836,7 @@ if (${JSON.stringify( assert.equal(payload.savedSession.resumable, false); assert.equal(payload.retainedRecoveryRecords.length, 1); assert.equal(payload.retainedRecoveryRecords[0].sandboxName, "my-assistant"); + assertRecoveryTuple(payload.retainedRecoveryRecords[0]); assert.equal( payload.commandNames.filter((command: string) => command.includes("sandbox create")) .length, @@ -836,7 +844,6 @@ if (${JSON.stringify( ); }; const assertPostCreateFinalizationRefusal = () => { - const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.sandboxName, null); assert.equal(payload.sandboxCreated, true); assert.equal(payload.deleted, false); @@ -851,9 +858,9 @@ if (${JSON.stringify( payload.savedSession.cancellationRecovery.sandboxIdentityFingerprint, identityFingerprint, ); + assertRecoveryTuple(payload.retainedRecoveryRecords[0]); }; const assertCancellationRecovery = () => { - const identityFingerprint = createHash("sha256").update("sbx-fresh-create").digest("hex"); assert.equal(payload.exitCode, 1); assert.equal(payload.sandboxName, "my-assistant"); assert.equal(payload.deleted, false); @@ -879,6 +886,7 @@ if (${JSON.stringify( payload.commandNames.some((command: string) => command.includes("sandbox delete")), false, ); + assertRecoveryTuple(payload.retainedRecoveryRecords[0]); assert.match(result.stderr, /preserved incomplete sandbox 'my-assistant'/u); assert.match(result.stderr, new RegExp(identityFingerprint, "u")); assert.match(result.stderr, /Do not delete the sandbox by mutable sandbox name/u); From 9275324bddb612403270a7424a7135d107bc085a Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 06:59:33 -0700 Subject: [PATCH 34/42] fix(onboard): serialize retained recovery state Signed-off-by: Apurv Kumaria --- src/lib/onboard.ts | 11 +- src/lib/onboard/entry-options.ts | 20 ++ .../onboard/portable-retirement-authority.ts | 8 +- ...onboard-session-cross-process-lock.test.ts | 94 ++++++++ src/lib/state/onboard-session.ts | 228 ++++++++++++------ .../retained-sandbox-recovery.ts | 190 +++++++++++++-- .../state/retained-sandbox-recovery.test.ts | 37 +++ .../onboard-fsm-live-slices.test.ts | 44 ++++ 8 files changed, 534 insertions(+), 98 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 2df60c25a74..5d2ab3ddfb5 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2750,13 +2750,9 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { ); setOnboardBrandingAgent(opts.agent || process.env.NEMOCLAW_AGENT || null); AUTO_YES = opts.autoYes === true || process.env.NEMOCLAW_YES === "1"; - const entryOptions = onboardEntryOptions.resolveDefaultRunEntryOptions( - opts, - onboardSession.loadSession(), - validateName, - process.env, - onboardSession.listRetainedSandboxRecoveryRecords().map((record) => record.sandboxName), - ); + const resolveEntryOptions = () => + onboardEntryOptions.resolveDefaultRunEntryOptionsFromState(opts, validateName, onboardSession); + const entryOptions = resolveEntryOptions(); const { fresh, nonInteractive, cannotPrompt, resume } = entryOptions; const { requestedFromDockerfile, requestedSandboxName } = entryOptions; NON_INTERACTIVE = nonInteractive; @@ -2796,6 +2792,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { preserveDeferredExitSession = false, preserveIncompleteSession = false; try { + resolveEntryOptions(); await portableRetirementEntry.run(async () => { const lockedRuntime = await resumeRuntime.prepare( opts, diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index 41fe67f56d2..b8f707b0ca1 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -67,6 +67,11 @@ type PersistedOnboardEntrySession = { readonly cancellationRecovery?: { readonly sandboxName: string } | null; }; +interface DefaultRunEntryState { + loadSession(): PersistedOnboardEntrySession | null; + listRetainedSandboxRecoveryRecords(): readonly { readonly sandboxName: string }[]; +} + type NonInteractiveEntryOptions = { nonInteractive?: boolean }; type ResumableEntryOptions = NonInteractiveEntryOptions & { resume?: boolean; @@ -194,6 +199,21 @@ export function resolveDefaultRunEntryOptions( ); } +export function resolveDefaultRunEntryOptionsFromState( + options: OnboardEntryOptionsInput["opts"] & { autoYes?: boolean; nonInteractive?: boolean }, + validateSandboxName: OnboardEntryOptionsDeps["validateName"], + state: DefaultRunEntryState, + env: NodeJS.ProcessEnv = process.env, +) { + return resolveDefaultRunEntryOptions( + options, + state.loadSession(), + validateSandboxName, + env, + state.listRetainedSandboxRecoveryRecords().map((record) => record.sandboxName), + ); +} + export function assertDefaultSandboxNameAllowed(sandboxName: string): void { if (!RESERVED_SANDBOX_NAMES.has(sandboxName)) return; console.error( diff --git a/src/lib/onboard/portable-retirement-authority.ts b/src/lib/onboard/portable-retirement-authority.ts index 4f94a679e69..8c6cac002e0 100644 --- a/src/lib/onboard/portable-retirement-authority.ts +++ b/src/lib/onboard/portable-retirement-authority.ts @@ -6,7 +6,12 @@ import fs from "node:fs"; import path from "node:path"; import { isDeepStrictEqual, TextDecoder } from "node:util"; -import { acquireOnboardLock, normalizeSession, releaseOnboardLock } from "../state/onboard-session"; +import { + acquireOnboardLock, + assertOnboardLockOwned, + normalizeSession, + releaseOnboardLock, +} from "../state/onboard-session"; import { assertHermesPortableUninstallCompleteForOnboarding } from "../state/hermes-portable-uninstall/journal"; import { inspectPortableOnboardSupersession, @@ -550,6 +555,7 @@ export function beginPortableOnboardRetirementEntry( options: PortableOnboardRetirementEntryOptions, ) { const ownsOnboardLock = !options.alreadyHeld; + if (!ownsOnboardLock) assertOnboardLockOwned(); const lockResult = ownsOnboardLock ? acquireOnboardLock(options.command) : { acquired: true as const }; diff --git a/src/lib/state/onboard-session-cross-process-lock.test.ts b/src/lib/state/onboard-session-cross-process-lock.test.ts index d8f501a1c47..0f4c4cc7619 100644 --- a/src/lib/state/onboard-session-cross-process-lock.test.ts +++ b/src/lib/state/onboard-session-cross-process-lock.test.ts @@ -39,6 +39,23 @@ afterEach(() => { }); describe("cross-process onboard lock", () => { + it("rejects caller-asserted onboarding lock ownership without a live descriptor (#9833)", async () => { + const authority = await import("../onboard/portable-retirement-authority"); + + expect(() => + authority.beginPortableOnboardRetirementEntry({ + alreadyHeld: true, + command: "nemoclaw onboard --resume", + displayName: "NemoClaw", + homeDir: tempHome, + loadRegistry: () => ({ defaultSandbox: null, sandboxes: {} }), + registryFile: path.join(tempHome, ".nemoclaw", "registry.json"), + sessionFile: session.SESSION_FILE, + withLifecycleLock: async (_sandboxName, operation) => await operation(), + }), + ).toThrow(/does not own.*onboarding lock/u); + }); + it("updates under a caller-owned onboard lock without releasing it", () => { session.saveSession( session.createSession({ @@ -159,4 +176,81 @@ describe("cross-process onboard lock", () => { await exited; } }); + + it("never reports two successful recovery writes after losing one record (#9833)", async () => { + const readyA = path.join(tempHome, "writer-a.ready"); + const readyB = path.join(tempHome, "writer-b.ready"); + const childScript = ` + const fs = require("node:fs"); + const session = require(process.argv[1]); + const ownReady = process.argv[2]; + const peerReady = process.argv[3]; + const role = process.argv[4]; + const originalWriteFileSync = fs.writeFileSync; + const wait = (milliseconds) => Atomics.wait( + new Int32Array(new SharedArrayBuffer(4)), + 0, + 0, + milliseconds, + ); + let synchronized = false; + fs.writeFileSync = (...args) => { + if (!synchronized && typeof args[0] === "number") { + synchronized = true; + originalWriteFileSync(ownReady, role); + const deadline = Date.now() + 750; + while (!fs.existsSync(peerReady) && Date.now() < deadline) wait(10); + if (role === "b") wait(100); + } + return originalWriteFileSync(...args); + }; + try { + const recorded = session.recordRetainedSandboxRecovery({ + sandboxName: "writer-" + role, + sandboxIdentityFingerprint: role.repeat(64), + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-" + role, + verifiedEffectivePolicyIdentity: null, + resources: { + sharedInferenceProviders: [], + sandboxScopedProviders: [], + credentialEnvironmentVariables: [], + }, + reason: "retained_after_sandbox_creation_failure", + }); + process.stdout.write(JSON.stringify({ ok: true, recordId: recorded.recordId })); + } catch (error) { + process.stdout.write(JSON.stringify({ ok: false, error: String(error) })); + } + `; + const runWriter = (role: "a" | "b", ownReady: string, peerReady: string) => + new Promise<{ ok: boolean }>((resolve, reject) => { + const child = spawn( + process.execPath, + ["--require", "tsx/cjs", "-e", childScript, sessionPath, ownReady, peerReady, role], + { env: { ...process.env, HOME: tempHome }, stdio: ["ignore", "pipe", "pipe"] }, + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += String(chunk))); + child.stderr.on("data", (chunk) => (stderr += String(chunk))); + child.once("error", reject); + child.once("close", (code) => { + code === 0 + ? resolve(JSON.parse(stdout) as { ok: boolean }) + : reject(new Error(`recovery writer exited ${String(code)}: ${stderr}`)); + }); + }); + + const results = await Promise.all([ + runWriter("a", readyA, readyB), + runWriter("b", readyB, readyA), + ]); + const successfulWrites = results.filter((result) => result.ok).length; + const records = session.listRetainedSandboxRecoveryRecords(); + + expect(successfulWrites).toBeGreaterThan(0); + expect(records).toHaveLength(successfulWrites); + }); }); diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 90216f7ddbe..c7b1388d762 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -397,7 +397,30 @@ export interface DebugSessionSummary { // ── Helpers ────────────────────────────────────────────────────── function ensureSessionDir(): void { + assertSessionDirectoryHasNoSymlinks(); fs.mkdirSync(SESSION_DIR, { recursive: true, mode: 0o700 }); + assertSessionDirectoryHasNoSymlinks(); + const stat = fs.lstatSync(SESSION_DIR); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new Error("NemoClaw onboarding state directory is not a secure directory."); + } +} + +function assertSessionDirectoryHasNoSymlinks(): void { + const home = path.resolve(process.env.HOME || "/tmp"); + let current = path.resolve(SESSION_DIR); + while (current !== home && current !== path.dirname(current)) { + try { + if (fs.lstatSync(current).isSymbolicLink()) { + throw new Error( + `NemoClaw onboarding state directory cannot be a symbolic link: ${current}`, + ); + } + } catch (error) { + if (!(isErrnoException(error) && error.code === "ENOENT")) throw error; + } + current = path.dirname(current); + } } export function sessionPath(): string { @@ -1236,6 +1259,46 @@ function lockHolderStillMatches(lock: LockInfo): boolean { // descriptor rather than a value re-read from disk. See #1281. let heldLockFd: number | null = null; +export function assertOnboardLockOwned(): void { + if (heldLockFd === null) { + throw new Error("This process does not own the NemoClaw onboarding lock."); + } + assertSessionDirectoryHasNoSymlinks(); + const descriptorStat = fs.fstatSync(heldLockFd); + const pathStat = fs.lstatSync(LOCK_FILE); + if ( + !descriptorStat.isFile() || + descriptorStat.nlink !== 1 || + pathStat.isSymbolicLink() || + !pathStat.isFile() || + pathStat.nlink !== 1 || + descriptorStat.dev !== pathStat.dev || + descriptorStat.ino !== pathStat.ino + ) { + throw new Error("NemoClaw onboarding lock ownership changed during the operation."); + } +} + +function withOwnedOnboardLock(command: string, operation: () => T): T { + const managesOnboardLock = heldLockFd === null; + if (managesOnboardLock) { + const lock = acquireOnboardLock(command); + if (!lock.acquired) { + throw new Error( + "Cannot update onboarding recovery while another onboarding run owns the lock.", + ); + } + } + try { + assertOnboardLockOwned(); + const result = operation(); + assertOnboardLockOwned(); + return result; + } finally { + if (managesOnboardLock) releaseOnboardLock(); + } +} + export function acquireOnboardLock(command: string | null = null): LockResult { ensureSessionDir(); const payload = JSON.stringify( @@ -1337,6 +1400,13 @@ export function acquireOnboardLock(command: string | null = null): LockResult { throw writeError; } heldLockFd = fd; + try { + assertOnboardLockOwned(); + } catch (error) { + heldLockFd = null; + fs.closeSync(fd); + throw error; + } return { acquired: true, lockFile: LOCK_FILE, stale: false }; } @@ -1691,13 +1761,18 @@ function persistIndependentRetainedSandboxRecovery( } export function listRetainedSandboxRecoveryRecords(): readonly RetainedSandboxRecoveryRecord[] { - return readRetainedSandboxRecoveryRecords(RETAINED_SANDBOX_RECOVERY_FILE); + if (heldLockFd !== null) assertOnboardLockOwned(); + const records = readRetainedSandboxRecoveryRecords(RETAINED_SANDBOX_RECOVERY_FILE); + if (heldLockFd !== null) assertOnboardLockOwned(); + return records; } export function recordRetainedSandboxRecovery( input: RecordRetainedSandboxRecoveryInput, ): RetainedSandboxRecoveryRecord { - return writeRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, input); + return withOwnedOnboardLock("nemoclaw retained sandbox recovery", () => + writeRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, input), + ); } export function markCancellationRecovery( @@ -1713,36 +1788,38 @@ export function markCancellationRecovery( ) { throw new Error("Cannot record cancellation recovery with invalid sandbox identity data."); } - const saved = updateSession((session) => { - if (session.sandboxName !== null && session.sandboxName !== sandboxName) { - throw new Error("Cannot record cancellation recovery for a different onboarding sandbox."); - } - const recordedAt = new Date().toISOString(); - session.sandboxName = sandboxName; - session.resumable = false; - session.status = CANCELLATION_RECOVERY_STATUS; - session.cancellationRecovery = { - reason: "cancelled_after_sandbox_creation", - sandboxName, - sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? null, - recordedAt, - }; - session.failure = { - step: session.lastStepStarted, - message: - "Onboarding was cancelled after sandbox creation; administrator recovery is required.", - recordedAt, - interrupted: true, - }; - return session; + return withOwnedOnboardLock("nemoclaw cancellation recovery", () => { + const saved = updateSession((session) => { + if (session.sandboxName !== null && session.sandboxName !== sandboxName) { + throw new Error("Cannot record cancellation recovery for a different onboarding sandbox."); + } + const recordedAt = new Date().toISOString(); + session.sandboxName = sandboxName; + session.resumable = false; + session.status = CANCELLATION_RECOVERY_STATUS; + session.cancellationRecovery = { + reason: "cancelled_after_sandbox_creation", + sandboxName, + sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? null, + recordedAt, + }; + session.failure = { + step: session.lastStepStarted, + message: + "Onboarding was cancelled after sandbox creation; administrator recovery is required.", + recordedAt, + interrupted: true, + }; + return session; + }); + persistIndependentRetainedSandboxRecovery( + saved, + "cancelled_after_sandbox_creation", + sandboxIdentityFingerprint ?? null, + context, + ); + return saved; }); - persistIndependentRetainedSandboxRecovery( - saved, - "cancelled_after_sandbox_creation", - sandboxIdentityFingerprint ?? null, - context, - ); - return saved; } export function markRetainedSandboxRecovery( @@ -1759,51 +1836,54 @@ export function markRetainedSandboxRecovery( ) { throw new Error("Cannot record retained sandbox recovery with invalid identity data."); } - const saved = updateSession((session) => { - if (session.sandboxName !== null && session.sandboxName !== sandboxName) { - throw new Error( - "Cannot record retained sandbox recovery for a different onboarding sandbox.", - ); + return withOwnedOnboardLock("nemoclaw retained sandbox recovery", () => { + const saved = updateSession((session) => { + if (session.sandboxName !== null && session.sandboxName !== sandboxName) { + throw new Error( + "Cannot record retained sandbox recovery for a different onboarding sandbox.", + ); + } + const recordedAt = new Date().toISOString(); + const sanitizedMessage = redactSensitiveText(message); + session.sandboxName = sandboxName; + session.resumable = false; + session.status = CANCELLATION_RECOVERY_STATUS; + session.cancellationRecovery = { + reason: "retained_after_sandbox_creation_failure", + sandboxName, + sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? null, + recordedAt, + }; + session.failure = { + step: session.lastStepStarted, + message: sanitizedMessage, + recordedAt, + interrupted: true, + }; + const sandboxStep = session.steps.sandbox; + if (sandboxStep) sandboxStep.error = sanitizedMessage; + return session; + }); + const reread = loadSession(); + if ( + reread?.sessionId !== saved.sessionId || + reread.status !== CANCELLATION_RECOVERY_STATUS || + reread.resumable !== false || + reread.cancellationRecovery?.reason !== "retained_after_sandbox_creation_failure" || + reread.cancellationRecovery.sandboxName !== sandboxName || + reread.cancellationRecovery.sandboxIdentityFingerprint !== + (sandboxIdentityFingerprint ?? null) + ) { + throw new Error("Retained sandbox recovery did not survive durable readback."); } - const recordedAt = new Date().toISOString(); - const sanitizedMessage = redactSensitiveText(message); - session.sandboxName = sandboxName; - session.resumable = false; - session.status = CANCELLATION_RECOVERY_STATUS; - session.cancellationRecovery = { - reason: "retained_after_sandbox_creation_failure", - sandboxName, - sandboxIdentityFingerprint: sandboxIdentityFingerprint ?? null, - recordedAt, - }; - session.failure = { - step: session.lastStepStarted, - message: sanitizedMessage, - recordedAt, - interrupted: true, - }; - const sandboxStep = session.steps.sandbox; - if (sandboxStep) sandboxStep.error = sanitizedMessage; - return session; + persistIndependentRetainedSandboxRecovery( + reread, + "retained_after_sandbox_creation_failure", + sandboxIdentityFingerprint ?? null, + context, + ); + return reread; }); - const reread = loadSession(); - if ( - reread?.sessionId !== saved.sessionId || - reread.status !== CANCELLATION_RECOVERY_STATUS || - reread.resumable !== false || - reread.cancellationRecovery?.reason !== "retained_after_sandbox_creation_failure" || - reread.cancellationRecovery.sandboxName !== sandboxName || - reread.cancellationRecovery.sandboxIdentityFingerprint !== (sandboxIdentityFingerprint ?? null) - ) { - throw new Error("Retained sandbox recovery did not survive durable readback."); - } - persistIndependentRetainedSandboxRecovery( - reread, - "retained_after_sandbox_creation_failure", - sandboxIdentityFingerprint ?? null, - context, - ); - return reread; } export type CompareAndSwapSessionResult = "updated" | "busy" | "mismatch"; diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index 3e037c2cef5..14088f38f08 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -65,6 +65,13 @@ interface RetainedSandboxRecoveryState { readonly resolutions: readonly RetainedSandboxAdministratorResolutionReceipt[]; } +interface RetainedSandboxStateDirectory { + readonly ancestors: readonly { readonly path: string; readonly stat: fs.Stats }[]; + readonly descriptor: number; + readonly path: string; + readonly stat: fs.Stats; +} + export interface RecordRetainedSandboxRecoveryInput { readonly sandboxName: string; readonly sandboxIdentityFingerprint: string | null; @@ -83,15 +90,130 @@ const emptyState = (): RetainedSandboxRecoveryState => ({ resolutions: [], }); +function sameFileIdentity(left: fs.Stats, right: fs.Stats): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +function stateDirectoryAncestors(directory: string): string[] { + const home = path.resolve(process.env.HOME ?? path.dirname(directory)); + const resolved = path.resolve(directory); + const relative = path.relative(home, resolved); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) { + return [resolved]; + } + const ancestors: string[] = []; + let current = home; + for (const component of relative.split(path.sep).filter(Boolean)) { + current = path.join(current, component); + ancestors.push(current); + } + return ancestors; +} + +function assertStateDirectoryComponent(candidate: string, stat: fs.Stats): void { + if (stat.isSymbolicLink()) { + throw new Error( + `Retained sandbox recovery state directory cannot be a symbolic link: ${candidate}`, + ); + } + if (!stat.isDirectory()) { + throw new Error(`Retained sandbox recovery state directory is not a directory: ${candidate}`); + } +} + +function openStateDirectory( + filePath: string, + create: boolean, +): RetainedSandboxStateDirectory | null { + const directory = path.dirname(filePath); + const ancestorPaths = stateDirectoryAncestors(directory); + for (const candidate of ancestorPaths) { + try { + assertStateDirectoryComponent(candidate, fs.lstatSync(candidate)); + } catch (error) { + if ( + error instanceof Error && + "code" in error && + (error as NodeJS.ErrnoException).code === "ENOENT" + ) { + continue; + } + throw error; + } + } + if (create) fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + + const ancestors: Array<{ path: string; stat: fs.Stats }> = []; + try { + for (const candidate of ancestorPaths) { + const stat = fs.lstatSync(candidate); + assertStateDirectoryComponent(candidate, stat); + ancestors.push({ path: candidate, stat }); + } + } catch (error) { + if ( + !create && + error instanceof Error && + "code" in error && + (error as NodeJS.ErrnoException).code === "ENOENT" + ) { + return null; + } + throw error; + } + + const flags = + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_DIRECTORY ?? 0); + const descriptor = fs.openSync(directory, flags); + try { + const descriptorStat = fs.fstatSync(descriptor); + const pathStat = fs.lstatSync(directory); + assertStateDirectoryComponent(directory, descriptorStat); + assertStateDirectoryComponent(directory, pathStat); + if (!sameFileIdentity(descriptorStat, pathStat)) { + throw new Error("Retained sandbox recovery state directory changed during validation."); + } + return { ancestors, descriptor, path: directory, stat: descriptorStat }; + } catch (error) { + fs.closeSync(descriptor); + throw error; + } +} + +function revalidateStateDirectory(directory: RetainedSandboxStateDirectory): void { + for (const ancestor of directory.ancestors) { + const current = fs.lstatSync(ancestor.path); + assertStateDirectoryComponent(ancestor.path, current); + if (!sameFileIdentity(ancestor.stat, current)) { + throw new Error("Retained sandbox recovery state directory changed during validation."); + } + } + const descriptorStat = fs.fstatSync(directory.descriptor); + const pathStat = fs.lstatSync(directory.path); + assertStateDirectoryComponent(directory.path, descriptorStat); + assertStateDirectoryComponent(directory.path, pathStat); + if ( + !sameFileIdentity(directory.stat, descriptorStat) || + !sameFileIdentity(directory.stat, pathStat) + ) { + throw new Error("Retained sandbox recovery state directory changed during validation."); + } +} + function isObjectRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function readStateFile(filePath: string): unknown { + const directory = openStateDirectory(filePath, false); + if (directory === null) return emptyState(); try { + revalidateStateDirectory(directory); const file = openRegularFileNoFollow(filePath); try { - return JSON.parse(file.readUtf8()); + const value = JSON.parse(file.readUtf8()); + revalidateStateDirectory(directory); + return value; } finally { file.close(); } @@ -112,13 +234,15 @@ function readStateFile(filePath: string): unknown { throw new Error("Retained sandbox recovery state cannot be a symbolic link."); } throw error; + } finally { + fs.closeSync(directory.descriptor); } } function writeStateFile(filePath: string, state: RetainedSandboxRecoveryState): void { - const directory = path.dirname(filePath); - fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const directory = openStateDirectory(filePath, true)!; try { + revalidateStateDirectory(directory); if (fs.lstatSync(filePath).isSymbolicLink()) { throw new Error("Retained sandbox recovery state cannot be a symbolic link."); } @@ -130,30 +254,64 @@ function writeStateFile(filePath: string, state: RetainedSandboxRecoveryState): (error as NodeJS.ErrnoException).code === "ENOENT" ) ) { + fs.closeSync(directory.descriptor); throw error; } } const temporary = path.join( - directory, + directory.path, `.retained-sandbox-recovery.${String(process.pid)}.${randomUUID()}.tmp`, ); + let descriptor: number | null = null; + let temporaryStat: fs.Stats | null = null; try { - fs.writeFileSync(temporary, JSON.stringify(state, null, 2), { mode: 0o600 }); - const descriptor = fs.openSync(temporary, "r"); - try { - fs.fsyncSync(descriptor); - } finally { - fs.closeSync(descriptor); + descriptor = fs.openSync( + temporary, + fs.constants.O_WRONLY | + fs.constants.O_CREAT | + fs.constants.O_EXCL | + (fs.constants.O_NOFOLLOW ?? 0), + 0o600, + ); + fs.writeFileSync(descriptor, JSON.stringify(state, null, 2)); + fs.fchmodSync(descriptor, 0o600); + fs.fsyncSync(descriptor); + const descriptorStat = fs.fstatSync(descriptor); + temporaryStat = descriptorStat; + const pathStat = fs.lstatSync(temporary); + if ( + !descriptorStat.isFile() || + descriptorStat.nlink !== 1 || + pathStat.isSymbolicLink() || + !pathStat.isFile() || + pathStat.nlink !== 1 || + !sameFileIdentity(descriptorStat, pathStat) + ) { + throw new Error("Retained sandbox recovery temporary state changed during validation."); } + fs.closeSync(descriptor); + descriptor = null; + revalidateStateDirectory(directory); fs.renameSync(temporary, filePath); - const directoryDescriptor = fs.openSync(directory, fs.constants.O_RDONLY); + revalidateStateDirectory(directory); + fs.fsyncSync(directory.descriptor); + } finally { + if (descriptor !== null) fs.closeSync(descriptor); try { - fs.fsyncSync(directoryDescriptor); - } finally { - fs.closeSync(directoryDescriptor); + revalidateStateDirectory(directory); + const pathStat = fs.lstatSync(temporary); + if ( + temporaryStat !== null && + pathStat.isFile() && + pathStat.nlink === 1 && + sameFileIdentity(temporaryStat, pathStat) + ) { + fs.unlinkSync(temporary); + } + } catch { + // Preserve the original result. Ambiguous paths are left untouched. } - } finally { - fs.rmSync(temporary, { force: true }); + fs.closeSync(directory.descriptor); } } diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index 92cbfa76cea..09c130c182c 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -120,6 +120,43 @@ describe("retained sandbox recovery state", () => { expect(fs.readFileSync(externalState, "utf8")).toBe(externalContents); }); + it("refuses a symbolic-link recovery state directory ancestor (#9833)", async () => { + const externalStateDirectory = path.join(home, "external-state"); + fs.mkdirSync(externalStateDirectory); + fs.symlinkSync(externalStateDirectory, path.join(home, ".nemoclaw"), "dir"); + const recovery = await import("./onboard-session"); + + expect(() => recovery.listRetainedSandboxRecoveryRecords()).toThrow(/symbolic link/u); + }); + + it("refuses recovery publication after the locked state directory is replaced (#9833)", async () => { + const recovery = await import("./onboard-session"); + expect(recovery.acquireOnboardLock("recovery directory replacement test").acquired).toBe(true); + const originalDirectory = path.dirname(recovery.RETAINED_SANDBOX_RECOVERY_FILE); + const displacedDirectory = `${originalDirectory}.displaced`; + const replacementDirectory = path.join(home, "replacement-state"); + fs.renameSync(originalDirectory, displacedDirectory); + fs.mkdirSync(replacementDirectory); + fs.symlinkSync(replacementDirectory, originalDirectory, "dir"); + + expect(() => + recovery.recordRetainedSandboxRecovery({ + sandboxName: "retained-sb", + sandboxIdentityFingerprint: "f".repeat(64), + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, + resources: evidence, + reason: "retained_after_sandbox_creation_failure", + }), + ).toThrow(/symbolic link|lock ownership changed/u); + expect(fs.existsSync(path.join(replacementDirectory, "retained-sandbox-recovery.json"))).toBe( + false, + ); + expect(fs.existsSync(path.join(displacedDirectory, "onboard.lock"))).toBe(true); + }); + it("does not expose a caller-supplied recovery resolution path (#9833)", async () => { const recovery = await import("./onboard-session"); const recoveryStore = await import("./onboard-session/retained-sandbox-recovery"); diff --git a/test/onboarding/onboard-fsm-live-slices.test.ts b/test/onboarding/onboard-fsm-live-slices.test.ts index 8252432bb74..d351c0d8cec 100644 --- a/test/onboarding/onboard-fsm-live-slices.test.ts +++ b/test/onboarding/onboard-fsm-live-slices.test.ts @@ -24,6 +24,7 @@ type ProbeMode = | "dashboard-port-composition" | "ordinary-policy-tier" | "providerless-staged-messaging" + | "stale-recovery-admission" | "ahead-core"; interface ProbeOptions { @@ -221,6 +222,7 @@ const coreFlowPhases = require(${coreFlowPhasesPath}); const registry = require(${registryPath}); const called = []; const sentinel = new Error("slice-called"); +const staleAdmissionExit = new Error("stale recovery admission refused"); if (scenario.mode === "dashboard-port-composition") { const finalizationHandlerDeps = require(${finalizationDepsPath}).finalizationHandlerDeps; @@ -447,6 +449,42 @@ if ( }); } +if (scenario.mode === "stale-recovery-admission") { + const listRetainedSandboxRecoveryRecords = + onboardSession.listRetainedSandboxRecoveryRecords; + let recoveryReads = 0; + onboardSession.listRetainedSandboxRecoveryRecords = () => { + recoveryReads += 1; + if (recoveryReads === 1) { + onboardSession.recordRetainedSandboxRecovery({ + sandboxName: "fsm-sandbox", + sandboxIdentityFingerprint: "a".repeat(64), + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "stale-admission-generation", + verifiedEffectivePolicyIdentity: null, + resources: { + sharedInferenceProviders: [], + sandboxScopedProviders: [], + credentialEnvironmentVariables: [], + }, + reason: "retained_after_sandbox_creation_failure", + }); + return []; + } + return listRetainedSandboxRecoveryRecords(); + }; + process.exit = () => { + throw staleAdmissionExit; + }; +} + +const ownsAuthoritativeOnboardLock = scenario.mode.startsWith("authoritative-"); +if (ownsAuthoritativeOnboardLock) { + const lock = onboardSession.acquireOnboardLock("authoritative rebuild fixture"); + if (!lock.acquired) throw new Error("authoritative rebuild fixture did not acquire onboard lock"); +} + const { onboard } = require(${onboardPath}); (async () => { @@ -471,9 +509,11 @@ const { onboard } = require(${onboardPath}); }); throw new Error("expected slice sentinel"); } catch (error) { + if (ownsAuthoritativeOnboardLock) onboardSession.releaseOnboardLock(); if ( error === sentinel || error?.message === sentinel.message || + (scenario.mode === "stale-recovery-admission" && error === staleAdmissionExit) || (scenario.mode === "endpoint-override" && error?.name === "OpenShellGatewayEndpointOverrideError") || (scenario.mode === "providerless-staged-messaging" && @@ -565,6 +605,10 @@ describe("live onboard FSM slice boundaries", () => { ); }); + it("rechecks retained sandbox admission after acquiring the onboarding lock (#9833)", () => { + assert.deepEqual(runSliceProbe({ slice: "initial", mode: "stale-recovery-admission" }), []); + }); + it("enters the core slice after the initial slice reaches provider selection", () => { assert.deepEqual(runSliceProbe({ slice: "core" }), ["initial:init", "core"]); }); From 3ae8f06a78e3c5f5d74b93c42e05ec605b36b711 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 08:51:25 -0700 Subject: [PATCH 35/42] fix(onboard): reject unsupported interceptor agents early Signed-off-by: Apurv Kumaria --- src/lib/onboard/sandbox-create/orchestration.ts | 4 ++++ .../onboard-fresh-create-identity.test.ts | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 6c59cfc3163..806e992dc19 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -315,6 +315,7 @@ export function assertApfCreateIntent( function assertProviderlessApfCreateInput(input: { readonly createIntent: SandboxCreateIntent | null; + readonly agent: AgentDefinition | null; readonly model: string; readonly provider: string; readonly preferredInferenceApi: string | null; @@ -324,7 +325,9 @@ function assertProviderlessApfCreateInput(input: { }): void { if (input.createIntent?.apfInterceptorRequested !== true) return; const resolved = input.createIntent.resolved; + const requestedAgent = input.agent?.name.trim().toLowerCase() ?? "openclaw"; const hasProviderIntent = + requestedAgent !== "openclaw" || input.webSearchConfig !== null || input.createIntent.reuseRegisteredCredentials === true || [ @@ -1085,6 +1088,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche assertApfCreateIntent(createIntent); assertProviderlessApfCreateInput({ createIntent, + agent, model, provider, preferredInferenceApi, diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 318c1ece693..86c2bd73157 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -36,6 +36,14 @@ describe("fresh create identity", () => { agent: null, expectedOutcome: "provider-refusal" as const, }, + { + title: "rejects a nondefault agent before credential reads or sandbox inspection (#9833)", + apfInterceptorRequested: true, + provider: null, + model: null, + agent: { name: "hermes" }, + expectedOutcome: "unsupported-agent-refusal" as const, + }, { title: "registers providerless APF only after identity, policy, and checkpoint verification (#9833)", @@ -633,6 +641,11 @@ if (${JSON.stringify( false, ); }; + const assertUnsupportedAgentRefusal = () => { + assertProviderBackedApfRefusal(); + assert.deepEqual(payload.commandNames, []); + assert.equal(payload.sandboxListCalls, 0); + }; const assertStagedMessagingRefusal = () => { assert.match( payload.creationError, @@ -979,6 +992,7 @@ if (${JSON.stringify( const assertions = { "managed-provider": assertManagedProviderCreation, "provider-refusal": assertProviderBackedApfRefusal, + "unsupported-agent-refusal": assertUnsupportedAgentRefusal, "providerless-apf": assertProviderlessApfCreation, "post-create-authority-refusal": assertPostCreateAuthorityRefusal, "post-create-runner-refusal": assertPostCreateRunnerRefusal, From 14ff3b8923d03254bb17cc6ad401cb00d7e616c8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 08:59:59 -0700 Subject: [PATCH 36/42] fix(onboard): refuse unsupported agent routes early Signed-off-by: Apurv Kumaria --- src/lib/onboard/machine/core-flow-phases.test.ts | 12 ++++++++---- src/lib/onboard/machine/core-flow-phases.ts | 5 +++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 689abbf7f91..a7d1e0c0867 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -587,11 +587,15 @@ describe("core onboard flow phases", () => { expect(events).toEqual(["credential-provider-effect", "sandbox-create"]); }); - it("rejects provider-backed APF before provider inference or sandbox effects", async () => { + it.each([ + ["provider-backed input", { model: "gpt-5.4", provider: "nvidia-prod" }], + ["a nondefault agent", { agent: { name: "hermes" }, model: null, provider: null }], + ])("rejects %s before provider inference or sandbox effects", async (_label, patch) => { const setupInference = vi.fn(async () => ({ ok: true as const })); + const reserveSandboxInferenceRoute = vi.fn(() => true); const createSandbox = vi.fn(async () => "created-sandbox"); const { providerInference: providerPhase } = createPhases({ - providerDeps: { setupInference }, + providerDeps: { reserveSandboxInferenceRoute, setupInference }, sandboxOptions: { apfInterceptorRequested: true }, sandboxDeps: { createSandbox }, }); @@ -600,14 +604,14 @@ describe("core onboard flow phases", () => { providerPhase.run( context({ fresh: true, - model: "gpt-5.4", - provider: "nvidia-prod", selectedMessagingChannels: [], + ...patch, }), ), ).rejects.toThrow(/supports providerless sandbox creation only/u); expect(setupInference).not.toHaveBeenCalled(); + expect(reserveSandboxInferenceRoute).not.toHaveBeenCalled(); expect(createSandbox).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index c733dabcf86..556ef4e016f 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -121,6 +121,10 @@ export function isCoreFlowCompleteBeforeFinalization(result: { } function hasProviderBackedApfIntent(context: OnboardFlowContext): boolean { + const requestedAgentName = (context.agent as { readonly name?: unknown } | null)?.name; + const requestsNondefaultAgent = + typeof requestedAgentName === "string" && + requestedAgentName.trim().toLowerCase() !== "openclaw"; const routeValues = [ context.provider, context.model, @@ -133,6 +137,7 @@ function hasProviderBackedApfIntent(context: OnboardFlowContext): boolean { context.nimContainer, ]; return ( + requestsNondefaultAgent || routeValues.some((value) => typeof value === "string" && value.trim().length > 0) || context.endpointSource != null || context.selectedMessagingChannels.length > 0 || From d625e9d4c233f3bf23ee944734210efd79726d10 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 09:04:18 -0700 Subject: [PATCH 37/42] fix(onboard): retain recovery on credential repair failure Signed-off-by: Apurv Kumaria --- .../sandbox-create/orchestration.test.ts | 5 ++ .../onboard/sandbox-create/orchestration.ts | 48 ++++++++++--------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 8faeb2d501a..95653c3f3e1 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -63,6 +63,7 @@ describe("created Hermes credential environment reconciliation", () => { return true; }, }, + vi.fn(), ); expect(events).toEqual([ @@ -89,6 +90,7 @@ describe("created Hermes credential environment reconciliation", () => { parseRestartCompletion: vi.fn(), waitForGateway, }, + vi.fn(), ); expect(restartGateway).not.toHaveBeenCalled(); @@ -96,6 +98,7 @@ describe("created Hermes credential environment reconciliation", () => { }); it("fails onboarding when the changed gateway cannot prove restart completion", () => { + const recordRecovery = vi.fn(); expect(() => reconcileCreatedHermesCredentialEnvironment( { sandboxName: "alpha", plan }, @@ -106,8 +109,10 @@ describe("created Hermes credential environment reconciliation", () => { parseRestartCompletion: () => null, waitForGateway: vi.fn(), }, + recordRecovery, ), ).toThrow("managed gateway restart did not complete"); + expect(recordRecovery).toHaveBeenCalledOnce(); }); }); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 806e992dc19..e8a1cdac152 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -539,32 +539,35 @@ export function reconcileCreatedHermesCredentialEnvironment( readonly plan: SandboxMessagingPlan | null; }, deps: CreatedHermesCredentialEnvReconciliationDeps, + recordRecovery: () => void, ): void { - if (input.plan?.agent !== "hermes") return; - - deps.revalidatePolicyAuthority( - `reconciling Hermes messaging credentials for sandbox '${input.sandboxName}'`, - ); - const reconciliation = deps.reconcileCredentialEnv(input.plan); - deps.revalidatePolicyAuthority( - `confirming Hermes messaging credential reconciliation for sandbox '${input.sandboxName}'`, - ); - if (!reconciliation.changed) return; + return runWithPostCreateRecovery(() => { + if (input.plan?.agent !== "hermes") return; - const restart = deps.restartGateway(input.sandboxName); - if (!deps.parseRestartCompletion(restart)) { - throw new Error( - `Hermes messaging credential reconciliation changed the gateway environment for sandbox '${input.sandboxName}', but the managed gateway restart did not complete.`, + deps.revalidatePolicyAuthority( + `reconciling Hermes messaging credentials for sandbox '${input.sandboxName}'`, ); - } - if (!deps.waitForGateway(input.sandboxName)) { - throw new Error( - `Hermes messaging credential reconciliation restarted sandbox '${input.sandboxName}', but the managed gateway did not remain healthy.`, + const reconciliation = deps.reconcileCredentialEnv(input.plan); + deps.revalidatePolicyAuthority( + `confirming Hermes messaging credential reconciliation for sandbox '${input.sandboxName}'`, ); - } - deps.revalidatePolicyAuthority( - `completing Hermes messaging credential reconciliation for sandbox '${input.sandboxName}'`, - ); + if (!reconciliation.changed) return; + + const restart = deps.restartGateway(input.sandboxName); + if (!deps.parseRestartCompletion(restart)) { + throw new Error( + `Hermes messaging credential reconciliation changed the gateway environment for sandbox '${input.sandboxName}', but the managed gateway restart did not complete.`, + ); + } + if (!deps.waitForGateway(input.sandboxName)) { + throw new Error( + `Hermes messaging credential reconciliation restarted sandbox '${input.sandboxName}', but the managed gateway did not remain healthy.`, + ); + } + deps.revalidatePolicyAuthority( + `completing Hermes messaging credential reconciliation for sandbox '${input.sandboxName}'`, + ); + }, recordRecovery); } /** @@ -2569,6 +2572,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche (args, options) => runOpenshell([...args], options), (operation) => revalidatePolicyAuthority(true, operation), ), + () => recordPostCreateRecovery("onboarding finalization"), ); } finally { cleanupInitialCreateSource(); From dbae39fda7799dffab26c424b6431fcfb7d6c7b1 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 09:08:26 -0700 Subject: [PATCH 38/42] docs(onboard): clarify fresh recovery selection Signed-off-by: Apurv Kumaria --- docs/reference/commands.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 27ea50f7c32..e6a47984d01 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -908,8 +908,9 @@ Rotate a credential only when identity-bound inspection proves that it was expos NemoClaw stores the recovery record independently from the active onboarding session. A fresh run with a different name can proceed without clearing that record, but automatic resume, explicit `--resume`, reuse, recreation, and fresh onboarding with the retained name remain blocked. NemoClaw has no supported operation in this release to clear the recovery record, so the retained name remains unavailable even after external recovery or removal. -Preserve the record as evidence, rerun the original onboarding command with the same required provider, model, agent, policy, and environment inputs, add `--fresh`, and use another available sandbox name. -`--fresh` starts a new session and does not retain those selections. +Preserve the record as evidence. +Start fresh onboarding with `--fresh` and another available sandbox name. +Select the required provider, model, agent, policy, and environment inputs again because `--fresh` does not retain them. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. In interactive mode, the wizard asks for confirmation before delete and recreate. From 4625e45d9eae9656f83d51ca155cc4680b8d3119 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 09:11:56 -0700 Subject: [PATCH 39/42] fix(onboard): bind retained recovery evidence Signed-off-by: Apurv Kumaria --- src/lib/onboard/sandbox-create/orchestration.test.ts | 7 +++++++ src/lib/onboard/sandbox-create/orchestration.ts | 2 +- src/lib/onboard/sandbox-gpu-create-flow.ts | 4 +++- src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 95653c3f3e1..5d14e856c4c 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -1250,6 +1250,7 @@ describe("sandbox create policy authority checks", () => { it("refuses continuation when identity changes during effective-policy verification (#9833)", async () => { const continuationEffect = vi.fn(); + const persistRetainedSandboxRecovery = vi.fn(() => true); const revalidate = vi .fn() .mockImplementationOnce(() => undefined) @@ -1273,11 +1274,17 @@ describe("sandbox create policy authority checks", () => { captureCreatedSandboxIdentity: () => exactIdentity, revalidateCreatedSandboxIdentity, ...verifiedPolicyBoundary(), + persistRetainedSandboxRecovery, cleanupTemporarySources: vi.fn(), }), ).rejects.toThrow("automatic sandbox cleanup was not safe"); expect(continuationEffect).not.toHaveBeenCalled(); + expect(persistRetainedSandboxRecovery).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("left sandbox 'alpha' in place"), + exactIdentity, + null, + ); }); it("fails closed when a create implementation skips the post-create gate (#9833)", async () => { diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index e8a1cdac152..ee567fc62ae 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -675,11 +675,11 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< `verifying effective policy for sandbox '${input.sandboxName}'`, ); const evidence = input.verifyCreatedPolicy(created, capturedIdentity); - observedPolicyEvidence = evidence; input.revalidateCreatedSandboxIdentity( capturedIdentity, `recording verified policy for sandbox '${input.sandboxName}'`, ); + observedPolicyEvidence = evidence; input.persistVerifiedPolicy(created, capturedIdentity, evidence); input.revalidateVerifiedPolicy( created, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 3702bfd3571..996f38c36d3 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -550,7 +550,9 @@ export async function runSandboxGpuCreateFlow( "Do not delete a sandbox by mutable name; use an identity-bound administrator recovery procedure."; let persisted = false; try { - persisted = persistRetainedSandboxRecovery(message); + persisted = evidence.liveIdentityFingerprint + ? persistRetainedSandboxRecovery(message, evidence.liveIdentityFingerprint) + : persistRetainedSandboxRecovery(message); } catch { persisted = false; } diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index ee354ffb235..f7471040d69 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -761,6 +761,7 @@ describe("created sandbox identity gate", () => { "u", ), ), + fingerprint, ); expect(input.persistRetainedSandboxRecovery).toHaveBeenCalledBefore(exit); const output = vi.mocked(console.error).mock.calls.flat().join("\n"); From eefb255f67a10c05a00ab3ea157505e7c86c8558 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 09:29:23 -0700 Subject: [PATCH 40/42] fix(onboard): use locked entry decision Signed-off-by: Apurv Kumaria --- src/lib/onboard.ts | 29 +++++++++-------- .../portable-resume-lock-boundary.test.ts | 8 +++-- .../onboard-fsm-live-slices.test.ts | 32 +++++++++++++++++++ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 5d2ab3ddfb5..652ab262106 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2752,25 +2752,15 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { AUTO_YES = opts.autoYes === true || process.env.NEMOCLAW_YES === "1"; const resolveEntryOptions = () => onboardEntryOptions.resolveDefaultRunEntryOptionsFromState(opts, validateName, onboardSession); - const entryOptions = resolveEntryOptions(); - const { fresh, nonInteractive, cannotPrompt, resume } = entryOptions; - const { requestedFromDockerfile, requestedSandboxName } = entryOptions; - NON_INTERACTIVE = nonInteractive; - const validatePolicyTierBeforeRuntime = - isNonInteractive() && !resume && opts.experimentalProfile !== "portable"; - if (validatePolicyTierBeforeRuntime) validatePolicyTierEnvEarly(); + const initialEntryOptions = resolveEntryOptions(); + NON_INTERACTIVE = initialEntryOptions.nonInteractive; RECREATE_SANDBOX = opts.recreateSandbox || process.env.NEMOCLAW_RECREATE_SANDBOX === "1"; _preflightDashboardPort = opts.controlUiPort ?? (process.env.NEMOCLAW_DASHBOARD_PORT != null ? DASHBOARD_PORT : null); onboardRuntimeBoundary.reset(); - const baseImageResolutionContext = baseImageResolutionFlow.createBaseImageResolutionContext({ - fresh, - initialHint: opts.baseImageResolutionHint, - initialPreResolvedMetadata: opts.preResolvedBaseImageMetadata, - }); const portableRetirementEntry = portableRetirementAuthority.beginPortableOnboardRetirementEntry({ alreadyHeld: opts.onboardLockAlreadyHeld === true, - command: `nemoclaw onboard${resume ? " --resume" : ""}${fresh ? " --fresh" : ""}${isNonInteractive() ? " --non-interactive" : ""}${requestedFromDockerfile ? ` --from ${requestedFromDockerfile}` : ""}`, + command: `nemoclaw onboard${initialEntryOptions.resume ? " --resume" : ""}${initialEntryOptions.fresh ? " --fresh" : ""}${initialEntryOptions.nonInteractive ? " --non-interactive" : ""}${initialEntryOptions.requestedFromDockerfile ? ` --from ${initialEntryOptions.requestedFromDockerfile}` : ""}`, displayName: cliDisplayName(), homeDir: process.env.HOME || os.homedir(), loadRegistry: registry.load, @@ -2792,8 +2782,19 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { preserveDeferredExitSession = false, preserveIncompleteSession = false; try { - resolveEntryOptions(); await portableRetirementEntry.run(async () => { + const entryOptions = resolveEntryOptions(); + const { fresh, nonInteractive, cannotPrompt, resume } = entryOptions; + const { requestedFromDockerfile, requestedSandboxName } = entryOptions; + NON_INTERACTIVE = nonInteractive; + const validatePolicyTierBeforeRuntime = + isNonInteractive() && !resume && opts.experimentalProfile !== "portable"; + if (validatePolicyTierBeforeRuntime) validatePolicyTierEnvEarly(); + const baseImageResolutionContext = baseImageResolutionFlow.createBaseImageResolutionContext({ + fresh, + initialHint: opts.baseImageResolutionHint, + initialPreResolvedMetadata: opts.preResolvedBaseImageMetadata, + }); const lockedRuntime = await resumeRuntime.prepare( opts, resume, diff --git a/src/lib/onboard/portable-resume-lock-boundary.test.ts b/src/lib/onboard/portable-resume-lock-boundary.test.ts index 5e6fcdd110c..86ee2edcd78 100644 --- a/src/lib/onboard/portable-resume-lock-boundary.test.ts +++ b/src/lib/onboard/portable-resume-lock-boundary.test.ts @@ -526,13 +526,14 @@ describe("portable resume command lock boundary", () => { return directory; }) as typeof fs.mkdtempSync); const harnessModule = await import("../../../test/helpers/rebuild-flow-generic-harness"); - const { rebuildOnboardDependencies } = + const { onboardSession: rebuildOnboardSession, rebuildOnboardDependencies } = await import("../../../test/helpers/rebuild-flow-harness"); const actualOnboard = rebuildOnboardDependencies.onboard.bind(rebuildOnboardDependencies) as ( options: import("./types").OnboardOptions, ) => Promise; const { retirement } = boundaryModules; let innerObserved = false; + let innerError = ""; const harness = harnessModule.createRebuildFlowHarness({ onboard: async (_session, options) => { const lockPath = retirement.portableHostFencePath(tempHome); @@ -545,16 +546,19 @@ describe("portable resume command lock boundary", () => { try { await actualOnboard(options); } catch (error) { + innerError = String(error); innerObserved = String(error).includes("retirement record"); expect(fs.lstatSync(lockPath, { bigint: true }).ino).toBe(outerInode); throw error; } }, }); + vi.mocked(rebuildOnboardSession.acquireOnboardLock).mockRestore(); + vi.mocked(rebuildOnboardSession.releaseOnboardLock).mockRestore(); try { await harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }).catch(() => {}); - expect(innerObserved).toBe(true); + expect(innerObserved, innerError).toBe(true); expect(fs.existsSync(retirement.portableHostFencePath(tempHome))).toBe(false); } finally { mkdtemp.mockRestore(); diff --git a/test/onboarding/onboard-fsm-live-slices.test.ts b/test/onboarding/onboard-fsm-live-slices.test.ts index d351c0d8cec..932a6584ad6 100644 --- a/test/onboarding/onboard-fsm-live-slices.test.ts +++ b/test/onboarding/onboard-fsm-live-slices.test.ts @@ -25,6 +25,7 @@ type ProbeMode = | "ordinary-policy-tier" | "providerless-staged-messaging" | "stale-recovery-admission" + | "stale-session-decision" | "ahead-core"; interface ProbeOptions { @@ -181,6 +182,12 @@ function runSliceProbe(options: ProbeOptions) { const sessionPath = JSON.stringify( path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), ); + const entryOptionsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "entry-options.ts"), + ); + const lockedRuntimePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "resume", "locked-runtime.ts"), + ); const preflightHandlerPath = JSON.stringify( path.join(repoRoot, "src", "lib", "onboard", "machine", "handlers", "preflight.ts"), ); @@ -215,6 +222,8 @@ const scenario = ${JSON.stringify(scenario)}; const flowSlices = require(${flowSlicesPath}); const { advanceTo, branchTo } = require(${resultPath}); const onboardSession = require(${sessionPath}); +const onboardEntryOptions = require(${entryOptionsPath}); +const lockedRuntime = require(${lockedRuntimePath}); const preflightHandlers = require(${preflightHandlerPath}); const providerHandlers = require(${providerHandlerPath}); const gatewayHandlers = require(${gatewayHandlerPath}); @@ -479,6 +488,23 @@ if (scenario.mode === "stale-recovery-admission") { }; } +if (scenario.mode === "stale-session-decision") { + const resolveEntryOptions = onboardEntryOptions.resolveDefaultRunEntryOptionsFromState; + let optionReads = 0; + onboardEntryOptions.resolveDefaultRunEntryOptionsFromState = (...args) => { + optionReads += 1; + const resolved = resolveEntryOptions(...args); + if (optionReads === 1) { + seedResumeSession("preflight", false); + } + return resolved; + }; + lockedRuntime.prepare = async (_opts, resume) => { + called.push("locked-resume:" + String(resume)); + throw sentinel; + }; +} + const ownsAuthoritativeOnboardLock = scenario.mode.startsWith("authoritative-"); if (ownsAuthoritativeOnboardLock) { const lock = onboardSession.acquireOnboardLock("authoritative rebuild fixture"); @@ -609,6 +635,12 @@ describe("live onboard FSM slice boundaries", () => { assert.deepEqual(runSliceProbe({ slice: "initial", mode: "stale-recovery-admission" }), []); }); + it("uses the session decision read after acquiring the onboarding lock (#9833)", () => { + assert.deepEqual(runSliceProbe({ slice: "initial", mode: "stale-session-decision" }), [ + "locked-resume:true", + ]); + }); + it("enters the core slice after the initial slice reaches provider selection", () => { assert.deepEqual(runSliceProbe({ slice: "core" }), ["initial:init", "core"]); }); From 310bfe761fe3237f3d37e557954af0f5c0873d5f Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 09:36:51 -0700 Subject: [PATCH 41/42] fix(state): pin onboarding state writes Signed-off-by: Apurv Kumaria --- src/lib/state/onboard-session.ts | 105 +++++++++++++++++- .../retained-sandbox-recovery.ts | 32 +++--- .../state/retained-sandbox-recovery.test.ts | 77 +++++++++++++ 3 files changed, 195 insertions(+), 19 deletions(-) diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index c7b1388d762..4f494d07d04 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -423,6 +423,62 @@ function assertSessionDirectoryHasNoSymlinks(): void { } } +interface PinnedSessionDirectory { + readonly descriptor: number; + readonly stat: fs.Stats; +} + +function sameSessionFileIdentity(left: fs.Stats, right: fs.Stats): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +function revalidatePinnedSessionDirectory(directory: PinnedSessionDirectory): void { + assertSessionDirectoryHasNoSymlinks(); + const descriptorStat = fs.fstatSync(directory.descriptor); + const pathStat = fs.lstatSync(SESSION_DIR); + if ( + !descriptorStat.isDirectory() || + pathStat.isSymbolicLink() || + !pathStat.isDirectory() || + !sameSessionFileIdentity(directory.stat, descriptorStat) || + !sameSessionFileIdentity(directory.stat, pathStat) + ) { + throw new Error("NemoClaw onboarding state directory changed during validation."); + } +} + +function openPinnedSessionDirectory(): PinnedSessionDirectory { + ensureSessionDir(); + const descriptor = fs.openSync( + SESSION_DIR, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_DIRECTORY ?? 0), + ); + try { + const directory = { descriptor, stat: fs.fstatSync(descriptor) }; + revalidatePinnedSessionDirectory(directory); + return directory; + } catch (error) { + fs.closeSync(descriptor); + throw error; + } +} + +function assertTemporarySessionFile(descriptor: number, temporary: string): fs.Stats { + const descriptorStat = fs.fstatSync(descriptor); + const pathStat = fs.lstatSync(temporary); + if ( + !descriptorStat.isFile() || + descriptorStat.nlink !== 1 || + pathStat.isSymbolicLink() || + !pathStat.isFile() || + pathStat.nlink !== 1 || + !sameSessionFileIdentity(descriptorStat, pathStat) + ) { + throw new Error("NemoClaw onboarding temporary state changed during validation."); + } + return descriptorStat; +} + export function sessionPath(): string { return SESSION_FILE; } @@ -1142,16 +1198,53 @@ function serializeSessionForDisk(session: Session): Record { export function saveSession(session: Session): Session { const normalized = normalizeSession(session) || createSession(); normalized.updatedAt = new Date().toISOString(); - ensureSessionDir(); + const directory = openPinnedSessionDirectory(); const tmpFile = path.join( SESSION_DIR, `.onboard-session.${process.pid}.${Date.now()}.${randomUUID()}.tmp`, ); - fs.writeFileSync(tmpFile, JSON.stringify(serializeSessionForDisk(normalized), null, 2), { - mode: 0o600, - }); - fs.renameSync(tmpFile, SESSION_FILE); - return normalized; + let descriptor: number | null = null; + let temporaryStat: fs.Stats | null = null; + try { + descriptor = fs.openSync( + tmpFile, + fs.constants.O_WRONLY | + fs.constants.O_CREAT | + fs.constants.O_EXCL | + (fs.constants.O_NOFOLLOW ?? 0), + 0o600, + ); + revalidatePinnedSessionDirectory(directory); + temporaryStat = assertTemporarySessionFile(descriptor, tmpFile); + fs.writeFileSync(descriptor, JSON.stringify(serializeSessionForDisk(normalized), null, 2)); + fs.fchmodSync(descriptor, 0o600); + fs.fsyncSync(descriptor); + temporaryStat = assertTemporarySessionFile(descriptor, tmpFile); + fs.closeSync(descriptor); + descriptor = null; + revalidatePinnedSessionDirectory(directory); + fs.renameSync(tmpFile, SESSION_FILE); + revalidatePinnedSessionDirectory(directory); + fs.fsyncSync(directory.descriptor); + return normalized; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + try { + revalidatePinnedSessionDirectory(directory); + const pathStat = fs.lstatSync(tmpFile); + if ( + temporaryStat !== null && + pathStat.isFile() && + pathStat.nlink === 1 && + sameSessionFileIdentity(temporaryStat, pathStat) + ) { + fs.unlinkSync(tmpFile); + } + } catch { + // Preserve the original result. Ambiguous paths are left untouched. + } + fs.closeSync(directory.descriptor); + } } export function clearSession(): void { diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index 14088f38f08..c5fe7552f77 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -239,6 +239,22 @@ function readStateFile(filePath: string): unknown { } } +function assertTemporaryStateFile(descriptor: number, temporary: string): fs.Stats { + const descriptorStat = fs.fstatSync(descriptor); + const pathStat = fs.lstatSync(temporary); + if ( + !descriptorStat.isFile() || + descriptorStat.nlink !== 1 || + pathStat.isSymbolicLink() || + !pathStat.isFile() || + pathStat.nlink !== 1 || + !sameFileIdentity(descriptorStat, pathStat) + ) { + throw new Error("Retained sandbox recovery temporary state changed during validation."); + } + return descriptorStat; +} + function writeStateFile(filePath: string, state: RetainedSandboxRecoveryState): void { const directory = openStateDirectory(filePath, true)!; try { @@ -273,22 +289,12 @@ function writeStateFile(filePath: string, state: RetainedSandboxRecoveryState): (fs.constants.O_NOFOLLOW ?? 0), 0o600, ); + revalidateStateDirectory(directory); + temporaryStat = assertTemporaryStateFile(descriptor, temporary); fs.writeFileSync(descriptor, JSON.stringify(state, null, 2)); fs.fchmodSync(descriptor, 0o600); fs.fsyncSync(descriptor); - const descriptorStat = fs.fstatSync(descriptor); - temporaryStat = descriptorStat; - const pathStat = fs.lstatSync(temporary); - if ( - !descriptorStat.isFile() || - descriptorStat.nlink !== 1 || - pathStat.isSymbolicLink() || - !pathStat.isFile() || - pathStat.nlink !== 1 || - !sameFileIdentity(descriptorStat, pathStat) - ) { - throw new Error("Retained sandbox recovery temporary state changed during validation."); - } + temporaryStat = assertTemporaryStateFile(descriptor, temporary); fs.closeSync(descriptor); descriptor = null; revalidateStateDirectory(directory); diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index 09c130c182c..605df8ac27e 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -157,6 +157,83 @@ describe("retained sandbox recovery state", () => { expect(fs.existsSync(path.join(displacedDirectory, "onboard.lock"))).toBe(true); }); + it("writes no recovery evidence after the state directory changes at temporary open (#9833)", async () => { + const recovery = await import("./onboard-session"); + const stateDirectory = path.dirname(recovery.RETAINED_SANDBOX_RECOVERY_FILE); + const displacedDirectory = `${stateDirectory}.displaced`; + const openSync = fs.openSync.bind(fs); + let replaced = false; + vi.spyOn(fs, "openSync").mockImplementation((file, flags, mode) => { + !replaced && + path.basename(String(file)).startsWith(".retained-sandbox-recovery.") && + (() => { + replaced = true; + fs.renameSync(stateDirectory, displacedDirectory); + fs.mkdirSync(stateDirectory, { mode: 0o700 }); + })(); + return openSync(file, flags, mode); + }); + + expect(() => + recovery.recordRetainedSandboxRecovery({ + sandboxName: "retained-sb", + sandboxIdentityFingerprint: "f".repeat(64), + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, + resources: evidence, + reason: "retained_after_sandbox_creation_failure", + }), + ).toThrow(/state directory changed|lock ownership changed/u); + expect(replaced).toBe(true); + expect( + fs + .readdirSync(stateDirectory) + .map((name) => fs.statSync(path.join(stateDirectory, name)).size) + .filter((size) => size > 0), + ).toEqual([]); + }); + + it("writes no session evidence after the state directory changes at temporary open (#9833)", async () => { + const recovery = await import("./onboard-session"); + const stateDirectory = path.dirname(recovery.SESSION_FILE); + const displacedDirectory = `${stateDirectory}.displaced`; + const openSync = fs.openSync.bind(fs); + let replaced = false; + vi.spyOn(fs, "openSync").mockImplementation((file, flags, mode) => { + !replaced && + path.basename(String(file)).startsWith(".onboard-session.") && + (() => { + replaced = true; + fs.renameSync(stateDirectory, displacedDirectory); + fs.mkdirSync(stateDirectory, { mode: 0o700 }); + })(); + return openSync(file, flags, mode); + }); + + expect(() => + recovery.markRetainedSandboxRecovery( + "retained-sb", + "Sandbox creation failed after identity verification.", + "f".repeat(64), + { + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, + }, + ), + ).toThrow(/state directory changed|lock ownership changed/u); + expect(replaced).toBe(true); + expect( + fs + .readdirSync(stateDirectory) + .map((name) => fs.statSync(path.join(stateDirectory, name)).size) + .filter((size) => size > 0), + ).toEqual([]); + }); + it("does not expose a caller-supplied recovery resolution path (#9833)", async () => { const recovery = await import("./onboard-session"); const recoveryStore = await import("./onboard-session/retained-sandbox-recovery"); From 1346f909e6ddb331f65426f30c4640b187dbe64a Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 27 Aug 2026 11:21:12 -0700 Subject: [PATCH 42/42] test(onboard): keep verified recovery providerless Signed-off-by: Apurv Kumaria --- .../onboard-reservation-recreate.test.ts | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/test/onboarding/onboard-reservation-recreate.test.ts b/test/onboarding/onboard-reservation-recreate.test.ts index 3c526f8b303..58adb17c3d5 100644 --- a/test/onboarding/onboard-reservation-recreate.test.ts +++ b/test/onboarding/onboard-reservation-recreate.test.ts @@ -265,7 +265,9 @@ const { createSandbox } = require(${onboardPath}); assert.equal( result.status, replaceBeforeCleanup ? 1 : 0, - result.stderr || result.error?.message || "onboarding subprocess returned an unexpected status", + result.stderr || + result.error?.message || + "onboarding subprocess returned an unexpected status", ); const payload = trailingJsonPayload<{ sandboxName: string | null; @@ -384,8 +386,8 @@ if (mode === "seed") { "my-assistant": { name: "my-assistant", gatewayName: "nemoclaw", - provider: "nvidia-prod", - model: "gpt-5.4", + provider: null, + model: null, endpointUrl: null, endpointSource: null, credentialEnv: null, @@ -403,8 +405,8 @@ if (mode === "seed") { intent: { agent: "openclaw", fromDockerfile: null, - provider: "nvidia-prod", - model: "gpt-5.4", + provider: null, + model: null, preferredInferenceApi: null, sandboxGpuConfig: null, gatewayName: "nemoclaw", @@ -419,8 +421,8 @@ if (mode === "seed") { const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", - provider: "nvidia-prod", - model: "gpt-5.4", + provider: null, + model: null, sessionId: "session-owner", durableRegistry: true, }); @@ -521,7 +523,7 @@ const { createSandbox } = require(${onboardPath}); const transaction = onboardSession.loadSession()?.checkpoint?.sandboxRecreate; if (!transaction) throw new Error("verified-create recovery has no lifecycle journal"); const createArgs = fixtureMocks.sandboxCreateArgsWithVerifiedReservation( - [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + [null, null, null, null, "my-assistant", null, null, null, null, null, null, null, []], createFixture, ); createArgs[15] = { @@ -590,7 +592,10 @@ createArgs[16] = async () => { assert.match(retained.error, /automatic sandbox cleanup was not safe/u); assert.equal(retained.registryEntry.pendingRouteReservation, true); assert.ok(retained.registryEntry.pendingPolicyVerification); - assert.match(retained.registryEntry.lifecycleLiveIdentityFingerprint ?? "", /^[0-9a-f]{64}$/u); + assert.match( + retained.registryEntry.lifecycleLiveIdentityFingerprint ?? "", + /^[0-9a-f]{64}$/u, + ); assert.equal(retained.journal.phase, "created"); assert.equal( retained.journal.targetLiveIdentityFingerprint, @@ -611,8 +616,16 @@ createArgs[16] = async () => { policyAuthority?: string; }; }>(second.stdout); - const createEvents = fs.readFileSync(createCountPath, "utf8").trim().split(/\n/u).filter(Boolean); - const effectEvents = fs.readFileSync(effectCountPath, "utf8").trim().split(/\n/u).filter(Boolean); + const createEvents = fs + .readFileSync(createCountPath, "utf8") + .trim() + .split(/\n/u) + .filter(Boolean); + const effectEvents = fs + .readFileSync(effectCountPath, "utf8") + .trim() + .split(/\n/u) + .filter(Boolean); assert.equal(createEvents.length, 1, "recovery must never create a second sandbox"); assert.match( recovered.error ?? "completed",