From b5fddb1a4005884b94e9da3f55989a7846ec83f3 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 13:01:06 -0700 Subject: [PATCH 01/24] fix(onboard): recover retained sandbox cleanup (#10547) Signed-off-by: Prekshi Vyas --- .../destroy-container-identity.test.ts | 30 ++++ src/lib/actions/sandbox/destroy-execution.ts | 68 +++++-- src/lib/actions/sandbox/destroy-presence.ts | 94 ++++++++-- .../destroy-retained-recovery-flow.test.ts | 169 ++++++++++++++++++ src/lib/actions/sandbox/destroy.ts | 85 ++++++++- src/lib/onboard/cancel-rollback.test.ts | 2 +- src/lib/onboard/cancel-rollback.ts | 9 +- src/lib/onboard/entry-options.test.ts | 24 +++ src/lib/onboard/entry-options.ts | 15 +- src/lib/onboard/lifecycle-contracts.md | 4 +- ...penshell-docker-sandbox-containers.test.ts | 67 +++++++ .../openshell-docker-sandbox-containers.ts | 56 ++++-- .../onboard/sandbox-create/orchestration.ts | 4 +- src/lib/state/onboard-session.ts | 40 +++++ .../retained-sandbox-recovery.ts | 22 +++ .../state/retained-sandbox-recovery.test.ts | 66 +++++-- test/helpers/destroy-flow-test-harness.ts | 12 ++ .../onboard-fresh-create-identity.test.ts | 2 +- 18 files changed, 693 insertions(+), 76 deletions(-) create mode 100644 src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts diff --git a/src/lib/actions/sandbox/destroy-container-identity.test.ts b/src/lib/actions/sandbox/destroy-container-identity.test.ts index a63f1f61fb..a0d3b52455 100644 --- a/src/lib/actions/sandbox/destroy-container-identity.test.ts +++ b/src/lib/actions/sandbox/destroy-container-identity.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; +import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; import { classifyDestroyContainerIdentity, type DestroyContainerIdentityVerdict, @@ -126,6 +127,35 @@ describe("classifyDestroyContainerIdentity", () => { expect(verdict.reason).toContain("2 managed containers"); }); + it("accepts every managed container bound to one retained sandbox identity (#10547)", () => { + const sandboxIdentityFingerprint = fingerprintOpenShellSandboxId(MANAGED.sandboxId)!; + const identities = [MANAGED, { ...MANAGED, id: "dddd000000000000" }]; + + expect( + classifyDestroyContainerIdentity( + "destroytest", + observeRows(identities), + sandboxIdentityFingerprint, + ), + ).toEqual({ status: "recovery", identities }); + }); + + it("refuses a retained recovery set that contains another sandbox identity (#10547)", () => { + const sandboxIdentityFingerprint = fingerprintOpenShellSandboxId(MANAGED.sandboxId)!; + const verdict = expectAmbiguous( + classifyDestroyContainerIdentity( + "destroytest", + observeRows([ + MANAGED, + { ...MANAGED, id: "dddd000000000000", sandboxId: "sb-replacement" }, + ]), + sandboxIdentityFingerprint, + ), + ); + + expect(verdict.reason).toContain("retained sandbox identity"); + }); + it.each([ ["workspace", { ...MANAGED, workspace: "" }], ["sandbox ID", { ...MANAGED, sandboxId: "" }], diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index a54be5f7bd..23444bd31f 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -33,9 +33,10 @@ import { readTimerMarker } from "../../shields/timer-control"; import type { SandboxEntry } from "../../state/registry"; import { classifyDestroyContainerIdentity, - isSameDestroyContainerIdentity, + isSameDestroyContainerIdentityProof, observeDestroyContainerIdentity, - removeExactDestroyContainerIdentity, + removeExactDestroyContainerIdentities, + type DestroyContainerIdentityProof, type SandboxNameLabeledContainer, } from "./destroy-presence"; import { type DestroyRunOpenshell, SANDBOX_DESTROY_TIMEOUT_MS } from "./destroy-gateway"; @@ -69,10 +70,11 @@ type SandboxDestroyExecutionInput = { sandbox: SandboxEntry | null; sandboxConfirmedAbsent: boolean; sandboxName: string; - // `undefined` delegates identity gating to the runtime provider. - // `null` records confirmed absence; an object records the one managed - // container observed by the pre-destroy guard. - expectedContainerIdentity?: SandboxNameLabeledContainer | null; + // `undefined` delegates identity gating to the runtime provider. An empty + // array records confirmed absence; other arrays contain the immutable + // Docker IDs qualified before destroy preparation. + expectedContainerIdentities?: readonly SandboxNameLabeledContainer[]; + expectedContainerIdentityFingerprint?: string; portableContainerAuthority?: PreparedPortableDemoSandboxDestroyAuthority; stopInferenceResources: () => void; runtimeProviders?: RuntimeProviderBundleRegistry; @@ -299,7 +301,8 @@ export async function executeSandboxDestroy({ sandbox, sandboxConfirmedAbsent, sandboxName, - expectedContainerIdentity, + expectedContainerIdentities, + expectedContainerIdentityFingerprint, portableContainerAuthority, stopInferenceResources, runtimeProviders = CURRENT_RUNTIME_PROVIDER_BUNDLES, @@ -312,6 +315,19 @@ export async function executeSandboxDestroy({ | { status: "ambiguous"; detail: string; subject?: string } | { status: "probe-failed"; detail: string; subject?: string }; const pendingPolicyVerification = sandbox?.pendingPolicyVerification; + const expectedContainerProof: DestroyContainerIdentityProof = + expectedContainerIdentities === undefined + ? { identity: undefined } + : { identities: expectedContainerIdentities }; + const proofFromVerdict = ( + verdict: ReturnType, + ): DestroyContainerIdentityProof | null => { + if (verdict.status === "clear") { + return { identities: verdict.identity === null ? [] : [verdict.identity] }; + } + if (verdict.status === "recovery") return { identities: verdict.identities }; + return null; + }; const inspectPendingPolicyVerificationContinuity = (): IdentityContinuity => { if (!pendingPolicyVerification) return { status: "match" }; if (!getSandbox) { @@ -326,6 +342,16 @@ export async function executeSandboxDestroy({ if (!isDeepStrictEqual(readCurrentCheckpoint(), pendingPolicyVerification)) { return { status: "changed", subject: "Pending policy verification authority" }; } + if ( + sandboxConfirmedAbsent && + expectedContainerIdentities !== undefined && + expectedContainerIdentityFingerprint === + pendingPolicyVerification.sandboxIdentityFingerprint + ) { + return isDeepStrictEqual(readCurrentCheckpoint(), pendingPolicyVerification) + ? { status: "match" } + : { status: "changed", subject: "Pending policy verification authority" }; + } const inspectIdentity = deps.inspectOpenShellSandboxIdentityFingerprint ?? inspectOpenShellSandboxIdentityFingerprint; @@ -362,12 +388,17 @@ export async function executeSandboxDestroy({ return { status: "probe-failed", detail: redactDestroyError(error) }; } } - if (expectedContainerIdentity === undefined) return { status: "match" }; + if (expectedContainerIdentities === undefined) return { status: "match" }; const verdict = classifyDestroyContainerIdentity( sandboxName, observeDestroyContainerIdentity(sandboxName), + expectedContainerIdentityFingerprint, ); - if (isSameDestroyContainerIdentity(expectedContainerIdentity, verdict)) { + const actualContainerProof = proofFromVerdict(verdict); + if ( + actualContainerProof && + isSameDestroyContainerIdentityProof(expectedContainerProof, actualContainerProof) + ) { return { status: "match" }; } if (verdict.status === "probe-failed") { @@ -510,14 +541,14 @@ export async function executeSandboxDestroy({ " Managed inference cleanup may already be partial; inspect or restart its resources before retrying.", ); } - // `expectedContainerIdentity === null` is a completed Docker identity + // An empty `expectedContainerIdentities` is a completed Docker identity // probe with zero matching containers. `undefined` means this runtime // does not use that probe (or Portable owns identity); skip hardening // only when OpenShell already proved absence. A live labeled Docker // identity still hardens even if the OpenShell list says absent. const sandboxRuntimeConfirmedAbsent = - expectedContainerIdentity === null || - (expectedContainerIdentity === undefined && sandboxConfirmedAbsent); + expectedContainerIdentities?.length === 0 || + (expectedContainerIdentities === undefined && sandboxConfirmedAbsent); let hardened: HardenedDeleteState; try { hardened = wipeAndHardenLiveSandbox( @@ -638,12 +669,19 @@ export async function executeSandboxDestroy({ }; } - if (!forcedLocalCleanup && (portableContainerAuthority || expectedContainerIdentity)) { + if ( + !forcedLocalCleanup && + (portableContainerAuthority || expectedContainerIdentities !== undefined) + ) { try { if (portableContainerAuthority) { portableContainerAuthority.verifyAbsent(); - } else if (expectedContainerIdentity) { - removeExactDestroyContainerIdentity(sandboxName, expectedContainerIdentity, console.log); + } else if (expectedContainerIdentities !== undefined) { + removeExactDestroyContainerIdentities( + sandboxName, + expectedContainerIdentities, + console.log, + ); } } catch (error) { const detail = redactDestroyError(error); diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts index 0077feb122..10cbfe51b9 100644 --- a/src/lib/actions/sandbox/destroy-presence.ts +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -7,7 +7,9 @@ import { OPENSHELL_SANDBOX_ID_LABEL, OPENSHELL_SANDBOX_NAME_LABEL, removeExactOpenShellDockerSandboxContainer, + removeExactOpenShellDockerSandboxContainers, } from "../../onboard/openshell-docker-sandbox-containers"; +import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; import { sanitizeReadinessText } from "../../readiness/sanitize"; import { type DockerSandboxIdentityObservation, @@ -35,6 +37,7 @@ export type SandboxNameLabeledContainer = { /** Verdict for whether destroy resolved one complete managed container identity. */ export type DestroyContainerIdentityVerdict = | { status: "clear"; identity: SandboxNameLabeledContainer | null } + | { status: "recovery"; identities: SandboxNameLabeledContainer[] } | { status: "probe-failed"; detail: string } | { status: "ambiguous"; @@ -47,14 +50,27 @@ export type DestroyContainerIdentityVerdict = export type AssertUnambiguousDestroyIdentityDeps = { providerId: string; redact: (detail: string) => string; + retainedSandboxIdentityFingerprint?: string; cliName?: string; - classify?: (sandboxName: string) => DestroyContainerIdentityVerdict; + classify?: ( + sandboxName: string, + retainedSandboxIdentityFingerprint?: string, + ) => DestroyContainerIdentityVerdict; error?: (message: string) => void; }; -export type DestroyContainerIdentityProof = { - identity: SandboxNameLabeledContainer | null | undefined; -}; +export type DestroyContainerIdentityProof = + | { identity: SandboxNameLabeledContainer | null | undefined } + | { identities: readonly SandboxNameLabeledContainer[] }; + +/** Normalize the legacy single-container proof and recovery set proof. */ +export function getDestroyContainerIdentities( + proof: DestroyContainerIdentityProof, +): readonly SandboxNameLabeledContainer[] | undefined { + if ("identities" in proof) return proof.identities; + if (proof.identity === undefined) return undefined; + return proof.identity === null ? [] : [proof.identity]; +} function observeDockerSandboxIdentities(sandboxName: string): DockerSandboxIdentityObservation { return inspectDockerSandboxIdentities(`${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, { @@ -80,6 +96,19 @@ export function removeExactDestroyContainerIdentity( removeExactOpenShellDockerSandboxContainer(sandboxName, expectedIdentity.id, log); } +/** Retire the exact container set qualified from one retained recovery fingerprint. */ +export function removeExactDestroyContainerIdentities( + sandboxName: string, + expectedIdentities: readonly SandboxNameLabeledContainer[], + log: (message: string) => void, +): void { + removeExactOpenShellDockerSandboxContainers( + sandboxName, + expectedIdentities.map((identity) => identity.id), + log, + ); +} + /** * Classify every Docker container carrying `openshell.ai/sandbox-name=`. * The query intentionally does not filter by managed-by so a foreign container @@ -88,6 +117,7 @@ export function removeExactDestroyContainerIdentity( export function classifyDestroyContainerIdentity( sandboxName: string, observation: DockerSandboxIdentityObservation, + retainedSandboxIdentityFingerprint?: string, ): DestroyContainerIdentityVerdict { if (observation.status === "probe-failed") { return { @@ -124,6 +154,30 @@ export function classifyDestroyContainerIdentity( managed, }; } + if (managed.length > 0 && retainedSandboxIdentityFingerprint !== undefined) { + const identityMatches = managed.every( + (row) => + row.workspace.length > 0 && + row.sandboxId.length > 0 && + fingerprintOpenShellSandboxId(row.sandboxId) === retainedSandboxIdentityFingerprint, + ); + const oneWorkspace = new Set(managed.map((row) => row.workspace)).size === 1; + if (!identityMatches || !oneWorkspace) { + return { + status: "ambiguous", + sandboxName, + reason: "one or more managed containers do not match the retained sandbox identity", + foreign, + managed, + }; + } + if (managed.length > 1) { + return { + status: "recovery", + identities: [...managed].sort((left, right) => left.id.localeCompare(right.id)), + }; + } + } if (managed.length !== 1) { return { status: "ambiguous", @@ -208,12 +262,18 @@ export function assertUnambiguousDestroyContainerIdentity( ): DestroyContainerIdentityProof | false { const classify = deps.classify ?? - ((name: string) => - classifyDestroyContainerIdentity(name, observeDestroyContainerIdentity(name))); + ((name: string, retainedSandboxIdentityFingerprint?: string) => + classifyDestroyContainerIdentity( + name, + observeDestroyContainerIdentity(name), + retainedSandboxIdentityFingerprint, + )); const error = deps.error ?? ((message: string) => console.error(` ${message}`)); if (deps.providerId !== "docker") return { identity: undefined }; - const verdict = classify(sandboxName); + const verdict = deps.retainedSandboxIdentityFingerprint + ? classify(sandboxName, deps.retainedSandboxIdentityFingerprint) + : classify(sandboxName); if (verdict.status === "ambiguous") { for (const line of formatAmbiguousDestroyIdentity(verdict, deps.cliName ?? "nemoclaw")) { error(line); @@ -228,7 +288,9 @@ export function assertUnambiguousDestroyContainerIdentity( ); return false; } - return { identity: verdict.identity }; + return verdict.status === "recovery" + ? { identities: verdict.identities } + : { identity: verdict.identity }; } /** Compare provider-owned identity proofs across two destroy checkpoints. */ @@ -236,12 +298,18 @@ export function isSameDestroyContainerIdentityProof( expected: DestroyContainerIdentityProof, actual: DestroyContainerIdentityProof, ): boolean { - if (expected.identity === undefined || actual.identity === undefined) { - return expected.identity === actual.identity; + const expectedIdentities = getDestroyContainerIdentities(expected); + const actualIdentities = getDestroyContainerIdentities(actual); + if (expectedIdentities === undefined || actualIdentities === undefined) { + return expectedIdentities === actualIdentities; } - return isSameDestroyContainerIdentity(expected.identity, { - status: "clear", - identity: actual.identity, + if (expectedIdentities.length !== actualIdentities.length) return false; + return expectedIdentities.every((identity, index) => { + const candidate = actualIdentities[index]; + return candidate !== undefined && isSameDestroyContainerIdentity(identity, { + status: "clear", + identity: candidate, + }); }); } diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts new file mode 100644 index 0000000000..6a47fc4f41 --- /dev/null +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; + +import { + createDestroyHarness, + resetDestroyModuleCache, +} from "../../../../test/helpers/destroy-flow-test-harness"; +import type { RetainedSandboxRecoveryRecord } from "../../state/onboard-session/retained-sandbox-recovery"; + +function retainedRecoveryRecord(sandboxId = "sb-alpha"): RetainedSandboxRecoveryRecord { + return { + schemaVersion: 1, + recordId: "f".repeat(64), + sandboxName: "alpha", + sandboxIdentityFingerprint: createHash("sha256").update(sandboxId).digest("hex"), + identityWasUnavailable: false, + gatewayName: "nemoclaw-19080", + gatewayPort: 19080, + lifecycleGeneration: "generation-alpha", + verifiedEffectivePolicyIdentity: null, + createAttemptNonce: "c".repeat(62), + policyCreationReceipt: null, + resources: { + sharedInferenceProviders: [], + sandboxScopedProviders: [], + credentialEnvironmentVariables: [], + }, + reason: "retained_after_sandbox_creation_failure", + recordedAt: "2026-08-28T00:00:00.000Z", + }; +} + +describe("destroySandbox retained recovery flow", () => { + let exitSpy: MockInstance; + let originalGatewayEnv: string | undefined; + + beforeEach(() => { + originalGatewayEnv = process.env.OPENSHELL_GATEWAY; + exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { + throw new Error(`process.exit(${code ?? 0})`); + }) as never); + }); + + afterEach(() => { + originalGatewayEnv === undefined + ? delete process.env.OPENSHELL_GATEWAY + : (process.env.OPENSHELL_GATEWAY = originalGatewayEnv); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + resetDestroyModuleCache(); + }); + + it( + "removes every container from one retained failed attempt and clears recovery (#10547)", + { timeout: 30_000 }, + async () => { + const recovery = retainedRecoveryRecord(); + const sandboxContainerId = "a".repeat(64); + const bootstrapContainerId = "b".repeat(64); + const identityRows = [sandboxContainerId, bootstrapContainerId] + .map((id) => `${id}\topenshell\tdefault\tsb-alpha`) + .join("\n"); + const harness = createDestroyHarness({ + dockerOrphanIds: [bootstrapContainerId], + dockerRunResult: { status: 0, stdout: identityRows }, + registryEntryOverrides: { + lifecycleGeneration: recovery.lifecycleGeneration!, + lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!, + }, + retainedRecoveryRecords: [recovery], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.dockerRunSpy).toHaveBeenCalledWith( + ["rm", "-f", bootstrapContainerId], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledWith(recovery); + expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); + expect(exitSpy).not.toHaveBeenCalled(); + }, + ); + + it( + "does not delete a live retained sandbox when Docker identity is absent (#10547)", + { timeout: 30_000 }, + async () => { + const recovery = retainedRecoveryRecord(); + const harness = createDestroyHarness({ + dockerRunResult: { status: 0, stdout: "" }, + registryEntryOverrides: { + lifecycleGeneration: recovery.lifecycleGeneration!, + lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!, + }, + retainedRecoveryRecords: [recovery], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("Docker exposed no container with the retained immutable identity"), + ); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.resolveRetainedSandboxRecoverySpy).not.toHaveBeenCalled(); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + }, + ); + + it( + "finishes retained cleanup after OpenShell already removed the sandbox (#10547)", + { timeout: 30_000 }, + async () => { + const recovery = retainedRecoveryRecord(); + const bootstrapContainerId = "b".repeat(64); + const pendingPolicyVerification = { + schemaVersion: 1 as const, + state: "verified-create" as const, + policyAuthority: "externally-managed" as const, + observedPolicyAuthority: "externally-managed" as const, + gatewayName: recovery.gatewayName, + gatewayPort: recovery.gatewayPort, + sandboxName: recovery.sandboxName, + lifecycleGeneration: recovery.lifecycleGeneration!, + sandboxIdentityFingerprint: recovery.sandboxIdentityFingerprint!, + createAttemptNonce: recovery.createAttemptNonce, + route: "none" as const, + policyHash: "policy-hash", + policyVersion: 1, + }; + const harness = createDestroyHarness({ + sandboxPresent: false, + dockerOrphanIds: [bootstrapContainerId], + dockerRunResult: { + status: 0, + stdout: `${bootstrapContainerId}\topenshell\tdefault\tsb-alpha`, + }, + registryEntryOverrides: { + lifecycleGeneration: recovery.lifecycleGeneration!, + lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!, + pendingPolicyVerification, + }, + retainedRecoveryRecords: [recovery], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.dockerRunSpy).toHaveBeenCalledWith( + ["rm", "-f", bootstrapContainerId], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledWith(recovery); + expect(exitSpy).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 2f1b91d26c..d55eeaf068 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -38,6 +38,7 @@ import { validateName } from "../../runner"; import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import * as onboardSession from "../../state/onboard-session"; +import type { RetainedSandboxRecoveryRecord } from "../../state/onboard-session/retained-sandbox-recovery"; import { resolveNemoclawStateDir } from "../../state/paths"; import * as registry from "../../state/registry"; import { @@ -55,6 +56,7 @@ import { shouldCleanupGatewayAfterConfirmedFinalDestroy } from "./destroy-gatewa import { assertUnambiguousDestroyContainerIdentity, classifyDestroySandboxPresence, + getDestroyContainerIdentities, isSameDestroyContainerIdentityProof, } from "./destroy-presence"; import { @@ -83,6 +85,38 @@ type RemoveSandboxRegistryEntryWithReceiptDeps = { removeSandboxWithReceipt?: typeof registry.removeSandboxWithReceipt; }; +function selectRetainedSandboxRecoveryAuthority( + sandboxName: string, + sandbox: registry.SandboxEntry | null, + records: readonly RetainedSandboxRecoveryRecord[], +): RetainedSandboxRecoveryRecord | null { + const matchesRegistryAuthority = (record: RetainedSandboxRecoveryRecord): boolean => { + if (record.sandboxIdentityFingerprint === null) return false; + if (!sandbox) return true; + const pending = sandbox.pendingPolicyVerification; + if (pending) { + return ( + record.gatewayName === pending.gatewayName && + record.gatewayPort === pending.gatewayPort && + record.lifecycleGeneration === pending.lifecycleGeneration && + record.sandboxIdentityFingerprint === pending.sandboxIdentityFingerprint && + (pending.createAttemptNonce === undefined || + record.createAttemptNonce === pending.createAttemptNonce) + ); + } + return ( + record.gatewayName === sandbox.gatewayName && + record.gatewayPort === sandbox.gatewayPort && + record.lifecycleGeneration === sandbox.lifecycleGeneration && + record.sandboxIdentityFingerprint === sandbox.lifecycleLiveIdentityFingerprint + ); + }; + const matching = records.filter( + (record) => record.sandboxName === sandboxName && matchesRegistryAuthority(record), + ); + return matching.length === 1 ? matching[0]! : null; +} + export type RemoveSandboxRegistryEntryOutcome = | { readonly status: "complete"; @@ -508,6 +542,14 @@ async function destroySandboxUnlocked( const normalized = normalizeDestroySandboxOptions(options); if (!(await confirmSandboxDestroy(sandboxName, normalized))) return; const destroySession = onboardSession.loadSession(); + const registeredSandbox = registry.getSandbox(sandboxName); + const retainedRecoveryAuthority = selectRetainedSandboxRecoveryAuthority( + sandboxName, + registeredSandbox, + onboardSession.listRetainedSandboxRecoveryRecords(), + ); + const retainedSandboxIdentityFingerprint = + retainedRecoveryAuthority?.sandboxIdentityFingerprint ?? undefined; let portableContainerAuthority: ReturnType; try { portableContainerAuthority = preparePortableDemoSandboxDestroyAuthority(sandboxName, () => { @@ -533,13 +575,18 @@ async function destroySandboxUnlocked( registry.getSandbox(sandboxName)?.openshellDriver, ), redact: redactDestroyError, + ...(retainedSandboxIdentityFingerprint + ? { retainedSandboxIdentityFingerprint } + : {}), }); const initialIdentity = portableContainerAuthority ? null : inspectContainerIdentity(); if (initialIdentity === false) { requestSandboxDestroyExit(1); } + const initialContainerIdentities = initialIdentity + ? getDestroyContainerIdentities(initialIdentity) + : undefined; - const registeredSandbox = registry.getSandbox(sandboxName); let preparedManagedLlamaCppCleanup: ReturnType< typeof prepareManagedLlamaCppRuntimeCleanupForSandbox > = null; @@ -594,6 +641,17 @@ async function destroySandboxUnlocked( let destroyPreflight: ReturnType; destroyPreflight = abortPreparedCleanupOnError(() => prepareSandboxDestroy(sandboxName)); const { cleanupGatewayName, runOpenshell, sandbox, sandboxConfirmedAbsent } = destroyPreflight; + if ( + retainedRecoveryAuthority && + initialContainerIdentities?.length === 0 && + !sandboxConfirmedAbsent + ) { + console.error( + ` Refusing to destroy retained sandbox '${sandboxName}': OpenShell still reports it present, but Docker exposed no container with the retained immutable identity. No sandbox resources were removed.`, + ); + preparedManagedLlamaCppCleanup?.abort(); + requestSandboxDestroyExit(1); + } // Recheck identity after pre-delete qualification and recoverable journal // publication reconciliation, before any sandbox runtime mutation. if (portableContainerAuthority) { @@ -635,7 +693,10 @@ async function destroySandboxUnlocked( sandbox, sandboxConfirmedAbsent, sandboxName, - expectedContainerIdentity: initialIdentity?.identity, + expectedContainerIdentities: initialContainerIdentities, + ...(retainedSandboxIdentityFingerprint + ? { expectedContainerIdentityFingerprint: retainedSandboxIdentityFingerprint } + : {}), ...(portableContainerAuthority ? { portableContainerAuthority } : {}), stopInferenceResources: () => stopSandboxInferenceResources(sandboxName, sandbox), }); @@ -898,7 +959,11 @@ async function destroySandboxUnlocked( ); } } - if (!routedSessionCleanupHandled && destroySession?.sandboxName === sandboxName) { + if ( + !retainedRecoveryAuthority && + !routedSessionCleanupHandled && + destroySession?.sandboxName === sandboxName + ) { const cleanupResult = onboardSession.compareAndSwapSession( (current) => current.sessionId === destroySession.sessionId && @@ -919,6 +984,20 @@ async function destroySandboxUnlocked( ); } } + if ( + deleteSucceededOrAlreadyGone && + retainedRecoveryAuthority + ) { + try { + onboardSession.resolveRetainedSandboxRecovery(retainedRecoveryAuthority); + } catch (error) { + console.error( + ` Sandbox '${sandboxName}' resources are gone, but NemoClaw could not clear its retained recovery record: ${redactDestroyError(error)}`, + ); + console.error(` Re-run '${CLI_NAME} ${sandboxName} destroy --yes' to finish local cleanup.`); + requestSandboxDestroyExit(1); + } + } if ( shouldCleanupGatewayAfterConfirmedFinalDestroy({ deleteSucceededOrAlreadyGone, diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 6fd504321c..7831ccba24 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -34,7 +34,7 @@ describe("createSandboxCancelRollback", () => { expect(guidance).toContain("Shared inference providers are gateway configuration"); expect(guidance).toContain("not sandbox cleanup targets"); expect(guidance).toContain("sandbox-scoped resources whose ownership is confirmed"); - expect(guidance).toContain("no supported operation to clear this recovery record"); + expect(guidance).toContain("nemoclaw new-sb destroy"); expect(guidance).toContain("credential environment name alone does not prove exposure"); expect(guidance).toContain("rotate a credential only when identity-bound inspection proves"); expect(guidance).not.toContain("rotate any credential"); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index f2d72f81af..be65ff4581 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { RetainedSandboxRecoveryContext } from "../state/onboard-session"; +import { cliName } from "./branding"; // Re-exported so the onboard entrypoint imports its sandbox default/cancel // lifecycle helpers from a single module. @@ -73,7 +74,13 @@ export function buildCancelRollbackMessage( " Sandbox-scoped provider registrations or gateway-bound credentials may remain when the durable recovery record lists them.", " Ask an OpenShell administrator to inspect the exact sandbox identity and remove only sandbox-scoped resources whose ownership is confirmed for this retained sandbox.", " A recorded credential environment name alone does not prove exposure; rotate a credential only when identity-bound inspection proves that it was exposed or attached to a retained sandbox-scoped resource.", - " NemoClaw has no supported operation to clear this recovery record; use a different sandbox name for later onboarding.", + ...(sandboxIdentityFingerprint + ? [ + ` Run '${cliName()} ${sandboxName} destroy' to verify and remove this failed attempt, then start fresh onboarding.`, + ] + : [ + " NemoClaw cannot clear this recovery record until an OpenShell administrator establishes the exact sandbox identity.", + ]), ]; } diff --git a/src/lib/onboard/entry-options.test.ts b/src/lib/onboard/entry-options.test.ts index ae40e6f3bc..fe8b7da9b2 100644 --- a/src/lib/onboard/entry-options.test.ts +++ b/src/lib/onboard/entry-options.test.ts @@ -290,6 +290,30 @@ describe("resolveOnboardEntryOptions", () => { expect(deps.error).not.toHaveBeenCalled(); }); + it("treats an explicit different name as fresh while recovery remains isolated (#10547)", () => { + const deps = createDeps(); + + const result = resolveOnboardEntryOptions( + { + opts: { sandboxName: "replacement-sb" }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "recovery_required", + persistedRecoverySandboxName: "retained-sb", + retainedRecoverySandboxNames: ["retained-sb"], + }, + deps, + ); + + expect(result).toMatchObject({ + fresh: true, + resume: false, + requestedSandboxName: "replacement-sb", + }); + expect(deps.error).not.toHaveBeenCalled(); + }); + it("rejects a different fresh name when recovery has no independent durable record", () => { const deps = createDeps(); diff --git a/src/lib/onboard/entry-options.ts b/src/lib/onboard/entry-options.ts index b8f707b0ca..6d2c98a796 100644 --- a/src/lib/onboard/entry-options.ts +++ b/src/lib/onboard/entry-options.ts @@ -282,7 +282,7 @@ export function resolveOnboardEntryOptions( deps: OnboardEntryOptionsDeps, ): ResolvedOnboardEntryOptions { const explicitResume = input.opts.resume === true; - const fresh = input.opts.fresh === true; + let fresh = input.opts.fresh === true; // The mutual-exclusion error applies only to the explicit flags — a leftover // in_progress session combined with an explicit `--fresh` is not a conflict // (fresh wins, see below), so it must not trip this guard. @@ -348,7 +348,7 @@ export function resolveOnboardEntryOptions( " Onboarding cannot continue while a retained sandbox recovery record is unresolved without an explicit different sandbox name.", ); deps.error( - " Use --fresh --name ; the retained sandbox recovery record stays unresolved.", + " Use --name ; the retained sandbox recovery record stays unresolved.", ); deps.exitProcess(1); } @@ -357,13 +357,20 @@ export function resolveOnboardEntryOptions( ` Onboarding cannot use retained sandbox '${recoveryEntryName}' while its identity-bound recovery record is unresolved.`, ); deps.error( - " Automatic and explicit resume, reuse, recreation, and same-name fresh onboarding remain disabled; NemoClaw has no supported operation to clear this recovery record.", + ` Run the destroy command for retained sandbox '${recoveryEntryName}' to remove the verified failed attempt; resume, reuse, recreation, and same-name fresh onboarding remain disabled until destroy completes.`, ); deps.exitProcess(1); } } if (input.persistedSessionStatus === "recovery_required") { const recoverySandboxName = input.persistedRecoverySandboxName?.trim() || null; + const canStartDifferentSandbox = + !explicitResume && + recoverySandboxName !== null && + requestedSandboxName !== null && + requestedSandboxName !== recoverySandboxName && + retainedRecoverySandboxNames.has(recoverySandboxName); + if (!fresh && canStartDifferentSandbox) fresh = true; if (!fresh) { deps.error( ` Onboarding cannot continue because cancellation preserved sandbox '${recoverySandboxName ?? "unknown"}' in recovery-only state.`, @@ -372,7 +379,7 @@ export function resolveOnboardEntryOptions( " Automatic and explicit resume, reuse, and recreation are disabled to protect the retained sandbox.", ); deps.error( - " Use --fresh --name ; the retained sandbox recovery record stays unresolved.", + " Use --name to start another sandbox. The retained sandbox recovery record stays unresolved.", ); deps.exitProcess(1); } diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 8fa6669f48..c9059a8bc5 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw leaves the sandbox running because the supported delete command targets its mutable name. Operators must preserve the durable sandbox identity fingerprint from the failure output and provide it to the OpenShell administrator. They must not delete the sandbox by name, even after comparing its identity. Contact the administrator for an identity-bound recovery or removal procedure. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when every immutable sandbox ID matches that fingerprint. It snapshots the matching container IDs, revalidates the set before deletion, removes only remaining members of that set, verifies their absence, and clears the matching recovery record. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. ## Effect-order flows @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because NemoClaw refuses the available mutable-name deletion command when it could target a replacement. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Cancellation after creation preserves the incomplete session, registry row, and an independent identity-bound recovery record. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Recovery removes only confirmed sandbox-scoped resources and rotates credentials only when inspection proves exposure or attachment to a retained resource; a recorded environment-variable name alone is not exposure evidence. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` uses that record to qualify immutable Docker container identities, complete interrupted cleanup, and retire the record. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Recovery removes only confirmed sandbox-scoped resources and rotates credentials only when inspection proves exposure or attachment to a retained resource; a recorded environment-variable name alone is not exposure evidence. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts index df081ac8be..fc6a11a5e7 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { queryOpenShellDockerSandboxRuntimeSnapshot, removeExactOpenShellDockerSandboxContainer, + removeExactOpenShellDockerSandboxContainers, } from "./openshell-docker-sandbox-containers"; const IMAGE_ID = `sha256:${"a".repeat(64)}`; @@ -31,6 +32,72 @@ describe("removeExactOpenShellDockerSandboxContainer", () => { expect(forceRemove).toHaveBeenCalledWith(expectedContainerId); }); + + it("removes every remaining container from one exact failed attempt (#10547)", () => { + const expectedContainerIds = ["a".repeat(64), "b".repeat(64)]; + let currentContainerIds = [...expectedContainerIds]; + const queryContainers = vi.fn(() => ({ ok: true as const, ids: [...currentContainerIds] })); + const forceRemove = vi.fn((containerId: string) => { + currentContainerIds = currentContainerIds.filter((candidate) => candidate !== containerId); + return { status: 0 }; + }); + + removeExactOpenShellDockerSandboxContainers( + "alpha", + expectedContainerIds, + vi.fn(), + { queryContainers, forceRemove }, + ); + + expect(forceRemove.mock.calls.map(([containerId]) => containerId)).toEqual( + expectedContainerIds, + ); + expect(currentContainerIds).toEqual([]); + }); + + it("continues cleanup when an earlier exact container is already absent (#10547)", () => { + const alreadyRemovedId = "a".repeat(64); + const remainingId = "b".repeat(64); + let currentContainerIds = [remainingId]; + const queryContainers = vi.fn(() => ({ ok: true as const, ids: [...currentContainerIds] })); + const forceRemove = vi.fn((containerId: string) => { + currentContainerIds = currentContainerIds.filter((candidate) => candidate !== containerId); + return { status: 0 }; + }); + + removeExactOpenShellDockerSandboxContainers( + "alpha", + [alreadyRemovedId, remainingId], + vi.fn(), + { queryContainers, forceRemove }, + ); + + expect(forceRemove).toHaveBeenCalledExactlyOnceWith(remainingId); + expect(currentContainerIds).toEqual([]); + }); + + it("does not remove a container outside the retained identity set (#10547)", () => { + const expectedContainerId = "a".repeat(64); + const replacementContainerId = "b".repeat(64); + const forceRemove = vi.fn(() => ({ status: 0 })); + + expect(() => + removeExactOpenShellDockerSandboxContainers( + "alpha", + [expectedContainerId], + vi.fn(), + { + queryContainers: vi.fn(() => ({ + ok: true as const, + ids: [replacementContainerId], + })), + forceRemove, + }, + ), + ).toThrow("refusing replacement cleanup"); + + expect(forceRemove).not.toHaveBeenCalled(); + }); }); function querySnapshot(fields: unknown, nvidiaVisibleDevices?: string) { diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index b5d6cb3866..3d6a7133d1 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -105,41 +105,63 @@ type StaleDockerOrphanCleanupDeps = { forceRemove?: (containerId: string) => { status?: number | null }; }; -/** Remove only the Docker container whose immutable ID passed the caller's authority check. */ -export function removeExactOpenShellDockerSandboxContainer( +/** Remove the remaining members of one immutable, prequalified Docker container set. */ +export function removeExactOpenShellDockerSandboxContainers( sandboxName: string, - expectedContainerId: string, + expectedContainerIds: readonly string[], log: (message: string) => void, deps: StaleDockerOrphanCleanupDeps = {}, ): void { + const expected = new Set(expectedContainerIds); + if ( + expected.size !== expectedContainerIds.length || + expectedContainerIds.some((containerId) => !/^[0-9a-f]{12,64}$/iu.test(containerId)) + ) { + throw new Error("exact Docker cleanup contains an invalid or duplicate container identity"); + } + const queryContainers = deps.queryContainers ?? queryOpenShellDockerSandboxContainers; const initial = queryContainers(sandboxName); if (!initial.ok) { throw new Error(`could not inspect the exact Docker cleanup target: ${initial.error}`); } - if (initial.ids.length === 0) return; - if (initial.ids.length !== 1 || initial.ids[0] !== expectedContainerId) { + const unexpected = initial.ids.filter((containerId) => !expected.has(containerId)); + if (unexpected.length > 0) { throw new Error( - `expected exactly labeled Docker container '${expectedContainerId}', found ` + - `${initial.ids.length === 0 ? "none" : initial.ids.join(", ")}; refusing replacement cleanup`, + `found labeled Docker container(s) outside the retained identity set: ${unexpected.join(", ")}; refusing replacement cleanup`, ); } - const removal = deps.forceRemove - ? deps.forceRemove(expectedContainerId) - : dockerRun(["rm", "-f", expectedContainerId], { - ignoreError: true, - suppressOutput: true, - timeout: STALE_DOCKER_ORPHAN_TIMEOUT_MS, - }); - if (Number(removal.status ?? 1) !== 0) { - throw new Error(`could not remove exact Docker container '${expectedContainerId}'`); + const remaining = new Set(initial.ids); + for (const containerId of expectedContainerIds) { + if (!remaining.has(containerId)) continue; + const removal = deps.forceRemove + ? deps.forceRemove(containerId) + : dockerRun(["rm", "-f", containerId], { + ignoreError: true, + suppressOutput: true, + timeout: STALE_DOCKER_ORPHAN_TIMEOUT_MS, + }); + if (Number(removal.status ?? 1) !== 0) { + throw new Error(`could not remove exact Docker container '${containerId}'`); + } + log(`Removed exact Docker container '${containerId}' after OpenShell sandbox deletion`); } + const confirmed = queryContainers(sandboxName); if (!confirmed.ok || confirmed.ids.length !== 0) { throw new Error("could not confirm exact Docker container removal"); } - log(`Removed exact Docker container '${expectedContainerId}' after OpenShell sandbox deletion`); +} + +/** Remove only the Docker container whose immutable ID passed the caller's authority check. */ +export function removeExactOpenShellDockerSandboxContainer( + sandboxName: string, + expectedContainerId: string, + log: (message: string) => void, + deps: StaleDockerOrphanCleanupDeps = {}, +): void { + removeExactOpenShellDockerSandboxContainers(sandboxName, [expectedContainerId], log, deps); } /** diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index f29693d319..c570b890f7 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -33,6 +33,7 @@ import type { } from "../managed-workload/hermes-state-volume"; import type { OwnedSandboxRecreateRuntime } from "../onboard-recreate-journal"; import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import { cliName } from "../branding"; import type { CreatedSandboxLifecycle, CreatedSandboxLifecycleRegistration, @@ -711,7 +712,8 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< const recoveryGuidance = `NemoClaw left sandbox '${input.sandboxName}' in place after post-create verification or finalization failed. ` + `${identityGuidance} NemoClaw did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. ` + - "Do not delete the sandbox by mutable sandbox name. Ask the OpenShell administrator to inspect the surviving sandbox and use an identity-bound recovery or removal procedure."; + `Do not delete the sandbox by mutable sandbox name. Run '${cliName()} ${input.sandboxName} destroy' to use the retained identity. ` + + "If destroy cannot prove that identity, stop. Ask the OpenShell administrator to inspect the surviving sandbox and use an identity-bound recovery or removal procedure."; const compensationErrors: unknown[] = []; if (input.persistRetainedSandboxRecovery) { try { diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 01ba28aace..f0d8dd56b0 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -10,6 +10,7 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; import type { SandboxPolicyAuthority } from "../adapters/openshell/policy-authority"; import { isErrnoException } from "../core/errno"; @@ -65,6 +66,7 @@ import { listRetainedSandboxRecoveryRecords as readRetainedSandboxRecoveryRecords, parseNemoClawPolicyCreationReceipt, recordRetainedSandboxRecovery as writeRetainedSandboxRecovery, + resolveRetainedSandboxRecovery as retireRetainedSandboxRecovery, retainedSandboxRecoveryFile, type RecordRetainedSandboxRecoveryInput, type RetainedSandboxRecoveryRecord, @@ -2087,6 +2089,44 @@ export function recordRetainedSandboxRecovery( ); } +function recoveryRecordMatchesSession( + record: RetainedSandboxRecoveryRecord, + recovery: SessionCancellationRecovery, +): boolean { + return ( + record.sandboxName === recovery.sandboxName && + record.sandboxIdentityFingerprint === recovery.sandboxIdentityFingerprint && + record.gatewayName === recovery.gatewayName && + record.gatewayPort === recovery.gatewayPort && + record.lifecycleGeneration === recovery.lifecycleGeneration && + record.createAttemptNonce === recovery.createAttemptNonce + ); +} + +/** Clear one recovery-only session after destroy verifies the retained resources absent. */ +export function resolveRetainedSandboxRecovery(record: RetainedSandboxRecoveryRecord): boolean { + return withOwnedOnboardLock("nemoclaw retained sandbox recovery completion", () => { + const recorded = readRetainedSandboxRecoveryRecords(RETAINED_SANDBOX_RECOVERY_FILE).find( + (candidate) => candidate.recordId === record.recordId, + ); + if (recorded && !isDeepStrictEqual(recorded, record)) { + throw new Error("Retained sandbox recovery authority changed before cleanup completed."); + } + const current = loadSession(); + if ( + current?.cancellationRecovery && + recoveryRecordMatchesSession(record, current.cancellationRecovery) + ) { + current.status = "failed"; + current.resumable = false; + current.sandboxName = null; + current.cancellationRecovery = null; + saveSession(current); + } + return retireRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, record); + }); +} + export function markCancellationRecovery( sandboxName: string, sandboxIdentityFingerprint: string | undefined, diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index fced883d05..25af2775db 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -4,6 +4,7 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; import { openRegularFileNoFollow } from "../../adapters/fs/regular-file"; import { @@ -546,3 +547,24 @@ export function recordRetainedSandboxRecovery( } return reread; } + +/** Retire only the unchanged record whose external resources were verified absent. */ +export function resolveRetainedSandboxRecovery( + filePath: string, + expected: RetainedSandboxRecoveryRecord, +): boolean { + const current = loadState(filePath); + const recorded = current.unresolved.find((candidate) => candidate.recordId === expected.recordId); + if (!recorded) return false; + if (!isDeepStrictEqual(recorded, expected)) { + throw new Error("Retained sandbox recovery authority changed before cleanup completed."); + } + writeStateFile(filePath, { + ...current, + unresolved: current.unresolved.filter((candidate) => candidate.recordId !== expected.recordId), + }); + if (loadState(filePath).unresolved.some((candidate) => candidate.recordId === expected.recordId)) { + throw new Error("Retained sandbox recovery record remained after verified cleanup."); + } + return true; +} diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index 6fd70af1d2..e9d9216148 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -245,9 +245,8 @@ describe("retained sandbox recovery state", () => { ).toEqual([]); }); - it("does not expose a caller-supplied recovery resolution path (#9833)", async () => { + it("retires only the exact retained recovery record after verified cleanup (#10547)", async () => { const recovery = await import("./onboard-session"); - const recoveryStore = await import("./onboard-session/retained-sandbox-recovery"); const fingerprint = "b".repeat(64); const recorded = recovery.recordRetainedSandboxRecovery({ sandboxName: "retained-sb", @@ -260,24 +259,55 @@ describe("retained sandbox recovery state", () => { resources: evidence, reason: "cancelled_after_sandbox_creation", }); - const unsupportedClear = (recovery as unknown as Record)[ - "resolveRetainedSandboxRecovery" - ]; - (unsupportedClear as undefined | ((input: Record) => unknown))?.({ - recordId: recorded.recordId, - receiptId: "c".repeat(64), - sandboxName: recorded.sandboxName, - sandboxIdentityFingerprint: fingerprint, - gatewayName: recorded.gatewayName, - gatewayPort: recorded.gatewayPort, - outcome: "removed_verified_identity", - }); - expect(unsupportedClear).toBeUndefined(); - expect( - (recoveryStore as unknown as Record)["resolveRetainedSandboxRecovery"], - ).toBeUndefined(); + expect(() => + recovery.resolveRetainedSandboxRecovery({ + ...recorded, + sandboxIdentityFingerprint: "d".repeat(64), + }), + ).toThrow(/changed before cleanup completed/u); expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([recorded]); + + expect(recovery.resolveRetainedSandboxRecovery(recorded)).toBe(true); + expect(recovery.resolveRetainedSandboxRecovery(recorded)).toBe(false); + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([]); }); + it("releases the matching recovery-only onboarding session after cleanup (#10547)", async () => { + const recovery = await import("./onboard-session"); + recovery.markRetainedSandboxRecovery( + "retained-sb", + "Sandbox creation failed after identity verification.", + "b".repeat(64), + { + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, + ...recoveryAuthority, + }, + ); + const [recorded] = recovery.listRetainedSandboxRecoveryRecords(); + + expect(() => + recovery.resolveRetainedSandboxRecovery({ + ...recorded!, + reason: "cancelled_after_sandbox_creation", + }), + ).toThrow(/changed before cleanup completed/u); + expect(recovery.loadSession()).toMatchObject({ + status: "recovery_required", + sandboxName: "retained-sb", + }); + + expect(recovery.resolveRetainedSandboxRecovery(recorded!)).toBe(true); + + expect(recovery.loadSession()).toMatchObject({ + status: "failed", + resumable: false, + sandboxName: null, + cancellationRecovery: null, + }); + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([]); + }); }); diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 2c2ce3e4e8..3c0e6abdee 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -8,6 +8,7 @@ import type { SandboxDestroyExecutionResult } from "../../src/lib/actions/sandbo import type { PreparedManagedLlamaCppRuntimeCleanup } from "../../src/lib/inference/local-model-profile/cleanup"; import type { ManagedHermesStateVolumeCleanupResult } from "../../src/lib/onboard/managed-workload/hermes-state-volume"; import type { Session } from "../../src/lib/state/onboard-session"; +import type { RetainedSandboxRecoveryRecord } from "../../src/lib/state/onboard-session/retained-sandbox-recovery"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../src/lib/state/registry"; type DestroySandbox = (typeof import("../../src/lib/actions/sandbox/destroy"))["destroySandbox"]; @@ -44,6 +45,7 @@ export type DestroyHarness = { promptSpy: MockInstance; removeManagedHermesStateVolumeSpy: MockInstance; removeSandboxSpy: MockInstance; + resolveRetainedSandboxRecoverySpy: MockInstance; retirePortableLifecycleReceiptSpy: MockInstance; portableDestroyRevalidateSpy: MockInstance; portableDestroyVerifyAbsentSpy: MockInstance; @@ -105,7 +107,9 @@ type DestroyHarnessOptions = { promptResponses?: string[]; provider?: string; registryEntryPresent?: boolean; + registryEntryOverrides?: Partial; registeredSandboxCount?: number; + retainedRecoveryRecords?: RetainedSandboxRecoveryRecord[]; replaceSessionAfterRegistryRemoval?: boolean; removeSandboxResult?: boolean; restoreMcpError?: string; @@ -319,6 +323,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }, } : {}), + ...options.registryEntryOverrides, }, ); let registeredSandboxCount = options.registeredSandboxCount ?? 0; @@ -370,6 +375,12 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }); vi.spyOn(modelRouterProcess, "isRouterHealthy").mockResolvedValue(false); vi.spyOn(onboardSession, "loadSession").mockImplementation(() => ({ ...sessionState })); + vi.spyOn(onboardSession, "listRetainedSandboxRecoveryRecords").mockReturnValue( + options.retainedRecoveryRecords ?? [], + ); + const resolveRetainedSandboxRecoverySpy = vi + .spyOn(onboardSession, "resolveRetainedSandboxRecovery") + .mockReturnValue(true); vi.spyOn(onboardSession, "acquireOnboardLock").mockImplementation(() => sessionLockBusy ? { @@ -628,6 +639,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr promptSpy, removeManagedHermesStateVolumeSpy, removeSandboxSpy, + resolveRetainedSandboxRecoverySpy, retirePortableLifecycleReceiptSpy, revokeHttpsPinRuntimeAdapterRouteSpy, restoreMcpBridgesAfterDestroyAbortSpy, diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 05a3e23c2c..a2ef55a596 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -982,7 +982,7 @@ if (${JSON.stringify( assert.match(result.stderr, /Shared inference providers are gateway configuration/u); assert.match(result.stderr, /not sandbox cleanup targets/u); assert.match(result.stderr, /sandbox-scoped resources whose ownership is confirmed/u); - assert.match(result.stderr, /no supported operation to clear this recovery record/u); + assert.match(result.stderr, /nemoclaw my-assistant destroy/u); assert.match(result.stderr, /credential environment name alone does not prove exposure/u); assert.match( result.stderr, From d88e1303f3bb72faffe7b3eed6e5045632c5bb1e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 13:30:01 -0700 Subject: [PATCH 02/24] fix(onboard): harden retained cleanup reconciliation (#10547) Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/destroy-presence.ts | 10 -- .../destroy-retained-recovery-flow.test.ts | 59 ++++++++++ src/lib/actions/sandbox/destroy.ts | 8 +- src/lib/onboard/lifecycle-contracts.md | 2 +- .../openshell-docker-sandbox-containers.ts | 98 +++++++++------- src/lib/state/onboard-session.ts | 22 ++-- .../state/retained-sandbox-recovery.test.ts | 29 +++++ test/helpers/destroy-flow-test-harness.ts | 107 ++++++++++++------ 8 files changed, 235 insertions(+), 100 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts index 10cbfe51b9..00a60daead 100644 --- a/src/lib/actions/sandbox/destroy-presence.ts +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -6,7 +6,6 @@ import { OPENSHELL_MANAGED_BY_VALUE, OPENSHELL_SANDBOX_ID_LABEL, OPENSHELL_SANDBOX_NAME_LABEL, - removeExactOpenShellDockerSandboxContainer, removeExactOpenShellDockerSandboxContainers, } from "../../onboard/openshell-docker-sandbox-containers"; import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; @@ -87,15 +86,6 @@ export function observeDestroyContainerIdentity( return observeDockerSandboxIdentities(sandboxName); } -/** Retire only the provider runtime bound to the pre-destroy identity proof. */ -export function removeExactDestroyContainerIdentity( - sandboxName: string, - expectedIdentity: SandboxNameLabeledContainer, - log: (message: string) => void, -): void { - removeExactOpenShellDockerSandboxContainer(sandboxName, expectedIdentity.id, log); -} - /** Retire the exact container set qualified from one retained recovery fingerprint. */ export function removeExactDestroyContainerIdentities( sandboxName: string, diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index 6a47fc4f41..9995022733 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -86,7 +86,66 @@ describe("destroySandbox retained recovery flow", () => { ); expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledWith(recovery); expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); + expect(harness.sessionState.sandboxName).toBeNull(); expect(exitSpy).not.toHaveBeenCalled(); + + const exactRemovalCallsAfterCleanup = harness.dockerRunSpy.mock.calls.filter( + ([args]) => Array.isArray(args) && args[0] === "rm" && args[1] === "-f", + ).length; + harness.setSandboxPresent(false); + harness.setRegistryEntryPresent(false); + harness.setRetainedRecoveryRecords([]); + harness.setDockerIdentityResult({ status: 0, stdout: "" }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect( + harness.dockerRunSpy.mock.calls.filter( + ([args]) => Array.isArray(args) && args[0] === "rm" && args[1] === "-f", + ), + ).toHaveLength(exactRemovalCallsAfterCleanup); + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledOnce(); + }, + ); + + it( + "preserves recovery when a foreign name-labeled container appears after continuity checks (#10547)", + { timeout: 30_000 }, + async () => { + const recovery = retainedRecoveryRecord(); + const sandboxContainerId = "a".repeat(64); + const bootstrapContainerId = "b".repeat(64); + const foreignContainerId = "e".repeat(64); + const identityRows = [sandboxContainerId, bootstrapContainerId] + .map((id) => `${id}\topenshell\tdefault\tsb-alpha`) + .join("\n"); + const harness = createDestroyHarness({ + dockerNameLabeledIds: [bootstrapContainerId, foreignContainerId], + dockerOrphanIds: [bootstrapContainerId], + dockerRunResult: { status: 0, stdout: identityRows }, + registryEntryOverrides: { + lifecycleGeneration: recovery.lifecycleGeneration!, + lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!, + }, + retainedRecoveryRecords: [recovery], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.dockerRunSpy).not.toHaveBeenCalledWith( + ["rm", "-f", expect.any(String)], + expect.anything(), + ); + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("outside the retained identity set"), + ); + expect(harness.resolveRetainedSandboxRecoverySpy).not.toHaveBeenCalled(); }, ); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index d55eeaf068..1c453dd7a6 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -959,8 +959,14 @@ async function destroySandboxUnlocked( ); } } + const retainedRecoveryOwnsDestroySession = retainedRecoveryAuthority + ? onboardSession.retainedSandboxRecoveryMatchesSession( + retainedRecoveryAuthority, + destroySession, + ) + : false; if ( - !retainedRecoveryAuthority && + !retainedRecoveryOwnsDestroySession && !routedSessionCleanupHandled && destroySession?.sandboxName === sandboxName ) { diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index c9059a8bc5..a3ac359bd9 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when every immutable sandbox ID matches that fingerprint. It snapshots the matching container IDs, revalidates the set before deletion, removes only remaining members of that set, verifies their absence, and clears the matching recovery record. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set before deletion, removes only remaining members of that set, verifies their absence, and clears the matching recovery record. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. ## Effect-order flows diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index 3d6a7133d1..d84d3c98a9 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -15,16 +15,16 @@ const STALE_DOCKER_ORPHAN_TIMEOUT_MS = 30_000; type DockerSandboxContainerQueryDeps = Pick; -function sandboxContainerFilterArgs(sandboxName: string, sandboxNamespace?: string): string[] { - const args = [ - "ps", - "-a", - "--no-trunc", - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--filter", - `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, - ]; +function sandboxContainerFilterArgs( + sandboxName: string, + sandboxNamespace?: string, + requireManagedBy = true, +): string[] { + const args = ["ps", "-a", "--no-trunc"]; + if (requireManagedBy) { + args.push("--filter", `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`); + } + args.push("--filter", `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`); if (sandboxNamespace !== undefined) { args.push("--filter", `label=${OPENSHELL_SANDBOX_NAMESPACE_LABEL}=${sandboxNamespace}`); } @@ -58,34 +58,21 @@ export type OpenShellDockerSandboxContainerQuery = | { ok: true; ids: string[] } | { ok: false; ids: []; error: string }; -/** - * Status-bearing lookup used when an empty container list is a safety proof. - * Unlike the best-effort discovery helper, this distinguishes Docker failure - * from a successful query with zero labeled matches. - */ -export function queryOpenShellDockerSandboxContainers( - sandboxName: string, - deps: DockerSandboxContainerQueryDeps = {}, - timeoutMs: number = DOCKER_SANDBOX_QUERY_TIMEOUT_MS, - sandboxNamespace?: string, +function queryDockerSandboxContainerIds( + filterArgs: readonly string[], + deps: DockerSandboxContainerQueryDeps, + timeoutMs: number, ): OpenShellDockerSandboxContainerQuery { const run = deps.dockerRun ?? dockerRun; const requestedTimeoutMs = Number.isFinite(timeoutMs) && timeoutMs > 0 ? Math.floor(timeoutMs) : DOCKER_SANDBOX_QUERY_TIMEOUT_MS; - const boundedTimeoutMs = Math.max( - 1, - Math.min(DOCKER_SANDBOX_QUERY_TIMEOUT_MS, requestedTimeoutMs), - ); - const result = run( - [...sandboxContainerFilterArgs(sandboxName, sandboxNamespace), "--format", "{{.ID}}"], - { - ignoreError: true, - suppressOutput: true, - timeout: boundedTimeoutMs, - }, - ); + const result = run([...filterArgs, "--format", "{{.ID}}"], { + ignoreError: true, + suppressOutput: true, + timeout: Math.max(1, Math.min(DOCKER_SANDBOX_QUERY_TIMEOUT_MS, requestedTimeoutMs)), + }); if (Number(result.status ?? 1) !== 0) { return { ok: false, @@ -93,11 +80,43 @@ export function queryOpenShellDockerSandboxContainers( error: commandResultText(result) || "docker ps did not complete successfully", }; } - const ids = String(result.stdout ?? "") - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - return { ok: true, ids }; + return { + ok: true, + ids: String(result.stdout ?? "") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean), + }; +} + +/** + * Status-bearing lookup used when an empty container list is a safety proof. + * Unlike the best-effort discovery helper, this distinguishes Docker failure + * from a successful query with zero labeled matches. + */ +export function queryOpenShellDockerSandboxContainers( + sandboxName: string, + deps: DockerSandboxContainerQueryDeps = {}, + timeoutMs: number = DOCKER_SANDBOX_QUERY_TIMEOUT_MS, + sandboxNamespace?: string, +): OpenShellDockerSandboxContainerQuery { + return queryDockerSandboxContainerIds( + sandboxContainerFilterArgs(sandboxName, sandboxNamespace), + deps, + timeoutMs, + ); +} + +function queryDockerSandboxNameLabeledContainers( + sandboxName: string, + deps: DockerSandboxContainerQueryDeps = {}, + timeoutMs: number = DOCKER_SANDBOX_QUERY_TIMEOUT_MS, +): OpenShellDockerSandboxContainerQuery { + return queryDockerSandboxContainerIds( + sandboxContainerFilterArgs(sandboxName, undefined, false), + deps, + timeoutMs, + ); } type StaleDockerOrphanCleanupDeps = { @@ -120,7 +139,10 @@ export function removeExactOpenShellDockerSandboxContainers( throw new Error("exact Docker cleanup contains an invalid or duplicate container identity"); } - const queryContainers = deps.queryContainers ?? queryOpenShellDockerSandboxContainers; + // Recovery retirement requires the mutable name itself to be unambiguous. + // Inspect every name-labeled container, including foreign containers that do + // not carry OpenShell's managed-by marker, while removing only qualified IDs. + const queryContainers = deps.queryContainers ?? queryDockerSandboxNameLabeledContainers; const initial = queryContainers(sandboxName); if (!initial.ok) { throw new Error(`could not inspect the exact Docker cleanup target: ${initial.error}`); diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index f0d8dd56b0..c6de8e6b00 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -10,7 +10,6 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { isDeepStrictEqual } from "node:util"; import type { SandboxPolicyAuthority } from "../adapters/openshell/policy-authority"; import { isErrnoException } from "../core/errno"; @@ -2089,10 +2088,12 @@ export function recordRetainedSandboxRecovery( ); } -function recoveryRecordMatchesSession( +export function retainedSandboxRecoveryMatchesSession( record: RetainedSandboxRecoveryRecord, - recovery: SessionCancellationRecovery, + session: Pick | null | undefined, ): boolean { + const recovery = session?.cancellationRecovery; + if (!recovery) return false; return ( record.sandboxName === recovery.sandboxName && record.sandboxIdentityFingerprint === recovery.sandboxIdentityFingerprint && @@ -2106,24 +2107,17 @@ function recoveryRecordMatchesSession( /** Clear one recovery-only session after destroy verifies the retained resources absent. */ export function resolveRetainedSandboxRecovery(record: RetainedSandboxRecoveryRecord): boolean { return withOwnedOnboardLock("nemoclaw retained sandbox recovery completion", () => { - const recorded = readRetainedSandboxRecoveryRecords(RETAINED_SANDBOX_RECOVERY_FILE).find( - (candidate) => candidate.recordId === record.recordId, - ); - if (recorded && !isDeepStrictEqual(recorded, record)) { - throw new Error("Retained sandbox recovery authority changed before cleanup completed."); - } + const retired = retireRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, record); + if (!retired) return false; const current = loadSession(); - if ( - current?.cancellationRecovery && - recoveryRecordMatchesSession(record, current.cancellationRecovery) - ) { + if (current && retainedSandboxRecoveryMatchesSession(record, current)) { current.status = "failed"; current.resumable = false; current.sandboxName = null; current.cancellationRecovery = null; saveSession(current); } - return retireRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, record); + return true; }); } diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index e9d9216148..df37acb892 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -310,4 +310,33 @@ describe("retained sandbox recovery state", () => { }); expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([]); }); + + it("keeps the recovery-only session when record retirement cannot be written (#10547)", async () => { + const recovery = await import("./onboard-session"); + recovery.markRetainedSandboxRecovery( + "retained-sb", + "Sandbox creation failed after identity verification.", + "b".repeat(64), + { + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, + ...recoveryAuthority, + }, + ); + const [recorded] = recovery.listRetainedSandboxRecoveryRecords(); + vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { + throw new Error("simulated recovery retirement write failure"); + }); + + expect(() => recovery.resolveRetainedSandboxRecovery(recorded!)).toThrow( + /simulated recovery retirement write failure/u, + ); + expect(recovery.loadSession()).toMatchObject({ + status: "recovery_required", + sandboxName: "retained-sb", + }); + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([recorded]); + }); }); diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 3c0e6abdee..2958dfaf22 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -54,6 +54,13 @@ export type DestroyHarness = { runOpenshellSpy: MockInstance; selectGatewaySpy: MockInstance; sessionState: Session; + setDockerIdentityResult: (result: { + status: number | null; + stdout?: string; + stderr?: string; + }) => void; + setRegistryEntryPresent: (present: boolean) => void; + setRetainedRecoveryRecords: (records: RetainedSandboxRecoveryRecord[]) => void; setSandboxPresent: (present: boolean) => void; shieldsDownSpy: MockInstance; stopAllSpy: MockInstance; @@ -72,6 +79,7 @@ type DestroyHarnessOptions = { deleteError?: Error; deleteOutput?: string; deleteStatus?: number | null; + dockerNameLabeledIds?: string[]; dockerPsOutput?: string; dockerOrphanIds?: string[]; dockerOrphanQueryStatus?: number | null; @@ -291,40 +299,40 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr detected: true, sessions: [{ pid: 1 }], }); - vi.spyOn(registry, "getSandbox").mockReturnValue( - options.registryEntryPresent === false - ? null - : { - ...sandboxEntry, - imageTag: options.imageTag === undefined ? sandboxEntry.imageTag : options.imageTag, - agent: options.agent ?? sandboxEntry.agent, - ...(options.provider ? { provider: options.provider } : {}), - ...(options.openshellDriver ? { openshellDriver: options.openshellDriver } : {}), - ...(options.endpointUrl ? { endpointUrl: options.endpointUrl } : {}), - ...(options.hostLocalInferenceReceipt !== undefined - ? { hostLocalInferenceReceipt: options.hostLocalInferenceReceipt } - : {}), - ...(options.hostLocalInferenceProvenance - ? { hostLocalInferenceProvenance: options.hostLocalInferenceProvenance } - : {}), - ...(options.workload ? { workload: options.workload } : {}), - ...(options.mcpServers?.length - ? { - mcp: { - bridges: Object.fromEntries( - options.mcpServers.map((server) => [ - server, - { - server, - ...(options.mcpAddState ? { addState: options.mcpAddState } : {}), - }, - ]), - ), + const configuredRegistryEntry = { + ...sandboxEntry, + imageTag: options.imageTag === undefined ? sandboxEntry.imageTag : options.imageTag, + agent: options.agent ?? sandboxEntry.agent, + ...(options.provider ? { provider: options.provider } : {}), + ...(options.openshellDriver ? { openshellDriver: options.openshellDriver } : {}), + ...(options.endpointUrl ? { endpointUrl: options.endpointUrl } : {}), + ...(options.hostLocalInferenceReceipt !== undefined + ? { hostLocalInferenceReceipt: options.hostLocalInferenceReceipt } + : {}), + ...(options.hostLocalInferenceProvenance + ? { hostLocalInferenceProvenance: options.hostLocalInferenceProvenance } + : {}), + ...(options.workload ? { workload: options.workload } : {}), + ...(options.mcpServers?.length + ? { + mcp: { + bridges: Object.fromEntries( + options.mcpServers.map((server) => [ + server, + { + server, + ...(options.mcpAddState ? { addState: options.mcpAddState } : {}), }, - } - : {}), - ...options.registryEntryOverrides, - }, + ]), + ), + }, + } + : {}), + ...options.registryEntryOverrides, + } as SandboxEntry; + let registryEntryPresent = options.registryEntryPresent !== false; + vi.spyOn(registry, "getSandbox").mockImplementation(() => + registryEntryPresent ? configuredRegistryEntry : null, ); let registeredSandboxCount = options.registeredSandboxCount ?? 0; vi.spyOn(registry, "listSandboxes").mockImplementation(() => ({ @@ -375,8 +383,9 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }); vi.spyOn(modelRouterProcess, "isRouterHealthy").mockResolvedValue(false); vi.spyOn(onboardSession, "loadSession").mockImplementation(() => ({ ...sessionState })); - vi.spyOn(onboardSession, "listRetainedSandboxRecoveryRecords").mockReturnValue( - options.retainedRecoveryRecords ?? [], + let retainedRecoveryRecords = [...(options.retainedRecoveryRecords ?? [])]; + vi.spyOn(onboardSession, "listRetainedSandboxRecoveryRecords").mockImplementation( + () => retainedRecoveryRecords, ); const resolveRetainedSandboxRecoverySpy = vi .spyOn(onboardSession, "resolveRetainedSandboxRecovery") @@ -463,6 +472,10 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }); let identityProbeCall = 0; let dockerOrphanIds = [...(options.dockerOrphanIds ?? [])]; + let dockerNameLabeledIds = [ + ...(options.dockerNameLabeledIds ?? options.dockerOrphanIds ?? []), + ]; + let dockerIdentityResult = options.dockerRunResult; const dockerRunSpy = vi.spyOn(dockerRun, "dockerRun").mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; const isDockerOrphanQuery = @@ -477,10 +490,23 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr stderr: "", } as ReturnType; } + const isDockerNameLabeledQuery = + argv[0] === "ps" && + !argv.includes("label=openshell.ai/managed-by=openshell") && + argv.includes("label=openshell.ai/sandbox-name=alpha") && + argv.at(-1) === "{{.ID}}"; + if (isDockerNameLabeledQuery) { + return { + status: options.dockerOrphanQueryStatus ?? 0, + stdout: dockerNameLabeledIds.join("\n"), + stderr: "", + } as ReturnType; + } if (argv[0] === "rm" && argv[1] === "-f") { const status = options.dockerRemoveStatus ?? 0; if (status === 0) { dockerOrphanIds = dockerOrphanIds.filter((id) => id !== argv[2]); + dockerNameLabeledIds = dockerNameLabeledIds.filter((id) => id !== argv[2]); } return { status, stdout: "", stderr: "" } as ReturnType; } @@ -502,7 +528,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }; const result = options.dockerRunResultSequence?.[identityProbeCall - 1] ?? - options.dockerRunResult ?? + dockerIdentityResult ?? defaultIdentityResult; return result as ReturnType; }); @@ -646,6 +672,15 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr runOpenshellSpy, selectGatewaySpy, sessionState, + setDockerIdentityResult: (result) => { + dockerIdentityResult = result; + }, + setRegistryEntryPresent: (present: boolean) => { + registryEntryPresent = present; + }, + setRetainedRecoveryRecords: (records) => { + retainedRecoveryRecords = [...records]; + }, setSandboxPresent: (present: boolean) => { sandboxPresent = present; }, From fc287e0fe9b280bf7e3b01f5c872b7c7ccf78376 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 13:46:38 -0700 Subject: [PATCH 03/24] refactor(destroy): unify container identity proof (#10547) Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/destroy-execution.ts | 4 +- src/lib/actions/sandbox/destroy-presence.ts | 61 ++++++++----------- src/lib/actions/sandbox/destroy.test.ts | 4 +- src/lib/actions/sandbox/destroy.ts | 5 +- ...penshell-docker-sandbox-containers.test.ts | 13 ++-- .../openshell-docker-sandbox-containers.ts | 10 --- 6 files changed, 35 insertions(+), 62 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 23444bd31f..1a69be1b46 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -316,9 +316,7 @@ export async function executeSandboxDestroy({ | { status: "probe-failed"; detail: string; subject?: string }; const pendingPolicyVerification = sandbox?.pendingPolicyVerification; const expectedContainerProof: DestroyContainerIdentityProof = - expectedContainerIdentities === undefined - ? { identity: undefined } - : { identities: expectedContainerIdentities }; + expectedContainerIdentities === undefined ? {} : { identities: expectedContainerIdentities }; const proofFromVerdict = ( verdict: ReturnType, ): DestroyContainerIdentityProof | null => { diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts index 00a60daead..0d504a769a 100644 --- a/src/lib/actions/sandbox/destroy-presence.ts +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -58,18 +58,12 @@ export type AssertUnambiguousDestroyIdentityDeps = { error?: (message: string) => void; }; -export type DestroyContainerIdentityProof = - | { identity: SandboxNameLabeledContainer | null | undefined } - | { identities: readonly SandboxNameLabeledContainer[] }; - -/** Normalize the legacy single-container proof and recovery set proof. */ -export function getDestroyContainerIdentities( - proof: DestroyContainerIdentityProof, -): readonly SandboxNameLabeledContainer[] | undefined { - if ("identities" in proof) return proof.identities; - if (proof.identity === undefined) return undefined; - return proof.identity === null ? [] : [proof.identity]; -} +export type DestroyContainerIdentityProof = { + // `undefined` delegates identity gating to the runtime provider. An empty + // array records confirmed Docker absence; other arrays contain the exact + // immutable Docker identities qualified for this destroy operation. + identities?: readonly SandboxNameLabeledContainer[]; +}; function observeDockerSandboxIdentities(sandboxName: string): DockerSandboxIdentityObservation { return inspectDockerSandboxIdentities(`${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, { @@ -197,21 +191,6 @@ export function classifyDestroyContainerIdentity( return { status: "clear", identity }; } -/** Require the same immutable container row, including the already-absent state. */ -export function isSameDestroyContainerIdentity( - expected: SandboxNameLabeledContainer | null, - verdict: DestroyContainerIdentityVerdict, -): boolean { - if (verdict.status !== "clear") return false; - if (expected === null || verdict.identity === null) return expected === verdict.identity; - return ( - expected.id === verdict.identity.id && - expected.managedBy === verdict.identity.managedBy && - expected.workspace === verdict.identity.workspace && - expected.sandboxId === verdict.identity.sandboxId - ); -} - /** Human-readable lines describing an ambiguous-identity refusal. */ export function formatAmbiguousDestroyIdentity( verdict: Extract, @@ -259,7 +238,7 @@ export function assertUnambiguousDestroyContainerIdentity( retainedSandboxIdentityFingerprint, )); const error = deps.error ?? ((message: string) => console.error(` ${message}`)); - if (deps.providerId !== "docker") return { identity: undefined }; + if (deps.providerId !== "docker") return {}; const verdict = deps.retainedSandboxIdentityFingerprint ? classify(sandboxName, deps.retainedSandboxIdentityFingerprint) @@ -278,9 +257,14 @@ export function assertUnambiguousDestroyContainerIdentity( ); return false; } - return verdict.status === "recovery" - ? { identities: verdict.identities } - : { identity: verdict.identity }; + return { + identities: + verdict.status === "recovery" + ? verdict.identities + : verdict.identity === null + ? [] + : [verdict.identity], + }; } /** Compare provider-owned identity proofs across two destroy checkpoints. */ @@ -288,18 +272,21 @@ export function isSameDestroyContainerIdentityProof( expected: DestroyContainerIdentityProof, actual: DestroyContainerIdentityProof, ): boolean { - const expectedIdentities = getDestroyContainerIdentities(expected); - const actualIdentities = getDestroyContainerIdentities(actual); + const expectedIdentities = expected.identities; + const actualIdentities = actual.identities; if (expectedIdentities === undefined || actualIdentities === undefined) { return expectedIdentities === actualIdentities; } if (expectedIdentities.length !== actualIdentities.length) return false; return expectedIdentities.every((identity, index) => { const candidate = actualIdentities[index]; - return candidate !== undefined && isSameDestroyContainerIdentity(identity, { - status: "clear", - identity: candidate, - }); + return ( + candidate !== undefined && + identity.id === candidate.id && + identity.managedBy === candidate.managedBy && + identity.workspace === candidate.workspace && + identity.sandboxId === candidate.sandboxId + ); }); } diff --git a/src/lib/actions/sandbox/destroy.test.ts b/src/lib/actions/sandbox/destroy.test.ts index f621f220f8..9af10ead97 100644 --- a/src/lib/actions/sandbox/destroy.test.ts +++ b/src/lib/actions/sandbox/destroy.test.ts @@ -110,7 +110,7 @@ describe("assertUnambiguousDestroyContainerIdentity (#8999)", () => { redact: String, classify: classify as never, }), - ).toEqual({ identity }); + ).toEqual({ identities: [identity] }); }); it("does not probe or block a non-Docker runtime provider", () => { @@ -120,7 +120,7 @@ describe("assertUnambiguousDestroyContainerIdentity (#8999)", () => { redact: String, classify: classify as never, }); - expect(proceed).toEqual({ identity: undefined }); + expect(proceed).toEqual({}); expect(classify).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 1c453dd7a6..98b44cdbf1 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -56,7 +56,6 @@ import { shouldCleanupGatewayAfterConfirmedFinalDestroy } from "./destroy-gatewa import { assertUnambiguousDestroyContainerIdentity, classifyDestroySandboxPresence, - getDestroyContainerIdentities, isSameDestroyContainerIdentityProof, } from "./destroy-presence"; import { @@ -583,9 +582,7 @@ async function destroySandboxUnlocked( if (initialIdentity === false) { requestSandboxDestroyExit(1); } - const initialContainerIdentities = initialIdentity - ? getDestroyContainerIdentities(initialIdentity) - : undefined; + const initialContainerIdentities = initialIdentity?.identities; let preparedManagedLlamaCppCleanup: ReturnType< typeof prepareManagedLlamaCppRuntimeCleanupForSandbox diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts index fc6a11a5e7..5ca5b4219b 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -4,7 +4,6 @@ import { describe, expect, it, vi } from "vitest"; import { queryOpenShellDockerSandboxRuntimeSnapshot, - removeExactOpenShellDockerSandboxContainer, removeExactOpenShellDockerSandboxContainers, } from "./openshell-docker-sandbox-containers"; @@ -14,7 +13,7 @@ const EMPTY_RUNTIME_FIELDS = [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "ru const ACTIVATED_CONTAINER_ID = "b".repeat(64); const ROLLBACK_CONTAINER_ID = "c".repeat(64); -describe("removeExactOpenShellDockerSandboxContainer", () => { +describe("removeExactOpenShellDockerSandboxContainers", () => { it("fails when Docker cannot confirm the exact container is absent after removal (#9073)", () => { const expectedContainerId = "a".repeat(64); const queryContainers = vi @@ -24,10 +23,12 @@ describe("removeExactOpenShellDockerSandboxContainer", () => { const forceRemove = vi.fn(() => ({ status: 0 })); expect(() => - removeExactOpenShellDockerSandboxContainer("alpha", expectedContainerId, vi.fn(), { - queryContainers, - forceRemove, - }), + removeExactOpenShellDockerSandboxContainers( + "alpha", + [expectedContainerId], + vi.fn(), + { queryContainers, forceRemove }, + ), ).toThrow("could not confirm exact Docker container removal"); expect(forceRemove).toHaveBeenCalledWith(expectedContainerId); diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index d84d3c98a9..7a8dc3feee 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -176,16 +176,6 @@ export function removeExactOpenShellDockerSandboxContainers( } } -/** Remove only the Docker container whose immutable ID passed the caller's authority check. */ -export function removeExactOpenShellDockerSandboxContainer( - sandboxName: string, - expectedContainerId: string, - log: (message: string) => void, - deps: StaleDockerOrphanCleanupDeps = {}, -): void { - removeExactOpenShellDockerSandboxContainers(sandboxName, [expectedContainerId], log, deps); -} - /** * Remove one exact Docker-owned orphan when a registry row outlives its OpenShell sandbox. * Docker labels are the remaining authority in this invalid state; see the focused #8720 tests. From 820374ff4a9e731668d3e1db8eeb824f3ddec02a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 14:14:36 -0700 Subject: [PATCH 04/24] fix(destroy): resolve retained identity conflicts (#10547) Signed-off-by: Prekshi Vyas --- .../destroy-retained-recovery-flow.test.ts | 70 +++++++++++++++++++ src/lib/actions/sandbox/destroy.ts | 29 +++++++- .../retained-sandbox-recovery.ts | 3 +- .../state/retained-sandbox-recovery.test.ts | 18 +++++ 4 files changed, 117 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index 9995022733..f60fe58d48 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -225,4 +225,74 @@ describe("destroySandbox retained recovery flow", () => { expect(exitSpy).not.toHaveBeenCalled(); }, ); + + it( + "selects only the retained record matching observed Docker identity without a registry row (#10547)", + { timeout: 30_000 }, + async () => { + const matchingRecovery = retainedRecoveryRecord(); + const olderRecovery = { + ...retainedRecoveryRecord("sb-older"), + recordId: "e".repeat(64), + lifecycleGeneration: "generation-older", + }; + const sandboxContainerId = "a".repeat(64); + const bootstrapContainerId = "b".repeat(64); + const identityRows = [sandboxContainerId, bootstrapContainerId] + .map((id) => `${id}\topenshell\tdefault\tsb-alpha`) + .join("\n"); + const harness = createDestroyHarness({ + registryEntryPresent: false, + dockerOrphanIds: [bootstrapContainerId], + dockerRunResult: { status: 0, stdout: identityRows }, + retainedRecoveryRecords: [olderRecovery, matchingRecovery], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.dockerRunSpy).toHaveBeenCalledWith( + ["rm", "-f", bootstrapContainerId], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledOnce(); + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledWith(matchingRecovery); + expect(harness.resolveRetainedSandboxRecoverySpy).not.toHaveBeenCalledWith(olderRecovery); + expect(exitSpy).not.toHaveBeenCalled(); + }, + ); + + it( + "refuses destroy when observed Docker identity matches multiple retained records (#10547)", + { timeout: 30_000 }, + async () => { + const firstRecovery = retainedRecoveryRecord(); + const secondRecovery = { + ...firstRecovery, + recordId: "e".repeat(64), + lifecycleGeneration: "generation-second", + createAttemptNonce: "d".repeat(62), + }; + const harness = createDestroyHarness({ + registryEntryPresent: false, + dockerRunResult: { + status: 0, + stdout: `${"a".repeat(64)}\topenshell\tdefault\tsb-alpha`, + }, + retainedRecoveryRecords: [firstRecovery, secondRecovery], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("could not select exactly one recovery record"), + ); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.resolveRetainedSandboxRecoverySpy).not.toHaveBeenCalled(); + }, + ); }); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 98b44cdbf1..226af832a9 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -55,8 +55,10 @@ import { cleanupGatewayAfterLastSandbox } from "./destroy-gateway"; import { shouldCleanupGatewayAfterConfirmedFinalDestroy } from "./destroy-gateway-cleanup"; import { assertUnambiguousDestroyContainerIdentity, + classifyDestroyContainerIdentity, classifyDestroySandboxPresence, isSameDestroyContainerIdentityProof, + observeDestroyContainerIdentity, } from "./destroy-presence"; import { prepareSandboxDestroy, @@ -113,6 +115,21 @@ function selectRetainedSandboxRecoveryAuthority( const matching = records.filter( (record) => record.sandboxName === sandboxName && matchesRegistryAuthority(record), ); + if (!sandbox && matching.length > 1) { + const observation = observeDestroyContainerIdentity(sandboxName); + const observedMatches = matching.filter((record) => { + const verdict = classifyDestroyContainerIdentity( + sandboxName, + observation, + record.sandboxIdentityFingerprint!, + ); + return ( + verdict.status === "recovery" || + (verdict.status === "clear" && verdict.identity !== null) + ); + }); + return observedMatches.length === 1 ? observedMatches[0]! : null; + } return matching.length === 1 ? matching[0]! : null; } @@ -542,11 +559,21 @@ async function destroySandboxUnlocked( if (!(await confirmSandboxDestroy(sandboxName, normalized))) return; const destroySession = onboardSession.loadSession(); const registeredSandbox = registry.getSandbox(sandboxName); + const retainedRecoveryRecords = onboardSession.listRetainedSandboxRecoveryRecords(); const retainedRecoveryAuthority = selectRetainedSandboxRecoveryAuthority( sandboxName, registeredSandbox, - onboardSession.listRetainedSandboxRecoveryRecords(), + retainedRecoveryRecords, ); + if ( + !retainedRecoveryAuthority && + retainedRecoveryRecords.some((record) => record.sandboxName === sandboxName) + ) { + console.error( + ` Refusing to destroy retained sandbox '${sandboxName}': NemoClaw could not select exactly one recovery record from the current immutable registry and Docker identities. No sandbox resources were removed. Resolve the identity conflict, then rerun '${CLI_NAME} ${sandboxName} destroy'.`, + ); + requestSandboxDestroyExit(1); + } const retainedSandboxIdentityFingerprint = retainedRecoveryAuthority?.sandboxIdentityFingerprint ?? undefined; let portableContainerAuthority: ReturnType; diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index 25af2775db..82fbcf92a7 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -11,14 +11,13 @@ import { parseNemoClawPolicyCreationReceipt, type NemoClawPolicyCreationReceipt, } from "../../policy/merge"; +import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../../sandbox-name-contract"; export { parseNemoClawPolicyCreationReceipt } from "../../policy/merge"; const SCHEMA_VERSION = 1; const FINGERPRINT_PATTERN = /^[0-9a-f]{64}$/u; const SAFE_EVIDENCE_PATTERN = /^[A-Za-z0-9._:@/-]{1,256}$/u; -const NAME_MAX_LENGTH = 63; -const NAME_VALID_PATTERN = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u; export function retainedSandboxRecoveryFile(sessionDirectory: string): string { return path.join(sessionDirectory, "retained-sandbox-recovery.json"); diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index df37acb892..a5d13f992c 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -88,6 +88,24 @@ describe("retained sandbox recovery state", () => { }); }); + it("rejects a recovery target outside the canonical sandbox-name contract", async () => { + const recovery = await import("./onboard-session"); + + expect(() => + recovery.recordRetainedSandboxRecovery({ + sandboxName: "1sandbox", + sandboxIdentityFingerprint: "a".repeat(64), + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + verifiedEffectivePolicyIdentity: null, + ...recoveryAuthority, + resources: evidence, + reason: "retained_after_sandbox_creation_failure", + }), + ).toThrow("Cannot persist invalid retained sandbox recovery evidence"); + }); + it("preserves distinct unresolved lifecycle tuples for one sandbox name (#9833)", async () => { const recovery = await import("./onboard-session"); const first = recovery.recordRetainedSandboxRecovery({ From 248750ce215c033e99245b76e99a45dca4b7bd37 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 15:23:04 -0700 Subject: [PATCH 05/24] fix(onboard): consolidate retained cleanup evidence Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 41 +++++------ src/lib/actions/sandbox/destroy-flow.test.ts | 10 ++- src/lib/actions/sandbox/destroy-presence.ts | 4 +- .../destroy-retained-recovery-flow.test.ts | 5 -- src/lib/onboard/cancel-rollback.test.ts | 6 +- src/lib/onboard/cancel-rollback.ts | 5 +- src/lib/onboard/lifecycle-contracts.md | 4 +- ...penshell-docker-sandbox-containers.test.ts | 53 +++++++++++---- .../openshell-docker-sandbox-containers.ts | 68 +++++++++++++------ ...onboard-session-cross-process-lock.test.ts | 5 -- src/lib/state/onboard-session.ts | 25 +------ .../retained-sandbox-recovery.ts | 29 +------- .../state/retained-sandbox-recovery.test.ts | 20 +----- test/helpers/destroy-flow-test-harness.ts | 35 +++++++++- .../onboard-fresh-create-identity.test.ts | 8 +-- .../onboard-fsm-live-slices.test.ts | 5 -- 16 files changed, 158 insertions(+), 165 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 8a3a425e8d..bdf2024c15 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -455,15 +455,19 @@ When that result is resumable, NemoClaw keeps the session `in_progress` at its l If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox and records its create-attempt label. When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. -Do not destroy the retained sandbox by name. -Ask an OpenShell administrator to verify the exact live durable ID before using an identity-bound removal procedure. -NemoClaw does not provide an operation to clear the recovery record, even after external removal. -Start another onboarding run with a different name: +Run `$$nemoclaw destroy` to complete identity-bound recovery. +Destroy proceeds only after it verifies one retained recovery record and the immutable Docker sandbox identity. +It removes only the qualified sandbox containers and clears the matching recovery record after verified cleanup. +A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. +Do not delete the retained sandbox manually by mutable name. +To onboard another sandbox while the record remains unresolved, supply a different explicit name: ```bash -$$nemoclaw onboard --fresh --name +$$nemoclaw onboard --name ``` +`--fresh` alone does not clear the recovery record or permit reuse of the retained sandbox name. + OpenClaw sessions also record the web search selection, messaging selection and non-secret settings, and resource profile. @@ -530,12 +534,12 @@ $$nemoclaw onboard --fresh --apf-interceptor --name my-apf-sandbox If post-create verification or native GPU fallback fails after OpenShell may have created the sandbox, NemoClaw preserves the incomplete sandbox because automatic deletion would use its mutable name. -Do not destroy that sandbox by name. -Retain the reported sandbox name, create-attempt label, and durable identity fingerprint for comparison only. -If OpenShell did not return the fingerprint, recovery remains blocked until an administrator resolves the create-attempt label to one exact sandbox. -Ask an OpenShell administrator to obtain the exact live durable ID, verify it against the fingerprint, and use an identity-bound removal procedure. +Run `$$nemoclaw destroy` to verify the retained immutable identity, remove only the qualified sandbox containers, and clear the matching recovery record. +A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. +Do not delete the retained sandbox manually by mutable name. +If OpenShell did not return an identity fingerprint, recovery remains blocked until an administrator resolves the create-attempt label to one exact sandbox. This onboarding mode does not support `--resume` or `--recreate-sandbox`, regardless of whether sandbox creation began. -After the administrator confirms identity-bound removal, repeat the original command with `--fresh` and a new name. +After destroy completes, repeat the original command with `--fresh`. @@ -938,18 +942,15 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. If you cancel a brand-new onboarding run at the policy-tier selector or either policy-preset selector after sandbox creation, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. NemoClaw reports the durable sandbox identity fingerprint when it is available. It does not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. -Do not delete the sandbox by mutable name. -Shared inference providers are gateway configuration, not sandbox cleanup targets. -Sandbox-scoped provider registrations or gateway-bound credentials may remain when the durable recovery record lists them. -Ask an OpenShell administrator to inspect the exact sandbox identity and remove only sandbox-scoped resources whose ownership is confirmed for the retained sandbox. -A credential environment name in the recovery record does not prove that its value was exposed. -Rotate a credential only when identity-bound inspection proves that it was exposed or attached to a retained sandbox-scoped resource. +Run `$$nemoclaw destroy` to complete cleanup through the retained immutable identity. +Destroy removes only the qualified sandbox containers and clears the matching recovery record after it verifies their absence. +If OpenShell already removed the sandbox, destroy can still clean up verified residual containers and retire the record. +A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. +Do not delete the sandbox manually by mutable name. NemoClaw stores the recovery record independently from the active onboarding session. A fresh run with a different name can proceed without clearing that record, but automatic resume, explicit `--resume`, reuse, recreation, and fresh onboarding with the retained name remain blocked. -NemoClaw has no supported operation in this release to clear the recovery record, so the retained name remains unavailable even after external recovery or removal. -Preserve the record as evidence. -Start fresh onboarding with `$$nemoclaw onboard --fresh --name `. -Select the required provider, model, agent, policy, and environment inputs again because `--fresh` does not retain them. +Start another onboarding run with `$$nemoclaw onboard --name `. +`--fresh` alone does not clear the recovery record or permit reuse of the retained sandbox name. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. In interactive mode, the wizard asks for confirmation before delete and recreate. diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 70698178f7..fc94c1bb2d 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -121,6 +121,7 @@ describe("destroySandbox flow", () => { switch (`${String(argv[0])}:${String(argv[1])}`) { case "sandbox:delete": trace.push("delete"); + harness.setSandboxPresent(false); return { status: 0, stdout: "", stderr: "" }; case "sandbox:list": trace.push("list"); @@ -177,6 +178,7 @@ describe("destroySandbox flow", () => { switch (`${String(argv[0])}:${String(argv[1])}`) { case "sandbox:delete": crossedDeleteBoundary = true; + harness.setSandboxPresent(false); return { status: 0, stdout: "", stderr: "" }; case "sandbox:list": return { @@ -992,7 +994,13 @@ describe("destroySandbox flow", () => { await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); - expect(trace.slice(-2)).toEqual([`probe:${String(identityProbeCalls)}`, "delete"]); + const deleteIndex = trace.indexOf("delete"); + expect(deleteIndex).toBeGreaterThan(0); + expect(trace[deleteIndex - 1]).toMatch(/^probe:/u); + expect(trace.slice(deleteIndex + 1)).toEqual([ + `probe:${String(identityProbeCalls - 1)}`, + `probe:${String(identityProbeCalls)}`, + ]); }); it("preserves provider and registry ownership when runtime authority is unknown", async () => { diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts index 0d504a769a..d0bae776bc 100644 --- a/src/lib/actions/sandbox/destroy-presence.ts +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -6,6 +6,7 @@ import { OPENSHELL_MANAGED_BY_VALUE, OPENSHELL_SANDBOX_ID_LABEL, OPENSHELL_SANDBOX_NAME_LABEL, + OPENSHELL_SANDBOX_WORKSPACE_LABEL, removeExactOpenShellDockerSandboxContainers, } from "../../onboard/openshell-docker-sandbox-containers"; import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; @@ -19,9 +20,6 @@ import { type OpenShellSandboxPresence, } from "../../adapters/openshell/sandbox-presence"; -/** Workspace label OpenShell stamps on every managed sandbox container. */ -export const OPENSHELL_SANDBOX_WORKSPACE_LABEL = "openshell.ai/sandbox-workspace"; - const IDENTITY_VALUE_MAX_LENGTH = 256; const IDENTITY_DIAGNOSTIC_MAX_LENGTH = 500; diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index f60fe58d48..31b7231905 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -24,11 +24,6 @@ function retainedRecoveryRecord(sandboxId = "sb-alpha"): RetainedSandboxRecovery verifiedEffectivePolicyIdentity: null, createAttemptNonce: "c".repeat(62), policyCreationReceipt: null, - resources: { - sharedInferenceProviders: [], - sandboxScopedProviders: [], - credentialEnvironmentVariables: [], - }, reason: "retained_after_sandbox_creation_failure", recordedAt: "2026-08-28T00:00:00.000Z", }; diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 7831ccba24..422799efbe 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -28,16 +28,12 @@ describe("createSandboxCancelRollback", () => { const guidance = log.mock.calls.flat().join("\n"); expect(guidance).toContain("preserved incomplete sandbox 'new-sb'"); expect(guidance).toContain(SANDBOX_FINGERPRINT); - expect(guidance).toContain("OpenShell administrator"); expect(guidance).toContain("did not run OpenShell's mutable-name deletion command"); expect(guidance).toContain("Do not delete the sandbox by mutable sandbox name"); expect(guidance).toContain("Shared inference providers are gateway configuration"); expect(guidance).toContain("not sandbox cleanup targets"); - expect(guidance).toContain("sandbox-scoped resources whose ownership is confirmed"); expect(guidance).toContain("nemoclaw new-sb destroy"); - expect(guidance).toContain("credential environment name alone does not prove exposure"); - expect(guidance).toContain("rotate a credential only when identity-bound inspection proves"); - expect(guidance).not.toContain("rotate any credential"); + expect(guidance).toContain("clear the matching recovery record"); }); it.each([ diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index be65ff4581..4feb1f20dc 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -71,12 +71,9 @@ export function buildCancelRollbackMessage( " NemoClaw did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox.", " Do not delete the sandbox by mutable sandbox name.", " Shared inference providers are gateway configuration and are not sandbox cleanup targets.", - " Sandbox-scoped provider registrations or gateway-bound credentials may remain when the durable recovery record lists them.", - " Ask an OpenShell administrator to inspect the exact sandbox identity and remove only sandbox-scoped resources whose ownership is confirmed for this retained sandbox.", - " A recorded credential environment name alone does not prove exposure; rotate a credential only when identity-bound inspection proves that it was exposed or attached to a retained sandbox-scoped resource.", ...(sandboxIdentityFingerprint ? [ - ` Run '${cliName()} ${sandboxName} destroy' to verify and remove this failed attempt, then start fresh onboarding.`, + ` Run '${cliName()} ${sandboxName} destroy' to verify the retained immutable identity, remove only its qualified resources, and clear the matching recovery record.`, ] : [ " NemoClaw cannot clear this recovery record until an OpenShell administrator establishes the exact sandbox identity.", diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index a3ac359bd9..af88df9817 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set before deletion, removes only remaining members of that set, verifies their absence, and clears the matching recovery record. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set before deletion, removes only remaining members of that set, verifies their absence, and clears the matching recovery record. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. ## Effect-order flows @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` uses that record to qualify immutable Docker container identities, complete interrupted cleanup, and retire the record. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Recovery removes only confirmed sandbox-scoped resources and rotates credentials only when inspection proves exposure or attachment to a retained resource; a recorded environment-variable name alone is not exposure evidence. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` uses that record to qualify immutable Docker container identities, complete interrupted cleanup, and retire the record. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts index 5ca5b4219b..4516c985c4 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -13,13 +13,26 @@ const EMPTY_RUNTIME_FIELDS = [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "ru const ACTIVATED_CONTAINER_ID = "b".repeat(64); const ROLLBACK_CONTAINER_ID = "c".repeat(64); +function observeContainerIds(ids: readonly string[], malformedRows = 0) { + return { + status: "observed" as const, + rows: ids.map((id) => ({ + id, + managedBy: "openshell", + workspace: "default", + sandboxId: "sb-alpha", + })), + malformedRows, + }; +} + describe("removeExactOpenShellDockerSandboxContainers", () => { it("fails when Docker cannot confirm the exact container is absent after removal (#9073)", () => { const expectedContainerId = "a".repeat(64); - const queryContainers = vi + const inspectContainers = vi .fn() - .mockReturnValueOnce({ ok: true, ids: [expectedContainerId] }) - .mockReturnValueOnce({ ok: true, ids: [expectedContainerId] }); + .mockReturnValueOnce(observeContainerIds([expectedContainerId])) + .mockReturnValueOnce(observeContainerIds([expectedContainerId])); const forceRemove = vi.fn(() => ({ status: 0 })); expect(() => @@ -27,7 +40,7 @@ describe("removeExactOpenShellDockerSandboxContainers", () => { "alpha", [expectedContainerId], vi.fn(), - { queryContainers, forceRemove }, + { inspectContainers, forceRemove }, ), ).toThrow("could not confirm exact Docker container removal"); @@ -37,7 +50,7 @@ describe("removeExactOpenShellDockerSandboxContainers", () => { it("removes every remaining container from one exact failed attempt (#10547)", () => { const expectedContainerIds = ["a".repeat(64), "b".repeat(64)]; let currentContainerIds = [...expectedContainerIds]; - const queryContainers = vi.fn(() => ({ ok: true as const, ids: [...currentContainerIds] })); + const inspectContainers = vi.fn(() => observeContainerIds(currentContainerIds)); const forceRemove = vi.fn((containerId: string) => { currentContainerIds = currentContainerIds.filter((candidate) => candidate !== containerId); return { status: 0 }; @@ -47,7 +60,7 @@ describe("removeExactOpenShellDockerSandboxContainers", () => { "alpha", expectedContainerIds, vi.fn(), - { queryContainers, forceRemove }, + { inspectContainers, forceRemove }, ); expect(forceRemove.mock.calls.map(([containerId]) => containerId)).toEqual( @@ -60,7 +73,7 @@ describe("removeExactOpenShellDockerSandboxContainers", () => { const alreadyRemovedId = "a".repeat(64); const remainingId = "b".repeat(64); let currentContainerIds = [remainingId]; - const queryContainers = vi.fn(() => ({ ok: true as const, ids: [...currentContainerIds] })); + const inspectContainers = vi.fn(() => observeContainerIds(currentContainerIds)); const forceRemove = vi.fn((containerId: string) => { currentContainerIds = currentContainerIds.filter((candidate) => candidate !== containerId); return { status: 0 }; @@ -70,7 +83,7 @@ describe("removeExactOpenShellDockerSandboxContainers", () => { "alpha", [alreadyRemovedId, remainingId], vi.fn(), - { queryContainers, forceRemove }, + { inspectContainers, forceRemove }, ); expect(forceRemove).toHaveBeenCalledExactlyOnceWith(remainingId); @@ -88,10 +101,7 @@ describe("removeExactOpenShellDockerSandboxContainers", () => { [expectedContainerId], vi.fn(), { - queryContainers: vi.fn(() => ({ - ok: true as const, - ids: [replacementContainerId], - })), + inspectContainers: vi.fn(() => observeContainerIds([replacementContainerId])), forceRemove, }, ), @@ -99,6 +109,25 @@ describe("removeExactOpenShellDockerSandboxContainers", () => { expect(forceRemove).not.toHaveBeenCalled(); }); + + it("rejects malformed Docker identity output during exact cleanup (#10547)", () => { + const expectedContainerId = "a".repeat(64); + const forceRemove = vi.fn(() => ({ status: 0 })); + + expect(() => + removeExactOpenShellDockerSandboxContainers( + "alpha", + [expectedContainerId], + vi.fn(), + { + inspectContainers: vi.fn(() => observeContainerIds([], 1)), + forceRemove, + }, + ), + ).toThrow("malformed container identity row"); + + expect(forceRemove).not.toHaveBeenCalled(); + }); }); function querySnapshot(fields: unknown, nvidiaVisibleDevices?: string) { diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index 7a8dc3feee..d5dcb34d5d 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import { dockerCapture, dockerRun } from "../adapters/docker"; +import { + inspectDockerSandboxIdentities, + type DockerSandboxIdentityObservation, +} from "../adapters/docker/inspect"; import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; @@ -9,6 +13,7 @@ export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; export const OPENSHELL_SANDBOX_NAMESPACE_LABEL = "openshell.ai/sandbox-namespace"; +export const OPENSHELL_SANDBOX_WORKSPACE_LABEL = "openshell.ai/sandbox-workspace"; const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; const STALE_DOCKER_ORPHAN_TIMEOUT_MS = 30_000; @@ -107,29 +112,51 @@ export function queryOpenShellDockerSandboxContainers( ); } -function queryDockerSandboxNameLabeledContainers( - sandboxName: string, - deps: DockerSandboxContainerQueryDeps = {}, - timeoutMs: number = DOCKER_SANDBOX_QUERY_TIMEOUT_MS, -): OpenShellDockerSandboxContainerQuery { - return queryDockerSandboxContainerIds( - sandboxContainerFilterArgs(sandboxName, undefined, false), - deps, - timeoutMs, - ); -} - type StaleDockerOrphanCleanupDeps = { queryContainers?: typeof queryOpenShellDockerSandboxContainers; forceRemove?: (containerId: string) => { status?: number | null }; }; +type ExactDockerContainerCleanupDeps = { + inspectContainers?: (sandboxName: string) => DockerSandboxIdentityObservation; + forceRemove?: (containerId: string) => { status?: number | null }; +}; + +function inspectDockerSandboxNameLabeledContainers( + sandboxName: string, +): DockerSandboxIdentityObservation { + return inspectDockerSandboxIdentities(`${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, { + managedBy: OPENSHELL_MANAGED_BY_LABEL, + workspace: OPENSHELL_SANDBOX_WORKSPACE_LABEL, + sandboxId: OPENSHELL_SANDBOX_ID_LABEL, + }); +} + +function exactCleanupContainerIds( + observation: DockerSandboxIdentityObservation, + phase: "inspect" | "confirm", +): string[] { + const failurePrefix = + phase === "inspect" + ? "could not inspect the exact Docker cleanup target" + : "could not confirm exact Docker container removal"; + if (observation.status === "probe-failed") { + throw new Error(`${failurePrefix}: ${observation.detail || "Docker identity probe failed"}`); + } + if (observation.malformedRows > 0) { + throw new Error( + `${failurePrefix}: Docker returned ${String(observation.malformedRows)} malformed container identity row(s)`, + ); + } + return observation.rows.map((row) => row.id); +} + /** Remove the remaining members of one immutable, prequalified Docker container set. */ export function removeExactOpenShellDockerSandboxContainers( sandboxName: string, expectedContainerIds: readonly string[], log: (message: string) => void, - deps: StaleDockerOrphanCleanupDeps = {}, + deps: ExactDockerContainerCleanupDeps = {}, ): void { const expected = new Set(expectedContainerIds); if ( @@ -142,19 +169,16 @@ export function removeExactOpenShellDockerSandboxContainers( // Recovery retirement requires the mutable name itself to be unambiguous. // Inspect every name-labeled container, including foreign containers that do // not carry OpenShell's managed-by marker, while removing only qualified IDs. - const queryContainers = deps.queryContainers ?? queryDockerSandboxNameLabeledContainers; - const initial = queryContainers(sandboxName); - if (!initial.ok) { - throw new Error(`could not inspect the exact Docker cleanup target: ${initial.error}`); - } - const unexpected = initial.ids.filter((containerId) => !expected.has(containerId)); + const inspectContainers = deps.inspectContainers ?? inspectDockerSandboxNameLabeledContainers; + const initialIds = exactCleanupContainerIds(inspectContainers(sandboxName), "inspect"); + const unexpected = initialIds.filter((containerId) => !expected.has(containerId)); if (unexpected.length > 0) { throw new Error( `found labeled Docker container(s) outside the retained identity set: ${unexpected.join(", ")}; refusing replacement cleanup`, ); } - const remaining = new Set(initial.ids); + const remaining = new Set(initialIds); for (const containerId of expectedContainerIds) { if (!remaining.has(containerId)) continue; const removal = deps.forceRemove @@ -170,8 +194,8 @@ export function removeExactOpenShellDockerSandboxContainers( log(`Removed exact Docker container '${containerId}' after OpenShell sandbox deletion`); } - const confirmed = queryContainers(sandboxName); - if (!confirmed.ok || confirmed.ids.length !== 0) { + const confirmedIds = exactCleanupContainerIds(inspectContainers(sandboxName), "confirm"); + if (confirmedIds.length !== 0) { throw new Error("could not confirm exact Docker container removal"); } } diff --git a/src/lib/state/onboard-session-cross-process-lock.test.ts b/src/lib/state/onboard-session-cross-process-lock.test.ts index aa37b509ea..4fd6018c00 100644 --- a/src/lib/state/onboard-session-cross-process-lock.test.ts +++ b/src/lib/state/onboard-session-cross-process-lock.test.ts @@ -346,11 +346,6 @@ describe("cross-process onboard lock", () => { verifiedEffectivePolicyIdentity: null, createAttemptNonce: "c".repeat(62), policyCreationReceipt: null, - resources: { - sharedInferenceProviders: [], - sandboxScopedProviders: [], - credentialEnvironmentVariables: [], - }, reason: "retained_after_sandbox_creation_failure", }); process.stdout.write(JSON.stringify({ ok: true, recordId: recorded.recordId })); diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index c6de8e6b00..e55e0532da 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -19,12 +19,7 @@ import { parseServingProfileProvenance, type ServingProfileProvenance, } from "../inference/serving/profile-provenance"; -import { - normalizeWebSearchConfig, - webSearchEnvFor, - webSearchProviderForConfig, - type WebSearchConfig, -} from "../inference/web-search"; +import { normalizeWebSearchConfig, type WebSearchConfig } from "../inference/web-search"; import type { SandboxMessagingPlan } from "../messaging/manifest"; import { compactSandboxMessagingPlanForPersistence } from "../messaging/persistence"; import { parseSandboxMessagingPlan } from "../messaging/plan-validation"; @@ -2004,22 +1999,6 @@ export interface RetainedSandboxRecoveryContext { readonly policyCreationReceipt: RetainedSandboxRecoveryRecord["policyCreationReceipt"]; } -function retainedSandboxResourceEvidence(session: Session) { - const messagingCredentialEnvironmentVariables = - session.messagingPlan?.credentialBindings.map((binding) => binding.providerEnvKey) ?? []; - return { - sharedInferenceProviders: session.provider ? [session.provider] : [], - sandboxScopedProviders: session.stagedCredentialProviders, - credentialEnvironmentVariables: [ - ...(session.credentialEnv ? [session.credentialEnv] : []), - ...(session.webSearchConfig - ? [webSearchEnvFor(webSearchProviderForConfig(session.webSearchConfig))] - : []), - ...messagingCredentialEnvironmentVariables, - ], - }; -} - function persistIndependentRetainedSandboxRecovery( session: Session, reason: RetainedSandboxRecoveryReason, @@ -2035,7 +2014,6 @@ function persistIndependentRetainedSandboxRecovery( verifiedEffectivePolicyIdentity: context.verifiedEffectivePolicyIdentity, createAttemptNonce: context.createAttemptNonce, policyCreationReceipt: context.policyCreationReceipt, - resources: retainedSandboxResourceEvidence(session), reason, }); } @@ -2065,7 +2043,6 @@ export function listRetainedSandboxRecoveryRecords(): readonly RetainedSandboxRe verifiedEffectivePolicyIdentity: recovery.verifiedEffectivePolicyIdentity, createAttemptNonce: recovery.createAttemptNonce, policyCreationReceipt: recovery.policyCreationReceipt, - resources: retainedSandboxResourceEvidence(current), reason: recovery.reason, recordedAt: recovery.recordedAt, }); diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index 82fbcf92a7..11c9412da0 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -27,12 +27,6 @@ export type RetainedSandboxRecoveryReason = | "cancelled_after_sandbox_creation" | "retained_after_sandbox_creation_failure"; -export interface RetainedSandboxResourceEvidence { - readonly sharedInferenceProviders: readonly string[]; - readonly sandboxScopedProviders: readonly string[]; - readonly credentialEnvironmentVariables: readonly string[]; -} - export interface RetainedSandboxVerifiedEffectivePolicyIdentity { readonly hash: string; readonly activeVersion: number; @@ -50,7 +44,6 @@ export interface RetainedSandboxRecoveryRecord { readonly verifiedEffectivePolicyIdentity: RetainedSandboxVerifiedEffectivePolicyIdentity | null; readonly createAttemptNonce: string; readonly policyCreationReceipt: NemoClawPolicyCreationReceipt | null; - readonly resources: RetainedSandboxResourceEvidence; readonly reason: RetainedSandboxRecoveryReason; readonly recordedAt: string; } @@ -76,7 +69,6 @@ export interface RecordRetainedSandboxRecoveryInput { readonly verifiedEffectivePolicyIdentity: RetainedSandboxVerifiedEffectivePolicyIdentity | null; readonly createAttemptNonce: string; readonly policyCreationReceipt: NemoClawPolicyCreationReceipt | null; - readonly resources: RetainedSandboxResourceEvidence; readonly reason: RetainedSandboxRecoveryReason; readonly recordedAt?: string; } @@ -335,20 +327,6 @@ function validGatewayPort(value: unknown): value is number { return Number.isInteger(value) && Number(value) >= 1024 && Number(value) <= 65535; } -function parseEvidence(value: unknown): RetainedSandboxResourceEvidence | null { - if (!isObjectRecord(value)) return null; - const parse = (candidate: unknown): string[] | null => - Array.isArray(candidate) && candidate.every(validSafeEvidence) - ? [...new Set(candidate)].sort() - : null; - const sharedInferenceProviders = parse(value.sharedInferenceProviders); - const sandboxScopedProviders = parse(value.sandboxScopedProviders); - const credentialEnvironmentVariables = parse(value.credentialEnvironmentVariables); - return sharedInferenceProviders && sandboxScopedProviders && credentialEnvironmentVariables - ? { sharedInferenceProviders, sandboxScopedProviders, credentialEnvironmentVariables } - : null; -} - function parseVerifiedEffectivePolicyIdentity( value: unknown, ): RetainedSandboxVerifiedEffectivePolicyIdentity | null | undefined { @@ -366,7 +344,6 @@ function parseVerifiedEffectivePolicyIdentity( function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { if (!isObjectRecord(value)) return null; - const resources = parseEvidence(value.resources); const fingerprint = value.sandboxIdentityFingerprint; const verifiedEffectivePolicyIdentity = parseVerifiedEffectivePolicyIdentity( value.verifiedEffectivePolicyIdentity, @@ -402,7 +379,6 @@ function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { policyCreationReceipt.sandboxIdentityFingerprint !== fingerprint || policyCreationReceipt.policyHash !== verifiedEffectivePolicyIdentity?.hash || policyCreationReceipt.policyVersion !== verifiedEffectivePolicyIdentity?.activeVersion)) || - !resources || !["cancelled_after_sandbox_creation", "retained_after_sandbox_creation_failure"].includes( String(reason), ) || @@ -422,7 +398,6 @@ function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { verifiedEffectivePolicyIdentity, createAttemptNonce: value.createAttemptNonce, policyCreationReceipt, - resources, reason: reason as RetainedSandboxRecoveryReason, recordedAt: value.recordedAt, }; @@ -469,8 +444,7 @@ function assertRecordInput(input: RecordRetainedSandboxRecoveryInput): void { !validGatewayPort(input.gatewayPort) || (input.lifecycleGeneration !== null && !validSafeEvidence(input.lifecycleGeneration)) || parseVerifiedEffectivePolicyIdentity(input.verifiedEffectivePolicyIdentity) === undefined || - !/^[0-9a-f]{62}$/u.test(input.createAttemptNonce) || - !parseEvidence(input.resources) + !/^[0-9a-f]{62}$/u.test(input.createAttemptNonce) ) { throw new Error("Cannot persist invalid retained sandbox recovery evidence."); } @@ -522,7 +496,6 @@ export function recordRetainedSandboxRecovery( policyCreationReceipt: input.policyCreationReceipt ? parseNemoClawPolicyCreationReceipt(input.policyCreationReceipt) : null, - resources: parseEvidence(input.resources)!, reason: input.reason, recordedAt: input.recordedAt ?? new Date().toISOString(), }; diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index a5d13f992c..7027b9c0b6 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -20,18 +20,13 @@ afterEach(() => { fs.rmSync(home, { recursive: true, force: true }); }); -const evidence = { - sharedInferenceProviders: ["nvidia"], - sandboxScopedProviders: ["sandbox-telegram"], - credentialEnvironmentVariables: ["NVIDIA_API_KEY", "TELEGRAM_BOT_TOKEN"], -} as const; const recoveryAuthority = { createAttemptNonce: "c".repeat(62), policyCreationReceipt: null, } as const; describe("retained sandbox recovery state", () => { - it("persists verified identity and secret-free resource evidence independently", async () => { + it("persists verified identity independently", async () => { const recovery = await import("./onboard-session"); const fingerprint = "a".repeat(64); const input = { @@ -42,7 +37,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: "00000000-0000-4000-8000-000000000001", verifiedEffectivePolicyIdentity: { hash: "sha256:policy-1", activeVersion: 1 }, ...recoveryAuthority, - resources: evidence, reason: "cancelled_after_sandbox_creation", recordedAt: "2026-08-27T00:00:00.000Z", } as const; @@ -55,7 +49,6 @@ describe("retained sandbox recovery state", () => { sandboxIdentityFingerprint: fingerprint, identityWasUnavailable: false, verifiedEffectivePolicyIdentity: input.verifiedEffectivePolicyIdentity, - resources: evidence, }); expect(fs.readFileSync(recovery.RETAINED_SANDBOX_RECOVERY_FILE, "utf8")).not.toContain( "secret-value", @@ -73,11 +66,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: null, verifiedEffectivePolicyIdentity: null, ...recoveryAuthority, - resources: { - sharedInferenceProviders: [], - sandboxScopedProviders: [], - credentialEnvironmentVariables: [], - }, reason: "retained_after_sandbox_creation_failure", }); @@ -100,7 +88,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: "00000000-0000-4000-8000-000000000001", verifiedEffectivePolicyIdentity: null, ...recoveryAuthority, - resources: evidence, reason: "retained_after_sandbox_creation_failure", }), ).toThrow("Cannot persist invalid retained sandbox recovery evidence"); @@ -116,7 +103,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: "00000000-0000-4000-8000-000000000001", verifiedEffectivePolicyIdentity: { hash: "sha256:policy-1", activeVersion: 1 }, ...recoveryAuthority, - resources: evidence, reason: "cancelled_after_sandbox_creation", }); const second = recovery.recordRetainedSandboxRecovery({ @@ -127,7 +113,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: "00000000-0000-4000-8000-000000000002", verifiedEffectivePolicyIdentity: { hash: "sha256:policy-2", activeVersion: 2 }, ...recoveryAuthority, - resources: evidence, reason: "retained_after_sandbox_creation_failure", }); @@ -174,7 +159,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: "generation-1", verifiedEffectivePolicyIdentity: null, ...recoveryAuthority, - resources: evidence, reason: "retained_after_sandbox_creation_failure", }), ).toThrow(/symbolic link|lock ownership changed/u); @@ -210,7 +194,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: "generation-1", verifiedEffectivePolicyIdentity: null, ...recoveryAuthority, - resources: evidence, reason: "retained_after_sandbox_creation_failure", }), ).toThrow(/state directory changed|lock ownership changed/u); @@ -274,7 +257,6 @@ describe("retained sandbox recovery state", () => { lifecycleGeneration: "generation-1", verifiedEffectivePolicyIdentity: null, ...recoveryAuthority, - resources: evidence, reason: "cancelled_after_sandbox_creation", }); diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 2958dfaf22..471306b1d0 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -161,7 +161,7 @@ export function resetDestroyModuleCache(): void { } export function traceDestroyBoundaryCalls( - harness: Pick, + harness: Pick, trace: string[], ): void { harness.runOpenshellSpy.mockImplementation((args: unknown) => { @@ -169,6 +169,7 @@ export function traceDestroyBoundaryCalls( switch (`${String(argv[0])}:${String(argv[1])}`) { case "sandbox:delete": trace.push("delete"); + harness.setSandboxPresent(false); return { status: 0, stdout: "", stderr: "" }; case "sandbox:list": return { status: 0, stdout: "[]", stderr: "" }; @@ -188,6 +189,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr const events: string[] = []; const lifecycleLockEvents: string[] = []; let sandboxPresent = options.sandboxPresent !== false; + let exactDockerCleanupPhase = false; let sessionLockBusy = false; const sessionState = { sessionId: "session-alpha", @@ -422,6 +424,8 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr return session; }); const gatewayPinsAtSandboxList: Array = []; + let identityProbeCall = 0; + let absentListIdentityProbeCall: number | null = null; const runOpenshellSpy = vi.spyOn(runtime, "runOpenshell").mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args : []; switch (`${String(argv[0])}:${String(argv[1])}`) { @@ -435,6 +439,9 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }; case "sandbox:list": gatewayPinsAtSandboxList.push(process.env.OPENSHELL_GATEWAY); + if (!sandboxPresent && absentListIdentityProbeCall === null) { + absentListIdentityProbeCall = identityProbeCall; + } return { status: 0, stdout: sandboxListJson(sandboxPresent ? ["alpha"] : []), @@ -442,6 +449,8 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }; case "sandbox:delete": events.push("delete"); + sandboxPresent = false; + exactDockerCleanupPhase = true; return { status: options.deleteStatus === undefined ? 0 : options.deleteStatus, stdout: options.deleteOutput ?? "", @@ -470,7 +479,6 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr : names; return matchedNames.length > 0 ? `${matchedNames.join("\n")}\n` : ""; }); - let identityProbeCall = 0; let dockerOrphanIds = [...(options.dockerOrphanIds ?? [])]; let dockerNameLabeledIds = [ ...(options.dockerNameLabeledIds ?? options.dockerOrphanIds ?? []), @@ -522,12 +530,32 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr } identityProbeCall += 1; options.onDockerRun?.(identityProbeCall); + // After OpenShell reports the sandbox absent, destroy performs one + // pre-mutation and five execution continuity observations before exact + // post-delete cleanup. Track that boundary relative to the list call so + // earlier record-selection observations do not affect the mock. Once + // entered, keep the phase across a failed cleanup retry. + if ( + absentListIdentityProbeCall !== null && + identityProbeCall - absentListIdentityProbeCall > 6 + ) { + exactDockerCleanupPhase = true; + } const defaultIdentityResult = { status: 0, stdout: sandboxPresent ? "aaaaaaaaaaaa\topenshell\tdefault\tsb-alpha" : "", }; + const sequencedResult = options.dockerRunResultSequence?.[identityProbeCall - 1]; + const exactCleanupResult = { + status: options.dockerOrphanQueryStatus ?? 0, + stdout: dockerNameLabeledIds + .map((id) => `${id}\topenshell\tdefault\tsb-alpha`) + .join("\n"), + stderr: "", + }; const result = - options.dockerRunResultSequence?.[identityProbeCall - 1] ?? + (exactDockerCleanupPhase ? exactCleanupResult : undefined) ?? + sequencedResult ?? dockerIdentityResult ?? defaultIdentityResult; return result as ReturnType; @@ -683,6 +711,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }, setSandboxPresent: (present: boolean) => { sandboxPresent = present; + if (!present) exactDockerCleanupPhase = true; }, shieldsDownSpy, stopAllSpy, diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index a2ef55a596..f3f757a27e 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -981,14 +981,8 @@ if (${JSON.stringify( assert.match(result.stderr, /Do not delete the sandbox by mutable sandbox name/u); assert.match(result.stderr, /Shared inference providers are gateway configuration/u); assert.match(result.stderr, /not sandbox cleanup targets/u); - assert.match(result.stderr, /sandbox-scoped resources whose ownership is confirmed/u); assert.match(result.stderr, /nemoclaw my-assistant destroy/u); - assert.match(result.stderr, /credential environment name alone does not prove exposure/u); - assert.match( - result.stderr, - /rotate a credential only when identity-bound inspection proves/u, - ); - assert.doesNotMatch(result.stderr, /rotate any credential/u); + assert.match(result.stderr, /clear the matching recovery record/u); const differentName = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, diff --git a/test/onboarding/onboard-fsm-live-slices.test.ts b/test/onboarding/onboard-fsm-live-slices.test.ts index 4a8ec615bc..11ecc8a260 100644 --- a/test/onboarding/onboard-fsm-live-slices.test.ts +++ b/test/onboarding/onboard-fsm-live-slices.test.ts @@ -478,11 +478,6 @@ if (scenario.mode === "stale-recovery-admission") { verifiedEffectivePolicyIdentity: null, createAttemptNonce: "c".repeat(62), policyCreationReceipt: null, - resources: { - sharedInferenceProviders: [], - sandboxScopedProviders: [], - credentialEnvironmentVariables: [], - }, reason: "retained_after_sandbox_creation_failure", }); return []; From 4b0f6366eed88bd93e0a6d5a74362b5485524d4a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 15:38:31 -0700 Subject: [PATCH 06/24] docs(onboard): clarify retained identity recovery Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 11 +++++++---- src/lib/actions/sandbox/destroy-execution.ts | 6 +++--- src/lib/actions/sandbox/destroy-presence.ts | 14 -------------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index bdf2024c15..37ff5849fe 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -455,10 +455,11 @@ When that result is resumable, NemoClaw keeps the session `in_progress` at its l If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox and records its create-attempt label. When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. -Run `$$nemoclaw destroy` to complete identity-bound recovery. -Destroy proceeds only after it verifies one retained recovery record and the immutable Docker sandbox identity. -It removes only the qualified sandbox containers and clears the matching recovery record after verified cleanup. +When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to complete identity-bound recovery. +Destroy verifies one retained recovery record and the immutable Docker sandbox identity, removes only the qualified sandbox containers, and clears the matching recovery record after verified cleanup. A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. +If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. +Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. Do not delete the retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, supply a different explicit name: @@ -942,10 +943,12 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. If you cancel a brand-new onboarding run at the policy-tier selector or either policy-preset selector after sandbox creation, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. NemoClaw reports the durable sandbox identity fingerprint when it is available. It does not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. -Run `$$nemoclaw destroy` to complete cleanup through the retained immutable identity. +When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to complete cleanup through the retained immutable identity. Destroy removes only the qualified sandbox containers and clears the matching recovery record after it verifies their absence. If OpenShell already removed the sandbox, destroy can still clean up verified residual containers and retire the record. A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. +If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. +Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. Do not delete the sandbox manually by mutable name. NemoClaw stores the recovery record independently from the active onboarding session. A fresh run with a different name can proceed without clearing that record, but automatic resume, explicit `--resume`, reuse, recreation, and fresh onboarding with the retained name remain blocked. diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 1a69be1b46..e5e6622df8 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -35,10 +35,10 @@ import { classifyDestroyContainerIdentity, isSameDestroyContainerIdentityProof, observeDestroyContainerIdentity, - removeExactDestroyContainerIdentities, type DestroyContainerIdentityProof, type SandboxNameLabeledContainer, } from "./destroy-presence"; +import { removeExactOpenShellDockerSandboxContainers } from "../../onboard/openshell-docker-sandbox-containers"; import { type DestroyRunOpenshell, SANDBOX_DESTROY_TIMEOUT_MS } from "./destroy-gateway"; import { finalizeMcpBridgesAfterSandboxDelete, @@ -675,9 +675,9 @@ export async function executeSandboxDestroy({ if (portableContainerAuthority) { portableContainerAuthority.verifyAbsent(); } else if (expectedContainerIdentities !== undefined) { - removeExactDestroyContainerIdentities( + removeExactOpenShellDockerSandboxContainers( sandboxName, - expectedContainerIdentities, + expectedContainerIdentities.map(({ id }) => id), console.log, ); } diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts index d0bae776bc..a29204f56f 100644 --- a/src/lib/actions/sandbox/destroy-presence.ts +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -7,7 +7,6 @@ import { OPENSHELL_SANDBOX_ID_LABEL, OPENSHELL_SANDBOX_NAME_LABEL, OPENSHELL_SANDBOX_WORKSPACE_LABEL, - removeExactOpenShellDockerSandboxContainers, } from "../../onboard/openshell-docker-sandbox-containers"; import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; import { sanitizeReadinessText } from "../../readiness/sanitize"; @@ -78,19 +77,6 @@ export function observeDestroyContainerIdentity( return observeDockerSandboxIdentities(sandboxName); } -/** Retire the exact container set qualified from one retained recovery fingerprint. */ -export function removeExactDestroyContainerIdentities( - sandboxName: string, - expectedIdentities: readonly SandboxNameLabeledContainer[], - log: (message: string) => void, -): void { - removeExactOpenShellDockerSandboxContainers( - sandboxName, - expectedIdentities.map((identity) => identity.id), - log, - ); -} - /** * Classify every Docker container carrying `openshell.ai/sandbox-name=`. * The query intentionally does not filter by managed-by so a foreign container From 90c182630f979a94785c8d2088c673e05f17e76a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 16:10:59 -0700 Subject: [PATCH 07/24] fix(destroy): revalidate retained sandbox identity (#10547) Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 2 + src/lib/actions/sandbox/destroy-execution.ts | 56 ++++++++++++++++--- .../destroy-retained-recovery-flow.test.ts | 38 ++++++++++++- src/lib/actions/sandbox/destroy.ts | 8 +++ src/lib/onboard/lifecycle-contracts.md | 2 +- src/lib/state/onboard-session.ts | 14 +++-- .../retained-sandbox-recovery.ts | 32 ++++++++--- .../state/retained-sandbox-recovery.test.ts | 51 +++++++++++++++-- test/helpers/destroy-flow-test-harness.ts | 13 ++++- .../onboard-fresh-create-identity.test.ts | 2 - 10 files changed, 187 insertions(+), 31 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 37ff5849fe..56cf5bd23c 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -460,6 +460,7 @@ Destroy verifies one retained recovery record and the immutable Docker sandbox i A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. +If NemoClaw reports that it could not save the recovery evidence, preserve the terminal output and ask the administrator to use its create-attempt label for the same identity-bound procedure. Do not delete the retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, supply a different explicit name: @@ -949,6 +950,7 @@ If OpenShell already removed the sandbox, destroy can still clean up verified re A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. +If NemoClaw reports that it could not save the recovery evidence, preserve the terminal output and ask the administrator to use its create-attempt label for the same identity-bound procedure. Do not delete the sandbox manually by mutable name. NemoClaw stores the recovery record independently from the active onboarding session. A fresh run with a different name can proceed without clearing that record, but automatic resume, explicit `--resume`, reuse, recreation, and fresh onboarding with the retained name remain blocked. diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index e5e6622df8..4891266e5d 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -75,6 +75,10 @@ type SandboxDestroyExecutionInput = { // Docker IDs qualified before destroy preparation. expectedContainerIdentities?: readonly SandboxNameLabeledContainer[]; expectedContainerIdentityFingerprint?: string; + retainedRecoveryIdentity?: { + readonly fingerprint: string; + readonly gatewayName: string; + }; portableContainerAuthority?: PreparedPortableDemoSandboxDestroyAuthority; stopInferenceResources: () => void; runtimeProviders?: RuntimeProviderBundleRegistry; @@ -303,6 +307,7 @@ export async function executeSandboxDestroy({ sandboxName, expectedContainerIdentities, expectedContainerIdentityFingerprint, + retainedRecoveryIdentity, portableContainerAuthority, stopInferenceResources, runtimeProviders = CURRENT_RUNTIME_PROVIDER_BUNDLES, @@ -375,9 +380,40 @@ export async function executeSandboxDestroy({ }; } }; + const inspectRetainedRecoveryContinuity = (): IdentityContinuity => { + if ( + !retainedRecoveryIdentity || + pendingPolicyVerification || + sandboxConfirmedAbsent + ) { + return { status: "match" }; + } + try { + const inspectIdentity = + deps.inspectOpenShellSandboxIdentityFingerprint ?? + inspectOpenShellSandboxIdentityFingerprint; + return inspectIdentity({ + sandboxName, + gatewayName: retainedRecoveryIdentity.gatewayName, + }) === retainedRecoveryIdentity.fingerprint + ? { status: "match" } + : { + status: "changed", + subject: "Retained recovery sandbox identity", + }; + } catch (error) { + return { + status: "probe-failed", + subject: "Retained recovery sandbox identity", + detail: redactDestroyError(error), + }; + } + }; const inspectIdentityContinuity = (): IdentityContinuity => { const pendingContinuity = inspectPendingPolicyVerificationContinuity(); if (pendingContinuity.status !== "match") return pendingContinuity; + const retainedContinuity = inspectRetainedRecoveryContinuity(); + if (retainedContinuity.status !== "match") return retainedContinuity; if (portableContainerAuthority) { try { portableContainerAuthority.revalidate(); @@ -614,18 +650,24 @@ export async function executeSandboxDestroy({ const deleteArgs = pendingPolicyVerification ? ["sandbox", "delete", "-g", pendingPolicyVerification.gatewayName, sandboxName] : ["sandbox", "delete", sandboxName]; - const deleteResult = runOpenshell(deleteArgs, { - ignoreError: true, - killSignal: "SIGKILL", - stdio: ["ignore", "pipe", "pipe"], - timeout: SANDBOX_DESTROY_TIMEOUT_MS, - }); + // A successful preflight absence is already the required OpenShell + // lifecycle proof. Do not issue a later mutable-name delete that could + // target a same-name replacement created after that observation. + const deleteResult: ReturnType = sandboxConfirmedAbsent + ? { status: 0, stdout: "", stderr: "" } + : runOpenshell(deleteArgs, { + ignoreError: true, + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "pipe"], + timeout: SANDBOX_DESTROY_TIMEOUT_MS, + }); const { output: capturedDeleteOutput, - alreadyGone, + alreadyGone: deleteReportedAlreadyGone, gatewayUnreachable, timedOut, } = getSandboxDeleteOutcome(deleteResult); + const alreadyGone = sandboxConfirmedAbsent || deleteReportedAlreadyGone; const deleteOutput = timedOut ? `OpenShell sandbox delete timed out after ${String(SANDBOX_DESTROY_TIMEOUT_MS / 1000)} seconds. Deletion could not be confirmed.` : capturedDeleteOutput; diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index 31b7231905..fbf86ff1e2 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -17,7 +17,6 @@ function retainedRecoveryRecord(sandboxId = "sb-alpha"): RetainedSandboxRecovery recordId: "f".repeat(64), sandboxName: "alpha", sandboxIdentityFingerprint: createHash("sha256").update(sandboxId).digest("hex"), - identityWasUnavailable: false, gatewayName: "nemoclaw-19080", gatewayPort: 19080, lifecycleGeneration: "generation-alpha", @@ -174,6 +173,43 @@ describe("destroySandbox retained recovery flow", () => { }, ); + it( + "does not delete a same-name OpenShell replacement while retained Docker identities remain (#10547)", + { timeout: 30_000 }, + async () => { + const recovery = retainedRecoveryRecord(); + const containerId = "a".repeat(64); + const harness = createDestroyHarness({ + dockerRunResult: { + status: 0, + stdout: `${containerId}\topenshell\tdefault\tsb-alpha`, + }, + openShellSandboxIdentityFingerprint: createHash("sha256") + .update("sb-replacement") + .digest("hex"), + registryEntryOverrides: { + lifecycleGeneration: recovery.lifecycleGeneration!, + lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!, + }, + retainedRecoveryRecords: [recovery], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("Retained recovery sandbox identity changed"), + ); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.resolveRetainedSandboxRecoverySpy).not.toHaveBeenCalled(); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + }, + ); + it( "finishes retained cleanup after OpenShell already removed the sandbox (#10547)", { timeout: 30_000 }, diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 226af832a9..491a519a1c 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -721,6 +721,14 @@ async function destroySandboxUnlocked( ...(retainedSandboxIdentityFingerprint ? { expectedContainerIdentityFingerprint: retainedSandboxIdentityFingerprint } : {}), + ...(retainedRecoveryAuthority && retainedSandboxIdentityFingerprint + ? { + retainedRecoveryIdentity: { + fingerprint: retainedSandboxIdentityFingerprint, + gatewayName: retainedRecoveryAuthority.gatewayName, + }, + } + : {}), ...(portableContainerAuthority ? { portableContainerAuthority } : {}), stopInferenceResources: () => stopSandboxInferenceResources(sandboxName, sandbox), }); diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index af88df9817..541f36c4f4 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set before deletion, removes only remaining members of that set, verifies their absence, and clears the matching recovery record. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set and the live OpenShell identity on the recorded gateway before mutable-name deletion, removes only remaining members of that set, and verifies their absence. Confirmed OpenShell absence skips the mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. ## Effect-order flows diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index e55e0532da..f39e6154ce 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -60,8 +60,9 @@ import { listRetainedSandboxRecoveryRecords as readRetainedSandboxRecoveryRecords, parseNemoClawPolicyCreationReceipt, recordRetainedSandboxRecovery as writeRetainedSandboxRecovery, - resolveRetainedSandboxRecovery as retireRetainedSandboxRecovery, + retainedSandboxRecoveryAuthorityIsCurrent, retainedSandboxRecoveryFile, + resolveRetainedSandboxRecovery as retireRetainedSandboxRecovery, type RecordRetainedSandboxRecoveryInput, type RetainedSandboxRecoveryRecord, type RetainedSandboxRecoveryReason, @@ -2084,8 +2085,9 @@ export function retainedSandboxRecoveryMatchesSession( /** Clear one recovery-only session after destroy verifies the retained resources absent. */ export function resolveRetainedSandboxRecovery(record: RetainedSandboxRecoveryRecord): boolean { return withOwnedOnboardLock("nemoclaw retained sandbox recovery completion", () => { - const retired = retireRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, record); - if (!retired) return false; + if (!retainedSandboxRecoveryAuthorityIsCurrent(RETAINED_SANDBOX_RECOVERY_FILE, record)) { + return false; + } const current = loadSession(); if (current && retainedSandboxRecoveryMatchesSession(record, current)) { current.status = "failed"; @@ -2094,7 +2096,11 @@ export function resolveRetainedSandboxRecovery(record: RetainedSandboxRecoveryRe current.cancellationRecovery = null; saveSession(current); } - return true; + // Release the recovery-only session first. If this write fails, the exact + // independent record remains available for a later completion attempt. If + // record retirement then fails, that record still blocks only the retained + // name while a different explicitly named onboarding run can proceed. + return retireRetainedSandboxRecovery(RETAINED_SANDBOX_RECOVERY_FILE, record); }); } diff --git a/src/lib/state/onboard-session/retained-sandbox-recovery.ts b/src/lib/state/onboard-session/retained-sandbox-recovery.ts index 11c9412da0..902f53d0dd 100644 --- a/src/lib/state/onboard-session/retained-sandbox-recovery.ts +++ b/src/lib/state/onboard-session/retained-sandbox-recovery.ts @@ -37,7 +37,6 @@ export interface RetainedSandboxRecoveryRecord { readonly recordId: string; readonly sandboxName: string; readonly sandboxIdentityFingerprint: string | null; - readonly identityWasUnavailable: boolean; readonly gatewayName: string; readonly gatewayPort: number; readonly lifecycleGeneration: string | null; @@ -364,7 +363,6 @@ function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { !validSandboxName(value.sandboxName) || (fingerprint !== null && (typeof fingerprint !== "string" || !FINGERPRINT_PATTERN.test(fingerprint))) || - value.identityWasUnavailable !== (fingerprint === null) || !validSafeEvidence(value.gatewayName) || !validGatewayPort(value.gatewayPort) || (value.lifecycleGeneration !== null && !validSafeEvidence(value.lifecycleGeneration)) || @@ -391,7 +389,6 @@ function parseRecord(value: unknown): RetainedSandboxRecoveryRecord | null { recordId: value.recordId, sandboxName: value.sandboxName, sandboxIdentityFingerprint: fingerprint, - identityWasUnavailable: fingerprint === null, gatewayName: value.gatewayName, gatewayPort: value.gatewayPort, lifecycleGeneration: value.lifecycleGeneration, @@ -485,7 +482,6 @@ export function recordRetainedSandboxRecovery( recordId: recoveryRecordId(input), sandboxName: input.sandboxName, sandboxIdentityFingerprint: input.sandboxIdentityFingerprint, - identityWasUnavailable: input.sandboxIdentityFingerprint === null, gatewayName: input.gatewayName, gatewayPort: input.gatewayPort, lifecycleGeneration: input.lifecycleGeneration, @@ -520,17 +516,35 @@ export function recordRetainedSandboxRecovery( return reread; } -/** Retire only the unchanged record whose external resources were verified absent. */ -export function resolveRetainedSandboxRecovery( - filePath: string, +function retainedSandboxRecoveryAuthorityMatchesState( + state: RetainedSandboxRecoveryState, expected: RetainedSandboxRecoveryRecord, ): boolean { - const current = loadState(filePath); - const recorded = current.unresolved.find((candidate) => candidate.recordId === expected.recordId); + const recorded = state.unresolved.find( + (candidate) => candidate.recordId === expected.recordId, + ); if (!recorded) return false; if (!isDeepStrictEqual(recorded, expected)) { throw new Error("Retained sandbox recovery authority changed before cleanup completed."); } + return true; +} + +/** Confirm that the exact cleanup authority is still present and unchanged. */ +export function retainedSandboxRecoveryAuthorityIsCurrent( + filePath: string, + expected: RetainedSandboxRecoveryRecord, +): boolean { + return retainedSandboxRecoveryAuthorityMatchesState(loadState(filePath), expected); +} + +/** Retire only the unchanged record whose external resources were verified absent. */ +export function resolveRetainedSandboxRecovery( + filePath: string, + expected: RetainedSandboxRecoveryRecord, +): boolean { + const current = loadState(filePath); + if (!retainedSandboxRecoveryAuthorityMatchesState(current, expected)) return false; writeStateFile(filePath, { ...current, unresolved: current.unresolved.filter((candidate) => candidate.recordId !== expected.recordId), diff --git a/src/lib/state/retained-sandbox-recovery.test.ts b/src/lib/state/retained-sandbox-recovery.test.ts index 7027b9c0b6..1c4669b1c9 100644 --- a/src/lib/state/retained-sandbox-recovery.test.ts +++ b/src/lib/state/retained-sandbox-recovery.test.ts @@ -47,7 +47,6 @@ describe("retained sandbox recovery state", () => { expect(recorded).toMatchObject({ sandboxName: "retained-sb", sandboxIdentityFingerprint: fingerprint, - identityWasUnavailable: false, verifiedEffectivePolicyIdentity: input.verifiedEffectivePolicyIdentity, }); expect(fs.readFileSync(recovery.RETAINED_SANDBOX_RECOVERY_FILE, "utf8")).not.toContain( @@ -71,7 +70,6 @@ describe("retained sandbox recovery state", () => { expect(recorded).toMatchObject({ sandboxIdentityFingerprint: null, - identityWasUnavailable: true, lifecycleGeneration: null, }); }); @@ -311,7 +309,7 @@ describe("retained sandbox recovery state", () => { expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([]); }); - it("keeps the recovery-only session when record retirement cannot be written (#10547)", async () => { + it("keeps the exact record when retirement fails after session release (#10547)", async () => { const recovery = await import("./onboard-session"); recovery.markRetainedSandboxRecovery( "retained-sb", @@ -326,13 +324,54 @@ describe("retained sandbox recovery state", () => { }, ); const [recorded] = recovery.listRetainedSandboxRecoveryRecords(); - vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { - throw new Error("simulated recovery retirement write failure"); - }); + const renameSync = fs.renameSync.bind(fs); + vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => + String(destination) === recovery.RETAINED_SANDBOX_RECOVERY_FILE + ? (() => { + throw new Error("simulated recovery retirement write failure"); + })() + : renameSync(source, destination), + ); expect(() => recovery.resolveRetainedSandboxRecovery(recorded!)).toThrow( /simulated recovery retirement write failure/u, ); + expect(recovery.loadSession()).toMatchObject({ + status: "failed", + resumable: false, + sandboxName: null, + cancellationRecovery: null, + }); + expect(recovery.listRetainedSandboxRecoveryRecords()).toEqual([recorded]); + }); + + it("preserves the exact record when recovery-only session release cannot be written (#10547)", async () => { + const recovery = await import("./onboard-session"); + recovery.markRetainedSandboxRecovery( + "retained-sb", + "Sandbox creation failed after identity verification.", + "b".repeat(64), + { + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, + ...recoveryAuthority, + }, + ); + const [recorded] = recovery.listRetainedSandboxRecoveryRecords(); + const renameSync = fs.renameSync.bind(fs); + vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => + String(destination) === recovery.SESSION_FILE + ? (() => { + throw new Error("simulated recovery session release write failure"); + })() + : renameSync(source, destination), + ); + + expect(() => recovery.resolveRetainedSandboxRecovery(recorded!)).toThrow( + /simulated recovery session release write failure/u, + ); expect(recovery.loadSession()).toMatchObject({ status: "recovery_required", sandboxName: "retained-sb", diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 471306b1d0..9425548463 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; import { createRequire } from "node:module"; import { expect, type MockInstance, vi } from "vitest"; @@ -107,6 +108,7 @@ type DestroyHarnessOptions = { mcpAddState?: "prepared"; mcpServers?: string[]; openshellDriver?: string; + openShellSandboxIdentityFingerprint?: string; portableCommandError?: string; portableDestroyAuthority?: boolean; portableDestroyPrepareError?: string; @@ -142,6 +144,10 @@ const sandboxEntry = { gatewayPort: 19080, }; +const DEFAULT_OPENSHELL_SANDBOX_IDENTITY_FINGERPRINT = createHash("sha256") + .update("sb-alpha") + .digest("hex"); + export function sandboxListJson(names: string[]): string { return JSON.stringify( names.map((name) => ({ @@ -172,7 +178,7 @@ export function traceDestroyBoundaryCalls( harness.setSandboxPresent(false); return { status: 0, stdout: "", stderr: "" }; case "sandbox:list": - return { status: 0, stdout: "[]", stderr: "" }; + return { status: 0, stdout: sandboxListJson(["alpha"]), stderr: "" }; default: return { status: 0, stdout: "", stderr: "" }; } @@ -221,6 +227,11 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr "../../state/mcp-lifecycle-lock.js", ) as typeof import("../../src/lib/state/mcp-lifecycle-lock"); const registry = requireSource("../../state/registry.js"); + const policyAuthority = requireSource("../../adapters/openshell/policy-authority.js"); + vi.spyOn(policyAuthority, "inspectOpenShellSandboxIdentityFingerprint").mockReturnValue( + options.openShellSandboxIdentityFingerprint ?? + DEFAULT_OPENSHELL_SANDBOX_IDENTITY_FINGERPRINT, + ); const destroyExecution = requireSource("./destroy-execution.js"); const destroyCommand = requireSource("../../../commands/sandbox/destroy.js").default; const destroyPreflight = requireSource("./destroy-preflight.js"); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index f3f757a27e..f26a56fb0d 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -821,7 +821,6 @@ if (${JSON.stringify( const record = payload.retainedRecoveryRecords[0]; assert.equal(record.sandboxName, "my-assistant"); assert.equal(record.sandboxIdentityFingerprint, identityFingerprint); - assert.equal(record.identityWasUnavailable, false); assert.equal(record.gatewayName, "nemoclaw-18080"); assert.equal(record.gatewayPort, 18080); assert.match(record.lifecycleGeneration, /^[0-9a-f-]{36}$/u); @@ -865,7 +864,6 @@ if (${JSON.stringify( assert.equal(payload.retainedRecoveryRecords.length, 1); const record = payload.retainedRecoveryRecords[0]; assert.equal(record.sandboxName, "my-assistant"); - assert.equal(record.identityWasUnavailable, false); assert.equal(record.reason, "retained_after_sandbox_creation_failure"); assertRecoveryTuple(record); }; From 2c75b80c818f4bf16d67300f0e4c1b106857f32d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 16:25:51 -0700 Subject: [PATCH 08/24] fix(destroy): refuse mutable retained deletion (#10547) Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 14 ++++--- src/lib/actions/sandbox/destroy-execution.ts | 36 ------------------ .../destroy-retained-recovery-flow.test.ts | 22 +++++------ src/lib/actions/sandbox/destroy.ts | 16 +------- src/lib/onboard/lifecycle-contracts.md | 4 +- test/helpers/destroy-flow-test-harness.ts | 38 ++++++------------- 6 files changed, 34 insertions(+), 96 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 56cf5bd23c..1123b2f963 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -455,8 +455,9 @@ When that result is resumable, NemoClaw keeps the session `in_progress` at its l If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox and records its create-attempt label. When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. -When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to complete identity-bound recovery. -Destroy verifies one retained recovery record and the immutable Docker sandbox identity, removes only the qualified sandbox containers, and clears the matching recovery record after verified cleanup. +When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt identity-bound recovery. +Destroy completes recovery only after OpenShell confirms the retained sandbox is absent. It then verifies one retained recovery record and the immutable Docker sandbox identity, removes only the qualified residual containers, and clears the matching recovery record after verified cleanup. +If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. Ask an OpenShell administrator to resolve the create-attempt label and remove that exact sandbox through an identity-bound procedure, then rerun destroy. A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. @@ -536,7 +537,8 @@ $$nemoclaw onboard --fresh --apf-interceptor --name my-apf-sandbox If post-create verification or native GPU fallback fails after OpenShell may have created the sandbox, NemoClaw preserves the incomplete sandbox because automatic deletion would use its mutable name. -Run `$$nemoclaw destroy` to verify the retained immutable identity, remove only the qualified sandbox containers, and clear the matching recovery record. +Run `$$nemoclaw destroy` to verify the retained immutable identity. If OpenShell confirms the retained sandbox is absent, destroy removes only the qualified residual containers and clears the matching recovery record. +If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources. Ask an OpenShell administrator to resolve the create-attempt label, remove that exact sandbox through an identity-bound procedure, and then rerun destroy. A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. Do not delete the retained sandbox manually by mutable name. If OpenShell did not return an identity fingerprint, recovery remains blocked until an administrator resolves the create-attempt label to one exact sandbox. @@ -944,9 +946,9 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. If you cancel a brand-new onboarding run at the policy-tier selector or either policy-preset selector after sandbox creation, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. NemoClaw reports the durable sandbox identity fingerprint when it is available. It does not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. -When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to complete cleanup through the retained immutable identity. -Destroy removes only the qualified sandbox containers and clears the matching recovery record after it verifies their absence. -If OpenShell already removed the sandbox, destroy can still clean up verified residual containers and retire the record. +When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt cleanup through the retained immutable identity. +If OpenShell already removed the sandbox, destroy removes only the qualified residual containers and clears the matching recovery record after it verifies their absence. +If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. Ask an OpenShell administrator to resolve the create-attempt label and remove that exact sandbox through an identity-bound procedure, then rerun destroy. A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index 4891266e5d..70a4389f93 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -75,10 +75,6 @@ type SandboxDestroyExecutionInput = { // Docker IDs qualified before destroy preparation. expectedContainerIdentities?: readonly SandboxNameLabeledContainer[]; expectedContainerIdentityFingerprint?: string; - retainedRecoveryIdentity?: { - readonly fingerprint: string; - readonly gatewayName: string; - }; portableContainerAuthority?: PreparedPortableDemoSandboxDestroyAuthority; stopInferenceResources: () => void; runtimeProviders?: RuntimeProviderBundleRegistry; @@ -307,7 +303,6 @@ export async function executeSandboxDestroy({ sandboxName, expectedContainerIdentities, expectedContainerIdentityFingerprint, - retainedRecoveryIdentity, portableContainerAuthority, stopInferenceResources, runtimeProviders = CURRENT_RUNTIME_PROVIDER_BUNDLES, @@ -380,40 +375,9 @@ export async function executeSandboxDestroy({ }; } }; - const inspectRetainedRecoveryContinuity = (): IdentityContinuity => { - if ( - !retainedRecoveryIdentity || - pendingPolicyVerification || - sandboxConfirmedAbsent - ) { - return { status: "match" }; - } - try { - const inspectIdentity = - deps.inspectOpenShellSandboxIdentityFingerprint ?? - inspectOpenShellSandboxIdentityFingerprint; - return inspectIdentity({ - sandboxName, - gatewayName: retainedRecoveryIdentity.gatewayName, - }) === retainedRecoveryIdentity.fingerprint - ? { status: "match" } - : { - status: "changed", - subject: "Retained recovery sandbox identity", - }; - } catch (error) { - return { - status: "probe-failed", - subject: "Retained recovery sandbox identity", - detail: redactDestroyError(error), - }; - } - }; const inspectIdentityContinuity = (): IdentityContinuity => { const pendingContinuity = inspectPendingPolicyVerificationContinuity(); if (pendingContinuity.status !== "match") return pendingContinuity; - const retainedContinuity = inspectRetainedRecoveryContinuity(); - if (retainedContinuity.status !== "match") return retainedContinuity; if (portableContainerAuthority) { try { portableContainerAuthority.revalidate(); diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index fbf86ff1e2..e801fba666 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -49,7 +49,7 @@ describe("destroySandbox retained recovery flow", () => { }); it( - "removes every container from one retained failed attempt and clears recovery (#10547)", + "removes every container after OpenShell confirms the retained sandbox absent (#10547)", { timeout: 30_000 }, async () => { const recovery = retainedRecoveryRecord(); @@ -59,6 +59,7 @@ describe("destroySandbox retained recovery flow", () => { .map((id) => `${id}\topenshell\tdefault\tsb-alpha`) .join("\n"); const harness = createDestroyHarness({ + sandboxPresent: false, dockerOrphanIds: [bootstrapContainerId], dockerRunResult: { status: 0, stdout: identityRows }, registryEntryOverrides: { @@ -70,9 +71,9 @@ describe("destroySandbox retained recovery flow", () => { await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), + expect.anything(), ); expect(harness.dockerRunSpy).toHaveBeenCalledWith( ["rm", "-f", bootstrapContainerId], @@ -114,6 +115,7 @@ describe("destroySandbox retained recovery flow", () => { .map((id) => `${id}\topenshell\tdefault\tsb-alpha`) .join("\n"); const harness = createDestroyHarness({ + sandboxPresent: false, dockerNameLabeledIds: [bootstrapContainerId, foreignContainerId], dockerOrphanIds: [bootstrapContainerId], dockerRunResult: { status: 0, stdout: identityRows }, @@ -128,9 +130,9 @@ describe("destroySandbox retained recovery flow", () => { "process.exit(1)", ); - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), + expect.anything(), ); expect(harness.dockerRunSpy).not.toHaveBeenCalledWith( ["rm", "-f", expect.any(String)], @@ -162,7 +164,7 @@ describe("destroySandbox retained recovery flow", () => { ); expect(harness.errorSpy).toHaveBeenCalledWith( - expect.stringContaining("Docker exposed no container with the retained immutable identity"), + expect.stringContaining("delete command accepts only the mutable sandbox name"), ); expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], @@ -174,7 +176,7 @@ describe("destroySandbox retained recovery flow", () => { ); it( - "does not delete a same-name OpenShell replacement while retained Docker identities remain (#10547)", + "does not issue mutable-name deletion for a live retained sandbox with matching identity (#10547)", { timeout: 30_000 }, async () => { const recovery = retainedRecoveryRecord(); @@ -184,9 +186,6 @@ describe("destroySandbox retained recovery flow", () => { status: 0, stdout: `${containerId}\topenshell\tdefault\tsb-alpha`, }, - openShellSandboxIdentityFingerprint: createHash("sha256") - .update("sb-replacement") - .digest("hex"), registryEntryOverrides: { lifecycleGeneration: recovery.lifecycleGeneration!, lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!, @@ -199,7 +198,7 @@ describe("destroySandbox retained recovery flow", () => { ); expect(harness.errorSpy).toHaveBeenCalledWith( - expect.stringContaining("Retained recovery sandbox identity changed"), + expect.stringContaining("cannot bind that deletion to the retained immutable identity"), ); expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], @@ -274,6 +273,7 @@ describe("destroySandbox retained recovery flow", () => { .join("\n"); const harness = createDestroyHarness({ registryEntryPresent: false, + sandboxPresent: false, dockerOrphanIds: [bootstrapContainerId], dockerRunResult: { status: 0, stdout: identityRows }, retainedRecoveryRecords: [olderRecovery, matchingRecovery], diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 491a519a1c..7296af25d2 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -665,13 +665,9 @@ async function destroySandboxUnlocked( let destroyPreflight: ReturnType; destroyPreflight = abortPreparedCleanupOnError(() => prepareSandboxDestroy(sandboxName)); const { cleanupGatewayName, runOpenshell, sandbox, sandboxConfirmedAbsent } = destroyPreflight; - if ( - retainedRecoveryAuthority && - initialContainerIdentities?.length === 0 && - !sandboxConfirmedAbsent - ) { + if (retainedRecoveryAuthority && !sandboxConfirmedAbsent) { console.error( - ` Refusing to destroy retained sandbox '${sandboxName}': OpenShell still reports it present, but Docker exposed no container with the retained immutable identity. No sandbox resources were removed.`, + ` Refusing to automatically delete retained sandbox '${sandboxName}': OpenShell still reports it present, but its delete command accepts only the mutable sandbox name. NemoClaw cannot bind that deletion to the retained immutable identity. No sandbox resources were removed. Ask an OpenShell administrator to resolve create-attempt label '${retainedRecoveryAuthority.createAttemptNonce}' to the exact sandbox and use an identity-bound removal procedure. After OpenShell confirms the retained sandbox is absent, rerun '${CLI_NAME} ${sandboxName} destroy --yes' to reconcile its verified Docker containers and recovery record.`, ); preparedManagedLlamaCppCleanup?.abort(); requestSandboxDestroyExit(1); @@ -721,14 +717,6 @@ async function destroySandboxUnlocked( ...(retainedSandboxIdentityFingerprint ? { expectedContainerIdentityFingerprint: retainedSandboxIdentityFingerprint } : {}), - ...(retainedRecoveryAuthority && retainedSandboxIdentityFingerprint - ? { - retainedRecoveryIdentity: { - fingerprint: retainedSandboxIdentityFingerprint, - gatewayName: retainedRecoveryAuthority.gatewayName, - }, - } - : {}), ...(portableContainerAuthority ? { portableContainerAuthority } : {}), stopInferenceResources: () => stopSandboxInferenceResources(sandboxName, sandbox), }); diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 541f36c4f4..cd8a14bd7f 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set and the live OpenShell identity on the recorded gateway before mutable-name deletion, removes only remaining members of that set, and verifies their absence. Confirmed OpenShell absence skips the mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to the create-attempt label for identity-bound removal. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. ## Effect-order flows @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` uses that record to qualify immutable Docker container identities, complete interrupted cleanup, and retire the record. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` refuses mutable-name deletion while that sandbox is live. After administrator identity-bound removal, destroy uses the record to qualify immutable Docker container identities, complete residual cleanup, and retire the record. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 9425548463..206183c953 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash } from "node:crypto"; import { createRequire } from "node:module"; import { expect, type MockInstance, vi } from "vitest"; @@ -108,7 +107,6 @@ type DestroyHarnessOptions = { mcpAddState?: "prepared"; mcpServers?: string[]; openshellDriver?: string; - openShellSandboxIdentityFingerprint?: string; portableCommandError?: string; portableDestroyAuthority?: boolean; portableDestroyPrepareError?: string; @@ -144,10 +142,6 @@ const sandboxEntry = { gatewayPort: 19080, }; -const DEFAULT_OPENSHELL_SANDBOX_IDENTITY_FINGERPRINT = createHash("sha256") - .update("sb-alpha") - .digest("hex"); - export function sandboxListJson(names: string[]): string { return JSON.stringify( names.map((name) => ({ @@ -227,11 +221,18 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr "../../state/mcp-lifecycle-lock.js", ) as typeof import("../../src/lib/state/mcp-lifecycle-lock"); const registry = requireSource("../../state/registry.js"); - const policyAuthority = requireSource("../../adapters/openshell/policy-authority.js"); - vi.spyOn(policyAuthority, "inspectOpenShellSandboxIdentityFingerprint").mockReturnValue( - options.openShellSandboxIdentityFingerprint ?? - DEFAULT_OPENSHELL_SANDBOX_IDENTITY_FINGERPRINT, + const openShellDockerContainers = requireSource( + "../../onboard/openshell-docker-sandbox-containers.js", ); + const removeExactDockerContainers = + openShellDockerContainers.removeExactOpenShellDockerSandboxContainers; + vi.spyOn( + openShellDockerContainers, + "removeExactOpenShellDockerSandboxContainers", + ).mockImplementation((...args: Parameters) => { + exactDockerCleanupPhase = true; + return removeExactDockerContainers(...args); + }); const destroyExecution = requireSource("./destroy-execution.js"); const destroyCommand = requireSource("../../../commands/sandbox/destroy.js").default; const destroyPreflight = requireSource("./destroy-preflight.js"); @@ -436,7 +437,6 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }); const gatewayPinsAtSandboxList: Array = []; let identityProbeCall = 0; - let absentListIdentityProbeCall: number | null = null; const runOpenshellSpy = vi.spyOn(runtime, "runOpenshell").mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args : []; switch (`${String(argv[0])}:${String(argv[1])}`) { @@ -450,9 +450,6 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }; case "sandbox:list": gatewayPinsAtSandboxList.push(process.env.OPENSHELL_GATEWAY); - if (!sandboxPresent && absentListIdentityProbeCall === null) { - absentListIdentityProbeCall = identityProbeCall; - } return { status: 0, stdout: sandboxListJson(sandboxPresent ? ["alpha"] : []), @@ -461,7 +458,6 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr case "sandbox:delete": events.push("delete"); sandboxPresent = false; - exactDockerCleanupPhase = true; return { status: options.deleteStatus === undefined ? 0 : options.deleteStatus, stdout: options.deleteOutput ?? "", @@ -541,17 +537,6 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr } identityProbeCall += 1; options.onDockerRun?.(identityProbeCall); - // After OpenShell reports the sandbox absent, destroy performs one - // pre-mutation and five execution continuity observations before exact - // post-delete cleanup. Track that boundary relative to the list call so - // earlier record-selection observations do not affect the mock. Once - // entered, keep the phase across a failed cleanup retry. - if ( - absentListIdentityProbeCall !== null && - identityProbeCall - absentListIdentityProbeCall > 6 - ) { - exactDockerCleanupPhase = true; - } const defaultIdentityResult = { status: 0, stdout: sandboxPresent ? "aaaaaaaaaaaa\topenshell\tdefault\tsb-alpha" : "", @@ -722,7 +707,6 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }, setSandboxPresent: (present: boolean) => { sandboxPresent = present; - if (!present) exactDockerCleanupPhase = true; }, shieldsDownSpy, stopAllSpy, From fd51e39600cc960bce0fa11b031f1a377ec26b7f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 16:54:50 -0700 Subject: [PATCH 09/24] fix(onboard): surface retained recovery identity Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 26 ++++--------- ...uild-baseline-transition-preflight.test.ts | 37 +++++++++++++++++++ .../sandbox/rebuild-preflight-guards.ts | 20 ++++++++++ .../sandbox/rebuild-preflight-phase.ts | 2 + src/lib/onboard/cancel-rollback.test.ts | 20 +++++++++- src/lib/onboard/cancel-rollback.ts | 10 ++++- src/lib/onboard/lifecycle-contracts.md | 2 +- .../sandbox-create/orchestration.test.ts | 6 ++- .../onboard/sandbox-create/orchestration.ts | 22 +++++++++++ .../onboard-fresh-create-identity.test.ts | 19 ++++++++++ 10 files changed, 139 insertions(+), 25 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 6bb30b9a8e..5dd44003cb 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -452,12 +452,14 @@ For a new or fresh session, `--yes` and `NEMOCLAW_YES=1` accept supported confir If onboarding returns without reaching the final `complete` state, the command exits with status `1`. When that result is resumable, NemoClaw keeps the session `in_progress` at its last checkpoint instead of marking it failed, so correct the reported condition and run `$$nemoclaw onboard --resume`. -If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox and records its create-attempt label. +#### Recover a retained sandbox + +If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox, records its create-attempt label, and prints the exact `ai.nvidia.nemoclaw.create-attempt=` selector. When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt identity-bound recovery. Destroy completes recovery only after OpenShell confirms the retained sandbox is absent. It then verifies one retained recovery record and the immutable Docker sandbox identity, removes only the qualified residual containers, and clears the matching recovery record after verified cleanup. -If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. Ask an OpenShell administrator to resolve the create-attempt label and remove that exact sandbox through an identity-bound procedure, then rerun destroy. +If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. Give the displayed create-attempt label to an OpenShell administrator, ask them to remove that exact sandbox through an identity-bound procedure, then rerun destroy. A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. @@ -537,11 +539,7 @@ $$nemoclaw onboard --fresh --apf-interceptor --name my-apf-sandbox If post-create verification or native GPU fallback fails after OpenShell may have created the sandbox, NemoClaw preserves the incomplete sandbox because automatic deletion would use its mutable name. -Run `$$nemoclaw destroy` to verify the retained immutable identity. If OpenShell confirms the retained sandbox is absent, destroy removes only the qualified residual containers and clears the matching recovery record. -If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources. Ask an OpenShell administrator to resolve the create-attempt label, remove that exact sandbox through an identity-bound procedure, and then rerun destroy. -A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. -Do not delete the retained sandbox manually by mutable name. -If OpenShell did not return an identity fingerprint, recovery remains blocked until an administrator resolves the create-attempt label to one exact sandbox. +Follow the [retained-sandbox recovery procedure](#recover-a-retained-sandbox); its OpenShell-absence, immutable Docker identity, and administrator requirements also apply to APF creation. This onboarding mode does not support `--resume` or `--recreate-sandbox`, regardless of whether sandbox creation began. After destroy completes, repeat the original command with `--fresh`. @@ -946,18 +944,8 @@ Pairing and `TELEGRAM_ALLOWED_IDS` still govern direct messages. If you cancel a brand-new onboarding run at the policy-tier selector or either policy-preset selector after sandbox creation, NemoClaw preserves the incomplete sandbox, registry entry, and onboarding session for identity-bound recovery. NemoClaw reports the durable sandbox identity fingerprint when it is available. It does not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. -When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt cleanup through the retained immutable identity. -If OpenShell already removed the sandbox, destroy removes only the qualified residual containers and clears the matching recovery record after it verifies their absence. -If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. Ask an OpenShell administrator to resolve the create-attempt label and remove that exact sandbox through an identity-bound procedure, then rerun destroy. -A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. -If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. -Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. -If NemoClaw reports that it could not save the recovery evidence, preserve the terminal output and ask the administrator to use its create-attempt label for the same identity-bound procedure. -Do not delete the sandbox manually by mutable name. -NemoClaw stores the recovery record independently from the active onboarding session. -A fresh run with a different name can proceed without clearing that record, but automatic resume, explicit `--resume`, reuse, recreation, and fresh onboarding with the retained name remain blocked. -Start another onboarding run with `$$nemoclaw onboard --name `. -`--fresh` alone does not clear the recovery record or permit reuse of the retained sandbox name. +Follow the [retained-sandbox recovery procedure](#recover-a-retained-sandbox) to reconcile this cancellation. +A fresh run with a different explicit name can continue while the cancelled sandbox name remains blocked. If you run onboarding again with the same sandbox name and choose a different inference provider or model, NemoClaw detects the drift and recreates the sandbox so the running agent config matches your selection. In interactive mode, the wizard asks for confirmation before delete and recreate. diff --git a/src/lib/actions/sandbox/rebuild-baseline-transition-preflight.test.ts b/src/lib/actions/sandbox/rebuild-baseline-transition-preflight.test.ts index 9fd0e7511c..f1fd945ffd 100644 --- a/src/lib/actions/sandbox/rebuild-baseline-transition-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-baseline-transition-preflight.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ confirmRebuildIntent: vi.fn(), countActiveSessions: vi.fn(), getSandbox: vi.fn(), + listRetainedRecovery: vi.fn(), prepareTargets: vi.fn(), })); @@ -17,6 +18,11 @@ vi.mock("../../state/registry", async (importOriginal) => ({ getSandbox: mocks.getSandbox, })); +vi.mock("../../state/onboard-session", async (importOriginal) => ({ + ...(await importOriginal()), + listRetainedSandboxRecoveryRecords: mocks.listRetainedRecovery, +})); + vi.mock("./mcp-bridge-state", async (importOriginal) => ({ ...(await importOriginal()), assertMcpDestroyNotPending: mocks.assertMcpDestroyNotPending, @@ -47,9 +53,39 @@ afterEach(() => { vi.restoreAllMocks(); }); +describe("rebuild retained sandbox recovery preflight (#10547)", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getSandbox.mockReturnValue({ name: "alpha", openshellDriver: "docker" }); + mocks.listRetainedRecovery.mockReturnValue([ + { + recordId: "f".repeat(64), + sandboxName: "alpha", + }, + ]); + }); + + it("stops before session probes, confirmation, MCP checks, or target preparation", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(runRebuildPreflightPhase("alpha", ["--yes"])).resolves.toBeNull(); + + expect(mocks.bail).toHaveBeenCalledWith( + "Retained sandbox recovery blocks rebuild for 'alpha'.", + 1, + ); + expect(error.mock.calls.flat().join("\n")).toContain("alpha destroy --yes"); + expect(mocks.countActiveSessions).not.toHaveBeenCalled(); + expect(mocks.assertMcpDestroyNotPending).not.toHaveBeenCalled(); + expect(mocks.confirmRebuildIntent).not.toHaveBeenCalled(); + expect(mocks.prepareTargets).not.toHaveBeenCalled(); + }); +}); + describe("rebuild baseline transition preflight (#7194)", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.listRetainedRecovery.mockReturnValue([]); mocks.getSandbox.mockReturnValue({ name: "alpha", baselineExclusionTransition: { @@ -84,6 +120,7 @@ describe("rebuild baseline transition preflight (#7194)", () => { describe("rebuild MCP destroy marker preflight (#7794)", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.listRetainedRecovery.mockReturnValue([]); mocks.getSandbox.mockReturnValue({ name: "alpha", agent: "openclaw", diff --git a/src/lib/actions/sandbox/rebuild-preflight-guards.ts b/src/lib/actions/sandbox/rebuild-preflight-guards.ts index 0d8ca2ae3a..c5c0c153dd 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-guards.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-guards.ts @@ -314,6 +314,26 @@ export function getRebuildSandboxEntryOrBail( return sb; } +/** Block rebuild before any live-state probe or cleanup can bypass retained recovery. */ +export function blockRebuildOnRetainedSandboxRecovery( + sandboxName: string, + bail: RebuildBail, +): boolean { + const retainedRecovery = onboardSession + .listRetainedSandboxRecoveryRecords() + .find((record) => record.sandboxName === sandboxName); + if (!retainedRecovery) return false; + + console.error( + ` Rebuild cannot use retained sandbox '${sandboxName}' while recovery record '${retainedRecovery.recordId}' is unresolved. No sandbox or Docker resources were removed.`, + ); + console.error( + ` Run '${CLI_NAME} ${sandboxName} destroy --yes'. If OpenShell still reports the sandbox present, follow destroy's create-attempt label guidance for identity-bound administrator removal.`, + ); + bail(`Retained sandbox recovery blocks rebuild for '${sandboxName}'.`, 1); + return true; +} + /** Keep the pending baseline-policy transaction guard identical at every rebuild boundary. */ export function blockRebuildOnPendingBaselineTransition( sandboxEntry: RebuildSandboxEntry, diff --git a/src/lib/actions/sandbox/rebuild-preflight-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-phase.ts index caa7b676ab..ecf1391e17 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-phase.ts @@ -43,6 +43,7 @@ import { acquireRebuildOnboardLock, assertRebuildEntryUnchanged, blockRebuildOnPendingBaselineTransition, + blockRebuildOnRetainedSandboxRecovery, checkRebuildGatewaySchemaPreflight, expectedRebuildEntryAfterVersionCheck, getRebuildSandboxEntryOrBail, @@ -137,6 +138,7 @@ export async function runRebuildPreflightPhase( } = createRebuildCommandContext(options, opts); const sandboxEntry = getRebuildSandboxEntryOrBail(sandboxName, bail); if (!sandboxEntry) return null; + if (blockRebuildOnRetainedSandboxRecovery(sandboxName, bail)) return null; if (blockRebuildOnPendingBaselineTransition(sandboxEntry, sandboxName, bail)) return null; const activeSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); // #6376: refuse a stuck MCP destroy transaction up front — before backup, diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 422799efbe..8b32616c4b 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -11,6 +11,14 @@ import { } from "./cancel-rollback"; const SANDBOX_FINGERPRINT = "a".repeat(64); +const RECOVERY_CONTEXT = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-alpha", + verifiedEffectivePolicyIdentity: null, + createAttemptNonce: "c".repeat(62), + policyCreationReceipt: null, +} as const; function createHarness() { const log = vi.fn(); @@ -21,13 +29,16 @@ describe("createSandboxCancelRollback", () => { it("preserves an armed cancelled sandbox and reports its captured identity (#9833)", () => { const { rollback, log } = createHarness(); - rollback.arm("new-sb", SANDBOX_FINGERPRINT); + rollback.arm("new-sb", SANDBOX_FINGERPRINT, RECOVERY_CONTEXT); rollback.markCancelled(); rollback.runIfArmed(); const guidance = log.mock.calls.flat().join("\n"); expect(guidance).toContain("preserved incomplete sandbox 'new-sb'"); expect(guidance).toContain(SANDBOX_FINGERPRINT); + expect(guidance).toContain( + `ai.nvidia.nemoclaw.create-attempt=${RECOVERY_CONTEXT.createAttemptNonce}`, + ); expect(guidance).toContain("did not run OpenShell's mutable-name deletion command"); expect(guidance).toContain("Do not delete the sandbox by mutable sandbox name"); expect(guidance).toContain("Shared inference providers are gateway configuration"); @@ -292,10 +303,15 @@ describe("makeOnboardCancelExit", () => { describe("buildCancelRollbackMessage", () => { it("preserves identity-bound recovery guidance", () => { - const message = buildCancelRollbackMessage("sb", SANDBOX_FINGERPRINT).join("\n"); + const message = buildCancelRollbackMessage( + "sb", + SANDBOX_FINGERPRINT, + RECOVERY_CONTEXT, + ).join("\n"); expect(message).toContain("preserved incomplete sandbox 'sb'"); expect(message).toContain(SANDBOX_FINGERPRINT); + expect(message).toContain(RECOVERY_CONTEXT.createAttemptNonce); expect(message).toContain("identity-bound inspection, recovery, or removal"); expect(message).not.toContain("openshell sandbox delete"); expect(message).not.toContain("cannot delete it by immutable identity"); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index 4feb1f20dc..03c63e7341 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../adapters/openshell/sandbox-identity"; import type { RetainedSandboxRecoveryContext } from "../state/onboard-session"; import { cliName } from "./branding"; @@ -55,10 +56,16 @@ export interface SandboxCancelRollback { export function buildCancelRollbackMessage( sandboxName: string, sandboxIdentityFingerprint?: string, + recoveryContext?: Pick, ): string[] { return [ "", ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}'.`, + ...(recoveryContext + ? [ + ` Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${recoveryContext.createAttemptNonce}`, + ] + : []), ...(sandboxIdentityFingerprint ? [ ` Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}`, @@ -73,7 +80,7 @@ export function buildCancelRollbackMessage( " Shared inference providers are gateway configuration and are not sandbox cleanup targets.", ...(sandboxIdentityFingerprint ? [ - ` Run '${cliName()} ${sandboxName} destroy' to verify the retained immutable identity, remove only its qualified resources, and clear the matching recovery record.`, + ` Run '${cliName()} ${sandboxName} destroy'. If OpenShell confirms the retained sandbox absent, destroy removes only verified residual containers and can clear the matching recovery record. If it is still live, give the displayed create-attempt label to an OpenShell administrator for identity-bound removal.`, ] : [ " NemoClaw cannot clear this recovery record until an OpenShell administrator establishes the exact sandbox identity.", @@ -204,6 +211,7 @@ export function createSandboxCancelRollback( for (const line of buildCancelRollbackMessage( sandboxName, identityFingerprint ?? undefined, + armedSandbox.context, )) { deps.log(line); } diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 7331888cc3..ba17d47d64 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to the create-attempt label for identity-bound removal. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint and reports the exact create-attempt label. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to that label for identity-bound removal. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Rebuild and same-name onboarding remain blocked until recovery completes. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. ## Effect-order flows diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 2b3bf8d4aa..e87c98387b 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -912,6 +912,7 @@ describe("sandbox create policy authority checks", () => { it("removes temporary sources but preserves the sandbox after final authority failure (#9833)", async () => { const events: string[] = []; + const createAttemptNonce = "c".repeat(62); const revalidate = vi.fn(() => events.push("create-check")); const error = await runSandboxCreateWithPolicyAuthorityChecks({ @@ -923,6 +924,7 @@ describe("sandbox create policy authority checks", () => { return "created"; }, ...exactIdentityBoundary(), + captureCreatedSandboxCreateAttemptNonce: () => createAttemptNonce, revalidateVerifiedPolicy: () => { events.push("ready-check"); throw new Error("external policy authority changed"); @@ -933,7 +935,7 @@ describe("sandbox create policy authority checks", () => { expect(error).toBeInstanceOf(AggregateError); expect((error as AggregateError).message).toMatch( new RegExp( - `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*did not run OpenShell's mutable-name deletion command.*Do not delete the sandbox by mutable sandbox name.*OpenShell administrator.*identity-bound recovery or removal procedure`, + `Create-attempt label: ai\\.nvidia\\.nemoclaw\\.create-attempt=${createAttemptNonce}.*left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*did not run OpenShell's mutable-name deletion command.*Do not delete the sandbox by mutable sandbox name.*OpenShell administrator.*identity-bound recovery or removal procedure`, "u", ), ); @@ -942,7 +944,7 @@ describe("sandbox create policy authority checks", () => { expect.objectContaining({ message: expect.stringMatching( new RegExp( - `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*did not run OpenShell's mutable-name deletion command.*Do not delete the sandbox by mutable sandbox name.*OpenShell administrator.*identity-bound recovery or removal procedure`, + `Create-attempt label: ai\\.nvidia\\.nemoclaw\\.create-attempt=${createAttemptNonce}.*left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*did not run OpenShell's mutable-name deletion command.*Do not delete the sandbox by mutable sandbox name.*OpenShell administrator.*identity-bound recovery or removal procedure`, "u", ), ), diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index c570b890f7..09b757cf65 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -12,6 +12,7 @@ import { } from "../../adapters/openshell/policy-authority"; import type { SandboxPolicyAuthority } from "../../adapters/openshell/policy-authority"; import { HERMES_PORTABLE_OPENSHELL_VERSION } from "../../adapters/openshell/resolve-shared"; +import { NEMOCLAW_CREATE_ATTEMPT_LABEL } from "../../adapters/openshell/sandbox-identity"; import type { AgentDefinition } from "../../agent/defs"; import type { WebSearchConfig } from "../../inference/web-search"; import type { SandboxMessagingPlan } from "../../messaging/manifest"; @@ -216,9 +217,11 @@ export function persistPostCreateRecovery(input: { ) => unknown | null; }): void { const message = + `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${input.recoveryContext.createAttemptNonce}. ` + `Sandbox '${input.sandboxName}' was retained after ${input.stage} failed. ` + `Gateway '${input.gatewayName}'. Lifecycle generation '${input.lifecycleGeneration}'. ` + "Do not delete the sandbox by mutable name; preserve it for identity-bound administrator recovery."; + console.error(` ${message}`); let persisted = false; try { persisted = persistRetainedSandboxRecoveryMessage( @@ -656,6 +659,7 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< readonly revalidate: (sandboxIsLive: boolean, operation: string) => void; readonly create: (verifyCreatedSandbox: (created: Created) => Promise) => Promise; readonly captureCreatedSandboxIdentity: (created: Created) => string; + readonly captureCreatedSandboxCreateAttemptNonce?: (created: Created) => string; readonly persistCreatedSandboxIdentity: (created: Created, exactIdentity: string) => void; readonly revalidateCreatedSandboxIdentity: (expectedIdentity: string, operation: string) => void; readonly verifyCreatedPolicy: (created: Created, exactIdentity: string) => Evidence; @@ -686,6 +690,7 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< }): Promise { input.revalidate(false, `creating sandbox '${input.sandboxName}'`); let exactIdentity: string | null = null; + let createAttemptNonce: string | null = null; let observedPolicyEvidence: Evidence | null = null; let observedCreatedSandbox: Created | null = null; let cleanupAttempted = false; @@ -709,7 +714,11 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< const identityGuidance = exactIdentity ? `Durable sandbox identity fingerprint: ${exactIdentity}. Use it only to compare the surviving sandbox with the failed create.` : "OpenShell did not return a durable sandbox identity fingerprint for comparison."; + const createAttemptGuidance = createAttemptNonce + ? `Create-attempt label: ${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${createAttemptNonce}. ` + : ""; const recoveryGuidance = + createAttemptGuidance + `NemoClaw left sandbox '${input.sandboxName}' in place after post-create verification or finalization failed. ` + `${identityGuidance} NemoClaw did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox. ` + `Do not delete the sandbox by mutable sandbox name. Run '${cliName()} ${input.sandboxName} destroy' to use the retained identity. ` + @@ -739,6 +748,16 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< const verifyCreatedSandbox = async (created: Created): Promise => { observedCreatedSandbox = created; try { + const capturedCreateAttemptNonce = input.captureCreatedSandboxCreateAttemptNonce?.(created); + if ( + capturedCreateAttemptNonce !== undefined && + !/^[0-9a-f]{62}$/u.test(capturedCreateAttemptNonce) + ) { + throw new Error( + `OpenShell did not return one exact create-attempt label for sandbox '${input.sandboxName}'.`, + ); + } + createAttemptNonce = capturedCreateAttemptNonce ?? null; const capturedIdentity = input.captureCreatedSandboxIdentity(created); if (!/^[0-9a-f]{64}$/u.test(capturedIdentity)) { throw new Error( @@ -2635,6 +2654,9 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche captureCreatedSandboxIdentity: ( identity: import("../sandbox-gpu-create-flow").CreatedSandboxIdentity, ) => identity.liveIdentityFingerprint, + captureCreatedSandboxCreateAttemptNonce: ( + identity: import("../sandbox-gpu-create-flow").CreatedSandboxIdentity, + ) => identity.createAttemptNonce, persistCreatedSandboxIdentity: (_identity, exactIdentity) => persistCreatedSandboxIdentity(exactIdentity), revalidateCreatedSandboxIdentity, diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index f423096a56..b526635f73 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -851,12 +851,25 @@ if (${JSON.stringify( false, ); }; + const assertCreateAttemptLabelReported = () => { + const match = payload.createCommand?.match( + /--label (ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62})/u, + ); + assert.ok(match?.[1], "expected the sandbox create-attempt label"); + assert.ok( + `${payload.creationError ?? ""}\n${result.stderr}`.includes( + `Create-attempt label: ${match[1]}`, + ), + "expected recovery output to report the exact create-attempt label", + ); + }; const assertPostCreateAuthorityRefusal = () => { assert.equal(payload.sandboxName, null); assert.equal(payload.sandboxCreated, true); assert.equal(payload.deleted, false); assert.match(payload.creationError, /left sandbox 'my-assistant' in place/u); assert.match(payload.creationError, new RegExp(identityFingerprint, "u")); + assertCreateAttemptLabelReported(); assert.match( payload.creationError, /did not run OpenShell's mutable-name deletion command because the name may now identify a replacement sandbox/u, @@ -892,6 +905,7 @@ if (${JSON.stringify( assert.equal(payload.deleted, false); assert.equal(payload.registeredSandbox, null); assert.match(payload.creationError, /automatic sandbox cleanup was not safe/u); + assertCreateAttemptLabelReported(); assert.equal(payload.savedSession.status, "recovery_required"); assert.equal(payload.savedSession.resumable, false); assert.equal( @@ -914,6 +928,7 @@ if (${JSON.stringify( assert.equal(payload.deleted, false); assert.equal(payload.registeredSandbox, null); assert.match(payload.creationError, /registry publication failed/u); + assertCreateAttemptLabelReported(); assert.equal(payload.savedSession.status, "recovery_required"); assert.equal(payload.savedSession.resumable, false); assert.equal( @@ -932,6 +947,7 @@ if (${JSON.stringify( assert.equal(payload.deleted, false); assert.equal(payload.registeredSandbox, null); assert.match(payload.creationError, /recovery record could not be persisted/u); + assertCreateAttemptLabelReported(); assert.equal(payload.savedSession.status, "recovery_required"); assert.equal(payload.savedSession.resumable, false); assert.equal( @@ -978,6 +994,7 @@ if (${JSON.stringify( assert.equal(payload.deleted, false); assert.equal(payload.registeredSandbox, null); assert.match(payload.creationError, /recovery record could not be persisted/u); + assertCreateAttemptLabelReported(); assert.equal(payload.savedSession.status, "recovery_required"); assert.equal(payload.savedSession.resumable, false); assert.equal(payload.retainedRecoveryRecords.length, 1); @@ -998,6 +1015,7 @@ if (${JSON.stringify( payload.creationError, /OpenShell sandbox policy authority inspection failed/u, ); + assertCreateAttemptLabelReported(); assert.equal(payload.savedSession.status, "recovery_required"); assert.equal(payload.savedSession.resumable, false); assert.equal( @@ -1040,6 +1058,7 @@ if (${JSON.stringify( assert.match(result.stderr, /not sandbox cleanup targets/u); assert.match(result.stderr, /nemoclaw my-assistant destroy/u); assert.match(result.stderr, /clear the matching recovery record/u); + assertCreateAttemptLabelReported(); const differentName = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, From 812bd03f8ee4cb44daadabf7814b89bf0020660b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 17:50:03 -0700 Subject: [PATCH 10/24] test(destroy): model verified container removal Signed-off-by: Prekshi Vyas --- test/cli/destroy-gateway-cleanup.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/cli/destroy-gateway-cleanup.test.ts b/test/cli/destroy-gateway-cleanup.test.ts index da17fe7e8c..11aa990a5a 100644 --- a/test/cli/destroy-gateway-cleanup.test.ts +++ b/test/cli/destroy-gateway-cleanup.test.ts @@ -9,7 +9,19 @@ import { describe, expect, it } from "vitest"; import { runWithEnv, testTimeoutOptions } from "./helpers"; const LIVE_DOCKER_IDENTITY = `#!/bin/sh -case "$*" in *'.Label '*) printf 'aaaaaaaaaaaa\topenshell\tdefault\tsb-alpha\n' ;; esac +removed_marker="$0.removed" +case "$1" in + ps) + if [ ! -e "$removed_marker" ]; then + printf 'aaaaaaaaaaaa\topenshell\tdefault\tsb-alpha\n' + fi + ;; + rm) + if [ "$2" = "-f" ] && [ "$3" = "aaaaaaaaaaaa" ]; then + : > "$removed_marker" + fi + ;; +esac exit 0 `; From 2a9f7e62305f2c38cedd8edfe6ea7f5fef255a90 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 18:03:53 -0700 Subject: [PATCH 11/24] fix(destroy): fail closed on recovery conflict Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/destroy-presence.ts | 12 +----- .../destroy-retained-recovery-flow.test.ts | 42 +++++++++++++++++++ src/lib/actions/sandbox/destroy.ts | 12 +++++- .../openshell-docker-sandbox-containers.ts | 2 +- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts index a29204f56f..78ef7b1017 100644 --- a/src/lib/actions/sandbox/destroy-presence.ts +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -7,12 +7,12 @@ import { OPENSHELL_SANDBOX_ID_LABEL, OPENSHELL_SANDBOX_NAME_LABEL, OPENSHELL_SANDBOX_WORKSPACE_LABEL, + inspectDockerSandboxNameLabeledContainers, } from "../../onboard/openshell-docker-sandbox-containers"; import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; import { sanitizeReadinessText } from "../../readiness/sanitize"; import { type DockerSandboxIdentityObservation, - inspectDockerSandboxIdentities, } from "../../adapters/docker/inspect"; import { classifyOpenShellSandboxPresence, @@ -62,19 +62,11 @@ export type DestroyContainerIdentityProof = { identities?: readonly SandboxNameLabeledContainer[]; }; -function observeDockerSandboxIdentities(sandboxName: string): DockerSandboxIdentityObservation { - return inspectDockerSandboxIdentities(`${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, { - managedBy: OPENSHELL_MANAGED_BY_LABEL, - workspace: OPENSHELL_SANDBOX_WORKSPACE_LABEL, - sandboxId: OPENSHELL_SANDBOX_ID_LABEL, - }); -} - /** Read the host observation consumed by the pure identity classifier. */ export function observeDestroyContainerIdentity( sandboxName: string, ): DockerSandboxIdentityObservation { - return observeDockerSandboxIdentities(sandboxName); + return inspectDockerSandboxNameLabeledContainers(sandboxName); } /** diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index e801fba666..fb10fa4e2b 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -256,6 +256,48 @@ describe("destroySandbox retained recovery flow", () => { }, ); + it( + "does not report success when retained recovery retirement loses authority (#10547)", + { timeout: 30_000 }, + async () => { + const recovery = retainedRecoveryRecord(); + const bootstrapContainerId = "b".repeat(64); + const harness = createDestroyHarness({ + sandboxPresent: false, + dockerOrphanIds: [bootstrapContainerId], + dockerRunResult: { + status: 0, + stdout: `${bootstrapContainerId}\topenshell\tdefault\tsb-alpha`, + }, + registryEntryOverrides: { + lifecycleGeneration: recovery.lifecycleGeneration!, + lifecycleLiveIdentityFingerprint: recovery.sandboxIdentityFingerprint!, + }, + retainedRecoveryRecords: [recovery], + }); + harness.resolveRetainedSandboxRecoverySpy.mockReturnValue(false); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledWith(recovery); + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("local recovery cleanup was not confirmed"), + ); + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("Resolve the recovery record conflict"), + ); + expect( + harness.logSpy.mock.calls.some(([message]) => + String(message).includes("Sandbox 'alpha' destroyed"), + ), + ).toBe(false); + expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); + expect(exitSpy).toHaveBeenCalledWith(1); + }, + ); + it( "selects only the retained record matching observed Docker identity without a registry row (#10547)", { timeout: 30_000 }, diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 7296af25d2..c353e18fdf 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -1014,8 +1014,9 @@ async function destroySandboxUnlocked( deleteSucceededOrAlreadyGone && retainedRecoveryAuthority ) { + let recoveryResolved: boolean; try { - onboardSession.resolveRetainedSandboxRecovery(retainedRecoveryAuthority); + recoveryResolved = onboardSession.resolveRetainedSandboxRecovery(retainedRecoveryAuthority); } catch (error) { console.error( ` Sandbox '${sandboxName}' resources are gone, but NemoClaw could not clear its retained recovery record: ${redactDestroyError(error)}`, @@ -1023,6 +1024,15 @@ async function destroySandboxUnlocked( console.error(` Re-run '${CLI_NAME} ${sandboxName} destroy --yes' to finish local cleanup.`); requestSandboxDestroyExit(1); } + if (!recoveryResolved) { + console.error( + ` Sandbox '${sandboxName}' resources are gone, but its retained recovery authority was no longer current, so local recovery cleanup was not confirmed.`, + ); + console.error( + ` NemoClaw preserved any current recovery state. Resolve the recovery record conflict, then re-run '${CLI_NAME} ${sandboxName} destroy --yes'.`, + ); + requestSandboxDestroyExit(1); + } } if ( shouldCleanupGatewayAfterConfirmedFinalDestroy({ diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index d5dcb34d5d..ef111b4a34 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -122,7 +122,7 @@ type ExactDockerContainerCleanupDeps = { forceRemove?: (containerId: string) => { status?: number | null }; }; -function inspectDockerSandboxNameLabeledContainers( +export function inspectDockerSandboxNameLabeledContainers( sandboxName: string, ): DockerSandboxIdentityObservation { return inspectDockerSandboxIdentities(`${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, { From 12bd5859556ead1f49ca4e5d215947da6512ab81 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 18:19:18 -0700 Subject: [PATCH 12/24] fix(destroy): preserve retained recovery retry Signed-off-by: Prekshi Vyas --- .../destroy-retained-recovery-flow.test.ts | 38 ++++++++++++++- src/lib/actions/sandbox/destroy.ts | 47 +++++++++++-------- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index fb10fa4e2b..077d03b4c3 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -257,7 +257,7 @@ describe("destroySandbox retained recovery flow", () => { ); it( - "does not report success when retained recovery retirement loses authority (#10547)", + "fails closed and then retires the lone record after recovery authority loss (#10547)", { timeout: 30_000 }, async () => { const recovery = retainedRecoveryRecord(); @@ -275,7 +275,12 @@ describe("destroySandbox retained recovery flow", () => { }, retainedRecoveryRecords: [recovery], }); - harness.resolveRetainedSandboxRecoverySpy.mockReturnValue(false); + harness.resolveRetainedSandboxRecoverySpy + .mockImplementation(() => { + harness.setRetainedRecoveryRecords([]); + return true; + }) + .mockReturnValueOnce(false); await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow( "process.exit(1)", @@ -295,6 +300,35 @@ describe("destroySandbox retained recovery flow", () => { ).toBe(false); expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); expect(exitSpy).toHaveBeenCalledWith(1); + + const exactRemovalCallsAfterFailure = harness.dockerRunSpy.mock.calls.filter( + ([args]) => Array.isArray(args) && args[0] === "rm" && args[1] === "-f", + ).length; + harness.setRegistryEntryPresent(false); + harness.setSandboxPresent(false); + harness.setDockerIdentityResult({ status: 0, stdout: "" }); + exitSpy.mockClear(); + harness.logSpy.mockClear(); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledTimes(2); + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenLastCalledWith(recovery); + expect( + harness.dockerRunSpy.mock.calls.filter( + ([args]) => Array.isArray(args) && args[0] === "rm" && args[1] === "-f", + ), + ).toHaveLength(exactRemovalCallsAfterFailure); + expect( + harness.logSpy.mock.calls.some(([message]) => + String(message).includes("Sandbox 'alpha' destroyed"), + ), + ).toBe(true); + expect(exitSpy).not.toHaveBeenCalled(); + + harness.logSpy.mockClear(); + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledTimes(2); }, ); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index c353e18fdf..5f047fa410 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -91,9 +91,33 @@ function selectRetainedSandboxRecoveryAuthority( sandbox: registry.SandboxEntry | null, records: readonly RetainedSandboxRecoveryRecord[], ): RetainedSandboxRecoveryRecord | null { + const candidates = records.filter( + (record) => + record.sandboxName === sandboxName && record.sandboxIdentityFingerprint !== null, + ); + if (!sandbox) { + // Once resource cleanup has removed the registry row, a retry must still + // select the lone durable record so the later Docker proof can confirm + // absence and retire it. Multiple records continue to require immutable + // Docker evidence so the mutable name never chooses between authorities. + if (candidates.length === 1) return candidates[0]!; + if (candidates.length === 0) return null; + const observation = observeDestroyContainerIdentity(sandboxName); + const observedMatches = candidates.filter((record) => { + const verdict = classifyDestroyContainerIdentity( + sandboxName, + observation, + record.sandboxIdentityFingerprint!, + ); + return ( + verdict.status === "recovery" || + (verdict.status === "clear" && verdict.identity !== null) + ); + }); + return observedMatches.length === 1 ? observedMatches[0]! : null; + } + const matchesRegistryAuthority = (record: RetainedSandboxRecoveryRecord): boolean => { - if (record.sandboxIdentityFingerprint === null) return false; - if (!sandbox) return true; const pending = sandbox.pendingPolicyVerification; if (pending) { return ( @@ -112,24 +136,7 @@ function selectRetainedSandboxRecoveryAuthority( record.sandboxIdentityFingerprint === sandbox.lifecycleLiveIdentityFingerprint ); }; - const matching = records.filter( - (record) => record.sandboxName === sandboxName && matchesRegistryAuthority(record), - ); - if (!sandbox && matching.length > 1) { - const observation = observeDestroyContainerIdentity(sandboxName); - const observedMatches = matching.filter((record) => { - const verdict = classifyDestroyContainerIdentity( - sandboxName, - observation, - record.sandboxIdentityFingerprint!, - ); - return ( - verdict.status === "recovery" || - (verdict.status === "clear" && verdict.identity !== null) - ); - }); - return observedMatches.length === 1 ? observedMatches[0]! : null; - } + const matching = candidates.filter(matchesRegistryAuthority); return matching.length === 1 ? matching[0]! : null; } From 19ab1211daa9f6fa5fe9cb4849f1d701d23d0014 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 18:46:38 -0700 Subject: [PATCH 13/24] test(e2e): reconcile retained onboarding cleanup --- test/e2e/live/mcp-bridge-cleanup.ts | 33 +++++++++++++++++++++++++++++ test/e2e/live/mcp-bridge.test.ts | 17 ++++++--------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/test/e2e/live/mcp-bridge-cleanup.ts b/test/e2e/live/mcp-bridge-cleanup.ts index 82f8b36471..0134ca6b0a 100644 --- a/test/e2e/live/mcp-bridge-cleanup.ts +++ b/test/e2e/live/mcp-bridge-cleanup.ts @@ -2,9 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; export type McpAdapter = "mcporter" | "hermes-config" | "deepagents-config"; @@ -17,6 +19,37 @@ export const MCP_MUTATION_TIMEOUT_MS: Record = { const MCP_BRIDGE_ALREADY_ABSENT = /No MCP servers are registered|No MCP server '.+' is registered|MCP server '.+' not found/iu; +/** Prepare a sandbox name exclusively owned by this isolated qualification job. */ +export async function prepareOwnedSandboxForOnboard( + host: HostCliClient, + sandbox: SandboxClient, + cleanup: CleanupRegistry, + sandboxName: string, +): Promise { + cleanup.trackSandbox(host, sandboxName, { + artifactName: "cleanup-destroy-sandbox", + timeoutMs: 15 * 60_000, + }); + // A failed onboard may leave a live sandbox that the production CLI safely + // refuses to delete by mutable name. Register the trusted administrator + // deletion last so LIFO cleanup removes OpenShell state before `destroy` + // reconciles the durable recovery record and identity-verified containers. + cleanup.trackDisposable(`delete owned OpenShell sandbox ${sandboxName}`, () => + sandbox.cleanupSandbox(sandboxName, { + artifactName: "cleanup-delete-openshell-sandbox", + timeoutMs: 15 * 60_000, + }), + ); + await sandbox.cleanupSandbox(sandboxName, { + artifactName: "precleanup-delete-openshell-sandbox", + timeoutMs: 15 * 60_000, + }); + await host.cleanupSandbox(sandboxName, { + artifactName: "precleanup-destroy-sandbox", + timeoutMs: 15 * 60_000, + }); +} + export async function cleanupMcpBridge( host: HostCliClient, sandboxName: string, diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 50b26fe286..08f11f57ce 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -30,6 +30,7 @@ import { cleanupMcpBridge, MCP_MUTATION_TIMEOUT_MS, type McpAdapter, + prepareOwnedSandboxForOnboard, removeMcpBridgeWithOneConcurrencyRetry, } from "./mcp-bridge-cleanup.ts"; import { @@ -121,6 +122,7 @@ function expectManagedImageQualificationReceipt(sandboxName: string, agent: McpA async function onboardAgent( host: HostCliClient, + sandbox: SandboxClient, cleanup: CleanupRegistry, endpointUrl: string, options: { @@ -131,14 +133,7 @@ async function onboardAgent( }, ): Promise { const corporateCaBundle = requireMcpBridgeTlsCaCert(); - cleanup.trackSandbox(host, options.sandboxName, { - artifactName: "cleanup-destroy-sandbox", - timeoutMs: 15 * 60_000, - }); - await host.cleanupSandbox(options.sandboxName, { - artifactName: "precleanup-destroy-sandbox", - timeoutMs: 15 * 60_000, - }); + await prepareOwnedSandboxForOnboard(host, sandbox, cleanup, options.sandboxName); const args = buildMcpBridgeOnboardArgs(); const commandOptions = { artifactName: options.artifactName, @@ -739,7 +734,7 @@ test("mcp-bridge", { const mcpUrl = fakeMcpTunnel.url; const decoyMcpUrl = decoyMcpTunnel.url; progress.phase("onboard OpenClaw and prove base policy"); - await onboardAgent(host, cleanup, endpointUrl, { + await onboardAgent(host, sandbox, cleanup, endpointUrl, { agent: "openclaw", sandboxName: OPENCLAW_SANDBOX_NAME, artifactName: "onboard-openclaw-mcp-bridge", @@ -1117,7 +1112,7 @@ mcpBridgeShardTest("hermes")( const endpointUrl = `http://${hostAddress}:${compatibleMock.port}/v1`; const mcpUrl = fakeMcpTunnel.url; progress.phase("onboard the Hermes MCP sandbox"); - await onboardAgent(host, cleanup, endpointUrl, { + await onboardAgent(host, sandbox, cleanup, endpointUrl, { agent: "hermes", sandboxName: HERMES_SANDBOX_NAME, artifactName: "onboard-hermes-mcp-bridge", @@ -1370,7 +1365,7 @@ mcpBridgeShardTest("deepagents")( mcpUrl, ); progress.phase("onboard the Deep Agents MCP sandbox"); - await onboardAgent(host, cleanup, endpointUrl, { + await onboardAgent(host, sandbox, cleanup, endpointUrl, { agent: "langchain-deepagents-code", sandboxName: DEEPAGENTS_SANDBOX_NAME, artifactName: "onboard-deepagents-mcp-bridge", From af6b521f8c8d659759c6ce0f7bcc1e7902afadff Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 18:57:26 -0700 Subject: [PATCH 14/24] test(e2e): prove retained cleanup ordering --- test/e2e/live/mcp-bridge-cleanup.ts | 4 +- test/e2e/mock-parity.json | 2 + test/e2e/support/mcp-bridge-cleanup.test.ts | 60 +++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 test/e2e/support/mcp-bridge-cleanup.test.ts diff --git a/test/e2e/live/mcp-bridge-cleanup.ts b/test/e2e/live/mcp-bridge-cleanup.ts index 0134ca6b0a..216d63c6ec 100644 --- a/test/e2e/live/mcp-bridge-cleanup.ts +++ b/test/e2e/live/mcp-bridge-cleanup.ts @@ -21,8 +21,8 @@ const MCP_BRIDGE_ALREADY_ABSENT = /** Prepare a sandbox name exclusively owned by this isolated qualification job. */ export async function prepareOwnedSandboxForOnboard( - host: HostCliClient, - sandbox: SandboxClient, + host: Pick, + sandbox: Pick, cleanup: CleanupRegistry, sandboxName: string, ): Promise { diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 92a7bdae3d..df8ddc3900 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -487,6 +487,7 @@ }, { "live": "test/e2e/live/mcp-bridge.test.ts", + "liveSources": ["test/e2e/live/mcp-bridge-cleanup.ts"], "fast": [ "src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts", "src/lib/actions/sandbox/mcp-bridge-adapter-teardown.test.ts", @@ -494,6 +495,7 @@ "src/lib/actions/sandbox/mcp-bridge-provider.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/mcp-bridge-cleanup.test.ts", "test/e2e/support/mcp-bridge-hermes-lifecycle.test.ts", "test/e2e/support/mcp-bridge-onboard-env.test.ts", "test/e2e/support/mcp-bridge-reliability.test.ts", diff --git a/test/e2e/support/mcp-bridge-cleanup.test.ts b/test/e2e/support/mcp-bridge-cleanup.test.ts new file mode 100644 index 0000000000..558d887dae --- /dev/null +++ b/test/e2e/support/mcp-bridge-cleanup.test.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { ShellProbeRunOptions } from "../fixtures/shell-probe.ts"; +import { prepareOwnedSandboxForOnboard } from "../live/mcp-bridge-cleanup.ts"; + +function cleanupClient(owner: string, calls: string[]) { + return { + cleanupSandbox: vi.fn(async (_name: string, options: ShellProbeRunOptions = {}) => { + calls.push(`${owner}:${options.artifactName}`); + }), + }; +} + +describe("MCP bridge owned-sandbox cleanup", () => { + it("deletes OpenShell state before reconciling NemoClaw recovery state", async () => { + const calls: string[] = []; + const host = cleanupClient("host", calls); + const sandbox = cleanupClient("openshell", calls); + const cleanup = new CleanupRegistry(); + + await prepareOwnedSandboxForOnboard(host, sandbox, cleanup, "e2e-mcp-bridge"); + expect(calls).toEqual([ + "openshell:precleanup-delete-openshell-sandbox", + "host:precleanup-destroy-sandbox", + ]); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([]); + expect(calls).toEqual([ + "openshell:precleanup-delete-openshell-sandbox", + "host:precleanup-destroy-sandbox", + "openshell:cleanup-delete-openshell-sandbox", + "host:cleanup-destroy-sandbox", + ]); + }); + + it("still attempts NemoClaw reconciliation when administrator deletion fails", async () => { + const calls: string[] = []; + const host = cleanupClient("host", calls); + const sandbox = cleanupClient("openshell", calls); + const cleanup = new CleanupRegistry(); + + await prepareOwnedSandboxForOnboard(host, sandbox, cleanup, "e2e-mcp-bridge"); + sandbox.cleanupSandbox.mockRejectedValueOnce(new Error("openshell cleanup failed")); + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([ + { + name: "delete owned OpenShell sandbox e2e-mcp-bridge", + message: "openshell cleanup failed", + }, + ]); + expect(calls.at(-1)).toBe("host:cleanup-destroy-sandbox"); + }); +}); From 92c91008d34446c77e3d2f766c9824b3cdd412e3 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 19:09:04 -0700 Subject: [PATCH 15/24] docs(recovery): cover unavailable identity evidence --- docs/reference/commands.mdx | 10 ++++++---- src/lib/onboard/lifecycle-contracts.md | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 5dd44003cb..09c36ab821 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -454,16 +454,18 @@ When that result is resumable, NemoClaw keeps the session `in_progress` at its l #### Recover a retained sandbox -If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox, records its create-attempt label, and prints the exact `ai.nvidia.nemoclaw.create-attempt=` selector. +If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox. +When OpenShell returns a create-attempt label, NemoClaw records it and prints the exact `ai.nvidia.nemoclaw.create-attempt=` selector. When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt identity-bound recovery. Destroy completes recovery only after OpenShell confirms the retained sandbox is absent. It then verifies one retained recovery record and the immutable Docker sandbox identity, removes only the qualified residual containers, and clears the matching recovery record after verified cleanup. -If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. Give the displayed create-attempt label to an OpenShell administrator, ask them to remove that exact sandbox through an identity-bound procedure, then rerun destroy. +If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. For a record with a durable identity fingerprint, give the displayed create-attempt label to an OpenShell administrator when present and ask them to remove that exact sandbox through an identity-bound procedure. If no label is available, preserve the terminal output and ask the administrator to identify the exact sandbox from gateway or controller evidence. After the administrator removes the exact sandbox, rerun destroy. A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. -Ask an OpenShell administrator to resolve the create-attempt label to one exact sandbox and use an identity-bound recovery or removal procedure. -If NemoClaw reports that it could not save the recovery evidence, preserve the terminal output and ask the administrator to use its create-attempt label for the same identity-bound procedure. +If the terminal output includes a create-attempt label, ask an OpenShell administrator to resolve it to one exact sandbox and use an identity-bound recovery or removal procedure. +If OpenShell returned neither a durable identity fingerprint nor a create-attempt label, automatic recovery is unavailable. Preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. +If NemoClaw reports that it could not save the recovery evidence, preserve the terminal output. Give the administrator its create-attempt label when present; otherwise ask them to identify the exact sandbox from gateway or controller evidence. Do not delete the retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, supply a different explicit name: diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index ba17d47d64..822c429c32 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint and reports the exact create-attempt label. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to that label for identity-bound removal. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Rebuild and same-name onboarding remain blocked until recovery completes. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint when available and reports the exact create-attempt label when OpenShell returned one. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to the create-attempt label for identity-bound removal when that label is available. If OpenShell returned neither a durable identity nor a create-attempt label, automatic recovery is unavailable; the operator preserves the terminal output and asks an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Rebuild and same-name onboarding remain blocked until recovery completes. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. ## Effect-order flows From f84bd7992c7844218e5d9f2d9ba880697c235035 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 19:19:31 -0700 Subject: [PATCH 16/24] fix(state): migrate retained recovery authority --- src/lib/state/legacy-port-migration.test.ts | 54 ++++++++++++++++++++- src/lib/state/legacy-port-migration.ts | 12 ++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/lib/state/legacy-port-migration.test.ts b/src/lib/state/legacy-port-migration.test.ts index 1036a34848..f9ed3f36f1 100644 --- a/src/lib/state/legacy-port-migration.test.ts +++ b/src/lib/state/legacy-port-migration.test.ts @@ -7,7 +7,15 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + type OnboardEntryOptionsDeps, + resolveOnboardEntryOptions, +} from "../onboard/entry-options"; import { migrateLegacyPortState } from "./legacy-port-migration"; +import { + listRetainedSandboxRecoveryRecords, + recordRetainedSandboxRecovery, +} from "./onboard-session/retained-sandbox-recovery"; const homes: string[] = []; @@ -32,7 +40,7 @@ afterEach(() => { }); describe("legacy non-default gateway state migration", () => { - it("partitions a selected registry and moves identity-bound session, credentials, and snapshots", () => { + it("moves a recovery-only session and its retained recovery authority", () => { const home = makeHome(); const shared = path.join(home, ".nemoclaw"); const selected = path.join(shared, "gateways", "9123"); @@ -56,9 +64,21 @@ describe("legacy non-default gateway state migration", () => { }); writeJson(path.join(shared, "onboard-session.json"), { sandboxName: "port-box", - status: "in_progress", + status: "recovery_required", metadata: { gatewayName: "nemoclaw-9123" }, }); + recordRetainedSandboxRecovery(path.join(shared, "retained-sandbox-recovery.json"), { + sandboxName: "port-box", + sandboxIdentityFingerprint: "a".repeat(64), + gatewayName: "nemoclaw-9123", + gatewayPort: 9123, + lifecycleGeneration: "generation-1", + verifiedEffectivePolicyIdentity: null, + createAttemptNonce: "b".repeat(62), + policyCreationReceipt: null, + reason: "retained_after_sandbox_creation_failure", + recordedAt: "2026-08-29T00:00:00.000Z", + }); writeJson(path.join(shared, "credentials.json"), { NVIDIA_API_KEY: "legacy-secret" }); writeJson(path.join(shared, "usage-notice.json"), { acceptedVersion: "1" }); writeJson(path.join(shared, "state", "default-forward.json"), { pid: 123 }); @@ -80,6 +100,36 @@ describe("legacy non-default gateway state migration", () => { ).toEqual(["port-box"]); expect(fs.existsSync(path.join(shared, "onboard-session.json"))).toBe(false); expect(fs.existsSync(path.join(selected, "onboard-session.json"))).toBe(true); + expect(fs.existsSync(path.join(shared, "retained-sandbox-recovery.json"))).toBe(false); + const recoveryRecords = listRetainedSandboxRecoveryRecords( + path.join(selected, "retained-sandbox-recovery.json"), + ); + expect(recoveryRecords.map((record) => record.sandboxName)).toEqual(["port-box"]); + const entryDeps: OnboardEntryOptionsDeps = { + isNonInteractive: () => false, + validateName: (name) => name, + reservedSandboxNames: new Set(), + cliDisplayName: () => "NemoClaw", + getNameValidationGuidance: () => [], + error: vi.fn(), + exitProcess: vi.fn(() => { + throw new Error("blocked retained recovery name"); + }), + }; + expect(() => + resolveOnboardEntryOptions( + { + opts: { fresh: true, sandboxName: "port-box" }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + persistedSessionStatus: "recovery_required", + persistedRecoverySandboxName: "port-box", + retainedRecoverySandboxNames: recoveryRecords.map((record) => record.sandboxName), + }, + entryDeps, + ), + ).toThrow("blocked retained recovery name"); expect(fs.existsSync(path.join(shared, "credentials.json"))).toBe(false); expect(fs.existsSync(path.join(selected, "credentials.json"))).toBe(true); expect(fs.existsSync(path.join(shared, "usage-notice.json"))).toBe(true); diff --git a/src/lib/state/legacy-port-migration.ts b/src/lib/state/legacy-port-migration.ts index 8bd03a686c..ba8eaa12bc 100644 --- a/src/lib/state/legacy-port-migration.ts +++ b/src/lib/state/legacy-port-migration.ts @@ -27,6 +27,7 @@ const MIGRATION_LOCK_STALE_MS = 10_000; const STALE_MIGRATION_INTENT_PATTERN = /^\.gateway-state-migration\.(?:preparing|completed)\.[1-9][0-9]*\.[1-9][0-9]*$/; const MAX_MIGRATABLE_JSON_BYTES = 16 * 1024 * 1024; +const RETAINED_SANDBOX_RECOVERY_ENTRY = "retained-sandbox-recovery.json"; const LEGACY_BUNDLE_ENTRIES = [ "backups", "blueprints", @@ -38,10 +39,11 @@ const LEGACY_BUNDLE_ENTRIES = [ "ollama-proxy-token", "onboard-failures", "openrouter-runtime-adapter.pid", + RETAINED_SANDBOX_RECOVERY_ENTRY, "state", "usage-notice.json", ] as const; -const SESSION_BOUND_ENTRIES = ["credentials.json"] as const; +const SESSION_BOUND_ENTRIES = ["credentials.json", RETAINED_SANDBOX_RECOVERY_ENTRY] as const; const HOST_SHARED_BUNDLE_ENTRIES = [ "ollama-auth-proxy.pid", "ollama-proxy-port", @@ -540,6 +542,13 @@ function applyMigrationIntent( } migrateSandboxBackups(home, sharedRoot, selectedRoot, intent.metadata.sandboxBackupNames); + if (intent.metadata.bundleEntries.includes(RETAINED_SANDBOX_RECOVERY_ENTRY)) { + resumeMovePath( + home, + path.join(sharedRoot, RETAINED_SANDBOX_RECOVERY_ENTRY), + path.join(selectedRoot, RETAINED_SANDBOX_RECOVERY_ENTRY), + ); + } if (intent.metadata.moveSession) { resumeMovePath( home, @@ -553,6 +562,7 @@ function applyMigrationIntent( } for (const entry of intent.metadata.bundleEntries) { + if (entry === RETAINED_SANDBOX_RECOVERY_ENTRY) continue; resumeMovePath(home, path.join(sharedRoot, entry), path.join(selectedRoot, entry)); } From 4e9f5d0bf8a46ff01f7d86c0bfaf6accb1abf73f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 19:44:04 -0700 Subject: [PATCH 17/24] fix(state): partition retained recovery migration --- docs/reference/commands.mdx | 4 +- src/lib/onboard/lifecycle-contracts.md | 2 +- src/lib/state/legacy-port-migration.test.ts | 144 ++++++++++++++------ src/lib/state/legacy-port-migration.ts | 139 +++++++++++++++++-- 4 files changed, 232 insertions(+), 57 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 09c36ab821..e9c7e5c4a0 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -455,7 +455,7 @@ When that result is resumable, NemoClaw keeps the session `in_progress` at its l #### Recover a retained sandbox If onboarding cannot complete after sandbox creation, NemoClaw preserves the sandbox. -When OpenShell returns a create-attempt label, NemoClaw records it and prints the exact `ai.nvidia.nemoclaw.create-attempt=` selector. +When available, NemoClaw records and prints the create-attempt label as the exact `ai.nvidia.nemoclaw.create-attempt=` selector. When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt identity-bound recovery. @@ -464,7 +464,7 @@ If OpenShell still reports the sandbox present, destroy preserves the record and A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. If the terminal output includes a create-attempt label, ask an OpenShell administrator to resolve it to one exact sandbox and use an identity-bound recovery or removal procedure. -If OpenShell returned neither a durable identity fingerprint nor a create-attempt label, automatic recovery is unavailable. Preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. +If neither a durable identity fingerprint nor a create-attempt label is available, automatic recovery is unavailable. Preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. If NemoClaw reports that it could not save the recovery evidence, preserve the terminal output. Give the administrator its create-attempt label when present; otherwise ask them to identify the exact sandbox from gateway or controller evidence. Do not delete the retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, supply a different explicit name: diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 822c429c32..8b59eadfd7 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,7 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint when available and reports the exact create-attempt label when OpenShell returned one. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to the create-attempt label for identity-bound removal when that label is available. If OpenShell returned neither a durable identity nor a create-attempt label, automatic recovery is unavailable; the operator preserves the terminal output and asks an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Rebuild and same-name onboarding remain blocked until recovery completes. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint when available and reports the exact create-attempt label when available. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to the create-attempt label for identity-bound removal when that label is available. If neither a durable identity nor a create-attempt label is available, automatic recovery is unavailable; the operator preserves the terminal output and asks an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Rebuild and same-name onboarding remain blocked until recovery completes. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. ## Effect-order flows diff --git a/src/lib/state/legacy-port-migration.test.ts b/src/lib/state/legacy-port-migration.test.ts index f9ed3f36f1..23327275ad 100644 --- a/src/lib/state/legacy-port-migration.test.ts +++ b/src/lib/state/legacy-port-migration.test.ts @@ -34,13 +34,62 @@ function readJson(filePath: string): Record { return JSON.parse(fs.readFileSync(filePath, "utf8")) as Record; } +function recordRecovery( + filePath: string, + sandboxName: string, + gatewayPort: number, + seed: string, +): void { + recordRetainedSandboxRecovery(filePath, { + sandboxName, + sandboxIdentityFingerprint: seed.repeat(64), + gatewayName: gatewayPort === 8080 ? "nemoclaw" : `nemoclaw-${String(gatewayPort)}`, + gatewayPort, + lifecycleGeneration: `generation-${seed}`, + verifiedEffectivePolicyIdentity: null, + createAttemptNonce: seed.repeat(62), + policyCreationReceipt: null, + reason: "retained_after_sandbox_creation_failure", + recordedAt: "2026-08-29T00:00:00.000Z", + }); +} + +function expectRetainedNameBlocked( + records: readonly { readonly sandboxName: string }[], + sandboxName: string, +): void { + const entryDeps: OnboardEntryOptionsDeps = { + isNonInteractive: () => false, + validateName: (name) => name, + reservedSandboxNames: new Set(), + cliDisplayName: () => "NemoClaw", + getNameValidationGuidance: () => [], + error: vi.fn(), + exitProcess: vi.fn(() => { + throw new Error("blocked retained recovery name"); + }), + }; + expect(() => + resolveOnboardEntryOptions( + { + opts: { fresh: true, sandboxName }, + env: {}, + stdinIsTty: true, + stdoutIsTty: true, + retainedRecoverySandboxNames: records.map((record) => record.sandboxName), + }, + entryDeps, + ), + ).toThrow("blocked retained recovery name"); +} + afterEach(() => { vi.restoreAllMocks(); for (const home of homes.splice(0)) fs.rmSync(home, { recursive: true, force: true }); }); describe("legacy non-default gateway state migration", () => { - it("moves a recovery-only session and its retained recovery authority", () => { + it("partitions a recovery-only session and mixed-gateway recovery authority", () => { const home = makeHome(); const shared = path.join(home, ".nemoclaw"); const selected = path.join(shared, "gateways", "9123"); @@ -67,18 +116,9 @@ describe("legacy non-default gateway state migration", () => { status: "recovery_required", metadata: { gatewayName: "nemoclaw-9123" }, }); - recordRetainedSandboxRecovery(path.join(shared, "retained-sandbox-recovery.json"), { - sandboxName: "port-box", - sandboxIdentityFingerprint: "a".repeat(64), - gatewayName: "nemoclaw-9123", - gatewayPort: 9123, - lifecycleGeneration: "generation-1", - verifiedEffectivePolicyIdentity: null, - createAttemptNonce: "b".repeat(62), - policyCreationReceipt: null, - reason: "retained_after_sandbox_creation_failure", - recordedAt: "2026-08-29T00:00:00.000Z", - }); + const sharedRecovery = path.join(shared, "retained-sandbox-recovery.json"); + recordRecovery(sharedRecovery, "port-box", 9123, "a"); + recordRecovery(sharedRecovery, "default-box", 8080, "b"); writeJson(path.join(shared, "credentials.json"), { NVIDIA_API_KEY: "legacy-secret" }); writeJson(path.join(shared, "usage-notice.json"), { acceptedVersion: "1" }); writeJson(path.join(shared, "state", "default-forward.json"), { pid: 123 }); @@ -100,36 +140,14 @@ describe("legacy non-default gateway state migration", () => { ).toEqual(["port-box"]); expect(fs.existsSync(path.join(shared, "onboard-session.json"))).toBe(false); expect(fs.existsSync(path.join(selected, "onboard-session.json"))).toBe(true); - expect(fs.existsSync(path.join(shared, "retained-sandbox-recovery.json"))).toBe(false); - const recoveryRecords = listRetainedSandboxRecoveryRecords( + const selectedRecoveryRecords = listRetainedSandboxRecoveryRecords( path.join(selected, "retained-sandbox-recovery.json"), ); - expect(recoveryRecords.map((record) => record.sandboxName)).toEqual(["port-box"]); - const entryDeps: OnboardEntryOptionsDeps = { - isNonInteractive: () => false, - validateName: (name) => name, - reservedSandboxNames: new Set(), - cliDisplayName: () => "NemoClaw", - getNameValidationGuidance: () => [], - error: vi.fn(), - exitProcess: vi.fn(() => { - throw new Error("blocked retained recovery name"); - }), - }; - expect(() => - resolveOnboardEntryOptions( - { - opts: { fresh: true, sandboxName: "port-box" }, - env: {}, - stdinIsTty: true, - stdoutIsTty: true, - persistedSessionStatus: "recovery_required", - persistedRecoverySandboxName: "port-box", - retainedRecoverySandboxNames: recoveryRecords.map((record) => record.sandboxName), - }, - entryDeps, - ), - ).toThrow("blocked retained recovery name"); + const remainingRecoveryRecords = listRetainedSandboxRecoveryRecords(sharedRecovery); + expect(selectedRecoveryRecords.map((record) => record.sandboxName)).toEqual(["port-box"]); + expect(remainingRecoveryRecords.map((record) => record.sandboxName)).toEqual(["default-box"]); + expectRetainedNameBlocked(selectedRecoveryRecords, "port-box"); + expectRetainedNameBlocked(remainingRecoveryRecords, "default-box"); expect(fs.existsSync(path.join(shared, "credentials.json"))).toBe(false); expect(fs.existsSync(path.join(selected, "credentials.json"))).toBe(true); expect(fs.existsSync(path.join(shared, "usage-notice.json"))).toBe(true); @@ -172,6 +190,31 @@ describe("legacy non-default gateway state migration", () => { expect(fs.existsSync(path.join(shared, "gateways", "9123", "sandboxes.json"))).toBe(false); }); + it("refuses conflicting retained recovery identity without mutating state", () => { + const home = makeHome(); + const shared = path.join(home, ".nemoclaw"); + const recoveryFile = path.join(shared, "retained-sandbox-recovery.json"); + recordRetainedSandboxRecovery(recoveryFile, { + sandboxName: "port-box", + sandboxIdentityFingerprint: "c".repeat(64), + gatewayName: "nemoclaw-9124", + gatewayPort: 9123, + lifecycleGeneration: "generation-c", + verifiedEffectivePolicyIdentity: null, + createAttemptNonce: "c".repeat(62), + policyCreationReceipt: null, + reason: "retained_after_sandbox_creation_failure", + recordedAt: "2026-08-29T00:00:00.000Z", + }); + const before = fs.readFileSync(recoveryFile, "utf8"); + + expect(() => migrateLegacyPortState({ home, gatewayPort: 9123 })).toThrow( + /conflicting gateway identity/, + ); + expect(fs.readFileSync(recoveryFile, "utf8")).toBe(before); + expect(fs.existsSync(path.join(shared, "gateways", "9123"))).toBe(false); + }); + it("preflights backup collisions before publishing the selected registry", () => { const home = makeHome(); const shared = path.join(home, ".nemoclaw"); @@ -219,6 +262,7 @@ describe("legacy non-default gateway state migration", () => { "port-box": { name: "port-box", gatewayName: "nemoclaw-9123", gatewayPort: 9123 }, }, }); + recordRecovery(path.join(shared, "retained-sandbox-recovery.json"), "port-box", 9123, "e"); writeJson(path.join(backupSource, "snapshot", "manifest.json"), {}); const renameSync = fs.renameSync.bind(fs); @@ -238,6 +282,12 @@ describe("legacy non-default gateway state migration", () => { expect(Object.keys(readJson(legacyRegistry).sandboxes as object)).toEqual(["default-box"]); expect(fs.existsSync(selectedRegistry)).toBe(false); + expect(fs.existsSync(path.join(shared, "retained-sandbox-recovery.json"))).toBe(false); + expect( + listRetainedSandboxRecoveryRecords( + path.join(selected, "retained-sandbox-recovery.json"), + ).map((record) => record.sandboxName), + ).toEqual(["port-box"]); expect(fs.existsSync(path.join(shared, ".gateway-state-migration"))).toBe(true); expect(() => migrateLegacyPortState({ home, gatewayPort: 8080 })).toThrow( /recoverable migration for gateway port 9123 is pending/, @@ -264,9 +314,10 @@ describe("legacy non-default gateway state migration", () => { }, ); - it("partitions provable rows but leaves credentials whose gateway ownership is ambiguous", () => { + it("partitions provable rows and recovery without a session but leaves credentials", () => { const home = makeHome(); const shared = path.join(home, ".nemoclaw"); + const selected = path.join(shared, "gateways", "9123"); writeJson(path.join(shared, "sandboxes.json"), { defaultSandbox: "default-box", sandboxes: { @@ -274,6 +325,7 @@ describe("legacy non-default gateway state migration", () => { "port-box": { name: "port-box", gatewayName: "nemoclaw-9123", gatewayPort: 9123 }, }, }); + recordRecovery(path.join(shared, "retained-sandbox-recovery.json"), "port-box", 9123, "d"); writeJson(path.join(shared, "credentials.json"), { NVIDIA_API_KEY: "ambiguous-secret" }); const result = migrateLegacyPortState({ home, gatewayPort: 9123 }); @@ -281,7 +333,13 @@ describe("legacy non-default gateway state migration", () => { expect(result.migratedSandboxNames).toEqual(["port-box"]); expect(result.warnings.join("\n")).toContain("Left ambiguous"); expect(fs.existsSync(path.join(shared, "credentials.json"))).toBe(true); - expect(fs.existsSync(path.join(shared, "gateways", "9123", "credentials.json"))).toBe(false); + expect(fs.existsSync(path.join(selected, "credentials.json"))).toBe(false); + expect(fs.existsSync(path.join(shared, "retained-sandbox-recovery.json"))).toBe(false); + const recoveryRecords = listRetainedSandboxRecoveryRecords( + path.join(selected, "retained-sandbox-recovery.json"), + ); + expect(recoveryRecords.map((record) => record.sandboxName)).toEqual(["port-box"]); + expectRetainedNameBlocked(recoveryRecords, "port-box"); }); it("moves singleton state when every legacy registry row belongs to the selected gateway", () => { diff --git a/src/lib/state/legacy-port-migration.ts b/src/lib/state/legacy-port-migration.ts index ba8eaa12bc..dd70d3da5e 100644 --- a/src/lib/state/legacy-port-migration.ts +++ b/src/lib/state/legacy-port-migration.ts @@ -15,19 +15,25 @@ import { readGatewayRegistryFile, registryEntryGatewayPort, } from "./gateway-registry"; +import { + listRetainedSandboxRecoveryRecords, + retainedSandboxRecoveryFile, + type RetainedSandboxRecoveryRecord, +} from "./onboard-session/retained-sandbox-recovery"; import { nemoclawStateRoot, resolveHome } from "./state-root"; const MIGRATION_LOCK = ".gateway-state-migration.lock"; const MIGRATION_INTENT = ".gateway-state-migration"; const MIGRATION_INTENT_METADATA = "intent.json"; +const MIGRATION_INTENT_REMAINING_RECOVERY = "remaining-retained-sandbox-recovery.json"; const MIGRATION_INTENT_SELECTED_REGISTRY = "selected-registry.json"; +const MIGRATION_INTENT_SELECTED_RECOVERY = "selected-retained-sandbox-recovery.json"; const MIGRATION_INTENT_REMAINING_REGISTRY = "remaining-registry.json"; const MIGRATION_INTENT_VERSION = 1; const MIGRATION_LOCK_STALE_MS = 10_000; const STALE_MIGRATION_INTENT_PATTERN = /^\.gateway-state-migration\.(?:preparing|completed)\.[1-9][0-9]*\.[1-9][0-9]*$/; const MAX_MIGRATABLE_JSON_BYTES = 16 * 1024 * 1024; -const RETAINED_SANDBOX_RECOVERY_ENTRY = "retained-sandbox-recovery.json"; const LEGACY_BUNDLE_ENTRIES = [ "backups", "blueprints", @@ -39,11 +45,10 @@ const LEGACY_BUNDLE_ENTRIES = [ "ollama-proxy-token", "onboard-failures", "openrouter-runtime-adapter.pid", - RETAINED_SANDBOX_RECOVERY_ENTRY, "state", "usage-notice.json", ] as const; -const SESSION_BOUND_ENTRIES = ["credentials.json", RETAINED_SANDBOX_RECOVERY_ENTRY] as const; +const SESSION_BOUND_ENTRIES = ["credentials.json"] as const; const HOST_SHARED_BUNDLE_ENTRIES = [ "ollama-auth-proxy.pid", "ollama-proxy-port", @@ -78,6 +83,13 @@ interface LegacyPortMigrationIntent { metadata: LegacyPortMigrationIntentMetadata; selectedRegistry: GatewayRegistryDocument; remainingRegistry: GatewayRegistryDocument | null; + selectedRecovery: RetainedRecoveryDocument | null; + remainingRecovery: RetainedRecoveryDocument | null; +} + +interface RetainedRecoveryDocument { + schemaVersion: 1; + unresolved: readonly RetainedSandboxRecoveryRecord[]; } function migrationError(message: string): Error { @@ -166,6 +178,44 @@ function readJsonNoFollow(home: string, filePath: string): unknown | null { } } +function retainedRecoveryDocument( + records: readonly RetainedSandboxRecoveryRecord[], +): RetainedRecoveryDocument { + return { schemaVersion: 1, unresolved: records }; +} + +function readRetainedRecoveryDocument( + home: string, + filePath: string, +): RetainedRecoveryDocument | null { + if (!lstatNoFollow(home, filePath)) return null; + let records: readonly RetainedSandboxRecoveryRecord[]; + try { + records = listRetainedSandboxRecoveryRecords(filePath); + } catch (error) { + throw migrationError( + `${filePath} is not valid retained sandbox recovery state: ${error instanceof Error ? error.message : String(error)}`, + ); + } + for (const record of records) { + const gatewayPort = resolveGatewayPortFromName(record.gatewayName); + if (gatewayPort === null || gatewayPort !== record.gatewayPort) { + throw migrationError( + `${filePath} record ${record.recordId} has conflicting gateway identity`, + ); + } + } + return retainedRecoveryDocument(records); +} + +function removeRetainedRecoveryFile(home: string, filePath: string): void { + const stat = lstatNoFollow(home, filePath); + if (!stat) return; + if (!stat.isFile()) throw migrationError(`${filePath} is not a regular file`); + fs.rmSync(filePath); + fsyncDirectory(path.dirname(filePath)); +} + function firstSandboxName(sandboxes: Record): string | null { return Object.keys(sandboxes).sort()[0] ?? null; } @@ -372,10 +422,19 @@ function readMigrationIntent(home: string, sharedRoot: string): LegacyPortMigrat rawMetadata.bundleEntries, "migration intent bundleEntries", ); + const selectedRecovery = readRetainedRecoveryDocument( + home, + path.join(intentDir, MIGRATION_INTENT_SELECTED_RECOVERY), + ); + const remainingRecovery = readRetainedRecoveryDocument( + home, + path.join(intentDir, MIGRATION_INTENT_REMAINING_RECOVERY), + ); if ( rawMetadata.rewriteLegacyRegistry !== selectedSandboxNames.length > 0 || (rawMetadata.moveSession && rawMetadata.warnAmbiguousSession) || - (selectedSandboxNames.length === 0 && !rawMetadata.moveSession) + (selectedSandboxNames.length === 0 && !rawMetadata.moveSession && !selectedRecovery) || + (selectedRecovery === null) !== (remainingRecovery === null) ) { throw migrationError("migration intent has inconsistent ownership metadata"); } @@ -390,6 +449,13 @@ function readMigrationIntent(home: string, sharedRoot: string): LegacyPortMigrat throw migrationError(`migration intent backup ${sandboxName} is not a selected sandbox`); } } + if ( + selectedRecovery?.unresolved.length === 0 || + selectedRecovery?.unresolved.some((record) => record.gatewayPort !== gatewayPort) || + remainingRecovery?.unresolved.some((record) => record.gatewayPort === gatewayPort) + ) { + throw migrationError("migration intent has inconsistent retained recovery ownership"); + } const selectedRegistryFile = path.join(intentDir, MIGRATION_INTENT_SELECTED_REGISTRY); const selectedRegistry = readGatewayRegistryFile(home, selectedRegistryFile); @@ -438,6 +504,8 @@ function readMigrationIntent(home: string, sharedRoot: string): LegacyPortMigrat }, selectedRegistry, remainingRegistry, + selectedRecovery, + remainingRecovery, }; } @@ -447,6 +515,8 @@ function createMigrationIntent( metadata: LegacyPortMigrationIntentMetadata, selectedRegistry: GatewayRegistryDocument, remainingRegistry: GatewayRegistryDocument | null, + selectedRecovery: RetainedRecoveryDocument | null, + remainingRecovery: RetainedRecoveryDocument | null, ): LegacyPortMigrationIntent { const intentDir = path.join(sharedRoot, MIGRATION_INTENT); if (lstatNoFollow(home, intentDir)) { @@ -457,6 +527,9 @@ function createMigrationIntent( if (metadata.rewriteLegacyRegistry && !remainingRegistry) { throw migrationError("migration intent is missing its remaining legacy registry"); } + if ((selectedRecovery === null) !== (remainingRecovery === null)) { + throw migrationError("migration intent is missing its retained recovery partition"); + } ensureRealDirectory(home, sharedRoot); const preparingDir = `${intentDir}.preparing.${String(process.pid)}.${String(Date.now())}`; @@ -476,6 +549,18 @@ function createMigrationIntent( remainingRegistry, ); } + if (selectedRecovery && remainingRecovery) { + writeJsonAtomic( + home, + path.join(preparingDir, MIGRATION_INTENT_SELECTED_RECOVERY), + selectedRecovery, + ); + writeJsonAtomic( + home, + path.join(preparingDir, MIGRATION_INTENT_REMAINING_RECOVERY), + remainingRecovery, + ); + } fsyncDirectory(preparingDir); fs.renameSync(preparingDir, intentDir); fsyncDirectory(sharedRoot); @@ -541,14 +626,23 @@ function applyMigrationIntent( writeJsonAtomic(home, legacyRegistryFile, intent.remainingRegistry); } - migrateSandboxBackups(home, sharedRoot, selectedRoot, intent.metadata.sandboxBackupNames); - if (intent.metadata.bundleEntries.includes(RETAINED_SANDBOX_RECOVERY_ENTRY)) { - resumeMovePath( + if (intent.selectedRecovery && intent.remainingRecovery) { + writeJsonAtomic( home, - path.join(sharedRoot, RETAINED_SANDBOX_RECOVERY_ENTRY), - path.join(selectedRoot, RETAINED_SANDBOX_RECOVERY_ENTRY), + retainedSandboxRecoveryFile(selectedRoot), + intent.selectedRecovery, ); + if (intent.remainingRecovery.unresolved.length > 0) { + writeJsonAtomic( + home, + retainedSandboxRecoveryFile(sharedRoot), + intent.remainingRecovery, + ); + } else { + removeRetainedRecoveryFile(home, retainedSandboxRecoveryFile(sharedRoot)); + } } + migrateSandboxBackups(home, sharedRoot, selectedRoot, intent.metadata.sandboxBackupNames); if (intent.metadata.moveSession) { resumeMovePath( home, @@ -562,7 +656,6 @@ function applyMigrationIntent( } for (const entry of intent.metadata.bundleEntries) { - if (entry === RETAINED_SANDBOX_RECOVERY_ENTRY) continue; resumeMovePath(home, path.join(sharedRoot, entry), path.join(selectedRoot, entry)); } @@ -692,10 +785,13 @@ export function migrateLegacyPortState( const legacyRegistry = readGatewayRegistryFile(home, legacyRegistryFile); const legacySessionFile = path.join(sharedRoot, "onboard-session.json"); const legacySession = readJsonNoFollow(home, legacySessionFile); + const legacyRecoveryFile = retainedSandboxRecoveryFile(sharedRoot); + const legacyRecoveryExists = lstatNoFollow(home, legacyRecoveryFile) !== null; if ( !pendingBeforeLock && !legacyRegistry && legacySession === null && + !legacyRecoveryExists && !staleIntentDirectoriesExist ) { return result; @@ -737,6 +833,18 @@ export function migrateLegacyPortState( } const session = readJsonNoFollow(home, legacySessionFile); + const recovery = readRetainedRecoveryDocument(home, legacyRecoveryFile); + const selectedRecoveryRecords = + recovery?.unresolved.filter((record) => record.gatewayPort === gatewayPort) ?? []; + const remainingRecoveryRecords = + recovery?.unresolved.filter((record) => record.gatewayPort !== gatewayPort) ?? []; + const selectedRecovery = + selectedRecoveryRecords.length > 0 + ? retainedRecoveryDocument(selectedRecoveryRecords) + : null; + const remainingRecovery = selectedRecovery + ? retainedRecoveryDocument(remainingRecoveryRecords) + : null; const recordedSessionPort = session === null ? null : sessionGatewayPort(session, registryPortsByName); const selectedNames = Object.keys(selectedEntries).sort(); @@ -746,7 +854,7 @@ export function migrateLegacyPortState( Object.keys(remainingEntries).length === 0 && (session === null || sessionBelongsToSelected); - if (selectedNames.length === 0 && !sessionBelongsToSelected) return result; + if (selectedNames.length === 0 && !sessionBelongsToSelected && !selectedRecovery) return result; const entriesToMove: readonly LegacyBundleEntry[] = wholeLegacyBundleBelongsToSelected ? MIGRATABLE_BUNDLE_ENTRIES @@ -779,6 +887,13 @@ export function migrateLegacyPortState( bundleEntries.push(entry); } } + if (selectedRecovery) { + preflightMovePath( + home, + legacyRecoveryFile, + retainedSandboxRecoveryFile(selectedRoot), + ); + } registryLocks.push(acquireDirectoryLock(home, `${selectedRegistryFile}.lock`)); const existingSelected = readGatewayRegistryFile(home, selectedRegistryFile); @@ -808,6 +923,8 @@ export function migrateLegacyPortState( }, selectedRegistry, remainingRegistry, + selectedRecovery, + remainingRecovery, ); return applyMigrationIntent( home, From ed383e54b38fcd84d2a5b06e524554768518ba1b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 19:56:56 -0700 Subject: [PATCH 18/24] fix(state): reject incomplete recovery intents --- src/lib/state/legacy-port-migration.test.ts | 48 +++++++++++++++++++++ src/lib/state/legacy-port-migration.ts | 11 +++++ 2 files changed, 59 insertions(+) diff --git a/src/lib/state/legacy-port-migration.test.ts b/src/lib/state/legacy-port-migration.test.ts index 23327275ad..997354c8ef 100644 --- a/src/lib/state/legacy-port-migration.test.ts +++ b/src/lib/state/legacy-port-migration.test.ts @@ -215,6 +215,54 @@ describe("legacy non-default gateway state migration", () => { expect(fs.existsSync(path.join(shared, "gateways", "9123"))).toBe(false); }); + it("refuses an older published intent that omitted retained recovery", () => { + const home = makeHome(); + const shared = path.join(home, ".nemoclaw"); + const migration = path.join(shared, ".gateway-state-migration"); + const legacyRegistry = path.join(shared, "sandboxes.json"); + const recoveryFile = path.join(shared, "retained-sandbox-recovery.json"); + writeJson(legacyRegistry, { + defaultSandbox: "default-box", + sandboxes: { + "default-box": { name: "default-box", gatewayName: "nemoclaw", gatewayPort: 8080 }, + "port-box": { name: "port-box", gatewayName: "nemoclaw-9123", gatewayPort: 9123 }, + }, + }); + writeJson(path.join(migration, "intent.json"), { + version: 1, + gatewayPort: 9123, + selectedSandboxNames: ["port-box"], + sandboxBackupNames: [], + moveSession: false, + bundleEntries: [], + warnAmbiguousSession: false, + rewriteLegacyRegistry: true, + }); + writeJson(path.join(migration, "selected-registry.json"), { + defaultSandbox: "port-box", + sandboxes: { + "port-box": { name: "port-box", gatewayName: "nemoclaw-9123", gatewayPort: 9123 }, + }, + }); + writeJson(path.join(migration, "remaining-registry.json"), { + defaultSandbox: "default-box", + sandboxes: { + "default-box": { name: "default-box", gatewayName: "nemoclaw", gatewayPort: 8080 }, + }, + }); + recordRecovery(recoveryFile, "port-box", 9123, "f"); + const registryBefore = fs.readFileSync(legacyRegistry, "utf8"); + const recoveryBefore = fs.readFileSync(recoveryFile, "utf8"); + + expect(() => migrateLegacyPortState({ home, gatewayPort: 9123 })).toThrow( + /intent predates retained recovery partitioning/, + ); + expect(fs.readFileSync(legacyRegistry, "utf8")).toBe(registryBefore); + expect(fs.readFileSync(recoveryFile, "utf8")).toBe(recoveryBefore); + expect(fs.existsSync(migration)).toBe(true); + expect(fs.existsSync(path.join(shared, "gateways", "9123"))).toBe(false); + }); + it("preflights backup collisions before publishing the selected registry", () => { const home = makeHome(); const shared = path.join(home, ".nemoclaw"); diff --git a/src/lib/state/legacy-port-migration.ts b/src/lib/state/legacy-port-migration.ts index dd70d3da5e..1f529a4423 100644 --- a/src/lib/state/legacy-port-migration.ts +++ b/src/lib/state/legacy-port-migration.ts @@ -489,6 +489,17 @@ function readMigrationIntent(home: string, sharedRoot: string): LegacyPortMigrat } } } + if ( + selectedRecovery === null && + readRetainedRecoveryDocument( + home, + retainedSandboxRecoveryFile(sharedRoot), + )?.unresolved.some((record) => record.gatewayPort === gatewayPort) + ) { + throw migrationError( + "published migration intent predates retained recovery partitioning; retained recovery remains safely in the shared root", + ); + } return { intentDir, From 12ea021ef94cebb6db41454ee7ee749bfe825105 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 20:37:57 -0700 Subject: [PATCH 19/24] test(e2e): initialize gateway before MCP cleanup --- test/e2e/live/mcp-bridge-cleanup.ts | 10 ++++++- test/e2e/support/mcp-bridge-cleanup.test.ts | 31 +++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/test/e2e/live/mcp-bridge-cleanup.ts b/test/e2e/live/mcp-bridge-cleanup.ts index 216d63c6ec..48f9828a46 100644 --- a/test/e2e/live/mcp-bridge-cleanup.ts +++ b/test/e2e/live/mcp-bridge-cleanup.ts @@ -21,7 +21,7 @@ const MCP_BRIDGE_ALREADY_ABSENT = /** Prepare a sandbox name exclusively owned by this isolated qualification job. */ export async function prepareOwnedSandboxForOnboard( - host: Pick, + host: Pick, sandbox: Pick, cleanup: CleanupRegistry, sandboxName: string, @@ -40,6 +40,14 @@ export async function prepareOwnedSandboxForOnboard( timeoutMs: 15 * 60_000, }), ); + // A fresh qualification runner has no active OpenShell gateway yet. Let the + // production CLI initialize it and perform any cleanup it can prove safe. + // Retained-state refusal remains non-fatal here because the identity-bound + // administrator deletion below is the isolated E2E fallback. + await host.bestEffortCleanupSandbox(sandboxName, { + artifactName: "precleanup-initialize-gateway", + timeoutMs: 15 * 60_000, + }); await sandbox.cleanupSandbox(sandboxName, { artifactName: "precleanup-delete-openshell-sandbox", timeoutMs: 15 * 60_000, diff --git a/test/e2e/support/mcp-bridge-cleanup.test.ts b/test/e2e/support/mcp-bridge-cleanup.test.ts index 558d887dae..3fb5bb8af5 100644 --- a/test/e2e/support/mcp-bridge-cleanup.test.ts +++ b/test/e2e/support/mcp-bridge-cleanup.test.ts @@ -8,15 +8,23 @@ import type { ShellProbeRunOptions } from "../fixtures/shell-probe.ts"; import { prepareOwnedSandboxForOnboard } from "../live/mcp-bridge-cleanup.ts"; function cleanupClient(owner: string, calls: string[]) { + const cleanupSandbox = vi.fn(async (_name: string, options: ShellProbeRunOptions = {}) => { + calls.push(`${owner}:${options.artifactName}`); + }); return { - cleanupSandbox: vi.fn(async (_name: string, options: ShellProbeRunOptions = {}) => { - calls.push(`${owner}:${options.artifactName}`); + cleanupSandbox, + bestEffortCleanupSandbox: vi.fn(async (name: string, options: ShellProbeRunOptions = {}) => { + try { + await cleanupSandbox(name, options); + } catch { + // Match HostCliClient: the administrator fallback must still run. + } }), }; } describe("MCP bridge owned-sandbox cleanup", () => { - it("deletes OpenShell state before reconciling NemoClaw recovery state", async () => { + it("initializes the gateway before administrator deletion and final reconciliation", async () => { const calls: string[] = []; const host = cleanupClient("host", calls); const sandbox = cleanupClient("openshell", calls); @@ -24,6 +32,7 @@ describe("MCP bridge owned-sandbox cleanup", () => { await prepareOwnedSandboxForOnboard(host, sandbox, cleanup, "e2e-mcp-bridge"); expect(calls).toEqual([ + "host:precleanup-initialize-gateway", "openshell:precleanup-delete-openshell-sandbox", "host:precleanup-destroy-sandbox", ]); @@ -32,6 +41,7 @@ describe("MCP bridge owned-sandbox cleanup", () => { expect(result.failures).toEqual([]); expect(calls).toEqual([ + "host:precleanup-initialize-gateway", "openshell:precleanup-delete-openshell-sandbox", "host:precleanup-destroy-sandbox", "openshell:cleanup-delete-openshell-sandbox", @@ -57,4 +67,19 @@ describe("MCP bridge owned-sandbox cleanup", () => { ]); expect(calls.at(-1)).toBe("host:cleanup-destroy-sandbox"); }); + + it("uses administrator deletion when safe gateway initialization refuses cleanup", async () => { + const calls: string[] = []; + const host = cleanupClient("host", calls); + const sandbox = cleanupClient("openshell", calls); + const cleanup = new CleanupRegistry(); + host.cleanupSandbox.mockRejectedValueOnce(new Error("retained identity requires recovery")); + + await prepareOwnedSandboxForOnboard(host, sandbox, cleanup, "e2e-mcp-bridge"); + + expect(calls).toEqual([ + "openshell:precleanup-delete-openshell-sandbox", + "host:precleanup-destroy-sandbox", + ]); + }); }); From 47709ad575a6eb8de1770ee2f1a93af8f1688285 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 20:59:52 -0700 Subject: [PATCH 20/24] test(e2e): bind MCP cleanup to gateway --- test/e2e/live/mcp-bridge-cleanup.ts | 12 ++++++++++++ test/e2e/support/mcp-bridge-cleanup.test.ts | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/test/e2e/live/mcp-bridge-cleanup.ts b/test/e2e/live/mcp-bridge-cleanup.ts index 48f9828a46..4e3adc87d0 100644 --- a/test/e2e/live/mcp-bridge-cleanup.ts +++ b/test/e2e/live/mcp-bridge-cleanup.ts @@ -19,6 +19,15 @@ export const MCP_MUTATION_TIMEOUT_MS: Record = { const MCP_BRIDGE_ALREADY_ABSENT = /No MCP servers are registered|No MCP server '.+' is registered|MCP server '.+' not found/iu; +function buildOwnedSandboxCleanupEnv(): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + // Bind trusted administrator cleanup to the gateway NemoClaw initialized. + // ShellProbe otherwise forwards only PATH, which hides gateway metadata. + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY?.trim() || "nemoclaw", + }; +} + /** Prepare a sandbox name exclusively owned by this isolated qualification job. */ export async function prepareOwnedSandboxForOnboard( host: Pick, @@ -26,6 +35,7 @@ export async function prepareOwnedSandboxForOnboard( cleanup: CleanupRegistry, sandboxName: string, ): Promise { + const openshellCleanupEnv = buildOwnedSandboxCleanupEnv(); cleanup.trackSandbox(host, sandboxName, { artifactName: "cleanup-destroy-sandbox", timeoutMs: 15 * 60_000, @@ -37,6 +47,7 @@ export async function prepareOwnedSandboxForOnboard( cleanup.trackDisposable(`delete owned OpenShell sandbox ${sandboxName}`, () => sandbox.cleanupSandbox(sandboxName, { artifactName: "cleanup-delete-openshell-sandbox", + env: openshellCleanupEnv, timeoutMs: 15 * 60_000, }), ); @@ -50,6 +61,7 @@ export async function prepareOwnedSandboxForOnboard( }); await sandbox.cleanupSandbox(sandboxName, { artifactName: "precleanup-delete-openshell-sandbox", + env: openshellCleanupEnv, timeoutMs: 15 * 60_000, }); await host.cleanupSandbox(sandboxName, { diff --git a/test/e2e/support/mcp-bridge-cleanup.test.ts b/test/e2e/support/mcp-bridge-cleanup.test.ts index 3fb5bb8af5..1963bbdf30 100644 --- a/test/e2e/support/mcp-bridge-cleanup.test.ts +++ b/test/e2e/support/mcp-bridge-cleanup.test.ts @@ -36,6 +36,16 @@ describe("MCP bridge owned-sandbox cleanup", () => { "openshell:precleanup-delete-openshell-sandbox", "host:precleanup-destroy-sandbox", ]); + expect(sandbox.cleanupSandbox).toHaveBeenNthCalledWith( + 1, + "e2e-mcp-bridge", + expect.objectContaining({ + env: expect.objectContaining({ + HOME: expect.any(String), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY?.trim() || "nemoclaw", + }), + }), + ); const result = await cleanup.runAll(); @@ -47,6 +57,16 @@ describe("MCP bridge owned-sandbox cleanup", () => { "openshell:cleanup-delete-openshell-sandbox", "host:cleanup-destroy-sandbox", ]); + expect(sandbox.cleanupSandbox).toHaveBeenNthCalledWith( + 2, + "e2e-mcp-bridge", + expect.objectContaining({ + env: expect.objectContaining({ + HOME: expect.any(String), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY?.trim() || "nemoclaw", + }), + }), + ); }); it("still attempts NemoClaw reconciliation when administrator deletion fails", async () => { From 9b5b1bbc0b252548b3e8490ac27a3671ca85b2bb Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 21:08:12 -0700 Subject: [PATCH 21/24] docs(recovery): qualify Docker identity guidance --- docs/reference/commands.mdx | 6 +++--- src/lib/onboard/lifecycle-contracts.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index e9c7e5c4a0..932182deec 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -459,9 +459,9 @@ When available, NemoClaw records and prints the create-attempt label as the exac When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt identity-bound recovery. -Destroy completes recovery only after OpenShell confirms the retained sandbox is absent. It then verifies one retained recovery record and the immutable Docker sandbox identity, removes only the qualified residual containers, and clears the matching recovery record after verified cleanup. +Destroy completes recovery only after OpenShell confirms the retained sandbox is absent. It then verifies one retained recovery record and its immutable runtime identity. For Docker-backed sandboxes, destroy additionally verifies the immutable Docker sandbox identity and removes only qualified residual containers. It clears the matching recovery record only after verified cleanup. If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. For a record with a durable identity fingerprint, give the displayed create-attempt label to an OpenShell administrator when present and ask them to remove that exact sandbox through an identity-bound procedure. If no label is available, preserve the terminal output and ask the administrator to identify the exact sandbox from gateway or controller evidence. After the administrator removes the exact sandbox, rerun destroy. -A foreign container, changed identity, failed Docker probe, or ambiguous recovery record stops cleanup and preserves the record. +An ambiguous recovery record or failed immutable runtime-identity check stops cleanup and preserves the record. For Docker-backed sandboxes, a foreign container, changed Docker identity, or failed Docker probe also stops cleanup and preserves the record. If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. If the terminal output includes a create-attempt label, ask an OpenShell administrator to resolve it to one exact sandbox and use an identity-bound recovery or removal procedure. If neither a durable identity fingerprint nor a create-attempt label is available, automatic recovery is unavailable. Preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. @@ -541,7 +541,7 @@ $$nemoclaw onboard --fresh --apf-interceptor --name my-apf-sandbox If post-create verification or native GPU fallback fails after OpenShell may have created the sandbox, NemoClaw preserves the incomplete sandbox because automatic deletion would use its mutable name. -Follow the [retained-sandbox recovery procedure](#recover-a-retained-sandbox); its OpenShell-absence, immutable Docker identity, and administrator requirements also apply to APF creation. +Follow the [retained-sandbox recovery procedure](#recover-a-retained-sandbox); its OpenShell-absence, immutable runtime-identity, and administrator requirements also apply to APF creation. For Docker-backed APF creation, its Docker container-identity requirement also applies. This onboarding mode does not support `--resume` or `--recreate-sandbox`, regardless of whether sandbox creation began. After destroy completes, repeat the original command with `--fresh`. diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 8b59eadfd7..500bea8ef0 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` refuses mutable-name deletion while that sandbox is live. After administrator identity-bound removal, destroy uses the record to qualify immutable Docker container identities, complete residual cleanup, and retire the record. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` refuses mutable-name deletion while that sandbox is live. After administrator identity-bound removal, destroy uses the record to qualify immutable runtime identity and, for Docker-backed sandboxes, exact container identities before residual cleanup and record retirement. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | From 9164a3c9a11ba58da9c0fd433a82551aab423a78 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 28 Aug 2026 21:14:15 -0700 Subject: [PATCH 22/24] docs(recovery): clarify fail-closed decisions --- docs/reference/commands.mdx | 19 +++++++++++-------- src/lib/onboard/lifecycle-contracts.md | 8 +++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 932182deec..e82ab561d8 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -459,14 +459,17 @@ When available, NemoClaw records and prints the create-attempt label as the exac When available, NemoClaw also records a durable identity fingerprint and verified policy evidence for recovery. Automatic and explicit resume, reuse, recreation, and fresh onboarding with that sandbox name remain blocked. When the recovery record contains a durable identity fingerprint, run `$$nemoclaw destroy` to attempt identity-bound recovery. -Destroy completes recovery only after OpenShell confirms the retained sandbox is absent. It then verifies one retained recovery record and its immutable runtime identity. For Docker-backed sandboxes, destroy additionally verifies the immutable Docker sandbox identity and removes only qualified residual containers. It clears the matching recovery record only after verified cleanup. -If OpenShell still reports the sandbox present, destroy preserves the record and removes no resources because OpenShell deletion accepts only the mutable sandbox name. For a record with a durable identity fingerprint, give the displayed create-attempt label to an OpenShell administrator when present and ask them to remove that exact sandbox through an identity-bound procedure. If no label is available, preserve the terminal output and ask the administrator to identify the exact sandbox from gateway or controller evidence. After the administrator removes the exact sandbox, rerun destroy. -An ambiguous recovery record or failed immutable runtime-identity check stops cleanup and preserves the record. For Docker-backed sandboxes, a foreign container, changed Docker identity, or failed Docker probe also stops cleanup and preserves the record. -If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. -If the terminal output includes a create-attempt label, ask an OpenShell administrator to resolve it to one exact sandbox and use an identity-bound recovery or removal procedure. -If neither a durable identity fingerprint nor a create-attempt label is available, automatic recovery is unavailable. Preserve the terminal output and ask an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. -If NemoClaw reports that it could not save the recovery evidence, preserve the terminal output. Give the administrator its create-attempt label when present; otherwise ask them to identify the exact sandbox from gateway or controller evidence. -Do not delete the retained sandbox manually by mutable name. + +Use the result from `destroy` to choose the next action: + +- If OpenShell still reports the sandbox present, `destroy` preserves the record and removes no resources. Do not delete the sandbox manually by mutable name. Give the displayed create-attempt label to an OpenShell administrator when present and ask them to remove that exact sandbox through an identity-bound procedure. Without a label, preserve the terminal output and ask the administrator to identify the exact sandbox from gateway or controller evidence. +- After the administrator removes the exact sandbox, rerun `$$nemoclaw destroy`. +- If OpenShell confirms the sandbox is absent, `destroy` verifies one retained recovery record and its immutable runtime identity. For Docker-backed sandboxes, it also verifies the immutable Docker sandbox identity and removes only qualified residual containers. It clears the matching recovery record only after verified cleanup. +- If the recovery record is ambiguous or immutable runtime-identity verification fails, cleanup stops and the record remains. For Docker-backed sandboxes, a foreign container, changed Docker identity, or failed Docker probe has the same fail-closed result. + +If OpenShell did not return a durable identity fingerprint, `destroy` cannot complete recovery. A create-attempt label can help an OpenShell administrator identify and remove the exact sandbox, but it does not let NemoClaw retire the record without immutable identity authority. If neither a fingerprint nor a label is available, preserve the terminal output and ask the administrator to identify the exact sandbox from gateway or controller evidence. If NemoClaw reports that it could not save recovery evidence, preserve the terminal output and follow the same escalation. + +This fail-closed record keeps only the affected sandbox name unavailable. It is not retired from mutable-name absence alone, and this command does not accept administrator-supplied identity authority. To onboard another sandbox while the record remains unresolved, supply a different explicit name: ```bash diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 500bea8ef0..9a232d8c48 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -70,7 +70,13 @@ Onboarding binds policy authority after gateway setup and before provider, crede The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint when available and reports the exact create-attempt label when available. Because OpenShell deletion accepts only a mutable sandbox name, retained recovery refuses automatic deletion while OpenShell reports the sandbox present and directs an administrator to the create-attempt label for identity-bound removal when that label is available. If neither a durable identity nor a create-attempt label is available, automatic recovery is unavailable; the operator preserves the terminal output and asks an OpenShell administrator to identify the exact sandbox from gateway or controller evidence. After OpenShell confirms absence, `destroy` accepts multiple managed Docker containers only when the fingerprint of every immutable sandbox ID equals the retained sandbox identity fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members of that set, and verifies their absence without issuing a mutable-name delete. Recovery completion releases the matching recovery-only session before retiring the exact independent record, so either write failure leaves the record available for retry. A foreign container, changed sandbox ID, failed probe, or changed recovery record stops cleanup. Rebuild and same-name onboarding remain blocked until recovery completes. Operators must not delete a retained sandbox manually by mutable name. To onboard another sandbox while the record remains unresolved, run `nemoclaw onboard --name `. `--fresh` alone does not clear the record or permit reuse of the retained name. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. + +After a post-create authority failure, NemoClaw retains the durable sandbox identity fingerprint when available and reports the exact create-attempt label when available. While OpenShell reports the sandbox present, recovery refuses automatic deletion because OpenShell deletion accepts only a mutable sandbox name. When a create-attempt label is available, the operator gives it to an administrator for identity-bound removal. Without a label, the operator preserves the terminal output and asks the administrator to identify the exact sandbox from gateway or controller evidence. + +After OpenShell confirms absence, `destroy` verifies the retained immutable runtime identity. For Docker-backed sandboxes, it accepts multiple managed containers only when every immutable sandbox ID has the retained fingerprint. It snapshots the matching container IDs, revalidates the set, removes only remaining members, and verifies their absence without issuing a mutable-name delete. A foreign container, changed identity, failed probe, ambiguous record, or changed recovery authority stops cleanup. + +Recovery completion releases the matching recovery-only session before retiring the exact independent record. Either write failure leaves the record available for retry. A record without an immutable fingerprint remains fail-closed for the affected sandbox name; mutable-name absence cannot retire it, and this flow accepts no administrator-supplied identity authority. Rebuild and same-name onboarding remain blocked, but `nemoclaw onboard --name ` starts an unrelated sandbox. `--fresh` alone does not clear the record or permit reuse of the retained name. The command reference owns the detailed operator procedure. ## Effect-order flows From 1ca2d4de1c6b32820b9b16edc0e5511875a71b7a Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 29 Aug 2026 08:37:56 -0700 Subject: [PATCH 23/24] fix(state): serialize retained recovery migration Signed-off-by: Senthil Ravichandran --- docs/reference/commands.mdx | 3 ++- src/lib/onboard/cancel-rollback.test.ts | 7 ++++++ src/lib/onboard/cancel-rollback.ts | 5 ++++- src/lib/state/legacy-port-migration.test.ts | 20 +++++++++++++++++ src/lib/state/legacy-port-migration.ts | 22 ++++++++++++++----- ...onboard-session-cross-process-lock.test.ts | 8 +++++++ src/lib/state/onboard-session.ts | 13 ++++++++++- test/cli/destroy-gateway-cleanup.test.ts | 1 + 8 files changed, 70 insertions(+), 9 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index e82ab561d8..656052fa25 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2660,7 +2660,8 @@ For one matching container, the command continues only when all these labels hav - A nonempty `openshell.ai/sandbox-workspace` - A nonempty `openshell.ai/sandbox-id` -If the initial inspection cannot complete, more than one container matches, a matching container has conflicting or incomplete labels, or Docker returns malformed identity data, `destroy` exits before changing sandbox resources. +In an ordinary destroy flow, if the initial inspection cannot complete, more than one container matches, a matching container has conflicting or incomplete labels, or Docker returns malformed identity data, `destroy` exits before changing sandbox resources. +Retained-sandbox recovery accepts multiple managed containers only when every immutable sandbox ID matches the retained recovery fingerprint. The identity checks still apply with `--force`, `--yes`, or `NEMOCLAW_NON_INTERACTIVE=1`; those controls authorize confirmation but do not authorize an unproven container identity. NemoClaw rechecks the identity after read-only preflight, before provider cleanup, and synchronously at the sandbox-deletion boundary. If a later recheck detects drift or fails, `destroy` refuses sandbox deletion, restores managed MCP preparation when possible, preserves local ownership state, and reports any earlier cleanup already performed. diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index 8b32616c4b..dd9d11cf41 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -316,4 +316,11 @@ describe("buildCancelRollbackMessage", () => { expect(message).not.toContain("openshell sandbox delete"); expect(message).not.toContain("cannot delete it by immutable identity"); }); + + it("does not refer to an undisplayed create-attempt label", () => { + const message = buildCancelRollbackMessage("sb", SANDBOX_FINGERPRINT).join("\n"); + + expect(message).toContain("preserve the displayed fingerprint"); + expect(message).not.toContain("displayed create-attempt label"); + }); }); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index 03c63e7341..90d7e9d219 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -80,7 +80,10 @@ export function buildCancelRollbackMessage( " Shared inference providers are gateway configuration and are not sandbox cleanup targets.", ...(sandboxIdentityFingerprint ? [ - ` Run '${cliName()} ${sandboxName} destroy'. If OpenShell confirms the retained sandbox absent, destroy removes only verified residual containers and can clear the matching recovery record. If it is still live, give the displayed create-attempt label to an OpenShell administrator for identity-bound removal.`, + ` Run '${cliName()} ${sandboxName} destroy'. If OpenShell confirms the retained sandbox absent, destroy removes only verified residual containers and can clear the matching recovery record.`, + recoveryContext + ? " If it is still live, give the displayed create-attempt label to an OpenShell administrator for identity-bound removal." + : " If it is still live, preserve the displayed fingerprint and ask an OpenShell administrator for identity-bound removal.", ] : [ " NemoClaw cannot clear this recovery record until an OpenShell administrator establishes the exact sandbox identity.", diff --git a/src/lib/state/legacy-port-migration.test.ts b/src/lib/state/legacy-port-migration.test.ts index 997354c8ef..fa20608c1f 100644 --- a/src/lib/state/legacy-port-migration.test.ts +++ b/src/lib/state/legacy-port-migration.test.ts @@ -409,6 +409,26 @@ describe("legacy non-default gateway state migration", () => { expect(fs.existsSync(path.join(selected, "credentials.json"))).toBe(true); }); + it.each([ + ["shared", (shared: string, _selected: string) => shared], + ["selected", (_shared: string, selected: string) => selected], + ])("refuses recovery-only migration while the %s onboarding lock is present", (_scope, root) => { + const home = makeHome(); + const shared = path.join(home, ".nemoclaw"); + const selected = path.join(shared, "gateways", "9123"); + const recoveryFile = path.join(shared, "retained-sandbox-recovery.json"); + recordRecovery(recoveryFile, "port-box", 9123, "d"); + const before = fs.readFileSync(recoveryFile, "utf8"); + fs.mkdirSync(root(shared, selected), { recursive: true }); + fs.writeFileSync(path.join(root(shared, selected), "onboard.lock"), "active writer"); + + expect(() => migrateLegacyPortState({ home, gatewayPort: 9123 })).toThrow( + /onboarding lock .* is present/u, + ); + expect(fs.readFileSync(recoveryFile, "utf8")).toBe(before); + expect(fs.existsSync(path.join(selected, "retained-sandbox-recovery.json"))).toBe(false); + }); + it.each( ["ollama-proxy-token", "ollama-proxy-port", "ollama-auth-proxy.pid"], )("keeps host-shared Ollama proxy state out of a non-default gateway migration [%s]", (entry) => { diff --git a/src/lib/state/legacy-port-migration.ts b/src/lib/state/legacy-port-migration.ts index 1f529a4423..be3c001799 100644 --- a/src/lib/state/legacy-port-migration.ts +++ b/src/lib/state/legacy-port-migration.ts @@ -742,6 +742,17 @@ function acquireDirectoryLock(home: string, lock: string): string { throw migrationError(`could not acquire ${lock}`); } +function assertOnboardStateUnlocked(home: string, stateRoots: readonly string[]): void { + for (const stateRoot of stateRoots) { + const activeLock = path.join(stateRoot, "onboard.lock"); + if (lstatNoFollow(home, activeLock)) { + throw migrationError( + `onboarding lock ${activeLock} is present; finish or stop that run before migrating state`, + ); + } + } +} + /** * Partition pre-segregation state into the selected non-default gateway root. * Registry rows move only when their persisted canonical gateway identity is @@ -783,6 +794,7 @@ export function migrateLegacyPortState( if (staleIntentDirectoriesExist) { const lock = acquireDirectoryLock(home, migrationLock); try { + assertOnboardStateUnlocked(home, [sharedRoot]); removeStaleMigrationIntentDirectories(home, sharedRoot); } finally { fs.rmSync(lock, { recursive: true, force: true }); @@ -811,6 +823,10 @@ export function migrateLegacyPortState( const lock = acquireDirectoryLock(home, migrationLock); const registryLocks: string[] = []; try { + // Onboard writers recheck the migration lock after claiming onboard.lock. + // Checking both roots while this lock is held closes the opposite side of + // the handshake and serializes session/recovery state with partitioning. + assertOnboardStateUnlocked(home, [sharedRoot, selectedRoot]); removeStaleMigrationIntentDirectories(home, sharedRoot); registryLocks.push(acquireDirectoryLock(home, `${legacyRegistryFile}.lock`)); const pendingIntent = readMigrationIntent(home, sharedRoot); @@ -874,12 +890,6 @@ export function migrateLegacyPortState( : []; let moveSession = false; if (sessionBelongsToSelected) { - const activeLock = path.join(sharedRoot, "onboard.lock"); - if (lstatNoFollow(home, activeLock)) { - throw migrationError( - `legacy onboarding lock ${activeLock} is present; finish or stop that run first`, - ); - } moveSession = preflightMovePath( home, legacySessionFile, diff --git a/src/lib/state/onboard-session-cross-process-lock.test.ts b/src/lib/state/onboard-session-cross-process-lock.test.ts index 4fd6018c00..8abb59ab1b 100644 --- a/src/lib/state/onboard-session-cross-process-lock.test.ts +++ b/src/lib/state/onboard-session-cross-process-lock.test.ts @@ -39,6 +39,14 @@ afterEach(() => { }); describe("cross-process onboard lock", () => { + it("releases its lock when legacy-state migration already owns the handshake", () => { + const migrationLock = path.join(tempHome, ".nemoclaw", ".gateway-state-migration.lock"); + fs.mkdirSync(migrationLock, { recursive: true }); + + expect(session.acquireOnboardLock("nemoclaw onboard").acquired).toBe(false); + expect(fs.existsSync(session.LOCK_FILE)).toBe(false); + }); + it("rejects caller-asserted onboarding lock ownership without a live descriptor (#9833)", async () => { const authority = await import("../onboard/portable-retirement-authority"); diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index f39e6154ce..1f247e9bdb 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -14,7 +14,7 @@ import path from "node:path"; import type { SandboxPolicyAuthority } from "../adapters/openshell/policy-authority"; import { isErrnoException } from "../core/errno"; import { isObjectRecord, type JsonObject, type JsonValue } from "../core/json-types"; -import { GATEWAY_PORT } from "../core/ports"; +import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../core/ports"; import { parseServingProfileProvenance, type ServingProfileProvenance, @@ -82,6 +82,10 @@ export const SESSION_DIR = nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY export const SESSION_FILE = path.join(SESSION_DIR, "onboard-session.json"); export const LOCK_FILE = path.join(SESSION_DIR, "onboard.lock"); export const RETAINED_SANDBOX_RECOVERY_FILE = retainedSandboxRecoveryFile(SESSION_DIR); +const LEGACY_STATE_MIGRATION_LOCK = path.join( + nemoclawStateRoot(process.env.HOME || "/tmp", DEFAULT_GATEWAY_PORT), + ".gateway-state-migration.lock", +); const SAFE_VLLM_INSTALL_MODEL = /^[A-Za-z0-9._:/-]+$/; export class InvalidPersistedPolicyAuthorityError extends Error {} @@ -1660,6 +1664,13 @@ export function acquireOnboardLock(command: string | null = null): LockResult { try { heldLockDirectory = openPinnedSessionDirectory(); assertOnboardLockOwned(); + // Legacy-port migration holds its lock before checking every onboard + // writer lock. Recheck here after atomically claiming onboard.lock so + // either the writer or the migrator wins, never both. + if (fs.existsSync(LEGACY_STATE_MIGRATION_LOCK)) { + releaseOnboardLock(); + return { acquired: false, lockFile: LOCK_FILE, stale: false }; + } } catch (error) { heldLockFd = null; if (heldLockDirectory !== null) fs.closeSync(heldLockDirectory.descriptor); diff --git a/test/cli/destroy-gateway-cleanup.test.ts b/test/cli/destroy-gateway-cleanup.test.ts index 11aa990a5a..3921b00a88 100644 --- a/test/cli/destroy-gateway-cleanup.test.ts +++ b/test/cli/destroy-gateway-cleanup.test.ts @@ -762,6 +762,7 @@ describe("CLI dispatch", () => { ].join("\n"), { mode: 0o755 }, ); + fs.writeFileSync(path.join(localBin, "docker"), LIVE_DOCKER_IDENTITY, { mode: 0o755 }); const r = runWithEnv("alpha destroy --yes", { HOME: home, From c9c4b5e8d97de90c5f2ee715f2b602730108f9e5 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 29 Aug 2026 08:52:39 -0700 Subject: [PATCH 24/24] fix(destroy): bind retained recovery gateway Signed-off-by: Senthil Ravichandran --- src/lib/actions/sandbox/destroy-preflight.ts | 27 ++++++++++++++++--- .../destroy-retained-recovery-flow.test.ts | 6 +++++ src/lib/actions/sandbox/destroy.ts | 4 ++- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/destroy-preflight.ts b/src/lib/actions/sandbox/destroy-preflight.ts index 496edb895b..ce24ed460b 100644 --- a/src/lib/actions/sandbox/destroy-preflight.ts +++ b/src/lib/actions/sandbox/destroy-preflight.ts @@ -23,7 +23,10 @@ import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { type DestroyRunOpenshell, selectGatewayForSandboxDestroy } from "./destroy-gateway"; import { classifyDestroySandboxPresence } from "./destroy-presence"; -import { getSandboxTargetGatewayName } from "./gateway-target"; +import { + getPersistedSandboxTargetGatewayName, + getSandboxTargetGatewayName, +} from "./gateway-target"; import { assertMcpAdapterConfigMutationsAllowed } from "./mcp-bridge-runtime-capabilities"; export type SandboxDestroyPreflight = { @@ -260,7 +263,10 @@ export async function stopModelRouterForDestroyedSandbox( return true; } -export function prepareSandboxDestroy(sandboxName: string): SandboxDestroyPreflight { +export function prepareSandboxDestroy( + sandboxName: string, + retainedRecoveryGatewayName?: string, +): SandboxDestroyPreflight { const sandbox = registry.getSandbox(sandboxName); console.log(` Deleting sandbox '${sandboxName}'...`); const { runOpenshell } = require("../../adapters/openshell/runtime") as { @@ -268,8 +274,21 @@ export function prepareSandboxDestroy(sandboxName: string): SandboxDestroyPrefli }; // Capture the sandbox gateway before destructive work, then pin every - // following OpenShell subprocess against that same registry-owned gateway. - const cleanupGatewayName = getSandboxTargetGatewayName(sandboxName); + // following OpenShell subprocess against that same durable authority. A + // retained recovery record remains authoritative after a partial destroy + // has already retired the registry row. + const registeredGatewayName = sandbox ? getPersistedSandboxTargetGatewayName(sandbox) : null; + if ( + retainedRecoveryGatewayName && + registeredGatewayName && + retainedRecoveryGatewayName !== registeredGatewayName + ) { + throw new Error( + `Refusing to destroy sandbox '${sandboxName}': retained recovery gateway '${retainedRecoveryGatewayName}' does not match registered gateway '${registeredGatewayName}'.`, + ); + } + const cleanupGatewayName = + retainedRecoveryGatewayName ?? registeredGatewayName ?? getSandboxTargetGatewayName(); selectGatewayForSandboxDestroy(sandboxName, cleanupGatewayName, runOpenshell); process.env.OPENSHELL_GATEWAY = cleanupGatewayName; diff --git a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts index 077d03b4c3..ac6345c20f 100644 --- a/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-retained-recovery-flow.test.ts @@ -364,6 +364,12 @@ describe("destroySandbox retained recovery flow", () => { expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledOnce(); expect(harness.resolveRetainedSandboxRecoverySpy).toHaveBeenCalledWith(matchingRecovery); expect(harness.resolveRetainedSandboxRecoverySpy).not.toHaveBeenCalledWith(olderRecovery); + expect(harness.selectGatewaySpy).toHaveBeenCalledWith( + "alpha", + matchingRecovery.gatewayName, + harness.runOpenshellSpy, + ); + expect(harness.gatewayPinsAtSandboxList).toEqual([matchingRecovery.gatewayName]); expect(exitSpy).not.toHaveBeenCalled(); }, ); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 5f047fa410..9aba963667 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -670,7 +670,9 @@ async function destroySandboxUnlocked( } }; let destroyPreflight: ReturnType; - destroyPreflight = abortPreparedCleanupOnError(() => prepareSandboxDestroy(sandboxName)); + destroyPreflight = abortPreparedCleanupOnError(() => + prepareSandboxDestroy(sandboxName, retainedRecoveryAuthority?.gatewayName), + ); const { cleanupGatewayName, runOpenshell, sandbox, sandboxConfirmedAbsent } = destroyPreflight; if (retainedRecoveryAuthority && !sandboxConfirmedAbsent) { console.error(