Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 27 additions & 3 deletions src/lib/actions/sandbox/destroy-execution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@

import { R, YW } from "../../cli/terminal-style";
import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy";
import { removePortableDemoSandboxLifecycleReceipt } from "../../onboard/experimental/portable-demo-lifecycle";
import {
type PreparedPortableDemoSandboxDestroyAuthority,
preparePortableDemoSandboxDestroyAuthority,
removePortableDemoSandboxLifecycleReceipt,
} from "../../onboard/experimental/portable-demo-lifecycle";
import {
CURRENT_RUNTIME_PROVIDER_BUNDLES,
type RuntimeProviderBundle,
Expand Down Expand Up @@ -50,6 +54,8 @@ export function retirePortableLifecycleAuthority(sandboxName: string): void {
removePortableDemoSandboxLifecycleReceipt(sandboxName);
}

export { preparePortableDemoSandboxDestroyAuthority };

type SandboxDestroyExecutionInput = {
cleanupShieldsArtifacts: (sandboxName: string) => void;
force: boolean;
Expand All @@ -63,6 +69,7 @@ type SandboxDestroyExecutionInput = {
// `null` records confirmed absence; an object records the one managed
// container observed by the pre-destroy guard.
expectedContainerIdentity?: SandboxNameLabeledContainer | null;
portableContainerAuthority?: PreparedPortableDemoSandboxDestroyAuthority;
stopInferenceResources: () => void;
runtimeProviders?: RuntimeProviderBundleRegistry;
deps?: {
Expand Down Expand Up @@ -91,6 +98,7 @@ export type SandboxDestroyExecutionResult =
hostLocalInferenceOwnershipRequiresGateway: boolean;
mcpOwnershipRequiresGateway: boolean;
mcpRecoveryFailure?: string;
portableLifecycleOwnershipRequiresGateway?: boolean;
shieldsRelockRequiresGateway: boolean;
hostLocalInferenceCleanupFailure?: string;
deleteConfirmed?: boolean;
Expand Down Expand Up @@ -281,6 +289,7 @@ export async function executeSandboxDestroy({
sandboxConfirmedAbsent,
sandboxName,
expectedContainerIdentity,
portableContainerAuthority,
stopInferenceResources,
runtimeProviders = CURRENT_RUNTIME_PROVIDER_BUNDLES,
deps = {},
Expand All @@ -292,6 +301,14 @@ export async function executeSandboxDestroy({
| { status: "ambiguous"; detail: string }
| { status: "probe-failed"; detail: string };
const inspectIdentityContinuity = (): IdentityContinuity => {
if (portableContainerAuthority) {
try {
portableContainerAuthority.revalidate();
return { status: "match" };
} catch (error) {
return { status: "probe-failed", detail: redactDestroyError(error) };
}
}
if (expectedContainerIdentity === undefined) return { status: "match" };
const verdict = classifyDestroyContainerIdentity(
sandboxName,
Expand Down Expand Up @@ -495,6 +512,7 @@ export async function executeSandboxDestroy({
force &&
!hasMcpOwnership &&
!hasHostLocalInferenceOwnership &&
portableContainerAuthority === undefined &&
!hardened.hardeningFailed;

if (deleteResult.status !== 0 && !alreadyGone && !forcedLocalCleanup) {
Expand All @@ -510,13 +528,19 @@ export async function executeSandboxDestroy({
gatewayUnreachable && hasHostLocalInferenceOwnership,
mcpOwnershipRequiresGateway: gatewayUnreachable && hasMcpOwnership,
mcpRecoveryFailure,
portableLifecycleOwnershipRequiresGateway:
gatewayUnreachable && portableContainerAuthority !== undefined,
shieldsRelockRequiresGateway: gatewayUnreachable && hardened.hardeningFailed,
};
}

if (!forcedLocalCleanup && expectedContainerIdentity) {
if (!forcedLocalCleanup && (portableContainerAuthority || expectedContainerIdentity)) {
try {
removeExactDestroyContainerIdentity(sandboxName, expectedContainerIdentity, console.log);
if (portableContainerAuthority) {
portableContainerAuthority.verifyAbsent();
} else if (expectedContainerIdentity) {
removeExactDestroyContainerIdentity(sandboxName, expectedContainerIdentity, console.log);
}
} catch (error) {
const detail = redactDestroyError(error);
return {
Expand Down
66 changes: 66 additions & 0 deletions src/lib/actions/sandbox/destroy-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,72 @@ describe("destroySandbox flow", () => {
expect(harness.retirePortableLifecycleReceiptSpy).not.toHaveBeenCalled();
});

it("revalidates schema-4 Portable identity at every destroy checkpoint without Docker preflight (#9189)", async () => {
const harness = createDestroyHarness({
dockerRunResult: { status: 0, stdout: `${"f".repeat(64)}\t\tforeign\t` },
portableDestroyAuthority: true,
});

await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined();

expect(harness.preparePortableDestroyAuthoritySpy).toHaveBeenCalledOnce();
expect(harness.portableDestroyRevalidateSpy).toHaveBeenCalledTimes(6);
expect(harness.portableDestroyVerifyAbsentSpy).toHaveBeenCalledOnce();
expect(harness.dockerRunSpy).not.toHaveBeenCalled();
expect(harness.portableDestroyVerifyAbsentSpy.mock.invocationCallOrder[0]).toBeLessThan(
harness.removeSandboxSpy.mock.invocationCallOrder[0],
);
expect(harness.removeSandboxSpy.mock.invocationCallOrder[0]).toBeLessThan(
harness.retirePortableLifecycleReceiptSpy.mock.invocationCallOrder[0],
);
});

it("fails closed on invalid schema-4 Portable receipt authority before Docker preflight (#9189)", async () => {
const harness = createDestroyHarness({
portableDestroyPrepareError: "portable receipt is invalid",
});

await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow(
"portable receipt is invalid",
);

expect(harness.dockerRunSpy).not.toHaveBeenCalled();
expect(harness.runOpenshellSpy).not.toHaveBeenCalled();
expect(harness.removeSandboxSpy).not.toHaveBeenCalled();
expect(harness.retirePortableLifecycleReceiptSpy).not.toHaveBeenCalled();
});

it("preserves the sandbox registry record and Portable receipt when Podman absence is unproven (#9189)", async () => {
const harness = createDestroyHarness({
portableDestroyAuthority: true,
portableDestroyVerifyAbsentError: "recorded Podman container remains",
});

await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)");

expect(harness.events).toContain("delete");
expect(harness.portableDestroyVerifyAbsentSpy).toHaveBeenCalledOnce();
expect(harness.removeSandboxSpy).not.toHaveBeenCalled();
expect(harness.retirePortableLifecycleReceiptSpy).not.toHaveBeenCalled();
});

it("refuses forced local cleanup while schema-4 Portable authority is retained (#9189)", async () => {
const harness = createDestroyHarness({
deleteStatus: 1,
deleteOutput: "error trying to connect: connection refused",
portableDestroyAuthority: true,
});

await expect(harness.destroySandbox("alpha", { force: true })).rejects.toThrow(
"process.exit(1)",
);

const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n");
expect(errorOutput).toContain("--force cannot discard this ownership state");
expect(harness.removeSandboxSpy).not.toHaveBeenCalled();
expect(harness.retirePortableLifecycleReceiptSpy).not.toHaveBeenCalled();
});

it(
"removes the owned managed Hermes state volume after confirmed sandbox deletion",
{ timeout: 30_000 },
Expand Down
50 changes: 40 additions & 10 deletions src/lib/actions/sandbox/destroy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import {
} from "./destroy-confirmation";
import {
executeSandboxDestroy,
preparePortableDemoSandboxDestroyAuthority,
redactDestroyError,
retirePortableLifecycleAuthority,
} from "./destroy-execution";
Expand Down Expand Up @@ -480,6 +481,16 @@ async function destroySandboxUnlocked(
const normalized = normalizeDestroySandboxOptions(options);
if (!(await confirmSandboxDestroy(sandboxName, normalized))) return;
const destroySession = onboardSession.loadSession();
const portableContainerAuthority = preparePortableDemoSandboxDestroyAuthority(sandboxName, () => {
const current = registry.getSandbox(sandboxName);
return current
? {
agent: current.agent,
lifecycleGeneration: current.lifecycleGeneration,
openshellDriver: current.openshellDriver,
}
: null;
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

const inspectContainerIdentity = () =>
assertUnambiguousDestroyContainerIdentity(sandboxName, {
Expand All @@ -489,25 +500,36 @@ async function destroySandboxUnlocked(
),
redact: redactDestroyError,
});
const initialIdentity = inspectContainerIdentity();
const initialIdentity = portableContainerAuthority ? null : inspectContainerIdentity();
if (initialIdentity === false) {
process.exit(1);
}

const { cleanupGatewayName, runOpenshell, sandbox, sandboxConfirmedAbsent } =
prepareSandboxDestroy(sandboxName);
// Recheck identity after read-only preflight and before local mutation.
const preMutationIdentity = inspectContainerIdentity();
if (
preMutationIdentity === false ||
!isSameDestroyContainerIdentityProof(initialIdentity, preMutationIdentity)
) {
if (preMutationIdentity !== false) {
if (portableContainerAuthority) {
try {
portableContainerAuthority.revalidate();
} catch (error) {
console.error(
` Refusing to destroy sandbox '${sandboxName}': Container identity changed during preflight. No sandbox resources were removed.`,
` Refusing to destroy sandbox '${sandboxName}': NemoClaw could not revalidate Portable container identity during preflight: ${redactDestroyError(error)}. NemoClaw removed no sandbox resources.`,
);
process.exit(1);
}
} else {
const preMutationIdentity = inspectContainerIdentity();
if (
preMutationIdentity === false ||
!isSameDestroyContainerIdentityProof(initialIdentity!, preMutationIdentity)
) {
if (preMutationIdentity !== false) {
console.error(
` Refusing to destroy sandbox '${sandboxName}': Container identity changed during preflight. No sandbox resources were removed.`,
);
}
process.exit(1);
}
process.exit(1);
}
const priorHttpsPinRouteId = parseHttpsPinRouteId(sandbox?.endpointUrl);
const destructiveResult = await executeSandboxDestroy({
Expand All @@ -519,7 +541,8 @@ async function destroySandboxUnlocked(
sandbox,
sandboxConfirmedAbsent,
sandboxName,
expectedContainerIdentity: initialIdentity.identity,
expectedContainerIdentity: initialIdentity?.identity,
...(portableContainerAuthority ? { portableContainerAuthority } : {}),
stopInferenceResources: () => stopSandboxInferenceResources(sandboxName, sandbox),
});
if (!destructiveResult.ok) {
Expand Down Expand Up @@ -565,6 +588,13 @@ async function destroySandboxUnlocked(
console.error(
` Start the gateway (run '${CLI_NAME} ${sandboxName} status'), then retry destroy; --force cannot safely discard MCP ownership.`,
);
} else if (destructiveResult.portableLifecycleOwnershipRequiresGateway) {
console.error(
` The OpenShell gateway is unreachable. NemoClaw preserved the sandbox registry record and schema-4 Portable receipt because OpenShell sandbox deletion and absence of the exact Podman container are unconfirmed.`,
);
console.error(
` Start the gateway by running '${CLI_NAME} ${sandboxName} status'. Retry destroy. --force cannot discard this ownership state until NemoClaw confirms both results.`,
);
} else {
console.error(
` The OpenShell gateway is unreachable. Start it (run '${CLI_NAME} ${sandboxName} status'),`,
Expand Down
Loading
Loading