diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index 202d8ddddea..f4873859e9e 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -565,6 +565,9 @@ describe("rebuild recreate shields state", () => { "bail: Recreate failed (stale-sandbox recovery).", ); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("Sandbox recreate error: inner onboard failed"), + ); expect(clearShieldsState).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts index eb2c9dd742e..146bf74f0c3 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-phase.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -15,10 +15,7 @@ import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; import { cloneSandboxHostMounts } from "../../state/registry/host-mount"; -import { - excludePolicyPresetsByName, - type RebuildBackupManifest, -} from "./rebuild-backup-phase"; +import { excludePolicyPresetsByName, type RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; import { isolateAmbientRecreateEnv } from "./rebuild-env-isolation"; @@ -288,7 +285,9 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): await rebuildOnboardDependencies.onboard({ ...recreateOptions, rebuildGatewayAuthority, - ...(Array.isArray(recreatePolicyPresets) ? { rebuildPolicyPresets: recreatePolicyPresets } : {}), + ...(Array.isArray(recreatePolicyPresets) + ? { rebuildPolicyPresets: recreatePolicyPresets } + : {}), ...(rebuildsHermesSandbox && backupManifest?.preservedEnv ? { rebuildPreservedEnv: backupManifest.preservedEnv } : {}), @@ -299,7 +298,12 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): onboardFailed = true; const message = error instanceof Error ? error.message : String(error); const name = error instanceof Error ? error.name : ""; - if (name !== "RebuildOnboardExit") log(`onboard() threw: ${message}`); + if (name !== "RebuildOnboardExit") { + log(`onboard() threw: ${message}`); + console.error( + ` ${_RD}Sandbox recreate error:${R} ${onboardSession.redactSensitiveText(message) ?? "Inner onboarding failed."}`, + ); + } } finally { process.exit = savedExit; restoreRebuildBaseImageOverride(); diff --git a/src/lib/adapters/openshell/sandbox-observer-cli.test.ts b/src/lib/adapters/openshell/sandbox-observer-cli.test.ts index 30a3fa74b5a..d355f9f6182 100644 --- a/src/lib/adapters/openshell/sandbox-observer-cli.test.ts +++ b/src/lib/adapters/openshell/sandbox-observer-cli.test.ts @@ -183,6 +183,7 @@ describe("CLI OpenShell sandbox observer", () => { it.each([ ["transport", "unreachable", captured(1, "", "client error (Connect): Connection refused")], ["transport", "unreachable", captured(1, "", "Status: Disconnected")], + ["transport", "unreachable", captured(1, "", "Unknown gateway 'nemoclaw'.")], ["transport", "identity_mismatch", captured(1, "", "handshake verification failed")], ["schema", undefined, captured(1, "", "protobuf decode error: invalid wire type")], ["command", "failed", captured(7, "", "unexpected opaque failure")], diff --git a/src/lib/adapters/openshell/sandbox-observer-cli.ts b/src/lib/adapters/openshell/sandbox-observer-cli.ts index c264083b066..8ab1b40e1ce 100644 --- a/src/lib/adapters/openshell/sandbox-observer-cli.ts +++ b/src/lib/adapters/openshell/sandbox-observer-cli.ts @@ -173,7 +173,7 @@ function commandError(result: CapturedSandboxCommandResult): OpenShellSandboxErr }; } if ( - /\b(?:connection refused|client error \(connect\)|tcp connect error|transport error|connection reset|connection aborted|connection closed|no active gateway|no gateway configured)\b|status:\s*disconnected/iu.test( + /\b(?:connection refused|client error \(connect\)|tcp connect error|transport error|connection reset|connection aborted|connection closed|no active gateway|no gateway configured|unknown gateway)\b|status:\s*disconnected/iu.test( output, ) ) { diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 5c917ebe1cb..10da13d91ea 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -148,7 +148,14 @@ describe("finalizeDockerGpuPatchBackup", () => { inspect: { event: "confirm running replacement", result: { status: 0, stdout: "true\n" } }, } as const; const dockerRun = vi.fn((args: readonly string[]) => { - const response = dockerResults[String(args[0]) as keyof typeof dockerResults]; + const namespaceInspect = + args[0] === "inspect" && String(args[4]).includes("sandbox-namespace"); + const response = namespaceInspect + ? { + event: "read replacement namespace", + result: { status: 0, stdout: "current-gateway\n" }, + } + : dockerResults[String(args[0]) as keyof typeof dockerResults]; events.push(response.event); return response.result; }); @@ -187,6 +194,7 @@ describe("finalizeDockerGpuPatchBackup", () => { "start replacement", "observe ready", "exec ready", + "read replacement namespace", "confirm sole replacement", "confirm running replacement", ]); @@ -260,15 +268,14 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(sleep).not.toHaveBeenCalled(); }); - it("accepts a retiring Error row only when the exact replacement has the OpenShell label (#9962)", () => { + it("scopes the final sole-container proof to the replacement gateway namespace", () => { const result = exactDeferredCreateResult(); - const dockerRunResults = { - inspect: { status: 0, stdout: "true\n" }, - ps: { status: 0, stdout: `${result.newContainerId}\n` }, - } as const; - const dockerRun = vi.fn( - (args: readonly string[]) => - dockerRunResults[String(args[0]) as keyof typeof dockerRunResults], + const dockerRun = vi.fn((args: readonly string[]) => + args[0] === "inspect" + ? String(args[4]).includes("sandbox-namespace") + ? { status: 0, stdout: "current-gateway\n" } + : { status: 0, stdout: "true\n" } + : { status: 0, stdout: `${result.newContainerId}\n` }, ); const outcome = finalizeDockerGpuPatchBackup( @@ -299,6 +306,25 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffAcknowledged: true, }); expect(dockerRun.mock.calls[0]?.[0]).toEqual([ + "ps", + "-a", + "--no-trunc", + "--filter", + `id=${result.newContainerId}`, + "--filter", + "label=openshell.ai/managed-by=openshell", + "--format", + "{{.ID}}", + ]); + expect(dockerRun.mock.calls[1]?.[0]).toEqual([ + "inspect", + "--type", + "container", + "--format", + '{{ index .Config.Labels "openshell.ai/sandbox-namespace" }}', + result.newContainerId, + ]); + expect(dockerRun.mock.calls[2]?.[0]).toEqual([ "ps", "-a", "--no-trunc", @@ -306,11 +332,55 @@ describe("finalizeDockerGpuPatchBackup", () => { "label=openshell.ai/managed-by=openshell", "--filter", "label=openshell.ai/sandbox-name=restored-name", + "--filter", + "label=openshell.ai/sandbox-namespace=current-gateway", "--format", "{{.ID}}", ]); }); + it("rejects multiple same-name containers within the replacement gateway namespace", () => { + const result = exactDeferredCreateResult(); + const dockerRun = vi.fn((args: readonly string[]) => + args[0] === "inspect" + ? String(args[4]).includes("sandbox-namespace") + ? { status: 0, stdout: "current-gateway\n" } + : { status: 0, stdout: "true\n" } + : args.includes("label=openshell.ai/sandbox-namespace=current-gateway") + ? { status: 0, stdout: `${result.newContainerId}\n${"c".repeat(64)}\n` } + : { status: 0, stdout: `${result.newContainerId}\n` }, + ); + + const outcome = finalizeDockerGpuPatchBackup( + { + result, + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 60, + }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: 0 })), + dockerRun, + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"), + runOpenshell: vi.fn((args: readonly string[]) => + args[1] === "list" + ? { status: 0, stdout: "beta 2026-08-23 01:40:35 Ready\n" } + : { status: 0 }, + ), + sleep: vi.fn(), + }, + ); + + expect(outcome).toMatchObject({ + backupRemoved: true, + lifecycleReleaseObserved: true, + replacementRestarted: true, + finalHandoffAcknowledged: false, + }); + }); + it.each([ ["a failed query", { status: 1, stderr: "daemon unavailable" }], ["no labeled replacement", { status: 0, stdout: "" }], diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 7a1c09fe3c1..89960a6a39a 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -39,7 +39,12 @@ import { waitForOpenShellFinalHandoff, waitForOpenShellSandboxLifecycleRelease, } from "./docker-gpu-supervisor-reconnect"; -import { queryOpenShellDockerSandboxContainers } from "./openshell-docker-sandbox-containers"; +import { + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_NAMESPACE_LABEL, + queryOpenShellDockerSandboxContainers, +} from "./openshell-docker-sandbox-containers"; export { restoreDockerGpuPatchBackupAfterRecreateFailure as rollbackDockerGpuPatchOnRecreateFailure, @@ -72,6 +77,45 @@ export type DockerGpuPatchFinalizeOutcome = { replacementPresence?: "absent" | "present" | "unknown"; }; +function isExactOpenShellReplacement( + replacementContainerId: string, + dockerRun: NonNullable, + timeoutMs: number, +): boolean { + const expectedContainerId = fullDockerContainerId(replacementContainerId); + if (!expectedContainerId || timeoutMs <= 0) return false; + try { + const query = dockerRun( + [ + "ps", + "-a", + "--no-trunc", + "--filter", + `id=${expectedContainerId}`, + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--format", + "{{.ID}}", + ], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.max(1, Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, Math.floor(timeoutMs))), + }, + ); + if (!hasZeroDockerExitStatus(query)) return false; + const containerIds = String(query.stdout ?? "") + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + return ( + containerIds.length === 1 && fullDockerContainerId(containerIds[0]) === expectedContainerId + ); + } catch { + return false; + } +} + function isExactRunningReplacement( sandboxName: string, replacementContainerId: string, @@ -82,7 +126,36 @@ function isExactRunningReplacement( if (!expectedContainerId || timeoutMs <= 0) return false; try { const deadline = Date.now() + timeoutMs; - const containers = queryOpenShellDockerSandboxContainers(sandboxName, { dockerRun }, timeoutMs); + const namespace = dockerRun( + [ + "inspect", + "--type", + "container", + "--format", + `{{ index .Config.Labels "${OPENSHELL_SANDBOX_NAMESPACE_LABEL}" }}`, + expectedContainerId, + ], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, timeoutMs), + }, + ); + const sandboxNamespace = String(namespace.stdout ?? "").trim(); + if ( + !hasZeroDockerExitStatus(namespace) || + !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u.test(sandboxNamespace) + ) { + return false; + } + let remainingMs = deadline - Date.now(); + if (remainingMs <= 0) return false; + const containers = queryOpenShellDockerSandboxContainers( + sandboxName, + { dockerRun }, + remainingMs, + sandboxNamespace, + ); if ( !containers.ok || containers.ids.length !== 1 || @@ -90,7 +163,7 @@ function isExactRunningReplacement( ) { return false; } - const remainingMs = deadline - Date.now(); + remainingMs = deadline - Date.now(); if (remainingMs <= 0) return false; const inspect = dockerRun( [ @@ -174,20 +247,12 @@ export function finalizeDockerGpuPatchBackup( { runOpenshell: deps.runOpenshell, sleep: deps.sleep, - soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) => { - const expectedContainerId = fullDockerContainerId(options.result.newContainerId); - if (!expectedContainerId || remainingMs <= 0) return false; - const containers = queryOpenShellDockerSandboxContainers( - options.sandboxName, - { dockerRun: resolved.dockerRun }, + soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) => + isExactOpenShellReplacement( + options.result.newContainerId, + resolved.dockerRun, remainingMs, - ); - return ( - containers.ok && - containers.ids.length === 1 && - fullDockerContainerId(containers.ids[0]) === expectedContainerId - ); - }, + ), }, ); if (!lifecycleReleaseObserved) { diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index eae99520491..07f71433d58 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -347,7 +347,7 @@ export interface PrepareOnboardSandboxWorkloadLaunchInput { readonly policyAuthority: MaterializeSandboxCreatePlanInput["policyAuthority"]; readonly deferSandboxEffectsUntilPolicyVerification?: boolean; readonly rebindMessagingTokenDefs: () => Promise; - readonly runProviderPreDeleteCleanup: () => void; + readonly runProviderPreDeleteCleanup: MaterializeSandboxCreatePlanInput["runProviderPreDeleteCleanup"]; readonly upsertMessagingProviders: MaterializeSandboxCreatePlanInput["upsertMessagingProviders"]; readonly getHermesToolGatewayProviderName: (sandboxName: string) => string; readonly discloseInitialSandboxPolicy: (policy: InitialSandboxPolicy) => void; @@ -379,7 +379,7 @@ export interface PreparedOnboardSandboxWorkloadLaunch { readonly messagingProviders: string[]; readonly gpuRoutePlan: SandboxCreateIntent["gpuRoutePlan"]; readonly compatibilityPolicyPath: string | null; - readonly activateDeferredProviderEffects: (() => readonly string[]) | null; + readonly activateDeferredProviderEffects: SandboxCreatePlan["activateDeferredProviderEffects"]; readonly initialGpuRoute: SelectedDockerGpuRoute; readonly sandboxReadyTimeoutSecs: number; readonly buildId: string; diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index 9ec45b91bb0..b5d6cb38663 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -8,14 +8,15 @@ export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; +export const OPENSHELL_SANDBOX_NAMESPACE_LABEL = "openshell.ai/sandbox-namespace"; const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; const STALE_DOCKER_ORPHAN_TIMEOUT_MS = 30_000; type DockerSandboxContainerQueryDeps = Pick; -function sandboxContainerFilterArgs(sandboxName: string): string[] { - return [ +function sandboxContainerFilterArgs(sandboxName: string, sandboxNamespace?: string): string[] { + const args = [ "ps", "-a", "--no-trunc", @@ -24,6 +25,10 @@ function sandboxContainerFilterArgs(sandboxName: string): string[] { "--filter", `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, ]; + if (sandboxNamespace !== undefined) { + args.push("--filter", `label=${OPENSHELL_SANDBOX_NAMESPACE_LABEL}=${sandboxNamespace}`); + } + return args; } function commandResultText(result: { @@ -62,6 +67,7 @@ export function queryOpenShellDockerSandboxContainers( sandboxName: string, deps: DockerSandboxContainerQueryDeps = {}, timeoutMs: number = DOCKER_SANDBOX_QUERY_TIMEOUT_MS, + sandboxNamespace?: string, ): OpenShellDockerSandboxContainerQuery { const run = deps.dockerRun ?? dockerRun; const requestedTimeoutMs = @@ -72,11 +78,14 @@ export function queryOpenShellDockerSandboxContainers( 1, Math.min(DOCKER_SANDBOX_QUERY_TIMEOUT_MS, requestedTimeoutMs), ); - const result = run([...sandboxContainerFilterArgs(sandboxName), "--format", "{{.ID}}"], { - ignoreError: true, - suppressOutput: true, - timeout: boundedTimeoutMs, - }); + const result = run( + [...sandboxContainerFilterArgs(sandboxName, sandboxNamespace), "--format", "{{.ID}}"], + { + ignoreError: true, + suppressOutput: true, + timeout: boundedTimeoutMs, + }, + ); if (Number(result.status ?? 1) !== 0) { return { ok: false, diff --git a/src/lib/onboard/sandbox-create-intent-types.ts b/src/lib/onboard/sandbox-create-intent-types.ts index 9126e99b4d5..77d92092689 100644 --- a/src/lib/onboard/sandbox-create-intent-types.ts +++ b/src/lib/onboard/sandbox-create-intent-types.ts @@ -101,7 +101,9 @@ export type MaterializeSandboxCreatePlanInput = { deferSandboxEffectsUntilPolicyVerification?: boolean; managedStateMount?: ManagedHermesStateVolumeMount | null; messagingTokenDefs: MessagingTokenDef[]; - runProviderPreDeleteCleanup(): void; + runProviderPreDeleteCleanup( + revalidatePolicyRequirements?: (operation: string) => void, + ): void; upsertMessagingProviders( tokenDefs: MessagingTokenDef[], options: { diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index 3b59c81f988..5cfc1fb7432 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -83,7 +83,9 @@ export type SandboxCreatePlan = { compatibilityPolicyPath: string | null; sandboxGpuLogMessage: string | null; /** One-shot provider activation owned by the post-create verification boundary. */ - activateDeferredProviderEffects: (() => readonly string[]) | null; + activateDeferredProviderEffects: + | ((revalidatePolicyRequirements: (operation: string) => void) => readonly string[]) + | null; }; function sameProviderNames(left: readonly string[], right: readonly string[]): boolean { @@ -340,13 +342,16 @@ export function materializeSandboxCreatePlan({ } } - const activateProviderEffects = (): readonly string[] => { - runProviderPreDeleteCleanup(); + const activateProviderEffects = ( + revalidatePolicyRequirements?: (operation: string) => void, + ): readonly string[] => { + runProviderPreDeleteCleanup(revalidatePolicyRequirements); const activatedMessagingProviders = filterMessagingProvidersForSandboxCreate( [ ...upsertMessagingProviders(enabledMessagingTokenDefs, { replaceExisting: true, allowedSandboxes: [intent.sandboxName], + ...(revalidatePolicyRequirements ? { revalidatePolicyRequirements } : {}), }), ...intent.reusableMessagingProviders, ], diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 1ed1c5c62d9..49e3e07d72a 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -620,6 +620,9 @@ describe("resolveSandboxCreateIntent", () => { policyTier: null, }); const events: string[] = []; + const revalidatePolicyRequirements = vi.fn((operation: string) => + events.push(`revalidate:${operation}`), + ); const plan = materializeSandboxCreatePlan({ intent, fromRef: "example.invalid/image@sha256:abc", @@ -630,8 +633,13 @@ describe("resolveSandboxCreateIntent", () => { policyPath: "/tmp/policy.yaml", appliedPresets: ["telegram"], }), - runProviderPreDeleteCleanup: () => events.push("cleanup"), - upsertMessagingProviders: vi.fn(() => { + runProviderPreDeleteCleanup: (revalidate) => { + expect(revalidate).toBe(revalidatePolicyRequirements); + revalidate?.("cleaning up providers"); + events.push("cleanup"); + }, + upsertMessagingProviders: vi.fn((_tokenDefs, options) => { + expect(options.revalidatePolicyRequirements).toBe(revalidatePolicyRequirements); events.push("upsert"); return ["sandbox-telegram-bridge"]; }), @@ -648,14 +656,19 @@ describe("resolveSandboxCreateIntent", () => { "sandbox-existing-discord", ]); - expect(plan.activateDeferredProviderEffects?.()).toEqual([ + expect(plan.activateDeferredProviderEffects?.(revalidatePolicyRequirements)).toEqual([ "nvidia-prod", "sandbox-telegram-bridge", "sandbox-existing-discord", "sandbox-hermes-tools", "custom-provider", ]); - expect(events).toEqual(["cleanup", "upsert", "hermes"]); + expect(events).toEqual([ + "revalidate:cleaning up providers", + "cleanup", + "upsert", + "hermes", + ]); }); it("keeps the NemoClaw policy on a managed create when effects are deferred (#9833)", () => { diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index 2dd47de85c4..3da5163dae4 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; +import { PolicyAuthorityRefusalError } from "../../adapters/openshell/policy-authority"; import type { SandboxEntry } from "../../state/registry"; import { applyManagedSandboxRebuildPolicyCarryForward, @@ -88,7 +89,10 @@ describe("deferred provider effect authority", () => { cleanupCreateSources: vi.fn(), }, runVerifiedSandboxCreateEffects: null, - activateDeferredProviderEffects: () => ["first", "second"], + activateDeferredProviderEffects: (revalidate) => { + revalidate("cleaning up providers for sandbox 'alpha'"); + return ["first", "second"]; + }, revalidatePolicyAuthorityBeforeCreate: vi.fn(), runOpenshell: runOpenshell as never, revalidateSandboxIdentity, @@ -123,6 +127,7 @@ describe("deferred provider effect authority", () => { "attaching provider 'second' to sandbox 'alpha'", ); expect(events).toContain("policy: attaching provider 'second' to sandbox 'alpha'"); + expect(events).toContain("policy: cleaning up providers for sandbox 'alpha'"); expect(events).not.toContain("sandbox provider attach -g nemoclaw alpha second"); }); }); @@ -602,26 +607,26 @@ describe("sandbox create policy authority checks", () => { const persistVerifiedPolicy = vi.fn(); const runVerifiedCreateEffects = vi.fn(); - await expect( - runSandboxCreateWithPolicyAuthorityChecks({ - sandboxName: "alpha", - revalidate: vi.fn(), - create: async (verifyCreatedSandbox) => { - await verifyCreatedSandbox("created"); - return "created"; - }, - captureCreatedSandboxIdentity: () => exactIdentity, - revalidateCreatedSandboxIdentity: vi.fn(), - verifyCreatedPolicy: () => { - throw new Error("policy verification failed"); - }, - persistVerifiedPolicy, - revalidateVerifiedPolicy: vi.fn(), - runVerifiedCreateEffects, - cleanupTemporarySources: vi.fn(), - }), - ).rejects.toThrow("automatic sandbox cleanup was not safe"); + const error = await runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + captureCreatedSandboxIdentity: () => exactIdentity, + revalidateCreatedSandboxIdentity: vi.fn(), + verifyCreatedPolicy: () => { + throw new PolicyAuthorityRefusalError("policy verification failed"); + }, + persistVerifiedPolicy, + revalidateVerifiedPolicy: vi.fn(), + runVerifiedCreateEffects, + cleanupTemporarySources: vi.fn(), + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).message).toContain("policy verification failed"); expect(persistVerifiedPolicy).not.toHaveBeenCalled(); expect(runVerifiedCreateEffects).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 3c643ad59bc..9d630e6daf7 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import type { SandboxCreateOrchestrationRuntime } from "../../onboard"; -import { assertRecordedPolicyAuthority } from "../../adapters/openshell/policy-authority"; +import { + assertRecordedPolicyAuthority, + isPolicyAuthorityRefusalError, +} from "../../adapters/openshell/policy-authority"; import { HERMES_PORTABLE_OPENSHELL_VERSION } from "../../adapters/openshell/resolve-shared"; import type { AgentDefinition } from "../../agent/defs"; import type { WebSearchConfig } from "../../inference/web-search"; @@ -215,6 +218,10 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< }; const refuseAfterCreate = (validationError: unknown): never => { const compensationErrors = cleanupTemporarySources(); + const validationDetail = + validationError instanceof Error && isPolicyAuthorityRefusalError(validationError) + ? validationError.message + : null; const identityGuidance = exactIdentity ? ` Durable sandbox identity fingerprint: ${exactIdentity}. Use it only to compare the surviving sandbox with the failed create. Do not delete the sandbox by name, even after this comparison. Contact the OpenShell administrator for an identity-bound recovery or removal procedure.` : " OpenShell did not return a durable identity for comparison. Do not delete the sandbox by name. Contact the OpenShell administrator for an identity-bound recovery or removal procedure."; @@ -225,7 +232,7 @@ export async function runSandboxCreateWithPolicyAuthorityChecks< ); throw new AggregateError( [validationError, ...compensationErrors], - "Sandbox policy authority validation failed after creation; automatic sandbox cleanup was not safe.", + `Sandbox policy authority validation failed after creation${validationDetail ? `: ${validationDetail}` : ""}; automatic sandbox cleanup was not safe.`, ); }; const verifyCreatedSandbox = async (created: Created): Promise => { @@ -432,7 +439,9 @@ export function createProviderEffectBoundary(input: { readonly preparationInput: ProviderPreparationInput; readonly preparationDeps: ProviderPreparationDeps; readonly runVerifiedSandboxCreateEffects: import("../types").VerifiedSandboxCreateEffects | null; - readonly activateDeferredProviderEffects: (() => readonly string[]) | null; + readonly activateDeferredProviderEffects: + | ((revalidatePolicyRequirements: (operation: string) => void) => readonly string[]) + | null; readonly revalidatePolicyAuthorityBeforeCreate: () => void; readonly runOpenshell: SandboxCreateOrchestrationRuntime["runOpenshell"]; readonly revalidateSandboxIdentity: (exactIdentity: string, operation: string) => void; @@ -468,7 +477,8 @@ export function createProviderEffectBoundary(input: { context.revalidatePolicyRequirements( `activating deferred providers for sandbox '${input.sandboxName}'`, ); - const providerNames = input.activateDeferredProviderEffects?.() ?? []; + const providerNames = + input.activateDeferredProviderEffects?.(context.revalidatePolicyRequirements) ?? []; validate(); context.revalidatePolicyRequirements( `publishing deferred providers for sandbox '${input.sandboxName}'`, @@ -1474,9 +1484,9 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ) ).messagingTokenDefs; }, - runProviderPreDeleteCleanup: () => { - revalidatePolicyAuthority( - createIntent?.deferSandboxEffectsUntilPolicyVerification === true, + runProviderPreDeleteCleanup: (verifiedPolicyRevalidation) => { + (verifiedPolicyRevalidation ?? + ((operation) => revalidatePolicyAuthority(false, operation)))( `cleaning up providers for sandbox '${sandboxName}'`, ); runSandboxProviderPreDeleteCleanup(sandboxName, { @@ -1489,8 +1499,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche upsertMessagingProviders(tokenDefs, { ...options, revalidatePolicyRequirements: (operation) => - revalidatePolicyAuthority( - createIntent?.deferSandboxEffectsUntilPolicyVerification === true, + (options.revalidatePolicyRequirements ?? + ((targetOperation) => revalidatePolicyAuthority(false, targetOperation)))( operation, ), }), diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 26bf2518f23..811a7c46c5f 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -165,8 +165,8 @@ describe("created sandbox identity gate", () => { "runtime-check", "revalidate:apply runtime patch for sandbox 'alpha'", "runtime-patch", - "revalidate:reconnect sandbox supervisor for 'alpha'", "reconnect", + "revalidate:reconnect sandbox supervisor for 'alpha'", "readiness", "revalidate:commit runtime readiness for sandbox 'alpha'", "commit", @@ -424,7 +424,7 @@ describe("created sandbox identity gate", () => { expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); }); - it("stops before supervisor work when the durable checkpoint drifts (#9833)", async () => { + it("reconnects before rejecting lifecycle drift from the transient recreate state (#9833)", async () => { let nonce = ""; const input = noGpuInput(); input.verifyCreatedSandboxBeforeEffects = vi.fn(); @@ -445,7 +445,7 @@ describe("created sandbox identity gate", () => { await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("checkpoint changed"); expect(patch.ensureApplied).toHaveBeenCalledOnce(); - expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); + expect(patch.waitForSupervisorReconnectIfNeeded).toHaveBeenCalledOnce(); expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index fd2c788607a..aad97c09960 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -708,8 +708,8 @@ export function createSandboxGpuCreateAttemptRunner( revalidatePostCreateEffect(`apply runtime patch for sandbox '${input.sandboxName}'`); await runtimePatch.ensureApplied(); } - revalidatePostCreateEffect(`reconnect sandbox supervisor for '${input.sandboxName}'`); await runtimePatch.waitForSupervisorReconnectIfNeeded(); + revalidatePostCreateEffect(`reconnect sandbox supervisor for '${input.sandboxName}'`); console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 6b6aac7fe30..611ba48a243 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -1033,6 +1033,49 @@ describe("source registry fingerprint", () => { } }); + it("survives owned MCP policy preparation while retaining policy authority", () => { + const sourceEntry: SandboxEntry = { + ...SOURCE_ENTRY, + lifecycleGeneration: TARGET_GENERATION, + lifecycleLiveIdentityFingerprint: SOURCE_ID, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + sandboxName: "alpha", + lifecycleGeneration: TARGET_GENERATION, + sandboxIdentityFingerprint: SOURCE_ID, + policyHash: "policy-before", + policyVersion: 1, + }, + policies: ["mcp-search"], + customPolicies: [{ name: "mcp-search", content: "network_policies: {}" }], + mcp: { bridges: {}, managedServerNames: ["search"] }, + }; + const journaled = fingerprintSandboxRegistryEntry(sourceEntry); + const preparedEntry: SandboxEntry = { + ...sourceEntry, + policyCreationReceipt: { + ...sourceEntry.policyCreationReceipt!, + policyHash: "policy-after", + policyVersion: 2, + }, + policies: [], + customPolicies: [], + mcp: { bridges: {}, managedServerNames: [] }, + }; + + expect(fingerprintSandboxRegistryEntry(preparedEntry)).toBe(journaled); + expect( + fingerprintSandboxRegistryEntry({ + ...preparedEntry, + policyAuthority: "externally-managed", + }), + ).not.toBe(journaled); + }); + it("changes when the row records another sandbox", async () => { const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recreate-journal-")); vi.stubEnv("HOME", home); diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index 1652cbfa4ba..f5476ff9ec5 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -198,10 +198,21 @@ const ROUTE_RESERVATION_FIELDS: readonly (keyof SandboxEntry)[] = [ "gatewayName", "gatewayPort", ]; +// Rebuild may update these independently receipt-bound projections before delete. +// The source fingerprint still binds policyAuthority and every sandbox, gateway, +// lifecycle, agent, and workload ownership field. +const RECEIPT_BOUND_PROJECTION_FIELDS: readonly (keyof SandboxEntry)[] = [ + "policyCreationReceipt", + "policies", + "customPolicies", + "mcp", +]; export function fingerprintSandboxRegistryEntry(entry: SandboxEntry): string { const durable: Record = { ...entry }; - for (const field of ROUTE_RESERVATION_FIELDS) delete durable[field]; + for (const field of [...ROUTE_RESERVATION_FIELDS, ...RECEIPT_BOUND_PROJECTION_FIELDS]) { + delete durable[field]; + } return fingerprintSandboxRecreateValue(durable); } diff --git a/src/lib/openshell-sandbox-list.test.ts b/src/lib/openshell-sandbox-list.test.ts index 74232075ea8..946bad679c3 100644 --- a/src/lib/openshell-sandbox-list.test.ts +++ b/src/lib/openshell-sandbox-list.test.ts @@ -114,14 +114,17 @@ describe("sandbox list gateway preflight and recovery (#6237)", () => { expect(exitSpy).not.toHaveBeenCalled(); }); - it("recovers an unreachable explicit gateway after its scoped observation (#6114)", async () => { + it("recovers a missing explicit gateway after its scoped observation (#6114)", async () => { const options = { gatewayName: "nemoclaw-12345" }; mocks.captureOpenshell - .mockReturnValueOnce({ status: 1, output: "Status: Disconnected" }) + .mockReturnValueOnce({ status: 1, output: "Unknown gateway 'nemoclaw-12345'." }) .mockReturnValueOnce({ status: 0, output: "alpha Ready" }); const result = await captureSandboxListWithGatewayPreflightOrExit(context, options); + expect(result).toEqual({ + sandboxes: [{ name: "alpha", phase: "Ready", readiness: "ready" }], + }); expect(mocks.detectPreflightIssue).toHaveBeenCalledWith(options); const expectedRecoveryOptions = { gatewayName: "nemoclaw-12345", @@ -138,6 +141,7 @@ describe("sandbox list gateway preflight and recovery (#6237)", () => { ["sandbox", "list", "-g", "nemoclaw-12345"], expect.anything(), ); + expect(mocks.captureOpenshell).toHaveBeenCalledTimes(2); expect(exitSpy).not.toHaveBeenCalled(); }); diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index e3c5695a435..41602c487db 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -1370,7 +1370,7 @@ function classifyPresetEntries(currentPolicy: string, presetEntries: string): Pr function policyDocumentsMatch(left: string, right: string): boolean { try { - return isDeepStrictEqual(YAML.parse(left), YAML.parse(right)); + return isDeepStrictEqual(parseOpenShellPolicy(left).policy, parseOpenShellPolicy(right).policy); } catch { return false; } diff --git a/src/lib/policy/policy-mutation-authority.test.ts b/src/lib/policy/policy-mutation-authority.test.ts index 4dd00301fdf..9a5c2f6eb7c 100644 --- a/src/lib/policy/policy-mutation-authority.test.ts +++ b/src/lib/policy/policy-mutation-authority.test.ts @@ -203,6 +203,10 @@ describe("PolicyMutationAuthority", () => { }); it("rotates the receipt after a matching policy mutation (#9833)", () => { + mocks.captureSandboxBasePolicy.mockImplementation( + () => `Version: 2\nHash: updated\nStatus: Effective\n---\n${liveBasePolicy}`, + ); + expect( applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { custom: { sourcePath: "/tmp/weather.yaml" }, diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index c3cdbf59c85..d07db0b8183 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -257,6 +257,7 @@ const ONBOARD_SANDBOX_INSPECT = { Labels: { "openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-name": "my-assistant", + "openshell.ai/sandbox-namespace": "test-gateway", }, Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], Cmd: [], @@ -636,13 +637,19 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { const normalized = normalizeCommand(command); if ( finalCommitReleased && - normalized.startsWith("docker ps -a --no-trunc ") && - normalized.includes("label=openshell.ai/sandbox-name=my-assistant") && - normalized.endsWith("--format {{.ID}}") + ((normalized.startsWith("docker ps -a --no-trunc ") && + normalized.includes("label=openshell.ai/sandbox-name=my-assistant") && + normalized.endsWith("--format {{.ID}}")) || + normalized === + `docker inspect --type container --format {{ index .Config.Labels "openshell.ai/sandbox-namespace" }} ${ONBOARD_SANDBOX_NEW_CONTAINER_ID}`) ) { return { status: 0, - stdout: Buffer.from(`${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`), + stdout: Buffer.from( + normalized.startsWith("docker inspect ") + ? "test-gateway\n" + : `${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`, + ), stderr: Buffer.alloc(0), }; }