diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 716dec32ec9..4369bd88ee4 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -8,7 +8,7 @@ "test/generation/generate-openclaw-config.test.ts": 1906, "test/installer-integration/install-preflight.test.ts": 3025, "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 4671, - "test/onboarding/onboard-messaging.test.ts": 2016, + "test/onboarding/onboard-messaging.test.ts": 1979, "test/onboarding/onboard-selection.test.ts": 4176 } } diff --git a/nemoclaw/src/blueprint/runner-identity.test.ts b/nemoclaw/src/blueprint/runner-identity.test.ts index 2799875615c..ff6a478c65d 100644 --- a/nemoclaw/src/blueprint/runner-identity.test.ts +++ b/nemoclaw/src/blueprint/runner-identity.test.ts @@ -20,7 +20,11 @@ import { MATCHING_RUNTIME_PROVIDER_LISTING, providersV2EnabledResult, resultWithBlueprintPolicyAuthority, + sandboxIdentityResult, + sequentialCommandResult, successResult, + TEST_SANDBOX_POLICY, + TEST_SANDBOX_POLICY_PATH, } from "./runner-test-fixtures.js"; const { store } = createRunnerFsStore(); @@ -31,14 +35,20 @@ vi.mock("node:os", async (importOriginal) => { const original = await importOriginal(); return { ...original, homedir: () => FAKE_HOME }; }); -vi.mock("node:crypto", () => ({ randomUUID: () => FIXED_RUN_UUID })); +vi.mock("node:crypto", async (importOriginal) => ({ + ...(await importOriginal()), + randomUUID: () => FIXED_RUN_UUID, +})); vi.mock("node:fs", async (importOriginal) => { const original = await importOriginal(); const memory = inMemoryFsMethods(store, { realpaths, spy: vi.fn }); return { ...original, existsSync: memory.existsSync, + closeSync: memory.closeSync, + fsyncSync: memory.fsyncSync, mkdirSync: memory.mkdirSync, + openSync: memory.openSync, readFileSync: memory.readFileSync, renameSync: memory.renameSync, writeFileSync: memory.writeFileSync, @@ -65,6 +75,18 @@ const matchingInferenceRoute = MATCHING_INFERENCE_ROUTE_LISTING; const success = successResult(); const providersV2Enabled = providersV2EnabledResult(); +const POLICY_BOUNDARY_COMMAND = "policy get -g test-gateway --full --output json test-sandbox"; + +function expectPolicyBoundaryImmediatelyBefore( + commands: readonly string[], + mutation: string | ((command: string) => boolean), +): void { + const index = commands.findIndex((command) => + typeof mutation === "string" ? command === mutation : mutation(command), + ); + expect(index).toBeGreaterThan(0); + expect(commands[index - 1]).toBe(POLICY_BOUNDARY_COMMAND); +} function responseQueue( overrides: Array<[string, Array<{ exitCode?: number; stdout: string; stderr: string }>]>, @@ -80,6 +102,7 @@ function responseQueue( "sandbox get test-sandbox", { exitCode: 0, stdout: "Name: test-sandbox\nPhase: Ready", stderr: "" }, ], + ["provider get test-provider", { exitCode: 0, stdout: matchingInferenceProvider, stderr: "" }], ["settings get --global --json", providersV2Enabled], ]); mockExeca.mockImplementation(async (_command: string, args: string[]) => { @@ -94,17 +117,6 @@ function responseQueue( }); } -function nonAuthorityCommandLines(): string[] { - const authorityCommands = new Set([ - "openshell status", - "openshell policy list -g test-gateway --global --limit 1", - "openshell policy get -g test-gateway --full --output json test-sandbox", - ]); - return mockExeca.mock.calls - .map(([command, args]) => [command, ...(args ?? [])].join(" ")) - .filter((command) => !authorityCommands.has(command)); -} - function blueprint(overrides: Record = {}): Parameters[1] { return { components: { @@ -136,9 +148,34 @@ function oktaIdentity(profilePath = "provider-profiles/okta-runtime-v1.yaml") { }; } +function managedPolicyAuthorityReceipt(sandboxName = "test-sandbox") { + return { + authority: "nemoclaw-managed", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + scope: "sandbox", + sandbox_name: sandboxName, + policy_creation_receipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "test-gateway", + gatewayPort: 8080, + sandboxName, + lifecycleGeneration: FIXED_RUN_UUID, + sandboxIdentityFingerprint: + "52aad66e236c4a522e5a9b5adb8234b8bbf780d3e4120ccffb0c3dd35ad63aab", + policyHash: "sha256:test-policy", + policyVersion: 1, + }, + }; +} + describe("blueprint identity wrapper", () => { beforeEach(() => { store.clear(); + store.set(TEST_SANDBOX_POLICY_PATH, { type: "file", content: TEST_SANDBOX_POLICY }); + vi.stubEnv("OPENSHELL_SANDBOX_POLICY", TEST_SANDBOX_POLICY_PATH); realpaths.clear(); vi.clearAllMocks(); mockExeca.mockImplementation(async (_command: string, args: string[]) => @@ -202,6 +239,7 @@ describe("blueprint identity wrapper", () => { delete process.env.OKTA_CLIENT_SECRET; delete process.env.NEMOCLAW_BLUEPRINT_PATH; vi.restoreAllMocks(); + vi.unstubAllEnvs(); }); it("accepts an opt-in provider-neutral Okta identity configuration", () => { @@ -321,6 +359,32 @@ describe("blueprint identity wrapper", () => { "provider refresh rotate acme-okta-runtime --credential-key OKTA_ACCESS_TOKEN", ), ); + expectPolicyBoundaryImmediatelyBefore(commands, (command) => + command.startsWith("provider profile import --file "), + ); + expectPolicyBoundaryImmediatelyBefore( + commands, + "provider create --name acme-okta-runtime --type okta-runtime-v1 --runtime-credentials", + ); + expectPolicyBoundaryImmediatelyBefore(commands, (command) => + command.startsWith("provider refresh configure "), + ); + expectPolicyBoundaryImmediatelyBefore( + commands, + "provider create --name test-provider --type openai --config OPENAI_BASE_URL=https://api.example.com/v1", + ); + expectPolicyBoundaryImmediatelyBefore( + commands, + "inference set --provider test-provider --model test-model", + ); + expectPolicyBoundaryImmediatelyBefore( + commands, + "sandbox provider attach test-sandbox acme-okta-runtime", + ); + expectPolicyBoundaryImmediatelyBefore( + commands, + "provider refresh rotate acme-okta-runtime --credential-key OKTA_ACCESS_TOKEN", + ); }); it("fails closed when an identity subprocess has no exit code", async () => { @@ -345,206 +409,90 @@ describe("blueprint identity wrapper", () => { ).not.toContain("refresh configure"); }); - it("fails before identity mutation when the target sandbox cannot be inspected", async () => { - process.env.OKTA_CLIENT_ID = "client-id"; - process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; - process.env.OKTA_CLIENT_SECRET = "client-secret"; - responseQueue([ - ["sandbox get test-sandbox", [failureResult("gateway configuration not found")]], - ]); - - await expect(actionApply("default", blueprint({ identity: oktaIdentity() }))).rejects.toThrow( - /Failed to inspect sandbox 'test-sandbox'.*gateway configuration not found/, - ); - - const commandLines = nonAuthorityCommandLines(); - expect(commandLines).toEqual(["openshell sandbox get test-sandbox"]); - }); - - it("fails before identity mutation when the target sandbox is not Ready", async () => { + it("establishes the policy receipt before the first identity mutation", async () => { process.env.OKTA_CLIENT_ID = "client-id"; process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; process.env.OKTA_CLIENT_SECRET = "client-secret"; responseQueue([ [ - "sandbox get test-sandbox", - [{ exitCode: 0, stdout: "Name: test-sandbox\nPhase: Provisioning", stderr: "" }], - ], - ]); - - await expect(actionApply("default", blueprint({ identity: oktaIdentity() }))).rejects.toThrow( - /Sandbox 'test-sandbox' is not reusable.*Ready phase.*Provisioning/, - ); - - const commandLines = nonAuthorityCommandLines(); - expect(commandLines).toEqual(["openshell sandbox get test-sandbox"]); - }); - - it.each([ - [ - "cannot be inspected", - failureResult("gateway route unavailable"), - /Failed to inspect sandbox 'test-sandbox' after concurrent creation.*gateway route unavailable/, - ], - [ - "is not Ready", - { exitCode: 0, stdout: "Name: test-sandbox\nPhase: Provisioning", stderr: "" }, - /Sandbox 'test-sandbox' is not reusable.*Ready phase.*Provisioning/, - ], - ])( - "fails closed when a concurrently created sandbox %s", - async (_label, racedSandbox, expectedError) => { - process.env.OKTA_CLIENT_ID = "client-id"; - process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; - process.env.OKTA_CLIENT_SECRET = "client-secret"; - responseQueue([ - ["sandbox get test-sandbox", [failureResult("sandbox not found"), racedSandbox]], - [ - "provider get acme-okta-runtime", - [ - failureResult("provider not found"), - ...Array.from({ length: 4 }, () => ({ - exitCode: 0, - stdout: matchingProvider, - stderr: "", - })), - ], - ], - [ - "sandbox create --from openclaw --name test-sandbox --forward 18789", - [failureResult("sandbox already exists")], - ], - ]); - - await expect(actionApply("default", blueprint({ identity: oktaIdentity() }))).rejects.toThrow( - expectedError, - ); - - const commands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); - expect(commands.filter((command) => command === "sandbox get test-sandbox")).toHaveLength(2); - expect(commands).not.toContain("sandbox provider attach test-sandbox acme-okta-runtime"); - expect(commands).toContain("provider delete acme-okta-runtime"); - }, - ); - - it("fails before identity mutation when a reused sandbox's inference provider cannot be inspected", async () => { - process.env.OKTA_CLIENT_ID = "client-id"; - process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; - process.env.OKTA_CLIENT_SECRET = "client-secret"; - responseQueue([ - [ - "sandbox get test-sandbox", - [{ exitCode: 0, stdout: "Name: test-sandbox\nPhase: Ready", stderr: "" }], - ], - ["provider get test-provider", [failureResult("gateway configuration not found")]], - ]); - - await expect(actionApply("default", blueprint({ identity: oktaIdentity() }))).rejects.toThrow( - /Failed to inspect inference provider 'test-provider'.*gateway configuration not found/, - ); - - const commandLines = nonAuthorityCommandLines(); - expect(commandLines).toEqual([ - "openshell sandbox get test-sandbox", - "openshell provider get test-provider", - ]); - }); - - it("rejects a mismatched inference provider before identity mutation for a new sandbox", async () => { - process.env.OKTA_CLIENT_ID = "client-id"; - process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; - process.env.OKTA_CLIENT_SECRET = "client-secret"; - responseQueue([ - [ - "provider get test-provider", + "provider get acme-okta-runtime", [ - { + failureResult("provider not found"), + ...Array.from({ length: 4 }, () => ({ exitCode: 0, - stdout: matchingInferenceProvider.replace("Type: openai", "Type: anthropic"), + stdout: matchingProvider, stderr: "", - }, + })), ], ], ]); - await expect(actionApply("default", blueprint({ identity: oktaIdentity() }))).rejects.toThrow( - /Inference provider 'test-provider' does not match the requested non-secret binding/, - ); + await actionApply("default", blueprint({ identity: oktaIdentity() })); - const commandLines = nonAuthorityCommandLines(); - expect(commandLines).toEqual([ - "openshell sandbox get test-sandbox", - "openshell provider get test-provider", - ]); + const commands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); + const createIndex = commands.indexOf( + "sandbox create -g test-gateway --from openclaw --name test-sandbox --policy /tmp/nemoclaw-test-policy.yaml --forward 18789", + ); + const firstReceiptValidation = commands.indexOf( + "sandbox get -g test-gateway test-sandbox", + createIndex + 1, + ); + const firstIdentityMutation = commands.indexOf( + "provider create --name acme-okta-runtime --type okta-runtime-v1 --runtime-credentials", + ); + expect(createIndex).toBeGreaterThan(-1); + expect(firstReceiptValidation).toBeGreaterThan(createIndex); + expect(firstIdentityMutation).toBeGreaterThan(firstReceiptValidation); }); - it("fails before identity mutation when a reused route cannot be inspected", async () => { + it("stops before identity mutation when the receipt sandbox identity changes", async () => { process.env.OKTA_CLIENT_ID = "client-id"; process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; process.env.OKTA_CLIENT_SECRET = "client-secret"; - responseQueue([ - [ - "sandbox get test-sandbox", - [{ exitCode: 0, stdout: "Name: test-sandbox\nPhase: Ready", stderr: "" }], - ], - [ - "provider get test-provider", - [{ exitCode: 0, stdout: matchingInferenceProvider, stderr: "" }], - ], - ["inference get", [failureResult("gateway route inspection unavailable")]], + const identityResult = sequentialCommandResult("sandbox get -g test-gateway test-sandbox", [ + sandboxIdentityResult("test-sandbox"), + sandboxIdentityResult("test-sandbox", "replacement-id"), ]); + mockExeca.mockImplementation( + async (_command: string, args: string[]) => + identityResult(args) ?? + resultWithBlueprintPolicyAuthority( + args, + args.join(" ") === "settings get --global --json" ? providersV2Enabled : success, + ), + ); await expect(actionApply("default", blueprint({ identity: oktaIdentity() }))).rejects.toThrow( - /Failed to inspect the active inference route.*gateway route inspection unavailable/, + /receipt does not match the live sandbox policy/u, ); - - const commandLines = nonAuthorityCommandLines(); - expect(commandLines).toEqual([ - "openshell sandbox get test-sandbox", - "openshell provider get test-provider", - "openshell inference get", - ]); + const commands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); + expect(commands).not.toContain( + "provider create --name acme-okta-runtime --type okta-runtime-v1 --runtime-credentials", + ); + expect(commands).not.toContain("sandbox provider attach test-sandbox acme-okta-runtime"); }); - it("revalidates the sandbox immediately before attaching runtime identity", async () => { + it("validates the policy receipt before inference-provider reuse inspection", async () => { process.env.OKTA_CLIENT_ID = "client-id"; process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; process.env.OKTA_CLIENT_SECRET = "client-secret"; responseQueue([ - [ - "sandbox get test-sandbox", - [ - { exitCode: 0, stdout: "Name: test-sandbox\nPhase: Ready", stderr: "" }, - { exitCode: 0, stdout: "Name: test-sandbox\nPhase: Provisioning", stderr: "" }, - ], - ], - [ - "provider get test-provider", - [{ exitCode: 0, stdout: matchingInferenceProvider, stderr: "" }], - ], - [ - "provider get acme-okta-runtime", - [ - failureResult("provider not found"), - ...Array.from({ length: 4 }, () => ({ - exitCode: 0, - stdout: matchingProvider, - stderr: "", - })), - ], - ], + ["provider get test-provider", [failureResult("gateway configuration not found")]], ]); await expect(actionApply("default", blueprint({ identity: oktaIdentity() }))).rejects.toThrow( - /Sandbox 'test-sandbox' is not reusable.*Ready phase.*Provisioning/, + /Failed to inspect inference provider 'test-provider'.*gateway configuration not found/u, ); - const commands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); - expect(commands.filter((command) => command === "sandbox get test-sandbox")).toHaveLength(2); - expect(commands).not.toContain("sandbox provider attach test-sandbox acme-okta-runtime"); - expect(commands).toContain("provider delete acme-okta-runtime"); + const receiptValidation = commands.indexOf("sandbox get -g test-gateway test-sandbox"); + const providerInspection = commands.indexOf("provider get test-provider"); + expect(receiptValidation).toBeGreaterThanOrEqual(0); + expect(providerInspection).toBeGreaterThanOrEqual(0); + expect(receiptValidation).toBeLessThan(providerInspection); + expect(commands).not.toContain( + "provider create --name acme-okta-runtime --type okta-runtime-v1 --runtime-credentials", + ); }); - it.each([ ["not configured", "Gateway inference:\n\n Not configured\n"], [ @@ -774,7 +722,7 @@ describe("blueprint identity wrapper", () => { ); expect(mockExeca).toHaveBeenCalledWith( "openshell", - ["sandbox", "remove", "test-sandbox"], + ["sandbox", "remove", "-g", "test-gateway", "test-sandbox"], expect.objectContaining({ reject: false }), ); const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json"))?.[1]; @@ -817,13 +765,24 @@ describe("blueprint identity wrapper", () => { }, }), ), - ).rejects.toThrow(/policy read rejected/); + ).rejects.toThrow("OpenShell policy receipt inspection failed."); const commands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); expect(commands).toContain("sandbox provider detach test-sandbox acme-okta-runtime"); expect(commands).toContain("provider delete acme-okta-runtime"); - expect(commands).toContain("sandbox remove test-sandbox"); + expect(commands).toContain("sandbox remove -g test-gateway test-sandbox"); expect(commands).toContain("provider delete test-provider"); + expectPolicyBoundaryImmediatelyBefore( + commands, + "sandbox provider detach test-sandbox acme-okta-runtime", + ); + expectPolicyBoundaryImmediatelyBefore(commands, "provider delete acme-okta-runtime"); + expectPolicyBoundaryImmediatelyBefore(commands, "provider delete test-provider"); + expectPolicyBoundaryImmediatelyBefore(commands, "sandbox stop -g test-gateway test-sandbox"); + expectPolicyBoundaryImmediatelyBefore(commands, "sandbox remove -g test-gateway test-sandbox"); + expect(commands.indexOf("provider delete test-provider")).toBeLessThan( + commands.indexOf("sandbox remove -g test-gateway test-sandbox"), + ); const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json"))?.[1]; expect(JSON.parse(planEntry!.content!)).toMatchObject({ sandbox_created_by_apply: false, @@ -864,12 +823,12 @@ describe("blueprint identity wrapper", () => { expect(mockExeca).toHaveBeenCalledWith( "openshell", - ["sandbox", "remove", "test-sandbox"], + ["sandbox", "remove", "-g", "test-gateway", "test-sandbox"], expect.objectContaining({ reject: false }), ); }); - it("persists reused sandbox ownership and preserves it during later rollback", async () => { + it("persists exact created-sandbox ownership and consumes it during rollback", async () => { process.env.OKTA_CLIENT_ID = "client-id"; process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; process.env.OKTA_CLIENT_SECRET = "client-secret"; @@ -898,8 +857,8 @@ describe("blueprint identity wrapper", () => { await actionApply("default", blueprint({ identity: oktaIdentity() })); const applyCommands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); - expect(applyCommands).not.toContain( - "sandbox create --from openclaw --name test-sandbox --forward 18789", + expect(applyCommands).toContain( + "sandbox create -g test-gateway --from openclaw --name test-sandbox --policy /tmp/nemoclaw-test-policy.yaml --forward 18789", ); expect(applyCommands).toContain("provider get test-provider"); expect(applyCommands).toContain("inference get"); @@ -912,11 +871,15 @@ describe("blueprint identity wrapper", () => { const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json"))?.[1]; expect(planEntry?.content).toBeDefined(); const plan = JSON.parse(planEntry!.content!); - expect(plan.sandbox_created_by_apply).toBe(false); + expect(plan.sandbox_created_by_apply).toBe(true); expect(plan.inference_provider_created_by_apply).toBe(false); mockExeca.mockClear(); responseQueue([ + [ + "provider get test-provider", + [{ exitCode: 0, stdout: matchingInferenceProvider, stderr: "" }], + ], [ "provider get acme-okta-runtime", [ @@ -928,10 +891,23 @@ describe("blueprint identity wrapper", () => { await actionRollback(plan.run_id); const rollbackCommands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); - expect(rollbackCommands).not.toContain("sandbox stop test-sandbox"); - expect(rollbackCommands).not.toContain("sandbox remove test-sandbox"); + expect(rollbackCommands).toContain("sandbox stop -g test-gateway test-sandbox"); + expect(rollbackCommands).toContain("sandbox remove -g test-gateway test-sandbox"); expect(rollbackCommands).toContain("sandbox provider detach test-sandbox acme-okta-runtime"); expect(rollbackCommands).toContain("provider delete acme-okta-runtime"); + expectPolicyBoundaryImmediatelyBefore( + rollbackCommands, + "sandbox provider detach test-sandbox acme-okta-runtime", + ); + expectPolicyBoundaryImmediatelyBefore(rollbackCommands, "provider delete acme-okta-runtime"); + expectPolicyBoundaryImmediatelyBefore( + rollbackCommands, + "sandbox stop -g test-gateway test-sandbox", + ); + expectPolicyBoundaryImmediatelyBefore( + rollbackCommands, + "sandbox remove -g test-gateway test-sandbox", + ); }); it("preserves a sandbox for a legacy plan without an ownership receipt", async () => { @@ -950,7 +926,7 @@ describe("blueprint identity wrapper", () => { expect(store.get(`${stateDir}/rolled_back`)?.content).toBeDefined(); }); - it("creates an explicitly missing inference provider for a reused sandbox", async () => { + it("creates an explicitly missing inference provider for a receipt-owned sandbox", async () => { process.env.OKTA_CLIENT_ID = "client-id"; process.env.OKTA_REFRESH_TOKEN = "refresh-secret"; process.env.OKTA_CLIENT_SECRET = "client-secret"; @@ -987,12 +963,16 @@ describe("blueprint identity wrapper", () => { ); const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json"))?.[1]; expect(JSON.parse(planEntry!.content!)).toMatchObject({ - sandbox_created_by_apply: false, + sandbox_created_by_apply: true, inference_provider_created_by_apply: true, }); mockExeca.mockClear(); responseQueue([ + [ + "provider get test-provider", + [{ exitCode: 0, stdout: matchingInferenceProvider, stderr: "" }], + ], [ "provider get acme-okta-runtime", [ @@ -1004,7 +984,19 @@ describe("blueprint identity wrapper", () => { await actionRollback(JSON.parse(planEntry!.content!).run_id); const rollbackCommands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); expect(rollbackCommands).toContain("provider delete test-provider"); - expect(rollbackCommands).not.toContain("sandbox remove test-sandbox"); + expect(rollbackCommands).toContain("sandbox remove -g test-gateway test-sandbox"); + expectPolicyBoundaryImmediatelyBefore(rollbackCommands, "provider delete test-provider"); + expectPolicyBoundaryImmediatelyBefore( + rollbackCommands, + "sandbox stop -g test-gateway test-sandbox", + ); + expectPolicyBoundaryImmediatelyBefore( + rollbackCommands, + "sandbox remove -g test-gateway test-sandbox", + ); + expect(rollbackCommands.indexOf("provider delete test-provider")).toBeLessThan( + rollbackCommands.indexOf("sandbox remove -g test-gateway test-sandbox"), + ); }); it("keeps an owned sandbox receipt retryable when removal fails", async () => { @@ -1015,9 +1007,12 @@ describe("blueprint identity wrapper", () => { content: JSON.stringify({ sandbox_name: "owned-sandbox", sandbox_created_by_apply: true, + policy_authority: managedPolicyAuthorityReceipt("owned-sandbox"), }), }); - responseQueue([["sandbox remove owned-sandbox", [failureResult("remove denied")]]]); + responseQueue([ + ["sandbox remove -g test-gateway owned-sandbox", [failureResult("remove denied")]], + ]); await expect(actionRollback("failed-sandbox-removal")).rejects.toThrow( /Failed to remove owned sandbox 'owned-sandbox': remove denied/, @@ -1033,10 +1028,17 @@ describe("blueprint identity wrapper", () => { content: JSON.stringify({ sandbox_name: "test-sandbox", inference_provider_created_by_apply: true, - inference: { provider_name: "test-provider" }, + inference: { provider_name: "test-provider", provider_type: "openai" }, + policy_authority: managedPolicyAuthorityReceipt(), }), }); - responseQueue([["provider delete test-provider", [failureResult("provider delete denied")]]]); + responseQueue([ + [ + "provider get test-provider", + [{ exitCode: 0, stdout: matchingInferenceProvider, stderr: "" }], + ], + ["provider delete test-provider", [failureResult("provider delete denied")]], + ]); await expect(actionRollback("failed-inference-provider-removal")).rejects.toThrow( /Failed to remove owned inference provider 'test-provider': provider delete denied/, @@ -1044,6 +1046,92 @@ describe("blueprint identity wrapper", () => { expect(store.get(`${stateDir}/rolled_back`)).toBeUndefined(); }); + it("preserves rollback resources when policy authority changes before provider removal", async () => { + const stateDir = "/fakehome/.nemoclaw/state/runs/provider-authority-drift"; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + sandbox_name: "test-sandbox", + inference_provider_created_by_apply: true, + inference: { provider_name: "test-provider", provider_type: "openai" }, + policy_authority: managedPolicyAuthorityReceipt(), + }), + }); + const policyResult = sequentialCommandResult(POLICY_BOUNDARY_COMMAND, [ + resultWithBlueprintPolicyAuthority(POLICY_BOUNDARY_COMMAND.split(" "), success), + { + ...resultWithBlueprintPolicyAuthority(POLICY_BOUNDARY_COMMAND.split(" "), success), + stdout: JSON.stringify({ + scope: "sandbox", + sandbox: "test-sandbox", + status: "effective", + policy_source: "sandbox", + hash: "sha256:replacement-policy", + active_version: 2, + policy: { version: 1, network_policies: {} }, + }), + }, + ]); + mockExeca.mockImplementation(async (_command: string, args: string[]) => + args.join(" ") === "provider get test-provider" + ? { exitCode: 0, stdout: matchingInferenceProvider, stderr: "" } + : (policyResult(args) ?? resultWithBlueprintPolicyAuthority(args, success)), + ); + + await expect(actionRollback("provider-authority-drift")).rejects.toThrow( + /receipt does not match the live sandbox policy/u, + ); + const commands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); + expect(commands).not.toContain("provider delete test-provider"); + expect(commands).not.toContain("sandbox stop -g test-gateway test-sandbox"); + expect(commands).not.toContain("sandbox remove -g test-gateway test-sandbox"); + }); + + it("preserves a stopped sandbox when authority changes before removal", async () => { + const stateDir = "/fakehome/.nemoclaw/state/runs/sandbox-authority-drift"; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + sandbox_name: "test-sandbox", + sandbox_created_by_apply: true, + policy_authority: managedPolicyAuthorityReceipt(), + }), + }); + const matchingPolicy = resultWithBlueprintPolicyAuthority( + POLICY_BOUNDARY_COMMAND.split(" "), + success, + ); + const policyResult = sequentialCommandResult(POLICY_BOUNDARY_COMMAND, [ + matchingPolicy, + matchingPolicy, + { + ...matchingPolicy, + stdout: JSON.stringify({ + scope: "sandbox", + sandbox: "test-sandbox", + status: "effective", + policy_source: "sandbox", + hash: "sha256:replacement-policy", + active_version: 2, + policy: { version: 1, network_policies: {} }, + }), + }, + ]); + mockExeca.mockImplementation( + async (_command: string, args: string[]) => + policyResult(args) ?? resultWithBlueprintPolicyAuthority(args, success), + ); + + await expect(actionRollback("sandbox-authority-drift")).rejects.toThrow( + /receipt does not match the live sandbox policy/u, + ); + const commands = mockExeca.mock.calls.map(([, args]) => (args ?? []).join(" ")); + expect(commands).toContain("sandbox stop -g test-gateway test-sandbox"); + expect(commands).not.toContain("sandbox remove -g test-gateway test-sandbox"); + }); + it("rejects an invalid owned inference provider receipt before mutation", async () => { const stateDir = "/fakehome/.nemoclaw/state/runs/invalid-inference-provider"; store.set(stateDir, { type: "dir" }); @@ -1083,6 +1171,7 @@ describe("blueprint identity wrapper", () => { "provider delete acme-okta-runtime", [failureResult("delete denied"), { exitCode: 0, stdout: "", stderr: "" }], ], + ["inference get", [failureResult("inference route not found")]], [ "inference set --provider test-provider --model test-model", [failureResult("route failed")], @@ -1168,11 +1257,16 @@ describe("blueprint identity wrapper", () => { sandbox_name: "test-sandbox", sandbox_created_by_apply: true, inference_provider_created_by_apply: true, - inference: { provider_name: "test-provider" }, + inference: { provider_name: "test-provider", provider_type: "openai" }, identity: receipt, + policy_authority: managedPolicyAuthorityReceipt(), }), }); responseQueue([ + [ + "provider get test-provider", + [{ exitCode: 0, stdout: matchingInferenceProvider, stderr: "" }], + ], [ "provider get acme-okta-runtime", [ @@ -1202,12 +1296,12 @@ describe("blueprint identity wrapper", () => { ); expect(mockExeca).toHaveBeenCalledWith( "openshell", - ["sandbox", "stop", "test-sandbox"], + ["sandbox", "stop", "-g", "test-gateway", "test-sandbox"], expect.objectContaining({ reject: false }), ); expect(mockExeca).toHaveBeenCalledWith( "openshell", - ["sandbox", "remove", "test-sandbox"], + ["sandbox", "remove", "-g", "test-gateway", "test-sandbox"], expect.objectContaining({ reject: false }), ); expect(mockExeca).toHaveBeenCalledWith( diff --git a/nemoclaw/src/blueprint/runner-mock-fixtures.test.ts b/nemoclaw/src/blueprint/runner-mock-fixtures.test.ts index 62cbda8fe60..13993ab6ac0 100644 --- a/nemoclaw/src/blueprint/runner-mock-fixtures.test.ts +++ b/nemoclaw/src/blueprint/runner-mock-fixtures.test.ts @@ -14,4 +14,19 @@ describe("blueprint runner mock fixtures", () => { memory.mkdirSync("/sandbox"); expect(memory.existsSync("/sandbox")).toBe(true); }); + + it("models durable file descriptors and missing filesystem entries (#9833)", () => { + const { store } = createRunnerFsStore(); + const memory = inMemoryFsMethods(store); + + expect(() => memory.openSync("/missing")).toThrow(/ENOENT/u); + expect(() => memory.renameSync("/missing", "/renamed")).toThrow(/ENOENT/u); + + memory.writeFileSync("/receipt", "complete"); + const fd = memory.openSync("/receipt"); + expect(() => memory.fsyncSync(fd)).not.toThrow(); + expect(() => memory.closeSync(fd)).not.toThrow(); + expect(() => memory.fsyncSync(fd)).toThrow(`EBADF: ${fd}`); + expect(() => memory.closeSync(fd)).toThrow(`EBADF: ${fd}`); + }); }); diff --git a/nemoclaw/src/blueprint/runner-mock-fixtures.ts b/nemoclaw/src/blueprint/runner-mock-fixtures.ts index 3e0d7aec5d9..f38caab9f46 100644 --- a/nemoclaw/src/blueprint/runner-mock-fixtures.ts +++ b/nemoclaw/src/blueprint/runner-mock-fixtures.ts @@ -22,7 +22,7 @@ export interface RunnerFsEntry { export const FAKE_HOME = "/fakehome"; /** The deterministic UUID every runner suite mocks crypto.randomUUID() to. */ -export const FIXED_RUN_UUID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; +export const FIXED_RUN_UUID = "aaaaaaaa-bbbb-4ccc-addd-eeeeeeeeeeee"; /** An in-memory filesystem store with its seeding helpers. */ export interface RunnerFsStore { @@ -59,6 +59,8 @@ export interface InMemoryFsOptions { */ export function inMemoryFsMethods(store: Map, options?: InMemoryFsOptions) { const spy = options?.spy ?? ((fn: T): T => fn); + let nextFd = 100; + const openFiles = new Map(); return { existsSync: (p: string) => store.has(p), mkdirSync: spy((p: string) => void store.set(p, { type: "dir" })), @@ -69,6 +71,18 @@ export function inMemoryFsMethods(store: Map, options?: I writeFileSync: spy((p: string, data: string) => { store.set(p, { type: "file", content: String(data) }); }), + openSync: spy((p: string) => { + if (!store.has(p)) return missingEntry(p); + const fd = nextFd++; + openFiles.set(fd, p); + return fd; + }), + fsyncSync: spy((fd: number) => { + if (!openFiles.has(fd)) throw new Error(`EBADF: ${fd}`); + }), + closeSync: spy((fd: number) => { + if (!openFiles.delete(fd)) throw new Error(`EBADF: ${fd}`); + }), renameSync: spy((source: string, destination: string) => { const entry = store.get(source); if (!entry) return missingEntry(source); @@ -141,3 +155,12 @@ export function createStdoutCapture(): StdoutCapture { }, }; } + +/** Returns a callback that throws on one numbered invocation. */ +export function throwOnCall(callNumber: number, error: Error): () => void { + let calls = 0; + return () => { + calls += 1; + if (calls === callNumber) throw error; + }; +} diff --git a/nemoclaw/src/blueprint/runner-name-validation.test.ts b/nemoclaw/src/blueprint/runner-name-validation.test.ts index 763444c2d4b..48edd614989 100644 --- a/nemoclaw/src/blueprint/runner-name-validation.test.ts +++ b/nemoclaw/src/blueprint/runner-name-validation.test.ts @@ -20,6 +20,8 @@ import { minimalBlueprint, resultWithBlueprintPolicyAuthority, successResult, + TEST_SANDBOX_POLICY, + TEST_SANDBOX_POLICY_PATH, } from "./runner-test-fixtures.js"; const { store, addFile, addDir } = createRunnerFsStore(); @@ -28,7 +30,8 @@ vi.mock("node:os", () => ({ homedir: () => FAKE_HOME, })); -vi.mock("node:crypto", () => ({ +vi.mock("node:crypto", async (importOriginal) => ({ + ...(await importOriginal()), randomUUID: () => FIXED_RUN_UUID, })); @@ -38,7 +41,10 @@ vi.mock("node:fs", async (importOriginal) => { return { ...original, existsSync: memory.existsSync, + closeSync: memory.closeSync, + fsyncSync: memory.fsyncSync, mkdirSync: memory.mkdirSync, + openSync: memory.openSync, readFileSync: memory.readFileSync, renameSync: memory.renameSync, writeFileSync: memory.writeFileSync, @@ -116,6 +122,8 @@ describe("blueprint name validation (fail-closed integration)", () => { beforeEach(() => { store.clear(); + addFile(TEST_SANDBOX_POLICY_PATH, TEST_SANDBOX_POLICY); + vi.stubEnv("OPENSHELL_SANDBOX_POLICY", TEST_SANDBOX_POLICY_PATH); stdout.reset(); vi.clearAllMocks(); vi.spyOn(process.stdout, "write").mockImplementation(stdout.write); @@ -126,35 +134,38 @@ describe("blueprint name validation (fail-closed integration)", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); }); - it.each( - INVALID_NAME_CASES, - )("apply rejects a malformed %s and creates no sandbox", async (_label, corrupt, expected) => { - const bp = minimalBlueprint(); - corrupt(bp.components as BlueprintComponents); - - await expect(actionApply("default", bp)).rejects.toThrow(expected); - // Validation runs before any command, so no provider or sandbox command - // may have executed — not merely no sandbox create. - expect(mockExeca).not.toHaveBeenCalled(); - expect(createCalls()).toEqual([]); - }); + it.each(INVALID_NAME_CASES)( + "apply rejects a malformed %s and creates no sandbox", + async (_label, corrupt, expected) => { + const bp = minimalBlueprint(); + corrupt(bp.components as BlueprintComponents); + + await expect(actionApply("default", bp)).rejects.toThrow(expected); + // Validation runs before any command, so no provider or sandbox command + // may have executed — not merely no sandbox create. + expect(mockExeca).not.toHaveBeenCalled(); + expect(createCalls()).toEqual([]); + }, + ); - it.each( - VALID_PROVIDER_NAMES, - )("apply accepts the supported provider name '%s'", async (providerName) => { - const bp = minimalBlueprint(); - const components = bp.components as BlueprintComponents; - components.inference.profiles.default.provider_name = providerName; - - await expect(actionApply("default", bp)).resolves.toBeUndefined(); - expect(mockExeca).toHaveBeenCalledWith( - "openshell", - expect.arrayContaining(["provider", "create", "--name", providerName]), - expect.any(Object), - ); - }); + it.each(VALID_PROVIDER_NAMES)( + "apply accepts the supported provider name '%s'", + async (providerName) => { + const bp = minimalBlueprint(); + const components = bp.components as BlueprintComponents; + components.inference.profiles.default.provider_name = providerName; + + await expect(actionApply("default", bp)).resolves.toBeUndefined(); + expect(mockExeca).toHaveBeenCalledWith( + "openshell", + expect.arrayContaining(["provider", "create", "--name", providerName]), + expect.any(Object), + ); + }, + ); it("rollback rejects a plan whose sandbox_name is not OpenShell-compatible", async () => { const runDir = `${RUNS_DIR}/nc-run-1`; diff --git a/nemoclaw/src/blueprint/runner-openshell-072-policy.test.ts b/nemoclaw/src/blueprint/runner-openshell-072-policy.test.ts index a9c3ff70580..638d96b9732 100644 --- a/nemoclaw/src/blueprint/runner-openshell-072-policy.test.ts +++ b/nemoclaw/src/blueprint/runner-openshell-072-policy.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; import type fs from "node:fs"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -13,20 +14,27 @@ import { FIXED_RUN_UUID, inMemoryFsMethods, resolvedEndpointFor, + throwOnCall, } from "./runner-mock-fixtures.js"; import { + createMutableSandboxPolicyResult, + gatewayInfoResult, gatewayStatusResult, globalPolicyAuthorityResult, minimalBlueprint, - resultWithBlueprintPolicyAuthority, + sandboxIdentityResult, sandboxPolicyAuthorityResult, + sequentialCommandResult, + TEST_SANDBOX_POLICY, + TEST_SANDBOX_POLICY_PATH, } from "./runner-test-fixtures.js"; const { store } = createRunnerFsStore(); const mockExeca = vi.fn(); const stdoutCapture = createStdoutCapture(); -vi.mock("node:crypto", () => ({ +vi.mock("node:crypto", async (importOriginal) => ({ + ...(await importOriginal()), randomUUID: () => FIXED_RUN_UUID, })); @@ -40,7 +48,10 @@ vi.mock("node:fs", async (importOriginal) => { return { ...original, existsSync: memory.existsSync, + closeSync: memory.closeSync, + fsyncSync: memory.fsyncSync, mkdirSync: memory.mkdirSync, + openSync: memory.openSync, readFileSync: memory.readFileSync, readdirSync: memory.readdirSync, renameSync: memory.renameSync, @@ -62,8 +73,8 @@ vi.mock("./ssrf.js", async (importOriginal) => { const { actionApply, actionReconcile, actionRollback, actionStatus, main } = await import("./runner.js"); -const { renameSync } = await import("node:fs"); -const mockedRenameSync = vi.mocked(renameSync); +const { fsyncSync } = await import("node:fs"); +const mockedFsyncSync = vi.mocked(fsyncSync); const BASE_POLICY = `version: 1 future_policy: @@ -124,13 +135,11 @@ function policySetCalls(): unknown[][] { ); } -function defaultCommandResult(args: string[]) { - return resultWithBlueprintPolicyAuthority(args, { - exitCode: 0, - stdout: "", - stderr: "", - }); -} +const readMergedPolicy = (): Record => { + const merged = [...store.entries()].find(([path]) => path.endsWith("/merged-policy.yaml")); + return YAML.parse(merged?.[1].content ?? TEST_SANDBOX_POLICY); +}; +let defaultCommandResult = createMutableSandboxPolicyResult(readMergedPolicy); function mergedPolicy(): Record { const key = [...store.keys()].find((candidate) => candidate.endsWith("/merged-policy.yaml")); @@ -173,6 +182,9 @@ function blueprint(): Parameters[1] { describe("OpenShell 0.0.72 blueprint policy round-trip", () => { beforeEach(() => { store.clear(); + defaultCommandResult = createMutableSandboxPolicyResult(readMergedPolicy); + store.set(TEST_SANDBOX_POLICY_PATH, { type: "file", content: TEST_SANDBOX_POLICY }); + vi.stubEnv("OPENSHELL_SANDBOX_POLICY", TEST_SANDBOX_POLICY_PATH); stdoutCapture.reset(); mockExeca.mockReset(); vi.spyOn(process.stdout, "write").mockImplementation(stdoutCapture.write); @@ -190,6 +202,7 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); }); it("preserves MCP, JSON-RPC, and unknown mapping sections without provider entries", async () => { @@ -198,7 +211,17 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { expect(mockExeca).toHaveBeenCalledWith( "openshell", ["policy", "get", "-g", "test-gateway", "--base", "test-sandbox"], - expect.objectContaining({ reject: false }), + expect.objectContaining({ maxBuffer: 1024 * 1024, reject: false, timeout: 30_000 }), + ); + expect(mockExeca).toHaveBeenCalledWith( + "openshell", + ["sandbox", "get", "-g", "test-gateway", "test-sandbox"], + expect.objectContaining({ maxBuffer: 1024 * 1024, reject: false, timeout: 30_000 }), + ); + expect(mockExeca).toHaveBeenCalledWith( + "openshell", + ["gateway", "info", "-g", "test-gateway"], + expect.objectContaining({ maxBuffer: 1024 * 1024, reject: false, timeout: 30_000 }), ); expect(mockExeca).not.toHaveBeenCalledWith( "openshell", @@ -231,6 +254,12 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { expected_authority: "nemoclaw-managed", policy_addition_names: ["nim_service"], }, + policy_authority: { + policy_creation_receipt: { + policyHash: "sha256:updated-policy", + policyVersion: 2, + }, + }, }); }); @@ -260,12 +289,9 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { const error = await actionApply("default", blueprint()).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(Error); - expect((error as Error).message).toContain( - "Failed to read current policy before applying additions", - ); - expect((error as Error).message).toContain("MY_API_KEY="); + expect((error as Error).message).toBe("OpenShell policy receipt inspection failed."); + expect((error as Error).message).not.toContain("MY_API_KEY"); expect((error as Error).message).not.toContain("super-secret"); - expect((error as Error).message).toContain("…"); expect((error as Error).message.length).toBeLessThan(600); expect(policySetCalls()).toEqual([]); }); @@ -336,20 +362,25 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { expect(policySetCalls()).toEqual([]); }); - it("uses exact external additions without mutating policy (#9833)", async () => { + it("keeps a verified global policy read-only before blueprint effects (#9833)", async () => { const bp = blueprint(); const additions = bp.components!.policy!.additions!; vi.stubEnv("OPENSHELL_SANDBOX_POLICY", "/tmp/caller-policy.yaml"); mockExeca.mockImplementation(async (_cmd: string, args: string[]) => args.join(" ") === "status" ? gatewayStatusResult("recorded-gateway") - : args.join(" ") === "policy list -g recorded-gateway --global --limit 1" - ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } - : args.join(" ") === "policy get -g recorded-gateway --global --full --output json" - ? globalPolicyAuthorityResult(additions) - : args.join(" ") === "policy get -g recorded-gateway --full --output json test-sandbox" - ? sandboxPolicyAuthorityResult("test-sandbox", "externally-managed", additions) - : { exitCode: 0, stdout: "", stderr: "" }, + : args.join(" ") === "gateway info -g recorded-gateway" + ? gatewayInfoResult() + : args.join(" ") === "policy list -g recorded-gateway --global --limit 1" + ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } + : args.join(" ") === "policy get -g recorded-gateway --global --full --output json" + ? globalPolicyAuthorityResult(additions) + : args.join(" ") === "sandbox get -g recorded-gateway test-sandbox" + ? sandboxIdentityResult("test-sandbox") + : args.join(" ") === + "policy get -g recorded-gateway --full --output json test-sandbox" + ? sandboxPolicyAuthorityResult("test-sandbox", "externally-managed", additions) + : { exitCode: 0, stdout: "", stderr: "" }, ); await actionApply("default", bp); @@ -369,11 +400,13 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { mockExeca.mockImplementation(async (_cmd: string, args: string[]) => args.join(" ") === "status" ? gatewayStatusResult() - : args.join(" ") === "policy list -g test-gateway --global --limit 1" - ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } - : args.join(" ") === "policy get -g test-gateway --global --full --output json" - ? globalPolicyAuthorityResult() - : { exitCode: 0, stdout: "", stderr: "" }, + : args.join(" ") === "gateway info -g test-gateway" + ? gatewayInfoResult() + : args.join(" ") === "policy list -g test-gateway --global --limit 1" + ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } + : args.join(" ") === "policy get -g test-gateway --global --full --output json" + ? globalPolicyAuthorityResult() + : { exitCode: 0, stdout: "", stderr: "" }, ); await expect(actionApply("default", blueprint())).rejects.toThrow( @@ -390,11 +423,13 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { mockExeca.mockImplementation(async (_cmd: string, args: string[]) => args.join(" ") === "status" ? gatewayStatusResult() - : args.join(" ") === "policy list -g test-gateway --global --limit 1" - ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } - : args.join(" ") === "policy get -g test-gateway --global --full --output json" - ? { exitCode: 0, stdout: "{", stderr: "" } - : { exitCode: 0, stdout: "", stderr: "" }, + : args.join(" ") === "gateway info -g test-gateway" + ? gatewayInfoResult() + : args.join(" ") === "policy list -g test-gateway --global --limit 1" + ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } + : args.join(" ") === "policy get -g test-gateway --global --full --output json" + ? { exitCode: 0, stdout: "{", stderr: "" } + : { exitCode: 0, stdout: "", stderr: "" }, ); await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( @@ -407,6 +442,242 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { ).toBe(false); }); + it("fails closed when global metadata does not name its policy source (#9833)", async () => { + const globalResult = globalPolicyAuthorityResult(); + const metadata = JSON.parse(globalResult.stdout) as Record; + delete metadata.policy_source; + const results = new Map([ + ["status", gatewayStatusResult()], + ["gateway info -g test-gateway", gatewayInfoResult()], + [ + "policy list -g test-gateway --global --limit 1", + { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" }, + ], + [ + "policy get -g test-gateway --global --full --output json", + { ...globalResult, stdout: JSON.stringify(metadata) }, + ], + ]); + mockExeca.mockImplementation( + async (_cmd: string, args: string[]) => + results.get(args.join(" ")) ?? { exitCode: 0, stdout: "", stderr: "" }, + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /invalid global policy authority metadata/u, + ); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "sandbox" && call[1][1] === "create", + ), + ).toBe(false); + }); + + it.each([ + [ + "a failed status read", + (args: string[]) => + args.join(" ") === "status" + ? { exitCode: 1, stdout: "", stderr: "gateway unavailable" } + : defaultCommandResult(args), + /Failed to inspect the active OpenShell gateway/u, + ], + [ + "a disconnected status", + (args: string[]) => + args.join(" ") === "status" + ? { exitCode: 0, stdout: "Status: Disconnected\nGateway: test-gateway\n", stderr: "" } + : defaultCommandResult(args), + /Failed to prove the active OpenShell gateway identity/u, + ], + [ + "a missing gateway endpoint", + (args: string[]) => + args.join(" ") === "gateway info -g test-gateway" + ? { exitCode: 0, stdout: "Gateway ready\n", stderr: "" } + : defaultCommandResult(args), + /did not report one gateway endpoint/u, + ], + [ + "an invalid gateway endpoint", + (args: string[]) => + args.join(" ") === "gateway info -g test-gateway" + ? { exitCode: 0, stdout: "Gateway endpoint: not-a-url\n", stderr: "" } + : defaultCommandResult(args), + /reported an invalid gateway endpoint/u, + ], + [ + "an unsupported gateway endpoint protocol", + (args: string[]) => + args.join(" ") === "gateway info -g test-gateway" + ? { exitCode: 0, stdout: "Gateway endpoint: ftp:\/\/127.0.0.1:8080\n", stderr: "" } + : defaultCommandResult(args), + /unsupported gateway endpoint protocol/u, + ], + [ + "an invalid gateway endpoint port", + (args: string[]) => + args.join(" ") === "gateway info -g test-gateway" + ? { exitCode: 0, stdout: "Gateway endpoint: http:\/\/127.0.0.1:0\n", stderr: "" } + : defaultCommandResult(args), + /reported an invalid gateway port/u, + ], + [ + "a non-loopback gateway endpoint", + (args: string[]) => + args.join(" ") === "gateway info -g test-gateway" + ? { exitCode: 0, stdout: "Gateway endpoint: http:\/\/192.0.2.10:8080\n", stderr: "" } + : defaultCommandResult(args), + /reported an unsupported local gateway endpoint/u, + ], + ])("stops before effects for %s (#9833)", async (_caseName, result, expected) => { + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => result(args)); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow(expected); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "sandbox" && call[1][1] === "create", + ), + ).toBe(false); + }); + + it("redacts a thrown gateway receipt inspection before effects (#9833)", async () => { + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args.join(" ") === "gateway info -g test-gateway" + ? Promise.reject(new Error("GATEWAY_TOKEN=must-not-appear")) + : defaultCommandResult(args), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + "OpenShell gateway receipt inspection failed.", + ); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "sandbox" && call[1][1] === "create", + ), + ).toBe(false); + }); + + it.each([ + [ + "throws", + async () => { + throw new Error("POLICY_TOKEN=must-not-appear"); + }, + ], + [ + "returns a failure", + async () => ({ exitCode: 1, stdout: "", stderr: "POLICY_TOKEN=must-not-appear" }), + ], + ])("redacts a global authority command that %s (#9833)", async (_caseName, result) => { + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args.join(" ") === "policy list -g test-gateway --global --limit 1" + ? result() + : defaultCommandResult(args), + ); + + const error = await actionApply("default", minimalBlueprint()).catch( + (caught: unknown) => caught, + ); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toBe( + "OpenShell global policy authority inspection failed. Policy-dependent operations must stop.", + ); + expect((error as Error).message).not.toContain("must-not-appear"); + }); + + it.each([ + ["empty", "", /empty global policy authority metadata/u], + [ + "invalid identity", + JSON.stringify({ + scope: "global", + status: "loaded", + policy_source: "global", + hash: "", + active_version: 0, + policy: {}, + }), + /invalid global policy authority metadata/u, + ], + [ + "non-mapping policy", + JSON.stringify({ + scope: "global", + status: "loaded", + policy_source: "global", + hash: "sha256:global-policy", + active_version: 1, + policy: [], + }), + /invalid global policy authority metadata/u, + ], + ])("fails closed on %s active global metadata (#9833)", async (_caseName, stdout, expected) => { + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args.join(" ") === "policy list -g test-gateway --global --limit 1" + ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } + : args.join(" ") === "policy get -g test-gateway --global --full --output json" + ? { exitCode: 0, stdout, stderr: "" } + : defaultCommandResult(args), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow(expected); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "sandbox" && call[1][1] === "create", + ), + ).toBe(false); + }); + + it("treats a superseded global revision as absent and supplies the managed policy (#9833)", async () => { + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args.join(" ") === "policy list -g test-gateway --global --limit 1" + ? { exitCode: 0, stdout: "VERSION STATUS\n1 superseded\n", stderr: "" } + : args.join(" ") === "policy get -g test-gateway --global --full --output json" + ? { + exitCode: 0, + stdout: JSON.stringify({ + scope: "global", + status: "superseded", + policy_source: "global", + }), + stderr: "", + } + : defaultCommandResult(args), + ); + + await actionApply("default", minimalBlueprint()); + + expect(mockExeca).toHaveBeenCalledWith( + "openshell", + expect.arrayContaining(["sandbox", "create", "--policy", TEST_SANDBOX_POLICY_PATH]), + expect.anything(), + ); + }); + + it.each([ + ["cannot be read", "/tmp/missing-policy.yaml", () => {}, /could not be read/u], + [ + "is invalid", + TEST_SANDBOX_POLICY_PATH, + () => store.set(TEST_SANDBOX_POLICY_PATH, { type: "file", content: "version: [" }), + /is invalid/u, + ], + ])( + "stops before create when the configured policy %s (#9833)", + async (_caseName, path, prepare, expected) => { + vi.stubEnv("OPENSHELL_SANDBOX_POLICY", path); + prepare(); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow(expected); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "sandbox" && call[1][1] === "create", + ), + ).toBe(false); + }, + ); + it("stops before provider and policy mutation when sandbox authority is malformed (#9833)", async () => { mockExeca.mockImplementation(async (_cmd: string, args: string[]) => args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox" @@ -425,366 +696,794 @@ describe("OpenShell 0.0.72 blueprint policy round-trip", () => { expect(policySetCalls()).toEqual([]); }); - it("rechecks authority immediately before a managed policy mutation (#9833)", async () => { - let sandboxAuthorityReads = 0; - mockExeca.mockImplementation(async (_cmd: string, args: string[]) => { - switch (args.join(" ")) { - case "sandbox create --from openclaw --name test-sandbox --forward 18789": - return { exitCode: 1, stdout: "", stderr: "sandbox already exists" }; - case "policy get -g test-gateway --base test-sandbox": - return { - exitCode: 0, - stdout: "Version: 1\nHash: sha256:test\n---\nversion: 1\nnetwork_policies: {}\n", - stderr: "", - }; - case "policy get -g test-gateway --full --output json test-sandbox": - sandboxAuthorityReads += 1; - return sandboxPolicyAuthorityResult( - "test-sandbox", - sandboxAuthorityReads < 3 ? "nemoclaw-managed" : "externally-managed", - ); - default: - return defaultCommandResult(args); - } - }); - - await expect(actionApply("default", blueprint())).rejects.toThrow(/policy authority changed/); - expect(policySetCalls()).toEqual([]); - expect(sandboxAuthorityReads).toBe(3); - }); - - it("records and reports an incomplete reused-sandbox policy transition (#9833)", async () => { - let sandboxAuthorityReads = 0; - mockExeca.mockImplementation(async (_cmd: string, args: string[]) => { - switch (args.join(" ")) { - case "sandbox create --from openclaw --name test-sandbox --forward 18789": - return { exitCode: 1, stdout: "", stderr: "sandbox already exists" }; - case "policy get -g test-gateway --base test-sandbox": - return { - exitCode: 0, - stdout: "Version: 1\nHash: sha256:test\n---\nversion: 1\nnetwork_policies: {}\n", - stderr: "", - }; - case "policy get -g test-gateway --full --output json test-sandbox": - sandboxAuthorityReads += 1; - return sandboxPolicyAuthorityResult( - "test-sandbox", - sandboxAuthorityReads < 4 ? "nemoclaw-managed" : "externally-managed", - ); - default: - return defaultCommandResult(args); - } - }); - - await expect(actionApply("default", blueprint())).rejects.toThrow(/policy authority changed/); - expect(policySetCalls()).toHaveLength(1); - expect(sandboxAuthorityReads).toBe(4); - - const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json")); - expect(planEntry).toBeDefined(); - const plan = JSON.parse(planEntry?.[1].content ?? "{}") as { - run_id: string; - sandbox_created_by_apply: boolean; - policy_transition: Record; + function policyCreationReceipt(overrides: Record = {}) { + return { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "test-gateway", + gatewayPort: 8080, + sandboxName: "test-sandbox", + lifecycleGeneration: FIXED_RUN_UUID, + sandboxIdentityFingerprint: createHash("sha256").update("sandbox-id-1").digest("hex"), + policyHash: "sha256:test-policy", + policyVersion: 1, + ...overrides, }; - expect(plan).toMatchObject({ - sandbox_created_by_apply: false, + } + + function targetPolicy(additions = blueprint().components!.policy!.additions!) { + return { version: 1, network_policies: additions }; + } + + function targetPolicyDigest(policy: Record): string { + const stable = (value: unknown): unknown => + Array.isArray(value) + ? value.map(stable) + : value !== null && typeof value === "object" + ? Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, child]) => [key, stable(child)]), + ) + : value; + return createHash("sha256") + .update(JSON.stringify(stable(policy))) + .digest("hex"); + } + + function managedPolicyAuthority() { + return { + authority: "nemoclaw-managed", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + scope: "sandbox", + sandbox_name: "test-sandbox", + policy_creation_receipt: policyCreationReceipt(), + }; + } + + function validReconciliationPlan() { + const policy = targetPolicy(); + return { + run_id: "reconcile-run", + sandbox_name: "test-sandbox", + sandbox_created_by_apply: true, + policy_additions: blueprint().components!.policy!.additions!, + policy_authority: managedPolicyAuthority(), policy_transition: { status: "incomplete", sandbox_name: "test-sandbox", gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, expected_authority: "nemoclaw-managed", policy_addition_names: ["nim_service"], + target_policy_digest: targetPolicyDigest(policy), }, - }); + }; + } - stdoutCapture.reset(); - actionStatus(plan.run_id); - expect(stdoutCapture.jsonOutput()).toMatchObject({ - policy_transition: { - status: "incomplete", + it("records ownership only for an explicit policy create on the exact gateway (#9833)", async () => { + await actionApply("default", minimalBlueprint()); + + expect(mockExeca).toHaveBeenCalledWith( + "openshell", + [ + "sandbox", + "create", + "-g", + "test-gateway", + "--from", + "openclaw", + "--name", + "test-sandbox", + "--policy", + TEST_SANDBOX_POLICY_PATH, + "--forward", + "18789", + ], + expect.objectContaining({ + reject: false, + env: expect.not.objectContaining({ OPENSHELL_SANDBOX_POLICY: expect.anything() }), + }), + ); + const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json")); + const plan = JSON.parse(planEntry?.[1].content ?? "{}"); + expect(plan).toMatchObject({ + sandbox_created_by_apply: true, + policy_authority: { + authority: "nemoclaw-managed", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, sandbox_name: "test-sandbox", - reconciliation_required: true, - reconciliation_action: expect.stringContaining("Run reconcile"), + policy_creation_receipt: { + origin: "sandbox-create", + gatewayName: "test-gateway", + gatewayPort: 8080, + sandboxName: "test-sandbox", + policyHash: "sha256:test-policy", + policyVersion: 1, + }, }, }); + expect(plan).not.toHaveProperty("policy_creation_transition"); + expect(JSON.stringify(plan.policy_authority)).not.toContain(TEST_SANDBOX_POLICY_PATH); + expect(JSON.stringify(plan.policy_authority)).not.toContain("network_policies"); + }); - await expect(actionRollback(plan.run_id)).rejects.toThrow( - /policy transition for reused sandbox "test-sandbox".*is incomplete.*reconcile/u, - ); - expect(store.has(`${planEntry?.[0].replace(/\/plan\.json$/u, "")}/rolled_back`)).toBe(false); + it("stops before create when no configured policy or verified global policy exists (#9833)", async () => { + vi.stubEnv("OPENSHELL_SANDBOX_POLICY", ""); - mockExeca.mockImplementation(async (_cmd: string, args: string[]) => - args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox" - ? sandboxPolicyAuthorityResult( - "test-sandbox", - "nemoclaw-managed", - blueprint().components!.policy!.additions!, - ) - : defaultCommandResult(args), + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /configured NemoClaw sandbox policy is required/u, ); - stdoutCapture.reset(); - await main(["reconcile", "--run-id", plan.run_id]); - expect(stdoutCapture.text()).toContain(`Policy transition for run ${plan.run_id} is complete.`); - expect(JSON.parse(store.get(planEntry?.[0] ?? "")?.content ?? "{}")).toMatchObject({ - policy_transition: { status: "complete" }, - }); - - await actionRollback(plan.run_id); - expect(store.has(`${planEntry?.[0].replace(/\/plan\.json$/u, "")}/rolled_back`)).toBe(true); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "sandbox" && call[1][1] === "create", + ), + ).toBe(false); }); - it("keeps a pending receipt when a reused-sandbox policy set fails (#9833)", async () => { - const diagnostic = `POLICY_TOKEN=super-secret ${"policy details ".repeat(80)}`; - const responses = new Map([ - [ - "sandbox create --from openclaw --name test-sandbox --forward 18789", - { exitCode: 1, stdout: "", stderr: "sandbox already exists" }, - ], - [ - "policy get -g test-gateway --base test-sandbox", - { - exitCode: 0, - stdout: policyOutput("version: 1\nnetwork_policies: {}\n"), - stderr: "", - }, - ], - ["policy set", { exitCode: 1, stdout: "", stderr: diagnostic }], - ]); - mockExeca.mockImplementation( - async (_cmd: string, args: string[]) => - responses.get(args.join(" ")) ?? - responses.get(args.slice(0, 2).join(" ")) ?? - defaultCommandResult(args), + it("does not mint ownership when sandbox create reports already exists (#9833)", async () => { + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args[0] === "sandbox" && args[1] === "create" + ? { exitCode: 1, stdout: "", stderr: "sandbox already exists" } + : defaultCommandResult(args), ); - const error = await actionApply("default", blueprint()).catch((caught: unknown) => caught); - expect(error).toBeInstanceOf(Error); - expect((error as Error).message).toContain("Failed to apply policy additions"); - expect((error as Error).message).toContain("POLICY_TOKEN="); - expect((error as Error).message).not.toContain("super-secret"); - expect((error as Error).message).toContain("…"); - expect((error as Error).message.length).toBeLessThan(600); - const planEntry = [...store.entries()].find(([, entry]) => - entry.content?.includes('"policy_transition"'), + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /already exists.*cannot establish NemoClaw policy ownership/u, ); - const plan = JSON.parse(planEntry?.[1].content ?? "{}") as { run_id: string }; + const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json")); + const plan = JSON.parse(planEntry?.[1].content ?? "{}"); expect(plan).toMatchObject({ sandbox_created_by_apply: false, - policy_transition: { - status: "pending", - sandbox_name: "test-sandbox", + policy_creation_transition: { + status: "incomplete", gateway: "test-gateway", - policy_addition_names: ["nim_service"], + gateway_host: "127.0.0.1", + gateway_port: 8080, + sandbox_name: "test-sandbox", }, }); + expect(plan).not.toHaveProperty("policy_authority"); + expect(policySetCalls()).toEqual([]); + expect( + mockExeca.mock.calls.some((call) => Array.isArray(call[1]) && call[1][0] === "provider"), + ).toBe(false); + }); + it("refuses conflicting sandbox identity fields before provider effects (#9833)", async () => { mockExeca.mockImplementation(async (_cmd: string, args: string[]) => - args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox" - ? sandboxPolicyAuthorityResult( - "test-sandbox", - "nemoclaw-managed", - blueprint().components!.policy!.additions!, - ) + args.join(" ") === "sandbox get -g test-gateway test-sandbox" + ? { + exitCode: 0, + stdout: "Name: test-sandbox\nId: sandbox-id\nId: replacement-id\nPhase: Ready\n", + stderr: "", + } : defaultCommandResult(args), ); - await actionReconcile(plan.run_id); - expect(JSON.parse(store.get(planEntry?.[0] ?? "")?.content ?? "{}")).toMatchObject({ - policy_transition: { status: "complete" }, - }); - }); - it.each([ - ["authority change", "externally-managed" as const, blueprint().components!.policy!.additions!], - ["missing addition", "nemoclaw-managed" as const, {}], - ])( - "retains an incomplete receipt after a reconciliation %s (#9833)", - async (_case, authority, observed) => { - const runId = "incomplete-transition"; - const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; - const additions = blueprint().components!.policy!.additions!; - store.set(stateDir, { type: "dir" }); - store.set(`${stateDir}/plan.json`, { - type: "file", - content: JSON.stringify({ - run_id: runId, - sandbox_name: "test-sandbox", - sandbox_created_by_apply: false, - policy_additions: additions, - policy_transition: { - status: "incomplete", - sandbox_name: "test-sandbox", - gateway: "recorded-gateway", - expected_authority: "nemoclaw-managed", - policy_addition_names: ["nim_service"], - }, - }), - }); - mockExeca.mockImplementation(async (_cmd: string, args: string[]) => - args.join(" ") === "policy get -g recorded-gateway --full --output json test-sandbox" - ? sandboxPolicyAuthorityResult("test-sandbox", authority, observed) - : defaultCommandResult(args), - ); + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /did not prove the immutable identity/u, + ); + expect( + mockExeca.mock.calls.some((call) => Array.isArray(call[1]) && call[1][0] === "provider"), + ).toBe(false); + }); - await expect(actionReconcile(runId)).rejects.toThrow(/Cannot reconcile/u); - expect(JSON.parse(store.get(`${stateDir}/plan.json`)?.content ?? "{}")).toMatchObject({ - policy_transition: { status: "incomplete" }, - }); - await expect(actionRollback(runId)).rejects.toThrow(/is incomplete/u); - }, - ); + it("stops before provider effects when the live sandbox identity changes (#9833)", async () => { + const identityResult = sequentialCommandResult("sandbox get -g test-gateway test-sandbox", [ + sandboxIdentityResult("test-sandbox"), + sandboxIdentityResult("test-sandbox", "replacement-id"), + ]); + mockExeca.mockImplementation( + async (_cmd: string, args: string[]) => identityResult(args) ?? defaultCommandResult(args), + ); - it("rejects an invalid persisted policy transition (#9833)", async () => { - const runId = "invalid-transition"; - const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; - store.set(stateDir, { type: "dir" }); - store.set(`${stateDir}/plan.json`, { - type: "file", - content: JSON.stringify({ - run_id: runId, - sandbox_name: "test-sandbox", - sandbox_created_by_apply: false, - policy_transition: { - status: "unknown", - sandbox_name: "test-sandbox", - gateway: "test-gateway", - expected_authority: "nemoclaw-managed", - policy_addition_names: ["nim_service"], - }, - }), + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /receipt does not match the live sandbox policy/u, + ); + expect( + mockExeca.mock.calls.some((call) => Array.isArray(call[1]) && call[1][0] === "provider"), + ).toBe(false); + const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json")); + expect(JSON.parse(planEntry?.[1].content ?? "{}")).toMatchObject({ + sandbox_created_by_apply: true, + policy_authority: { authority: "nemoclaw-managed" }, }); + }); - stdoutCapture.reset(); - actionStatus(runId); - expect(stdoutCapture.jsonOutput()).toMatchObject({ - run_id: runId, - status: "unknown", - receipt_error_kind: "invalid", - run_directory: stateDir, - recovery: expect.stringContaining("Do not reconstruct plan.json"), + it("stops before provider effects when the gateway binding changes (#9833)", async () => { + const gatewayResult = sequentialCommandResult("gateway info -g test-gateway", [ + gatewayInfoResult(), + gatewayInfoResult(), + gatewayInfoResult(9090), + ]); + mockExeca.mockImplementation( + async (_cmd: string, args: string[]) => gatewayResult(args) ?? defaultCommandResult(args), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /gateway endpoint no longer matches the durable policy receipt/u, + ); + expect( + mockExeca.mock.calls.some((call) => Array.isArray(call[1]) && call[1][0] === "provider"), + ).toBe(false); + }); + + it("rejects a non-loopback gateway substitution on the receipt port (#9833)", async () => { + const gatewayResult = sequentialCommandResult("gateway info -g test-gateway", [ + gatewayInfoResult(), + gatewayInfoResult(), + gatewayInfoResult(8080, "gateway.example.com"), + ]); + mockExeca.mockImplementation( + async (_cmd: string, args: string[]) => gatewayResult(args) ?? defaultCommandResult(args), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /unsupported local gateway endpoint/u, + ); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "provider" && call[1][1] === "create", + ), + ).toBe(false); + }); + + it("stops before provider effects when the live policy identity changes (#9833)", async () => { + const policyResult = sequentialCommandResult( + "policy get -g test-gateway --full --output json test-sandbox", + [ + sandboxPolicyAuthorityResult( + "test-sandbox", + "nemoclaw-managed", + {}, + { version: 1, network_policies: {} }, + ), + sandboxPolicyAuthorityResult( + "test-sandbox", + "nemoclaw-managed", + {}, + { version: 1, network_policies: {} }, + "sha256:replacement-policy", + ), + ], + ); + mockExeca.mockImplementation( + async (_cmd: string, args: string[]) => policyResult(args) ?? defaultCommandResult(args), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /receipt does not match the live sandbox policy/u, + ); + expect( + mockExeca.mock.calls.some((call) => Array.isArray(call[1]) && call[1][0] === "provider"), + ).toBe(false); + }); + + it("rejects a created sandbox whose effective policy was not supplied by NemoClaw (#9833)", async () => { + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox" + ? sandboxPolicyAuthorityResult("test-sandbox", "externally-managed") + : defaultCommandResult(args), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /did not prove the exact policy supplied by this NemoClaw create transaction/u, + ); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "provider" && call[1][1] === "create", + ), + ).toBe(false); + }); + + it("rejects managed receipt reuse after the policy becomes global (#9833)", async () => { + const policyResult = sequentialCommandResult( + "policy get -g test-gateway --full --output json test-sandbox", + [ + sandboxPolicyAuthorityResult("test-sandbox"), + sandboxPolicyAuthorityResult("test-sandbox", "externally-managed"), + ], + ); + mockExeca.mockImplementation( + async (_cmd: string, args: string[]) => policyResult(args) ?? defaultCommandResult(args), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /live sandbox policy is no longer sandbox-scoped/u, + ); + expect( + mockExeca.mock.calls.some( + (call) => Array.isArray(call[1]) && call[1][0] === "provider" && call[1][1] === "create", + ), + ).toBe(false); + }); + + it("rejects a verified external boundary after its gateway binding changes (#9833)", async () => { + const additions = blueprint().components!.policy!.additions!; + const gatewayResult = sequentialCommandResult("gateway info -g test-gateway", [ + gatewayInfoResult(), + gatewayInfoResult(), + gatewayInfoResult(9090), + ]); + mockExeca.mockImplementation( + async (_cmd: string, args: string[]) => + gatewayResult(args) ?? + (args.join(" ") === "policy list -g test-gateway --global --limit 1" + ? { exitCode: 0, stdout: "VERSION STATUS\n1 loaded\n", stderr: "" } + : args.join(" ") === "policy get -g test-gateway --global --full --output json" + ? globalPolicyAuthorityResult(additions) + : args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox" + ? sandboxPolicyAuthorityResult("test-sandbox", "externally-managed", additions) + : defaultCommandResult(args)), + ); + + await expect(actionApply("default", blueprint())).rejects.toThrow( + /verified external policy boundary no longer matches/u, + ); + expect(policySetCalls()).toEqual([]); + }); + + it("reports incomplete creation when receipt directory durability fails (#9833)", async () => { + mockedFsyncSync.mockImplementation( + throwOnCall(6, new Error("simulated receipt directory fsync failure")), + ); + + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /simulated receipt directory fsync failure/u, + ); + expect(stdoutCapture.text()).not.toContain("Apply complete"); + const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json")); + const plan = JSON.parse(planEntry?.[1].content ?? "{}"); + expect(plan).toMatchObject({ + sandbox_created_by_apply: true, + policy_creation_transition: { status: "incomplete" }, }); - await expect(actionReconcile(runId)).rejects.toThrow(/policy transition receipt is invalid/u); - await expect(actionRollback(runId)).rejects.toThrow(/policy transition receipt is invalid/u); - expect(mockExeca).not.toHaveBeenCalled(); + expect(plan).not.toHaveProperty("policy_authority"); }); - const reconciliationPlan = { - run_id: "invalid-reconciliation", - sandbox_name: "test-sandbox", - sandbox_created_by_apply: false, - policy_additions: blueprint().components!.policy!.additions!, - policy_transition: { - status: "incomplete", - sandbox_name: "test-sandbox", - gateway: "test-gateway", - expected_authority: "nemoclaw-managed", - policy_addition_names: ["nim_service"], - }, - }; + it("reports incomplete mutation when receipt rotation durability fails (#9833)", async () => { + mockedFsyncSync.mockImplementation( + throwOnCall(14, new Error("simulated rotated receipt directory fsync failure")), + ); + + await expect(actionApply("default", blueprint())).rejects.toThrow( + /simulated rotated receipt directory fsync failure/u, + ); + expect(stdoutCapture.text()).not.toContain("Apply complete"); + const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json")); + expect(JSON.parse(planEntry?.[1].content ?? "{}")).toMatchObject({ + policy_authority: { + policy_creation_receipt: { + policyHash: "sha256:updated-policy", + policyVersion: 2, + }, + }, + policy_transition: { status: "incomplete" }, + }); + }); + + it("keeps the policy transition incomplete when policy set fails (#9833)", async () => { + const diagnostic = `POLICY_TOKEN=super-secret ${"policy details ".repeat(80)}`; + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args[0] === "policy" && args[1] === "set" + ? { exitCode: 1, stdout: "", stderr: diagnostic } + : args.join(" ") === "policy get -g test-gateway --base test-sandbox" + ? { exitCode: 0, stdout: policyOutput(TEST_SANDBOX_POLICY), stderr: "" } + : defaultCommandResult(args), + ); + + const error = await actionApply("default", blueprint()).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("Failed to apply policy additions"); + expect((error as Error).message).toContain("POLICY_TOKEN="); + expect((error as Error).message).not.toContain("super-secret"); + const planEntry = [...store.entries()].find(([path]) => path.endsWith("/plan.json")); + const plan = JSON.parse(planEntry?.[1].content ?? "{}"); + expect(plan).toMatchObject({ + policy_transition: { + status: "pending", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + policy_addition_names: ["nim_service"], + }, + }); + }); it.each([ - ["a non-object body", [], /plan\.json must contain a JSON object/u], + ["a non-object plan", []], + ["a missing authority receipt", { ...validReconciliationPlan(), policy_authority: undefined }], [ - "an apply-owned sandbox", - { ...reconciliationPlan, sandbox_created_by_apply: true }, - /requires a reused sandbox/u, + "an external authority receipt", + { + ...validReconciliationPlan(), + policy_authority: { + authority: "externally-managed", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + scope: "global", + }, + }, ], [ - "a mismatched sandbox", - { ...reconciliationPlan, sandbox_name: "other-sandbox" }, - /sandbox does not match/u, + "an invalid transition receipt", + { ...validReconciliationPlan(), policy_transition: undefined }, ], [ - "invalid policy additions", - { ...reconciliationPlan, policy_additions: [] }, - /policy additions are invalid/u, + "a transition for another sandbox", + { + ...validReconciliationPlan(), + policy_transition: { + ...validReconciliationPlan().policy_transition, + sandbox_name: "replacement-sandbox", + }, + }, ], [ - "mismatched policy addition names", + "a transition for another gateway", { - ...reconciliationPlan, + ...validReconciliationPlan(), policy_transition: { - ...reconciliationPlan.policy_transition, - policy_addition_names: ["other"], + ...validReconciliationPlan().policy_transition, + gateway: "replacement-gateway", }, }, - /additions do not match/u, ], - ])("rejects a reconciliation plan with %s (#9833)", async (_case, plan, expected) => { - const runId = "invalid-reconciliation"; + ["invalid policy additions", { ...validReconciliationPlan(), policy_additions: [] }], + ["mismatched policy additions", { ...validReconciliationPlan(), policy_additions: {} }], + [ + "a policy target with a mismatched digest", + { + ...validReconciliationPlan(), + policy_transition: { + ...validReconciliationPlan().policy_transition, + target_policy_digest: "a".repeat(64), + }, + }, + ], + ])("refuses reconciliation from %s (#9833)", async (caseName, plan) => { + const runId = `invalid-reconciliation-${caseName.replaceAll(" ", "-")}`; const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; store.set(stateDir, { type: "dir" }); store.set(`${stateDir}/plan.json`, { type: "file", content: JSON.stringify(plan) }); + store.set(`${stateDir}/merged-policy.yaml`, { + type: "file", + content: YAML.stringify(targetPolicy()), + }); - await expect(actionReconcile(runId)).rejects.toThrow(expected); - }); - - it("rejects reconciliation for a missing run (#9833)", async () => { - await expect(actionReconcile("missing-reconciliation")).rejects.toThrow(/not found/u); + await expect(actionReconcile(runId)).rejects.toThrow( + new RegExp(`Cannot read reconciliation plan for run ${runId}`), + ); + expect(mockExeca).not.toHaveBeenCalled(); }); - it("preserves the previous receipt when reconciliation replacement is interrupted (#9833)", async () => { - const runId = "interrupted-reconciliation"; + it("reconciles only the exact intended policy and rotates the receipt (#9833)", async () => { + const runId = "incomplete-transition"; const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; - const planFile = `${stateDir}/plan.json`; + const policy = targetPolicy(); store.set(stateDir, { type: "dir" }); - store.set(planFile, { + store.set(`${stateDir}/merged-policy.yaml`, { type: "file", - content: JSON.stringify({ ...reconciliationPlan, run_id: runId }), + content: YAML.stringify(policy), + }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + sandbox_name: "test-sandbox", + sandbox_created_by_apply: true, + inference_provider_created_by_apply: false, + policy_additions: blueprint().components!.policy!.additions!, + policy_authority: managedPolicyAuthority(), + policy_transition: { + status: "incomplete", + sandbox_name: "test-sandbox", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + expected_authority: "nemoclaw-managed", + policy_addition_names: ["nim_service"], + target_policy_digest: targetPolicyDigest(policy), + }, + }), }); mockExeca.mockImplementation(async (_cmd: string, args: string[]) => args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox" ? sandboxPolicyAuthorityResult( "test-sandbox", "nemoclaw-managed", - blueprint().components!.policy!.additions!, + policy.network_policies, + policy, ) : defaultCommandResult(args), ); - mockedRenameSync.mockImplementationOnce(() => { - throw new Error("simulated interrupted receipt replacement"); + + await main(["reconcile", "--run-id", runId]); + expect(JSON.parse(store.get(`${stateDir}/plan.json`)?.content ?? "{}")).toMatchObject({ + policy_authority: { + policy_creation_receipt: { + policyHash: "sha256:test-policy", + policyVersion: 1, + }, + }, + policy_transition: { status: "complete" }, + }); + }); + + it("refuses reconcile when the durable target does not match live policy (#9833)", async () => { + const runId = "mismatched-transition"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + const policy = targetPolicy(); + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/merged-policy.yaml`, { + type: "file", + content: YAML.stringify(policy), + }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + sandbox_name: "test-sandbox", + sandbox_created_by_apply: true, + policy_additions: blueprint().components!.policy!.additions!, + policy_authority: managedPolicyAuthority(), + policy_transition: { + status: "incomplete", + sandbox_name: "test-sandbox", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + expected_authority: "nemoclaw-managed", + policy_addition_names: ["nim_service"], + target_policy_digest: targetPolicyDigest(policy), + }, + }), }); - await expect(actionReconcile(runId)).rejects.toThrow(/interrupted receipt replacement/u); - expect(JSON.parse(store.get(planFile)?.content ?? "{}")).toMatchObject({ + await expect(actionReconcile(runId)).rejects.toThrow(/exact intended policy/u); + expect(JSON.parse(store.get(`${stateDir}/plan.json`)?.content ?? "{}")).toMatchObject({ policy_transition: { status: "incomplete" }, }); + }); + + it("rejects an invalid persisted policy creation transition (#9833)", async () => { + const runId = "invalid-creation"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + sandbox_name: "test-sandbox", + policy_creation_transition: { + status: "complete", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + sandbox_name: "test-sandbox", + lifecycle_generation: FIXED_RUN_UUID, + }, + }), + }); stdoutCapture.reset(); actionStatus(runId); expect(stdoutCapture.jsonOutput()).toMatchObject({ - policy_transition: { status: "incomplete", reconciliation_required: true }, + run_id: runId, + status: "unknown", + receipt_error_kind: "invalid", }); - await expect(actionRollback(runId)).rejects.toThrow(/is incomplete.*reconcile/u); + await expect(actionRollback(runId)).rejects.toThrow(/policy creation transition is invalid/u); + expect(mockExeca).not.toHaveBeenCalled(); + }); - await actionReconcile(runId); - expect(JSON.parse(store.get(planFile)?.content ?? "{}")).toMatchObject({ - policy_transition: { status: "complete" }, + it("reports a non-object run receipt as invalid (#9833)", () => { + const runId = "non-object-receipt"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { type: "file", content: "[]" }); + + stdoutCapture.reset(); + actionStatus(runId); + + expect(stdoutCapture.jsonOutput()).toMatchObject({ + run_id: runId, + status: "unknown", + receipt_error_kind: "invalid", + }); + }); + + it("reports a pending policy creation checkpoint without granting authority (#9833)", () => { + const runId = "pending-policy-creation"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + policy_creation_transition: { + status: "pending", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + sandbox_name: "test-sandbox", + lifecycle_generation: FIXED_RUN_UUID, + }, + }), + }); + + stdoutCapture.reset(); + actionStatus(runId); + + expect(stdoutCapture.jsonOutput()).toMatchObject({ + run_id: runId, + policy_creation_transition: { status: "pending" }, + }); + expect(stdoutCapture.jsonOutput()).not.toHaveProperty("policy_authority"); + }); + + it("reports an invalid policy transition receipt without exposing it (#9833)", () => { + const runId = "invalid-policy-transition"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + policy_transition: { status: "pending" }, + }), + }); + + stdoutCapture.reset(); + actionStatus(runId); + + expect(stdoutCapture.jsonOutput()).toMatchObject({ + run_id: runId, + status: "unknown", + receipt_error_kind: "invalid", + }); + }); + + it("reports the exact recovery action for an incomplete policy transition (#9833)", () => { + const runId = "incomplete-policy-transition"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ ...validReconciliationPlan(), run_id: runId }), + }); + + stdoutCapture.reset(); + actionStatus(runId); + + expect(stdoutCapture.jsonOutput()).toMatchObject({ + run_id: runId, + policy_transition: { + status: "incomplete", + reconciliation_required: true, + reconciliation_action: expect.stringContaining("reconcile"), + }, + }); + }); + + it("rejects a malformed managed receipt before rollback mutation (#9833)", async () => { + const runId = "malformed-receipt"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + sandbox_name: "test-sandbox", + sandbox_created_by_apply: true, + policy_authority: { + ...managedPolicyAuthority(), + policy_creation_receipt: { status: "pending" }, + }, + }), }); + + await expect(actionRollback(runId)).rejects.toThrow(/policy authority receipt is invalid/u); + expect(mockExeca).not.toHaveBeenCalled(); }); - it("treats a complete reconciliation as idempotent and requires a CLI run ID (#9833)", async () => { - const runId = "complete-reconciliation"; + it("rejects a receipt whose wrapper names a different sandbox (#9833)", async () => { + const runId = "mismatched-wrapper"; const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; store.set(stateDir, { type: "dir" }); store.set(`${stateDir}/plan.json`, { type: "file", content: JSON.stringify({ - ...reconciliationPlan, run_id: runId, + sandbox_name: "replacement-sandbox", + sandbox_created_by_apply: true, + policy_authority: { + ...managedPolicyAuthority(), + sandbox_name: "replacement-sandbox", + }, + }), + }); + + await expect(actionRollback(runId)).rejects.toThrow(/policy authority receipt is invalid/u); + expect(mockExeca).not.toHaveBeenCalled(); + }); + + it("does not report an unexpected receipt field through status (#9833)", () => { + const runId = "extended-receipt"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + policy_authority: { + ...managedPolicyAuthority(), + credential_value: "must-not-appear", + }, + }), + }); + + stdoutCapture.reset(); + actionStatus(runId); + expect(stdoutCapture.jsonOutput()).toMatchObject({ + run_id: runId, + status: "unknown", + receipt_error_kind: "invalid", + }); + expect(stdoutCapture.text()).not.toContain("must-not-appear"); + }); + + it("revalidates a complete transition before reporting it complete (#9833)", async () => { + const runId = "complete-transition"; + const stateDir = `${FAKE_HOME}/.nemoclaw/state/runs/${runId}`; + const policy = targetPolicy(); + store.set(stateDir, { type: "dir" }); + store.set(`${stateDir}/merged-policy.yaml`, { + type: "file", + content: YAML.stringify(policy), + }); + store.set(`${stateDir}/plan.json`, { + type: "file", + content: JSON.stringify({ + run_id: runId, + sandbox_name: "test-sandbox", + sandbox_created_by_apply: true, + policy_additions: blueprint().components!.policy!.additions!, + policy_authority: managedPolicyAuthority(), policy_transition: { - ...reconciliationPlan.policy_transition, status: "complete", + sandbox_name: "test-sandbox", + gateway: "test-gateway", + gateway_host: "127.0.0.1", + gateway_port: 8080, + expected_authority: "nemoclaw-managed", + policy_addition_names: ["nim_service"], + target_policy_digest: targetPolicyDigest(policy), }, }), }); + mockExeca.mockImplementation(async (_cmd: string, args: string[]) => + args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox" + ? sandboxPolicyAuthorityResult( + "test-sandbox", + "nemoclaw-managed", + policy.network_policies, + policy, + ) + : defaultCommandResult(args), + ); await actionReconcile(runId); - expect(stdoutCapture.text()).toContain( `Policy transition for run ${runId} is already complete.`, ); - expect(mockExeca).not.toHaveBeenCalled(); await expect(main(["reconcile"])).rejects.toThrow(/--run-id is required/u); }); }); diff --git a/nemoclaw/src/blueprint/runner-test-fixtures.ts b/nemoclaw/src/blueprint/runner-test-fixtures.ts index 1c577538096..490d7de1a99 100644 --- a/nemoclaw/src/blueprint/runner-test-fixtures.ts +++ b/nemoclaw/src/blueprint/runner-test-fixtures.ts @@ -99,7 +99,11 @@ export function successResult(): { return { exitCode: 0, stdout: "", stderr: "" }; } -type CommandResult = { exitCode: number; stdout: string; stderr: string }; +export type CommandResult = { exitCode: number; stdout: string; stderr: string }; + +/** The configured policy used by blueprint runner tests that create a sandbox. */ +export const TEST_SANDBOX_POLICY_PATH = "/tmp/nemoclaw-test-policy.yaml"; +export const TEST_SANDBOX_POLICY = "version: 1\nnetwork_policies: {}\n"; /** The connected gateway identity reported by `openshell status`. */ export function gatewayStatusResult(gateway = "test-gateway"): CommandResult { @@ -110,11 +114,36 @@ export function gatewayStatusResult(gateway = "test-gateway"): CommandResult { }; } +/** The stable endpoint binding reported by `openshell gateway info`. */ +export function gatewayInfoResult(port = 8080, host = "127.0.0.1"): CommandResult { + return { + exitCode: 0, + stdout: `Gateway endpoint: http://${host}:${port}\n`, + stderr: "", + }; +} + +/** The immutable identity and lifecycle state reported for one sandbox. */ +export function sandboxIdentityResult( + sandboxName: string, + id = "sandbox-id-1", + phase = "Ready", +): CommandResult { + return { + exitCode: 0, + stdout: `Name: ${sandboxName}\nId: ${id}\nPhase: ${phase}\n`, + stderr: "", + }; +} + /** Machine-readable effective policy metadata for one sandbox. */ export function sandboxPolicyAuthorityResult( sandboxName: string, authority: "nemoclaw-managed" | "externally-managed" = "nemoclaw-managed", networkPolicies: Record = {}, + effectivePolicy: Record = { version: 1, network_policies: networkPolicies }, + policyHash = "sha256:test-policy", + policyVersion = 1, ): CommandResult { return { exitCode: 0, @@ -123,7 +152,9 @@ export function sandboxPolicyAuthorityResult( sandbox: sandboxName, status: "effective", policy_source: authority === "nemoclaw-managed" ? "sandbox" : "global", - policy: { version: 1, network_policies: networkPolicies }, + hash: policyHash, + active_version: policyVersion, + policy: effectivePolicy, }), stderr: "", }; @@ -139,6 +170,8 @@ export function globalPolicyAuthorityResult( scope: "global", status: "loaded", policy_source: "global", + hash: "sha256:global-policy", + active_version: 1, policy: { version: 1, network_policies: networkPolicies }, }), stderr: "", @@ -153,18 +186,81 @@ export function resultWithBlueprintPolicyAuthority( ): CommandResult { return args.join(" ") === "status" ? gatewayStatusResult(gateway) - : args.join(" ") === `policy list -g ${gateway} --global --limit 1` - ? successResult() - : args[0] === "policy" && - args[1] === "get" && - args[2] === "-g" && - args[3] === gateway && - args[4] === "--full" && - args[5] === "--output" && - args[6] === "json" && - typeof args[7] === "string" - ? sandboxPolicyAuthorityResult(args[7]) - : fallback; + : args.join(" ") === `gateway info -g ${gateway}` + ? gatewayInfoResult() + : args.join(" ") === `policy list -g ${gateway} --global --limit 1` + ? successResult() + : args[0] === "policy" && + args[1] === "get" && + args[2] === "-g" && + args[3] === gateway && + args[4] === "--full" && + args[5] === "--output" && + args[6] === "json" && + typeof args[7] === "string" + ? sandboxPolicyAuthorityResult(args[7]) + : args[0] === "sandbox" && + args[1] === "get" && + args[2] === "-g" && + args[3] === gateway && + typeof args[4] === "string" + ? sandboxIdentityResult(args[4]) + : fallback; +} + +/** Returns configured results for successive calls to one exact command. */ +export function sequentialCommandResult( + command: string, + results: readonly CommandResult[], +): (args: readonly string[]) => CommandResult | undefined { + let callCount = 0; + return (args) => { + if (args.join(" ") !== command) return undefined; + const result = results[Math.min(callCount, results.length - 1)]; + callCount += 1; + return result; + }; +} + +/** Tracks the effective sandbox policy after a blueprint policy mutation. */ +export function createMutableSandboxPolicyResult( + readMergedPolicy: () => Record, +): (args: readonly string[]) => CommandResult { + let livePolicy: Record = { version: 1, network_policies: {} }; + let livePolicyHash = "sha256:test-policy"; + let livePolicyVersion = 1; + return (args) => { + if (args[0] === "policy" && args[1] === "set") { + livePolicy = readMergedPolicy(); + livePolicyHash = "sha256:updated-policy"; + livePolicyVersion = 2; + return successResult(); + } + if (args.join(" ") === "policy get -g test-gateway --full --output json test-sandbox") { + return sandboxPolicyAuthorityResult( + "test-sandbox", + "nemoclaw-managed", + (livePolicy.network_policies as Record | undefined) ?? {}, + livePolicy, + livePolicyHash, + livePolicyVersion, + ); + } + if (args[0] === "provider" && args[1] === "get" && typeof args[2] === "string") { + return { + exitCode: 0, + stdout: [ + `Name: ${args[2]}`, + "Type: openai", + "Credential keys: OPENAI_API_KEY", + "Config keys: OPENAI_BASE_URL", + "", + ].join("\n"), + stderr: "", + }; + } + return resultWithBlueprintPolicyAuthority(args, successResult()); + }; } /** A failed command result carrying only stderr. */ diff --git a/nemoclaw/src/blueprint/runner.test.ts b/nemoclaw/src/blueprint/runner.test.ts index 685066a7154..3cd970d8439 100644 --- a/nemoclaw/src/blueprint/runner.test.ts +++ b/nemoclaw/src/blueprint/runner.test.ts @@ -18,6 +18,8 @@ import { resultForCommandFailure, resultWithBlueprintPolicyAuthority, routedBlueprint, + TEST_SANDBOX_POLICY, + TEST_SANDBOX_POLICY_PATH, } from "./runner-test-fixtures.js"; // ── In-memory filesystem ──────────────────────────────────────── @@ -39,7 +41,10 @@ vi.mock("node:fs", async (importOriginal) => { return { ...original, existsSync: memory.existsSync, + closeSync: memory.closeSync, + fsyncSync: memory.fsyncSync, mkdirSync: memory.mkdirSync, + openSync: memory.openSync, readFileSync: vi.fn(memory.readFileSync), renameSync: memory.renameSync, writeFileSync: memory.writeFileSync, @@ -100,6 +105,8 @@ function mockCurrentPolicy(stdout: string): void { describe("runner", () => { beforeEach(() => { store.clear(); + addFile(TEST_SANDBOX_POLICY_PATH, TEST_SANDBOX_POLICY); + vi.stubEnv("OPENSHELL_SANDBOX_POLICY", TEST_SANDBOX_POLICY_PATH); stdoutCapture.reset(); vi.clearAllMocks(); delete process.env.NEMOCLAW_BLUEPRINT_PATH; @@ -551,7 +558,20 @@ describe("runner", () => { expect(mockExeca).toHaveBeenCalledWith( "openshell", - ["sandbox", "create", "--from", "openclaw", "--name", "test-sandbox", "--forward", "18789"], + [ + "sandbox", + "create", + "-g", + "test-gateway", + "--from", + "openclaw", + "--name", + "test-sandbox", + "--policy", + TEST_SANDBOX_POLICY_PATH, + "--forward", + "18789", + ], expect.objectContaining({ reject: false }), ); }); @@ -608,113 +628,35 @@ describe("runner", () => { }); it("compensates an owned inference provider when inference set fails (#6703)", async () => { - mockExeca.mockImplementation(async (_cmd: string, args: string[]) => - resultForCommandFailure(args, ["inference", "set"], "inference route rejected"), - ); - - await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( - /Failed to set inference route .*model 'gpt-4'.*inference route rejected/i, - ); - - expect(hasPlanJson()).toBe(true); - expect(mockExeca).toHaveBeenCalledWith( - "openshell", - ["provider", "delete", "my-provider"], - expect.objectContaining({ reject: false }), - ); - expect(stdoutText()).not.toContain("Apply complete"); - expect(stdoutText()).not.toContain("PROGRESS:100"); - }); - - it("applies blueprint policy additions by merging into the base policy", async () => { - const bp = minimalBlueprint({ - components: { - inference: { - profiles: { - default: { - provider_type: "openai", - provider_name: "my-provider", - endpoint: "https://api.example.com/v1", - model: "gpt-4", - credential_env: "MY_API_KEY", - }, - }, - }, - sandbox: { - image: "openclaw", - name: "test-sandbox", - forward_ports: [18789], - }, - policy: { - additions: { - nim_service: { - name: "nim_service", - endpoints: [ - { - host: "integrate.api.nvidia.com", - port: 443, - access: "full", - }, - ], - }, - }, - }, - }, - }); mockExeca.mockImplementation(async (_cmd: string, args: string[]) => { - if (args.join(" ") === "policy get -g test-gateway --base test-sandbox") { + if (args.join(" ") === "provider get my-provider") { return { exitCode: 0, stdout: [ - "Version: 1", - "Hash: sha256:test", - "---", - "version: 1", - "network_policies:", - " existing_service:", - " mode: allow", - " endpoints:", - " - https://api.example.com", + "Name: my-provider", + "Type: openai", + "Credential keys: ", + "Config keys: OPENAI_BASE_URL", "", ].join("\n"), stderr: "", }; } - return resultWithBlueprintPolicyAuthority(args, { - exitCode: 0, - stdout: "", - stderr: "", - }); + return resultForCommandFailure(args, ["inference", "set"], "inference route rejected"); }); - await actionApply("default", bp); + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /Failed to set inference route .*model 'gpt-4'.*inference route rejected/i, + ); + expect(hasPlanJson()).toBe(true); expect(mockExeca).toHaveBeenCalledWith( "openshell", - [ - "policy", - "set", - "-g", - "test-gateway", - "--policy", - expect.stringContaining("merged-policy.yaml"), - "--wait", - "test-sandbox", - ], + ["provider", "delete", "my-provider"], expect.objectContaining({ reject: false }), ); - - const mergedPolicyKey = [...store.keys()].find( - (k) => k.endsWith("/merged-policy.yaml") || k.endsWith("\\merged-policy.yaml"), - ); - if (!mergedPolicyKey) throw new Error("merged policy file not written"); - const mergedEntry = store.get(mergedPolicyKey); - if (!mergedEntry?.content) throw new Error("merged policy file is empty"); - const merged = YAML.parse(mergedEntry.content) as { - network_policies?: Record; - }; - expect(merged.network_policies).toHaveProperty("existing_service"); - expect(merged.network_policies).toHaveProperty("nim_service"); + expect(stdoutText()).not.toContain("Apply complete"); + expect(stdoutText()).not.toContain("PROGRESS:100"); }); it("fails closed when the live policy cannot be parsed", async () => { @@ -806,13 +748,15 @@ describe("runner", () => { expect(policyCalls.some((call) => call[1][1] === "set")).toBe(false); }); - it("reuses sandbox when 'already exists' error", async () => { + it("refuses to claim policy ownership when sandbox already exists", async () => { mockExeca.mockImplementation(async (_cmd: string, args: string[]) => resultForCommandFailure(args, ["sandbox", "create"], "already exists"), ); - await actionApply("default", minimalBlueprint()); - expect(stdoutText()).toContain("already exists, reusing"); + await expect(actionApply("default", minimalBlueprint())).rejects.toThrow( + /already exists.*cannot establish NemoClaw policy ownership/u, + ); + expect(stdoutText()).not.toContain("Apply complete"); }); it("throws when sandbox creation fails with other error", async () => { diff --git a/nemoclaw/src/blueprint/runner.ts b/nemoclaw/src/blueprint/runner.ts index 21744e0c7bb..25179a7ac7c 100644 --- a/nemoclaw/src/blueprint/runner.ts +++ b/nemoclaw/src/blueprint/runner.ts @@ -12,17 +12,20 @@ * - exit code 0 = success, non-zero = failure */ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { + closeSync, existsSync, + fsyncSync, mkdirSync, + openSync, readdirSync, readFileSync, renameSync, writeFileSync, } from "node:fs"; import { homedir } from "node:os"; -import { join, sep } from "node:path"; +import { dirname, join, sep } from "node:path"; import { execa } from "execa"; import YAML from "yaml"; @@ -30,6 +33,7 @@ import YAML from "yaml"; import { DASHBOARD_PORT } from "../lib/ports.js"; import { buildSubprocessEnv } from "../lib/subprocess-env.js"; import { isPlainObject, type UnknownRecord } from "../shared/object-record.js"; +import * as importedOpenShellGatewayEndpointBoundary from "../shared/openshell-gateway-endpoint-boundary.cjs"; import * as importedOpenShellExternalTargetBoundary from "../shared/openshell-external-target-boundary.cjs"; import * as importedOpenShellPolicyBoundary from "../shared/openshell-policy-boundary.cjs"; import * as importedSandboxName from "../shared/sandbox-name.cjs"; @@ -64,13 +68,23 @@ const sourceOrGeneratedOpenShellPolicyBoundary = }; const { assertExternalPolicyRequirementContainment, - assertMatchingPolicyAuthority, + assertNemoClawPolicyCreationReceiptMatches, assertPolicyRequirementContainment, + openShellPolicyValuesEqual, + parseNemoClawPolicyCreationReceipt, parseOpenShellPolicy, parseSandboxPolicyAuthorityMetadata, withoutProviderComposedPolicies, } = sourceOrGeneratedOpenShellPolicyBoundary.default ?? sourceOrGeneratedOpenShellPolicyBoundary; +const sourceOrGeneratedOpenShellGatewayEndpointBoundary = + importedOpenShellGatewayEndpointBoundary as typeof importedOpenShellGatewayEndpointBoundary & { + default?: typeof importedOpenShellGatewayEndpointBoundary; + }; +const { isManagedGatewayEndpointHost, parseSingleManagedGatewayEndpoint } = + sourceOrGeneratedOpenShellGatewayEndpointBoundary.default ?? + sourceOrGeneratedOpenShellGatewayEndpointBoundary; + const sourceOrGeneratedOpenShellExternalTargetBoundary = importedOpenShellExternalTargetBoundary as typeof importedOpenShellExternalTargetBoundary & { default?: typeof importedOpenShellExternalTargetBoundary; @@ -95,6 +109,7 @@ type RollbackPlanSource = { inference?: unknown; identity?: unknown; policy_authority?: unknown; + policy_creation_transition?: unknown; policy_transition?: unknown; }; type ReconciliationPlanSource = RollbackPlanSource & { @@ -131,20 +146,53 @@ type PolicyAdditions = { [name: string]: PolicyAddition }; type BlueprintPolicyAuthorityInspection = import("../shared/openshell-policy-boundary.cjs").SandboxPolicyAuthorityInspection; +type NemoClawPolicyCreationReceipt = + import("../shared/openshell-policy-boundary.cjs").NemoClawPolicyCreationReceipt; + +type GatewayBinding = { + name: string; + host: string; + port: number; +}; -type BlueprintPolicyAuthorityReceipt = { - authority: BlueprintPolicyAuthorityInspection["authority"]; +type BlueprintPolicyAuthorityReceipt = + | { + authority: "externally-managed"; + gateway: string; + gateway_host: string; + gateway_port: number; + scope: "global" | "sandbox"; + sandbox_name?: string; + sandbox_identity_fingerprint?: string; + } + | { + authority: "nemoclaw-managed"; + gateway: string; + gateway_host: string; + gateway_port: number; + scope: "sandbox"; + sandbox_name: string; + policy_creation_receipt: NemoClawPolicyCreationReceipt; + }; + +type BlueprintPolicyCreationTransition = { + status: "pending" | "incomplete"; gateway: string; - scope: "global" | "sandbox"; - sandbox_name?: string; + gateway_host: string; + gateway_port: number; + sandbox_name: string; + lifecycle_generation: string; }; type BlueprintPolicyTransitionReceipt = { status: "pending" | "incomplete" | "complete"; sandbox_name: string; gateway: string; + gateway_host: string; + gateway_port: number; expected_authority: "nemoclaw-managed"; policy_addition_names: string[]; + target_policy_digest: string; }; type StatusPolicyTransition = BlueprintPolicyTransitionReceipt & { @@ -160,12 +208,50 @@ const REST_PROTOCOLS = new Set(["rest"]); const ENDPOINT_ENFORCEMENT_MODES = new Set(["enforce", "audit"]); const ENDPOINT_TLS_MODES = new Set(["terminate", "passthrough", "skip"]); const MISSING_SANDBOX_PATTERN = /\b(?:not found|does not exist)\b/i; -const MISSING_SANDBOX_INSPECTION_PATTERN = - /(?:\bsandbox\b[^\r\n]*\b(?:not found|does not exist)\b|\b(?:not found|does not exist)\b[^\r\n]*\bsandbox\b)/i; const MISSING_PROVIDER_INSPECTION_PATTERN = /(?:\bprovider\b[^\r\n]*\b(?:not found|does not exist)\b|\b(?:not found|does not exist)\b[^\r\n]*\bprovider\b|\bunknown provider\b)/i; const POLICY_AUTHORITY_MAX_BYTES = 1024 * 1024; const POLICY_AUTHORITY_TIMEOUT_MS = 30_000; +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu; +const MANAGED_POLICY_AUTHORITY_KEYS = [ + "authority", + "gateway", + "gateway_host", + "gateway_port", + "scope", + "sandbox_name", + "policy_creation_receipt", +] as const; +const EXTERNAL_GLOBAL_POLICY_AUTHORITY_KEYS = [ + "authority", + "gateway", + "gateway_host", + "gateway_port", + "scope", +] as const; +const EXTERNAL_SANDBOX_POLICY_AUTHORITY_KEYS = [ + ...EXTERNAL_GLOBAL_POLICY_AUTHORITY_KEYS, + "sandbox_name", + "sandbox_identity_fingerprint", +] as const; +const POLICY_CREATION_TRANSITION_KEYS = [ + "status", + "gateway", + "gateway_host", + "gateway_port", + "sandbox_name", + "lifecycle_generation", +] as const; +const POLICY_TRANSITION_KEYS = [ + "status", + "sandbox_name", + "gateway", + "gateway_host", + "gateway_port", + "expected_authority", + "policy_addition_names", + "target_policy_digest", +] as const; interface InferenceRouteBinding { provider: string; @@ -173,19 +259,6 @@ interface InferenceRouteBinding { timeoutSeconds?: number; } -function assertReusableRuntimeIdentitySandbox(output: string, expectedName: string): void { - const lines = output.replace(/\u001b\[[0-9;]*m/g, "").split(/\r?\n/); - const nameLine = lines.find((line) => /^\s*Name:/i.test(line)); - const phaseLine = lines.find((line) => /^\s*Phase:/i.test(line)); - const name = /^\s*Name:\s*(.+)$/i.exec(nameLine ?? "")?.[1]?.trim(); - const phase = /^\s*Phase:\s*(.+)$/i.exec(phaseLine ?? "")?.[1]?.trim(); - if (name !== expectedName || phase !== "Ready") { - throw new Error( - `Sandbox '${expectedName}' is not reusable for runtime identity apply: expected exact name and Ready phase, received ${boundedCommandError(output)}`, - ); - } -} - function parseInferenceRouteBinding(output: string): InferenceRouteBinding | null { const lines = output.replace(/\u001b\[[0-9;]*m/g, "").split(/\r?\n/); let inGatewayInference = false; @@ -476,6 +549,29 @@ function readRollbackInferenceProviderName(value: RollbackPlanSource): string { return assertValidProviderName(value.inference.provider_name, "rollback inference provider name"); } +function readRollbackInferenceProviderBinding(value: RollbackPlanSource): { + name: string; + type: string; + requiresEndpointConfig: boolean; +} { + const name = readRollbackInferenceProviderName(value); + if ( + !isPlainObject(value.inference) || + typeof value.inference.provider_type !== "string" || + value.inference.provider_type.trim() === "" + ) { + throw new Error( + "rollback plan inference.provider_type must be a non-empty string for an owned provider", + ); + } + return { + name, + type: value.inference.provider_type, + requiresEndpointConfig: + typeof value.inference.endpoint === "string" && value.inference.endpoint !== "", + }; +} + function assertReusableInferenceProvider( output: string, expected: { @@ -503,6 +599,23 @@ function assertReusableInferenceProvider( } } +async function inspectOwnedInferenceProvider( + expected: Parameters[1], +): Promise<"absent" | "matching"> { + const result = await runCmd(["openshell", "provider", "get", expected.name], { + reject: false, + }); + const output = `${result.stderr}\n${result.stdout}`; + if (result.exitCode !== 0) { + if (MISSING_PROVIDER_INSPECTION_PATTERN.test(output)) return "absent"; + throw new Error( + `Failed to inspect inference provider '${expected.name}' before cleanup: ${boundedCommandError(output)}`, + ); + } + assertReusableInferenceProvider(result.stdout, expected); + return "matching"; +} + // ── Utilities ─────────────────────────────────────────────────── export function emitRunId(): string { @@ -637,7 +750,7 @@ async function runCmd( }; } -async function inspectActiveGatewayIdentity(): Promise { +async function inspectActiveGatewayBinding(): Promise { const result = await runCmd(["openshell", "status"], { reject: false }); const output = `${result.stderr}\n${result.stdout}`; if (result.exitCode !== 0) { @@ -655,7 +768,8 @@ async function inspectActiveGatewayIdentity(): Promise { `Failed to prove the active OpenShell gateway identity: ${boundedCommandError(output)}`, ); } - return assertValidName(gateways[0], "OpenShell gateway name"); + const name = assertValidName(gateways[0], "OpenShell gateway name"); + return { name, ...(await inspectGatewayEndpoint(name)) }; } async function runBlueprintPolicyAuthorityCommand( @@ -686,10 +800,58 @@ async function runBlueprintPolicyAuthorityCommand( return result; } +async function runBlueprintReceiptInspectionCommand( + command: string[], + subject: "gateway" | "policy" | "sandbox", +): Promise>> { + let result: Awaited>; + try { + result = await runCmd(command, { + maxBuffer: POLICY_AUTHORITY_MAX_BYTES, + reject: false, + timeout: POLICY_AUTHORITY_TIMEOUT_MS, + }); + } catch { + throw new Error(`OpenShell ${subject} receipt inspection failed.`); + } + if ( + result.exitCode !== 0 || + Buffer.byteLength(result.stdout, "utf8") + Buffer.byteLength(result.stderr, "utf8") > + POLICY_AUTHORITY_MAX_BYTES + ) { + throw new Error(`OpenShell ${subject} receipt inspection failed.`); + } + return result; +} + +function parseGlobalPolicyIdentity( + metadata: UnknownRecord, +): BlueprintPolicyAuthorityInspection["policyIdentity"] { + if ( + typeof metadata.hash !== "string" || + metadata.hash.length === 0 || + typeof metadata.active_version !== "number" || + !Number.isSafeInteger(metadata.active_version) || + metadata.active_version <= 0 + ) { + throw new Error( + "OpenShell returned invalid global policy authority metadata. Policy-dependent operations must stop.", + ); + } + return { hash: metadata.hash, activeVersion: metadata.active_version }; +} + +async function inspectBlueprintPolicyAuthority( + gateway: string, +): Promise; +async function inspectBlueprintPolicyAuthority( + gateway: string, + sandboxName: string, +): Promise; async function inspectBlueprintPolicyAuthority( gateway: string, sandboxName?: string, -): Promise { +): Promise { const subject = sandboxName === undefined ? "global" : "sandbox"; if (sandboxName === undefined) { const history = await runBlueprintPolicyAuthorityCommand( @@ -697,7 +859,7 @@ async function inspectBlueprintPolicyAuthority( subject, ); if (history.stdout.trim().length === 0) { - return { authority: "nemoclaw-managed", effectivePolicy: {} }; + return null; } } const command = @@ -723,7 +885,7 @@ async function inspectBlueprintPolicyAuthority( !isPlainObject(metadata) || metadata.scope !== "global" || (metadata.status !== "loaded" && metadata.status !== "superseded") || - (metadata.policy_source !== undefined && metadata.policy_source !== "global") || + metadata.policy_source !== "global" || Object.hasOwn(metadata, "sandbox") ) { throw new Error( @@ -731,14 +893,18 @@ async function inspectBlueprintPolicyAuthority( ); } if (metadata.status === "superseded") { - return { authority: "nemoclaw-managed", effectivePolicy: {} }; + return null; } if (!isPlainObject(metadata.policy)) { throw new Error( "OpenShell returned invalid global policy authority metadata. Policy-dependent operations must stop.", ); } - return { authority: "externally-managed", effectivePolicy: metadata.policy }; + return { + authority: "externally-managed", + effectivePolicy: metadata.policy, + policyIdentity: parseGlobalPolicyIdentity(metadata), + }; } try { return parseSandboxPolicyAuthorityMetadata(result.stdout, sandboxName); @@ -748,18 +914,6 @@ async function inspectBlueprintPolicyAuthority( } } -function assertBlueprintPolicyAuthorityMatches( - recorded: BlueprintPolicyAuthorityInspection, - observed: BlueprintPolicyAuthorityInspection, -): void { - try { - assertMatchingPolicyAuthority(recorded.authority, observed.authority); - } catch (error) { - const detail = error instanceof Error ? error.message : "policy authority is invalid"; - throw new Error(`Refusing to apply blueprint policy additions because ${detail}.`); - } -} - function assertBlueprintExternalPolicyRequirements( inspection: BlueprintPolicyAuthorityInspection, additions: PolicyAdditions, @@ -790,6 +944,179 @@ function assertBlueprintPolicyRequirements( } } +function readConfiguredSandboxPolicy(): { + path: string; + policy: UnknownRecord; +} | null { + const path = process.env.OPENSHELL_SANDBOX_POLICY?.trim(); + if (!path) return null; + let raw: string; + try { + raw = readFileSync(path, "utf-8"); + } catch { + throw new Error("The configured NemoClaw sandbox policy could not be read"); + } + try { + return { path, policy: parseOpenShellPolicy(raw).policy }; + } catch { + throw new Error("The configured NemoClaw sandbox policy is invalid"); + } +} + +function stablePolicyValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stablePolicyValue); + if (!isPlainObject(value)) return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, stablePolicyValue(value[key])]), + ); +} + +function policyDigest(policy: unknown): string { + return createHash("sha256") + .update(JSON.stringify(stablePolicyValue(policy))) + .digest("hex"); +} + +function policyForOwnershipProof(policy: UnknownRecord): UnknownRecord { + const networkPolicies = isPlainObject(policy.network_policies) + ? withoutProviderComposedPolicies(policy.network_policies) + : policy.network_policies; + return { ...policy, network_policies: networkPolicies }; +} + +async function inspectGatewayEndpoint(name: string): Promise<{ host: string; port: number }> { + const info = await runBlueprintReceiptInspectionCommand( + ["openshell", "gateway", "info", "-g", name], + "gateway", + ); + return parseSingleManagedGatewayEndpoint(`${info.stderr}\n${info.stdout}`); +} + +async function inspectSandboxIdentityFingerprint( + gateway: string, + sandboxName: string, + requireReady = true, +): Promise { + const result = await runBlueprintReceiptInspectionCommand( + ["openshell", "sandbox", "get", "-g", gateway, sandboxName], + "sandbox", + ); + const output = `${result.stderr}\n${result.stdout}`; + const lines = output.replace(/\u001b\[[0-9;]*m/g, "").split(/\r?\n/); + const names = lines + .map((line) => /^\s*Name:\s*(.+?)\s*$/i.exec(line)?.[1]) + .filter((value): value is string => Boolean(value)); + const ids = lines + .map((line) => /^\s*Id:\s*(.+?)\s*$/i.exec(line)?.[1]) + .filter((value): value is string => Boolean(value)); + const phases = lines + .map((line) => /^\s*Phase:\s*(.+?)\s*$/i.exec(line)?.[1]) + .filter((value): value is string => Boolean(value)); + if ( + names.length !== 1 || + names[0] !== sandboxName || + ids.length !== 1 || + phases.length !== 1 || + (requireReady && phases[0] !== "Ready") + ) { + throw new Error( + `OpenShell did not prove the immutable identity of${requireReady ? " Ready" : ""} sandbox ${JSON.stringify(sandboxName)}`, + ); + } + return createHash("sha256").update(ids[0]).digest("hex"); +} + +function managedInspection( + inspection: BlueprintPolicyAuthorityInspection, +): BlueprintPolicyAuthorityInspection { + return { ...inspection, authority: "nemoclaw-managed" }; +} + +async function validateManagedPolicyReceipt( + value: unknown, + expectedGatewayEndpoint: { host: string; port: number }, + requireReady = true, +): Promise<{ + receipt: NemoClawPolicyCreationReceipt; + inspection: BlueprintPolicyAuthorityInspection; +}> { + const receipt = parseNemoClawPolicyCreationReceipt(value); + const gatewayEndpoint = await inspectGatewayEndpoint(receipt.gatewayName); + if ( + gatewayEndpoint.host !== expectedGatewayEndpoint.host || + gatewayEndpoint.port !== expectedGatewayEndpoint.port + ) { + throw new Error("The OpenShell gateway endpoint no longer matches the durable policy receipt"); + } + const sandboxIdentityFingerprint = await inspectSandboxIdentityFingerprint( + receipt.gatewayName, + receipt.sandboxName, + requireReady, + ); + const inspection = await inspectBlueprintPolicyAuthority( + receipt.gatewayName, + receipt.sandboxName, + ); + if (inspection.authority !== "owner-unknown") { + throw new Error("The live sandbox policy is no longer sandbox-scoped"); + } + assertNemoClawPolicyCreationReceiptMatches(receipt, { + origin: "sandbox-create", + gatewayName: receipt.gatewayName, + gatewayPort: gatewayEndpoint.port, + sandboxName: receipt.sandboxName, + lifecycleGeneration: receipt.lifecycleGeneration, + sandboxIdentityFingerprint, + policyHash: inspection.policyIdentity.hash, + policyVersion: inspection.policyIdentity.activeVersion, + }); + return { receipt, inspection: managedInspection(inspection) }; +} + +async function inspectReceiptSandboxBinding( + value: unknown, + expectedGatewayEndpoint: { host: string; port: number }, +): Promise<{ + receipt: NemoClawPolicyCreationReceipt; + inspection: BlueprintPolicyAuthorityInspection; +}> { + const receipt = parseNemoClawPolicyCreationReceipt(value); + const gatewayEndpoint = await inspectGatewayEndpoint(receipt.gatewayName); + if ( + gatewayEndpoint.host !== expectedGatewayEndpoint.host || + gatewayEndpoint.port !== expectedGatewayEndpoint.port + ) { + throw new Error("The OpenShell gateway endpoint no longer matches the durable policy receipt"); + } + const sandboxIdentityFingerprint = await inspectSandboxIdentityFingerprint( + receipt.gatewayName, + receipt.sandboxName, + ); + assertNemoClawPolicyCreationReceiptMatches(receipt, { + origin: "sandbox-create", + gatewayName: receipt.gatewayName, + gatewayPort: gatewayEndpoint.port, + sandboxName: receipt.sandboxName, + lifecycleGeneration: receipt.lifecycleGeneration, + sandboxIdentityFingerprint, + policyHash: receipt.policyHash, + policyVersion: receipt.policyVersion, + }); + const inspection = await inspectBlueprintPolicyAuthority( + receipt.gatewayName, + receipt.sandboxName, + ); + if (inspection.authority !== "owner-unknown") { + throw new Error("The live sandbox policy is no longer sandbox-scoped"); + } + return { + receipt, + inspection, + }; +} + async function runRuntimeIdentityCommand( args: string[], options?: RuntimeIdentityCommandOptions, @@ -808,9 +1135,29 @@ async function runRuntimeIdentityCommand( }; } -function runtimeIdentityCommandDeps(): RuntimeIdentityCommandDeps { +function isRuntimeIdentityMutationCommand(args: readonly string[]): boolean { + const command = args.slice(1).join(" "); + return ( + command.startsWith("provider profile import ") || + command.startsWith("provider create ") || + command.startsWith("provider delete ") || + command.startsWith("provider refresh configure ") || + command.startsWith("provider refresh rotate ") || + command.startsWith("sandbox provider attach ") || + command.startsWith("sandbox provider detach ") + ); +} + +function runtimeIdentityCommandDeps( + validateBeforeMutation?: () => Promise, +): RuntimeIdentityCommandDeps { return { - run: runRuntimeIdentityCommand, + run: async (args, options) => { + if (validateBeforeMutation && isRuntimeIdentityMutationCommand(args)) { + await validateBeforeMutation(); + } + return runRuntimeIdentityCommand(args, options); + }, formatError: boundedCommandError, }; } @@ -818,9 +1165,10 @@ function runtimeIdentityCommandDeps(): RuntimeIdentityCommandDeps { function runtimeIdentityDeps( persistReceipt: (receipt: RuntimeIdentityReceipt) => void, profilePolicy?: RuntimeIdentityProfilePolicy, + validateBeforeMutation?: () => Promise, ): RuntimeIdentityDeps { return { - ...runtimeIdentityCommandDeps(), + ...runtimeIdentityCommandDeps(validateBeforeMutation), validateEndpointUrl, persistReceipt, blueprintPath: process.env.NEMOCLAW_BLUEPRINT_PATH ?? ".", @@ -936,7 +1284,8 @@ interface PersistedRunPlan { sandbox_created_by_apply: boolean; inference_provider_created_by_apply: boolean; policy_additions: PolicyAdditions; - policy_authority: BlueprintPolicyAuthorityReceipt; + policy_authority?: BlueprintPolicyAuthorityReceipt; + policy_creation_transition?: BlueprintPolicyCreationTransition; policy_transition?: BlueprintPolicyTransitionReceipt; inference: SafeInferencePlan; identity?: RuntimeIdentityReceipt; @@ -956,6 +1305,7 @@ type StatusRunPlan = { inference_provider_created_by_apply?: boolean; policy_additions?: PolicyAdditions; policy_authority?: BlueprintPolicyAuthorityReceipt; + policy_creation_transition?: BlueprintPolicyCreationTransition; policy_transition?: StatusPolicyTransition; inference?: SafeInferencePlan; identity?: RuntimeIdentityReceipt; @@ -979,13 +1329,53 @@ function isBlueprintPolicyAuthorityReceipt( if ( (value.authority !== "nemoclaw-managed" && value.authority !== "externally-managed") || (value.scope !== "global" && value.scope !== "sandbox") || - !isValidName(value.gateway) + !isValidName(value.gateway) || + !isManagedGatewayEndpointHost(value.gateway_host) || + !isValidPort(value.gateway_port) ) { return false; } + if (value.authority === "nemoclaw-managed") { + if ( + value.scope !== "sandbox" || + !isValidName(value.sandbox_name) || + !hasOnlyKeys(value, MANAGED_POLICY_AUTHORITY_KEYS) + ) { + return false; + } + try { + const receipt = parseNemoClawPolicyCreationReceipt(value.policy_creation_receipt); + return ( + receipt.gatewayName === value.gateway && + receipt.gatewayPort === value.gateway_port && + receipt.sandboxName === value.sandbox_name + ); + } catch { + return false; + } + } return value.scope === "global" - ? value.sandbox_name === undefined - : isValidName(value.sandbox_name); + ? hasOnlyKeys(value, EXTERNAL_GLOBAL_POLICY_AUTHORITY_KEYS) + : hasOnlyKeys(value, EXTERNAL_SANDBOX_POLICY_AUTHORITY_KEYS) && + isValidName(value.sandbox_name) && + typeof value.sandbox_identity_fingerprint === "string" && + /^[a-f0-9]{64}$/u.test(value.sandbox_identity_fingerprint); +} + +function isBlueprintPolicyCreationTransition( + value: unknown, +): value is BlueprintPolicyCreationTransition { + return ( + isPlainObject(value) && + hasOnlyKeys(value, POLICY_CREATION_TRANSITION_KEYS) && + (value.status === "pending" || value.status === "incomplete") && + isValidName(value.gateway) && + isManagedGatewayEndpointHost(value.gateway_host) && + isValidPort(value.gateway_port) && + isValidName(value.sandbox_name) && + typeof value.lifecycle_generation === "string" && + UUID_PATTERN.test(value.lifecycle_generation) + ); } function isBlueprintPolicyTransitionReceipt( @@ -993,10 +1383,15 @@ function isBlueprintPolicyTransitionReceipt( ): value is BlueprintPolicyTransitionReceipt { if (!isPlainObject(value)) return false; if ( + !hasOnlyKeys(value, POLICY_TRANSITION_KEYS) || (value.status !== "pending" && value.status !== "incomplete" && value.status !== "complete") || !isValidName(value.sandbox_name) || !isValidName(value.gateway) || + !isManagedGatewayEndpointHost(value.gateway_host) || + !isValidPort(value.gateway_port) || value.expected_authority !== "nemoclaw-managed" || + typeof value.target_policy_digest !== "string" || + !/^[a-f0-9]{64}$/u.test(value.target_policy_digest) || !Array.isArray(value.policy_addition_names) || value.policy_addition_names.length === 0 || !value.policy_addition_names.every( @@ -1008,6 +1403,45 @@ function isBlueprintPolicyTransitionReceipt( return new Set(value.policy_addition_names).size === value.policy_addition_names.length; } +async function validateBlueprintPolicyAuthorityReceipt( + value: unknown, + sandboxName: string, + requireReady = true, +): Promise { + if ( + !isBlueprintPolicyAuthorityReceipt(value) || + value.scope !== "sandbox" || + value.sandbox_name !== sandboxName + ) { + throw new Error("A complete sandbox policy boundary receipt is required"); + } + if (value.authority === "nemoclaw-managed") { + return ( + await validateManagedPolicyReceipt( + value.policy_creation_receipt, + { host: value.gateway_host, port: value.gateway_port }, + requireReady, + ) + ).inspection; + } + const liveEndpoint = await inspectGatewayEndpoint(value.gateway); + const liveFingerprint = await inspectSandboxIdentityFingerprint( + value.gateway, + sandboxName, + requireReady, + ); + const livePolicy = await inspectBlueprintPolicyAuthority(value.gateway, sandboxName); + if ( + liveEndpoint.host !== value.gateway_host || + liveEndpoint.port !== value.gateway_port || + liveFingerprint !== value.sandbox_identity_fingerprint || + livePolicy.authority !== "externally-managed" + ) { + throw new Error("The verified external policy boundary no longer matches the live sandbox"); + } + return livePolicy; +} + function buildSafeInferencePlan(source: InferenceProfile | UnknownRecord): SafeInferencePlan { return { provider_type: optionalString(source.provider_type), @@ -1060,7 +1494,8 @@ function buildPersistedRunPlan(args: { sandboxCreatedByApply: boolean; inferenceProviderCreatedByApply: boolean; policyAdditions: PolicyAdditions; - policyAuthorityReceipt: BlueprintPolicyAuthorityReceipt; + policyAuthorityReceipt?: BlueprintPolicyAuthorityReceipt; + policyCreationTransition?: BlueprintPolicyCreationTransition; policyTransition?: BlueprintPolicyTransitionReceipt; inferenceCfg: InferenceProfile; runtimeIdentityReceipt?: RuntimeIdentityReceipt; @@ -1073,10 +1508,15 @@ function buildPersistedRunPlan(args: { sandbox_created_by_apply: args.sandboxCreatedByApply, inference_provider_created_by_apply: args.inferenceProviderCreatedByApply, policy_additions: args.policyAdditions, - policy_authority: args.policyAuthorityReceipt, inference: buildSafeInferencePlan(args.inferenceCfg), timestamp: args.timestamp, }; + if (args.policyAuthorityReceipt) { + plan.policy_authority = args.policyAuthorityReceipt; + } + if (args.policyCreationTransition) { + plan.policy_creation_transition = args.policyCreationTransition; + } if (args.runtimeIdentityReceipt) { plan.identity = args.runtimeIdentityReceipt; } @@ -1089,7 +1529,19 @@ function buildPersistedRunPlan(args: { function persistRunReceipt(planFile: string, plan: unknown): void { const pendingFile = `${planFile}.pending`; writeFileSync(pendingFile, JSON.stringify(plan, null, 2), { encoding: "utf-8", mode: 0o600 }); + const pendingFd = openSync(pendingFile, "r"); + try { + fsyncSync(pendingFd); + } finally { + closeSync(pendingFd); + } renameSync(pendingFile, planFile); + const stateDirFd = openSync(dirname(planFile), "r"); + try { + fsyncSync(stateDirFd); + } finally { + closeSync(stateDirFd); + } } function buildStatusRunPlan(source: unknown, fallbackRunId: string): StatusRunPlan | null { @@ -1141,9 +1593,24 @@ function buildStatusRunPlan(source: unknown, fallbackRunId: string): StatusRunPl if (isPolicyAdditions(source.policy_additions)) { safePlan.policy_additions = source.policy_additions; } + if ( + source.policy_authority !== undefined && + !isBlueprintPolicyAuthorityReceipt(source.policy_authority) + ) { + return null; + } if (isBlueprintPolicyAuthorityReceipt(source.policy_authority)) { safePlan.policy_authority = source.policy_authority; } + if ( + source.policy_creation_transition !== undefined && + !isBlueprintPolicyCreationTransition(source.policy_creation_transition) + ) { + return null; + } + if (isBlueprintPolicyCreationTransition(source.policy_creation_transition)) { + safePlan.policy_creation_transition = source.policy_creation_transition; + } if ( source.policy_transition !== undefined && !isBlueprintPolicyTransitionReceipt(source.policy_transition) @@ -1323,21 +1790,36 @@ export async function actionApply( if (credentialEnv) { credential = process.env[credentialEnv] ?? credentialDefault; } - const policyGateway = await inspectActiveGatewayIdentity(); - const initialPolicyAuthority = await inspectBlueprintPolicyAuthority(policyGateway); + const policyGateway = await inspectActiveGatewayBinding(); + const initialPolicyAuthority = await inspectBlueprintPolicyAuthority(policyGateway.name); + if (initialPolicyAuthority) { + assertBlueprintExternalPolicyRequirements(initialPolicyAuthority, policyAdditions); + } + const configuredSandboxPolicy = initialPolicyAuthority ? null : readConfiguredSandboxPolicy(); + if (!initialPolicyAuthority && !configuredSandboxPolicy) { + throw new Error( + "A configured NemoClaw sandbox policy is required before the blueprint can create or mutate resources.", + ); + } const stateDir = join(homedir(), ".nemoclaw", "state", "runs", rid); mkdirSync(stateDir, { recursive: true }); let runtimeIdentityReceipt: RuntimeIdentityReceipt | undefined; - let policyAuthorityReceipt: BlueprintPolicyAuthorityReceipt = { - authority: initialPolicyAuthority.authority, - gateway: policyGateway, - scope: "global", - }; + let policyAuthorityReceipt: BlueprintPolicyAuthorityReceipt | undefined = initialPolicyAuthority + ? { + authority: "externally-managed", + gateway: policyGateway.name, + gateway_host: policyGateway.host, + gateway_port: policyGateway.port, + scope: "global", + } + : undefined; + let policyCreationTransition: BlueprintPolicyCreationTransition | undefined; let sandboxPolicyAuthority: BlueprintPolicyAuthorityInspection | null = null; let policyTransition: BlueprintPolicyTransitionReceipt | undefined; let sandboxCreatedByApply = false; let inferenceProviderCreatedByApply = false; + let runtimeIdentityCleanupAllowed = true; const persistRunPlan = (): void => { persistRunReceipt( join(stateDir, "plan.json"), @@ -1349,6 +1831,7 @@ export async function actionApply( inferenceProviderCreatedByApply, policyAdditions, policyAuthorityReceipt, + policyCreationTransition, policyTransition, inferenceCfg, runtimeIdentityReceipt, @@ -1356,31 +1839,204 @@ export async function actionApply( }), ); }; - const identityDeps = runtimeIdentityDeps((receipt) => { - runtimeIdentityReceipt = receipt; - persistRunPlan(); - }, options?.runtimeIdentityProfilePolicy); - persistRunPlan(); - assertBlueprintExternalPolicyRequirements(initialPolicyAuthority, policyAdditions); + const requireUsablePolicyBoundary = async (): Promise => { + const inspection = await validateBlueprintPolicyAuthorityReceipt( + policyAuthorityReceipt, + sandboxName, + ); + assertBlueprintExternalPolicyRequirements(inspection, policyAdditions); + sandboxPolicyAuthority = inspection; + return inspection; + }; + const requireCreatePolicyBoundary = async (): Promise => { + const liveGateway = await inspectActiveGatewayBinding(); + if ( + liveGateway.name !== policyGateway.name || + liveGateway.host !== policyGateway.host || + liveGateway.port !== policyGateway.port + ) { + throw new Error("The OpenShell gateway binding changed before sandbox creation."); + } + const liveGlobalPolicy = await inspectBlueprintPolicyAuthority(policyGateway.name); + if (initialPolicyAuthority === null) { + if (liveGlobalPolicy !== null) { + throw new Error("The OpenShell global policy boundary changed before sandbox creation."); + } + return; + } + if ( + liveGlobalPolicy === null || + liveGlobalPolicy.authority !== "externally-managed" || + liveGlobalPolicy.policyIdentity.hash !== initialPolicyAuthority.policyIdentity.hash || + liveGlobalPolicy.policyIdentity.activeVersion !== + initialPolicyAuthority.policyIdentity.activeVersion || + !openShellPolicyValuesEqual( + liveGlobalPolicy.effectivePolicy, + initialPolicyAuthority.effectivePolicy, + ) + ) { + throw new Error("The OpenShell global policy boundary changed before sandbox creation."); + } + }; + const identityDeps = runtimeIdentityDeps( + (receipt) => { + const previousReceipt = runtimeIdentityReceipt; + runtimeIdentityReceipt = receipt; + try { + persistRunPlan(); + } catch (error) { + runtimeIdentityReceipt = previousReceipt; + runtimeIdentityCleanupAllowed = false; + throw error; + } + }, + options?.runtimeIdentityProfilePolicy, + requireUsablePolicyBoundary, + ); try { - let reuseExistingSandbox = false; let reuseExistingInferenceProvider = false; let reuseExistingInferenceRoute = false; - if (runtimeIdentityConfig) { - const sandboxResult = await runCmd(["openshell", "sandbox", "get", sandboxName], { - reject: false, - }); - const sandboxOutput = `${sandboxResult.stderr}\n${sandboxResult.stdout}`; - if (sandboxResult.exitCode === 0) { - assertReusableRuntimeIdentitySandbox(sandboxResult.stdout, sandboxName); - reuseExistingSandbox = true; - } else if (!MISSING_SANDBOX_INSPECTION_PATTERN.test(sandboxOutput)) { + let policyCreationReceipt: NemoClawPolicyCreationReceipt | undefined; + progress(20, "Creating OpenClaw sandbox"); + const lifecycleGeneration = randomUUID(); + if (configuredSandboxPolicy) { + policyCreationTransition = { + status: "pending", + gateway: policyGateway.name, + gateway_host: policyGateway.host, + gateway_port: policyGateway.port, + sandbox_name: sandboxName, + lifecycle_generation: lifecycleGeneration, + }; + persistRunPlan(); + } + const createArgs = [ + "openshell", + "sandbox", + "create", + "-g", + policyGateway.name, + "--from", + sandboxImage, + "--name", + sandboxName, + ]; + if (configuredSandboxPolicy) { + createArgs.push("--policy", configuredSandboxPolicy.path); + } + for (const port of forwardPorts) { + createArgs.push("--forward", String(port)); + } + + await requireCreatePolicyBoundary(); + const createResult = await runCmd(createArgs, { + omitSandboxPolicy: true, + reject: false, + }); + if (createResult.exitCode !== 0) { + if (createResult.stderr.includes("already exists")) { + if (configuredSandboxPolicy) { + policyCreationTransition = { ...policyCreationTransition!, status: "incomplete" }; + persistRunPlan(); + throw new Error( + `Sandbox ${JSON.stringify(sandboxName)} already exists, so this create transaction cannot establish NemoClaw policy ownership.`, + ); + } + log(`Sandbox '${sandboxName}' already exists, reusing under verified global policy.`); + } else { + if (configuredSandboxPolicy) { + policyCreationTransition = { ...policyCreationTransition!, status: "incomplete" }; + persistRunPlan(); + } + throw new Error(`Failed to create sandbox: ${boundedCommandError(createResult.stderr)}`); + } + } else { + sandboxCreatedByApply = true; + if (configuredSandboxPolicy) { + policyCreationTransition = { ...policyCreationTransition!, status: "incomplete" }; + } + persistRunPlan(); + } + + const sandboxIdentityFingerprint = await inspectSandboxIdentityFingerprint( + policyGateway.name, + sandboxName, + ); + const observedPolicyAuthority = await inspectBlueprintPolicyAuthority( + policyGateway.name, + sandboxName, + ); + if (configuredSandboxPolicy) { + if ( + observedPolicyAuthority.authority !== "owner-unknown" || + !openShellPolicyValuesEqual( + policyForOwnershipProof(configuredSandboxPolicy.policy), + policyForOwnershipProof(observedPolicyAuthority.effectivePolicy), + ) + ) { throw new Error( - `Failed to inspect sandbox '${sandboxName}' before runtime identity apply: ${boundedCommandError(sandboxOutput)}`, + "The created sandbox did not prove the exact policy supplied by this NemoClaw create transaction.", ); } + policyCreationReceipt = { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: policyGateway.name, + gatewayPort: policyGateway.port, + sandboxName, + lifecycleGeneration, + sandboxIdentityFingerprint, + policyHash: observedPolicyAuthority.policyIdentity.hash, + policyVersion: observedPolicyAuthority.policyIdentity.activeVersion, + }; + parseNemoClawPolicyCreationReceipt(policyCreationReceipt); + sandboxPolicyAuthority = managedInspection(observedPolicyAuthority); + policyAuthorityReceipt = { + authority: "nemoclaw-managed", + gateway: policyGateway.name, + gateway_host: policyGateway.host, + gateway_port: policyGateway.port, + scope: "sandbox", + sandbox_name: sandboxName, + policy_creation_receipt: policyCreationReceipt, + }; + } else { + if (observedPolicyAuthority.authority !== "externally-managed") { + throw new Error("The created sandbox did not retain the verified global policy boundary."); + } + assertBlueprintExternalPolicyRequirements(observedPolicyAuthority, policyAdditions); + sandboxPolicyAuthority = observedPolicyAuthority; + policyAuthorityReceipt = { + authority: "externally-managed", + gateway: policyGateway.name, + gateway_host: policyGateway.host, + gateway_port: policyGateway.port, + scope: "sandbox", + sandbox_name: sandboxName, + sandbox_identity_fingerprint: sandboxIdentityFingerprint, + }; + } + policyCreationTransition = undefined; + try { + persistRunPlan(); + } catch (error) { + policyAuthorityReceipt = undefined; + sandboxPolicyAuthority = null; + policyCreationTransition = { + status: "incomplete", + gateway: policyGateway.name, + gateway_host: policyGateway.host, + gateway_port: policyGateway.port, + sandbox_name: sandboxName, + lifecycle_generation: lifecycleGeneration, + }; + persistRunPlan(); + throw error; + } + await requireUsablePolicyBoundary(); + if (runtimeIdentityConfig) { const providerResult = await runCmd(["openshell", "provider", "get", providerName], { reject: false, }); @@ -1398,21 +2054,17 @@ export async function actionApply( `Failed to inspect inference provider '${providerName}' before runtime identity apply: ${boundedCommandError(providerOutput)}`, ); } - - if (reuseExistingSandbox && reuseExistingInferenceProvider) { - const routeResult = await runCmd(["openshell", "inference", "get"], { - reject: false, - }); - const routeOutput = `${routeResult.stderr}\n${routeResult.stdout}`; + if (reuseExistingInferenceProvider) { + const routeResult = await runCmd(["openshell", "inference", "get"], { reject: false }); if (routeResult.exitCode !== 0) { throw new Error( - `Failed to inspect the active inference route before runtime identity apply: ${boundedCommandError(routeOutput)}`, + `Failed to inspect the active inference route before runtime identity apply: ${boundedCommandError(`${routeResult.stderr}\n${routeResult.stdout}`)}`, ); } const activeRoute = parseInferenceRouteBinding(routeResult.stdout); if (!activeRoute && !isUnconfiguredInferenceRoute(routeResult.stdout)) { throw new Error( - `Failed to parse the active inference route before runtime identity apply: ${boundedCommandError(routeOutput)}`, + "Failed to parse the active inference route before runtime identity apply", ); } reuseExistingInferenceRoute = @@ -1421,84 +2073,16 @@ export async function actionApply( (inferenceCfg.timeout_secs === undefined || activeRoute.timeoutSeconds === inferenceCfg.timeout_secs); } - - progress(10, "Configuring runtime identity"); - // Establish durable state before the first identity mutation, then update - // the receipt after each acquired resource. - persistRunPlan(); + progress(30, "Configuring runtime identity"); + await requireUsablePolicyBoundary(); runtimeIdentityReceipt = await prepareRuntimeIdentity(runtimeIdentityConfig, identityDeps); persistRunPlan(); } - progress(20, "Creating OpenClaw sandbox"); - if (reuseExistingSandbox) { - log(`Sandbox '${sandboxName}' already exists, reusing.`); - } else { - const createArgs = [ - "openshell", - "sandbox", - "create", - "--from", - sandboxImage, - "--name", - sandboxName, - ]; - for (const port of forwardPorts) { - createArgs.push("--forward", String(port)); - } - - const createResult = await runCmd(createArgs, { - omitSandboxPolicy: initialPolicyAuthority.authority === "externally-managed", - reject: false, - }); - sandboxCreatedByApply = createResult.exitCode === 0; - if (sandboxCreatedByApply) { - // Persist ownership immediately so a later-process rollback stays safe - // if apply is interrupted before its final state write. - persistRunPlan(); - } - if (createResult.exitCode !== 0) { - if (createResult.stderr.includes("already exists")) { - if (runtimeIdentityConfig) { - const racedSandbox = await runCmd(["openshell", "sandbox", "get", sandboxName], { - reject: false, - }); - if (racedSandbox.exitCode !== 0) { - throw new Error( - `Failed to inspect sandbox '${sandboxName}' after concurrent creation: ${boundedCommandError(`${racedSandbox.stderr}\n${racedSandbox.stdout}`)}`, - ); - } - assertReusableRuntimeIdentitySandbox(racedSandbox.stdout, sandboxName); - } - log(`Sandbox '${sandboxName}' already exists, reusing.`); - } else { - throw new Error(`Failed to create sandbox: ${createResult.stderr}`); - } - } - } - - { - const observedPolicyAuthority = await inspectBlueprintPolicyAuthority( - policyGateway, - sandboxName, - ); - if (sandboxCreatedByApply) { - assertBlueprintPolicyAuthorityMatches(initialPolicyAuthority, observedPolicyAuthority); - } - sandboxPolicyAuthority = observedPolicyAuthority; - policyAuthorityReceipt = { - authority: observedPolicyAuthority.authority, - gateway: policyGateway, - scope: "sandbox", - sandbox_name: sandboxName, - }; - persistRunPlan(); - assertBlueprintExternalPolicyRequirements(observedPolicyAuthority, policyAdditions); - } - // Keep runtime credentials unattached until OpenShell accepts the // sandbox's requested inference route. progress(50, "Configuring inference provider"); + await requireUsablePolicyBoundary(); if (reuseExistingInferenceProvider) { log(`Provider '${providerName}' already exists, reusing.`); } else { @@ -1522,6 +2106,7 @@ export async function actionApply( providerArgs.push("--config", `OPENAI_BASE_URL=${endpoint}`); } + await requireUsablePolicyBoundary(); const providerResult = await execa(providerArgs[0], providerArgs.slice(1), { reject: false, stdout: "pipe", @@ -1562,11 +2147,17 @@ export async function actionApply( } else { inferenceProviderCreatedByApply = true; // Persist ownership before a later route or policy mutation can fail. - persistRunPlan(); + try { + persistRunPlan(); + } catch (error) { + inferenceProviderCreatedByApply = false; + throw error; + } } } progress(70, "Setting inference route"); + await requireUsablePolicyBoundary(); if (reuseExistingInferenceRoute) { log(`Inference route '${providerName} / ${model}' is already active, reusing.`); } else { @@ -1582,6 +2173,7 @@ export async function actionApply( if (inferenceCfg.timeout_secs !== undefined) { inferenceArgs.push("--timeout", String(inferenceCfg.timeout_secs)); } + await requireUsablePolicyBoundary(); const inferenceResult = await runCmd(inferenceArgs, { reject: false }); // Another required mutation: without a routed provider the sandbox cannot // perform inference, so a non-zero result must abort the apply. (#6703) @@ -1593,15 +2185,7 @@ export async function actionApply( } if (runtimeIdentityReceipt) { - const attachmentSandbox = await runCmd(["openshell", "sandbox", "get", sandboxName], { - reject: false, - }); - if (attachmentSandbox.exitCode !== 0) { - throw new Error( - `Failed to inspect sandbox '${sandboxName}' immediately before runtime identity attachment: ${boundedCommandError(`${attachmentSandbox.stderr}\n${attachmentSandbox.stdout}`)}`, - ); - } - assertReusableRuntimeIdentitySandbox(attachmentSandbox.stdout, sandboxName); + await requireUsablePolicyBoundary(); const attachmentCreated = await attachRuntimeIdentity( runtimeIdentityReceipt, sandboxName, @@ -1611,7 +2195,12 @@ export async function actionApply( ...runtimeIdentityReceipt, attachment_created: attachmentCreated, }; - persistRunPlan(); + try { + persistRunPlan(); + } catch (error) { + runtimeIdentityCleanupAllowed = false; + throw error; + } await mintRuntimeIdentityCredential(runtimeIdentityReceipt, identityDeps); } @@ -1619,23 +2208,14 @@ export async function actionApply( if (!sandboxPolicyAuthority) { throw new Error("Sandbox policy authority is unavailable before applying additions."); } - const observedPolicyAuthority = await inspectBlueprintPolicyAuthority( - policyGateway, - sandboxName, - ); - assertBlueprintPolicyAuthorityMatches(sandboxPolicyAuthority, observedPolicyAuthority); + const observedPolicyAuthority = await requireUsablePolicyBoundary(); assertBlueprintExternalPolicyRequirements(observedPolicyAuthority, policyAdditions); if (observedPolicyAuthority.authority === "nemoclaw-managed") { progress(78, "Applying policy additions"); - const currentPolicy = await runCmd( - ["openshell", "policy", "get", "-g", policyGateway, "--base", sandboxName], - { reject: false }, + const currentPolicy = await runBlueprintReceiptInspectionCommand( + ["openshell", "policy", "get", "-g", policyGateway.name, "--base", sandboxName], + "policy", ); - if (currentPolicy.exitCode !== 0) { - throw new Error( - `Failed to read current policy before applying additions: ${boundedCommandError(currentPolicy.stderr)}`, - ); - } const mergedPolicyFile = join(stateDir, "merged-policy.yaml"); writeFileSync( @@ -1647,23 +2227,27 @@ export async function actionApply( }, ); - const beforeMutation = await inspectBlueprintPolicyAuthority(policyGateway, sandboxName); - assertBlueprintPolicyAuthorityMatches(sandboxPolicyAuthority, beforeMutation); + const mergedPolicy = parseOpenShellPolicy(readFileSync(mergedPolicyFile, "utf-8")).policy; + await requireUsablePolicyBoundary(); policyTransition = { status: "pending", sandbox_name: sandboxName, - gateway: policyGateway, + gateway: policyGateway.name, + gateway_host: policyGateway.host, + gateway_port: policyGateway.port, expected_authority: "nemoclaw-managed", policy_addition_names: Object.keys(policyAdditions).sort(), + target_policy_digest: policyDigest(mergedPolicy), }; persistRunPlan(); + await requireUsablePolicyBoundary(); const policySet = await runCmd( [ "openshell", "policy", "set", "-g", - policyGateway, + policyGateway.name, "--policy", mergedPolicyFile, "--wait", @@ -1678,6 +2262,55 @@ export async function actionApply( } policyTransition = { ...policyTransition, status: "incomplete" }; persistRunPlan(); + const afterMutationIdentity = await inspectSandboxIdentityFingerprint( + policyGateway.name, + sandboxName, + ); + const afterMutation = await inspectBlueprintPolicyAuthority( + policyGateway.name, + sandboxName, + ); + const afterMutationGateway = await inspectGatewayEndpoint(policyGateway.name); + if (!policyCreationReceipt) { + throw new Error("The NemoClaw policy creation receipt is unavailable after mutation"); + } + if ( + afterMutationGateway.host !== policyGateway.host || + afterMutationGateway.port !== policyGateway.port || + afterMutationIdentity !== policyCreationReceipt.sandboxIdentityFingerprint || + afterMutation.authority !== "owner-unknown" || + !openShellPolicyValuesEqual( + policyForOwnershipProof(afterMutation.effectivePolicy), + policyForOwnershipProof(mergedPolicy), + ) + ) { + throw new Error( + "OpenShell did not prove the exact sandbox and effective policy after the NemoClaw policy mutation.", + ); + } + const rotatedReceipt: NemoClawPolicyCreationReceipt = { + ...policyCreationReceipt, + policyHash: afterMutation.policyIdentity.hash, + policyVersion: afterMutation.policyIdentity.activeVersion, + }; + policyAuthorityReceipt = { + authority: "nemoclaw-managed", + gateway: policyGateway.name, + gateway_host: policyGateway.host, + gateway_port: policyGateway.port, + scope: "sandbox", + sandbox_name: sandboxName, + policy_creation_receipt: rotatedReceipt, + }; + sandboxPolicyAuthority = managedInspection(afterMutation); + policyTransition = { ...policyTransition, status: "complete" }; + try { + persistRunPlan(); + } catch (error) { + policyTransition = { ...policyTransition, status: "incomplete" }; + persistRunPlan(); + throw error; + } } } @@ -1685,8 +2318,7 @@ export async function actionApply( if (!sandboxPolicyAuthority) { throw new Error("Sandbox policy authority is unavailable before saving run state."); } - const finalPolicyAuthority = await inspectBlueprintPolicyAuthority(policyGateway, sandboxName); - assertBlueprintPolicyAuthorityMatches(sandboxPolicyAuthority, finalPolicyAuthority); + const finalPolicyAuthority = await requireUsablePolicyBoundary(); assertBlueprintExternalPolicyRequirements(finalPolicyAuthority, policyAdditions); if (policyTransition) { policyTransition = { ...policyTransition, status: "complete" }; @@ -1698,7 +2330,8 @@ export async function actionApply( log(`Inference: ${providerName} -> ${model} @ ${endpoint}`); } catch (error) { const cleanupFailures: string[] = []; - if (runtimeIdentityReceipt) { + let preserveSandboxForCleanup = false; + if (runtimeIdentityReceipt && runtimeIdentityCleanupAllowed) { try { await compensateRuntimeIdentityApply(runtimeIdentityReceipt, sandboxName, identityDeps); runtimeIdentityReceipt = { @@ -1708,41 +2341,75 @@ export async function actionApply( }; persistRunPlan(); } catch (cleanupError) { + preserveSandboxForCleanup = true; cleanupFailures.push( cleanupError instanceof Error ? cleanupError.message : String(cleanupError), ); } + } else if (runtimeIdentityReceipt) { + preserveSandboxForCleanup = true; + cleanupFailures.push( + "Runtime identity resources were preserved because their latest ownership state was not durably recorded.", + ); } - if (sandboxCreatedByApply) { - await runCmd(["openshell", "sandbox", "stop", sandboxName], { reject: false }); - const remove = await runCmd(["openshell", "sandbox", "remove", sandboxName], { - reject: false, - }); - if (remove.exitCode === 0 || MISSING_SANDBOX_PATTERN.test(remove.stderr)) { - sandboxCreatedByApply = false; - if (policyTransition) { - policyTransition = { ...policyTransition, status: "complete" }; + if (inferenceProviderCreatedByApply) { + try { + const providerState = await inspectOwnedInferenceProvider({ + name: providerName, + type: providerType, + requiresEndpointConfig: endpoint !== "", + requiresCredential: credential !== "", + }); + if (providerState === "absent") { + inferenceProviderCreatedByApply = false; + persistRunPlan(); + } else { + await requireUsablePolicyBoundary(); + const removeProvider = await runCmd(["openshell", "provider", "delete", providerName], { + reject: false, + }); + if ( + removeProvider.exitCode !== 0 && + !MISSING_PROVIDER_INSPECTION_PATTERN.test(removeProvider.stderr) + ) { + throw new Error( + `Failed to remove inference provider '${providerName}': ${boundedCommandError(removeProvider.stderr)}`, + ); + } + inferenceProviderCreatedByApply = false; + persistRunPlan(); } - persistRunPlan(); - } else { + } catch (cleanupError) { + preserveSandboxForCleanup = true; cleanupFailures.push( - `Failed to remove sandbox '${sandboxName}': ${boundedCommandError(remove.stderr)}`, + cleanupError instanceof Error ? cleanupError.message : String(cleanupError), ); } } - if (inferenceProviderCreatedByApply && !sandboxCreatedByApply) { - const removeProvider = await runCmd(["openshell", "provider", "delete", providerName], { - reject: false, - }); - if ( - removeProvider.exitCode === 0 || - MISSING_PROVIDER_INSPECTION_PATTERN.test(removeProvider.stderr) - ) { - inferenceProviderCreatedByApply = false; + if (sandboxCreatedByApply && !preserveSandboxForCleanup) { + try { + await requireUsablePolicyBoundary(); + await runCmd(["openshell", "sandbox", "stop", "-g", policyGateway.name, sandboxName], { + reject: false, + }); + await validateBlueprintPolicyAuthorityReceipt(policyAuthorityReceipt, sandboxName, false); + const remove = await runCmd( + ["openshell", "sandbox", "remove", "-g", policyGateway.name, sandboxName], + { reject: false }, + ); + if (remove.exitCode !== 0 && !MISSING_SANDBOX_PATTERN.test(remove.stderr)) { + throw new Error( + `Failed to remove sandbox '${sandboxName}': ${boundedCommandError(remove.stderr)}`, + ); + } + sandboxCreatedByApply = false; + if (policyCreationTransition) { + policyAuthorityReceipt = undefined; + } persistRunPlan(); - } else { + } catch (cleanupError) { cleanupFailures.push( - `Failed to remove inference provider '${providerName}': ${boundedCommandError(removeProvider.stderr)}`, + `Sandbox ${JSON.stringify(sandboxName)} may still exist because its exact creation receipt could not be validated for cleanup: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`, ); } } @@ -1861,16 +2528,22 @@ export async function actionReconcile(rid: string): Promise { let plan: ReconciliationPlanSource; let transition: BlueprintPolicyTransitionReceipt; let additions: PolicyAdditions; + let authorityReceipt: BlueprintPolicyAuthorityReceipt; + let targetPolicy: UnknownRecord; try { const parsedPlan: unknown = JSON.parse(readFileSync(planFile, "utf-8")); if (!isPlainObject(parsedPlan)) { throw new Error("plan.json must contain a JSON object"); } plan = parsedPlan; - if (plan.sandbox_created_by_apply !== false) { - throw new Error("policy reconciliation requires a reused sandbox"); - } const sandboxName = readRollbackSandboxName(plan); + if (!isBlueprintPolicyAuthorityReceipt(plan.policy_authority)) { + throw new Error("policy authority receipt is invalid"); + } + authorityReceipt = plan.policy_authority; + if (authorityReceipt.authority !== "nemoclaw-managed") { + throw new Error("policy reconciliation requires a NemoClaw policy creation receipt"); + } if (!isBlueprintPolicyTransitionReceipt(plan.policy_transition)) { throw new Error("policy transition receipt is invalid"); } @@ -1878,6 +2551,14 @@ export async function actionReconcile(rid: string): Promise { if (transition.sandbox_name !== sandboxName) { throw new Error("policy transition sandbox does not match the run plan"); } + if ( + transition.gateway !== authorityReceipt.gateway || + transition.gateway_host !== authorityReceipt.gateway_host || + transition.gateway_port !== authorityReceipt.gateway_port || + transition.sandbox_name !== authorityReceipt.sandbox_name + ) { + throw new Error("policy transition boundary does not match the policy creation receipt"); + } if (!isPolicyAdditions(plan.policy_additions)) { throw new Error("policy additions are invalid"); } @@ -1890,30 +2571,61 @@ export async function actionReconcile(rid: string): Promise { ) { throw new Error("policy transition additions do not match the run plan"); } + targetPolicy = parseOpenShellPolicy( + readFileSync(join(stateDir, "merged-policy.yaml"), "utf-8"), + ).policy; + if (policyDigest(targetPolicy) !== transition.target_policy_digest) { + throw new Error("policy transition target does not match its durable digest"); + } } catch (error) { const detail = error instanceof Error ? error.message : String(error); throw new Error(`Cannot read reconciliation plan for run ${rid}: ${detail}`); } if (transition.status === "complete") { + const validated = await validateManagedPolicyReceipt(authorityReceipt.policy_creation_receipt, { + host: authorityReceipt.gateway_host, + port: authorityReceipt.gateway_port, + }); + if ( + !openShellPolicyValuesEqual( + policyForOwnershipProof(validated.inspection.effectivePolicy), + policyForOwnershipProof(targetPolicy), + ) + ) { + throw new Error("Cannot reconcile the blueprint policy transition: live policy changed."); + } log(`Policy transition for run ${rid} is already complete.`); return; } - const observed = await inspectBlueprintPolicyAuthority( - transition.gateway, - transition.sandbox_name, - ); - try { - assertMatchingPolicyAuthority(transition.expected_authority, observed.authority); - } catch (error) { - const detail = error instanceof Error ? error.message : "policy authority is invalid"; - throw new Error(`Cannot reconcile the blueprint policy transition: ${detail}.`); + const observed = await inspectReceiptSandboxBinding(authorityReceipt.policy_creation_receipt, { + host: authorityReceipt.gateway_host, + port: authorityReceipt.gateway_port, + }); + if ( + !openShellPolicyValuesEqual( + policyForOwnershipProof(observed.inspection.effectivePolicy), + policyForOwnershipProof(targetPolicy), + ) + ) { + throw new Error( + "Cannot reconcile the blueprint policy transition: OpenShell did not prove the exact intended policy.", + ); } - assertBlueprintPolicyRequirements(observed, additions); + assertBlueprintPolicyRequirements(managedInspection(observed.inspection), additions); + const rotatedReceipt: NemoClawPolicyCreationReceipt = { + ...observed.receipt, + policyHash: observed.inspection.policyIdentity.hash, + policyVersion: observed.inspection.policyIdentity.activeVersion, + }; persistRunReceipt(planFile, { ...plan, + policy_authority: { + ...authorityReceipt, + policy_creation_receipt: rotatedReceipt, + }, policy_transition: { ...transition, status: "complete" }, }); log(`Policy transition for run ${rid} is complete.`); @@ -1935,7 +2647,11 @@ export async function actionRollback(rid: string): Promise { let sandboxCreatedByApply = false; let inferenceProviderCreatedByApply = false; let inferenceProviderName: string | undefined; + let inferenceProviderBinding: + | { name: string; type: string; requiresEndpointConfig: boolean } + | undefined; let runtimeIdentityReceipt: RuntimeIdentityReceipt | undefined; + let policyAuthorityReceipt: BlueprintPolicyAuthorityReceipt | undefined; let policyTransition: BlueprintPolicyTransitionReceipt | undefined; try { const planData = readFileSync(planFile, "utf-8"); @@ -1948,7 +2664,8 @@ export async function actionRollback(rid: string): Promise { sandboxCreatedByApply = rollbackPlan?.sandbox_created_by_apply === true; inferenceProviderCreatedByApply = rollbackPlan?.inference_provider_created_by_apply === true; if (inferenceProviderCreatedByApply) { - inferenceProviderName = readRollbackInferenceProviderName(rollbackPlan!); + inferenceProviderBinding = readRollbackInferenceProviderBinding(rollbackPlan!); + inferenceProviderName = inferenceProviderBinding.name; } if (rollbackPlan?.identity !== undefined) { if (!isRuntimeIdentityReceipt(rollbackPlan.identity)) { @@ -1956,6 +2673,20 @@ export async function actionRollback(rid: string): Promise { } runtimeIdentityReceipt = rollbackPlan.identity; } + if (rollbackPlan?.policy_creation_transition !== undefined) { + if (!isBlueprintPolicyCreationTransition(rollbackPlan.policy_creation_transition)) { + throw new Error("policy creation transition is invalid"); + } + throw new Error( + "policy creation is incomplete, so sandbox ownership is unavailable for rollback", + ); + } + if (rollbackPlan?.policy_authority !== undefined) { + if (!isBlueprintPolicyAuthorityReceipt(rollbackPlan.policy_authority)) { + throw new Error("policy authority receipt is invalid"); + } + policyAuthorityReceipt = rollbackPlan.policy_authority; + } if (rollbackPlan?.policy_transition !== undefined) { if (!isBlueprintPolicyTransitionReceipt(rollbackPlan.policy_transition)) { throw new Error("policy transition receipt is invalid"); @@ -1967,25 +2698,79 @@ export async function actionRollback(rid: string): Promise { throw new Error(`Cannot read rollback plan for run ${rid}: ${detail}`); } - if (!sandboxCreatedByApply && policyTransition && policyTransition.status !== "complete") { + if (policyTransition && policyTransition.status !== "complete") { throw new Error( `Cannot roll back run ${rid}: the policy transition for reused sandbox ${JSON.stringify(policyTransition.sandbox_name)} through gateway ${JSON.stringify(policyTransition.gateway)} is ${policyTransition.status}. ${POLICY_TRANSITION_RECONCILIATION_ACTION}`, ); } + const requireRollbackPolicyBoundary = async (requireReady = true): Promise => { + try { + await validateBlueprintPolicyAuthorityReceipt( + policyAuthorityReceipt, + sandboxName, + requireReady, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Cannot roll back run ${rid}: ${detail}.`); + } + }; + + if ( + runtimeIdentityReceipt !== undefined || + sandboxCreatedByApply || + inferenceProviderCreatedByApply + ) { + await requireRollbackPolicyBoundary(); + } + if (runtimeIdentityReceipt) { progress(20, `Removing runtime identity provider ${runtimeIdentityReceipt.provider_name}`); - await removeRuntimeIdentity(runtimeIdentityReceipt, sandboxName, runtimeIdentityCommandDeps()); + await removeRuntimeIdentity( + runtimeIdentityReceipt, + sandboxName, + runtimeIdentityCommandDeps(requireRollbackPolicyBoundary), + ); + } + + if (inferenceProviderCreatedByApply) { + progress(40, `Removing inference provider ${inferenceProviderName!}`); + const providerState = await inspectOwnedInferenceProvider({ + ...inferenceProviderBinding!, + requiresCredential: false, + }); + if (providerState === "matching") { + await requireRollbackPolicyBoundary(); + const removeProvider = await runCmd( + ["openshell", "provider", "delete", inferenceProviderName!], + { reject: false }, + ); + if ( + removeProvider.exitCode !== 0 && + !MISSING_PROVIDER_INSPECTION_PATTERN.test(removeProvider.stderr) + ) { + throw new Error( + `Failed to remove owned inference provider '${inferenceProviderName!}': ${boundedCommandError(removeProvider.stderr)}`, + ); + } + } } if (sandboxCreatedByApply) { - progress(30, `Stopping sandbox ${sandboxName}`); - const stop = await runCmd(["openshell", "sandbox", "stop", sandboxName], { reject: false }); + progress(60, `Stopping sandbox ${sandboxName}`); + await requireRollbackPolicyBoundary(); + const stop = await runCmd( + ["openshell", "sandbox", "stop", "-g", policyAuthorityReceipt!.gateway, sandboxName], + { reject: false }, + ); - progress(60, `Removing sandbox ${sandboxName}`); - const remove = await runCmd(["openshell", "sandbox", "remove", sandboxName], { - reject: false, - }); + progress(70, `Removing sandbox ${sandboxName}`); + await requireRollbackPolicyBoundary(false); + const remove = await runCmd( + ["openshell", "sandbox", "remove", "-g", policyAuthorityReceipt!.gateway, sandboxName], + { reject: false }, + ); if (remove.exitCode !== 0 && !MISSING_SANDBOX_PATTERN.test(remove.stderr)) { const stopFailure = stop.exitCode !== 0 && !MISSING_SANDBOX_PATTERN.test(stop.stderr) @@ -1997,23 +2782,7 @@ export async function actionRollback(rid: string): Promise { ); } } else { - progress(60, `Preserving unowned sandbox ${sandboxName}`); - } - - if (inferenceProviderCreatedByApply) { - progress(80, `Removing inference provider ${inferenceProviderName!}`); - const removeProvider = await runCmd( - ["openshell", "provider", "delete", inferenceProviderName!], - { reject: false }, - ); - if ( - removeProvider.exitCode !== 0 && - !MISSING_PROVIDER_INSPECTION_PATTERN.test(removeProvider.stderr) - ) { - throw new Error( - `Failed to remove owned inference provider '${inferenceProviderName!}': ${boundedCommandError(removeProvider.stderr)}`, - ); - } + progress(70, `Preserving unowned sandbox ${sandboxName}`); } progress(90, "Cleaning up run state"); diff --git a/nemoclaw/src/blueprint/runtime-identity.test.ts b/nemoclaw/src/blueprint/runtime-identity.test.ts index 6988c45f2d7..87c42b7975a 100644 --- a/nemoclaw/src/blueprint/runtime-identity.test.ts +++ b/nemoclaw/src/blueprint/runtime-identity.test.ts @@ -62,10 +62,7 @@ const matchingProviderResult: RuntimeIdentityCommandResult = { }; const configuredProviderResult: RuntimeIdentityCommandResult = { exitCode: 0, - stdout: matchingProvider.replace( - "Credential keys: OKTA_ACCESS_TOKEN", - "Credential keys: ", - ), + stdout: matchingProvider.replace("Credential keys: OKTA_ACCESS_TOKEN", "Credential keys: "), stderr: "", }; const configuredRefreshResult: RuntimeIdentityCommandResult = { @@ -300,15 +297,14 @@ describe("runtime identity contract", () => { expect(resolveRuntimeIdentityProfilePath(config.profile_path, root)).toBe(profilePath); }); - it.each([ - "/absolute-profile.yaml", - "../outside-profile.yaml", - "missing-profile.yaml", - ])("rejects an unsafe or missing profile path: %s", (candidate) => { - expect(() => resolveRuntimeIdentityProfilePath(candidate, root)).toThrow( - /must (?:be relative|stay inside|name an existing file)/, - ); - }); + it.each(["/absolute-profile.yaml", "../outside-profile.yaml", "missing-profile.yaml"])( + "rejects an unsafe or missing profile path: %s", + (candidate) => { + expect(() => resolveRuntimeIdentityProfilePath(candidate, root)).toThrow( + /must (?:be relative|stay inside|name an existing file)/, + ); + }, + ); it("rejects a directory and an outward symlink as profiles", () => { mkdirSync(join(root, "provider-profiles", "directory")); @@ -555,15 +551,18 @@ describe("runtime identity contract", () => { entraProfileDocument.replace("login.microsoftonline.com", "graph.microsoft.com"), /refresh token_url host 'graph\.microsoft\.com' is outside/, ], - ])("rejects an Entra profile that changes the reviewed %s", async (_boundary, profile, message) => { - writeFileSync(join(root, entraConfig.profile_path), profile); - environment.ENTRA_CLIENT_ID = "entra-client-id"; - environment.ENTRA_REFRESH_TOKEN = "entra-refresh-secret"; - environment.ENTRA_CLIENT_SECRET = "entra-client-secret"; - - await expect(prepareRuntimeIdentity(entraConfig, deps)).rejects.toThrow(message); - expect(calls).toEqual([]); - }); + ])( + "rejects an Entra profile that changes the reviewed %s", + async (_boundary, profile, message) => { + writeFileSync(join(root, entraConfig.profile_path), profile); + environment.ENTRA_CLIENT_ID = "entra-client-id"; + environment.ENTRA_REFRESH_TOKEN = "entra-refresh-secret"; + environment.ENTRA_CLIENT_SECRET = "entra-client-secret"; + + await expect(prepareRuntimeIdentity(entraConfig, deps)).rejects.toThrow(message); + expect(calls).toEqual([]); + }, + ); it("rejects DNS-backed destinations unless the reviewed profile policy owns DNS", async () => { deps.validateEndpointUrl = async () => ({ dnsResolved: true }); @@ -868,6 +867,20 @@ describe("runtime identity contract", () => { ); }); + it("preserves a created provider when its ownership receipt is not durable (#9833)", async () => { + responses.set("provider get acme-okta-runtime", [missingProvider]); + deps.persistReceipt = () => { + throw new Error("receipt persistence failed"); + }; + + await expect(prepareRuntimeIdentity(config, deps)).rejects.toThrow( + "receipt persistence failed", + ); + const commands = calls.map(({ args }) => commandKey(args)); + expect(commands).not.toContain("provider delete acme-okta-runtime"); + expect(commands.join("\n")).not.toContain("provider refresh configure"); + }); + it("redacts client ID and secret material from refresh configuration errors", async () => { responses.set("provider get acme-okta-runtime", [missingProvider, matchingProviderResult]); responses.set( @@ -961,10 +974,9 @@ describe("runtime identity contract", () => { it("attaches a provider whose refresh is configured before its first mint", async () => { responses.set("provider get acme-okta-runtime", [configuredProviderResult]); - responses.set( - "provider refresh status acme-okta-runtime --credential-key OKTA_ACCESS_TOKEN", - [configuredRefreshResult], - ); + responses.set("provider refresh status acme-okta-runtime --credential-key OKTA_ACCESS_TOKEN", [ + configuredRefreshResult, + ]); await expect(attachRuntimeIdentity(createdReceipt, "sandbox", deps)).resolves.toBe(true); expect(calls.map(({ args }) => commandKey(args))).toEqual([ diff --git a/nemoclaw/src/blueprint/runtime-identity.ts b/nemoclaw/src/blueprint/runtime-identity.ts index dee37c1a6d3..5a9f5b43918 100644 --- a/nemoclaw/src/blueprint/runtime-identity.ts +++ b/nemoclaw/src/blueprint/runtime-identity.ts @@ -985,8 +985,11 @@ export async function prepareRuntimeIdentity( `Failed to create runtime identity provider '${config.provider_name}': ${deps.formatError(commandOutput(providerCreate))}`, ); } - providerAcquired = true; deps.persistReceipt(receipt); + // Cleanup authority begins only after the ownership receipt is durable. If + // persistence fails, preserve the provider instead of deleting a resource + // that the next process cannot prove this run created. + providerAcquired = true; const refreshArgs = [ "openshell", diff --git a/nemoclaw/src/shared/openshell-gateway-endpoint-boundary.cts b/nemoclaw/src/shared/openshell-gateway-endpoint-boundary.cts new file mode 100644 index 00000000000..f4a7a62a834 --- /dev/null +++ b/nemoclaw/src/shared/openshell-gateway-endpoint-boundary.cts @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const ANSI_RE = /\x1b\[[0-9;]*m/gu; +const ENDPOINT_RE = /^\s*(?:Gateway endpoint|Server):(.*)$/gimu; + +export type ManagedGatewayEndpointBinding = "match" | "mismatch" | "not-applicable" | "unknown"; + +export interface ManagedGatewayEndpoint { + host: "127.0.0.1" | "localhost" | "[::1]"; + port: number; +} + +function stripAnsi(value: string): string { + return value.replace(ANSI_RE, ""); +} + +export function isManagedGatewayEndpointHost( + value: unknown, +): value is ManagedGatewayEndpoint["host"] { + return value === "127.0.0.1" || value === "localhost" || value === "[::1]"; +} + +function parseLoopbackEndpoint(value: string): URL | null { + if (!value || /\s/u.test(value)) return null; + let endpoint: URL; + try { + endpoint = new URL(value); + } catch { + return null; + } + const localProtocol = endpoint.protocol === "https:" || endpoint.protocol === "http:"; + const localHost = isManagedGatewayEndpointHost(endpoint.hostname); + if ( + !localProtocol || + !localHost || + endpoint.username !== "" || + endpoint.password !== "" || + endpoint.pathname !== "/" || + endpoint.search !== "" || + endpoint.hash !== "" + ) { + return null; + } + return endpoint; +} + +function endpointPort(endpoint: URL): number | null { + const value = endpoint.port || (endpoint.protocol === "https:" ? "443" : "80"); + const port = Number(value); + return Number.isSafeInteger(port) && port > 0 && port <= 65_535 ? port : null; +} + +export function classifyManagedGatewayEndpointBinding( + outputs: readonly string[], + expectedGatewayPort: number, +): Exclude { + let observedEndpoint = false; + for (const output of outputs) { + for (const match of stripAnsi(output).matchAll(ENDPOINT_RE)) { + observedEndpoint = true; + const endpoint = parseLoopbackEndpoint(match[1]?.trim() ?? ""); + if (!endpoint || endpointPort(endpoint) !== expectedGatewayPort) return "mismatch"; + } + } + return observedEndpoint ? "match" : "unknown"; +} + +export function parseSingleManagedGatewayEndpoint(output: string): ManagedGatewayEndpoint { + const endpointValues = [...stripAnsi(output).matchAll(ENDPOINT_RE)].map( + (match) => match[1]?.trim() ?? "", + ); + if (endpointValues.length !== 1) { + throw new Error("OpenShell gateway info did not report one gateway endpoint"); + } + let parsed: URL; + try { + parsed = new URL(endpointValues[0]); + } catch { + throw new Error("OpenShell gateway info reported an invalid gateway endpoint"); + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new Error("OpenShell gateway info reported an unsupported gateway endpoint protocol"); + } + const endpoint = parseLoopbackEndpoint(endpointValues[0]); + if (!endpoint) { + throw new Error("OpenShell gateway info reported an unsupported local gateway endpoint"); + } + const port = endpointPort(endpoint); + if (port === null) { + throw new Error("OpenShell gateway info reported an invalid gateway port"); + } + return { + host: endpoint.hostname as ManagedGatewayEndpoint["host"], + port, + }; +} + +export function parseSingleManagedGatewayEndpointPort(output: string): number { + return parseSingleManagedGatewayEndpoint(output).port; +} diff --git a/nemoclaw/src/shared/openshell-gateway-endpoint-boundary.test.ts b/nemoclaw/src/shared/openshell-gateway-endpoint-boundary.test.ts new file mode 100644 index 00000000000..2154178703a --- /dev/null +++ b/nemoclaw/src/shared/openshell-gateway-endpoint-boundary.test.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + classifyManagedGatewayEndpointBinding, + parseSingleManagedGatewayEndpoint, + parseSingleManagedGatewayEndpointPort, +} from "./openshell-gateway-endpoint-boundary.cjs"; + +describe("OpenShell managed gateway endpoint boundary", () => { + it.each([ + ["http://127.0.0.1:8080", 8080], + ["https://localhost", 443], + ["http://[::1]:9090", 9090], + ])("accepts one credential-free loopback endpoint %s (#9833)", (endpoint, port) => { + const output = `Gateway endpoint: ${endpoint}\n`; + expect(parseSingleManagedGatewayEndpointPort(output)).toBe(port); + expect(parseSingleManagedGatewayEndpoint(output).port).toBe(port); + expect(classifyManagedGatewayEndpointBinding([output], port)).toBe("match"); + }); + + it("returns the normalized loopback host for durable receipt binding (#9833)", () => { + expect(parseSingleManagedGatewayEndpoint("Gateway endpoint: HTTP://LOCALHOST:8080\n")).toEqual({ + host: "localhost", + port: 8080, + }); + }); + + it.each([ + "http://192.0.2.10:8080", + "http://gateway.example.com:8080", + "http://user:password@127.0.0.1:8080", + "http://127.0.0.1:8080/path", + ])("rejects substituted gateway endpoint %s (#9833)", (endpoint) => { + const output = `Gateway endpoint: ${endpoint}\n`; + expect(() => parseSingleManagedGatewayEndpointPort(output)).toThrow( + "unsupported local gateway endpoint", + ); + expect(classifyManagedGatewayEndpointBinding([output], 8080)).toBe("mismatch"); + }); + + it("rejects ambiguous gateway endpoint output (#9833)", () => { + const output = [ + "Gateway endpoint: http://127.0.0.1:8080", + "Server: http://127.0.0.1:8080", + ].join("\n"); + expect(() => parseSingleManagedGatewayEndpointPort(output)).toThrow( + "did not report one gateway endpoint", + ); + }); + + it.each(["", "not-a-url"])( + "rejects an empty or malformed gateway endpoint %j (#9833)", + (endpoint) => { + const output = `Gateway endpoint: ${endpoint}\n`; + expect(() => parseSingleManagedGatewayEndpointPort(output)).toThrow( + "invalid gateway endpoint", + ); + expect(classifyManagedGatewayEndpointBinding([output], 8080)).toBe("mismatch"); + }, + ); +}); diff --git a/nemoclaw/src/shared/openshell-policy-boundary.cts b/nemoclaw/src/shared/openshell-policy-boundary.cts index d752f615c27..2dbde34e940 100644 --- a/nemoclaw/src/shared/openshell-policy-boundary.cts +++ b/nemoclaw/src/shared/openshell-policy-boundary.cts @@ -15,13 +15,42 @@ export interface ParsedOpenShellPolicy { readonly policy: ValidatedOpenShellPolicyMapping; } -export type OpenShellPolicyAuthority = "nemoclaw-managed" | "externally-managed"; +export type OpenShellPolicyAuthority = "nemoclaw-managed" | "externally-managed" | "owner-unknown"; + +export interface OpenShellPolicyIdentity { + readonly hash: string; + readonly activeVersion: number; +} + +/** Secret-free proof that NemoClaw created and verified one sandbox policy. */ +export interface NemoClawPolicyCreationReceipt { + readonly schemaVersion: 1; + readonly origin: "sandbox-create"; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly sandboxName: string; + readonly lifecycleGeneration: string; + readonly sandboxIdentityFingerprint: string; + readonly policyHash: string; + readonly policyVersion: number; +} export interface SandboxPolicyAuthorityInspection { readonly authority: OpenShellPolicyAuthority; readonly effectivePolicy: OpenShellPolicyMapping; + readonly policyIdentity: OpenShellPolicyIdentity; } +/** Result of checking whether a verified active global policy exists. */ +export type ActiveGlobalPolicyInspection = + | { readonly state: "absent" } + | { + readonly state: "active"; + readonly inspection: SandboxPolicyAuthorityInspection & { + readonly authority: "externally-managed"; + }; + }; + const MISSING_POLICY_DOCUMENT = "Current policy from openshell policy get --base does not contain a policy YAML document"; @@ -30,7 +59,96 @@ function isMapping(value: unknown): value is OpenShellPolicyMapping { } function isPolicyAuthority(value: unknown): value is OpenShellPolicyAuthority { - return value === "nemoclaw-managed" || value === "externally-managed"; + return ( + value === "nemoclaw-managed" || value === "externally-managed" || value === "owner-unknown" + ); +} + +const RECEIPT_KEYS = new Set([ + "schemaVersion", + "origin", + "gatewayName", + "gatewayPort", + "sandboxName", + "lifecycleGeneration", + "sandboxIdentityFingerprint", + "policyHash", + "policyVersion", +]); +const RECEIPT_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; +const RECEIPT_POLICY_HASH_PATTERN = /^[A-Za-z0-9][A-Za-z0-9:._-]{0,255}$/u; +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu; + +function positiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0; +} + +function parsePolicyIdentity(metadata: OpenShellPolicyMapping): OpenShellPolicyIdentity { + if ( + typeof metadata.hash !== "string" || + !RECEIPT_POLICY_HASH_PATTERN.test(metadata.hash) || + !positiveInteger(metadata.active_version) + ) { + throw new Error("OpenShell returned invalid sandbox policy identity metadata"); + } + return { hash: metadata.hash, activeVersion: metadata.active_version }; +} + +/** Parse a complete receipt. Pending, extended, or malformed values fail closed. */ +export function parseNemoClawPolicyCreationReceipt(value: unknown): NemoClawPolicyCreationReceipt { + if ( + !isMapping(value) || + Object.keys(value).some((key) => !RECEIPT_KEYS.has(key)) || + value.schemaVersion !== 1 || + value.origin !== "sandbox-create" || + typeof value.gatewayName !== "string" || + !RECEIPT_NAME_PATTERN.test(value.gatewayName) || + !positiveInteger(value.gatewayPort) || + value.gatewayPort > 65_535 || + typeof value.sandboxName !== "string" || + !RECEIPT_NAME_PATTERN.test(value.sandboxName) || + typeof value.lifecycleGeneration !== "string" || + !UUID_PATTERN.test(value.lifecycleGeneration) || + typeof value.sandboxIdentityFingerprint !== "string" || + !SHA256_PATTERN.test(value.sandboxIdentityFingerprint) || + typeof value.policyHash !== "string" || + !RECEIPT_POLICY_HASH_PATTERN.test(value.policyHash) || + !positiveInteger(value.policyVersion) + ) { + throw new Error("NemoClaw policy creation receipt is unavailable or invalid"); + } + return { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: value.gatewayName, + gatewayPort: value.gatewayPort, + sandboxName: value.sandboxName, + lifecycleGeneration: value.lifecycleGeneration, + sandboxIdentityFingerprint: value.sandboxIdentityFingerprint, + policyHash: value.policyHash, + policyVersion: value.policyVersion, + }; +} + +/** Require a receipt to describe the exact live policy boundary being used. */ +export function assertNemoClawPolicyCreationReceiptMatches( + value: unknown, + expected: Omit, +): NemoClawPolicyCreationReceipt { + const receipt = parseNemoClawPolicyCreationReceipt(value); + if ( + receipt.gatewayName !== expected.gatewayName || + receipt.gatewayPort !== expected.gatewayPort || + receipt.sandboxName !== expected.sandboxName || + receipt.lifecycleGeneration !== expected.lifecycleGeneration || + receipt.sandboxIdentityFingerprint !== expected.sandboxIdentityFingerprint || + receipt.policyHash !== expected.policyHash || + receipt.policyVersion !== expected.policyVersion + ) { + throw new Error("NemoClaw policy creation receipt does not match the live sandbox policy"); + } + return receipt; } function parseJsonMapping(source: string, invalidMessage: string): OpenShellPolicyMapping { @@ -68,15 +186,16 @@ export function parseSandboxPolicyAuthorityMetadata( throw new Error("OpenShell returned invalid sandbox policy authority metadata"); } return { - authority: metadata.policy_source === "sandbox" ? "nemoclaw-managed" : "externally-managed", + authority: metadata.policy_source === "sandbox" ? "owner-unknown" : "externally-managed", effectivePolicy: metadata.policy, + policyIdentity: parsePolicyIdentity(metadata), }; } -/** Parse machine-readable global policy metadata after history proves a revision exists. */ -export function parseGlobalPolicyAuthorityMetadata( +/** Parse one global policy revision without treating absence as NemoClaw ownership. */ +export function parseActiveGlobalPolicyAuthorityMetadata( raw: string, -): SandboxPolicyAuthorityInspection { +): ActiveGlobalPolicyInspection { if (raw.trim().length === 0) { throw new Error("OpenShell returned empty global policy authority metadata"); } @@ -86,26 +205,32 @@ export function parseGlobalPolicyAuthorityMetadata( ); if ( metadata.scope !== "global" || - Object.hasOwn(metadata, "sandbox") || - metadata.policy_source !== "global" + (metadata.status !== "loaded" && metadata.status !== "superseded") || + metadata.policy_source !== "global" || + Object.hasOwn(metadata, "sandbox") ) { throw new Error("OpenShell returned invalid global policy authority metadata"); } - if (metadata.status === "superseded") { - return { authority: "nemoclaw-managed", effectivePolicy: {} }; - } - if (metadata.status !== "loaded" || !isMapping(metadata.policy)) { + if (metadata.status === "superseded") return { state: "absent" }; + if (!isMapping(metadata.policy)) { throw new Error("OpenShell returned invalid global policy authority metadata"); } - return { authority: "externally-managed", effectivePolicy: metadata.policy }; + return { + state: "active", + inspection: { + authority: "externally-managed", + effectivePolicy: metadata.policy, + policyIdentity: parsePolicyIdentity(metadata), + }, + }; } /** Require durable and observed policy authority to describe the same owner. */ export function assertMatchingPolicyAuthority(recorded: unknown, observed: unknown): void { - if (!isPolicyAuthority(recorded)) { + if (!isPolicyAuthority(recorded) || recorded === "owner-unknown") { throw new Error("the recorded policy authority is unavailable or invalid"); } - if (!isPolicyAuthority(observed)) { + if (!isPolicyAuthority(observed) || observed === "owner-unknown") { throw new Error("the observed OpenShell policy authority is unavailable or invalid"); } if (recorded !== observed) { @@ -140,6 +265,11 @@ function policyValuesEqual(left: unknown, right: unknown): boolean { ); } +/** Compare two parsed policy values without serializing their contents. */ +export function openShellPolicyValuesEqual(left: unknown, right: unknown): boolean { + return policyValuesEqual(left, right); +} + function formatPolicyKeys(keys: readonly string[]): string { return keys.map((key) => JSON.stringify(key)).join(", "); } @@ -225,6 +355,9 @@ export function assertExternalPolicyRequirementContainment( if (!isPolicyAuthority(inspection.authority)) { throw new Error("the observed OpenShell policy authority is invalid"); } + if (inspection.authority === "owner-unknown") { + throw new Error("the observed OpenShell policy authority is unknown"); + } if (inspection.authority === "nemoclaw-managed") return; assertPolicyRequirementContainmentForOwner( inspection, diff --git a/nemoclaw/src/shared/openshell-policy-boundary.test.ts b/nemoclaw/src/shared/openshell-policy-boundary.test.ts index 331267970af..30568d1c2ed 100644 --- a/nemoclaw/src/shared/openshell-policy-boundary.test.ts +++ b/nemoclaw/src/shared/openshell-policy-boundary.test.ts @@ -7,8 +7,10 @@ import YAML from "yaml"; import { assertExternalPolicyRequirementContainment, assertMatchingPolicyAuthority, + assertNemoClawPolicyCreationReceiptMatches, assertPolicyRequirementContainment, - parseGlobalPolicyAuthorityMetadata, + parseActiveGlobalPolicyAuthorityMetadata, + parseNemoClawPolicyCreationReceipt, parseOpenShellPolicy, parseSandboxPolicyAuthorityMetadata, stripProviderComposedPolicies, @@ -88,42 +90,127 @@ const POLICY_CASES = [ describe("sandbox policy authority boundary", () => { const policy = { version: 1, network_policies: { required: { allow: true } } }; + const metadata = (policySource: "sandbox" | "global", sandbox = "alpha") => + JSON.stringify({ + scope: "sandbox", + sandbox, + status: "effective", + policy_source: policySource, + active_version: 7, + hash: "sha256:effective", + policy, + }); it.each([ - ["sandbox", "nemoclaw-managed"], + ["sandbox", "owner-unknown"], ["global", "externally-managed"], ] as const)("classifies the %s policy source as %s", (policySource, authority) => { + expect(parseSandboxPolicyAuthorityMetadata(metadata(policySource), "alpha")).toEqual({ + authority, + effectivePolicy: policy, + policyIdentity: { activeVersion: 7, hash: "sha256:effective" }, + }); + }); + + it.each([ + ["empty", " \n\t", /empty sandbox policy authority metadata/u], + ["malformed", "{", /malformed sandbox policy authority metadata/u], + ["non-object", "[]", /malformed sandbox policy authority metadata/u], + ["mismatched", metadata("sandbox", "beta"), /invalid sandbox policy authority metadata/u], + ])("rejects %s sandbox authority metadata", (_caseName, raw, expected) => { + expect(() => parseSandboxPolicyAuthorityMetadata(raw, "alpha")).toThrow(expected); + }); + + it("accepts a loaded global policy as active external authority (#9833)", () => { expect( - parseSandboxPolicyAuthorityMetadata( + parseActiveGlobalPolicyAuthorityMetadata( JSON.stringify({ - scope: "sandbox", - sandbox: "alpha", - status: "effective", - policy_source: policySource, + scope: "global", + status: "loaded", + policy_source: "global", + active_version: 9, + hash: "sha256:global", policy, }), - "alpha", ), - ).toEqual({ authority, effectivePolicy: policy }); + ).toEqual({ + state: "active", + inspection: { + authority: "externally-managed", + effectivePolicy: policy, + policyIdentity: { activeVersion: 9, hash: "sha256:global" }, + }, + }); + }); + + it("treats a superseded global revision as absent without requiring an identity (#9833)", () => { + expect( + parseActiveGlobalPolicyAuthorityMetadata( + JSON.stringify({ + scope: "global", + status: "superseded", + policy_source: "global", + }), + ), + ).toEqual({ state: "absent" }); }); it.each([ - ["empty", " \n\t", /empty sandbox policy authority metadata/u], - ["malformed", "{", /malformed sandbox policy authority metadata/u], - ["non-object", "[]", /malformed sandbox policy authority metadata/u], + ["empty output", " \n\t", /empty global policy authority metadata/u], + ["malformed JSON", "{", /malformed global policy authority metadata/u], + ["non-object JSON", "[]", /malformed global policy authority metadata/u], + [ + "invalid authority metadata", + JSON.stringify({ scope: "sandbox", status: "loaded", policy_source: "global" }), + /invalid global policy authority metadata/u, + ], + [ + "sandbox-scoped global metadata", + JSON.stringify({ + scope: "global", + status: "loaded", + policy_source: "global", + sandbox: "alpha", + }), + /invalid global policy authority metadata/u, + ], [ - "mismatched", + "missing loaded policy", JSON.stringify({ - scope: "sandbox", - sandbox: "beta", - status: "effective", - policy_source: "sandbox", + scope: "global", + status: "loaded", + policy_source: "global", + active_version: 9, + hash: "sha256:global", + }), + /invalid global policy authority metadata/u, + ], + [ + "empty policy identity", + JSON.stringify({ + scope: "global", + status: "loaded", + policy_source: "global", + active_version: 9, + hash: "", policy, }), - /invalid sandbox policy authority metadata/u, + /invalid sandbox policy identity metadata/u, ], - ])("rejects %s sandbox authority metadata", (_caseName, raw, expected) => { - expect(() => parseSandboxPolicyAuthorityMetadata(raw, "alpha")).toThrow(expected); + [ + "malformed policy identity", + JSON.stringify({ + scope: "global", + status: "loaded", + policy_source: "global", + active_version: "9", + hash: "sha256:global", + policy, + }), + /invalid sandbox policy identity metadata/u, + ], + ])("rejects %s for global policy authority inspection (#9833)", (_name, raw, expected) => { + expect(() => parseActiveGlobalPolicyAuthorityMetadata(raw)).toThrow(expected); }); it("accepts matching authority and rejects invalid or changed authority", () => { @@ -144,6 +231,7 @@ describe("sandbox policy authority boundary", () => { it("requires external entries and sections while allowing unrelated content", () => { const inspection = { authority: "externally-managed" as const, + policyIdentity: { activeVersion: 7, hash: "sha256:effective" }, effectivePolicy: { version: 9, filesystem_policy: { read_only: true }, @@ -169,7 +257,11 @@ describe("sandbox policy authority boundary", () => { ); expect(() => assertExternalPolicyRequirementContainment( - { authority: "unknown" as never, effectivePolicy: {} }, + { + authority: "unknown" as never, + effectivePolicy: {}, + policyIdentity: { activeVersion: 7, hash: "sha256:effective" }, + }, {}, ), ).toThrow(/observed OpenShell policy authority is invalid/u); @@ -184,6 +276,7 @@ describe("sandbox policy authority boundary", () => { const inspection = { authority: "nemoclaw-managed" as const, effectivePolicy: { network_policies: { required: { allow: true } } }, + policyIdentity: { activeVersion: 7, hash: "sha256:effective" }, }; expect(() => assertPolicyRequirementContainment(inspection, { @@ -195,43 +288,44 @@ describe("sandbox policy authority boundary", () => { network_policies: { missing: { allow: true } }, }), ).toThrow(/missing entries "missing"/u); - }); -}); -describe("global policy authority boundary", () => { - const policy = { version: 1, network_policies: { required: { allow: true } } }; - - it("classifies a loaded global policy as externally managed", () => { - expect( - parseGlobalPolicyAuthorityMetadata( - JSON.stringify({ - scope: "global", - status: "loaded", - policy_source: "global", - policy, - }), + expect(() => + assertPolicyRequirementContainment( + { ...inspection, authority: "invalid" as never }, + { network_policies: {} }, ), - ).toEqual({ authority: "externally-managed", effectivePolicy: policy }); - }); - - it("classifies a superseded global revision as NemoClaw-managed", () => { - expect( - parseGlobalPolicyAuthorityMetadata( - JSON.stringify({ scope: "global", status: "superseded", policy_source: "global" }), + ).toThrow(/observed OpenShell policy authority is invalid/u); + expect(() => + assertExternalPolicyRequirementContainment( + { ...inspection, authority: "owner-unknown" }, + { network_policies: {} }, ), - ).toEqual({ authority: "nemoclaw-managed", effectivePolicy: {} }); + ).toThrow(/observed OpenShell policy authority is unknown/u); }); - it.each([ - ["empty", ""], - ["malformed", "{"], - ["wrong scope", JSON.stringify({ scope: "sandbox", status: "loaded", policy })], - ["sandbox identity", JSON.stringify({ scope: "global", sandbox: "alpha", status: "loaded", policy })], - ["missing source", JSON.stringify({ scope: "global", status: "loaded", policy })], - ["unknown source", JSON.stringify({ scope: "global", status: "loaded", policy_source: "sandbox", policy })], - ["unknown status", JSON.stringify({ scope: "global", status: "pending", policy })], - ])("rejects %s global authority metadata", (_caseName, raw) => { - expect(() => parseGlobalPolicyAuthorityMetadata(raw)).toThrow(/global policy authority metadata/u); + it("accepts only a complete secret-free receipt for the exact live policy", () => { + const receipt = { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "alpha", + lifecycleGeneration: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + sandboxIdentityFingerprint: "b".repeat(64), + policyHash: "sha256:effective", + policyVersion: 7, + }; + expect(parseNemoClawPolicyCreationReceipt(receipt)).toEqual(receipt); + expect(() => + assertNemoClawPolicyCreationReceiptMatches(receipt, { + ...receipt, + policyHash: "sha256:drifted", + }), + ).toThrow(/does not match the live sandbox policy/u); + expect(() => parseNemoClawPolicyCreationReceipt({ ...receipt, status: "pending" })).toThrow( + /unavailable or invalid/u, + ); + expect(JSON.stringify(receipt)).not.toMatch(/credential|network_policies|secret/u); }); }); diff --git a/nemoclaw/tsconfig.shared.json b/nemoclaw/tsconfig.shared.json index 7183ad50979..682ecb15493 100644 --- a/nemoclaw/tsconfig.shared.json +++ b/nemoclaw/tsconfig.shared.json @@ -7,6 +7,7 @@ "include": [ "src/shared/banner-boundary.cts", "src/shared/credential-filter-boundary.cts", + "src/shared/openshell-gateway-endpoint-boundary.cts", "src/shared/openshell-external-target-boundary.cts", "src/shared/openshell-policy-boundary.cts", "src/shared/private-networks-boundary.cts", diff --git a/scripts/checks/openshell-policy-mutation-read.mts b/scripts/checks/openshell-policy-mutation-read.mts index 1288daaf967..c539fe5b6cc 100644 --- a/scripts/checks/openshell-policy-mutation-read.mts +++ b/scripts/checks/openshell-policy-mutation-read.mts @@ -99,8 +99,8 @@ const NON_MUTATION_POLICY_READS: readonly AuditedPolicyReadFile[] = [ { relativePath: "src/lib/adapters/openshell/policy-authority.ts", expectedReads: [ + unclassifiedBase("captureSandboxBasePolicy"), unclassifiedFull("inspectSandboxPolicyAuthority"), - unclassifiedFull("inspectGlobalPolicyAuthority"), ], }, { @@ -117,8 +117,9 @@ const NON_MUTATION_POLICY_READS: readonly AuditedPolicyReadFile[] = [ { relativePath: "src/lib/policy/commands.ts", expectedReads: [ + unclassifiedFull("buildGlobalPolicyGetFullJsonArgs"), { - site: "buildPolicyGetCommand", + site: "buildPolicyGetArgs", view: "base", failureHandling: "unclassified", }, @@ -127,8 +128,7 @@ const NON_MUTATION_POLICY_READS: readonly AuditedPolicyReadFile[] = [ view: "full", failureHandling: "unclassified", }, - unclassifiedFull("buildPolicyGetFullJsonCommand"), - unclassifiedFull("buildGlobalPolicyGetFullJsonCommand"), + unclassifiedFull("buildPolicyGetFullJsonArgs"), ], }, ] as const; @@ -141,9 +141,10 @@ export interface DiscoveredPolicyReadSite { const POLICY_GET_BUILDERS = new Map([ ["buildPolicyGetCommand", "base"], + ["buildPolicyGetArgs", "base"], ["buildPolicyGetFullCommand", "full"], ["buildPolicyGetFullJsonCommand", "full"], - ["buildGlobalPolicyGetFullJsonCommand", "full"], + ["buildPolicyGetFullJsonArgs", "full"], ]); interface PolicyBuilderBindings { diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 7716eda8c50..1175d79883f 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -67,6 +67,7 @@ import { const MANAGED_AGENTS = new Set(SHIPPED_MANAGED_IMAGE_AGENTS); const MODEL = "nvidia/nemotron-3-ultra-550b-a55b"; +const GATEWAY_NAME = "nemoclaw"; const GATEWAY_PORT = 8080; const IMMUTABLE_MANIFEST_REFERENCE_RE = /^([^\s@]+)@(sha256:[a-f0-9]{64})$/u; const MANAGED_AGENT_BASE_POLICIES: Record = { @@ -904,7 +905,7 @@ async function run { type IdentityContinuity = | { status: "match" } - | { status: "changed" } - | { status: "ambiguous"; detail: string } - | { status: "probe-failed"; detail: string }; + | { status: "changed"; subject?: string } + | { status: "ambiguous"; detail: string; subject?: string } + | { status: "probe-failed"; detail: string; subject?: string }; + const pendingPolicyVerification = sandbox?.pendingPolicyVerification; + const inspectPendingPolicyVerificationContinuity = (): IdentityContinuity => { + if (!pendingPolicyVerification) return { status: "match" }; + if (!getSandbox) { + return { + status: "probe-failed", + subject: "Pending policy verification sandbox identity", + detail: "an exact registry reader is unavailable", + }; + } + const readCurrentCheckpoint = () => getSandbox(sandboxName)?.pendingPolicyVerification; + try { + if (!isDeepStrictEqual(readCurrentCheckpoint(), pendingPolicyVerification)) { + return { status: "changed", subject: "Pending policy verification authority" }; + } + const inspectIdentity = + deps.inspectOpenShellSandboxIdentityFingerprint ?? + inspectOpenShellSandboxIdentityFingerprint; + const liveFingerprint = inspectIdentity({ + sandboxName, + gatewayName: pendingPolicyVerification.gatewayName, + }); + if ( + liveFingerprint !== pendingPolicyVerification.sandboxIdentityFingerprint || + !isDeepStrictEqual(readCurrentCheckpoint(), pendingPolicyVerification) + ) { + return { + status: "changed", + subject: "Pending policy verification sandbox identity", + }; + } + return { status: "match" }; + } catch (error) { + return { + status: "probe-failed", + subject: "Pending policy verification sandbox identity", + detail: redactDestroyError(error), + }; + } + }; const inspectIdentityContinuity = (): IdentityContinuity => { + const pendingContinuity = inspectPendingPolicyVerificationContinuity(); + if (pendingContinuity.status !== "match") return pendingContinuity; if (portableContainerAuthority) { try { portableContainerAuthority.revalidate(); @@ -334,21 +380,24 @@ export async function executeSandboxDestroy({ continuity: Exclude, mcpRecoveryFailure?: string, earlierCleanupDetail = "", - ): SandboxDestroyExecutionResult => ({ - ok: false, - deleteOutput: - continuity.status === "probe-failed" - ? `Container identity could not be inspected ${phase}: ${continuity.detail}. No sandbox delete was attempted.${earlierCleanupDetail}` - : continuity.status === "ambiguous" - ? `Container identity became ambiguous ${phase}: ${continuity.detail}. No sandbox delete was attempted.${earlierCleanupDetail}` - : `Container identity changed ${phase}; no sandbox delete was attempted.${earlierCleanupDetail}`, - exitCode: 1, - gatewayUnreachable: false, - hostLocalInferenceOwnershipRequiresGateway: false, - mcpOwnershipRequiresGateway: false, - mcpRecoveryFailure, - shieldsRelockRequiresGateway: false, - }); + ): SandboxDestroyExecutionResult => { + const subject = continuity.subject ?? "Container identity"; + return { + ok: false, + deleteOutput: + continuity.status === "probe-failed" + ? `${subject} could not be inspected ${phase}: ${continuity.detail}. No sandbox delete was attempted.${earlierCleanupDetail}` + : continuity.status === "ambiguous" + ? `${subject} became ambiguous ${phase}: ${continuity.detail}. No sandbox delete was attempted.${earlierCleanupDetail}` + : `${subject} changed ${phase}; no sandbox delete was attempted.${earlierCleanupDetail}`, + exitCode: 1, + gatewayUnreachable: false, + hostLocalInferenceOwnershipRequiresGateway: false, + mcpOwnershipRequiresGateway: false, + mcpRecoveryFailure, + shieldsRelockRequiresGateway: false, + }; + }; const initialContinuity = inspectIdentityContinuity(); if (initialContinuity.status !== "match") { return identityRefusalResult("before destroy preparation", initialContinuity); @@ -517,7 +566,10 @@ export async function executeSandboxDestroy({ ` Managed inference cleanup and workspace wipe or hardening may already have run; inspect those resources before retrying.${detachedDetail}`, ); } - const deleteResult = runOpenshell(["sandbox", "delete", sandboxName], { + const deleteArgs = pendingPolicyVerification + ? ["sandbox", "delete", "-g", pendingPolicyVerification.gatewayName, sandboxName] + : ["sandbox", "delete", sandboxName]; + const deleteResult = runOpenshell(deleteArgs, { ignoreError: true, killSignal: "SIGKILL", stdio: ["ignore", "pipe", "pipe"], diff --git a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts index 161a96a28cc..00da34149b3 100644 --- a/src/lib/actions/sandbox/destroy-host-local-inference.test.ts +++ b/src/lib/actions/sandbox/destroy-host-local-inference.test.ts @@ -19,6 +19,31 @@ import { executeSandboxDestroy } from "./destroy-execution"; const AUTHORITY_ID = `mxc-endpoint:${"a".repeat(64)}`; const BINDING_SHA256 = "d".repeat(64); const MODEL = "qwen3.5-9b"; +const SANDBOX_FINGERPRINT = "a".repeat(64); +type ExternalPendingPolicyVerification = Extract< + NonNullable, + { policyAuthority: "externally-managed" } +>; + +function pendingPolicyVerification( + overrides: Partial = {}, +): ExternalPendingPolicyVerification { + return { + schemaVersion: 1, + state: "verified-create", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "alpha", + lifecycleGeneration: "alpha-generation-1", + sandboxIdentityFingerprint: SANDBOX_FINGERPRINT, + route: "none", + policyHash: "policy-hash", + policyVersion: 1, + policyAuthority: "externally-managed", + observedPolicyAuthority: "externally-managed", + ...overrides, + }; +} function receipt( service: "ollama" | "nim" | "vllm" = "vllm", @@ -167,6 +192,7 @@ async function runDestroy( sandboxConfirmedAbsent?: boolean; force?: boolean; includeRegistryReaders?: boolean; + inspectSandboxIdentityFingerprint?: () => string; lifecycleOptions?: NonNullable< NonNullable< Parameters[0]["deps"] @@ -213,6 +239,11 @@ async function runDestroy( ...(options.lifecycleOptions ? { hostLocalInferenceLifecycleOptions: options.lifecycleOptions } : {}), + ...(options.inspectSandboxIdentityFingerprint + ? { + inspectOpenShellSandboxIdentityFingerprint: options.inspectSandboxIdentityFingerprint, + } + : {}), readTimerMarker: () => null, wipeSandboxState: () => undefined, }, @@ -238,6 +269,93 @@ describe("sandbox destroy host-local inference transaction", () => { }; } + it.each([ + ["changes", () => "b".repeat(64)], + [ + "cannot be inspected", + () => { + throw new Error("identity unavailable"); + }, + ], + ])("preserves a pending create when its sandbox identity %s", async (_case, inspect) => { + const runtimeProvider = provider(); + const entry = sandbox("alpha", receipt(), { + pendingPolicyVerification: pendingPolicyVerification(), + }); + + const { result, runOpenshell, stopInferenceResources } = await runDestroy(runtimeProvider, { + entry, + inspectSandboxIdentityFingerprint: inspect, + }); + + expect(result).toMatchObject({ + ok: false, + deleteOutput: expect.stringContaining("Pending policy verification sandbox identity"), + }); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(stopInferenceResources).not.toHaveBeenCalled(); + expect(runtimeProvider.events).toEqual([]); + }); + + it("re-reads a matching pending checkpoint and gateway-scopes its delete", async () => { + const runtimeProvider = provider(); + const entry = sandbox("alpha", receipt(), { + pendingPolicyVerification: pendingPolicyVerification(), + }); + const inspect = vi.fn(() => SANDBOX_FINGERPRINT); + + const { getSandbox, result, runOpenshell } = await runDestroy(runtimeProvider, { + entry, + inspectSandboxIdentityFingerprint: inspect, + }); + + expect(result).toMatchObject({ ok: true }); + expect(inspect.mock.calls.length).toBeGreaterThanOrEqual(5); + expect(getSandbox.mock.calls.length).toBeGreaterThanOrEqual(10); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("preserves a pending create when its checkpoint changes during identity inspection", async () => { + const runtimeProvider = provider(); + const entry = sandbox("alpha", receipt(), { + pendingPolicyVerification: pendingPolicyVerification(), + }); + const getSandbox = vi + .fn() + .mockReturnValueOnce(entry) + .mockReturnValueOnce({ + ...entry, + pendingPolicyVerification: pendingPolicyVerification({ policyVersion: 2 }), + }); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + const stopInferenceResources = vi.fn(); + + const result = await executeSandboxDestroy({ + cleanupShieldsArtifacts: vi.fn(), + force: false, + getSandbox, + listSandboxes: () => ({ sandboxes: [entry] }), + runOpenshell, + sandbox: entry, + sandboxConfirmedAbsent: false, + sandboxName: "alpha", + stopInferenceResources, + runtimeProviders: { mxc: runtimeProvider.bundle }, + deps: { + inspectOpenShellSandboxIdentityFingerprint: () => SANDBOX_FINGERPRINT, + readTimerMarker: () => null, + wipeSandboxState: () => undefined, + }, + }); + + expect(result).toMatchObject({ ok: false }); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(stopInferenceResources).not.toHaveBeenCalled(); + }); + function explicitLlamaSandbox(value = explicitLlamaReceipt()): SandboxEntry { const serialized = serializeHostLocalInferenceReceipt(value); return { diff --git a/src/lib/actions/sandbox/snapshot-restore-clone-ports.test.ts b/src/lib/actions/sandbox/snapshot-restore-clone-ports.test.ts index c68c6fd4f80..7ee33289684 100644 --- a/src/lib/actions/sandbox/snapshot-restore-clone-ports.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-clone-ports.test.ts @@ -55,6 +55,21 @@ describe("runSandboxSnapshot restore: clone port identity", () => { model: "nvidia/model-a", dashboardPort: 18790, gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + sandboxName: "alpha", + lifecycleGeneration: "00000000-0000-4000-8000-000000000001", + sandboxIdentityFingerprint: "a".repeat(64), + policyHash: "policy-alpha", + policyVersion: 1, + }, } : registeredClone, ); @@ -94,6 +109,8 @@ describe("runSandboxSnapshot restore: clone port identity", () => { undefined, { pending: true }, ); + expect(registeredClone).not.toHaveProperty("policyAuthority"); + expect(registeredClone).not.toHaveProperty("policyCreationReceipt"); }); it("keeps a --force destination when the source gateway binding is invalid (#7227)", async () => { diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index afa5c6cb4ca..cacf6e50622 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -63,6 +63,8 @@ export type SandboxRecord = { preferredInferenceApi?: string | null; lifecycleGeneration?: string; lifecycleLiveIdentityFingerprint?: string; + policyAuthority?: SandboxEntry["policyAuthority"]; + policyCreationReceipt?: SandboxEntry["policyCreationReceipt"]; hostLocalInferenceReceipt?: string | null; hostLocalInferenceProvenance?: SandboxHostLocalInferenceProvenance; dashboardPort?: number | null; diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 54fe0fc1554..241403f054e 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -591,10 +591,15 @@ async function autoCreateSandboxFromSource( releaseCloneHostLocalReservation(); failUnregisteredSnapshotClone(dstName, sourceGatewayName); } + const { + policyAuthority: _sourcePolicyAuthority, + policyCreationReceipt: _sourcePolicyCreationReceipt, + ...cloneSourceEntry + } = srcEntry as SandboxEntry; try { registry.registerSandbox( { - ...srcEntry, + ...cloneSourceEntry, name: dstName, createdAt: new Date().toISOString(), policies: [], @@ -829,10 +834,9 @@ function reconcilePendingSnapshotClone( return "removed"; } - const get = captureOpenshell( - ["sandbox", "get", "-g", sourceGatewayName, targetSandbox], - { ignoreError: true }, - ); + const get = captureOpenshell(["sandbox", "get", "-g", sourceGatewayName, targetSandbox], { + ignoreError: true, + }); if (get.status !== 0) { throw new SnapshotCommandError( `Cannot reconcile pending clone '${targetSandbox}' because its live identity could not be read.`, diff --git a/src/lib/adapters/openshell/client.ts b/src/lib/adapters/openshell/client.ts index 14ca2134699..80e83e9003c 100644 --- a/src/lib/adapters/openshell/client.ts +++ b/src/lib/adapters/openshell/client.ts @@ -13,6 +13,9 @@ import { import { redirectInheritedChildStdoutToStderr } from "../../cli/stdout-guard"; import { buildSubprocessEnv } from "../../subprocess-env"; import { processTreeBoundedOpenshellInvocation } from "./process-tree-timeout"; +import { classifyManagedGatewayEndpointBinding } from "../../../../nemoclaw/dist/shared/openshell-gateway-endpoint-boundary.cjs"; + +export { classifyManagedGatewayEndpointBinding }; export { openshellSandboxSshHost, resolveOpenshellSandboxSshHost } from "./sandbox-ssh-host"; @@ -91,6 +94,9 @@ export function stripAnsi(value = ""): string { return String(value).replace(ANSI_RE, ""); } +export type ManagedGatewayEndpointBinding = + import("../../../../nemoclaw/dist/shared/openshell-gateway-endpoint-boundary.cjs").ManagedGatewayEndpointBinding; + function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } diff --git a/src/lib/adapters/openshell/gateway-drift.ts b/src/lib/adapters/openshell/gateway-drift.ts index c5fbdad6952..9f1d33bb9df 100644 --- a/src/lib/adapters/openshell/gateway-drift.ts +++ b/src/lib/adapters/openshell/gateway-drift.ts @@ -22,18 +22,24 @@ import { isGatewayHealthy, } from "../../state/gateway"; import { dockerContainerInspectFormat } from "../docker"; -import { parseVersionFromText, stripAnsi } from "./client"; +import { + classifyManagedGatewayEndpointBinding, + type ManagedGatewayEndpointBinding, + parseVersionFromText, + stripAnsi, +} from "./client"; import { resolveOpenshell } from "./resolve"; import { captureOpenshell, getInstalledOpenshellVersionOrNull } from "./runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "./timeouts"; -export type { GatewayReuseState }; +export type { GatewayReuseState, ManagedGatewayEndpointBinding }; // Gateway observation consumers use the established drift adapter as their // OpenShell parsing and state boundary. export { getGatewayReuseState, isGatewayHealthy, OPENSHELL_PROBE_TIMEOUT_MS, + classifyManagedGatewayEndpointBinding, parseVersionFromText, stripAnsi, }; diff --git a/src/lib/adapters/openshell/policy-authority.test.ts b/src/lib/adapters/openshell/policy-authority.test.ts index 86cfff2efed..1b49da59936 100644 --- a/src/lib/adapters/openshell/policy-authority.test.ts +++ b/src/lib/adapters/openshell/policy-authority.test.ts @@ -1,16 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; + import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import * as openshellResolveModule from "./resolve"; +import * as openshellRuntimeModule from "./runtime"; import { assertExternalPolicyRequirements, assertRecordedPolicyAuthority, - inspectGlobalPolicyAuthority, + inspectActiveGlobalPolicy, + inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority, isExternalPolicyAuthorityRefusalError, - type PolicyAuthorityCapture, policyAuthorityInternals, type SandboxPolicyAuthorityInspection, } from "./policy-authority"; @@ -19,34 +21,31 @@ function captureResult( stdout: string, overrides: Partial<{ stderr: string; - exitCode: number | null; - timedOut: boolean; + status: number | null; + error: Error; }> = {}, ) { return { + status: overrides.status === undefined ? 0 : overrides.status, + output: stdout, stdout, stderr: overrides.stderr ?? "", - exitCode: overrides.exitCode ?? 0, - timedOut: overrides.timedOut ?? false, + ...(overrides.error ? { error: overrides.error } : {}), }; } +function captureError(code: string, message: string): Error { + return Object.assign(new Error(message), { code }); +} + function sandboxMetadata(overrides: Record = {}): Record { return { scope: "sandbox", sandbox: "alpha", status: "effective", policy_source: "sandbox", - policy: { version: 1, network_policies: { baseline: { endpoints: ["base.test"] } } }, - ...overrides, - }; -} - -function globalMetadata(overrides: Record = {}): Record { - return { - scope: "global", - status: "loaded", - policy_source: "global", + hash: "policy-alpha", + active_version: 7, policy: { version: 1, network_policies: { baseline: { endpoints: ["base.test"] } } }, ...overrides, }; @@ -65,151 +64,117 @@ function errorFrom(action: () => unknown): Error { describe("OpenShell policy authority inspection", () => { beforeEach(() => { vi.restoreAllMocks(); - vi.spyOn(openshellResolveModule, "resolveOpenshell").mockReturnValue(null); }); afterEach(() => { vi.restoreAllMocks(); }); - it("recognizes a sandbox-scoped effective policy as NemoClaw-managed (#9833)", () => { - const runCaptureEx = vi.fn(() => - captureResult(JSON.stringify(sandboxMetadata())), - ); + it("leaves a sandbox-scoped effective policy owner unknown (#9833)", () => { + const captureOpenshell = vi + .spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValue(captureResult(JSON.stringify(sandboxMetadata()))); - expect(inspectSandboxPolicyAuthority({ sandboxName: "alpha", runCaptureEx })).toEqual({ - authority: "nemoclaw-managed", + expect(inspectSandboxPolicyAuthority({ sandboxName: "alpha" })).toEqual({ + authority: "owner-unknown", effectivePolicy: { version: 1, network_policies: { baseline: { endpoints: ["base.test"] } }, }, + policyIdentity: { hash: "policy-alpha", activeVersion: 7 }, }); - expect(runCaptureEx).toHaveBeenCalledWith( - ["openshell", "policy", "get", "--full", "--output", "json", "alpha"], - { + expect(captureOpenshell).toHaveBeenCalledWith( + ["policy", "get", "--full", "--output", "json", "alpha"], + expect.objectContaining({ + ignoreError: true, + includeStreams: true, maxBuffer: policyAuthorityInternals.captureMaxBytes, + replaceEnv: true, timeout: policyAuthorityInternals.captureTimeoutMs, - }, + }), ); }); it("recognizes a global policy source as externally managed on the recorded gateway (#9833)", () => { const policy = { version: 1, network_policies: { required: { endpoints: ["api.test"] } } }; - const runCaptureEx = vi.fn(() => - captureResult(JSON.stringify(sandboxMetadata({ policy_source: "global", policy }))), - ); + const captureOpenshell = vi + .spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValue( + captureResult(JSON.stringify(sandboxMetadata({ policy_source: "global", policy }))), + ); expect( inspectSandboxPolicyAuthority({ sandboxName: "alpha", gatewayName: "nemoclaw-18080", - runCaptureEx, }), - ).toEqual({ authority: "externally-managed", effectivePolicy: policy }); - expect(runCaptureEx.mock.calls[0]?.[0]).toEqual([ - "openshell", - "policy", - "get", - "-g", - "nemoclaw-18080", - "--full", - "--output", - "json", - "alpha", - ]); - }); - - it("uses empty bounded global history as managed authority (#9833)", () => { - const runCaptureEx = vi.fn(() => captureResult("")); - - expect(inspectGlobalPolicyAuthority({ gatewayName: "nemoclaw-18080", runCaptureEx })).toEqual({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }); - expect(runCaptureEx).toHaveBeenCalledTimes(1); - expect(runCaptureEx.mock.calls[0]?.[0]).toEqual([ - "openshell", - "policy", - "list", - "-g", - "nemoclaw-18080", - "--global", - "--limit", - "1", - ]); - }); - - it("uses loaded global metadata as external authority (#9833)", () => { - const policy = { version: 1, network_policies: { required: { endpoints: ["api.test"] } } }; - const runCaptureEx = vi - .fn() - .mockReturnValueOnce(captureResult("revision-1")) - .mockReturnValueOnce(captureResult(JSON.stringify(globalMetadata({ policy })))); - - expect(inspectGlobalPolicyAuthority({ gatewayName: "nemoclaw-18080", runCaptureEx })).toEqual({ + ).toEqual({ authority: "externally-managed", effectivePolicy: policy, + policyIdentity: { hash: "policy-alpha", activeVersion: 7 }, }); - expect(runCaptureEx.mock.calls[1]?.[0]).toEqual([ - "openshell", + expect(captureOpenshell.mock.calls[0]?.[0]).toEqual([ "policy", "get", "-g", "nemoclaw-18080", - "--global", "--full", "--output", "json", + "alpha", ]); }); - it("treats a superseded global revision as managed authority (#9833)", () => { - const runCaptureEx = vi - .fn() - .mockReturnValueOnce(captureResult("revision-1")) - .mockReturnValueOnce(captureResult(JSON.stringify(globalMetadata({ status: "superseded" })))); - - expect(inspectGlobalPolicyAuthority({ runCaptureEx })).toEqual({ - authority: "nemoclaw-managed", - effectivePolicy: {}, + it("reads an active global policy through the bounded selected-gateway boundary (#9833)", () => { + const policy = { version: 1, network_policies: { required: { endpoints: ["api.test"] } } }; + const captureOpenshell = vi + .spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(captureResult("global revision 7")) + .mockReturnValueOnce( + captureResult( + JSON.stringify({ + scope: "global", + status: "loaded", + policy_source: "global", + hash: "global-policy", + active_version: 7, + policy, + }), + ), + ); + + expect(inspectActiveGlobalPolicy({ gatewayName: "nemoclaw-18080" })).toEqual({ + state: "active", + inspection: { + authority: "externally-managed", + effectivePolicy: policy, + policyIdentity: { hash: "global-policy", activeVersion: 7 }, + }, }); - }); - - it.each([ - ["malformed metadata", "{"], - ["wrong scope", JSON.stringify(globalMetadata({ scope: "sandbox" }))], - ["unknown status", JSON.stringify(globalMetadata({ status: "pending" }))], - ])("fails closed on %s after global history is present (#9833)", (_caseName, raw) => { - const secret = "captured-global-policy-secret"; - const runCaptureEx = vi - .fn() - .mockReturnValueOnce(captureResult("revision-1")) - .mockReturnValueOnce(captureResult(`${raw}${secret}`)); - - const error = errorFrom(() => inspectGlobalPolicyAuthority({ runCaptureEx })); - expect(error.message).toContain("inspection failed"); - expect(error.message).not.toContain(secret); + expect(captureOpenshell.mock.calls.map(([args]) => args)).toEqual([ + ["policy", "list", "-g", "nemoclaw-18080", "--global", "--limit", "1"], + ["policy", "get", "-g", "nemoclaw-18080", "--global", "--full", "--output", "json"], + ]); }); it("rejects invalid sandbox and gateway identities before querying policy (#9833)", () => { - const runCaptureEx = vi.fn(() => - captureResult(JSON.stringify(sandboxMetadata())), - ); + const captureOpenshell = vi + .spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValue(captureResult(JSON.stringify(sandboxMetadata()))); - expect(() => inspectSandboxPolicyAuthority({ sandboxName: "--global", runCaptureEx })).toThrow( + expect(() => inspectSandboxPolicyAuthority({ sandboxName: "--global" })).toThrow( /Invalid sandbox name/, ); expect(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha", gatewayName: "invalid gateway", - runCaptureEx, }), ).toThrow(/Invalid gateway name/); - expect(() => - inspectSandboxPolicyAuthority({ sandboxName: "alpha", gatewayName: "", runCaptureEx }), - ).toThrow(/gateway name is required/); - expect(runCaptureEx).not.toHaveBeenCalled(); + expect(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha", gatewayName: "" })).toThrow( + /gateway name is required/, + ); + expect(captureOpenshell).not.toHaveBeenCalled(); }); it.each([ @@ -218,65 +183,115 @@ describe("OpenShell policy authority inspection", () => { ["an unknown source", sandboxMetadata({ policy_source: "unknown" })], ])("rejects sandbox metadata with %s (#9833)", (_caseName, metadata) => { const secret = "captured-policy-secret"; - const runCaptureEx = vi.fn(() => + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell").mockReturnValue( captureResult(JSON.stringify({ ...metadata, diagnostic: secret })), ); - const error = errorFrom(() => - inspectSandboxPolicyAuthority({ sandboxName: "alpha", runCaptureEx }), - ); + const error = errorFrom(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha" })); expect(error.message).toContain("inspection failed"); expect(error.message).not.toContain(secret); }); it.each(["", " \n\t"])("fails closed when sandbox policy output is empty (%j) (#9833)", (raw) => { - const runCaptureEx = vi.fn(() => captureResult(raw)); + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell").mockReturnValue( + captureResult(raw), + ); - expect(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha", runCaptureEx })).toThrow( + expect(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha" })).toThrow( /empty sandbox policy authority metadata/u, ); }); it.each([ - ["a nonzero exit", { exitCode: 7 }], - ["a timeout", { timedOut: true }], + ["a nonzero exit", { status: 7 }], + ["a timeout", { status: null, error: captureError("ETIMEDOUT", "captured-timeout-secret") }], ["malformed JSON", {}], ])("fails closed without exposing output after %s (#9833)", (_caseName, overrides) => { - const runCaptureEx = vi.fn(() => + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell").mockReturnValue( captureResult('{"secret":"captured-stdout-secret"', { ...overrides, stderr: "captured-stderr-secret", }), ); - const error = errorFrom(() => - inspectSandboxPolicyAuthority({ sandboxName: "alpha", runCaptureEx }), - ); + const error = errorFrom(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha" })); expect(error.message).not.toContain("captured-stdout-secret"); expect(error.message).not.toContain("captured-stderr-secret"); }); it("replaces a thrown capture diagnostic instead of exposing command output (#9833)", () => { - const runCaptureEx = vi.fn(() => { + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell").mockImplementation(() => { throw new Error("captured-policy-secret"); }); - const error = errorFrom(() => - inspectSandboxPolicyAuthority({ sandboxName: "alpha", runCaptureEx }), - ); + const error = errorFrom(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha" })); expect(error.message).toContain("could not run"); expect(error.message).not.toContain("captured-policy-secret"); }); it("rejects a captured policy response that exceeds the byte limit (#9833)", () => { - const oversized = "x".repeat(policyAuthorityInternals.captureMaxBytes + 1); - const runCaptureEx = vi.fn(() => captureResult(oversized)); + const secret = "captured-oversized-secret"; + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell").mockReturnValue( + captureResult(secret, { + status: null, + error: captureError("ENOBUFS", secret), + }), + ); + + const error = errorFrom(() => inspectSandboxPolicyAuthority({ sandboxName: "alpha" })); + expect(error.message).toContain("capture limit"); + expect(error.message).not.toContain(secret); + }); + + it("reads one sandbox identity through the canonical bounded adapter (#9833)", () => { + const captureOpenshell = vi + .spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValue(captureResult("Name: alpha\nId: sandbox-alpha\nPhase: Ready\n")); + + expect( + inspectOpenShellSandboxIdentityFingerprint({ + sandboxName: "alpha", + gatewayName: "nemoclaw-18080", + }), + ).toBe(createHash("sha256").update("sandbox-alpha").digest("hex")); + expect(captureOpenshell).toHaveBeenCalledWith( + ["sandbox", "get", "-g", "nemoclaw-18080", "alpha"], + expect.objectContaining({ + ignoreError: true, + includeStreams: true, + maxBuffer: policyAuthorityInternals.captureMaxBytes, + replaceEnv: true, + timeout: policyAuthorityInternals.captureTimeoutMs, + }), + ); + }); + it("rejects an ambiguous or failed sandbox identity without exposing output (#9833)", () => { + const captureOpenshell = vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell"); + captureOpenshell.mockReturnValueOnce( + captureResult("Name: alpha\nId: first-secret\nId: second-secret\nPhase: Ready\n"), + ); + expect(() => + inspectOpenShellSandboxIdentityFingerprint({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + }), + ).toThrow("OpenShell did not return one exact durable sandbox ID"); + + captureOpenshell.mockReturnValueOnce( + captureResult("captured-stdout-secret", { + status: 7, + stderr: "captured-stderr-secret", + }), + ); const error = errorFrom(() => - inspectSandboxPolicyAuthority({ sandboxName: "alpha", runCaptureEx }), + inspectOpenShellSandboxIdentityFingerprint({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + }), ); - expect(error.message).toContain("capture limit"); - expect(error.message).not.toContain(oversized.slice(0, 32)); + expect(error.message).not.toContain("captured-stdout-secret"); + expect(error.message).not.toContain("captured-stderr-secret"); }); }); @@ -335,6 +350,7 @@ describe("externally managed policy requirements", () => { }; const inspection: SandboxPolicyAuthorityInspection = { authority: "externally-managed", + policyIdentity: { hash: "policy-alpha", activeVersion: 7 }, effectivePolicy: { version: 9, filesystem_policy: { read_only: ["/observed-secret"] }, @@ -365,7 +381,11 @@ describe("externally managed policy requirements", () => { it("leaves NemoClaw-managed requirements to the mutation path (#9833)", () => { expect(() => assertExternalPolicyRequirements({ - inspection: { authority: "nemoclaw-managed", effectivePolicy: {} }, + inspection: { + authority: "nemoclaw-managed", + effectivePolicy: {}, + policyIdentity: { hash: "policy-alpha", activeVersion: 7 }, + }, requiredPolicy: { network_policies: { required: { endpoints: ["api.test"] } } }, operation: "apply a preset", }), diff --git a/src/lib/adapters/openshell/policy-authority.ts b/src/lib/adapters/openshell/policy-authority.ts index f50dad20cf4..5fba5ebc995 100644 --- a/src/lib/adapters/openshell/policy-authority.ts +++ b/src/lib/adapters/openshell/policy-authority.ts @@ -8,19 +8,23 @@ import { NAME_MAX_LENGTH, } from "../../sandbox-name-contract"; import { - buildGlobalPolicyGetFullJsonCommand, - buildGlobalPolicyListCommand, - buildPolicyGetFullJsonCommand, + buildGlobalPolicyGetFullJsonArgs, + buildGlobalPolicyListArgs, + buildPolicyGetArgs, + buildPolicyGetFullJsonArgs, } from "../../policy/commands"; import { assertExternalPolicyRequirementContainment, assertMatchingPolicyAuthority, + assertPolicyRequirementContainment, + parseActiveGlobalPolicyAuthorityMetadata, + type ActiveGlobalPolicyInspection, type OpenShellPolicyAuthority, - parseGlobalPolicyAuthorityMetadata, parseSandboxPolicyAuthorityMetadata, type SandboxPolicyAuthorityInspection as CanonicalSandboxPolicyAuthorityInspection, } from "../../policy/merge"; -import { runCaptureEx } from "./runtime"; +import * as openshellRuntime from "./runtime"; +import { fingerprintOpenShellSandboxLiveIdentity } from "./sandbox-identity"; const POLICY_AUTHORITY_CAPTURE_MAX_BYTES = 1024 * 1024; const POLICY_AUTHORITY_CAPTURE_TIMEOUT_MS = 30_000; @@ -28,6 +32,7 @@ type JsonObject = Record; export type SandboxPolicyAuthority = OpenShellPolicyAuthority; export type SandboxPolicyAuthorityInspection = CanonicalSandboxPolicyAuthorityInspection; +export type { ActiveGlobalPolicyInspection } from "../../policy/merge"; const POLICY_AUTHORITY_REFUSAL_CODE = "NEMOCLAW_POLICY_AUTHORITY_REFUSAL"; @@ -60,27 +65,13 @@ export function isExternalPolicyAuthorityRefusalError(error: unknown): boolean { ); } -interface PolicyAuthorityCaptureResult { - readonly stdout: string; - readonly stderr?: string; - readonly exitCode: number | null; - readonly timedOut: boolean; -} - -export type PolicyAuthorityCapture = ( - command: readonly string[], - options?: { readonly maxBuffer?: number; readonly timeout?: number }, -) => PolicyAuthorityCaptureResult; - interface SandboxPolicyAuthorityInspectionOptions { readonly sandboxName: string; readonly gatewayName?: string; - readonly runCaptureEx?: PolicyAuthorityCapture; } -interface GlobalPolicyAuthorityInspectionOptions { +interface ActiveGlobalPolicyInspectionOptions { readonly gatewayName?: string; - readonly runCaptureEx?: PolicyAuthorityCapture; } function validatePolicyAuthorityName(name: string, label: string): string { @@ -104,67 +95,76 @@ function isObject(value: unknown): value is JsonObject { return typeof value === "object" && value !== null && !Array.isArray(value); } -function failInspection(subject: "sandbox" | "global", reason: string): never { +function failInspection(subject: "sandbox" | "global" | "gateway", reason: string): never { throw new PolicyAuthorityRefusalError( `OpenShell ${subject} policy authority inspection failed: ${reason}. Policy-dependent operations must stop.`, ); } -function capturePolicyQuery( - command: readonly string[], - capture: PolicyAuthorityCapture, - subject: "sandbox" | "global", - queryKind: "machine-readable policy" | "policy history", -): { readonly stdout: string; readonly stderr: string } { - let result: PolicyAuthorityCaptureResult; +function captureBoundedOpenShell( + args: string[], + subject: "sandbox" | "global" | "gateway", +): ReturnType { try { - result = capture(command, { + return openshellRuntime.captureResolvedOpenshell(args, { + env: openshellRuntime.buildOpenShellSubprocessEnv(), + ignoreError: true, + includeStreams: true, maxBuffer: POLICY_AUTHORITY_CAPTURE_MAX_BYTES, + replaceEnv: true, timeout: POLICY_AUTHORITY_CAPTURE_TIMEOUT_MS, }); } catch { - failInspection(subject, `the ${queryKind} query could not run`); + failInspection(subject, "the policy query could not run"); } +} +function captureAuthorityRead( + args: string[], + subject: "sandbox" | "global" | "gateway", +): { readonly output: string; readonly stdout: string; readonly stderr: string } { + const result = captureBoundedOpenShell(args, subject); if ( !isObject(result) || + typeof result.output !== "string" || typeof result.stdout !== "string" || - (result.stderr !== undefined && typeof result.stderr !== "string") + typeof result.stderr !== "string" ) { - failInspection(subject, `the ${queryKind} query returned an invalid capture result`); + failInspection(subject, "the policy query returned an invalid result"); } - const stderr = result.stderr ?? ""; - if ( - Buffer.byteLength(result.stdout, "utf8") + Buffer.byteLength(stderr, "utf8") > - POLICY_AUTHORITY_CAPTURE_MAX_BYTES - ) { - failInspection(subject, `the ${queryKind} response exceeded the capture limit`); + const errorCode = (result.error as NodeJS.ErrnoException | undefined)?.code; + if (errorCode === "ETIMEDOUT") { + failInspection(subject, "the policy query timed out"); } - if (result.timedOut === true) { - failInspection(subject, `the ${queryKind} query timed out`); + if (errorCode === "ENOBUFS") { + failInspection(subject, "the policy response exceeded the capture limit"); } - if (result.timedOut !== false || result.exitCode !== 0) { - failInspection(subject, `the ${queryKind} query did not complete successfully`); + if (result.error) { + failInspection(subject, "the policy query could not run"); } - return { stdout: result.stdout, stderr }; + if (result.status !== 0) { + failInspection(subject, "the policy query did not complete successfully"); + } + return { output: result.output, stdout: result.stdout, stderr: result.stderr }; +} + +function capturePolicyRead(args: string[], subject: "sandbox" | "global"): string { + return captureAuthorityRead(args, subject).stdout; } /** Inspect the effective policy source for one live sandbox. */ export function inspectSandboxPolicyAuthority({ sandboxName, gatewayName, - runCaptureEx: capture = runCaptureEx, }: SandboxPolicyAuthorityInspectionOptions): SandboxPolicyAuthorityInspection { const validatedSandboxName = validatePolicyAuthorityName(sandboxName, "sandbox name"); const validatedGatewayName = gatewayName === undefined ? undefined : validatePolicyAuthorityName(gatewayName, "gateway name"); - const { stdout: raw } = capturePolicyQuery( - buildPolicyGetFullJsonCommand(validatedSandboxName, validatedGatewayName), - capture, + const raw = capturePolicyRead( + buildPolicyGetFullJsonArgs(validatedSandboxName, validatedGatewayName), "sandbox", - "machine-readable policy", ); try { return parseSandboxPolicyAuthorityMetadata(raw, validatedSandboxName); @@ -176,32 +176,19 @@ export function inspectSandboxPolicyAuthority({ } } -/** Inspect whether an active global policy will manage a sandbox created next. */ -export function inspectGlobalPolicyAuthority({ +/** Inspect active global policy presence without assigning absent policy ownership. */ +export function inspectActiveGlobalPolicy({ gatewayName, - runCaptureEx: capture = runCaptureEx, -}: GlobalPolicyAuthorityInspectionOptions = {}): SandboxPolicyAuthorityInspection { +}: ActiveGlobalPolicyInspectionOptions = {}): ActiveGlobalPolicyInspection { const validatedGatewayName = gatewayName === undefined ? undefined : validatePolicyAuthorityName(gatewayName, "gateway name"); - const { stdout: history } = capturePolicyQuery( - buildGlobalPolicyListCommand(validatedGatewayName), - capture, - "global", - "policy history", - ); - if (history.trim().length === 0) { - return { authority: "nemoclaw-managed", effectivePolicy: {} }; - } - const { stdout: raw } = capturePolicyQuery( - buildGlobalPolicyGetFullJsonCommand(validatedGatewayName), - capture, - "global", - "machine-readable policy", - ); + const history = capturePolicyRead(buildGlobalPolicyListArgs(validatedGatewayName), "global"); + if (history.trim().length === 0) return { state: "absent" }; + const raw = capturePolicyRead(buildGlobalPolicyGetFullJsonArgs(validatedGatewayName), "global"); try { - return parseGlobalPolicyAuthorityMetadata(raw); + return parseActiveGlobalPolicyAuthorityMetadata(raw); } catch (error) { failInspection( "global", @@ -210,6 +197,67 @@ export function inspectGlobalPolicyAuthority({ } } +/** Read one sandbox base policy through the same bounded OpenShell adapter. */ +export function captureSandboxBasePolicy(sandboxName: string, gatewayName: string): string { + return capturePolicyRead( + buildPolicyGetArgs( + validatePolicyAuthorityName(sandboxName, "sandbox name"), + validatePolicyAuthorityName(gatewayName, "gateway name"), + ), + "sandbox", + ); +} + +/** Read and fingerprint one sandbox ID without exposing the ID in diagnostics. */ +export function inspectOpenShellSandboxIdentityFingerprint(options: { + readonly sandboxName: string; + readonly gatewayName: string; +}): string { + const gatewayName = validatePolicyAuthorityName(options.gatewayName, "gateway name"); + const sandboxName = validatePolicyAuthorityName(options.sandboxName, "sandbox name"); + let result: ReturnType; + try { + result = captureBoundedOpenShell(["sandbox", "get", "-g", gatewayName, sandboxName], "sandbox"); + } catch { + throw new Error("OpenShell sandbox identity inspection could not run"); + } + if ( + !isObject(result) || + typeof result.stdout !== "string" || + result.error !== undefined || + result.status !== 0 + ) { + throw new Error("OpenShell sandbox identity inspection did not complete successfully"); + } + const fingerprint = fingerprintOpenShellSandboxLiveIdentity(result.stdout); + if (fingerprint === null) { + throw new Error("OpenShell did not return one exact durable sandbox ID"); + } + return fingerprint; +} + +/** Require the named live OpenShell gateway to expose the receipt-bound local port. */ +export function assertOpenShellGatewayPortBinding(options: { + readonly gatewayName: string; + readonly gatewayPort: number; +}): void { + const gatewayName = validatePolicyAuthorityName(options.gatewayName, "gateway name"); + if ( + !Number.isSafeInteger(options.gatewayPort) || + options.gatewayPort < 1 || + options.gatewayPort > 65_535 + ) { + failInspection("gateway", "the expected gateway port is invalid"); + } + const result = captureAuthorityRead(["gateway", "info", "-g", gatewayName], "gateway"); + if ( + openshellRuntime.classifyManagedGatewayEndpointBinding([result.output], options.gatewayPort) !== + "match" + ) { + failInspection("gateway", "the live endpoint does not match the recorded gateway port"); + } +} + function operationLabel(operation: string): string { return typeof operation === "string" && operation.trim().length > 0 ? operation.trim() @@ -260,6 +308,30 @@ export function assertExternalPolicyRequirements({ } } +/** Verify required entries without assigning ownership to a sandbox-scoped policy. */ +export function assertObservedPolicyRequirements({ + inspection, + requiredPolicy, + operation, + sandboxName, +}: { + readonly inspection: SandboxPolicyAuthorityInspection; + readonly requiredPolicy: JsonObject; + readonly operation: string; + readonly sandboxName?: string; +}): void { + const label = operationLabel(operation); + const target = sandboxName ? ` for sandbox ${JSON.stringify(sandboxName)}` : ""; + try { + assertPolicyRequirementContainment(inspection, requiredPolicy); + } catch (error) { + const detail = error instanceof Error ? error.message : "the policy requirement is invalid"; + throw new PolicyAuthorityRefusalError( + `Refusing to ${label}${target}: ${detail}. The verified policy must supply the exact required entries.`, + ); + } +} + export const policyAuthorityInternals = { captureMaxBytes: POLICY_AUTHORITY_CAPTURE_MAX_BYTES, captureTimeoutMs: POLICY_AUTHORITY_CAPTURE_TIMEOUT_MS, diff --git a/src/lib/adapters/openshell/runtime.ts b/src/lib/adapters/openshell/runtime.ts index ada04453913..c24e656996b 100644 --- a/src/lib/adapters/openshell/runtime.ts +++ b/src/lib/adapters/openshell/runtime.ts @@ -18,6 +18,7 @@ import { OPENSHELL_OPERATION_TIMEOUT_MS, OPENSHELL_PROBE_TIMEOUT_MS } from "./ti type CommandArgs = string[]; export { buildOpenShellSubprocessEnv, OPENSHELL_OPERATION_TIMEOUT_MS }; +export { classifyManagedGatewayEndpointBinding } from "./client"; export { runCaptureEx } from "../../runner"; type RunnerOptions = { diff --git a/src/lib/adapters/openshell/sandbox-identity.test.ts b/src/lib/adapters/openshell/sandbox-identity.test.ts index d0e4fdef4fb..68bad3921cf 100644 --- a/src/lib/adapters/openshell/sandbox-identity.test.ts +++ b/src/lib/adapters/openshell/sandbox-identity.test.ts @@ -10,9 +10,29 @@ import { fingerprintOpenShellSandboxId, fingerprintOpenShellSandboxLiveIdentity, isOpenShellSandboxId, + NEMOCLAW_CREATE_ATTEMPT_LABEL, + NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, parseOpenShellSandboxId, + resolveCreatedOpenShellSandboxId, } from "./sandbox-identity"; +const CREATE_ATTEMPT_NONCE = "a".repeat(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); + +function sandboxListJson(overrides: Record = {}): string { + return JSON.stringify([ + { + id: "sandbox-alpha", + name: "alpha", + labels: { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: CREATE_ATTEMPT_NONCE }, + resource_version: 1, + created_at: "2026-08-25T00:00:00Z", + phase: "Ready", + current_policy_version: 1, + ...overrides, + }, + ]); +} + describe("OpenShell sandbox identity parsing", () => { it("accepts one exact durable ID with optional terminal color", () => { expect(parseOpenShellSandboxId("Name: alpha\nID: sandbox-alpha\n")).toBe("sandbox-alpha"); @@ -35,6 +55,7 @@ describe("OpenShell sandbox identity parsing", () => { createHash("sha256").update("sandbox-alpha").digest("hex"), ); expect(fingerprintOpenShellSandboxLiveIdentity("Name: alpha\nPhase: Ready\n")).toBeNull(); + expect(fingerprintOpenShellSandboxLiveIdentity("Id: first\nId: second\n")).toBeNull(); expect(fingerprintOpenShellSandboxLiveIdentity("ID: first\nID: second\n")).toBeNull(); expect(fingerprintOpenShellSandboxId("sandbox-alpha")).toBe( createHash("sha256").update("sandbox-alpha").digest("hex"), @@ -48,6 +69,90 @@ describe("OpenShell sandbox identity parsing", () => { }); describe("OpenShell sandbox identity reading", () => { + it("binds the first ID to an exact create-attempt label (#9833)", () => { + const runCaptureOpenshell = vi.fn(() => sandboxListJson()); + + expect( + resolveCreatedOpenShellSandboxId({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + createAttemptNonce: CREATE_ATTEMPT_NONCE, + runCaptureOpenshell, + }), + ).toBe("sandbox-alpha"); + expect(runCaptureOpenshell).toHaveBeenCalledExactlyOnceWith( + [ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${CREATE_ATTEMPT_NONCE}`, + "--output", + "json", + "--limit", + "2", + ], + { + ignoreError: false, + timeout: 30_000, + maxBuffer: 1024 * 1024, + killSignal: "SIGKILL", + killProcessTreeOnTimeout: true, + }, + ); + }); + + it.each(["a".repeat(61), "a".repeat(63), "a".repeat(64), "g".repeat(62)])( + "refuses a create-attempt nonce outside the label-compatible contract (#9833)", + (createAttemptNonce) => { + const runCaptureOpenshell = vi.fn(); + + expect(() => + resolveCreatedOpenShellSandboxId({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + createAttemptNonce, + runCaptureOpenshell, + }), + ).toThrow("OpenShell sandbox create-attempt identity is invalid."); + expect(runCaptureOpenshell).not.toHaveBeenCalled(); + }, + ); + + it.each([ + ["same-name replacement", sandboxListJson({ labels: {} })], + ["different name", sandboxListJson({ name: "bravo" })], + ["ambiguous rows", `${sandboxListJson().slice(0, -1)},${sandboxListJson().slice(1)}`], + ["malformed row", sandboxListJson({ id: "invalid/id" })], + ["oversized row", sandboxListJson({ id: "a".repeat(513) })], + ])("refuses %s without disclosing captured metadata (#9833)", (_case, output) => { + const outputCanary = "captured-metadata-canary"; + const capturedRows = (JSON.parse(output) as Array>).map((row) => ({ + ...row, + diagnostic: outputCanary, + })); + const runCaptureOpenshell = vi.fn(() => JSON.stringify(capturedRows)); + + let caught: unknown; + try { + resolveCreatedOpenShellSandboxId({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + createAttemptNonce: CREATE_ATTEMPT_NONCE, + runCaptureOpenshell, + }); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect(String(caught)).toContain( + "OpenShell did not return the exact created identity for sandbox 'alpha'", + ); + expect(String(caught)).not.toContain(outputCanary); + expect(String(caught)).not.toContain(CREATE_ATTEMPT_NONCE); + }); + it("reads each sandbox ID once per process (#9316)", () => { const runCommand = vi.fn(() => ({ status: 0, stdout: "Name: alpha\nID: sandbox-alpha\n" })); const readSandboxId = createOpenshellSandboxIdReader("/usr/bin/openshell", runCommand); diff --git a/src/lib/adapters/openshell/sandbox-identity.ts b/src/lib/adapters/openshell/sandbox-identity.ts index f7378107928..5d1d59d38dc 100644 --- a/src/lib/adapters/openshell/sandbox-identity.ts +++ b/src/lib/adapters/openshell/sandbox-identity.ts @@ -22,6 +22,54 @@ export function fingerprintOpenShellSandboxId(sandboxId: string): string | null : null; } +export const NEMOCLAW_CREATE_ATTEMPT_LABEL = "ai.nvidia.nemoclaw.create-attempt" as const; +export const NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH = 62 as const; + +export interface OpenShellSandboxListJsonRow { + readonly id: string; + readonly name: string; + readonly labels: Readonly>; + readonly resource_version: number; + readonly created_at: string; + readonly phase: string; + readonly current_policy_version: number; +} + +function isStrictSandboxListJsonRow(value: unknown): value is OpenShellSandboxListJsonRow { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const row = value as Record; + const labels = row.labels; + return ( + isOpenShellSandboxId(row.id) && + typeof row.name === "string" && + row.name.length > 0 && + row.name.trim() === row.name && + !!labels && + typeof labels === "object" && + !Array.isArray(labels) && + Object.values(labels as Record).every((label) => typeof label === "string") && + typeof row.resource_version === "number" && + Number.isFinite(row.resource_version) && + typeof row.created_at === "string" && + typeof row.phase === "string" && + row.phase.length > 0 && + typeof row.current_policy_version === "number" && + Number.isFinite(row.current_policy_version) + ); +} + +export function parseStrictOpenShellSandboxListJson( + output: string, +): readonly OpenShellSandboxListJsonRow[] | null { + let rows: unknown; + try { + rows = JSON.parse(output); + } catch { + return null; + } + return Array.isArray(rows) && rows.every(isStrictSandboxListJsonRow) ? rows : null; +} + export function parseOpenShellSandboxId(output: string): string | null { const matches = [ ...String(output) @@ -52,6 +100,62 @@ export function resolveOpenShellSandboxId( return sandboxId; } +/** + * Bind the first accepted sandbox ID to one create attempt. The random label is + * supplied on `sandbox create`; a same-name replacement without that label is + * rejected before the caller can run post-create effects. + */ +export function resolveCreatedOpenShellSandboxId(input: { + readonly sandboxName: string; + readonly gatewayName: string; + readonly createAttemptNonce: string; + readonly runCaptureOpenshell: (args: string[], options?: Record) => string; +}): string { + if (!/^[0-9a-f]{62}$/u.test(input.createAttemptNonce)) { + throw new Error("OpenShell sandbox create-attempt identity is invalid."); + } + let output: string; + try { + output = input.runCaptureOpenshell( + [ + "sandbox", + "list", + "-g", + input.gatewayName, + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${input.createAttemptNonce}`, + "--output", + "json", + "--limit", + "2", + ], + { + ignoreError: false, + timeout: 30_000, + maxBuffer: 1024 * 1024, + killSignal: "SIGKILL", + killProcessTreeOnTimeout: true, + }, + ); + } catch { + throw new Error( + `OpenShell did not return the exact created identity for sandbox '${input.sandboxName}'.`, + ); + } + const rows = parseStrictOpenShellSandboxListJson(output); + const row = rows?.length === 1 ? rows[0] : null; + if ( + !row || + row.name !== input.sandboxName || + row.labels[NEMOCLAW_CREATE_ATTEMPT_LABEL] !== input.createAttemptNonce + ) { + throw new Error( + `OpenShell did not return the exact created identity for sandbox '${input.sandboxName}'.`, + ); + } + return row.id; +} + /** * Read sandbox IDs from the OpenShell CLI, memoized per process. * diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 744ed8f7184..8fb7b4e9643 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -34,6 +34,7 @@ export function createGpuFlowInput(): SandboxGpuCreateFlowInput { initialGpuRoute: "native", compatibilityPolicyPath: "/tmp/compatibility-policy.yaml", dockerDriverGateway: true, + gatewayName: "nemoclaw", gatewayPort: 8080, sandboxReadyTimeoutSecs: 60, createArgv: ["openshell", "sandbox", "create", "--gpu"], diff --git a/src/lib/onboard/cancel-rollback.test.ts b/src/lib/onboard/cancel-rollback.test.ts index cd76afbd813..64c83eef5ad 100644 --- a/src/lib/onboard/cancel-rollback.test.ts +++ b/src/lib/onboard/cancel-rollback.test.ts @@ -10,6 +10,40 @@ import { makeOnboardCancelExit, type SandboxCancelRollbackDeps, } from "./cancel-rollback"; +import type { SandboxEntry } from "../state/registry"; + +const SANDBOX_FINGERPRINT = "a".repeat(64); +type ExternalPendingPolicyVerification = Extract< + NonNullable, + { policyAuthority: "externally-managed" } +>; + +function pendingSandboxEntry( + overrides: Partial = {}, +): SandboxEntry { + return { + name: "new-sb", + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: SANDBOX_FINGERPRINT, + pendingPolicyVerification: { + schemaVersion: 1, + state: "verified-create", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "new-sb", + lifecycleGeneration: "generation-1", + sandboxIdentityFingerprint: SANDBOX_FINGERPRINT, + route: "none", + policyHash: "policy-hash", + policyVersion: 1, + policyAuthority: "externally-managed", + observedPolicyAuthority: "externally-managed", + ...overrides, + }, + }; +} function createDeps(overrides: Partial = {}) { const calls = { @@ -50,7 +84,7 @@ describe("createSandboxCancelRollback", () => { ); }); - it("still unregisters and prints manual cleanup when container delete fails", () => { + it("preserves recovery state when container deletion is not confirmed", () => { const { deps, calls } = createDeps({ deleteSandboxContainer: vi.fn(() => false) }); const rollback = createSandboxCancelRollback(deps); @@ -58,13 +92,12 @@ describe("createSandboxCancelRollback", () => { rollback.markCancelled(); rollback.runIfArmed(); - expect(calls.removeFromRegistry).toHaveBeenCalledWith("new-sb"); - expect(calls.log).toHaveBeenCalledWith( - expect.stringContaining("unregistered incomplete sandbox 'new-sb'"), - ); + expect(calls.removeFromRegistry).not.toHaveBeenCalled(); + expect(calls.clearSession).not.toHaveBeenCalled(); expect(calls.log).toHaveBeenCalledWith( - expect.stringContaining('openshell sandbox delete "new-sb"'), + expect.stringContaining("preserved incomplete sandbox 'new-sb'"), ); + expect(calls.log.mock.calls.flat().join("\n")).not.toContain("openshell sandbox delete"); }); it("does NOT roll back on a non-cancel exit (armed but not cancelled)", () => { @@ -152,7 +185,7 @@ describe("installSandboxCancelRollback", () => { const rollback = installSandboxCancelRollback({ runOpenshell, - registry: { removeSandbox }, + registry: { getSandbox: () => null, removeSandbox }, clearOnboardSession: () => {}, registerExitHandler: (h) => exitHandlers.push(h), }); @@ -176,7 +209,7 @@ describe("installSandboxCancelRollback", () => { const rollback = installSandboxCancelRollback({ runOpenshell, - registry: { removeSandbox }, + registry: { getSandbox: () => null, removeSandbox }, clearOnboardSession: () => {}, registerExitHandler: (h) => exitHandlers.push(h), }); @@ -186,6 +219,129 @@ describe("installSandboxCancelRollback", () => { expect(runOpenshell).not.toHaveBeenCalled(); expect(removeSandbox).not.toHaveBeenCalled(); }); + + it("deletes a pending create only after its exact identity and checkpoint are re-read", () => { + const entry = pendingSandboxEntry(); + const getSandbox = vi.fn(() => entry); + const runOpenshell = vi.fn(() => ({ status: 0 })); + const removeSandbox = vi.fn(); + const clearOnboardSession = vi.fn(); + const inspectIdentity = vi.fn(() => SANDBOX_FINGERPRINT); + const exitHandlers: Array<() => void> = []; + + const rollback = installSandboxCancelRollback({ + runOpenshell, + registry: { getSandbox, removeSandbox }, + clearOnboardSession, + inspectOpenShellSandboxIdentityFingerprint: inspectIdentity, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + rollback.arm("new-sb"); + rollback.markCancelled(); + exitHandlers[0](); + + expect(getSandbox).toHaveBeenCalledTimes(2); + expect(inspectIdentity).toHaveBeenCalledWith({ + sandboxName: "new-sb", + gatewayName: "nemoclaw", + }); + expect(runOpenshell).toHaveBeenCalledWith(["sandbox", "delete", "-g", "nemoclaw", "new-sb"], { + ignoreError: true, + }); + expect(removeSandbox).toHaveBeenCalledWith("new-sb"); + expect(clearOnboardSession).toHaveBeenCalledOnce(); + }); + + it("preserves a pending create and its fingerprint when deletion is not confirmed", () => { + const entry = pendingSandboxEntry(); + const runOpenshell = vi.fn(() => ({ status: 1 })); + const removeSandbox = vi.fn(); + const clearOnboardSession = vi.fn(); + const log = vi.fn(); + const exitHandlers: Array<() => void> = []; + + const rollback = installSandboxCancelRollback({ + runOpenshell, + registry: { getSandbox: () => entry, removeSandbox }, + clearOnboardSession, + inspectOpenShellSandboxIdentityFingerprint: () => SANDBOX_FINGERPRINT, + log, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + rollback.arm("new-sb"); + rollback.markCancelled(); + exitHandlers[0](); + + expect(runOpenshell).toHaveBeenCalledWith(["sandbox", "delete", "-g", "nemoclaw", "new-sb"], { + ignoreError: true, + }); + expect(removeSandbox).not.toHaveBeenCalled(); + expect(clearOnboardSession).not.toHaveBeenCalled(); + const guidance = log.mock.calls.flat().join("\n"); + expect(guidance).toContain(SANDBOX_FINGERPRINT); + expect(guidance).toContain("identity-bound recovery"); + expect(guidance).not.toContain("openshell sandbox delete"); + }); + + it.each([ + ["does not match", () => "b".repeat(64)], + [ + "cannot be inspected", + () => { + throw new Error("identity unavailable"); + }, + ], + ])("preserves a pending create when its exact identity %s", (_case, inspect) => { + const entry = pendingSandboxEntry(); + const runOpenshell = vi.fn(() => ({ status: 0 })); + const removeSandbox = vi.fn(); + const clearOnboardSession = vi.fn(); + const log = vi.fn(); + const exitHandlers: Array<() => void> = []; + + const rollback = installSandboxCancelRollback({ + runOpenshell, + registry: { getSandbox: () => entry, removeSandbox }, + clearOnboardSession, + inspectOpenShellSandboxIdentityFingerprint: vi.fn(inspect), + log, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + rollback.arm("new-sb"); + rollback.markCancelled(); + exitHandlers[0](); + + expect(runOpenshell).not.toHaveBeenCalled(); + expect(removeSandbox).not.toHaveBeenCalled(); + expect(clearOnboardSession).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join("\n")).toContain("preserved incomplete sandbox 'new-sb'"); + }); + + it("preserves a pending create when its durable checkpoint changes during inspection", () => { + const entry = pendingSandboxEntry(); + const changed = pendingSandboxEntry({ policyVersion: 2 }); + const getSandbox = vi.fn().mockReturnValueOnce(entry).mockReturnValueOnce(changed); + const runOpenshell = vi.fn(() => ({ status: 0 })); + const removeSandbox = vi.fn(); + const clearOnboardSession = vi.fn(); + const exitHandlers: Array<() => void> = []; + + const rollback = installSandboxCancelRollback({ + runOpenshell, + registry: { getSandbox, removeSandbox }, + clearOnboardSession, + inspectOpenShellSandboxIdentityFingerprint: () => SANDBOX_FINGERPRINT, + registerExitHandler: (handler) => exitHandlers.push(handler), + }); + rollback.arm("new-sb"); + rollback.markCancelled(); + exitHandlers[0](); + + expect(getSandbox).toHaveBeenCalledTimes(2); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(removeSandbox).not.toHaveBeenCalled(); + expect(clearOnboardSession).not.toHaveBeenCalled(); + }); }); describe("makeOnboardCancelExit", () => { @@ -211,9 +367,11 @@ describe("buildCancelRollbackMessage", () => { expect(lines.join("\n")).not.toContain("openshell sandbox delete"); }); - it("falls back to manual cleanup guidance when the delete failed", () => { - const lines = buildCancelRollbackMessage("sb", false); - expect(lines.join("\n")).toContain("unregistered incomplete sandbox 'sb'"); - expect(lines.join("\n")).toContain('openshell sandbox delete "sb"'); + it("preserves identity-bound recovery guidance when the delete failed", () => { + const lines = buildCancelRollbackMessage("sb", false, SANDBOX_FINGERPRINT); + expect(lines.join("\n")).toContain("preserved incomplete sandbox 'sb'"); + expect(lines.join("\n")).toContain(SANDBOX_FINGERPRINT); + expect(lines.join("\n")).toContain("identity-bound recovery"); + expect(lines.join("\n")).not.toContain("openshell sandbox delete"); }); }); diff --git a/src/lib/onboard/cancel-rollback.ts b/src/lib/onboard/cancel-rollback.ts index 956ee724466..d3f76541ce1 100644 --- a/src/lib/onboard/cancel-rollback.ts +++ b/src/lib/onboard/cancel-rollback.ts @@ -1,6 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isDeepStrictEqual } from "node:util"; + +import { inspectOpenShellSandboxIdentityFingerprint } from "../adapters/openshell/policy-authority"; +import type { SandboxEntry } from "../state/registry"; + // Re-exported so the onboard entrypoint imports its sandbox default/cancel // lifecycle helpers from a single module. export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preservation"; @@ -24,7 +29,12 @@ export { restoreDefaultAfterRecreate, wasSandboxDefault } from "./default-preser */ export interface SandboxCancelRollbackDeps { /** Delete the OpenShell sandbox container. Returns true when the delete succeeded. */ - deleteSandboxContainer(sandboxName: string): boolean; + deleteSandboxContainer(sandboxName: string, gatewayName?: string): boolean; + /** Prove a pending create checkpoint immediately before its name-based delete. */ + prepareSandboxContainerDeletion?(sandboxName: string): { + readonly gatewayName: string; + readonly sandboxIdentityFingerprint: string; + } | null; /** Remove the sandbox entry from the NemoClaw registry (clears default). */ removeSandboxFromRegistry(sandboxName: string): void; /** @@ -53,6 +63,7 @@ export interface SandboxCancelRollback { export function buildCancelRollbackMessage( sandboxName: string, deleteSucceeded: boolean, + sandboxIdentityFingerprint?: string, ): string[] { if (deleteSucceeded) { return [ @@ -62,15 +73,24 @@ export function buildCancelRollbackMessage( } return [ "", - ` Onboarding cancelled — unregistered incomplete sandbox '${sandboxName}'.`, - " The sandbox container may still be running. Remove it with:", - ` openshell sandbox delete "${sandboxName}"`, + ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}' because OpenShell did not confirm its deletion.`, + ...(sandboxIdentityFingerprint + ? [ + ` Durable sandbox identity fingerprint: ${sandboxIdentityFingerprint}`, + " Preserve this fingerprint and give it to an OpenShell administrator for identity-bound recovery.", + ] + : [" Preserve its registry and onboarding recovery state for identity-bound recovery."]), + " Do not delete this sandbox by mutable sandbox name.", ]; } export interface InstallSandboxCancelRollbackOptions { runOpenshell: (args: string[], opts: { ignoreError: boolean }) => { status: number | null }; - registry: { removeSandbox(name: string): void }; + registry: { + getSandbox(name: string): SandboxEntry | null; + removeSandbox(name: string): void; + }; + inspectOpenShellSandboxIdentityFingerprint?: typeof inspectOpenShellSandboxIdentityFingerprint; clearOnboardSession: () => void; log?: (message: string) => void; /** Override for tests; defaults to `process.on("exit", ...)`. */ @@ -90,8 +110,33 @@ export function installSandboxCancelRollback( opts: InstallSandboxCancelRollbackOptions, ): SandboxCancelRollback { const rollback = createSandboxCancelRollback({ - deleteSandboxContainer: (name) => - opts.runOpenshell(["sandbox", "delete", name], { ignoreError: true }).status === 0, + prepareSandboxContainerDeletion: (name) => { + const checkpoint = opts.registry.getSandbox(name)?.pendingPolicyVerification; + if (!checkpoint) return null; + const inspectIdentity = + opts.inspectOpenShellSandboxIdentityFingerprint ?? + inspectOpenShellSandboxIdentityFingerprint; + const liveFingerprint = inspectIdentity({ + sandboxName: name, + gatewayName: checkpoint.gatewayName, + }); + const currentCheckpoint = opts.registry.getSandbox(name)?.pendingPolicyVerification; + if ( + liveFingerprint !== checkpoint.sandboxIdentityFingerprint || + !isDeepStrictEqual(currentCheckpoint, checkpoint) + ) { + throw new Error("pending sandbox policy verification identity changed"); + } + return { + gatewayName: checkpoint.gatewayName, + sandboxIdentityFingerprint: checkpoint.sandboxIdentityFingerprint, + }; + }, + deleteSandboxContainer: (name, gatewayName) => + opts.runOpenshell( + gatewayName ? ["sandbox", "delete", "-g", gatewayName, name] : ["sandbox", "delete", name], + { ignoreError: true }, + ).status === 0, removeSandboxFromRegistry: (name) => opts.registry.removeSandbox(name), clearOnboardSession: opts.clearOnboardSession, log: opts.log ?? ((message) => console.error(message)), @@ -148,14 +193,42 @@ export function createSandboxCancelRollback( const sandboxName = armedSandboxName; armedSandboxName = null; - // Delete the container first, then unregister regardless of the delete - // result — leaving a registry entry pointing at a half-built sandbox is - // worse than an orphaned container the operator can clean up manually. - const deleteSucceeded = deps.deleteSandboxContainer(sandboxName); + let deletionAuthority: { + readonly gatewayName: string; + readonly sandboxIdentityFingerprint: string; + } | null = null; + try { + deletionAuthority = deps.prepareSandboxContainerDeletion?.(sandboxName) ?? null; + } catch { + deps.log(""); + deps.log( + ` Onboarding cancelled — preserved incomplete sandbox '${sandboxName}' because its durable creation identity could not be proved immediately before deletion.`, + ); + return; + } + + let deleteSucceeded = false; + try { + deleteSucceeded = deletionAuthority + ? deps.deleteSandboxContainer(sandboxName, deletionAuthority.gatewayName) + : deps.deleteSandboxContainer(sandboxName); + } catch { + deleteSucceeded = false; + } + if (!deleteSucceeded) { + for (const line of buildCancelRollbackMessage( + sandboxName, + false, + deletionAuthority?.sandboxIdentityFingerprint, + )) { + deps.log(line); + } + return; + } deps.removeSandboxFromRegistry(sandboxName); // Discard the aborted session so `nemoclaw list` recovery doesn't resurrect it. deps.clearOnboardSession(); - for (const line of buildCancelRollbackMessage(sandboxName, deleteSucceeded)) { + for (const line of buildCancelRollbackMessage(sandboxName, true)) { deps.log(line); } }, diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index 432ec845713..7ed71dfee61 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -13,10 +13,12 @@ import * as sandboxState from "../state/sandbox"; import { createCreatedSandboxCompletionActions, createOnboardCreatedSandboxCompletion, + createOnboardCreatedSandboxRegistration, finalizeCreatedSandbox, } from "./created-sandbox-finalization"; import { getDcodeSelectionDrift } from "./dcode-selection-drift"; import type { HermesPortableConfiguredReceipt } from "./experimental/hermes-portable-receipt"; +import { pendingSandboxPolicyVerificationForBoundary } from "./sandbox-create/policy-creation-receipt"; import type { SandboxGpuCreateFlowResult } from "./sandbox-gpu-create-flow"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import type { CreatedSandboxRegistrationInput } from "./sandbox-registration"; @@ -29,6 +31,32 @@ afterEach(() => { vi.restoreAllMocks(); }); +describe("created sandbox registration authority", () => { + it("refuses legacy Hermes resume without an in-process verified create checkpoint (#9833)", async () => { + const complete = vi.fn(); + const cleanupBuildContext = vi.fn(); + const register = createOnboardCreatedSandboxRegistration({ + completion: { complete }, + createdLifecycle: {} as never, + cleanupBuildContext, + manageDashboard: false, + sandboxGpuEnabled: false, + }); + + await expect( + register( + null, + { lifecycleGeneration: "generation-1" } as HermesPortableConfiguredReceipt, + "a".repeat(64), + vi.fn(), + ), + ).rejects.toThrow(/without a verified create checkpoint from this process/u); + + expect(cleanupBuildContext).not.toHaveBeenCalled(); + expect(complete).not.toHaveBeenCalled(); + }); +}); + function executable(file: string, contents: string): void { fs.writeFileSync(file, contents, { mode: 0o755 }); } @@ -107,12 +135,17 @@ function makeRestoreFixture(): { ].join("\n"), ); - const pythonResult = spawnSync("python3", ["-c", "import sys; print(sys.executable)"], { - encoding: "utf8", - }); - expect(pythonResult.status, `Python 3 is required: ${pythonResult.stderr}`).toBe(0); - expect(pythonResult.stdout.trim(), "Python 3 executable path is required").not.toBe(""); - const hostPython = pythonResult.stdout.trim(); + const pythonResult = ["python3.13", "python3.12", "python3.11", "python3"] + .map((candidate) => + spawnSync( + candidate, + ["-c", "import sys; assert sys.version_info >= (3, 11); print(sys.executable)"], + { encoding: "utf8" }, + ), + ) + .find((result) => result.status === 0 && result.stdout.trim().length > 0); + expect(pythonResult, "Python 3.11 or newer is required").toBeDefined(); + const hostPython = pythonResult!.stdout.trim(); const python = path.join(bin, "python3"); executable( python, @@ -299,6 +332,34 @@ describe("created DCode sandbox finalization", () => { it("passes the fresh create endpoint through the production completion constructor (#9555)", async () => { const endpointUrl = "https://openrouter.ai/api/v1"; const model = "nvidia/nemotron-3-ultra-550b-a55b"; + const policyCreationReceipt = { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "dcode", + lifecycleGeneration: "generation-1", + sandboxIdentityFingerprint: "a".repeat(64), + policyHash: "sha256:effective", + policyVersion: 1, + }; + const verifiedPolicyBoundary = { + registration: { + policyAuthority: "nemoclaw-managed" as const, + policyCreationReceipt, + observedPolicyAuthority: "owner-unknown" as const, + }, + sandboxName: "dcode", + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + route: "native" as const, + }; + const verifiedCreate = { + reservation: {} as never, + checkpoint: pendingSandboxPolicyVerificationForBoundary(verifiedPolicyBoundary), + } as NonNullable; const runCaptureOpenshell = vi.fn(() => [ "Sandbox: dcode", @@ -355,9 +416,17 @@ describe("created DCode sandbox finalization", () => { null, { gatewayName: "nemoclaw", gatewayPort: 8080 }, { - initialSandboxPolicy: { appliedPresets: ["personal-open-internet"] }, + initialSandboxPolicy: { + appliedPresets: ["personal-open-internet"], + policyPath: "/private/initial-policy.yaml", + }, + compatibilityPolicyPath: null, policyTier: null, + policyAuthority: "nemoclaw-managed", dashboardRemoteBindPrepared: false, + getVerifiedPolicyBoundary: () => verifiedPolicyBoundary, + getVerifiedCreateRegistrationAuthority: () => verifiedCreate, + revalidatePolicyAuthority: vi.fn(), }, null, "build-1", @@ -393,7 +462,19 @@ describe("created DCode sandbox finalization", () => { release: null, fallbackDiagnostic: null, }, + null, vi.fn(), + vi.fn((input) => ({ + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + sandboxName: input.sandboxName, + lifecycleGeneration: input.lifecycleGeneration, + sandboxIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, + policyHash: "sha256:effective", + policyVersion: 1, + })), ] as unknown as Parameters; const completion = createOnboardCreatedSandboxCompletion(...completionArgs); const created = { @@ -467,7 +548,8 @@ describe("created DCode sandbox finalization", () => { expect(register).not.toHaveBeenCalled(); expect(error).toHaveBeenCalledWith(expect.stringContaining("sandbox still exists")); expect(error).toHaveBeenCalledWith(expect.stringContaining("rebuild is unsafe")); - expect(error).toHaveBeenCalledWith(expect.stringContaining('openshell sandbox delete "dcode"')); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Verify its durable identity")); + expect(error.mock.calls.flat().join("\n")).not.toContain("openshell sandbox delete"); expect(error).toHaveBeenCalledWith(expect.stringContaining("nemoclaw onboard")); }); @@ -523,14 +605,12 @@ describe("created DCode sandbox finalization", () => { " Workspace state restoration did not complete. Registry metadata was not updated.", ); expect(error).toHaveBeenCalledWith( - " Remove the unregistered sandbox from its owning gateway before retrying:", - ); - expect(error).toHaveBeenCalledWith( - ' openshell sandbox delete -g "nemoclaw" "dcode"', + " NemoClaw left unregistered sandbox 'dcode' in place because OpenShell can delete it only by mutable name.", ); expect(error).toHaveBeenCalledWith( - " Rerun the original onboarding command after the deletion succeeds.", + " Verify its durable identity before manual cleanup; do not act by name alone.", ); + expect(error.mock.calls.flat().join("\n")).not.toContain("openshell sandbox delete"); expect(error).toHaveBeenCalledWith( ` Keep the snapshot for manual recovery: ${fixture.backupPath}`, ); @@ -761,7 +841,8 @@ describe("created OpenClaw sandbox finalization", () => { expect(error).toHaveBeenCalledWith( " State was not restored and registry metadata was not updated.", ); - expect(error).toHaveBeenCalledWith(' openshell sandbox delete "openclaw"'); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Verify its durable identity")); + expect(error.mock.calls.flat().join("\n")).not.toContain("openshell sandbox delete"); expect(error).toHaveBeenCalledWith(" Manual recovery: /tmp/managed-openclaw-backup"); }); @@ -806,7 +887,8 @@ describe("created OpenClaw sandbox finalization", () => { expect(error).toHaveBeenCalledWith( " State was not restored and registry metadata was not updated.", ); - expect(error).toHaveBeenCalledWith(' openshell sandbox delete "openclaw"'); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Verify its durable identity")); + expect(error.mock.calls.flat().join("\n")).not.toContain("openshell sandbox delete"); expect(error).toHaveBeenCalledWith( " Then rerun the original `nemoclaw onboard --from ` command.", ); @@ -860,7 +942,8 @@ describe("created OpenClaw sandbox finalization", () => { expect(error).toHaveBeenCalledWith( expect.stringContaining(sandboxState.OPENCLAW_IMAGE_PLUGIN_PROVENANCE_RESTORE_ERROR), ); - expect(error).toHaveBeenCalledWith(' openshell sandbox delete "openclaw"'); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Verify its durable identity")); + expect(error.mock.calls.flat().join("\n")).not.toContain("openshell sandbox delete"); expect(error).toHaveBeenCalledWith( " Then rerun the original `nemoclaw onboard --from ` command.", ); @@ -893,6 +976,34 @@ describe("created sandbox completion actions", () => { order.push("registry"); return input as unknown as SandboxEntry; }); + const policyCreationReceipt = { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "hermes", + lifecycleGeneration: "generation-1", + sandboxIdentityFingerprint: "a".repeat(64), + policyHash: "sha256:effective", + policyVersion: 1, + }; + const verifiedPolicyBoundary = { + registration: { + policyAuthority: "nemoclaw-managed" as const, + policyCreationReceipt, + observedPolicyAuthority: "owner-unknown" as const, + }, + sandboxName: "hermes", + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + route: "native" as const, + }; + const verifiedCreate = { + reservation: {} as never, + checkpoint: pendingSandboxPolicyVerificationForBoundary(verifiedPolicyBoundary), + } as NonNullable; const completion = createCreatedSandboxCompletionActions( { finalization: { @@ -935,7 +1046,12 @@ describe("created sandbox completion actions", () => { hermesToolGateways: [], gatewayName: "nemoclaw", gatewayPort: 8080, - reservationSessionId: "session-owner", + }, + policy: { + initialPolicyPath: "/private/initial-policy.yaml", + compatibilityPolicyPath: "/private/compatibility-policy.yaml", + getVerifiedPolicyBoundary: () => verifiedPolicyBoundary, + getVerifiedCreateRegistrationAuthority: () => verifiedCreate, }, gpu: { config: gpuConfig, @@ -1039,10 +1155,11 @@ describe("created sandbox completion actions", () => { ); expect(order).toEqual([ + "lifecycle-capture", + "lifecycle-revalidate", "gpu", ...(manageDashboard ? ["dashboard-release", "dashboard-forward", "dashboard-hermes"] : []), ...(schema5 ? [] : ["workload"]), - "lifecycle-capture", "lifecycle-revalidate", "registry", ]); @@ -1051,11 +1168,15 @@ describe("created sandbox completion actions", () => { expect.objectContaining({ imageTag: "hermes:test", hermesPortableLifecycle: schema5, - reservationSessionId: "session-owner", appliedPolicies: ["personal-open-internet"], dashboardPort: manageDashboard ? 8644 : 0, lifecycleGeneration: "generation-1", lifecycleLiveIdentityFingerprint: "a".repeat(64), + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: expect.objectContaining({ + policyHash: "sha256:effective", + }), + verifiedCreate, runtimeFields: expect.objectContaining({ sandboxGpuProof: gpuProof }), }), ); diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index c3dc66aef5f..1116ab3a93d 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; import type { OpenClawImagePluginInstall, @@ -20,6 +21,7 @@ import { import * as sandboxState from "../state/sandbox"; import * as buildContext from "../build-context"; import { resolveSandboxImageTagFromCreateOutput } from "../domain/sandbox/image-tag"; +import { restoreDefaultAfterRecreate } from "./default-preservation"; import { createDcodeSelectionDriftReader } from "./dcode-selection-drift"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; @@ -27,7 +29,14 @@ import type { HermesPortableConfiguredReceipt } from "./experimental/hermes-port import { warnIfLandlockUnsupported } from "./landlock-warning"; import * as managedWorkloadOnboard from "./managed-workload/onboard-orchestration"; import { printMessagingProviderMissing } from "./preflight-messages"; +import { + pendingSandboxPolicyVerificationForBoundary, +} from "./sandbox-create/policy-creation-receipt"; import type { SandboxGpuCreateFlowResult } from "./sandbox-gpu-create-flow"; +import type { + VerifiedSandboxPolicyBoundary, + VerifiedSandboxPolicyRegistration, +} from "./types"; import type { SelectionDrift } from "./selection-drift"; import { applyOnboardVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; import { @@ -94,11 +103,20 @@ type RegistrationSeed = Omit< | "lifecycleGeneration" | "lifecycleLiveIdentityFingerprint" | "inferenceRouteReservation" + | "verifiedCreate" >; export interface CreatedSandboxCompletionOptions { readonly finalization: CreatedSandboxFinalizationOptions; readonly registration: RegistrationSeed; + readonly policy: { + readonly initialPolicyPath: string; + readonly compatibilityPolicyPath: string | null; + readonly getVerifiedPolicyBoundary: () => VerifiedSandboxPolicyBoundary; + readonly getVerifiedCreateRegistrationAuthority: () => NonNullable< + CreatedSandboxRegistrationInput["verifiedCreate"] + >; + }; readonly gpu: { readonly config: Parameters< typeof dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAndCommitAfterReady @@ -121,7 +139,6 @@ export interface CreatedSandboxCompletionOptions { chatUiUrl: string, options: { rollbackSandboxOnFailure: true; - gatewayName: string; revalidatePolicyAuthority?: (operation: string) => void; }, ) => number; @@ -185,6 +202,11 @@ export function createOnboardCreatedSandboxRegistration(input: { if (!created && !configuredReceipt) { throw new Error("Sandbox registration requires create or Hermes receipt authority."); } + if (!created) { + throw new Error( + "Hermes portable resume cannot publish sandbox authority without a verified create checkpoint from this process.", + ); + } input.cleanupBuildContext(); const providerGpuDisposition = !input.sandboxGpuEnabled ? "disabled" @@ -224,11 +246,13 @@ export function createOnboardCreatedSandboxRegistration(input: { export function completeOrdinaryOnboardSandboxCreation( input: { readonly sandboxName: string; + readonly sandboxWasLiveDefault: boolean; readonly runtimeFields: RegistrationSeed["runtimeFields"]; readonly messagingProviders: readonly string[]; readonly liveExists: boolean; }, deps: { + readonly setDefault: (sandboxName: string) => void; readonly runFile: (command: string, args: string[], options: { ignoreError: true }) => unknown; readonly scriptsDir: string; readonly gatewayName: string; @@ -241,6 +265,7 @@ export function completeOrdinaryOnboardSandboxCreation( }, ): string { deps.revalidatePolicyAuthority(`completing sandbox '${input.sandboxName}'`); + restoreDefaultAfterRecreate(deps.setDefault, input.sandboxName, input.sandboxWasLiveDefault); deps.revalidatePolicyAuthority(`starting DNS setup for sandbox '${input.sandboxName}'`); if (input.runtimeFields.openshellDriver === "kubernetes") { console.log(" Setting up sandbox DNS proxy..."); @@ -335,7 +360,6 @@ export function createCreatedSandboxCompletionActions( ); dashboardPort = options.dashboard.ensureForward(options.finalization.sandboxName, chatUiUrl, { rollbackSandboxOnFailure: true, - gatewayName: options.registration.gatewayName, revalidatePolicyAuthority: deps.revalidatePolicyAuthority, }); deps.revalidatePolicyAuthority?.( @@ -369,6 +393,20 @@ export function createCreatedSandboxCompletionActions( lifecycle, inferenceRouteReservation, ) => { + const verifiedLifecycle = lifecycle.revalidate( + lifecycle.capture(resolveLifecycleRegistrationFields()), + ); + const verifiedPolicyBoundary = options.policy.getVerifiedPolicyBoundary(); + assertVerifiedPolicyBoundaryMatchesLifecycle( + verifiedPolicyBoundary, + options.finalization.sandboxName, + options.registration.gatewayName, + options.registration.gatewayPort, + verifiedLifecycle, + ); + deps.revalidatePolicyAuthority?.( + `finalizing verified policy for sandbox '${options.finalization.sandboxName}'`, + ); if (providerGpuDisposition === "created") { deps.revalidatePolicyAuthority?.( `committing GPU capability for sandbox '${options.finalization.sandboxName}'`, @@ -392,40 +430,97 @@ export function createCreatedSandboxCompletionActions( firstCreateOutput: created?.firstCreateOutput ?? "", createOutput: created?.createResult.output ?? "", }); - const verifiedLifecycle = lifecycle.revalidate( - lifecycle.capture(resolveLifecycleRegistrationFields()), + const finalLifecycle = lifecycle.revalidate(verifiedLifecycle); + assertVerifiedPolicyBoundaryMatchesLifecycle( + verifiedPolicyBoundary, + options.finalization.sandboxName, + options.registration.gatewayName, + options.registration.gatewayPort, + finalLifecycle, ); + deps.revalidatePolicyAuthority?.( + `publishing sandbox '${options.finalization.sandboxName}' registry authority`, + ); + const verifiedPolicyRegistration = verifiedPolicyBoundary.registration; return finalizeCreatedSandbox( { ...options.finalization, gatewayName: options.registration.gatewayName }, { - ...deps, - register: (openclawImagePluginInstalls) => - (deps.registerCreatedSandbox ?? registerCreatedSandbox)({ - ...options.registration, - runtimeFields: { - ...options.registration.runtimeFields, - sandboxGpuProof: - options.gpu.config.sandboxGpuProof ?? - options.registration.runtimeFields.sandboxGpuProof, - openshellVersion: - configuredReceipt?.openshellExecutableAuthority.version ?? - options.registration.runtimeFields.openshellVersion, - }, - hermesPortableLifecycle: configuredReceipt !== null, - imageTag: resolved.resolvedImageTag, - workload: resolved.workloadReceipt, - openclawImagePluginInstalls, - hermesDashboardState, - dashboardPort, - ...verifiedLifecycle, - inferenceRouteReservation, - }), + ...deps, + register: (openclawImagePluginInstalls) => { + const verifiedCreate = options.policy.getVerifiedCreateRegistrationAuthority(); + assertVerifiedCreateMatchesPolicyBoundary(verifiedPolicyBoundary, verifiedCreate); + return (deps.registerCreatedSandbox ?? registerCreatedSandbox)({ + ...options.registration, + runtimeFields: { + ...options.registration.runtimeFields, + sandboxGpuProof: + options.gpu.config.sandboxGpuProof ?? + options.registration.runtimeFields.sandboxGpuProof, + openshellVersion: + configuredReceipt?.openshellExecutableAuthority.version ?? + options.registration.runtimeFields.openshellVersion, + }, + hermesPortableLifecycle: configuredReceipt !== null, + imageTag: resolved.resolvedImageTag, + workload: resolved.workloadReceipt, + openclawImagePluginInstalls, + hermesDashboardState, + dashboardPort, + ...finalLifecycle, + appliedPolicies: + verifiedPolicyRegistration.policyAuthority === "externally-managed" + ? [] + : options.registration.appliedPolicies, + policyAuthority: verifiedPolicyRegistration.policyAuthority, + ...(verifiedPolicyRegistration.policyAuthority === "nemoclaw-managed" + ? { + policyCreationReceipt: + verifiedPolicyRegistration.policyCreationReceipt, + } + : {}), + inferenceRouteReservation, + verifiedCreate, + }); + }, }, ); }, }; } +function assertVerifiedCreateMatchesPolicyBoundary( + boundary: VerifiedSandboxPolicyBoundary, + verifiedCreate: NonNullable, +): void { + if ( + !isDeepStrictEqual( + verifiedCreate.checkpoint, + pendingSandboxPolicyVerificationForBoundary(boundary), + ) + ) { + throw new Error("Verified sandbox create checkpoint does not match final policy authority."); + } +} + +function assertVerifiedPolicyBoundaryMatchesLifecycle( + boundary: VerifiedSandboxPolicyBoundary, + sandboxName: string, + gatewayName: string, + gatewayPort: number, + lifecycle: CreatedSandboxLifecycleRegistration, +): VerifiedSandboxPolicyRegistration { + if ( + boundary.sandboxName !== sandboxName || + boundary.gatewayName !== gatewayName || + boundary.gatewayPort !== gatewayPort || + boundary.lifecycleGeneration !== lifecycle.lifecycleGeneration || + boundary.lifecycleLiveIdentityFingerprint !== lifecycle.lifecycleLiveIdentityFingerprint + ) { + throw new Error("Verified sandbox policy authority does not match the final lifecycle."); + } + return boundary.registration; +} + type OnboardCreateIntent = { readonly endpointSource?: RegistrationSeed["inferenceSelection"]["endpointSource"]; readonly observabilityEnabled?: boolean; @@ -469,10 +564,19 @@ type OnboardGatewayBinding = { readonly gatewayName: string; readonly gatewayPort: number; }; -type OnboardPreparedPolicy = Pick< +type OnboardPreparedPolicy = Omit< + Pick< managedWorkloadOnboard.PreparedOnboardSandboxWorkloadLaunch, "initialSandboxPolicy" | "policyTier" | "policyAuthority" | "dashboardRemoteBindPrepared" + >, + "policyAuthority" > & { + readonly policyAuthority: NonNullable; + readonly compatibilityPolicyPath: string | null; + readonly getVerifiedPolicyBoundary: () => VerifiedSandboxPolicyBoundary; + readonly getVerifiedCreateRegistrationAuthority: () => NonNullable< + CreatedSandboxRegistrationInput["verifiedCreate"] + >; readonly revalidatePolicyAuthority: (operation: string) => void; }; @@ -509,7 +613,6 @@ export function createOnboardCreatedSandboxCompletion( ensureHermesDashboardForward: CreatedSandboxCompletionOptions["dashboard"]["ensureHermesForward"], workloadRuntime: WorkloadResolutionInput["runtime"], workload: WorkloadResolutionInput["workload"], - reservationSessionId: string | null, note: (message: string) => void, ): CreatedSandboxCompletionActions { const { provider, model, preferredInferenceApi, endpointUrl } = inference; @@ -564,7 +667,13 @@ export function createOnboardCreatedSandboxCompletion( hermesApiPort, ...gateway, hostMounts: resolvedCreateIntent.hostMounts, - reservationSessionId: reservationSessionId ?? undefined, + }, + policy: { + initialPolicyPath: preparedPolicy.initialSandboxPolicy.policyPath, + compatibilityPolicyPath: preparedPolicy.compatibilityPolicyPath, + getVerifiedPolicyBoundary: preparedPolicy.getVerifiedPolicyBoundary, + getVerifiedCreateRegistrationAuthority: + preparedPolicy.getVerifiedCreateRegistrationAuthority, }, gpu: { config: gpuConfig, @@ -616,6 +725,12 @@ export function finalizeCreatedSandbox( options: CreatedSandboxFinalizationOptions, deps: CreatedSandboxFinalizationDeps, ): SandboxEntry | void { + const reportUnregisteredSandboxRecovery = (): void => { + deps.error( + ` NemoClaw left unregistered sandbox '${options.sandboxName}' in place because OpenShell can delete it only by mutable name.`, + ); + deps.error(" Verify its durable identity before manual cleanup; do not act by name alone."); + }; let freshOpenClawImagePluginInstalls: readonly OpenClawImagePluginInstall[] | undefined; if (options.discoverOpenClawImagePluginInstalls === true) { const discovery = deps.discoverFreshOpenClawImagePluginInstalls(options.sandboxName); @@ -624,8 +739,7 @@ export function finalizeCreatedSandbox( ` OpenClaw image plugin discovery failed for sandbox '${options.sandboxName}': ${discovery.error}`, ); deps.error(" State was not restored and registry metadata was not updated."); - deps.error(" Remove the unregistered sandbox before retrying:"); - deps.error(` openshell sandbox delete ${JSON.stringify(options.sandboxName)}`); + reportUnregisteredSandboxRecovery(); deps.error(" Then rerun the original `nemoclaw onboard --from ` command."); if (options.restoreBackupPath) deps.error(` Manual recovery: ${options.restoreBackupPath}`); return deps.exitProcess(1); @@ -664,8 +778,7 @@ export function finalizeCreatedSandbox( ` Managed snapshot restore is deferred for newly created sandbox '${options.sandboxName}' until its runtime authority can be bound before registry publication.`, ); deps.error(" State was not restored and registry metadata was not updated."); - deps.error(" Remove the unregistered sandbox before retrying:"); - deps.error(` openshell sandbox delete ${JSON.stringify(options.sandboxName)}`); + reportUnregisteredSandboxRecovery(); deps.error(` Manual recovery: ${options.restoreBackupPath}`); return deps.exitProcess(1); } @@ -676,8 +789,7 @@ export function finalizeCreatedSandbox( deps.error( " The sandbox still exists, but registry metadata was not updated because a future rebuild would be unsafe.", ); - deps.error(" Remove the unregistered sandbox before retrying:"); - deps.error(` openshell sandbox delete ${JSON.stringify(options.sandboxName)}`); + reportUnregisteredSandboxRecovery(); deps.error(" Then rerun the original `nemoclaw onboard --from ` command."); deps.error(` Manual recovery: ${options.restoreBackupPath}`); return deps.exitProcess(1); @@ -698,17 +810,10 @@ export function finalizeCreatedSandbox( deps.error(` Failed files: ${restore.failedFiles.join(", ")}`); } if (restore.error) deps.error(` Restore reason: ${restore.error}`); - deps.error(" Workspace state restoration did not complete. Registry metadata was not updated."); - const gatewayName = options.gatewayName?.trim(); - if (gatewayName) { - deps.error(" Remove the unregistered sandbox from its owning gateway before retrying:"); - deps.error( - ` openshell sandbox delete -g ${JSON.stringify(gatewayName)} ${JSON.stringify(options.sandboxName)}`, - ); - deps.error(" Rerun the original onboarding command after the deletion succeeds."); - } else { - deps.error(" The owning OpenShell gateway is unknown. Do not delete a same-name sandbox."); - } + deps.error( + " Workspace state restoration did not complete. Registry metadata was not updated.", + ); + reportUnregisteredSandboxRecovery(); deps.error(` Keep the snapshot for manual recovery: ${options.restoreBackupPath}`); return deps.exitProcess(1); } @@ -729,8 +834,7 @@ export function finalizeCreatedSandbox( deps.error( " A NemoClaw rebuild is unsafe here because no verified registry metadata exists.", ); - deps.error(" Remove the unregistered sandbox before retrying:"); - deps.error(` openshell sandbox delete ${JSON.stringify(options.sandboxName)}`); + reportUnregisteredSandboxRecovery(); deps.error(" Then rerun the original `nemoclaw onboard` command."); if (options.restoreBackupPath) { deps.error(` Manual recovery: ${options.restoreBackupPath}`); diff --git a/src/lib/onboard/experimental/hermes-portable-policy-authority.ts b/src/lib/onboard/experimental/hermes-portable-policy-authority.ts index 94bca31a63b..a1f2cab2d5c 100644 --- a/src/lib/onboard/experimental/hermes-portable-policy-authority.ts +++ b/src/lib/onboard/experimental/hermes-portable-policy-authority.ts @@ -10,30 +10,11 @@ import { mergePresetNamesIntoPolicy } from "../../policy"; import { parseOpenShellPolicy } from "../../policy/merge"; import type { SandboxEntry } from "../../state/registry/types"; import { ensureRequiredTierPolicyPresets } from "../policy-tier-suppression"; +import { isOpenShellNativeGpuBaselineEnrichment } from "../sandbox-gpu-route-policy"; const UTF8 = new TextDecoder("utf-8", { fatal: true }); const MAX_POLICY_BYTES = 256 * 1024; const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u; -const PROC_PATH = "/proc"; -const OPENSHELL_PROXY_REQUIRED_READ_ONLY_PATHS = new Set([ - "/usr", - "/lib", - "/etc", - "/app", - "/var/log", - "/dev/urandom", -]); -const OPENSHELL_PROXY_REQUIRED_READ_WRITE_PATHS = new Set(["/tmp"]); -const OPENSHELL_GPU_READ_ONLY_PATHS = new Set(["/run/nvidia-persistenced", "/usr/lib/wsl"]); -const OPENSHELL_GPU_READ_WRITE_PATHS = new Set([ - "/dev/nvidiactl", - "/dev/nvidia-uvm", - "/dev/nvidia-uvm-tools", - "/dev/nvidia-modeset", - "/dev/dxg", - PROC_PATH, -]); -const OPENSHELL_GPU_DEVICE_PATH = /^\/dev\/nvidia[0-9]+$/u; export interface HermesPortablePolicyCaptureResult { readonly status: number | null; @@ -136,113 +117,6 @@ function semanticDigest(policy: Record): string { .digest("hex"); } -function policyWithoutFilesystemPolicy(policy: Record): Record { - const result: Record = Object.create(null); - for (const key of Object.keys(policy)) { - if (key !== "filesystem_policy") result[key] = policy[key]; - } - return result; -} - -function filesystemPolicyWithoutPaths(policy: Record): Record { - const result: Record = Object.create(null); - for (const key of Object.keys(policy)) { - if (key !== "read_only" && key !== "read_write") result[key] = policy[key]; - } - return result; -} - -function readUniquePaths( - policy: Record, - key: "read_only" | "read_write", -): Set | null { - const value = policy[key]; - if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) return null; - const paths = new Set(value); - return paths.size === value.length ? paths : null; -} - -function isSubset(subset: ReadonlySet, superset: ReadonlySet): boolean { - return [...subset].every((entry) => superset.has(entry)); -} - -function setsOverlap(left: ReadonlySet, right: ReadonlySet): boolean { - return [...left].some((entry) => right.has(entry)); -} - -function isOpenShellNativeGpuBaselineEnrichment( - intended: Record, - live: Record, -): boolean { - if ( - semanticDigest(policyWithoutFilesystemPolicy(intended)) !== - semanticDigest(policyWithoutFilesystemPolicy(live)) - ) { - return false; - } - const intendedFilesystem = intended.filesystem_policy; - const liveFilesystem = live.filesystem_policy; - if ( - !intendedFilesystem || - typeof intendedFilesystem !== "object" || - Array.isArray(intendedFilesystem) || - !liveFilesystem || - typeof liveFilesystem !== "object" || - Array.isArray(liveFilesystem) - ) { - return false; - } - const intendedFilesystemRecord = intendedFilesystem as Record; - const liveFilesystemRecord = liveFilesystem as Record; - if ( - semanticDigest(filesystemPolicyWithoutPaths(intendedFilesystemRecord)) !== - semanticDigest(filesystemPolicyWithoutPaths(liveFilesystemRecord)) - ) { - return false; - } - const intendedReadOnly = readUniquePaths(intendedFilesystemRecord, "read_only"); - const intendedReadWrite = readUniquePaths(intendedFilesystemRecord, "read_write"); - const liveReadOnly = readUniquePaths(liveFilesystemRecord, "read_only"); - const liveReadWrite = readUniquePaths(liveFilesystemRecord, "read_write"); - if (!intendedReadOnly || !intendedReadWrite || !liveReadOnly || !liveReadWrite) return false; - if ( - setsOverlap(intendedReadOnly, intendedReadWrite) || - setsOverlap(liveReadOnly, liveReadWrite) - ) { - return false; - } - - // The native direct-GPU create policy deliberately omits /proc so OpenShell - // can add it read-write only after it observes the GPU devices. Treat that - // exact omission as the authority signal; ordinary/non-GPU policies retain - // /proc read-only and cannot use this exception. - if (intendedReadOnly.has(PROC_PATH) || intendedReadWrite.has(PROC_PATH)) return false; - if ( - !isSubset(OPENSHELL_PROXY_REQUIRED_READ_ONLY_PATHS, intendedReadOnly) || - !isSubset(OPENSHELL_PROXY_REQUIRED_READ_WRITE_PATHS, intendedReadWrite) - ) { - return false; - } - if (liveReadOnly.has(PROC_PATH) || !liveReadWrite.has(PROC_PATH)) return false; - if (!isSubset(intendedReadOnly, liveReadOnly) || !isSubset(intendedReadWrite, liveReadWrite)) { - return false; - } - - for (const path of liveReadOnly) { - if (!intendedReadOnly.has(path) && !OPENSHELL_GPU_READ_ONLY_PATHS.has(path)) return false; - } - for (const path of liveReadWrite) { - if ( - !intendedReadWrite.has(path) && - !OPENSHELL_GPU_READ_WRITE_PATHS.has(path) && - !OPENSHELL_GPU_DEVICE_PATH.test(path) - ) { - return false; - } - } - return true; -} - function rejectReservedCreateEntries(policy: Record): void { const policies = policy.network_policies; if (!policies || typeof policies !== "object" || Array.isArray(policies)) return; diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 836fa634877..00ca0efcf7c 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -19,7 +19,7 @@ Related guides: [`README.md`](README.md) describes package placement, [`machine/ | **apply** | Effectful phase that binds credentials and live capabilities | `bindMessagingTokenDefs`; create, rebuild, and mutation executors | | **checkpoint** | Durable, secret-minimized state from which a later process can continue | onboard session and machine snapshot; registry; backup/recovery manifests | | **result** | Handler outcome: advance, retry, branch, pause, complete, or fail | `OnboardStateResult`, applied by `OnboardRuntime` through `OnboardRuntimeBoundary` | -| **compensation** | Effect that undoes or limits a partial apply | failed-create deletion, `cancel-rollback.ts`, `rollbackChannelAdd`, recovery-registry restore | +| **compensation** | Effect that undoes or limits a partial apply | failed-create source cleanup, `cancel-rollback.ts`, `rollbackChannelAdd`, recovery-registry restore | | **reconcile** | Align recorded and live state without replaying the full journey | sandbox drift checks, `reconcileSandboxMessaging`, `mergeOpenClawRestoredConfig` | A **lifecycle contribution** is an internal, data-first plan input such as a policy preset, provider binding, package, host forward, resource profile, or runtime setup entry. It is not a public SDK. A **managed agent package** is recorded desired state for an agent runtime. An **agent-native plugin** is interpreted by the selected agent. “NemoClaw plugin SDK” remains reserved for the #6229 decision. @@ -66,11 +66,11 @@ The machine still cannot resume inside gateway startup, an individual credential ## Onboarding policy authority -Onboarding binds policy authority after gateway setup and before provider, credential, service, registry, or sandbox changes. Empty global policy history means NemoClaw manages the sandbox policy. An active global policy means an external owner manages it. Missing, malformed, unavailable, or contradictory OpenShell metadata stops onboarding. +Onboarding binds policy authority after gateway setup and before provider, credential, service, registry, or sandbox changes. Empty global policy history establishes no observed owner. NemoClaw ownership begins only after an exact sandbox creation receipt binds the created identity and effective policy. An active global policy means an external owner manages it. Missing, malformed, unavailable, or contradictory OpenShell metadata stops onboarding. The session records the decision before later effects. A live sandbox must agree with both the saved session and registry entry. Onboarding rechecks that agreement before each policy-dependent change and after the created sandbox reaches Ready. -NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw leaves the sandbox running because the supported OpenShell delete command targets its mutable name. Before reporting the refusal, the session lifecycle journal records the sandbox name, owning gateway name and port, and a hash of the observed OpenShell identity. Recovery re-observes the sandbox on that gateway and rejects reuse or cleanup when the live identity differs. Operators must not delete that sandbox by name alone. They must ask the OpenShell administrator for recovery guidance before removal. +NemoClaw-managed onboarding keeps the existing policy creation and attribution behavior. Externally managed onboarding verifies that the effective policy contains every requirement for the selected agent, provider, messaging channels, observability, GPU mode, and web search setup. It does not pass a policy file, export `OPENSHELL_SANDBOX_POLICY`, change policy, or record NemoClaw policy attribution. After a post-create authority failure, NemoClaw leaves the sandbox running because the supported delete command targets its mutable name. Operators must preserve the durable sandbox identity fingerprint from the failure output and provide it to the OpenShell administrator. They must not delete the sandbox by name, even after comparing its identity. Contact the administrator for an identity-bound recovery or removal procedure. ## Effect-order flows @@ -125,7 +125,7 @@ runtime mutation | Journey and entry | Desired state, planning, and assembly | Visible and destructive boundaries | Checkpoint and secret boundary | Compensation, coverage, and gaps | |---|---|---|---|---| -| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | A non-Docker-GPU readiness failure attempts to delete the failed sandbox; the Docker-GPU patch path preserves it and emits patch-specific recovery diagnostics. Temp build-context cleanup is attempted inline with an exit-handler fallback; cancel rollback applies only to a brand-new sandbox. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, and `sandbox-create-plan.test.ts`. Gap: gateway upserts can outlive a failed/interrupted create. | +| **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox because OpenShell cannot condition deletion on its durable identity. Exact provider-owned GPU cleanup can proceed through its owner receipt. Temporary policy and build-context cleanup remains best effort. Cancellation clears only the aborted session and leaves the registry row unchanged. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, and policy-authority tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | | **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 0a8e3c761d4..64380d08d20 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -335,8 +335,8 @@ describe("core onboard flow phases", () => { return durableSession; }); const createSandbox = vi.fn(async (...args: unknown[]) => { - const authority = args.at(-2) as { sessionId?: unknown } | null; - const createIntent = args.at(-1) as { + const authority = args.at(-3) as { sessionId?: unknown } | null; + const createIntent = args.at(-2) as { endpointSource?: InferenceEndpointSource | null; }; const reservation = getSandbox(sandboxName); @@ -511,8 +511,9 @@ describe("core onboard flow phases", () => { credentialEnv: "NVIDIA_INFERENCE_API_KEY", }), ); - expect(createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + expect(createSandbox.mock.calls[0]?.at(-2)).toMatchObject({ compatibleEndpointReasoning: "true", + deferSandboxEffectsUntilPolicyVerification: true, resolved: { inferenceProvider: "compatible-endpoint", extraProviders: ["current-provider"], @@ -538,12 +539,80 @@ describe("core onboard flow phases", () => { const providerResult = await providerPhase.run(context({ agent: { name: "hermes" } })); await sandboxPhase.run(providerResult.context); - expect(createSandbox.mock.calls[0]?.at(-1)).toMatchObject({ + expect(createSandbox.mock.calls[0]?.at(-2)).toMatchObject({ rebuildPreservedEnv, rebuildPolicyPresets, }); }); + it("defers normal-entry credential provider effects until policy verification and checkpoint publication", async () => { + const events: string[] = []; + const stageSandboxCredentialProviders = vi.fn(async () => { + events.push("credential-provider-effect"); + return []; + }); + const createSandbox = vi.fn(async (...args: unknown[]) => { + const createIntent = args.at(-2) as { + deferSandboxEffectsUntilPolicyVerification?: boolean; + resolved?: { policy?: { basePolicyPath?: string } }; + }; + const runVerifiedEffects = args.at(-1) as + | ((context: { + revalidatePolicyRequirements: (operation: string) => void; + }) => Promise) + | undefined; + expect(createIntent).toMatchObject({ + deferSandboxEffectsUntilPolicyVerification: true, + resolved: { policy: { basePolicyPath: "/repo/policy.yaml" } }, + }); + expect(runVerifiedEffects).toEqual(expect.any(Function)); + expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); + events.push("policy-verified", "checkpoint-published"); + await runVerifiedEffects?.({ + revalidatePolicyRequirements: () => events.push("policy-checkpoint-revalidated"), + }); + return "created-sandbox"; + }); + const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ + sandboxDeps: { createSandbox, stageSandboxCredentialProviders }, + }); + + const providerResult = await providerPhase.run(context()); + await sandboxPhase.run(providerResult.context); + + expect(events).toEqual([ + "policy-verified", + "checkpoint-published", + "policy-checkpoint-revalidated", + "credential-provider-effect", + ]); + }); + + it.each([ + ["post-create policy verification", "policy verification refused"], + ["durable checkpoint publication", "checkpoint publication refused"], + ])("withholds normal-entry provider effects when %s fails", async (_boundary, failure) => { + const stageSandboxCredentialProviders = vi.fn(async () => []); + const createSandbox = vi.fn(async (...args: unknown[]) => { + const createIntent = args.at(-2) as { + deferSandboxEffectsUntilPolicyVerification?: boolean; + }; + const runVerifiedEffects = args.at(-1); + expect(createIntent.deferSandboxEffectsUntilPolicyVerification).toBe(true); + expect(runVerifiedEffects).toEqual(expect.any(Function)); + expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); + throw new Error(failure); + }); + const { providerInference: providerPhase, sandbox: sandboxPhase } = createPhases({ + sandboxDeps: { createSandbox, stageSandboxCredentialProviders }, + }); + + const providerResult = await providerPhase.run(context()); + await expect(sandboxPhase.run(providerResult.context)).rejects.toThrow(failure); + + expect(stageSandboxCredentialProviders).not.toHaveBeenCalled(); + }); + it("passes fresh context through to provider setup recovery policy", async () => { const setupNim = vi.fn(async () => ({ model: "nvidia/test", diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 91cbf7810ed..6d1514fac38 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -230,6 +230,7 @@ export function createSandboxOnboardFlowPhase< hermesPortableLifecycle: options.hermesPortableLifecycle === true, authoritativeResumeConfig: options.authoritativeResumeConfig, authoritativePolicyTier: options.authoritativePolicyTier, + deferSandboxEffectsUntilPolicyVerification: true, recreateJournalTargetIntentFingerprint: options.recreateJournalTargetIntentFingerprint, endpointSource: endpointProvenance.endpointSource, diff --git a/src/lib/onboard/machine/handlers/provider-inference-policy-authority.test.ts b/src/lib/onboard/machine/handlers/provider-inference-policy-authority.test.ts index 8eb63375af1..1cb20e51ca9 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-policy-authority.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-policy-authority.test.ts @@ -111,28 +111,22 @@ describe("provider inference policy authority", () => { _recoverySessionId, revalidatePolicyRequirements, ) => { - revalidatePolicyRequirements?.( - { - provider: "ollama-local", - model: "llama3.1", - endpointUrl: "http://127.0.0.1:11434/v1", - credentialEnv: null, - preferredInferenceApi: "openai-completions", - }, - "install managed local runtime", - ); - return { + const selection = { ...baseSelection, provider: "ollama-local", model: "llama3.1", endpointUrl: "http://127.0.0.1:11434/v1", credentialEnv: null, }; + expect(revalidatePolicyRequirements).toBeTypeOf("function"); + revalidatePolicyRequirements!(selection, "install managed local runtime"); + return selection; }, ); - const policyChecks = new Map([["install managed local runtime", refuseExternalPolicy]]); const preflightPolicyRequirements = vi.fn((requirements: { operation: string }) => - policyChecks.get(requirements.operation)?.(), + requirements.operation === "install managed local runtime" + ? refuseExternalPolicy() + : undefined, ); const { deps, calls } = createDeps({ setupNim, preflightPolicyRequirements }); @@ -178,19 +172,7 @@ describe("provider inference policy authority", () => { expect(calls.complete).not.toHaveBeenCalledWith("inference", expect.any(Object)); }); - it("withholds inference success when authority changes after deferred selection (#9833)", async () => { - const session = createSession({ - sandboxName: "alpha", - provider: "nvidia-prod", - model: "nvidia/model", - sandboxPromptProgress: { - sandboxName: true, - webSearch: false, - messaging: false, - resourceProfile: false, - }, - }); - session.steps.provider_selection.status = "failed"; + it("withholds durable inference success when final policy authority changes (#9833)", async () => { const preflightPolicyRequirements = vi.fn((requirements: { operation: string }) => requirements.operation === "record successful inference configuration" ? refuseExternalPolicy() @@ -198,14 +180,11 @@ describe("provider inference policy authority", () => { ); const { deps, calls } = createDeps({ preflightPolicyRequirements }); - await expect( - handleProviderInferenceState({ - ...baseOptions(deps, session), - sandboxName: "alpha", - }), - ).rejects.toThrow(/external policy authority must supply/u); + await expect(handleProviderInferenceState(baseOptions(deps))).rejects.toThrow( + /external policy authority must supply/u, + ); - expect(calls.complete).toHaveBeenCalledWith("provider_selection", expect.any(Object)); + expect(calls.setupInference).toHaveBeenCalledOnce(); expect(calls.complete).not.toHaveBeenCalledWith("inference", expect.any(Object)); }); diff --git a/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts b/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts index ab567c40bec..6b6c76e5884 100644 --- a/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-route-mutation-lock.test.ts @@ -101,7 +101,6 @@ describe("sandbox registration route transaction", () => { "guard", "create", "registry", - "publish", ]); }); diff --git a/src/lib/onboard/machine/handlers/sandbox-route-publication.test.ts b/src/lib/onboard/machine/handlers/sandbox-route-publication.test.ts index a8f862f11f5..da2308d6126 100644 --- a/src/lib/onboard/machine/handlers/sandbox-route-publication.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-route-publication.test.ts @@ -100,7 +100,7 @@ describe("sandbox route publication", () => { }); }); - it("rejects Ready sandbox reuse after route reservation ownership changes", async () => { + it("passes route ownership to the verified create boundary after reservation ownership changes", async () => { const session = createSession({ sandboxName: "saved" }); session.steps.sandbox.status = "complete"; const registryEntry: SandboxEntry = { @@ -113,9 +113,13 @@ describe("sandbox route publication", () => { const finalizeSandboxRouteReservation = vi.fn( (_name: string, sessionId: string) => sessionId === registryEntry.reservationSessionId, ); + const createSandbox = vi.fn(async (...args: unknown[]) => { + expect(args[14]).toEqual({ sessionId: session.sessionId }); + throw new Error("route reservation changed at the verified create boundary"); + }); const { deps, calls } = createDeps( { - createSandbox: vi.fn(async () => "saved"), + createSandbox, finalizeSandboxRouteReservation, getSandboxReuseState: () => "ready", getSandboxRegistryEntry: () => registryEntry, @@ -129,15 +133,11 @@ describe("sandbox route publication", () => { resume: true, sandboxName: "saved", }), - ).rejects.toThrow("exit 1"); + ).rejects.toThrow("route reservation changed at the verified create boundary"); - expect(calls.error).toHaveBeenCalledWith( - " Error: sandbox 'saved' inference route reservation changed while onboarding was in progress. Retry onboarding.", - ); - expect(finalizeSandboxRouteReservation).toHaveBeenCalledExactlyOnceWith( - "saved", - session.sessionId, - ); + expect(createSandbox).toHaveBeenCalledOnce(); + expect(finalizeSandboxRouteReservation).not.toHaveBeenCalled(); + expect(calls.updateSandbox).not.toHaveBeenCalled(); expect(registryEntry).toMatchObject({ pendingRouteReservation: true, reservationSessionId: "session-new", diff --git a/src/lib/onboard/machine/handlers/sandbox.test.ts b/src/lib/onboard/machine/handlers/sandbox.test.ts index 2bda762147d..02af94c08f1 100644 --- a/src/lib/onboard/machine/handlers/sandbox.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox.test.ts @@ -89,10 +89,7 @@ describe("handleSandboxState", () => { extraProviders: [], }, ); - expect(calls.finalizeRouteReservation).toHaveBeenCalledWith( - "my-assistant", - expect.any(String), - ); + expect(calls.finalizeRouteReservation).not.toHaveBeenCalled(); expect(calls.updateSandbox).toHaveBeenCalledWith( "my-assistant", expect.objectContaining({ model: "model", provider: "provider" }), diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 4946adbc313..d384cca6337 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -195,6 +195,8 @@ export interface SandboxStateOptions< authoritativeResumeConfig?: boolean; /** Internal rebuild tier that must govern create-time and resumed policy selection. */ authoritativePolicyTier?: string | null; + /** Keep provider and credential effects behind the exact post-create policy gate. */ + deferSandboxEffectsUntilPolicyVerification?: boolean; /** Endpoint source to preserve during an authoritative rebuild. */ endpointSource?: InferenceEndpointSource | null; /** Internal rebuild target fingerprint recorded by the journal opened before deletion. */ @@ -369,6 +371,7 @@ export interface SandboxStateOptions< hermesAuthMethod: HermesAuthMethod | null, inferenceRouteReservationAuthority: InferenceRouteReservationAuthority | null, createIntent: CompleteSandboxCreateIntent, + runVerifiedSandboxCreateEffects?: import("../../types").VerifiedSandboxCreateEffects, ): Promise; finalizeSandboxRouteReservation(sandboxName: string, sessionId: string): boolean; updateSandboxRegistry(sandboxName: string, updates: Record): void; @@ -378,7 +381,11 @@ export interface SandboxStateOptions< ): Record; recordStepComplete(stepName: string, updates: SessionUpdates): Promise; toSessionUpdates(updates: Record): SessionUpdates; - skippedStepMessage(stepName: string, detail?: string | null, reason?: "resume" | "reuse"): void; + skippedStepMessage( + stepName: string, + detail?: string | null, + reason?: "resume" | "reuse", + ): void; recordStateSkipped( state: "sandbox", metadata?: Record | null, @@ -547,6 +554,12 @@ function applyAuthoritativeRebuildPolicyPresets( }; } +function deferredSandboxEffectsIntent(enabled: boolean): { + readonly deferSandboxEffectsUntilPolicyVerification?: true; +} { + return enabled ? { deferSandboxEffectsUntilPolicyVerification: true } : {}; +} + type SandboxCreationDecision = Exclude; type CompleteSandboxCreateIntent = SandboxCreateIntent & { readonly resolved: ResolvedSandboxCreateIntent; @@ -1154,13 +1167,12 @@ class SandboxStateFlow< sandboxName: string, ): void { const sessionId = state.session?.sessionId; - if (sessionId && this.deps.finalizeSandboxRouteReservation(sandboxName, sessionId)) { - return; - } + if (sessionId && this.deps.finalizeSandboxRouteReservation(sandboxName, sessionId)) return; this.deps.error( ` Error: sandbox '${sandboxName}' inference route reservation changed while onboarding was in progress. Retry onboarding.`, ); this.deps.exitProcess(1); + throw new Error("exitProcess returned after route reservation ownership changed"); } private assertRegistryMessagingPlanUnchanged( @@ -1548,6 +1560,7 @@ class SandboxStateFlow< checkpoint: OnboardCheckpoint | null, session: Session | null, force = false, + verifiedPolicyRevalidation?: (operation: string) => void, ): Promise { if ( !this.resumesSandboxPrompts || @@ -1585,14 +1598,16 @@ class SandboxStateFlow< const registeredProviders = await this.deps.withGatewayRouteMutationLock( this.options.gatewayName, async () => { - const revalidatePolicyRequirements = (operation: string) => - this.revalidatePolicyRequirements( - sandboxName, - selectedMessagingChannels, - webSearchConfig, - session, - operation, - ); + const revalidatePolicyRequirements = + verifiedPolicyRevalidation ?? + ((operation: string) => + this.revalidatePolicyRequirements( + sandboxName, + selectedMessagingChannels, + webSearchConfig, + session, + operation, + )); revalidatePolicyRequirements( `register credential providers for sandbox ${JSON.stringify(sandboxName)}`, ); @@ -1644,6 +1659,7 @@ class SandboxStateFlow< requiredBindings: readonly CheckpointProviderBinding[], registryAuthoritySnapshot: RegistryMessagingAuthority, force: boolean, + verifiedPolicyRevalidation?: (operation: string) => void, ): Promise { if (state.selectedMessagingChannels.length === 0) return; const stage = async () => { @@ -1658,6 +1674,7 @@ class SandboxStateFlow< state.session?.checkpoint ?? null, state.session, force, + verifiedPolicyRevalidation, ); }; if (this.deps.withSandboxMutationLock) { @@ -1667,6 +1684,58 @@ class SandboxStateFlow< } } + private async activateCredentialProvidersForCreate( + state: SandboxStepState, + sandboxName: string, + webSearchProviderBindings: readonly CheckpointProviderBinding[], + messagingProviderBindings: readonly CheckpointProviderBinding[], + registryMessagingAuthority: RegistryMessagingAuthority, + forceMessagingProviderRegistration: boolean, + verifiedPolicyRevalidation?: (operation: string) => void, + ): Promise> { + await this.registerCompletedCredentialProviders( + sandboxName, + [], + state.selectedMessagingChannels, + state.webSearchConfig, + webSearchProviderBindings, + "web_search_provider", + state.session?.checkpoint ?? null, + state.session, + false, + verifiedPolicyRevalidation, + ); + let nextState = this.checkpointProviderEffectGroup( + state, + "web_search_provider", + webSearchProviderBindings, + ); + await this.stageMessagingProvidersForCreate( + sandboxName, + nextState, + messagingProviderBindings, + registryMessagingAuthority, + forceMessagingProviderRegistration, + verifiedPolicyRevalidation, + ); + nextState = this.checkpointProviderEffectGroup( + nextState, + "messaging_providers", + messagingProviderBindings, + ); + if (this.resumesSandboxPrompts) { + const session = this.deps.updateSession((current) => { + recordCheckpointProviderEffectGroups(current, { + webSearch: webSearchProviderBindings, + messaging: messagingProviderBindings, + }); + return current; + }); + nextState = { ...nextState, session }; + } + return nextState; + } + private async resolveResourceProfile(state: SandboxStepState): Promise<{ state: SandboxStepState; resourceProfile: ResourceProfile | null; @@ -1765,6 +1834,9 @@ class SandboxStateFlow< ...(this.options.authoritativePolicyTier !== undefined ? { policyTier: this.options.authoritativePolicyTier } : {}), + ...deferredSandboxEffectsIntent( + this.options.deferSandboxEffectsUntilPolicyVerification === true, + ), ...(this.options.rebuildPreservedEnv ? { rebuildPreservedEnv: this.options.rebuildPreservedEnv } : {}), @@ -1997,9 +2069,13 @@ class SandboxStateFlow< messagingPlan: SandboxMessagingPlan | null, registryMessagingAuthoritySnapshot: RegistryMessagingAuthority, decision: SandboxCreationDecision, + activateVerifiedCredentialProviders?: ( + state: SandboxStepState, + revalidatePolicyRequirements: (operation: string) => void, + ) => Promise>, ): Promise> { const resourceSelection = await this.resolveResourceProfile(initialState); - const state = resourceSelection.state; + let state = resourceSelection.state; const resourceProfile = resourceSelection.resourceProfile; const effectiveHermesToolGateways = effectiveHermesToolGatewaysForWebSearch( this.options.agent as { name?: string } | null, @@ -2061,7 +2137,10 @@ class SandboxStateFlow< let sandboxName: string; try { revalidatePolicyRequirements(`create sandbox '${requestedSandboxName}'`); - if (this.options.fresh) { + if ( + this.options.fresh && + this.options.deferSandboxEffectsUntilPolicyVerification !== true + ) { this.deps.stopStaleDashboardListenersForSandbox( this.deps.listRegistrySandboxes().sandboxes, requestedSandboxName, @@ -2090,6 +2169,24 @@ class SandboxStateFlow< this.options.hermesAuthMethod, this.options.session ? { sessionId: this.options.session.sessionId } : null, effectiveCreateIntent, + ...(activateVerifiedCredentialProviders + ? [ + async ( + verifiedContext: import("../../types").VerifiedSandboxCreateEffectsContext, + ) => { + if (this.options.fresh) { + this.deps.stopStaleDashboardListenersForSandbox( + this.deps.listRegistrySandboxes().sandboxes, + requestedSandboxName, + ); + } + state = await activateVerifiedCredentialProviders( + state, + verifiedContext.revalidatePolicyRequirements, + ); + }, + ] + : []), ), ); } catch (error) { @@ -2148,7 +2245,6 @@ class SandboxStateFlow< sandboxName, createIntent, ); - this.finalizeInferenceRouteReservation(state, sandboxName); return { ...state, sandboxName, session: recordedSession }; }; const withGatewayLock = () => @@ -2292,46 +2388,25 @@ class SandboxStateFlow< messagingProviderBindings, ); nextState = this.checkpointMessaging(nextState, messaging); - await this.registerCompletedCredentialProviders( - requestedSandboxName, - [], - nextState.selectedMessagingChannels, - nextState.webSearchConfig, - webSearchProviderBindings, - "web_search_provider", - nextState.session?.checkpoint ?? null, - nextState.session, - ); - nextState = this.checkpointProviderEffectGroup( - nextState, - "web_search_provider", - webSearchProviderBindings, - ); - await this.stageMessagingProvidersForCreate( - requestedSandboxName, - nextState, - messagingProviderBindings, - registryMessagingAuthority, - shouldForceMessagingProviderRegistration( - messagingCredentialChanged, - messagingCredentialBaseline, - messaging.plan, - ), - ); - nextState = this.checkpointProviderEffectGroup( - nextState, - "messaging_providers", - messagingProviderBindings, - ); - if (this.resumesSandboxPrompts) { - const session = this.deps.updateSession((current) => { - recordCheckpointProviderEffectGroups(current, { - webSearch: webSearchProviderBindings, - messaging: messagingProviderBindings, - }); - return current; - }); - nextState = { ...nextState, session }; + const activateCredentialProviders = ( + state: SandboxStepState, + verifiedPolicyRevalidation?: (operation: string) => void, + ) => + this.activateCredentialProvidersForCreate( + state, + requestedSandboxName, + webSearchProviderBindings, + messagingProviderBindings, + registryMessagingAuthority, + shouldForceMessagingProviderRegistration( + messagingCredentialChanged, + messagingCredentialBaseline, + messaging.plan, + ), + verifiedPolicyRevalidation, + ); + if (this.options.deferSandboxEffectsUntilPolicyVerification !== true) { + nextState = await activateCredentialProviders(nextState); } return this.createAndRecordSandbox( nextState, @@ -2339,6 +2414,9 @@ class SandboxStateFlow< messaging.plan, registryMessagingAuthority, decision, + this.options.deferSandboxEffectsUntilPolicyVerification === true + ? activateCredentialProviders + : undefined, ); } diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index 186f0d1fe42..72ee82fe656 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -266,6 +266,7 @@ function operationsHarness( providerId: string, events: string[], failAt: FailurePhase = null, + previousLiveIdentityFingerprint = "fingerprint-old", ): ManagedWorkloadRebuildProviderOperations { const bound = { schemaVersion: 1 as const, @@ -276,7 +277,7 @@ function operationsHarness( ...bound, previousRuntimeHandle: "runtime-old-exact", preparationHandle: "preparation-exact", - previousLiveIdentityFingerprint: "fingerprint-old", + previousLiveIdentityFingerprint, }; const staged: StagedManagedWorkloadReplacement = { ...bound, @@ -345,13 +346,19 @@ function transactionHarness( providerId: string, failAt: FailurePhase = null, platform: (typeof PLATFORMS)[number] = "linux/amd64", + previousEntryOverrides: Partial = {}, ) { const events: string[] = []; - const oldEntry = previousEntry(agent, providerId, platform); + const oldEntry = { ...previousEntry(agent, providerId, platform), ...previousEntryOverrides }; let currentEntry = structuredClone(oldEntry); let providerPreparationCompleted = false; let ambiguousPersistenceReadback = false; - const operations = operationsHarness(providerId, events, failAt); + const operations = operationsHarness( + providerId, + events, + failAt, + oldEntry.lifecycleLiveIdentityFingerprint, + ); const prepare = operations.prepare; operations.prepare = vi.fn(async (plan) => { const prepared = await prepare(plan); @@ -631,41 +638,44 @@ describe("managed workload rebuild transaction", () => { PLATFORMS.map((platform) => [agent, provider, platform] as const), ), ), - )("atomically rebuilds %s through the socket-free %s provider contract on %s", async (agent, provider, platform) => { - const harness = transactionHarness(agent, provider, null, platform); - - const result = await harness.run(); - - expect(result).toMatchObject({ - status: "committed", - previousCleanup: "complete", - entry: { - agent, - openshellDriver: provider, - model: "nvidia/nemotron-new", - fromDockerfile: null, - lifecycleGeneration: "generation-new", - lifecycleLiveIdentityFingerprint: "fingerprint-new", - workload: { - kind: "managed-image", - platform, - release: NEW_RELEASE, - shared: true, + )( + "atomically rebuilds %s through the socket-free %s provider contract on %s", + async (agent, provider, platform) => { + const harness = transactionHarness(agent, provider, null, platform); + + const result = await harness.run(); + + expect(result).toMatchObject({ + status: "committed", + previousCleanup: "complete", + entry: { + agent, + openshellDriver: provider, + model: "nvidia/nemotron-new", + fromDockerfile: null, + lifecycleGeneration: "generation-new", + lifecycleLiveIdentityFingerprint: "fingerprint-new", + workload: { + kind: "managed-image", + platform, + release: NEW_RELEASE, + shared: true, + }, }, - }, - }); - expect(harness.events).toEqual([ - "prepare", - "create", - "readiness", - "restore", - "provider-rebind", - "registry-commit", - "retire:runtime-old-exact", - ]); - expect(harness.currentEntry()).toEqual(result.entry); - expect(harness.currentEntry().imageTag).not.toBe(harness.oldEntry.imageTag); - }); + }); + expect(harness.events).toEqual([ + "prepare", + "create", + "readiness", + "restore", + "provider-rebind", + "registry-commit", + "retire:runtime-old-exact", + ]); + expect(harness.currentEntry()).toEqual(result.entry); + expect(harness.currentEntry().imageTag).not.toBe(harness.oldEntry.imageTag); + }, + ); it.each([ ["prepare", false], @@ -689,6 +699,33 @@ describe("managed workload rebuild transaction", () => { ); }); + it("publishes a replacement without carrying the previous policy receipt (#9833)", async () => { + const lifecycleGeneration = "00000000-0000-4000-8000-000000000001"; + const sandboxIdentityFingerprint = "a".repeat(64); + const harness = transactionHarness("openclaw", "mxc", null, "linux/amd64", { + lifecycleGeneration, + lifecycleLiveIdentityFingerprint: sandboxIdentityFingerprint, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "rebuild-openclaw", + lifecycleGeneration, + sandboxIdentityFingerprint, + policyHash: "policy-old", + policyVersion: 1, + }, + }); + + const result = await harness.run(); + + expect(result.entry.lifecycleGeneration).toBe("generation-new"); + expect(result.entry).not.toHaveProperty("policyAuthority"); + expect(result.entry).not.toHaveProperty("policyCreationReceipt"); + }); + it("rolls back a not-ready replacement by exact staged handle", async () => { const harness = transactionHarness("hermes", "docker", "readiness"); @@ -705,27 +742,23 @@ describe("managed workload rebuild transaction", () => { expect(harness.currentEntry().lifecycleGeneration).toBe("generation-old"); }); - it.each(INVALID_PROVIDER_ARTIFACT_CASES)("$name and stops at the invalid transition", async ({ - phase, - install, - events, - notCalled, - abortCalls, - rollbackCalls, - }) => { - const harness = transactionHarness("langchain-deepagents-code", "mxc"); - install(harness.operations); + it.each(INVALID_PROVIDER_ARTIFACT_CASES)( + "$name and stops at the invalid transition", + async ({ phase, install, events, notCalled, abortCalls, rollbackCalls }) => { + const harness = transactionHarness("langchain-deepagents-code", "mxc"); + install(harness.operations); - await expect(harness.run()).rejects.toMatchObject({ phase }); + await expect(harness.run()).rejects.toMatchObject({ phase }); - expect(harness.currentEntry()).toEqual(harness.oldEntry); - expect(harness.events).toEqual(events); - expect(harness.operations.abortPreparation).toHaveBeenCalledTimes(abortCalls); - expect(harness.operations.rollback).toHaveBeenCalledTimes(rollbackCalls); - notCalled.forEach((operation) => { - expect(harness.operations[operation]).not.toHaveBeenCalled(); - }); - }); + expect(harness.currentEntry()).toEqual(harness.oldEntry); + expect(harness.events).toEqual(events); + expect(harness.operations.abortPreparation).toHaveBeenCalledTimes(abortCalls); + expect(harness.operations.rollback).toHaveBeenCalledTimes(rollbackCalls); + notCalled.forEach((operation) => { + expect(harness.operations[operation]).not.toHaveBeenCalled(); + }); + }, + ); it("aborts preparation when durable registry metadata drifts during preparation", async () => { const events: string[] = []; diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts index 31a4e156e64..73ddb90eacc 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterAll, describe, expect, it, vi } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createHermesStateVolumeDockerHarness } from "../__test-helpers__/hermes-state-volume"; @@ -14,9 +14,6 @@ const preparationState = vi.hoisted(() => ({ useUnavailableCatalog: false, })); const prepareSandboxWorkloadSource = vi.hoisted(() => vi.fn()); -const INSTALLED_REVISION = vi.hoisted(() => "d".repeat(40)); -const releaseRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-release-root-")); -fs.writeFileSync(path.join(releaseRoot, ".version"), "0.0.0\n"); vi.mock("../workload/preparation", async (importOriginal) => { const original = await importOriginal(); @@ -33,10 +30,7 @@ vi.mock("../workload/preparation", async (importOriginal) => { return { ...original, prepareSandboxWorkloadSource }; }); -vi.mock("../../core/version", () => ({ - getBuildIdentity: () => ({ nemoclawVersion: "0.0.0", sourceRevision: INSTALLED_REVISION }), - getVersion: () => "v0.0.0", -})); +vi.mock("../../core/version", () => ({ getVersion: () => "v0.0.0" })); import { createManagedHermesStateVolumeOnboardLifecycle, @@ -77,7 +71,7 @@ function createFreshOnboardingRuntime( agentName: "openclaw", legacyDockerfilePath: "agents/openclaw/Dockerfile", customDockerfilePath: null, - rootDir: releaseRoot, + rootDir: "/tmp/nemoclaw", model: "model", provider: "provider", preferredInferenceApi: null, @@ -124,10 +118,6 @@ async function expectUnsupportedHermesPortableSources( } describe("managed workload onboard orchestration", () => { - afterAll(() => { - fs.rmSync(releaseRoot, { force: true, recursive: true }); - }); - it("activates stock managed images only for shipped agents outside Portable", () => { expect( shouldActivateStockManagedRuntime({ @@ -305,17 +295,6 @@ describe("managed workload onboard orchestration", () => { expect(prepareSandboxWorkloadSource.mock.calls[0]?.[0]).not.toHaveProperty("catalogRevision"); }); - it("retains an exact installed revision outside GitHub Actions", async () => { - const { prepared, runtime } = createFreshOnboardingRuntime({ - NEMOCLAW_INSTALL_REF: INSTALLED_REVISION, - }); - - await expect(runtime.ensurePreparedWorkload()).resolves.toBe(prepared); - expect(prepareSandboxWorkloadSource).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ catalogRevision: INSTALLED_REVISION }), - ); - }); - it("resolves final-image patch metadata after managed build-context staging", async () => { const resolutionMetadata = { key: "published-dcode-base" }; let staged = false; diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index c6d3f6a41bd..eae99520491 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -54,7 +54,6 @@ import { import { getSandboxReadyTimeoutSecs } from "../sandbox-gpu-create"; import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; import { - installedManagedImageCatalogRevision, liveE2eManagedImageCatalog, liveE2eManagedImageRevision, type PreparedSandboxWorkloadSource, @@ -241,16 +240,11 @@ export function createManagedWorkloadOnboardRuntime( let preparedProfile: BuiltManagedStartupOnboardProfile | null = null; const ensurePreparedWorkload = async (): Promise => { - const liveCatalogRevision = liveE2eManagedImageRevision(input.startupProfile.environment); + const catalogRevision = liveE2eManagedImageRevision(input.startupProfile.environment); const liveCatalog = liveE2eManagedImageCatalog(input.startupProfile.environment); - if (liveCatalogRevision && liveCatalog) { + if (catalogRevision && liveCatalog) { throw new Error("live E2E managed-image revision and catalog authority conflict"); } - const catalogRevision = - liveCatalogRevision ?? - (liveCatalog || input.tempManagedRuntimeCatalog || input.managedWorkloadRebuild - ? null - : installedManagedImageCatalogRevision(input.startupProfile.environment, input.rootDir)); preparedWorkloadPromise ??= input.managedWorkloadRebuild ? Promise.resolve( prepareSandboxWorkloadSourceFromRebuildHandoff( @@ -351,6 +345,7 @@ export interface PrepareOnboardSandboxWorkloadLaunchInput { readonly plan: { readonly intent: SandboxCreateIntent; readonly policyAuthority: MaterializeSandboxCreatePlanInput["policyAuthority"]; + readonly deferSandboxEffectsUntilPolicyVerification?: boolean; readonly rebindMessagingTokenDefs: () => Promise; readonly runProviderPreDeleteCleanup: () => void; readonly upsertMessagingProviders: MaterializeSandboxCreatePlanInput["upsertMessagingProviders"]; @@ -384,6 +379,7 @@ export interface PreparedOnboardSandboxWorkloadLaunch { readonly messagingProviders: string[]; readonly gpuRoutePlan: SandboxCreateIntent["gpuRoutePlan"]; readonly compatibilityPolicyPath: string | null; + readonly activateDeferredProviderEffects: (() => readonly string[]) | null; readonly initialGpuRoute: SelectedDockerGpuRoute; readonly sandboxReadyTimeoutSecs: number; readonly buildId: string; @@ -425,6 +421,8 @@ export async function prepareOnboardSandboxWorkloadLaunch( intent: input.plan.intent, fromRef, policyAuthority: input.plan.policyAuthority, + deferSandboxEffectsUntilPolicyVerification: + input.plan.deferSandboxEffectsUntilPolicyVerification, messagingTokenDefs: [...messagingTokenDefs], runProviderPreDeleteCleanup: input.plan.runProviderPreDeleteCleanup, upsertMessagingProviders: input.plan.upsertMessagingProviders, @@ -510,6 +508,7 @@ export async function prepareOnboardSandboxWorkloadLaunch( messagingProviders: createPlan.messagingProviders, gpuRoutePlan: createPlan.gpuRoutePlan, compatibilityPolicyPath: createPlan.compatibilityPolicyPath, + activateDeferredProviderEffects: createPlan.activateDeferredProviderEffects, initialGpuRoute, sandboxReadyTimeoutSecs, buildId, diff --git a/src/lib/onboard/managed-workload/rebuild/commit.ts b/src/lib/onboard/managed-workload/rebuild/commit.ts index 80d14cccc08..c555dadffa7 100644 --- a/src/lib/onboard/managed-workload/rebuild/commit.ts +++ b/src/lib/onboard/managed-workload/rebuild/commit.ts @@ -73,8 +73,13 @@ export function materializeManagedWorkloadReplacementEntry( plan: ManagedWorkloadRebuildPlan, replacement: ReboundManagedWorkloadReplacement, ): SandboxEntry { + const { + policyAuthority: _previousPolicyAuthority, + policyCreationReceipt: _previousPolicyCreationReceipt, + ...retainedPreviousEntry + } = previousEntry; return structuredClone({ - ...previousEntry, + ...retainedPreviousEntry, ...plan.replacementMetadata, name: plan.sandboxName, pendingRouteReservation: undefined, diff --git a/src/lib/onboard/managed-workload/rebuild/plan.ts b/src/lib/onboard/managed-workload/rebuild/plan.ts index 8dfb8efb234..836d2e7d19e 100644 --- a/src/lib/onboard/managed-workload/rebuild/plan.ts +++ b/src/lib/onboard/managed-workload/rebuild/plan.ts @@ -29,6 +29,8 @@ const PROTECTED_REBUILD_METADATA_FIELDS = new Set([ "workload", "lifecycleGeneration", "lifecycleLiveIdentityFingerprint", + "policyAuthority", + "policyCreationReceipt", ]); function safeReplacementMetadata( diff --git a/src/lib/onboard/policy-authority/preflight.test.ts b/src/lib/onboard/policy-authority/preflight.test.ts index 54f3811fdbb..0be1d3c69f9 100644 --- a/src/lib/onboard/policy-authority/preflight.test.ts +++ b/src/lib/onboard/policy-authority/preflight.test.ts @@ -3,8 +3,14 @@ import { describe, expect, it, vi } from "vitest"; +import { + managedPolicyInspection, + managedSandboxEntry, + SANDBOX_IDENTITY, +} from "../../../../test/helpers/managed-policy-receipt-fixture"; + import { PolicyAuthorityRefusalError } from "../../adapters/openshell/policy-authority"; -import { loadAgent } from "../../agent/defs"; +import type { SandboxEntry } from "../../state/registry"; import { createOnboardPolicyAuthorityBindings, qualifySandboxPolicyAuthority, @@ -20,37 +26,31 @@ const requiredPolicy = { }; describe("sandbox policy authority preflight", () => { - it("preflights policy authority for the default OpenClaw agent (#9833)", () => { - const inspectSandboxForCreate = vi.fn(() => ({ existingEntry: null, liveExists: false })); - const loadDefaultAgent = vi.fn((name: string) => loadAgent(name)); + it("normalizes a null default agent before policy preflight (#9833)", () => { + const openclaw = { name: "openclaw" }; + const loadAgent = vi.fn(() => openclaw); + const getAgentPolicyPath = vi.fn(() => "/unused/openclaw-policy.yaml"); const bindings = createOnboardPolicyAuthorityBindings( { - GATEWAY_NAME: "nemoclaw-18080", - ROOT: "/unused", - agentDefs: { loadAgent: loadDefaultAgent }, - agentOnboard: { getAgentPolicyPath: vi.fn(() => null) }, - inspectSandboxForCreate, + GATEWAY_NAME: "nemoclaw", + ROOT: "/repo", + agentDefs: { loadAgent: loadAgent as never }, + agentOnboard: { getAgentPolicyPath: getAgentPolicyPath as never }, + inspectSandboxForCreate: () => ({ existingEntry: null, liveExists: false }), onboardSession: { loadSession: () => null, - updateSession: (mutator) => { - const session: { policyAuthority?: "nemoclaw-managed" | "externally-managed" } = {}; - mutator(session); - return session; - }, + updateSession: vi.fn(), }, }, null, { - inspectGlobalPolicyAuthority: () => ({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }), + inspectActiveGlobalPolicy: () => ({ state: "absent" }), }, ); expect(() => bindings.preflightPolicyRequirements({ - gatewayName: "nemoclaw-18080", + gatewayName: "nemoclaw", sandboxName: null, agent: null, selectedMessagingChannels: [], @@ -59,11 +59,10 @@ describe("sandbox policy authority preflight", () => { provider: null, webSearchConfig: null, observabilityEnabled: false, - operation: "select an inference provider", + operation: "prepare the default sandbox", }), ).not.toThrow(); - expect(loadDefaultAgent).toHaveBeenCalledExactlyOnceWith("openclaw"); - expect(inspectSandboxForCreate).toHaveBeenCalledExactlyOnceWith("my-assistant"); + expect(loadAgent).toHaveBeenCalledExactlyOnceWith("openclaw"); }); it("includes every final selected policy requirement (#9833)", () => { @@ -97,12 +96,7 @@ describe("sandbox policy authority preflight", () => { effectivePolicy: { network_policies: { required_route: { endpoints: ["example.com"] } }, }, - })); - const inspectGlobal = vi.fn(() => ({ - authority: "externally-managed" as const, - effectivePolicy: { - network_policies: { required_route: { endpoints: ["example.com"] } }, - }, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, })); const result = qualifySandboxPolicyAuthority( @@ -115,7 +109,6 @@ describe("sandbox policy authority preflight", () => { operation: "prepare sandbox 'demo'", }, { - inspectGlobalPolicyAuthority: inspectGlobal, inspectSandboxPolicyAuthority: inspectSandbox, }, ); @@ -125,7 +118,39 @@ describe("sandbox policy authority preflight", () => { sandboxName: "demo", gatewayName: "nemoclaw", }); - expect(inspectGlobal).toHaveBeenCalledWith({ gatewayName: "nemoclaw" }); + }); + + it.each([ + ["malformed YAML", "version: [unterminated"], + ["non-mapping YAML", "- version: 1"], + ["an invalid policy root", "unexpected: true"], + ])("rejects %s through the canonical required-policy parser (#9833)", (_case, source) => { + const cleanup = vi.fn(() => true); + + expect(() => + qualifySandboxPolicyAuthority( + { + sandboxName: "demo", + gatewayName: "nemoclaw", + liveExists: true, + recordedAuthorities: ["externally-managed"], + prepareRequiredPolicy: () => ({ + ...requiredPolicy, + sourceBytes: Buffer.from(source), + cleanup, + }), + operation: "verify external policy", + }, + { + inspectSandboxPolicyAuthority: () => ({ + authority: "externally-managed", + effectivePolicy: { network_policies: {} }, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, + }), + }, + ), + ).toThrow(/required sandbox policy is invalid/u); + expect(cleanup).toHaveBeenCalledOnce(); }); it("stops before cleanup-owning callers when external requirements are missing (#9833)", () => { @@ -142,9 +167,13 @@ describe("sandbox policy authority preflight", () => { operation: "create sandbox 'demo'", }, { - inspectGlobalPolicyAuthority: () => ({ - authority: "externally-managed", - effectivePolicy: { network_policies: {} }, + inspectActiveGlobalPolicy: () => ({ + state: "active", + inspection: { + authority: "externally-managed", + effectivePolicy: { network_policies: {} }, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, + }, }), }, ), @@ -167,9 +196,13 @@ describe("sandbox policy authority preflight", () => { operation: "create sandbox 'demo'", }, { - inspectGlobalPolicyAuthority: () => ({ - authority: "externally-managed", - effectivePolicy: { network_policies: {} }, + inspectActiveGlobalPolicy: () => ({ + state: "active", + inspection: { + authority: "externally-managed", + effectivePolicy: { network_policies: {} }, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, + }, }), }, ); @@ -204,12 +237,9 @@ describe("sandbox policy authority preflight", () => { }, { inspectSandboxPolicyAuthority: () => ({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }), - inspectGlobalPolicyAuthority: () => ({ authority: "externally-managed", - effectivePolicy: { network_policies: {} }, + effectivePolicy: {}, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, }), }, ), @@ -231,13 +261,64 @@ describe("sandbox policy authority preflight", () => { operation: "create sandbox 'demo'", }, { - inspectGlobalPolicyAuthority: () => ({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }), + inspectActiveGlobalPolicy: () => ({ state: "absent" }), }, ).authority, ).toBe("nemoclaw-managed"); expect(prepareRequiredPolicy).not.toHaveBeenCalled(); }); + + it("refuses a recorded managed owner without a durable gateway port (#9833)", () => { + const recorded = { + ...managedSandboxEntry("demo"), + gatewayPort: undefined, + } as unknown as SandboxEntry; + const assertGatewayBinding = vi.fn(); + + expect(() => + qualifySandboxPolicyAuthority( + { + sandboxName: "demo", + gatewayName: "nemoclaw", + liveExists: true, + recordedAuthorities: ["nemoclaw-managed"], + recordedSandbox: recorded, + prepareRequiredPolicy: () => requiredPolicy, + operation: "reuse sandbox 'demo'", + }, + { + inspectSandboxPolicyAuthority: managedPolicyInspection, + assertOpenShellGatewayPortBinding: assertGatewayBinding, + }, + ), + ).toThrow(/ownership is not durably verified/u); + expect(assertGatewayBinding).not.toHaveBeenCalled(); + }); + + it("refuses when the recorded route becomes pending during live verification (#9833)", () => { + const recorded = managedSandboxEntry("demo"); + + expect(() => + qualifySandboxPolicyAuthority( + { + sandboxName: "demo", + gatewayName: "nemoclaw", + liveExists: true, + recordedAuthorities: ["nemoclaw-managed"], + recordedSandbox: recorded, + readRecordedSandbox: () => ({ + ...recorded, + pendingRouteReservation: true, + }), + prepareRequiredPolicy: () => requiredPolicy, + operation: "reuse sandbox 'demo'", + }, + { + inspectSandboxPolicyAuthority: managedPolicyInspection, + inspectOpenShellSandboxIdentityFingerprint: () => SANDBOX_IDENTITY, + assertOpenShellGatewayPortBinding: vi.fn(), + }, + ), + ).toThrow(/recorded sandbox policy boundary changed/u); + }); }); diff --git a/src/lib/onboard/policy-authority/preflight.ts b/src/lib/onboard/policy-authority/preflight.ts index 90429005373..9e24764d6df 100644 --- a/src/lib/onboard/policy-authority/preflight.ts +++ b/src/lib/onboard/policy-authority/preflight.ts @@ -3,18 +3,25 @@ import fs from "node:fs"; import path from "node:path"; -import YAML from "yaml"; +import { isDeepStrictEqual } from "node:util"; import type { AgentDefinition } from "../../agent/defs"; import { assertExternalPolicyRequirements, + assertObservedPolicyRequirements, + assertOpenShellGatewayPortBinding, assertRecordedPolicyAuthority, - inspectGlobalPolicyAuthority, + inspectActiveGlobalPolicy, + inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority, PolicyAuthorityRefusalError, type SandboxPolicyAuthority, type SandboxPolicyAuthorityInspection, } from "../../adapters/openshell/policy-authority"; +import { + assertNemoClawPolicyCreationReceiptMatches, + parseOpenShellPolicy, +} from "../../policy/merge"; import type { SandboxEntry } from "../../state/registry"; import { type InitialSandboxPolicy, prepareInitialSandboxCreatePolicy } from "../initial-policy"; import { requiredObservabilityPolicyPresets } from "../observability-policy-presets"; @@ -26,10 +33,21 @@ const { LOCAL_INFERENCE_POLICY_PROVIDERS } = require("../providers") as { }; type PolicyAuthorityInspectionDeps = { - readonly inspectGlobalPolicyAuthority?: typeof inspectGlobalPolicyAuthority; + readonly inspectActiveGlobalPolicy?: typeof inspectActiveGlobalPolicy; + readonly inspectOpenShellSandboxIdentityFingerprint?: typeof inspectOpenShellSandboxIdentityFingerprint; + readonly assertOpenShellGatewayPortBinding?: typeof assertOpenShellGatewayPortBinding; readonly inspectSandboxPolicyAuthority?: typeof inspectSandboxPolicyAuthority; }; +type RecordedPolicyAuthority = Exclude; + +export type QualifiedSandboxPolicyAuthority = + | { readonly authority: "nemoclaw-managed" } + | { + readonly authority: "externally-managed"; + readonly inspection: SandboxPolicyAuthorityInspection; + }; + /** Bind the global policy authority before provider selection can mutate gateway state. */ export function qualifyGlobalPolicyAuthority( input: { @@ -37,28 +55,27 @@ export function qualifyGlobalPolicyAuthority( readonly recordedAuthority?: SandboxPolicyAuthority | null; readonly operation: string; }, - deps: Pick = {}, -): SandboxPolicyAuthorityInspection { - const inspection = (deps.inspectGlobalPolicyAuthority ?? inspectGlobalPolicyAuthority)({ + deps: Pick = {}, +): QualifiedSandboxPolicyAuthority { + const presence = (deps.inspectActiveGlobalPolicy ?? inspectActiveGlobalPolicy)({ gatewayName: input.gatewayName, }); + const authority: RecordedPolicyAuthority = + presence.state === "active" ? "externally-managed" : "nemoclaw-managed"; if (input.recordedAuthority) { - assertRecordedPolicyAuthority(input.recordedAuthority, inspection.authority, input.operation); + assertRecordedPolicyAuthority(input.recordedAuthority, authority, input.operation); } - return inspection; + return presence.state === "active" + ? { authority: "externally-managed", inspection: presence.inspection } + : { authority: "nemoclaw-managed" }; } function parseRequiredPolicy(content: string, operation: string): Record { - let parsed: unknown; try { - parsed = YAML.parse(content); + return parseOpenShellPolicy(content).policy; } catch { throw new Error(`Refusing to ${operation}: the required sandbox policy is invalid.`); } - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - throw new Error(`Refusing to ${operation}: the required sandbox policy is invalid.`); - } - return parsed as Record; } function readInitialPolicy(policy: InitialSandboxPolicy, operation: string): string { @@ -103,28 +120,44 @@ export function qualifySandboxPolicyAuthority( readonly gatewayName: string; readonly liveExists: boolean; readonly recordedAuthorities: readonly (SandboxPolicyAuthority | null | undefined)[]; + readonly recordedSandbox?: SandboxEntry | null; + readonly readRecordedSandbox?: (sandboxName: string) => SandboxEntry | null; readonly prepareRequiredPolicy: () => InitialSandboxPolicy; readonly operation: string; }, deps: PolicyAuthorityInspectionDeps = {}, -): SandboxPolicyAuthorityInspection { +): QualifiedSandboxPolicyAuthority { const sandboxInspection = input.liveExists ? (deps.inspectSandboxPolicyAuthority ?? inspectSandboxPolicyAuthority)({ sandboxName: input.sandboxName, gatewayName: input.gatewayName, }) : null; - const globalInspection = (deps.inspectGlobalPolicyAuthority ?? inspectGlobalPolicyAuthority)({ - gatewayName: input.gatewayName, - }); - if (sandboxInspection) { - assertRecordedPolicyAuthority( - sandboxInspection.authority, - globalInspection.authority, - input.operation, + let inspection: QualifiedSandboxPolicyAuthority; + if (!sandboxInspection) { + inspection = qualifyGlobalPolicyAuthority( + { gatewayName: input.gatewayName, operation: input.operation }, + deps, + ); + } else if (sandboxInspection.authority === "externally-managed") { + inspection = { authority: "externally-managed", inspection: sandboxInspection }; + } else if (sandboxInspection.authority === "owner-unknown") { + inspection = qualifyRecordedSandboxPolicyAuthority( + { + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + recordedSandbox: input.recordedSandbox ?? null, + readRecordedSandbox: input.readRecordedSandbox, + inspection: sandboxInspection, + operation: input.operation, + }, + deps, + ); + } else { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the observed sandbox policy authority is invalid.`, ); } - const inspection = sandboxInspection ?? globalInspection; for (const recorded of input.recordedAuthorities) { if (recorded) { @@ -140,29 +173,144 @@ export function qualifySandboxPolicyAuthority( readInitialPolicy(requiredPolicy, input.operation), input.operation, ); - for (const observed of sandboxInspection - ? [sandboxInspection, globalInspection] - : [globalInspection]) { - assertExternalPolicyRequirements({ - inspection: observed, - requiredPolicy: parsedPolicy, - operation: input.operation, - sandboxName: input.sandboxName, - }); - } + const observed = inspection.inspection; + const assertRequirements = + observed.authority === "owner-unknown" + ? assertObservedPolicyRequirements + : assertExternalPolicyRequirements; + assertRequirements({ + inspection: observed, + requiredPolicy: parsedPolicy, + operation: input.operation, + sandboxName: input.sandboxName, + }); } catch (error) { primaryError = error; } + let cleanupError: unknown; try { cleanupRequirement(requiredPolicy, input.operation); - } catch (cleanupError) { - if (primaryError === undefined) throw cleanupError; - throw attachCleanupFailure(primaryError, cleanupError); + } catch (error) { + cleanupError = error; } - if (primaryError !== undefined) throw primaryError; + if (primaryError !== undefined) { + if (cleanupError !== undefined) { + throw attachCleanupFailure(primaryError, cleanupError); + } + throw primaryError; + } + if (cleanupError !== undefined) throw cleanupError; return inspection; } +function qualifyRecordedSandboxPolicyAuthority( + input: { + readonly sandboxName: string; + readonly gatewayName: string; + readonly recordedSandbox: SandboxEntry | null; + readonly readRecordedSandbox?: (sandboxName: string) => SandboxEntry | null; + readonly inspection: SandboxPolicyAuthorityInspection; + readonly operation: string; + }, + deps: PolicyAuthorityInspectionDeps, +): QualifiedSandboxPolicyAuthority { + const recorded = input.recordedSandbox; + const gatewayPort = recorded?.gatewayPort; + if ( + !recorded?.policyAuthority || + recorded.pendingRouteReservation === true || + recorded.gatewayName !== input.gatewayName || + typeof gatewayPort !== "number" || + !Number.isSafeInteger(gatewayPort) || + typeof recorded.lifecycleGeneration !== "string" || + typeof recorded.lifecycleLiveIdentityFingerprint !== "string" + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: sandbox-scoped policy ownership is not durably verified.`, + "owner-unknown", + ); + } + const inspectIdentity = + deps.inspectOpenShellSandboxIdentityFingerprint ?? inspectOpenShellSandboxIdentityFingerprint; + (deps.assertOpenShellGatewayPortBinding ?? assertOpenShellGatewayPortBinding)({ + gatewayName: input.gatewayName, + gatewayPort, + }); + const beforeIdentity = inspectIdentity({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + }); + if (beforeIdentity !== recorded.lifecycleLiveIdentityFingerprint) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the live sandbox identity does not match the recorded lifecycle.`, + "owner-unknown", + ); + } + const confirmedInspection = ( + deps.inspectSandboxPolicyAuthority ?? inspectSandboxPolicyAuthority + )({ sandboxName: input.sandboxName, gatewayName: input.gatewayName }); + const afterIdentity = inspectIdentity({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + }); + if ( + beforeIdentity !== afterIdentity || + confirmedInspection.authority !== "owner-unknown" || + confirmedInspection.policyIdentity.hash !== input.inspection.policyIdentity.hash || + confirmedInspection.policyIdentity.activeVersion !== input.inspection.policyIdentity.activeVersion + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the sandbox or policy identity changed during verification.`, + "owner-unknown", + ); + } + if (recorded.policyAuthority === "nemoclaw-managed") { + try { + assertNemoClawPolicyCreationReceiptMatches(recorded.policyCreationReceipt, { + origin: "sandbox-create", + gatewayName: input.gatewayName, + gatewayPort, + sandboxName: input.sandboxName, + lifecycleGeneration: recorded.lifecycleGeneration, + sandboxIdentityFingerprint: afterIdentity, + policyHash: confirmedInspection.policyIdentity.hash, + policyVersion: confirmedInspection.policyIdentity.activeVersion, + }); + } catch { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the NemoClaw policy creation receipt does not match the live sandbox policy.`, + "owner-unknown", + ); + } + } + const confirmedRecorded = input.readRecordedSandbox + ? input.readRecordedSandbox(input.sandboxName) + : recorded; + if ( + !confirmedRecorded || + confirmedRecorded.pendingRouteReservation === true || + confirmedRecorded.policyAuthority !== recorded.policyAuthority || + !isDeepStrictEqual(confirmedRecorded.policyCreationReceipt, recorded.policyCreationReceipt) || + confirmedRecorded.lifecycleGeneration !== recorded.lifecycleGeneration || + confirmedRecorded.lifecycleLiveIdentityFingerprint !== + recorded.lifecycleLiveIdentityFingerprint || + confirmedRecorded.gatewayName !== recorded.gatewayName || + confirmedRecorded.gatewayPort !== recorded.gatewayPort + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the recorded sandbox policy boundary changed during live verification.`, + "owner-unknown", + ); + } + if (recorded.policyAuthority === "nemoclaw-managed") { + return { authority: "nemoclaw-managed" }; + } + return { + authority: "externally-managed", + inspection: confirmedInspection, + }; +} + type ProviderPolicyRequirements = { readonly gatewayName: string; readonly sandboxName: string | null; @@ -262,6 +410,8 @@ export function createOnboardPolicyAuthorityBindings runtime.inspectSandboxForCreate(name).existingEntry, operation: requirements.operation, prepareRequiredPolicy: () => prepareInitialSandboxCreatePolicy( @@ -298,7 +448,8 @@ export function createOnboardPolicyAuthorityBindings { - current.policyAuthority = inspection.authority; + current.policyAuthority = + inspection.authority === "externally-managed" ? "externally-managed" : null; if (inspection.authority === "externally-managed") current.policyPresets = null; }); }, diff --git a/src/lib/onboard/sandbox-create-intent-types.ts b/src/lib/onboard/sandbox-create-intent-types.ts index 09aed1cadac..9126e99b4d5 100644 --- a/src/lib/onboard/sandbox-create-intent-types.ts +++ b/src/lib/onboard/sandbox-create-intent-types.ts @@ -97,6 +97,8 @@ export type MaterializeSandboxCreatePlanInput = { intent: SandboxCreateIntent; fromRef: string; policyAuthority: SandboxPolicyAuthority; + /** Keep provider mutations and attachments behind the exact post-create policy gate. */ + deferSandboxEffectsUntilPolicyVerification?: boolean; managedStateMount?: ManagedHermesStateVolumeMount | null; messagingTokenDefs: MessagingTokenDef[]; runProviderPreDeleteCleanup(): void; diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index 7f5886aaddf..3b59c81f988 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -82,8 +82,17 @@ export type SandboxCreatePlan = { gpuRoutePlan: SandboxCreateIntent["gpuRoutePlan"]; compatibilityPolicyPath: string | null; sandboxGpuLogMessage: string | null; + /** One-shot provider activation owned by the post-create verification boundary. */ + activateDeferredProviderEffects: (() => readonly string[]) | null; }; +function sameProviderNames(left: readonly string[], right: readonly string[]): boolean { + return ( + left.length === right.length && + left.every((providerName, index) => providerName === right[index]) + ); +} + export function selectHermesPortableExtraProviderPlan( hermesPortable: boolean, requested: readonly string[] | undefined, @@ -257,6 +266,7 @@ export function materializeSandboxCreatePlan({ intent, fromRef, policyAuthority, + deferSandboxEffectsUntilPolicyVerification = false, managedStateMount, messagingTokenDefs, runProviderPreDeleteCleanup, @@ -299,10 +309,15 @@ export function materializeSandboxCreatePlan({ ...intent.resourceCreateArgs, ]; - const hermesToolGatewayProvider = - intent.hermesToolGateways.length > 0 - ? getHermesToolGatewayProviderName(intent.sandboxName) - : null; + let hermesToolGatewayProvider: string | null | undefined; + const resolveHermesToolGatewayProvider = (): string | null => { + if (hermesToolGatewayProvider !== undefined) return hermesToolGatewayProvider; + hermesToolGatewayProvider = + intent.hermesToolGateways.length > 0 + ? getHermesToolGatewayProviderName(intent.sandboxName) + : null; + return hermesToolGatewayProvider; + }; const plannedMessagingProviders = filterMessagingProvidersForSandboxCreate( [ ...enabledMessagingTokenDefs.filter(({ token }) => Boolean(token)).map(({ name }) => name), @@ -315,7 +330,7 @@ export function materializeSandboxCreatePlan({ if (policyAuthority === "nemoclaw-managed") { assertCredentialBindingProvidersAttached( initialSandboxPolicy, - buildCreateProviderSet(intent, plannedMessagingProviders, hermesToolGatewayProvider), + buildCreateProviderSet(intent, plannedMessagingProviders, resolveHermesToolGatewayProvider()), ); try { discloseInitialSandboxPolicy?.(initialSandboxPolicy); @@ -325,29 +340,39 @@ export function materializeSandboxCreatePlan({ } } - runProviderPreDeleteCleanup(); - const messagingProviders = filterMessagingProvidersForSandboxCreate( - [ - ...upsertMessagingProviders(enabledMessagingTokenDefs, { - replaceExisting: true, - allowedSandboxes: [intent.sandboxName], - }), - ...intent.reusableMessagingProviders, - ], - intent.messagingProviderRequests, - intent.policy.activeMessagingChannels, - intent.disabledChannelNames, - ); - const createProviders = buildCreateProviderSet( - intent, - messagingProviders, - hermesToolGatewayProvider, - ); - if (policyAuthority === "nemoclaw-managed") { - assertCredentialBindingProvidersAttached(initialSandboxPolicy, createProviders); - } - for (const provider of createProviders) { - createArgs.push("--provider", provider); + const activateProviderEffects = (): readonly string[] => { + runProviderPreDeleteCleanup(); + const activatedMessagingProviders = filterMessagingProvidersForSandboxCreate( + [ + ...upsertMessagingProviders(enabledMessagingTokenDefs, { + replaceExisting: true, + allowedSandboxes: [intent.sandboxName], + }), + ...intent.reusableMessagingProviders, + ], + intent.messagingProviderRequests, + intent.policy.activeMessagingChannels, + intent.disabledChannelNames, + ); + const createProviders = buildCreateProviderSet( + intent, + activatedMessagingProviders, + resolveHermesToolGatewayProvider(), + ); + if (policyAuthority === "nemoclaw-managed") { + assertCredentialBindingProvidersAttached(initialSandboxPolicy, createProviders); + } + if (!sameProviderNames(activatedMessagingProviders, plannedMessagingProviders)) { + throw new Error( + `Provider activation for sandbox '${intent.sandboxName}' did not match its verified create plan.`, + ); + } + return [...createProviders]; + }; + if (!deferSandboxEffectsUntilPolicyVerification) { + for (const provider of activateProviderEffects()) { + createArgs.push("--provider", provider); + } } return { @@ -356,10 +381,13 @@ export function materializeSandboxCreatePlan({ policyTier: intent.policy.options.policyTier, policyAuthority, createArgs, - messagingProviders, + messagingProviders: plannedMessagingProviders, gpuRoutePlan: intent.gpuRoutePlan, compatibilityPolicyPath, sandboxGpuLogMessage: intent.sandboxGpuLogMessage, + activateDeferredProviderEffects: deferSandboxEffectsUntilPolicyVerification + ? activateProviderEffects + : null, }; } @@ -371,9 +399,7 @@ export function materializeHermesPortableCreatePlan(input: { }): SandboxCreatePlan { const { intent, fromRef, policyAuthority } = input; if (policyAuthority !== "nemoclaw-managed") { - throw new Error( - "Hermes portable sandbox creation requires NemoClaw-managed policy authority.", - ); + throw new Error("Hermes portable sandbox creation requires NemoClaw-managed policy authority."); } if ( intent.policy.options.agentName !== "hermes" || @@ -427,5 +453,6 @@ export function materializeHermesPortableCreatePlan(input: { gpuRoutePlan: intent.gpuRoutePlan, compatibilityPolicyPath: null, sandboxGpuLogMessage: intent.sandboxGpuLogMessage, + activateDeferredProviderEffects: null, }; } diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 4f1b4424677..1ed1c5c62d9 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -588,6 +588,118 @@ describe("resolveSandboxCreateIntent", () => { expect(JSON.stringify(intent)).toBe(serializedIntent); }); + it("defers every provider effect and create attachment until activation (#9833)", () => { + const tokenDefs = [ + { + name: "sandbox-telegram-bridge", + envKey: "TELEGRAM_BOT_TOKEN", + token: "telegram-super-secret", + }, + ]; + const intent = resolveSandboxCreateIntent({ + basePolicyPath: "/repo/policy.yaml", + sandboxName: "sandbox", + inferenceProvider: "nvidia-prod", + channels, + enabledChannels: ["telegram"], + disabledChannelNames: new Set(), + messagingProviderRequests: resolveSandboxCreateMessagingProviderRequests( + tokenDefs, + () => "telegram", + ), + primaryMessagingCredentialEnvKeys: ["TELEGRAM_BOT_TOKEN"], + reusableMessagingChannels: [], + reusableMessagingProviders: ["sandbox-existing-discord"], + extraProviders: ["custom-provider"], + hermesToolGateways: ["github"], + sandboxGpuConfig: disabledSandboxGpuConfig, + gpuCreateArgs: [], + gpuRoutePlan: "none", + sandboxGpuLogMessage: null, + agentName: "hermes", + policyTier: null, + }); + const events: string[] = []; + const plan = materializeSandboxCreatePlan({ + intent, + fromRef: "example.invalid/image@sha256:abc", + policyAuthority: "externally-managed", + deferSandboxEffectsUntilPolicyVerification: true, + messagingTokenDefs: tokenDefs, + prepareInitialSandboxCreatePolicy: () => ({ + policyPath: "/tmp/policy.yaml", + appliedPresets: ["telegram"], + }), + runProviderPreDeleteCleanup: () => events.push("cleanup"), + upsertMessagingProviders: vi.fn(() => { + events.push("upsert"); + return ["sandbox-telegram-bridge"]; + }), + getHermesToolGatewayProviderName: () => { + events.push("hermes"); + return "sandbox-hermes-tools"; + }, + }); + + expect(events).toEqual([]); + expect(plan.createArgs).not.toContain("--provider"); + expect(plan.messagingProviders).toEqual([ + "sandbox-telegram-bridge", + "sandbox-existing-discord", + ]); + + expect(plan.activateDeferredProviderEffects?.()).toEqual([ + "nvidia-prod", + "sandbox-telegram-bridge", + "sandbox-existing-discord", + "sandbox-hermes-tools", + "custom-provider", + ]); + expect(events).toEqual(["cleanup", "upsert", "hermes"]); + }); + + it("keeps the NemoClaw policy on a managed create when effects are deferred (#9833)", () => { + const intent = resolveSandboxCreateIntent({ + basePolicyPath: "/repo/policy.yaml", + sandboxName: "sandbox", + inferenceProvider: "nvidia-prod", + channels, + enabledChannels: [], + disabledChannelNames: new Set(), + messagingProviderRequests: [], + primaryMessagingCredentialEnvKeys: [], + reusableMessagingChannels: [], + reusableMessagingProviders: [], + extraProviders: [], + hermesToolGateways: [], + sandboxGpuConfig: disabledSandboxGpuConfig, + gpuCreateArgs: [], + gpuRoutePlan: "none", + sandboxGpuLogMessage: null, + agentName: "openclaw", + policyTier: null, + }); + const plan = materializeSandboxCreatePlan({ + intent, + fromRef: "example.invalid/image@sha256:abc", + policyAuthority: "nemoclaw-managed", + deferSandboxEffectsUntilPolicyVerification: true, + messagingTokenDefs: [], + prepareInitialSandboxCreatePolicy: () => ({ + policyPath: "/tmp/policy.yaml", + appliedPresets: [], + }), + runProviderPreDeleteCleanup: vi.fn(), + upsertMessagingProviders: vi.fn(() => []), + getHermesToolGatewayProviderName: vi.fn(), + }); + + expect(plan.createArgs).toEqual( + expect.arrayContaining(["--policy", "/tmp/policy.yaml"]), + ); + expect(plan.createArgs).not.toContain("--provider"); + }); + it("omits caller policy when external authority owns sandbox policy (#9833)", () => { const intent = resolveSandboxCreateIntent({ basePolicyPath: "/repo/policy.yaml", diff --git a/src/lib/onboard/sandbox-create/orchestration.test.ts b/src/lib/onboard/sandbox-create/orchestration.test.ts index a328ffa8149..51631b86325 100644 --- a/src/lib/onboard/sandbox-create/orchestration.test.ts +++ b/src/lib/onboard/sandbox-create/orchestration.test.ts @@ -3,26 +3,125 @@ import { describe, expect, it, vi } from "vitest"; -import { decisionSelected } from "../../state/onboard-checkpoint-decision"; -import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; -import { createSession, normalizeSession } from "../../state/onboard-session"; import type { SandboxEntry } from "../../state/registry"; -import { - beginSandboxRecreateTransaction, - createCreatedSandboxLifecycle, - createSandboxRecreateRuntime, - fingerprintSandboxRecreateValue, - type SandboxRecreateObservation, -} from "../sandbox-recreate-transaction"; import { applyManagedSandboxRebuildPolicyCarryForward, + backfillVerifiedExternalSandboxPolicyAuthority, completeHermesPortableSandboxRegistration, + createProviderEffectBoundary, hasManagedMcpRebuildHandoff, readManagedDcodeCreateSelectionDrift, readSandboxRecreateRegistryEntry, runSandboxCreateWithPolicyAuthorityChecks, } from "./orchestration"; +describe("deferred provider effect authority", () => { + it("refuses a second provider attachment after policy authority changes (#9833)", async () => { + const events: string[] = []; + const recordPolicyCheck = (operation: string) => { + events.push(`policy: ${operation}`); + }; + const revalidatePolicyRequirements = vi.fn(recordPolicyCheck); + const runOpenshell = vi.fn((args: string[]) => { + events.push(args.join(" ")); + revalidatePolicyRequirements + .mockImplementationOnce(recordPolicyCheck) + .mockImplementationOnce((operation) => { + recordPolicyCheck(operation); + throw new Error("policy authority changed after the first provider attachment"); + }); + return { status: 0 }; + }); + const revalidateSandboxIdentity = vi.fn((_exactIdentity: string, operation: string) => { + events.push(`identity: ${operation}`); + }); + const boundary = createProviderEffectBoundary({ + deferred: true, + sandboxName: "alpha", + gatewayName: "nemoclaw", + preparationInput: { + openshellDriver: "docker", + inferenceProvider: null, + messagingProviders: [], + messagingProviderRequests: [], + extraProviders: [], + gatewayName: "nemoclaw", + }, + preparationDeps: { + providerExistsInGateway: vi.fn(() => true), + runOpenshell: runOpenshell as never, + cleanupCreateSources: vi.fn(), + }, + runVerifiedSandboxCreateEffects: null, + activateDeferredProviderEffects: () => ["first", "second"], + revalidatePolicyAuthorityBeforeCreate: vi.fn(), + runOpenshell: runOpenshell as never, + revalidateSandboxIdentity, + }); + const runAfterVerifiedCreate = boundary.runAfterVerifiedCreate; + expect(runAfterVerifiedCreate).toBeTypeOf("function"); + + await expect( + runAfterVerifiedCreate?.({ + registration: { + policyAuthority: "externally-managed", + policyCreationReceipt: null, + observedPolicyAuthority: "externally-managed", + policyIdentity: { hash: "b".repeat(64), activeVersion: 1 }, + }, + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 18790, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + route: "direct" as never, + revalidatePolicyRequirements, + }), + ).rejects.toThrow("policy authority changed after the first provider attachment"); + + expect(runOpenshell).toHaveBeenCalledExactlyOnceWith( + ["sandbox", "provider", "attach", "-g", "nemoclaw", "alpha", "first"], + { ignoreError: true, suppressOutput: true }, + ); + expect(revalidateSandboxIdentity).toHaveBeenCalledWith( + "a".repeat(64), + "attaching provider 'second' to sandbox 'alpha'", + ); + expect(events).toContain("policy: attaching provider 'second' to sandbox 'alpha'"); + expect(events).not.toContain("sandbox provider attach -g nemoclaw alpha second"); + }); +}); + +describe("policy authority backfill", () => { + it("does not assign managed authority before completed sandbox registration (#9833)", () => { + const updateSandbox = vi.fn(() => true); + + backfillVerifiedExternalSandboxPolicyAuthority({ + sandboxName: "alpha", + existingEntry: { name: "alpha", pendingRouteReservation: true }, + policyAuthority: "nemoclaw-managed", + updateSandbox, + }); + + expect(updateSandbox).not.toHaveBeenCalled(); + }); + + it("records verified external authority on an unattributed existing row (#9833)", () => { + const updateSandbox = vi.fn(() => true); + + backfillVerifiedExternalSandboxPolicyAuthority({ + sandboxName: "alpha", + existingEntry: { name: "alpha" }, + policyAuthority: "externally-managed", + updateSandbox, + }); + + expect(updateSandbox).toHaveBeenCalledExactlyOnceWith("alpha", { + policyAuthority: "externally-managed", + }); + }); +}); + describe("managed MCP rebuild handoff", () => { const targetIntentFingerprint = "a".repeat(64); const recreateTransaction = { @@ -238,12 +337,17 @@ describe("Hermes portable registration adapter", () => { }); describe("sandbox create policy authority checks", () => { - const CREATED_IDENTITY = "a".repeat(64); - const REPLACEMENT_IDENTITY = "b".repeat(64); - const createdObservation = { - state: "ready" as const, - liveIdentityFingerprint: CREATED_IDENTITY, - }; + const exactIdentity = "a".repeat(64); + const verifiedPolicyBoundary = () => ({ + verifyCreatedPolicy: vi.fn(() => "verified"), + persistVerifiedPolicy: vi.fn(), + revalidateVerifiedPolicy: vi.fn(), + }); + const exactIdentityBoundary = () => ({ + captureCreatedSandboxIdentity: vi.fn(() => exactIdentity), + revalidateCreatedSandboxIdentity: vi.fn(), + ...verifiedPolicyBoundary(), + }); it("refuses sandbox creation before mutation when the final check fails (#9833)", async () => { const create = vi.fn(async () => "created"); @@ -254,9 +358,8 @@ describe("sandbox create policy authority checks", () => { revalidate: () => { throw new Error("external policy authority must supply the selected route"); }, + ...exactIdentityBoundary(), create, - captureCreatedSandboxLiveIdentity: vi.fn(() => CREATED_IDENTITY), - observeCreatedSandbox: vi.fn(() => createdObservation), cleanupTemporarySources: vi.fn(), }), ).rejects.toThrow(/external policy authority must supply/u); @@ -268,15 +371,22 @@ describe("sandbox create policy authority checks", () => { const result = await runSandboxCreateWithPolicyAuthorityChecks({ sandboxName: "alpha", revalidate: (sandboxIsLive) => events.push(sandboxIsLive ? "ready-check" : "create-check"), - create: async () => { + create: async (verifyCreatedSandbox) => { events.push("create"); + await verifyCreatedSandbox("created"); return "created"; }, - captureCreatedSandboxLiveIdentity: () => { + captureCreatedSandboxIdentity: () => { events.push("capture-identity"); - return CREATED_IDENTITY; + return exactIdentity; + }, + revalidateCreatedSandboxIdentity: () => events.push("identity-check"), + verifyCreatedPolicy: () => { + events.push("policy-check"); + return "verified"; }, - observeCreatedSandbox: vi.fn(() => createdObservation), + persistVerifiedPolicy: () => events.push("persist-checkpoint"), + revalidateVerifiedPolicy: () => events.push("revalidate-checkpoint"), cleanupTemporarySources: vi.fn(), }); events.push("register"); @@ -286,91 +396,72 @@ describe("sandbox create policy authority checks", () => { "create-check", "create", "capture-identity", - "ready-check", + "identity-check", + "policy-check", + "identity-check", + "persist-checkpoint", + "revalidate-checkpoint", + "identity-check", + "identity-check", "register", ]); }); - it("removes create sources and reports the surviving sandbox when final validation fails (#9833)", async () => { + it("removes temporary sources but preserves the sandbox after final authority failure (#9833)", async () => { const events: string[] = []; - const validationError = new Error("external policy authority changed"); - const revalidate = vi - .fn() - .mockImplementationOnce(() => events.push("create-check")) - .mockImplementationOnce(() => { - events.push("ready-check"); - throw validationError; - }); - - await expect( - runSandboxCreateWithPolicyAuthorityChecks({ - sandboxName: "alpha", - revalidate, - create: async () => { - events.push("create"); - return "created"; - }, - captureCreatedSandboxLiveIdentity: () => { - events.push("capture-identity"); - return CREATED_IDENTITY; - }, - observeCreatedSandbox: () => { - events.push("observe-created"); - return createdObservation; - }, - cleanupTemporarySources: () => { - events.push("cleanup-sources"); - }, - }), - ).rejects.toThrow("cleanup did not complete"); + const revalidate = vi.fn(() => events.push("create-check")); - expect(events).toEqual([ - "create-check", - "create", - "capture-identity", - "ready-check", - "cleanup-sources", - "observe-created", - ]); - }); - - it("reports both source cleanup failure and the surviving sandbox (#9833)", async () => { - const revalidate = vi - .fn() - .mockImplementationOnce(() => undefined) - .mockImplementationOnce(() => { + const error = await runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate, + create: async (verifyCreatedSandbox) => { + events.push("create"); + await verifyCreatedSandbox("created"); + return "created"; + }, + ...exactIdentityBoundary(), + revalidateVerifiedPolicy: () => { + events.push("ready-check"); throw new Error("external policy authority changed"); - }); + }, + cleanupTemporarySources: () => events.push("cleanup-sources"), + }).catch((caught: unknown) => caught); - await expect( - runSandboxCreateWithPolicyAuthorityChecks({ - sandboxName: "alpha", - revalidate, - create: async () => "created", - captureCreatedSandboxLiveIdentity: () => CREATED_IDENTITY, - observeCreatedSandbox: () => createdObservation, - cleanupTemporarySources: () => { - throw new Error("temporary source cleanup failed"); - }, - }), - ).rejects.toThrow("cleanup did not complete"); + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringMatching( + new RegExp( + `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*Do not delete the sandbox by name, even after this comparison.*Contact the OpenShell administrator for an identity-bound recovery or removal procedure`, + "u", + ), + ), + }), + ]), + ); + expect(events).toEqual(["create-check", "create", "ready-check", "cleanup-sources"]); }); - it("does not delete a matching identity through its mutable name (#9833)", async () => { - const observeCreatedSandbox = vi.fn(() => createdObservation); - const revalidate = vi - .fn() - .mockImplementationOnce(() => undefined) - .mockImplementationOnce(() => { - throw new Error("external policy authority changed"); - }); + it("does not delete a same-name replacement after final authority failure (#9833)", async () => { + let sandboxIdentity = "created"; + const revalidate = vi.fn(); + const revalidateCreatedSandboxIdentity = vi.fn(); const error = await runSandboxCreateWithPolicyAuthorityChecks({ sandboxName: "alpha", revalidate, - create: async () => "created", - captureCreatedSandboxLiveIdentity: () => CREATED_IDENTITY, - observeCreatedSandbox, + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + captureCreatedSandboxIdentity: () => exactIdentity, + revalidateCreatedSandboxIdentity, + ...verifiedPolicyBoundary(), + revalidateVerifiedPolicy: () => { + sandboxIdentity = "replacement"; + throw new Error("sandbox identity changed"); + }, cleanupTemporarySources: vi.fn(), }).catch((caught: unknown) => caught); @@ -378,181 +469,259 @@ describe("sandbox create policy authority checks", () => { expect((error as AggregateError).errors).toEqual( expect.arrayContaining([ expect.objectContaining({ - message: expect.stringContaining("deletion targets its mutable name"), + message: expect.stringMatching( + new RegExp( + `left sandbox 'alpha' in place.*identity fingerprint: ${exactIdentity}.*Do not delete the sandbox by name, even after this comparison`, + "u", + ), + ), }), ]), ); - expect(observeCreatedSandbox).toHaveBeenCalledOnce(); + expect(revalidateCreatedSandboxIdentity).toHaveBeenNthCalledWith( + 1, + exactIdentity, + "verifying effective policy for sandbox 'alpha'", + ); + expect(revalidateCreatedSandboxIdentity).toHaveBeenNthCalledWith( + 2, + exactIdentity, + "recording verified policy for sandbox 'alpha'", + ); + expect(sandboxIdentity).toBe("replacement"); }); - it("does not delete a replacement observed before cleanup (#9833)", async () => { - const revalidate = vi - .fn() - .mockImplementationOnce(() => undefined) - .mockImplementationOnce(() => { - throw new Error("external policy authority changed"); - }); + it("reports temporary source cleanup failure with sandbox preservation (#9833)", async () => { + const revalidate = vi.fn(); const error = await runSandboxCreateWithPolicyAuthorityChecks({ sandboxName: "alpha", revalidate, - create: async () => "created", - captureCreatedSandboxLiveIdentity: () => CREATED_IDENTITY, - observeCreatedSandbox: () => ({ - state: "ready", - liveIdentityFingerprint: REPLACEMENT_IDENTITY, - }), - cleanupTemporarySources: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + ...exactIdentityBoundary(), + revalidateVerifiedPolicy: () => { + throw new Error("external policy authority changed"); + }, + cleanupTemporarySources: () => { + throw new Error("temporary source cleanup failed"); + }, }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(AggregateError); expect((error as AggregateError).errors).toEqual( expect.arrayContaining([ - expect.objectContaining({ message: expect.stringContaining("live identity changed") }), + expect.objectContaining({ message: "temporary source cleanup failed" }), + expect.objectContaining({ message: expect.stringContaining("left sandbox 'alpha'") }), ]), ); }); - it("persists refusal recovery identity and rejects a same-name replacement after reload (#9833)", async () => { - const gatewayName = "nemoclaw-31818"; - const gatewayPort = 31818; - const transactionId = "11111111-1111-4111-8111-111111111111"; - const targetGeneration = "22222222-2222-4222-8222-222222222222"; - const targetIntentFingerprint = fingerprintSandboxRecreateValue("fresh-create"); - const session = createSession({ sandboxName: "alpha", agent: "openclaw" }); - session.checkpoint = { - ...deriveCheckpointFromSession(session), - sandboxIdentity: decisionSelected({ name: "alpha", agent: "openclaw" }), - gatewayAuthority: decisionSelected({ - gatewayName, - gatewayPort, - mode: "nemoclaw-managed", - source: "standalone", - endpoint: null, - stateDir: null, - supervisor: null, - requiredCapabilities: [], - }), - }; - const transaction = beginSandboxRecreateTransaction(session, { + it("runs continuation effects only after policy and identity verification (#9833)", async () => { + const events: string[] = []; + + const result = await runSandboxCreateWithPolicyAuthorityChecks({ sandboxName: "alpha", - gatewayName, - gatewayPort, - sourceEntry: null, - observation: { state: "missing", liveIdentityFingerprint: null }, - targetIntentFingerprint, - id: transactionId, - targetGeneration, - }); - let observation: SandboxRecreateObservation = { - state: "missing", - liveIdentityFingerprint: null, - }; - const sessionStore = { - loadSession: () => session, - updateSession: (mutator: (current: typeof session) => typeof session | void) => { - mutator(session); - return session; + revalidate: (sandboxIsLive) => events.push(sandboxIsLive ? "policy" : "preflight"), + create: async (verifyCreatedSandbox) => { + events.push("create-without-policy"); + await verifyCreatedSandbox({ sandboxName: "alpha" }); + return "complete"; }, - }; - const request = { - id: transaction.id, - targetGeneration: transaction.targetGeneration, - targetIntentFingerprint: transaction.targetIntentFingerprint, - }; - const runtime = createSandboxRecreateRuntime( - sessionStore, - request, - "alpha", - gatewayName, - null, - () => observation, - vi.fn(), - ); - runtime.advance("creating"); - const lifecycle = createCreatedSandboxLifecycle( - runtime, - { sandboxName: "alpha", gatewayName }, - () => observation, - ); - observation = createdObservation; - const revalidate = vi - .fn() - .mockImplementationOnce(() => undefined) - .mockImplementationOnce(() => { - throw new Error("external policy authority changed"); - }); + runVerifiedCreateEffects: async () => { + events.push("provider-effects"); + }, + captureCreatedSandboxIdentity: () => { + events.push("capture"); + return exactIdentity; + }, + revalidateCreatedSandboxIdentity: () => events.push("identity"), + verifyCreatedPolicy: () => { + events.push("policy"); + return "verified"; + }, + persistVerifiedPolicy: () => events.push("checkpoint"), + revalidateVerifiedPolicy: () => events.push("checkpoint-revalidate"), + cleanupTemporarySources: vi.fn(), + }); + + expect(result).toBe("complete"); + expect(events).toEqual([ + "preflight", + "create-without-policy", + "capture", + "identity", + "policy", + "identity", + "checkpoint", + "checkpoint-revalidate", + "provider-effects", + "identity", + "identity", + ]); + }); + + it("withholds checkpoint and effects when post-create policy verification fails (#9833)", async () => { + const persistVerifiedPolicy = vi.fn(); + const runVerifiedCreateEffects = vi.fn(); await expect( runSandboxCreateWithPolicyAuthorityChecks({ sandboxName: "alpha", - revalidate, - create: async () => "created", - captureCreatedSandboxLiveIdentity: () => - lifecycle.capture({ lifecycleGeneration: targetGeneration }) - .lifecycleLiveIdentityFingerprint, - observeCreatedSandbox: () => observation, + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + captureCreatedSandboxIdentity: () => exactIdentity, + revalidateCreatedSandboxIdentity: vi.fn(), + verifyCreatedPolicy: () => { + throw new Error("policy verification failed"); + }, + persistVerifiedPolicy, + revalidateVerifiedPolicy: vi.fn(), + runVerifiedCreateEffects, cleanupTemporarySources: vi.fn(), }), - ).rejects.toThrow("cleanup did not complete"); + ).rejects.toThrow("automatic sandbox cleanup was not safe"); - expect(session.checkpoint?.sandboxRecreate).toMatchObject({ - sandboxName: "alpha", - gatewayName, - gatewayPort, - phase: "created", - targetLiveIdentityFingerprint: CREATED_IDENTITY, - }); - const restored = normalizeSession(JSON.parse(JSON.stringify(session))); - expect(restored).not.toBeNull(); - const restoredSession = restored!; - expect(restoredSession.checkpoint?.sandboxRecreate).toMatchObject(request); - observation = { state: "ready", liveIdentityFingerprint: REPLACEMENT_IDENTITY }; - expect(() => - createSandboxRecreateRuntime( - { - loadSession: () => restoredSession, - updateSession: (mutator) => { - mutator(restoredSession); - return restoredSession; - }, + expect(persistVerifiedPolicy).not.toHaveBeenCalled(); + expect(runVerifiedCreateEffects).not.toHaveBeenCalled(); + }); + + it("withholds effects when durable checkpoint persistence fails (#9833)", async () => { + const revalidateVerifiedPolicy = vi.fn(); + const runVerifiedCreateEffects = vi.fn(); + + await expect( + runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; }, - request, - "alpha", - gatewayName, - null, - () => observation, - vi.fn(), - ), - ).toThrow(/not the journaled created sandbox/u); + captureCreatedSandboxIdentity: () => exactIdentity, + revalidateCreatedSandboxIdentity: vi.fn(), + verifyCreatedPolicy: () => "verified", + persistVerifiedPolicy: () => { + throw new Error("checkpoint persistence failed"); + }, + revalidateVerifiedPolicy, + runVerifiedCreateEffects, + cleanupTemporarySources: vi.fn(), + }), + ).rejects.toThrow("automatic sandbox cleanup was not safe"); + + expect(revalidateVerifiedPolicy).not.toHaveBeenCalled(); + expect(runVerifiedCreateEffects).not.toHaveBeenCalled(); + }); + + it("retains the checkpoint and withholds effects when its reread fails (#9833)", async () => { + const persistVerifiedPolicy = vi.fn(); + const runVerifiedCreateEffects = vi.fn(); + + await expect( + runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + captureCreatedSandboxIdentity: () => exactIdentity, + revalidateCreatedSandboxIdentity: vi.fn(), + verifyCreatedPolicy: () => "verified", + persistVerifiedPolicy, + revalidateVerifiedPolicy: () => { + throw new Error("durable checkpoint missing"); + }, + runVerifiedCreateEffects, + cleanupTemporarySources: vi.fn(), + }), + ).rejects.toThrow("automatic sandbox cleanup was not safe"); + + expect(persistVerifiedPolicy).toHaveBeenCalledOnce(); + expect(runVerifiedCreateEffects).not.toHaveBeenCalled(); }); - it("leaves the named sandbox in place when its created identity was not captured (#9833)", async () => { - const observeCreatedSandbox = vi.fn(() => createdObservation); + it("retains the durable checkpoint when a deferred effect fails (#9833)", async () => { + const persistVerifiedPolicy = vi.fn(); + + await expect( + runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate: vi.fn(), + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + return "created"; + }, + captureCreatedSandboxIdentity: () => exactIdentity, + revalidateCreatedSandboxIdentity: vi.fn(), + verifyCreatedPolicy: () => "verified", + persistVerifiedPolicy, + revalidateVerifiedPolicy: vi.fn(), + runVerifiedCreateEffects: async () => { + throw new Error("provider effect failed"); + }, + cleanupTemporarySources: vi.fn(), + }), + ).rejects.toThrow("automatic sandbox cleanup was not safe"); + + expect(persistVerifiedPolicy).toHaveBeenCalledOnce(); + }); + + it("refuses continuation when identity changes during effective-policy verification (#9833)", async () => { + const continuationEffect = vi.fn(); const revalidate = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => undefined); + const revalidateCreatedSandboxIdentity = vi .fn() .mockImplementationOnce(() => undefined) .mockImplementationOnce(() => { - throw new Error("external policy authority changed"); + throw new Error("sandbox identity changed"); }); + await expect( + runSandboxCreateWithPolicyAuthorityChecks({ + sandboxName: "alpha", + revalidate, + create: async (verifyCreatedSandbox) => { + await verifyCreatedSandbox("created"); + continuationEffect(); + return "created"; + }, + captureCreatedSandboxIdentity: () => exactIdentity, + revalidateCreatedSandboxIdentity, + ...verifiedPolicyBoundary(), + cleanupTemporarySources: vi.fn(), + }), + ).rejects.toThrow("automatic sandbox cleanup was not safe"); + + expect(continuationEffect).not.toHaveBeenCalled(); + }); + + it("fails closed when a create implementation skips the post-create gate (#9833)", async () => { + const cleanupTemporarySources = vi.fn(); + const error = await runSandboxCreateWithPolicyAuthorityChecks({ sandboxName: "alpha", - revalidate, + revalidate: vi.fn(), create: async () => "created", - captureCreatedSandboxLiveIdentity: () => { - throw new Error("identity probe failed"); - }, - observeCreatedSandbox, - cleanupTemporarySources: vi.fn(), + ...exactIdentityBoundary(), + cleanupTemporarySources, }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(AggregateError); expect((error as AggregateError).errors).toEqual( expect.arrayContaining([ - expect.objectContaining({ message: expect.stringContaining("identity was not captured") }), + expect.objectContaining({ message: expect.stringContaining("post-create verification") }), ]), ); - expect(observeCreatedSandbox).not.toHaveBeenCalled(); + expect(cleanupTemporarySources).toHaveBeenCalledOnce(); }); }); diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 6e3bb33fc86..0d44010322c 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -8,6 +8,10 @@ import type { AgentDefinition } from "../../agent/defs"; import type { WebSearchConfig } from "../../inference/web-search"; import type { BackupResult } from "../../state/sandbox"; import type { SandboxEntry } from "../../state/registry"; +import type { + PendingSandboxPolicyVerification, + QualifiedPendingSandboxCreateReservation, +} from "../../state/registry"; import type { HermesAuthMethod } from "../hermes-auth"; import * as policyAuthorityPreflight from "../policy-authority/preflight"; import type { PreparedSandboxBuildContext } from "../build-context-stage"; @@ -15,10 +19,21 @@ import type { DcodeSelectionDriftReader } from "../dcode-selection-drift"; import type { OwnedSandboxRecreateRuntime } from "../onboard-recreate-journal"; import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; import type { PortableOnboardRuntimeContext } from "../session-bootstrap"; -import type { InferenceRouteReservationAuthority, SandboxCreateIntent } from "../types"; -import type { SandboxRecreateObservation } from "../sandbox-recreate-transaction"; +import type { + InferenceRouteReservationAuthority, + SandboxCreateIntent, + VerifiedSandboxCreateEffectsContext, + VerifiedSandboxPolicyBoundary, + VerifiedSandboxPolicyRegistration, +} from "../types"; import * as sandboxCreatePlanMaterialization from "../sandbox-create-plan-materialization"; import { + pendingSandboxPolicyVerificationForBoundary, + revalidateCreatedSandboxPolicyRegistration, + verifyCreatedSandboxPolicyRegistration, +} from "./policy-creation-receipt"; +import { + attachProvidersAfterSandboxCreation, publishAttachedProvidersBeforeDockerSandboxCreation, validateAttachedMessagingProvidersBeforeSandboxCreation, } from "./provider-publication"; @@ -119,75 +134,135 @@ export async function completeHermesPortableSandboxRegistration(input: { return registered; } -/** Revalidate create-time authority before mutation and against the named Ready sandbox. */ -export async function runSandboxCreateWithPolicyAuthorityChecks(input: { +/** + * Keep every effect after an unverified create behind one exact-identity policy gate. + * + * The create callback owns the create transaction. It must await the supplied gate + * immediately after OpenShell returns the exact created identity and before provider, + * credential, service, runtime, registry, or completion effects. + */ +export async function runSandboxCreateWithPolicyAuthorityChecks< + Created, + Evidence, + Result = Created, +>(input: { readonly sandboxName: string; readonly revalidate: (sandboxIsLive: boolean, operation: string) => void; - readonly create: () => Promise; - readonly captureCreatedSandboxLiveIdentity: (created: Result) => string; - readonly observeCreatedSandbox: () => SandboxRecreateObservation; + readonly create: (verifyCreatedSandbox: (created: Created) => Promise) => Promise; + readonly captureCreatedSandboxIdentity: (created: Created) => string; + readonly revalidateCreatedSandboxIdentity: (expectedIdentity: string, operation: string) => void; + readonly verifyCreatedPolicy: (created: Created, exactIdentity: string) => Evidence; + readonly persistVerifiedPolicy: ( + created: Created, + exactIdentity: string, + evidence: Evidence, + ) => void; + readonly revalidateVerifiedPolicy: ( + created: Created, + exactIdentity: string, + evidence: Evidence, + operation: string, + ) => void; + readonly runVerifiedCreateEffects?: ( + created: Created, + exactIdentity: string, + evidence: Evidence, + ) => Promise; readonly cleanupTemporarySources: () => void; }): Promise { input.revalidate(false, `creating sandbox '${input.sandboxName}'`); - const created = await input.create(); - let createdLiveIdentityFingerprint: string | null = null; - let createdIdentityCaptureError: unknown = null; - try { - const captured = input.captureCreatedSandboxLiveIdentity(created); - if (!/^[a-f0-9]{64}$/u.test(captured)) { - throw new Error("OpenShell did not return one exact durable sandbox identity."); - } - createdLiveIdentityFingerprint = captured; - } catch (error) { - createdIdentityCaptureError = error; - } - try { - input.revalidate(true, `finalizing sandbox '${input.sandboxName}'`); - } catch (validationError) { - const compensationErrors: unknown[] = []; + let exactIdentity: string | null = null; + let cleanupAttempted = false; + const cleanupTemporarySources = (): unknown[] => { + if (cleanupAttempted) return []; + cleanupAttempted = true; try { input.cleanupTemporarySources(); + return []; } catch (error) { - compensationErrors.push(error); + return [error]; } - if (!createdLiveIdentityFingerprint) { - compensationErrors.push( - new Error( - `Cannot safely remove sandbox '${input.sandboxName}' after policy authority validation failed because its exact created identity was not captured. NemoClaw left the named sandbox in place.`, - createdIdentityCaptureError === null ? undefined : { cause: createdIdentityCaptureError }, - ), - ); - } else { - try { - const beforeDelete = input.observeCreatedSandbox(); - if (beforeDelete.state !== "missing") { - if (!beforeDelete.liveIdentityFingerprint) { - throw new Error( - `Cannot safely remove sandbox '${input.sandboxName}' after policy authority validation failed because its current identity is ambiguous. NemoClaw left the named sandbox in place.`, - ); - } - if (beforeDelete.liveIdentityFingerprint !== createdLiveIdentityFingerprint) { - throw new Error( - `Cannot safely remove sandbox '${input.sandboxName}' after policy authority validation failed because its live identity changed. NemoClaw left the replacement in place.`, - ); - } - throw new Error( - `Cannot safely remove sandbox '${input.sandboxName}' after policy authority validation failed because OpenShell deletion targets its mutable name rather than the captured identity. NemoClaw left the created sandbox in place for manual identity-checked cleanup.`, - ); - } - } catch (error) { - compensationErrors.push(error); + }; + const refuseAfterCreate = (validationError: unknown): never => { + const compensationErrors = cleanupTemporarySources(); + const identityGuidance = exactIdentity + ? ` Durable sandbox identity fingerprint: ${exactIdentity}. Use it only to compare the surviving sandbox with the failed create. Do not delete the sandbox by name, even after this comparison. Contact the OpenShell administrator for an identity-bound recovery or removal procedure.` + : " OpenShell did not return a durable identity for comparison. Do not delete the sandbox by name. Contact the OpenShell administrator for an identity-bound recovery or removal procedure."; + compensationErrors.push( + new Error( + `NemoClaw left sandbox '${input.sandboxName}' in place after policy authority validation failed because OpenShell can delete it only by mutable name.${identityGuidance}`, + ), + ); + throw new AggregateError( + [validationError, ...compensationErrors], + "Sandbox policy authority validation failed after creation; automatic sandbox cleanup was not safe.", + ); + }; + const verifyCreatedSandbox = async (created: Created): Promise => { + try { + const capturedIdentity = input.captureCreatedSandboxIdentity(created); + if (!/^[0-9a-f]{64}$/u.test(capturedIdentity)) { + throw new Error( + `OpenShell did not return one exact durable identity for sandbox '${input.sandboxName}'.`, + ); } + exactIdentity = capturedIdentity; + input.revalidateCreatedSandboxIdentity( + capturedIdentity, + `verifying effective policy for sandbox '${input.sandboxName}'`, + ); + const evidence = input.verifyCreatedPolicy(created, capturedIdentity); + input.revalidateCreatedSandboxIdentity( + capturedIdentity, + `recording verified policy for sandbox '${input.sandboxName}'`, + ); + input.persistVerifiedPolicy(created, capturedIdentity, evidence); + input.revalidateVerifiedPolicy( + created, + capturedIdentity, + evidence, + `continuing onboarding for sandbox '${input.sandboxName}'`, + ); + await input.runVerifiedCreateEffects?.(created, capturedIdentity, evidence); + input.revalidateCreatedSandboxIdentity( + capturedIdentity, + `confirming verified effects for sandbox '${input.sandboxName}'`, + ); + return capturedIdentity; + } catch (validationError) { + return refuseAfterCreate(validationError); } - if (compensationErrors.length > 0) { + }; + let result: Result; + try { + result = await input.create(verifyCreatedSandbox); + } catch (error) { + const cleanupErrors = cleanupTemporarySources(); + if (cleanupErrors.length > 0) { throw new AggregateError( - [validationError, ...compensationErrors], - "Sandbox policy authority validation failed after creation, and cleanup did not complete.", + [error, ...cleanupErrors], + "Sandbox creation failed, and temporary source cleanup did not complete.", ); } - throw validationError; + throw error; + } + const verifiedIdentity = exactIdentity; + if (verifiedIdentity === null) { + return refuseAfterCreate( + new Error( + `Sandbox '${input.sandboxName}' creation returned before its exact post-create verification boundary ran.`, + ), + ); } - return created; + try { + input.revalidateCreatedSandboxIdentity( + verifiedIdentity, + `completing sandbox creation for '${input.sandboxName}'`, + ); + } catch (validationError) { + return refuseAfterCreate(validationError); + } + return result; } function assertHermesPortablePolicyAuthority( @@ -219,7 +294,7 @@ function hasPreservedManagedMcpRebuildHandoff( return Boolean(preservedMcpState) && hasManagedMcpRebuildHandoff(createIntent); } -function backfillExistingSandboxPolicyAuthority(input: { +export function backfillVerifiedExternalSandboxPolicyAuthority(input: { readonly sandboxName: string; readonly existingEntry: SandboxEntry | null; readonly policyAuthority: "nemoclaw-managed" | "externally-managed"; @@ -228,7 +303,13 @@ function backfillExistingSandboxPolicyAuthority(input: { updates: { policyAuthority: "nemoclaw-managed" | "externally-managed" }, ) => boolean; }): void { - if (!input.existingEntry || input.existingEntry.policyAuthority) return; + if ( + !input.existingEntry || + input.existingEntry.policyAuthority || + input.policyAuthority !== "externally-managed" + ) { + return; + } if (input.updateSandbox(input.sandboxName, { policyAuthority: input.policyAuthority })) return; throw new Error(`Could not record policy authority for sandbox '${input.sandboxName}'.`); } @@ -299,6 +380,91 @@ export function proveRecreateSourceBeforePolicyCarryForward(input: { return runtime; } +type ProviderPreparationInput = Parameters< + typeof validateAttachedMessagingProvidersBeforeSandboxCreation +>[0]; +type ProviderPreparationDeps = Parameters< + typeof validateAttachedMessagingProvidersBeforeSandboxCreation +>[1]; + +type ProviderEffectBoundary = { + readonly validateBeforeCreate: () => void; + readonly publishBeforeCreate: () => void; + readonly runAfterVerifiedCreate: + | ((context: VerifiedSandboxCreateEffectsContext) => Promise) + | undefined; +}; + +export function createProviderEffectBoundary(input: { + readonly deferred: boolean; + readonly sandboxName: string; + readonly gatewayName: string; + readonly preparationInput: ProviderPreparationInput; + readonly preparationDeps: ProviderPreparationDeps; + readonly runVerifiedSandboxCreateEffects: import("../types").VerifiedSandboxCreateEffects | null; + readonly activateDeferredProviderEffects: (() => readonly string[]) | null; + readonly revalidatePolicyAuthorityBeforeCreate: () => void; + readonly runOpenshell: SandboxCreateOrchestrationRuntime["runOpenshell"]; + readonly revalidateSandboxIdentity: (exactIdentity: string, operation: string) => void; +}): ProviderEffectBoundary { + const validate = () => + validateAttachedMessagingProvidersBeforeSandboxCreation( + input.preparationInput, + input.preparationDeps, + ); + const publish = () => + publishAttachedProvidersBeforeDockerSandboxCreation( + input.preparationInput, + input.preparationDeps, + ); + if (!input.deferred) { + return { + validateBeforeCreate: validate, + publishBeforeCreate: () => { + input.revalidatePolicyAuthorityBeforeCreate(); + publish(); + }, + runAfterVerifiedCreate: undefined, + }; + } + return { + validateBeforeCreate: () => undefined, + publishBeforeCreate: () => undefined, + runAfterVerifiedCreate: async (context) => { + context.revalidatePolicyRequirements( + `starting deferred provider effects for sandbox '${input.sandboxName}'`, + ); + await input.runVerifiedSandboxCreateEffects?.(context); + context.revalidatePolicyRequirements( + `activating deferred providers for sandbox '${input.sandboxName}'`, + ); + const providerNames = input.activateDeferredProviderEffects?.() ?? []; + validate(); + context.revalidatePolicyRequirements( + `publishing deferred providers for sandbox '${input.sandboxName}'`, + ); + publish(); + context.revalidatePolicyRequirements( + `attaching deferred providers to sandbox '${input.sandboxName}'`, + ); + attachProvidersAfterSandboxCreation( + { + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + providerNames, + }, + { + runOpenshell: input.runOpenshell, + revalidateSandboxIdentity: (operation) => { + input.revalidateSandboxIdentity(context.lifecycleLiveIdentityFingerprint, operation); + context.revalidatePolicyRequirements(operation); + }, + }, + ); + }, + }; +} + export function readSandboxRecreateRegistryEntry(input: { readonly sandboxName: string; readonly recreateTransaction: boolean; @@ -356,6 +522,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche hermesAuthMethod: HermesAuthMethod | null = null, inferenceRouteReservationAuthority: InferenceRouteReservationAuthority | null = null, createIntent: import("../types").SandboxCreateIntent | null = null, + runVerifiedSandboxCreateEffects: import("../types").VerifiedSandboxCreateEffects | null = null, preparedBuildContext: PreparedSandboxBuildContext | null = null, ) { const portableRuntimeAuthority = portableRuntimeContext?.authority ?? null; @@ -451,6 +618,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche validateName, verifyDirectSandboxGpu, waitForSandboxRecreateDeleteAbsence, + wasSandboxDefault, updateReusedSandboxMetadata, getSandboxInferenceConfig, redact, @@ -677,17 +845,21 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche const qualifyPolicyAuthority = ( sandboxIsLive = liveExists, operation = `prepare sandbox '${sandboxName}'`, - ) => - policyAuthorityPreflight.qualifySandboxPolicyAuthority({ + ) => { + const recordedSandbox = sandboxIsLive ? registry.getSandbox(sandboxName) : existingEntry; + return policyAuthorityPreflight.qualifySandboxPolicyAuthority({ sandboxName, gatewayName: GATEWAY_NAME, liveExists: sandboxIsLive, recordedAuthorities: [existingEntry?.policyAuthority, sessionPolicyAuthority], + recordedSandbox, + readRecordedSandbox: registry.getSandbox, prepareRequiredPolicy: () => sandboxCreatePlanMaterialization.prepareSandboxCreatePolicy(policyRequirementIntent) .initialSandboxPolicy, operation, }); + }; const policyAuthorityInspection = qualifyPolicyAuthority(); const resolvedPolicyAuthority = policyAuthorityInspection.authority; const revalidatePolicyAuthority = (sandboxIsLive: boolean, operation: string): void => { @@ -698,15 +870,16 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche agentCreateInput.hermesPortableLifecycle, resolvedPolicyAuthority, ); - backfillExistingSandboxPolicyAuthority({ + backfillVerifiedExternalSandboxPolicyAuthority({ sandboxName, existingEntry, policyAuthority: resolvedPolicyAuthority, updateSandbox: registry.updateSandbox, }); onboardSession.updateSession((session) => { - session.policyAuthority = resolvedPolicyAuthority; - if (resolvedPolicyAuthority === "externally-managed") session.policyPresets = null; + session.policyAuthority = + resolvedPolicyAuthority === "externally-managed" ? "externally-managed" : null; + if (session.policyAuthority === "externally-managed") session.policyPresets = null; }); const recreateRegistryEntry = readSandboxRecreateRegistryEntry({ sandboxName, @@ -764,12 +937,15 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche releaseDashboardPort: dashboardPortReservationScope.release, revalidatePolicyRequirements: (operation) => revalidatePolicyAuthority(true, operation), })); - note(` [reuse] Skipping sandbox (${sandboxName})`); }; if (recreateRuntime.acceptedTarget) { await restoreReusedSandboxDashboard(true); return sandboxName; } + // #4614: capture default AFTER prune so a stale registry row isn't read as a live sandbox. + const sandboxWasLiveDefault = + liveExists && wasSandboxDefault(registry.getDefault(), sandboxName); + let pendingStateRestore: BackupResult | null = null; let notReadyRecreateInProgress = false; const customOpenClawImage = @@ -950,6 +1126,9 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche " [non-interactive] Set NEMOCLAW_RECREATE_SANDBOX=1 (or --recreate-sandbox) to force recreation.", ); } else { + note( + ` [non-interactive] Sandbox '${sandboxName}' exists and is ready — reusing it`, + ); note( " Pass --recreate-sandbox or set NEMOCLAW_RECREATE_SANDBOX=1 to force recreation.", ); @@ -1220,6 +1399,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche plan: { intent: resolvedCreateIntent, policyAuthority: resolvedPolicyAuthority, + deferSandboxEffectsUntilPolicyVerification: + createIntent?.deferSandboxEffectsUntilPolicyVerification === true, rebindMessagingTokenDefs: async () => { revalidatePolicyAuthority( false, @@ -1242,7 +1423,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }, runProviderPreDeleteCleanup: () => { revalidatePolicyAuthority( - false, + createIntent?.deferSandboxEffectsUntilPolicyVerification === true, `cleaning up providers for sandbox '${sandboxName}'`, ); runSandboxProviderPreDeleteCleanup(sandboxName, { @@ -1255,7 +1436,10 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche upsertMessagingProviders(tokenDefs, { ...options, revalidatePolicyRequirements: (operation) => - revalidatePolicyAuthority(false, operation), + revalidatePolicyAuthority( + createIntent?.deferSandboxEffectsUntilPolicyVerification === true, + operation, + ), }), getHermesToolGatewayProviderName: (targetSandbox) => getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), @@ -1301,6 +1485,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche messagingProviders, gpuRoutePlan, compatibilityPolicyPath, + activateDeferredProviderEffects, initialGpuRoute, sandboxReadyTimeoutSecs, buildId, @@ -1365,21 +1550,158 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche : null; const createFlowEnvironment = hermesGpuAuthority?.env ?? sandboxEnv; const createGpuVerifier = hermesGpuAuthority?.verify ?? verifyDirectSandboxGpu; + let verifiedPolicyGate: VerifiedSandboxPolicyBoundary | null = null; + let pendingPolicyVerification: PendingSandboxPolicyVerification | null = null; + let admittedCreateReservation: QualifiedPendingSandboxCreateReservation | null = null; + const policySourcePathForRoute = ( + route: import("../docker-gpu-route").SelectedDockerGpuRoute, + ): string => { + const policySourcePath = + route === "compatibility" ? compatibilityPolicyPath : initialSandboxPolicy.policyPath; + if (!policySourcePath) { + throw new Error("Sandbox creation has no exact policy source for its selected route."); + } + return policySourcePath; + }; + const revalidateCreatedSandboxIdentity = ( + expectedIdentity: string, + operation: string, + ): void => { + sandboxRecreateTransaction.revalidateCreatedSandboxLifecycleRegistration( + { sandboxName, gatewayName: GATEWAY_NAME }, + { + lifecycleGeneration: createdSandboxLifecycle.generation, + lifecycleLiveIdentityFingerprint: expectedIdentity, + }, + getSandboxRecreateObservation, + ); + }; + const requireVerifiedPolicyGate = (): NonNullable => { + if (!verifiedPolicyGate) { + throw new Error("Sandbox creation has no verified post-create policy boundary."); + } + return verifiedPolicyGate; + }; + const requirePendingPolicyVerification = (): PendingSandboxPolicyVerification => { + if (!pendingPolicyVerification) { + throw new Error("Sandbox creation has no durable verified policy checkpoint."); + } + return pendingPolicyVerification; + }; + const requireCreateReservation = (): QualifiedPendingSandboxCreateReservation => { + if (!admittedCreateReservation) { + throw new Error("Sandbox creation has no exact inference route reservation."); + } + return admittedCreateReservation; + }; + const admitCreateReservation = (): QualifiedPendingSandboxCreateReservation => { + if (!inferenceRouteReservationAuthority?.sessionId) { + throw new Error("Sandbox creation requires current inference route reservation authority."); + } + return registry.qualifyPendingSandboxCreateReservation( + { + sandboxName, + gatewayName: GATEWAY_NAME, + sessionId: inferenceRouteReservationAuthority.sessionId, + selection: sandboxRegistration.selection( + sandboxName, + provider, + model, + preferredInferenceApi, + createIntent?.endpointSource ?? null, + ), + }, + registry.getSandbox(sandboxName), + ); + }; + const revalidateVerifiedPolicyRegistration = ( + boundary: VerifiedSandboxPolicyBoundary, + operation: string, + ): void => { + revalidateCreatedSandboxIdentity(boundary.lifecycleLiveIdentityFingerprint, operation); + revalidateCreatedSandboxPolicyRegistration({ + sandboxName, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + lifecycleGeneration: createdSandboxLifecycle.generation, + lifecycleLiveIdentityFingerprint: boundary.lifecycleLiveIdentityFingerprint, + policySourcePath: policySourcePathForRoute(boundary.route), + route: boundary.route, + operation, + registration: boundary.registration, + }); + revalidateCreatedSandboxIdentity(boundary.lifecycleLiveIdentityFingerprint, operation); + registry.requireCurrentPendingSandboxPolicyVerification( + requireCreateReservation(), + pendingSandboxPolicyVerificationForBoundary(boundary), + ); + }; const runCreateFlow = async ( attemptCreateArgv: string[], hermesPortableReadyCapture?: import("../sandbox-gpu-create-flow").HermesPortableReadyCapture, hermesPortableReadyRunner?: import("../sandbox-gpu-create-flow").HermesPortableReadyRunner, createWorkingDirectory?: string, - ) => - runSandboxCreateWithPolicyAuthorityChecks({ + runDeferredProviderEffects?: (context: VerifiedSandboxCreateEffectsContext) => Promise, + ) => { + admittedCreateReservation = admitCreateReservation(); + return runSandboxCreateWithPolicyAuthorityChecks< + import("../sandbox-gpu-create-flow").CreatedSandboxIdentity, + VerifiedSandboxPolicyBoundary, + import("../sandbox-gpu-create-flow").SandboxGpuCreateFlowResult + >({ sandboxName, revalidate: revalidatePolicyAuthority, - captureCreatedSandboxLiveIdentity: (created) => - createdSandboxLifecycle.capture(created.lifecycleRegistrationFields) - .lifecycleLiveIdentityFingerprint, - observeCreatedSandbox: () => getSandboxRecreateObservation(sandboxName, GATEWAY_NAME), + captureCreatedSandboxIdentity: ( + identity: import("../sandbox-gpu-create-flow").CreatedSandboxIdentity, + ) => identity.liveIdentityFingerprint, + revalidateCreatedSandboxIdentity, + verifyCreatedPolicy: ( + identity: import("../sandbox-gpu-create-flow").CreatedSandboxIdentity, + ) => { + const registration = verifyCreatedSandboxPolicyRegistration({ + sandboxName, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + lifecycleGeneration: createdSandboxLifecycle.generation, + lifecycleLiveIdentityFingerprint: identity.liveIdentityFingerprint, + policySourcePath: policySourcePathForRoute(identity.route), + route: identity.route, + plannedAuthority: resolvedPolicyAuthority, + operation: `verify effective policy for sandbox '${sandboxName}'`, + }); + return { + registration, + sandboxName, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + lifecycleGeneration: createdSandboxLifecycle.generation, + lifecycleLiveIdentityFingerprint: identity.liveIdentityFingerprint, + route: identity.route, + }; + }, + persistVerifiedPolicy: (_identity, _exactIdentity, boundary) => { + const checkpoint = pendingSandboxPolicyVerificationForBoundary(boundary); + registry.recordPendingSandboxPolicyVerification(requireCreateReservation(), checkpoint, { + ...(pendingPolicyVerification ? { expected: pendingPolicyVerification } : {}), + }); + pendingPolicyVerification = checkpoint; + verifiedPolicyGate = boundary; + }, + revalidateVerifiedPolicy: (_identity, _exactIdentity, boundary, operation) => { + revalidateVerifiedPolicyRegistration(boundary, operation); + }, cleanupTemporarySources: cleanupSandboxCreateSources, - create: () => + runVerifiedCreateEffects: runDeferredProviderEffects + ? async (_identity, _exactIdentity, boundary) => { + const context: VerifiedSandboxCreateEffectsContext = { + ...boundary, + revalidatePolicyRequirements: (operation) => + revalidateVerifiedPolicyRegistration(boundary, operation), + }; + await runDeferredProviderEffects(context); + } + : undefined, + create: (verifyCreatedSandbox) => sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, @@ -1388,6 +1710,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche gpuRoutePlan, initialGpuRoute, compatibilityPolicyPath, + gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, sandboxReadyTimeoutSecs, createArgv: attemptCreateArgv, @@ -1400,6 +1723,11 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche restoreBackupPath, terminalAgent: agentDefs.isTerminalAgent(agent), managedBootstrap, + verifyCreatedSandboxBeforeEffects: async (identity) => { + await verifyCreatedSandbox(identity); + }, + revalidateVerifiedSandboxBeforeEffect: (operation) => + revalidateVerifiedPolicyRegistration(requireVerifiedPolicyGate(), operation), ...agentCreateInput, }, { @@ -1411,6 +1739,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }, ), }); + }; const cleanupBuildContext = sandboxGpuCreateFlow.createSandboxBuildContextCleanup(legacyBuildContext); @@ -1469,9 +1798,17 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche { gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, { initialSandboxPolicy, + compatibilityPolicyPath, policyTier: resolvedCreatePolicyTier, policyAuthority: resolvedPolicyAuthority, - revalidatePolicyAuthority: (operation) => revalidatePolicyAuthority(true, operation), + getVerifiedPolicyBoundary: requireVerifiedPolicyGate, + getVerifiedCreateRegistrationAuthority: () => ({ + reservation: requireCreateReservation(), + checkpoint: requirePendingPolicyVerification(), + }), + revalidatePolicyAuthority: (operation) => { + revalidateVerifiedPolicyRegistration(requireVerifiedPolicyGate(), operation); + }, dashboardRemoteBindPrepared, }, prebuild.imageRef, @@ -1489,7 +1826,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche hermesDashboardForwarding.ensureForState, managedWorkloadRuntime, preparedSandboxWorkload, - inferenceRouteReservationAuthority?.sessionId ?? null, note, ); const completeCreatedSandboxRegistration = createOnboardCreatedSandboxRegistration({ @@ -1516,10 +1852,31 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche cleanupBuildContext(); }, }; - validateAttachedMessagingProvidersBeforeSandboxCreation( - providerPreparationInput, - providerPreparationDeps, - ); + const providerEffectBoundary = createProviderEffectBoundary({ + deferred: createIntent?.deferSandboxEffectsUntilPolicyVerification === true, + sandboxName, + gatewayName: GATEWAY_NAME, + preparationInput: providerPreparationInput, + preparationDeps: providerPreparationDeps, + runVerifiedSandboxCreateEffects, + activateDeferredProviderEffects, + revalidatePolicyAuthorityBeforeCreate: () => + revalidatePolicyAuthority( + false, + `publishing providers before creating sandbox gateway '${GATEWAY_NAME}'`, + ), + runOpenshell, + revalidateSandboxIdentity: (exactIdentity, operation) => + sandboxRecreateTransaction.revalidateCreatedSandboxLifecycleRegistration( + { sandboxName, gatewayName: GATEWAY_NAME }, + { + lifecycleGeneration: createdSandboxLifecycle.generation, + lifecycleLiveIdentityFingerprint: exactIdentity, + }, + getSandboxRecreateObservation, + ), + }); + providerEffectBoundary.validateBeforeCreate(); if (hermesPortableAuthority) { if (!portableRuntimeContext?.environmentScope) { @@ -1598,17 +1955,19 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }); cleanupBuildContext(); } else { - revalidatePolicyAuthority( - false, - `publishing providers before creating sandbox gateway '${GATEWAY_NAME}'`, - ); - publishAttachedProvidersBeforeDockerSandboxCreation( - providerPreparationInput, - providerPreparationDeps, + providerEffectBoundary.publishBeforeCreate(); + const created = await runCreateFlow( + createArgv, + undefined, + undefined, + undefined, + providerEffectBoundary.runAfterVerifiedCreate, ); - const created = await runCreateFlow(createArgv); - cleanupInitialCreateSource(); - await completeCreatedSandboxRegistration(created, null); + try { + await completeCreatedSandboxRegistration(created, null); + } finally { + cleanupInitialCreateSource(); + } } hermesStateVolumeLifecycle.commit(); if ("complete" in recreateRuntime) recreateRuntime.complete(); @@ -1616,11 +1975,13 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche return completeOrdinaryOnboardSandboxCreation( { sandboxName, + sandboxWasLiveDefault, runtimeFields: sandboxRuntimeFields, messagingProviders, liveExists, }, { + setDefault: registry.setDefault, runFile, scriptsDir: SCRIPTS, gatewayName: GATEWAY_NAME, diff --git a/src/lib/onboard/sandbox-create/policy-creation-receipt.test.ts b/src/lib/onboard/sandbox-create/policy-creation-receipt.test.ts new file mode 100644 index 00000000000..52aee7b7043 --- /dev/null +++ b/src/lib/onboard/sandbox-create/policy-creation-receipt.test.ts @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import * as openshellRuntimeModule from "../../adapters/openshell/runtime"; +import { verifyCreatedSandboxPolicyCreationReceipt } from "./policy-creation-receipt"; + +const POLICY = "version: 1\nnetwork_policies:\n github:\n endpoints: []\n"; +const NATIVE_GPU_POLICY = `version: 1 +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /etc + - /app + - /var/log + - /dev/urandom + read_write: + - /tmp +network_policies: + github: + endpoints: [] +`; +const ENRICHED_NATIVE_GPU_POLICY = NATIVE_GPU_POLICY.replace( + " read_write:\n - /tmp\n", + ` read_write: + - /tmp + - /proc + - /dev/nvidiactl + - /dev/nvidia0 +`, +); +const INPUT = { + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + lifecycleLiveIdentityFingerprint: "b".repeat(64), + policySourcePath: "/private/policy.yaml", + route: "none" as const, +}; + +function metadata(overrides: Partial> = {}): { + status: number; + output: string; + stdout: string; + stderr: string; +} { + const stdout = JSON.stringify({ + scope: "sandbox", + sandbox: "alpha", + status: "effective", + policy_source: "sandbox", + active_version: 4, + hash: "sha256:effective", + policy: { + version: 1, + network_policies: { github: { endpoints: [] } }, + }, + ...overrides, + }); + return { + status: 0, + output: stdout, + stdout, + stderr: "", + }; +} + +function gatewayInfo(): { status: number; output: string; stdout: string; stderr: string } { + const output = "Gateway endpoint: http://127.0.0.1:8080\n"; + return { status: 0, output, stdout: output, stderr: "" }; +} + +describe("created sandbox policy receipt", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("binds the exact supplied policy to the verified create identity (#9833)", () => { + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(gatewayInfo()) + .mockReturnValueOnce(metadata()) + .mockReturnValueOnce({ status: 0, output: POLICY, stdout: POLICY, stderr: "" }) + .mockReturnValueOnce(metadata()); + const receipt = verifyCreatedSandboxPolicyCreationReceipt(INPUT, { + readFile: vi.fn(() => POLICY) as never, + }); + + expect(receipt).toEqual({ + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "alpha", + lifecycleGeneration: INPUT.lifecycleGeneration, + sandboxIdentityFingerprint: INPUT.lifecycleLiveIdentityFingerprint, + policyHash: "sha256:effective", + policyVersion: 4, + }); + expect(JSON.stringify(receipt)).not.toMatch(/github|credential|endpoints/u); + }); + + it("refuses a live base policy that differs from the create source (#9833)", () => { + const captureOpenshell = vi + .spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(gatewayInfo()) + .mockReturnValueOnce(metadata()) + .mockReturnValueOnce({ + status: 0, + output: "version: 1\nnetwork_policies: {}\n", + stdout: "version: 1\nnetwork_policies: {}\n", + stderr: "", + }) + .mockReturnValueOnce(metadata()); + expect(() => + verifyCreatedSandboxPolicyCreationReceipt(INPUT, { + readFile: vi.fn(() => POLICY) as never, + }), + ).toThrow(/live base policy does not match/u); + expect(captureOpenshell).toHaveBeenCalledTimes(4); + }); + + it("binds the documented native-GPU policy enrichment to the create receipt (#9833)", () => { + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(gatewayInfo()) + .mockReturnValueOnce(metadata()) + .mockReturnValueOnce({ + status: 0, + output: ENRICHED_NATIVE_GPU_POLICY, + stdout: ENRICHED_NATIVE_GPU_POLICY, + stderr: "", + }) + .mockReturnValueOnce(metadata()); + + expect( + verifyCreatedSandboxPolicyCreationReceipt( + { ...INPUT, route: "native" }, + { + readFile: vi.fn(() => NATIVE_GPU_POLICY) as never, + }, + ), + ).toMatchObject({ + policyHash: "sha256:effective", + policyVersion: 4, + }); + }); + + it.each([ + { + label: "the create route does not use native GPU injection", + input: INPUT, + livePolicy: ENRICHED_NATIVE_GPU_POLICY, + }, + { + label: "the live policy contains an arbitrary added path", + input: { ...INPUT, route: "native" as const }, + livePolicy: ENRICHED_NATIVE_GPU_POLICY.replace("/dev/nvidia0", "/home"), + }, + ])("refuses native-GPU policy enrichment when $label (#9833)", ({ input, livePolicy }) => { + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(gatewayInfo()) + .mockReturnValueOnce(metadata()) + .mockReturnValueOnce({ status: 0, output: livePolicy, stdout: livePolicy, stderr: "" }) + .mockReturnValueOnce(metadata()); + + expect(() => + verifyCreatedSandboxPolicyCreationReceipt(input, { + readFile: vi.fn(() => NATIVE_GPU_POLICY) as never, + }), + ).toThrow(/live base policy does not match/u); + }); + + it("does not claim a verified global policy as NemoClaw-created (#9833)", () => { + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(gatewayInfo()) + .mockReturnValueOnce(metadata({ policy_source: "global" })) + .mockReturnValueOnce({ status: 0, output: POLICY, stdout: POLICY, stderr: "" }) + .mockReturnValueOnce(metadata({ policy_source: "global" })); + expect(() => + verifyCreatedSandboxPolicyCreationReceipt(INPUT, { + readFile: vi.fn(() => POLICY) as never, + }), + ).toThrow(/does not report.*sandbox-scoped/u); + }); + + it("refuses incomplete OpenShell policy identity without exposing policy contents (#9833)", () => { + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(gatewayInfo()) + .mockReturnValue(metadata({ hash: "" })); + let error: unknown; + try { + verifyCreatedSandboxPolicyCreationReceipt(INPUT, { + readFile: vi.fn(() => POLICY) as never, + }); + } catch (caught) { + error = caught; + } + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("policy authority inspection failed"); + expect((error as Error).message).not.toContain("github"); + }); + + it("refuses an effective policy identity that changes during verification (#9833)", () => { + vi.spyOn(openshellRuntimeModule, "captureResolvedOpenshell") + .mockReturnValueOnce(gatewayInfo()) + .mockReturnValueOnce(metadata()) + .mockReturnValueOnce({ status: 0, output: POLICY, stdout: POLICY, stderr: "" }) + .mockReturnValueOnce(metadata({ hash: "sha256:replacement", active_version: 5 })); + + expect(() => + verifyCreatedSandboxPolicyCreationReceipt(INPUT, { + readFile: vi.fn(() => POLICY) as never, + }), + ).toThrow(/policy identity changed/u); + }); +}); diff --git a/src/lib/onboard/sandbox-create/policy-creation-receipt.ts b/src/lib/onboard/sandbox-create/policy-creation-receipt.ts new file mode 100644 index 00000000000..38c7ff070bd --- /dev/null +++ b/src/lib/onboard/sandbox-create/policy-creation-receipt.ts @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { + assertExternalPolicyRequirements, + assertOpenShellGatewayPortBinding, + captureSandboxBasePolicy, + inspectSandboxPolicyAuthority, + PolicyAuthorityRefusalError, + type SandboxPolicyAuthority, +} from "../../adapters/openshell/policy-authority"; +import type { NemoClawPolicyCreationReceipt } from "../../policy/merge"; +import type { PendingSandboxPolicyVerification } from "../../state/registry/types"; +import { + assertNemoClawPolicyCreationReceiptMatches, + openShellPolicyValuesEqual, + parseNemoClawPolicyCreationReceipt, + parseOpenShellPolicy, +} from "../../policy/merge"; +import type { SelectedDockerGpuRoute } from "../docker-gpu-route"; +import { isOpenShellNativeGpuBaselineEnrichment } from "../sandbox-gpu-route-policy"; +import type { VerifiedSandboxPolicyBoundary, VerifiedSandboxPolicyRegistration } from "../types"; + +export interface CreatedSandboxPolicyReceiptInput { + readonly sandboxName: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly lifecycleGeneration: string; + readonly lifecycleLiveIdentityFingerprint: string; + readonly policySourcePath: string; + readonly route: SelectedDockerGpuRoute; +} + +export interface CreatedSandboxPolicyReceiptDeps { + readonly readFile?: typeof fs.readFileSync; +} + +export interface CreatedSandboxPolicyRegistrationInput extends CreatedSandboxPolicyReceiptInput { + readonly plannedAuthority: Exclude; + readonly operation: string; +} + +/** Flatten one in-memory verified boundary into its non-authorizing durable checkpoint. */ +export function pendingSandboxPolicyVerificationForBoundary( + boundary: VerifiedSandboxPolicyBoundary, +): PendingSandboxPolicyVerification { + const common = { + schemaVersion: 1 as const, + state: "verified-create" as const, + gatewayName: boundary.gatewayName, + gatewayPort: boundary.gatewayPort, + sandboxName: boundary.sandboxName, + lifecycleGeneration: boundary.lifecycleGeneration, + sandboxIdentityFingerprint: boundary.lifecycleLiveIdentityFingerprint, + route: boundary.route, + }; + const registration = boundary.registration; + if (registration.policyAuthority === "nemoclaw-managed") { + return { + ...common, + policyAuthority: "nemoclaw-managed", + observedPolicyAuthority: "owner-unknown", + policyHash: registration.policyCreationReceipt.policyHash, + policyVersion: registration.policyCreationReceipt.policyVersion, + policyCreationReceipt: registration.policyCreationReceipt, + }; + } + return { + ...common, + policyAuthority: "externally-managed", + observedPolicyAuthority: registration.observedPolicyAuthority, + policyHash: registration.policyIdentity.hash, + policyVersion: registration.policyIdentity.activeVersion, + }; +} + +function refusal(reason: string): never { + throw new PolicyAuthorityRefusalError( + `Cannot record NemoClaw policy ownership: ${reason}. The sandbox remains owner-unknown and policy mutation is disabled.`, + ); +} + +/** + * Bind one successful create to its exact sandbox and effective policy. + * Policy bytes are compared in memory and never enter the receipt or error. + */ +export function verifyCreatedSandboxPolicyCreationReceipt( + input: CreatedSandboxPolicyReceiptInput, + deps: CreatedSandboxPolicyReceiptDeps = {}, +): NemoClawPolicyCreationReceipt { + assertOpenShellGatewayPortBinding({ + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + }); + const readFile = deps.readFile ?? fs.readFileSync; + let intendedPolicy: ReturnType["policy"]; + try { + intendedPolicy = parseOpenShellPolicy(readFile(input.policySourcePath, "utf8")).policy; + } catch { + refusal("the intended base policy could not be read"); + } + + const before = inspectSandboxPolicyAuthority({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + }); + let liveBasePolicy: ReturnType["policy"]; + try { + liveBasePolicy = parseOpenShellPolicy( + captureSandboxBasePolicy(input.sandboxName, input.gatewayName), + ).policy; + } catch { + refusal("the live base policy could not be compared"); + } + const after = inspectSandboxPolicyAuthority({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + }); + if (before.authority !== "owner-unknown" || after.authority !== "owner-unknown") { + refusal("OpenShell does not report the verified policy as sandbox-scoped"); + } + if ( + before.policyIdentity.hash !== after.policyIdentity.hash || + before.policyIdentity.activeVersion !== after.policyIdentity.activeVersion + ) { + refusal("the effective policy identity changed during receipt verification"); + } + if ( + !openShellPolicyValuesEqual(intendedPolicy, liveBasePolicy) && + !( + input.route === "native" && + isOpenShellNativeGpuBaselineEnrichment(intendedPolicy, liveBasePolicy) + ) + ) { + refusal("the live base policy does not match the policy supplied by this create transaction"); + } + try { + return parseNemoClawPolicyCreationReceipt({ + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + sandboxName: input.sandboxName, + lifecycleGeneration: input.lifecycleGeneration, + sandboxIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, + policyHash: after.policyIdentity.hash, + policyVersion: after.policyIdentity.activeVersion, + }); + } catch { + refusal("the verified sandbox or policy identity is incomplete"); + } +} + +function readRequiredPolicy( + policySourcePath: string, + operation: string, + deps: CreatedSandboxPolicyReceiptDeps, +): ReturnType["policy"] { + try { + return parseOpenShellPolicy((deps.readFile ?? fs.readFileSync)(policySourcePath, "utf8")) + .policy; + } catch { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the required sandbox policy could not be read.`, + ); + } +} + +function verifyExternalPolicyBoundary( + input: CreatedSandboxPolicyRegistrationInput, + deps: CreatedSandboxPolicyReceiptDeps, +): VerifiedSandboxPolicyRegistration { + assertOpenShellGatewayPortBinding({ + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + }); + const requiredPolicy = readRequiredPolicy(input.policySourcePath, input.operation, deps); + const before = inspectSandboxPolicyAuthority({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + }); + if (before.authority !== "externally-managed") { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the created sandbox policy authority is not externally controlled.`, + before.authority, + ); + } + assertExternalPolicyRequirements({ + inspection: before, + requiredPolicy, + operation: input.operation, + sandboxName: input.sandboxName, + }); + const after = inspectSandboxPolicyAuthority({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + }); + if ( + after.authority !== before.authority || + after.policyIdentity.hash !== before.policyIdentity.hash || + after.policyIdentity.activeVersion !== before.policyIdentity.activeVersion + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the effective sandbox policy changed during verification.`, + after.authority, + ); + } + assertExternalPolicyRequirements({ + inspection: after, + requiredPolicy, + operation: input.operation, + sandboxName: input.sandboxName, + }); + return { + policyAuthority: "externally-managed", + policyCreationReceipt: null, + observedPolicyAuthority: "externally-managed", + policyIdentity: { ...before.policyIdentity }, + }; +} + +/** Prove the post-create policy before any unrelated create effects run. */ +export function verifyCreatedSandboxPolicyRegistration( + input: CreatedSandboxPolicyRegistrationInput, + deps: CreatedSandboxPolicyReceiptDeps = {}, +): VerifiedSandboxPolicyRegistration { + if (input.plannedAuthority === "nemoclaw-managed") { + return { + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: verifyCreatedSandboxPolicyCreationReceipt(input, deps), + observedPolicyAuthority: "owner-unknown", + }; + } + return verifyExternalPolicyBoundary(input, deps); +} + +/** Revalidate one in-memory gate result against the exact live policy identity. */ +export function revalidateCreatedSandboxPolicyRegistration( + input: Omit & { + readonly registration: VerifiedSandboxPolicyRegistration; + }, + deps: CreatedSandboxPolicyReceiptDeps = {}, +): VerifiedSandboxPolicyRegistration { + assertOpenShellGatewayPortBinding({ + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + }); + const before = inspectSandboxPolicyAuthority({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + }); + const registration = input.registration; + if (before.authority !== registration.observedPolicyAuthority) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the effective sandbox policy authority changed after verification.`, + before.authority, + ); + } + if (registration.policyAuthority === "nemoclaw-managed") { + try { + assertNemoClawPolicyCreationReceiptMatches(registration.policyCreationReceipt, { + origin: "sandbox-create", + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + sandboxName: input.sandboxName, + lifecycleGeneration: input.lifecycleGeneration, + sandboxIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, + policyHash: before.policyIdentity.hash, + policyVersion: before.policyIdentity.activeVersion, + }); + } catch { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the NemoClaw policy creation receipt no longer matches the live sandbox policy.`, + "owner-unknown", + ); + } + return registration; + } + if ( + before.policyIdentity.hash !== registration.policyIdentity.hash || + before.policyIdentity.activeVersion !== registration.policyIdentity.activeVersion + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${input.operation}: the effective sandbox policy identity changed after verification.`, + before.authority, + ); + } + const requiredPolicy = readRequiredPolicy(input.policySourcePath, input.operation, deps); + assertExternalPolicyRequirements({ + inspection: before, + requiredPolicy, + operation: input.operation, + sandboxName: input.sandboxName, + }); + return registration; +} diff --git a/src/lib/onboard/sandbox-create/provider-publication.test.ts b/src/lib/onboard/sandbox-create/provider-publication.test.ts index 000bd04506a..b7dbdd0aa10 100644 --- a/src/lib/onboard/sandbox-create/provider-publication.test.ts +++ b/src/lib/onboard/sandbox-create/provider-publication.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { MESSAGING_CREDENTIAL_PROVIDER_TYPE } from "../../messaging/provider-profile"; import { + attachProvidersAfterSandboxCreation, publishAttachedProvidersBeforeDockerSandboxCreation, validateAttachedMessagingProvidersBeforeSandboxCreation, } from "./provider-publication"; @@ -116,6 +117,58 @@ function prepareProviders( } describe("sandbox provider preparation", () => { + it("identity-fences every deferred provider attachment (#9833)", () => { + const events: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + events.push(args.join(" ")); + return { status: 0 }; + }); + + attachProvidersAfterSandboxCreation( + { + sandboxName: "alpha", + gatewayName: "nemoclaw", + providerNames: ["inference", "alpha-telegram"], + }, + { + runOpenshell: runOpenshell as never, + revalidateSandboxIdentity: (operation) => events.push(operation), + }, + ); + + expect(events).toEqual([ + "attaching provider 'inference' to sandbox 'alpha'", + "sandbox provider attach -g nemoclaw alpha inference", + "confirming provider 'inference' on sandbox 'alpha'", + "attaching provider 'alpha-telegram' to sandbox 'alpha'", + "sandbox provider attach -g nemoclaw alpha alpha-telegram", + "confirming provider 'alpha-telegram' on sandbox 'alpha'", + ]); + }); + + it("redacts deferred attachment command output on failure (#9833)", () => { + const revalidateSandboxIdentity = vi.fn(); + + expect(() => + attachProvidersAfterSandboxCreation( + { + sandboxName: "alpha", + gatewayName: "nemoclaw", + providerNames: ["alpha-telegram"], + }, + { + runOpenshell: vi.fn(() => ({ + status: 1, + stdout: "secret-stdout", + stderr: "secret-stderr", + })) as never, + revalidateSandboxIdentity, + }, + ), + ).toThrow("OpenShell did not attach provider 'alpha-telegram' to the verified sandbox."); + expect(revalidateSandboxIdentity).toHaveBeenCalledOnce(); + }); + it("confirms an exact messaging binding before and after publication (#9875)", () => { const harness = createHarness(); diff --git a/src/lib/onboard/sandbox-create/provider-publication.ts b/src/lib/onboard/sandbox-create/provider-publication.ts index 2f94763b8e6..e50249c13d3 100644 --- a/src/lib/onboard/sandbox-create/provider-publication.ts +++ b/src/lib/onboard/sandbox-create/provider-publication.ts @@ -27,6 +27,16 @@ type ProviderPreparationDeps = Pick< readonly cleanupCreateSources: () => void; }; +type DeferredProviderAttachmentInput = { + readonly sandboxName: string; + readonly gatewayName: string; + readonly providerNames: readonly string[]; +}; + +type DeferredProviderAttachmentDeps = Pick & { + readonly revalidateSandboxIdentity: (operation: string) => void; +}; + function expectedMessagingBindings(input: ProviderPreparationInput) { return new Map( input.messagingProviderRequests @@ -137,3 +147,27 @@ export function publishAttachedProvidersBeforeDockerSandboxCreation( ); } } + +/** Attach the planned providers only after the created sandbox passed its exact policy gate. */ +export function attachProvidersAfterSandboxCreation( + input: DeferredProviderAttachmentInput, + deps: DeferredProviderAttachmentDeps, +): void { + for (const providerName of input.providerNames) { + deps.revalidateSandboxIdentity( + `attaching provider '${providerName}' to sandbox '${input.sandboxName}'`, + ); + const attached = deps.runOpenshell( + ["sandbox", "provider", "attach", "-g", input.gatewayName, input.sandboxName, providerName], + { ignoreError: true, suppressOutput: true }, + ); + if (attached.status !== 0) { + throw new Error( + `OpenShell did not attach provider '${providerName}' to the verified sandbox.`, + ); + } + deps.revalidateSandboxIdentity( + `confirming provider '${providerName}' on sandbox '${input.sandboxName}'`, + ); + } +} diff --git a/src/lib/onboard/sandbox-fresh-readiness.test.ts b/src/lib/onboard/sandbox-fresh-readiness.test.ts index ad571847d61..6380999db8b 100644 --- a/src/lib/onboard/sandbox-fresh-readiness.test.ts +++ b/src/lib/onboard/sandbox-fresh-readiness.test.ts @@ -145,16 +145,16 @@ describe("fresh sandbox executable readiness", () => { await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); - expect(deps.runOpenshell).toHaveBeenCalledWith( + expect(deps.runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), + expect.anything(), ); expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: null, }); }); - it("removes a fresh sandbox when sandbox get omits a durable ID (#9050)", async () => { + it("preserves a fresh sandbox when sandbox get omits a durable ID (#9050)", async () => { const deps = createDeps(); vi.mocked(deps.runOpenshell).mockImplementation( createSequencedOpenShellRunner([ @@ -169,9 +169,9 @@ describe("fresh sandbox executable readiness", () => { ["sandbox", "exec", "--name", "alpha", "--", "true"], expect.anything(), ); - expect(deps.runOpenshell).toHaveBeenCalledWith( + expect(deps.runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true, suppressOutput: true }), + expect.anything(), ); expect(console.error).toHaveBeenCalledWith( " NemoClaw could not verify that sandbox 'alpha' returned a durable ID and accepted commands.", @@ -196,9 +196,9 @@ describe("fresh sandbox executable readiness", () => { ["sandbox", "exec", "--name", "alpha", "--", "true"], expect.anything(), ); - expect(deps.runOpenshell).toHaveBeenCalledWith( + expect(deps.runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), + expect.anything(), ); }); @@ -231,9 +231,9 @@ describe("fresh sandbox executable readiness", () => { expect(identityOptions?.timeout).toBeLessThanOrEqual(500); expect(executableOptions?.timeout).toBeGreaterThan(0); expect(executableOptions?.timeout).toBeLessThanOrEqual(500); - expect(deps.runOpenshell).toHaveBeenCalledWith( + expect(deps.runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), + expect.anything(), ); expect(mocks.printSandboxCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: null, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 00f3969d7e0..08ef87b805a 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -967,10 +967,8 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); - expect(deps.runOpenshell).toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), - ); + expect(deps.runOpenshell).not.toHaveBeenCalled(); + expect(errorOutput()).toContain("Verify the sandbox identity before manual cleanup"); expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); }); @@ -1376,19 +1374,21 @@ describe("runSandboxGpuCreateFlow cleanup and provenance", () => { ); }); - it("reports manual cleanup when ordinary readiness deletion fails (#6110)", async () => { + it("preserves the sandbox when ordinary readiness cleanup is name-only (#6110)", async () => { mockReadinessFailure(); const deps = createDeps(); vi.mocked(deps.runOpenshell).mockReturnValue({ status: 7, stderr: "gateway unavailable" }); await expectFlowExit(createInput(), deps); const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain("could not be removed automatically"); - expect(output).toContain('Manual cleanup: openshell sandbox delete "alpha"'); + expect(deps.runOpenshell).not.toHaveBeenCalled(); + expect(output).toContain("left sandbox 'alpha' in place"); + expect(output).toContain("Verify the sandbox identity before manual cleanup"); + expect(output).not.toContain("openshell sandbox delete"); expect(output).not.toContain("Retry: nemoclaw onboard"); }); - it("treats an already-absent sandbox as successful ordinary readiness cleanup", async () => { + it("does not infer absence through a mutable-name readiness cleanup", async () => { mockReadinessFailure(); const deps = createDeps(); vi.mocked(deps.runOpenshell).mockReturnValue({ @@ -1398,8 +1398,10 @@ describe("runSandboxGpuCreateFlow cleanup and provenance", () => { await expectFlowExit(createInput(), deps); const output = vi.mocked(console.error).mock.calls.flat().join("\n"); - expect(output).toContain("Retry: nemoclaw onboard"); - expect(output).not.toContain("could not be removed automatically"); + expect(deps.runOpenshell).not.toHaveBeenCalled(); + expect(output).toContain("left sandbox 'alpha' in place"); + expect(output).toContain("Verify the sandbox identity before manual cleanup"); + expect(output).not.toContain("Retry: nemoclaw onboard"); }); it("fully redacts command diagnostics when cleanup cannot be proven safe", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index b0637393a5a..f7981fe8ed7 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -215,6 +215,7 @@ export interface SandboxGpuCreateFlowInput { initialGpuRoute: SelectedDockerGpuRoute; compatibilityPolicyPath: string | null; dockerDriverGateway: boolean; + gatewayName: string; gatewayPort: number; sandboxReadyTimeoutSecs: number; createArgv: string[]; @@ -246,6 +247,19 @@ export interface SandboxGpuCreateFlowInput { readonly expectedSupervisorArgv: readonly string[]; } | null; requiredUlimits?: readonly DockerUlimit[] | null; + /** + * Verify the exact sandbox created by each attempt before runtime activation, + * readiness, GPU, service, dashboard, or registry effects continue. + */ + verifyCreatedSandboxBeforeEffects?: (identity: CreatedSandboxIdentity) => void | Promise; + /** Re-read the exact durable policy checkpoint before each post-create effect. */ + revalidateVerifiedSandboxBeforeEffect?: (operation: string) => void; +} + +export interface CreatedSandboxIdentity { + readonly sandboxId: string; + readonly liveIdentityFingerprint: string; + readonly route: SelectedDockerGpuRoute; } export interface SandboxGpuCreateFlowDeps { @@ -477,13 +491,20 @@ export async function runSandboxGpuCreateFlow( ? ` Managed bootstrap retained exact owner-cleanup authority for sandbox '${input.sandboxName}'. Do not delete a runtime by mutable sandbox name; preserve it for identity-bound recovery.` : hermesPortableLifecycle ? ` Hermes portable sandbox '${input.sandboxName}' did not complete receipt-owned creation. Preserve its lifecycle receipt and resume onboarding after correcting the reported failure.` - : ` Manual cleanup: openshell sandbox delete "${input.sandboxName}"`, + : ` Sandbox '${input.sandboxName}' may still exist. Verify its durable identity before manual cleanup; do not act by mutable name alone.`, ); process.exit(1); } let portableLifecycleGeneration = attemptRunner.state.portableLifecycleGeneration; if (!input.portableLifecycle && !input.hermesPortableLifecycle && !portableLifecycleGeneration) { + if (input.verifyCreatedSandboxBeforeEffects) { + const revalidate = input.revalidateVerifiedSandboxBeforeEffect; + if (!revalidate) { + throw new Error("Verified sandbox creation has no post-create effect revalidation."); + } + revalidate(`record portable lifecycle for sandbox '${input.sandboxName}'`); + } try { portableLifecycleGeneration = (deps.installPortableDemoLifecycle ?? installPortableDemoSandboxLifecycle)( diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts new file mode 100644 index 00000000000..85e553056a9 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -0,0 +1,367 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + streamSandboxCreate: vi.fn(), + waitForCreatedSandboxReadyWithTrace: vi.fn(), + printReadinessFailure: vi.fn(), + enforceDockerGpuPatchPreserveNetwork: vi.fn(), + verifyGpuSandboxAccessAfterReady: vi.fn(), + createDockerGpuSandboxCreatePatch: vi.fn(), + printSandboxCreateFailureDiagnostics: vi.fn(), + collectDockerGpuPatchDiagnostics: vi.fn(), + queryOpenShellDockerSandboxContainers: vi.fn(), + queryOpenShellDockerSandboxRuntimeSnapshot: vi.fn(), +})); + +vi.mock("../sandbox/create-stream", () => ({ + streamSandboxCreate: mocks.streamSandboxCreate, +})); + +vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({ + ...(await importOriginal()), + waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, + printReadinessFailure: mocks.printReadinessFailure, +})); + +vi.mock("./docker-gpu-local-inference", () => ({ + enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, + verifyGpuSandboxAccessAfterReady: mocks.verifyGpuSandboxAccessAfterReady, +})); + +vi.mock("./docker-gpu-sandbox-create", () => ({ + createDockerGpuSandboxCreatePatch: mocks.createDockerGpuSandboxCreatePatch, +})); + +vi.mock("./sandbox-create-failure", () => ({ + printSandboxCreateFailureDiagnostics: mocks.printSandboxCreateFailureDiagnostics, +})); + +vi.mock("./docker-gpu-patch", async (importOriginal) => ({ + ...(await importOriginal()), + collectDockerGpuPatchDiagnostics: mocks.collectDockerGpuPatchDiagnostics, +})); + +vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ + ...(await importOriginal()), + queryOpenShellDockerSandboxContainers: mocks.queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, +})); + +import { + NEMOCLAW_CREATE_ATTEMPT_LABEL, + NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, +} from "../adapters/openshell/sandbox-identity"; +import { + createGpuFlowDeps, + createGpuFlowInput, + createGpuPatchFixture, + resetGpuFlowMocks, + setupGpuFlowMocks, +} from "./__test-helpers__/sandbox-gpu-create-flow"; +import { runSandboxGpuCreateFlow } from "./sandbox-gpu-create-flow"; + +function sandboxListJson(sandboxId: string, labels: Readonly>): string { + return JSON.stringify([ + { + id: sandboxId, + name: "alpha", + labels, + resource_version: 1, + created_at: "2026-08-25T00:00:00Z", + phase: "Ready", + current_policy_version: 1, + }, + ]); +} + +function createAttemptNonce(args: readonly string[]): string { + const labelIndex = args.indexOf("--label"); + return (args[labelIndex + 1] ?? "").slice(NEMOCLAW_CREATE_ATTEMPT_LABEL.length + 1); +} + +function noGpuInput() { + const input = createGpuFlowInput(); + input.sandboxGpuConfig = { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + input.createArgv = ["openshell", "sandbox", "create", "--name", "alpha", "--", "agent"]; + return input; +} + +function refuseEffectStartingWith(prefix: string): (operation: string) => void { + return (operation) => { + expect(operation, "checkpoint changed").not.toMatch(new RegExp(`^${prefix}`, "u")); + }; +} + +beforeEach(() => setupGpuFlowMocks(mocks)); +afterEach(resetGpuFlowMocks); + +describe("created sandbox identity gate", () => { + it("verifies the exact created sandbox before runtime and readiness effects (#9833)", async () => { + const events: string[] = []; + let nonce = ""; + const input = noGpuInput(); + const patch = createGpuPatchFixture(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(async (identity) => { + events.push("verify-created"); + expect(identity).toEqual({ + sandboxId: "alpha-sandbox-id", + liveIdentityFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/u), + route: "none", + }); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn((operation) => + events.push(`revalidate:${operation}`), + ); + patch.exitOnPatchError.mockImplementation(() => events.push("runtime-check")); + patch.ensureApplied.mockImplementation(() => events.push("runtime-patch")); + patch.waitForSupervisorReconnectIfNeeded.mockImplementation(() => events.push("reconnect")); + patch.commitAfterReady.mockImplementation(() => events.push("commit")); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { + events.push("create"); + expect(options.onPoll).toBeUndefined(); + expect(args.indexOf("--label")).toBeGreaterThan(0); + expect(args.indexOf("--label")).toBeLessThan(args.indexOf("--")); + nonce = createAttemptNonce(args); + expect(nonce).toMatch(/^[0-9a-f]{62}$/u); + expect(nonce).toHaveLength(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH); + expect(nonce.length).toBeLessThanOrEqual(63); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockImplementation(() => { + events.push("readiness"); + return { ready: true, reason: "ready", failurePhase: null }; + }); + const deps = createGpuFlowDeps(); + deps.installPortableDemoLifecycle = vi.fn(() => { + events.push("portable-lifecycle"); + return "generation-1"; + }); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(events).toEqual([ + "create", + "verify-created", + "revalidate:validate runtime patch for sandbox 'alpha'", + "runtime-check", + "revalidate:apply runtime patch for sandbox 'alpha'", + "runtime-patch", + "revalidate:reconnect sandbox supervisor for 'alpha'", + "reconnect", + "readiness", + "revalidate:commit runtime readiness for sandbox 'alpha'", + "commit", + "revalidate:record portable lifecycle for sandbox 'alpha'", + "portable-lifecycle", + ]); + expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( + [ + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + "--output", + "json", + "--limit", + "2", + ], + { + ignoreError: false, + timeout: 30_000, + maxBuffer: 1024 * 1024, + killSignal: "SIGKILL", + killProcessTreeOnTimeout: true, + }, + ); + }); + + it("rejects a same-name replacement before post-create effects (#9833)", async () => { + let nonce = ""; + const outputCanary = "replacement-output-must-not-be-reported"; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("replacement-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: "0".repeat(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH), + untrusted: outputCanary, + }), + ); + + const error = await runSandboxGpuCreateFlow(input, deps).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain( + "did not return one exact durable sandbox identity before post-create effects", + ); + expect(String(error)).not.toContain(nonce); + expect(String(error)).not.toContain(outputCanary); + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + + it("requires checkpoint revalidation before the first post-create effect (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "has no post-create effect revalidation", + ); + + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + + it("uses a distinct identity label for each create attempt (#9833)", async () => { + const input = createGpuFlowInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const nonces: string[] = []; + mocks.streamSandboxCreate + .mockImplementationOnce(async (_command, args) => { + nonces.push(createAttemptNonce(args)); + return { + status: 1, + output: "error: unexpected argument '--gpu' found", + sawProgress: false, + }; + }) + .mockImplementationOnce(async (_command, args) => { + nonces.push(createAttemptNonce(args)); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { + [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonces[1] ?? "", + }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(nonces).toHaveLength(2); + expect(nonces[0]).toMatch(/^[0-9a-f]{62}$/u); + expect(nonces[1]).toMatch(/^[0-9a-f]{62}$/u); + expect(nonces[0]).not.toBe(nonces[1]); + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); + }); + + it("stops before a runtime patch when the durable checkpoint drifts (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn( + refuseEffectStartingWith("apply runtime patch"), + ); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("checkpoint changed"); + + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); + }); + + it("stops before supervisor work when the durable checkpoint drifts (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn( + refuseEffectStartingWith("reconnect sandbox supervisor"), + ); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("checkpoint changed"); + + expect(patch.ensureApplied).toHaveBeenCalledOnce(); + expect(patch.waitForSupervisorReconnectIfNeeded).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); + }); + + it("stops before GPU proof when the durable checkpoint drifts (#9833)", async () => { + let nonce = ""; + const input = createGpuFlowInput(); + input.gpuRoutePlan = "native-only"; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn( + refuseEffectStartingWith("verify GPU access"), + ); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("checkpoint changed"); + + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledOnce(); + expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); + expect(patch.commitAfterReady).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index fcfb7dd8cbe..8b769796bfb 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -1,18 +1,21 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { randomBytes } from "node:crypto"; + import { + NEMOCLAW_CREATE_ATTEMPT_LABEL, + NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH, parseOpenShellSandboxId, + resolveCreatedOpenShellSandboxId, resolveOpenShellSandboxId, } from "../adapters/openshell/sandbox-identity"; import { printSandboxCreateRecoveryHints } from "../build-context"; -import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; import { streamSandboxCreate } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; import { getSandboxFailurePhase, isSandboxReady } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; import { classifySandboxCreateFailure } from "../validation"; -import { cliName } from "./branding"; import { reportSandboxCreateFailure } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; @@ -36,6 +39,7 @@ import type { SandboxGpuCreateFlowDeps, SandboxGpuCreateFlowInput, } from "./sandbox-gpu-create-flow"; +import { fingerprintSandboxRecreateValue } from "./sandbox-recreate-transaction"; import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; import { SANDBOX_RECREATE_PROBE_TIMEOUT_MS } from "./sandbox-recreate-probe"; import type { CreatedSandboxReadyIdentityCheck } from "./sandbox-readiness-tracing"; @@ -142,6 +146,27 @@ type OpenShellSandboxIdentityProbe = | { state: "not_ready" } | { state: "failed" }; +function addCreateAttemptIdentityLabel(argv: readonly string[], nonce: string): string[] { + const optionEnd = argv.indexOf("--"); + const insertAt = optionEnd === -1 ? argv.length : optionEnd; + const labelPrefix = `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=`; + for (let index = 0; index < insertAt; index += 1) { + const argument = argv[index] ?? ""; + if ( + (argument === "--label" && (argv[index + 1] ?? "").startsWith(labelPrefix)) || + argument.startsWith(`--label=${labelPrefix}`) + ) { + throw new Error("OpenShell create arguments override NemoClaw's reserved identity label."); + } + } + return [ + ...argv.slice(0, insertAt), + "--label", + `${NEMOCLAW_CREATE_ATTEMPT_LABEL}=${nonce}`, + ...argv.slice(insertAt), + ]; +} + function remainingReadinessProbeTimeout(getRemainingMs: () => number): number | null { const remainingMs = Math.floor(getRemainingMs()); return remainingMs > 0 ? Math.min(SANDBOX_RECREATE_PROBE_TIMEOUT_MS, remainingMs) : null; @@ -172,6 +197,19 @@ function probeExactOpenShellSandboxId( : { state: "failed" }; } +async function verifyCreatedSandboxBeforeEffects( + sandboxId: string, + route: SelectedDockerGpuRoute, + input: SandboxGpuCreateFlowInput, +): Promise { + if (!input.verifyCreatedSandboxBeforeEffects) return; + await input.verifyCreatedSandboxBeforeEffects({ + sandboxId, + liveIdentityFingerprint: fingerprintSandboxRecreateValue(sandboxId), + route, + }); +} + function checkRecreatedSandboxReadyIdentity( sandboxName: string, expectedSandboxId: string, @@ -256,6 +294,14 @@ export function createSandboxGpuCreateAttemptRunner( nativeRuntimeSnapshot: null, portableLifecycleGeneration: null, }; + const revalidatePostCreateEffect = (operation: string): void => { + if (!input.verifyCreatedSandboxBeforeEffects) return; + const revalidate = input.revalidateVerifiedSandboxBeforeEffect; + if (!revalidate) { + throw new Error("Verified sandbox creation has no post-create effect revalidation."); + } + revalidate(operation); + }; const managedRouting = input.managedBootstrap?.runtimeProvider.bootstrap.createOnboardRouting({ sandboxName: input.sandboxName, openshellArgv: deps.openshellArgv, @@ -273,6 +319,7 @@ export function createSandboxGpuCreateAttemptRunner( managedRouting?.nativeFallbackHasCleanBaseline ?? (nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0); const runAttempt = async (route: SelectedDockerGpuRoute) => { + const deferPostCreateEffects = input.verifyCreatedSandboxBeforeEffects !== undefined; const compatibility = route === "compatibility"; if (compatibility && input.initialGpuRoute === "native") { console.warn( @@ -284,7 +331,13 @@ export function createSandboxGpuCreateAttemptRunner( } const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; const managedBootstrap = input.managedBootstrap ?? null; - const attemptArgv = state.compatibilityArgv ?? input.createArgv; + const unboundAttemptArgv = state.compatibilityArgv ?? input.createArgv; + const createAttemptNonce = deferPostCreateEffects + ? randomBytes(NEMOCLAW_CREATE_ATTEMPT_NONCE_HEX_LENGTH / 2).toString("hex") + : null; + const attemptArgv = createAttemptNonce + ? addCreateAttemptIdentityLabel(unboundAttemptArgv, createAttemptNonce) + : unboundAttemptArgv; const attemptBootstrapIdentity = state.compatibilityBootstrapIdentity ?? managedBootstrap?.bootstrapIdentity ?? null; const attemptHeldWorkloadArgv = @@ -369,7 +422,7 @@ export function createSandboxGpuCreateAttemptRunner( console.warn(` ⚠ ${message}`), ); } - await managedLifecycle?.prepareNetwork(); + if (!deferPostCreateEffects) await managedLifecycle?.prepareNetwork(); const [createExecutable, ...createExecutableArgs] = managedLifecycle?.launchArgv ?? attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); const streamCreate = () => @@ -382,9 +435,13 @@ export function createSandboxGpuCreateAttemptRunner( }); return isSandboxReady(list, input.sandboxName); }, - onPoll: () => { - if (!deferRestartSafeCutover) void runtimePatch.maybeApplyDuringCreate(); - }, + ...(deferPostCreateEffects + ? {} + : { + onPoll: () => { + if (!deferRestartSafeCutover) void runtimePatch.maybeApplyDuringCreate(); + }, + }), readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent({ isTerminalAgent: input.terminalAgent, startupRunsDuringCreate: managedLifecycle === null, @@ -400,6 +457,7 @@ export function createSandboxGpuCreateAttemptRunner( }); let createResult: Awaited>; let managedIncompleteCreateRecovered = false; + let createdSandboxVerified = false; if (managedBootstrap && managedLifecycle) { try { createResult = await managedLifecycle.runCreate( @@ -453,7 +511,14 @@ export function createSandboxGpuCreateAttemptRunner( } let sandboxId: string; try { - sandboxId = resolveOpenShellSandboxId(input.sandboxName, deps.runCaptureOpenshell); + sandboxId = createAttemptNonce + ? resolveCreatedOpenShellSandboxId({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + createAttemptNonce, + runCaptureOpenshell: deps.runCaptureOpenshell, + }) + : resolveOpenShellSandboxId(input.sandboxName, deps.runCaptureOpenshell); } catch (error) { throw new Error( createFailure?.kind === "sandbox_create_incomplete" @@ -462,6 +527,14 @@ export function createSandboxGpuCreateAttemptRunner( { cause: error }, ); } + await verifyCreatedSandboxBeforeEffects(sandboxId, route, input); + createdSandboxVerified = true; + if (deferPostCreateEffects) { + revalidatePostCreateEffect( + `activate managed sandbox network for '${input.sandboxName}'`, + ); + await managedLifecycle.prepareNetwork(); + } managedIncompleteCreateRecovered = createFailure?.kind === "sandbox_create_incomplete"; return { value: result, @@ -485,7 +558,7 @@ export function createSandboxGpuCreateAttemptRunner( createResult = await streamCreate(); } if (!state.firstCreateOutput) state.firstCreateOutput = createResult.output; - await runtimePatch.exitOnPatchError(); + if (!deferPostCreateEffects) await runtimePatch.exitOnPatchError(); if (createResult.status !== 0) { const failure = classifySandboxCreateFailure(createResult.output); if (failure.kind === "sandbox_create_incomplete") { @@ -560,6 +633,31 @@ export function createSandboxGpuCreateAttemptRunner( ); } } + if (!createdSandboxVerified && deferPostCreateEffects) { + if (!createAttemptNonce) { + throw new Error("Sandbox create-attempt identity was not generated."); + } + let sandboxId: string; + try { + sandboxId = resolveCreatedOpenShellSandboxId({ + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + createAttemptNonce, + runCaptureOpenshell: deps.runCaptureOpenshell, + }); + } catch (error) { + throw new Error( + `Sandbox '${input.sandboxName}' was created, but OpenShell did not return one exact durable sandbox identity before post-create effects.`, + { cause: error }, + ); + } + await verifyCreatedSandboxBeforeEffects(sandboxId, route, input); + createdSandboxVerified = true; + } + if (deferPostCreateEffects) { + revalidatePostCreateEffect(`validate runtime patch for sandbox '${input.sandboxName}'`); + await runtimePatch.exitOnPatchError(); + } const preRecreateIdentity = deferRestartSafeCutover ? probeExactOpenShellSandboxId(input.sandboxName, deps) : null; @@ -575,7 +673,11 @@ export function createSandboxGpuCreateAttemptRunner( }); process.exit(createResult.status === 0 ? 1 : createResult.status); } - if (!portableLifecycle || managedLifecycle) await runtimePatch.ensureApplied(); + if (!portableLifecycle || managedLifecycle) { + revalidatePostCreateEffect(`apply runtime patch for sandbox '${input.sandboxName}'`); + await runtimePatch.ensureApplied(); + } + revalidatePostCreateEffect(`reconnect sandbox supervisor for '${input.sandboxName}'`); await runtimePatch.waitForSupervisorReconnectIfNeeded(); console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ @@ -653,23 +755,15 @@ export function createSandboxGpuCreateAttemptRunner( " NemoClaw left the portable sandbox in place because it could not verify the exact runtime identity.", ); } else { - const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { - ignoreError: true, - suppressOutput: true, - }); - const { alreadyGone } = getSandboxDeleteOutcome({ - status: deletion.status ?? null, - stdout: String(deletion.stdout ?? ""), - stderr: String(deletion.stderr ?? ""), - }); - if (Number(deletion.status ?? 1) !== 0 && !alreadyGone) { - console.error(" The failed sandbox could not be removed automatically."); - console.error(` Manual cleanup: openshell sandbox delete "${input.sandboxName}"`); - } else console.error(` Retry: ${cliName()} onboard`); + console.error( + ` NemoClaw left sandbox '${input.sandboxName}' in place because OpenShell can delete it only by mutable name.`, + ); + console.error(" Verify the sandbox identity before manual cleanup."); } process.exit(createResult.status === 0 ? 1 : createResult.status); } if (input.sandboxGpuConfig.sandboxGpuEnabled) { + revalidatePostCreateEffect(`verify GPU access for sandbox '${input.sandboxName}'`); const deferNativeProofFailure = route === "native" && input.gpuRoutePlan === "native-with-fallback" && @@ -731,11 +825,15 @@ export function createSandboxGpuCreateAttemptRunner( throw new Error("Sandbox GPU proof returned failed status."); } } - if (portableRuntimePatch) await portableRuntimePatch.ensureApplied(); + if (portableRuntimePatch) { + revalidatePostCreateEffect(`install portable lifecycle for sandbox '${input.sandboxName}'`); + await portableRuntimePatch.ensureApplied(); + } // GPU-enabled cutover stays reversible until the caller also proves the // configured host-local inference path. Non-GPU workloads have completed // their final authoritative Ready gate here. if (!input.sandboxGpuConfig.sandboxGpuEnabled) { + revalidatePostCreateEffect(`commit runtime readiness for sandbox '${input.sandboxName}'`); await runtimePatch.commitAfterReady(); } return { diff --git a/src/lib/onboard/sandbox-gpu-route-policy.ts b/src/lib/onboard/sandbox-gpu-route-policy.ts index ffd0617ef31..dbbf5dbe7e5 100644 --- a/src/lib/onboard/sandbox-gpu-route-policy.ts +++ b/src/lib/onboard/sandbox-gpu-route-policy.ts @@ -7,6 +7,28 @@ import { initialDockerGpuRoute, } from "./docker-gpu-route"; import type { InitialSandboxPolicy } from "./initial-policy"; +import { openShellPolicyValuesEqual } from "../policy/merge"; + +const PROC_PATH = "/proc"; +const OPENSHELL_PROXY_REQUIRED_READ_ONLY_PATHS = new Set([ + "/usr", + "/lib", + "/etc", + "/app", + "/var/log", + "/dev/urandom", +]); +const OPENSHELL_PROXY_REQUIRED_READ_WRITE_PATHS = new Set(["/tmp"]); +const OPENSHELL_GPU_READ_ONLY_PATHS = new Set(["/run/nvidia-persistenced", "/usr/lib/wsl"]); +const OPENSHELL_GPU_READ_WRITE_PATHS = new Set([ + "/dev/nvidiactl", + "/dev/nvidia-uvm", + "/dev/nvidia-uvm-tools", + "/dev/nvidia-modeset", + "/dev/dxg", + PROC_PATH, +]); +const OPENSHELL_GPU_DEVICE_PATH = /^\/dev\/nvidia[0-9]+$/u; type PrepareInitialSandboxCreatePolicy = typeof import("./initial-policy").prepareInitialSandboxCreatePolicy; @@ -17,6 +39,116 @@ export type SandboxGpuRoutePolicies = { compatibilityPolicyPath: string | null; }; +function policyWithoutFilesystemPolicy(policy: Record): Record { + const result: Record = Object.create(null); + for (const key of Object.keys(policy)) { + if (key !== "filesystem_policy") result[key] = policy[key]; + } + return result; +} + +function filesystemPolicyWithoutPaths(policy: Record): Record { + const result: Record = Object.create(null); + for (const key of Object.keys(policy)) { + if (key !== "read_only" && key !== "read_write") result[key] = policy[key]; + } + return result; +} + +function readUniquePaths( + policy: Record, + key: "read_only" | "read_write", +): Set | null { + const value = policy[key]; + if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) return null; + const paths = new Set(value); + return paths.size === value.length ? paths : null; +} + +function isSubset(subset: ReadonlySet, superset: ReadonlySet): boolean { + return [...subset].every((entry) => superset.has(entry)); +} + +function setsOverlap(left: ReadonlySet, right: ReadonlySet): boolean { + return [...left].some((entry) => right.has(entry)); +} + +/** Accept only the documented filesystem additions made by OpenShell native-GPU injection. */ +export function isOpenShellNativeGpuBaselineEnrichment( + intended: Record, + live: Record, +): boolean { + if ( + !openShellPolicyValuesEqual( + policyWithoutFilesystemPolicy(intended), + policyWithoutFilesystemPolicy(live), + ) + ) { + return false; + } + const intendedFilesystem = intended.filesystem_policy; + const liveFilesystem = live.filesystem_policy; + if ( + !intendedFilesystem || + typeof intendedFilesystem !== "object" || + Array.isArray(intendedFilesystem) || + !liveFilesystem || + typeof liveFilesystem !== "object" || + Array.isArray(liveFilesystem) + ) { + return false; + } + const intendedFilesystemRecord = intendedFilesystem as Record; + const liveFilesystemRecord = liveFilesystem as Record; + if ( + !openShellPolicyValuesEqual( + filesystemPolicyWithoutPaths(intendedFilesystemRecord), + filesystemPolicyWithoutPaths(liveFilesystemRecord), + ) + ) { + return false; + } + const intendedReadOnly = readUniquePaths(intendedFilesystemRecord, "read_only"); + const intendedReadWrite = readUniquePaths(intendedFilesystemRecord, "read_write"); + const liveReadOnly = readUniquePaths(liveFilesystemRecord, "read_only"); + const liveReadWrite = readUniquePaths(liveFilesystemRecord, "read_write"); + if (!intendedReadOnly || !intendedReadWrite || !liveReadOnly || !liveReadWrite) return false; + if ( + setsOverlap(intendedReadOnly, intendedReadWrite) || + setsOverlap(liveReadOnly, liveReadWrite) + ) { + return false; + } + + // The native direct-GPU create policy deliberately omits /proc so OpenShell can add it + // read-write only after it observes GPU devices. Ordinary policies retain /proc read-only. + if (intendedReadOnly.has(PROC_PATH) || intendedReadWrite.has(PROC_PATH)) return false; + if ( + !isSubset(OPENSHELL_PROXY_REQUIRED_READ_ONLY_PATHS, intendedReadOnly) || + !isSubset(OPENSHELL_PROXY_REQUIRED_READ_WRITE_PATHS, intendedReadWrite) + ) { + return false; + } + if (liveReadOnly.has(PROC_PATH) || !liveReadWrite.has(PROC_PATH)) return false; + if (!isSubset(intendedReadOnly, liveReadOnly) || !isSubset(intendedReadWrite, liveReadWrite)) { + return false; + } + + for (const path of liveReadOnly) { + if (!intendedReadOnly.has(path) && !OPENSHELL_GPU_READ_ONLY_PATHS.has(path)) return false; + } + for (const path of liveReadWrite) { + if ( + !intendedReadWrite.has(path) && + !OPENSHELL_GPU_READ_WRITE_PATHS.has(path) && + !OPENSHELL_GPU_DEVICE_PATH.test(path) + ) { + return false; + } + } + return true; +} + /** * Materialize narrow native and compatibility fallback policies before sandbox-create side * effects. `preparePolicy` may create secure temporary files; every successful result carries its diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 69729ba909a..6b6aac7fe30 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -911,6 +911,8 @@ describe("source registry fingerprint", () => { vi.resetModules(); try { const registry = await import("../state/registry"); + const lifecycleGeneration = "00000000-0000-4000-8000-000000000001"; + const lifecycleLiveIdentityFingerprint = "a".repeat(64); registry.registerSandbox({ name: "alpha", agent: "openclaw", @@ -924,10 +926,23 @@ describe("source registry fingerprint", () => { preferredInferenceApi: "openai-responses", gatewayName: "nemoclaw", gatewayPort: 8080, + lifecycleGeneration, + lifecycleLiveIdentityFingerprint, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "alpha", + lifecycleGeneration, + sandboxIdentityFingerprint: lifecycleLiveIdentityFingerprint, + policyHash: "policy-alpha", + policyVersion: 1, + }, }); - const journaled = fingerprintSandboxRegistryEntry( - registry.getSandbox("alpha") as SandboxEntry, - ); + const sourceEntry = registry.getSandbox("alpha") as SandboxEntry; + const journaled = fingerprintSandboxRegistryEntry(sourceEntry); const hostLocalInferenceReceipt = serializedHostLocalInferenceReceipt("podman"); expect( @@ -945,7 +960,16 @@ describe("source registry fingerprint", () => { ).toBe(true); const reserved = registry.getSandbox("alpha") as SandboxEntry; expect(reserved.hostLocalInferenceReceipt).toBe(hostLocalInferenceReceipt); + expect(reserved.policyAuthority).toBe("nemoclaw-managed"); + expect(reserved.policyCreationReceipt).toEqual( + expect.objectContaining({ lifecycleGeneration, policyHash: "policy-alpha" }), + ); expect(fingerprintSandboxRegistryEntry(reserved)).toBe(journaled); + + registry.restoreSandboxEntry(sourceEntry); + expect(registry.getSandbox("alpha")?.policyCreationReceipt).toEqual( + sourceEntry.policyCreationReceipt, + ); } finally { await fs.rm(home, { recursive: true, force: true }); } @@ -975,7 +999,11 @@ describe("source registry fingerprint", () => { hostLocalInferenceReceipt, hostLocalInferenceProvenance, }; - registry.reserveSandboxInferenceRoute("alpha", route); + const { + hostLocalInferenceReceipt: _reservedReceipt, + hostLocalInferenceProvenance: _reservedProvenance, + ...registeredRoute + } = route; registry.registerSandbox({ name: "alpha", agent: "openclaw", @@ -983,7 +1011,7 @@ describe("source registry fingerprint", () => { createdAt: ISO, imageTag: "nemoclaw/openclaw:2026.3.11", lifecycleGeneration: "alpha-generation-1", - ...route, + ...registeredRoute, }); const journaled = fingerprintSandboxRegistryEntry( registry.getSandbox("alpha") as SandboxEntry, diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index ce67743d544..5b9b5bd74f1 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -699,18 +699,14 @@ describe("registerCreatedSandbox", () => { expect(classifyPortableLifecycleReceipt).not.toHaveBeenCalled(); }); - it("stages an owned registration until onboarding commits the sandbox step", () => { + it("publishes ordinary registrations without verified-create transaction options", () => { const registerSandbox = vi.fn(); const entry = registerCreatedSandbox({ ...createdRegistryEntryInput({ lifecycleGeneration: "generation-1" }), - reservationSessionId: "session-owner", registerSandbox, }); - expect(registerSandbox).toHaveBeenCalledExactlyOnceWith(entry, undefined, { - pending: true, - reservationSessionId: "session-owner", - }); + expect(registerSandbox).toHaveBeenCalledExactlyOnceWith(entry); }); it("persists lifecycle identity for a non-OpenClaw agent", () => { diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index fbc4fbbad39..d594d7765ba 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -3,10 +3,12 @@ import { isDeepStrictEqual } from "node:util"; -import type { SandboxPolicyAuthority } from "../adapters/openshell/policy-authority"; import type { AgentDefinition } from "../agent/defs"; import type { InferenceEndpointSource, InferenceSelection } from "../inference/selection"; -import { inferenceSelectionRegistryFields, normalizeInferenceSelection } from "../inference/selection"; +import { + inferenceSelectionRegistryFields, + normalizeInferenceSelection, +} from "../inference/selection"; import { type WebSearchConfig, webSearchProviderForConfig } from "../inference/web-search"; import * as onboardSession from "../state/onboard-session"; import type { OpenClawImagePluginInstall } from "../state/openclaw-plugin-restore"; @@ -70,7 +72,6 @@ export interface CreatedSandboxRegistryEntryInput { hostLocalInferenceProvenance?: SandboxEntry["hostLocalInferenceProvenance"]; openclawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; appliedPolicies: string[]; - policyAuthority?: SandboxPolicyAuthority; toolDisclosure?: ToolDisclosure; observabilityEnabled?: boolean; dcodeAutoApprovalMode?: DcodeAutoApprovalMode; @@ -98,19 +99,24 @@ export interface CreatedSandboxRegistryEntryInput { lifecycleLiveIdentityFingerprint?: string; gatewayName: string; gatewayPort: number; + policyAuthority?: SandboxEntry["policyAuthority"]; + policyCreationReceipt?: SandboxEntry["policyCreationReceipt"]; hostMounts?: readonly import("../state/registry/types").SandboxHostMount[]; } export interface CreatedSandboxRegistrationInput extends CreatedSandboxRegistryEntryInput { portableLifecycle?: boolean; + reservationSessionId?: string; environment?: NodeJS.ProcessEnv; classifyPortableLifecycleReceipt?: typeof classifyPortableLifecycleReceipt; inferenceRouteReservation?: QualifiedSandboxInferenceRouteReservation; - reservationSessionId?: string; + verifiedCreate?: NonNullable< + NonNullable[2]>["verifiedCreate"] + >; registerSandbox?( entry: SandboxEntry, routeReservation?: QualifiedSandboxInferenceRouteReservation, - options?: { pending?: boolean; reservationSessionId?: string }, + options?: Parameters[2], ): SandboxEntry | void; runtimeProviders?: RuntimeProviderBundleRegistry; } @@ -277,6 +283,9 @@ export function buildCreatedSandboxRegistryEntry( : {}), policies: input.appliedPolicies, ...(input.policyAuthority !== undefined ? { policyAuthority: input.policyAuthority } : {}), + ...(input.policyCreationReceipt !== undefined + ? { policyCreationReceipt: input.policyCreationReceipt } + : {}), baselineExclusions: input.baselineExclusions?.map((exclusion) => ({ ...exclusion })), toolDisclosure: input.toolDisclosure ?? DEFAULT_TOOL_DISCLOSURE, observabilityEnabled: input.observabilityEnabled === true, @@ -384,12 +393,19 @@ export function registerCreatedSandbox(input: CreatedSandboxRegistrationInput): ); } const writeRegistry = input.registerSandbox ?? registry.registerSandbox; - const pendingOptions = input.reservationSessionId - ? { pending: true as const, reservationSessionId: input.reservationSessionId } - : undefined; + const pendingOptions = + input.reservationSessionId && !input.verifiedCreate + ? { + pending: true as const, + reservationSessionId: input.reservationSessionId, + } + : undefined; + const registrationOptions = input.verifiedCreate + ? { verifiedCreate: input.verifiedCreate } + : pendingOptions; const registered = - input.inferenceRouteReservation || pendingOptions - ? writeRegistry(entry, input.inferenceRouteReservation, pendingOptions) + input.inferenceRouteReservation || registrationOptions + ? writeRegistry(entry, input.inferenceRouteReservation, registrationOptions) : writeRegistry(entry); return registered ?? entry; } diff --git a/src/lib/onboard/setup-inference-policy-authority.test.ts b/src/lib/onboard/setup-inference-policy-authority.test.ts index 876b693fe61..c7ed74f017d 100644 --- a/src/lib/onboard/setup-inference-policy-authority.test.ts +++ b/src/lib/onboard/setup-inference-policy-authority.test.ts @@ -83,6 +83,9 @@ describe("onboard inference policy authority mutation edges", () => { expect(deps.checkGatewayRouteCompatibility).not.toHaveBeenCalled(); expect(deps.upsertProvider).not.toHaveBeenCalled(); + expect(deps.updateSandbox).not.toHaveBeenCalled(); + expect(deps.verifyOnboardInferenceSmoke).not.toHaveBeenCalled(); + expect(deps.log).not.toHaveBeenCalled(); }); it("withholds success when authority changes after inference smoke (#9833)", async () => { diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 4ff82c70fe1..3f44ba39afd 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -594,13 +594,17 @@ export function createSetupInference( if (sandboxName && !options.revalidatePolicyRequirements) { throw new Error("Sandbox inference setup requires policy authority revalidation."); } + const revalidatePolicyRequirements = (operation: string): void => { + if (!sandboxName) return; + options.revalidatePolicyRequirements?.(operation); + }; const gatewayName = options.gatewayName ?? deps.getGatewayName(); - const revalidatePolicyRequirements = options.revalidatePolicyRequirements ?? (() => {}); const endpointSource = options.endpointSource === undefined ? "onboard" : options.endpointSource; const routedProvider = deps.isRoutedInferenceProvider?.(provider) === true; const usesBedrockRuntimeAdapter = provider === "compatible-anthropic-endpoint" && isBedrockRuntimeEndpoint(endpointUrl); + let shouldLogSuccessfulRoute = false; const withInferenceMutationLocks = (operation: () => Promise | T): Promise => deps.withGatewayRouteMutationLock(gatewayName, () => { if (!routedProvider) return operation(); @@ -1108,6 +1112,7 @@ export function createSetupInference( } } revalidatePolicyRequirements("report successful inference provider setup"); + shouldLogSuccessfulRoute = true; return { ok: true as const }; } catch (error) { if (!hostLocalRoute || !hostLocalSelection) throw error; @@ -1166,7 +1171,7 @@ export function createSetupInference( deps, revalidatePolicyRequirements, ); - if ("ok" in result && !usesBedrockRuntimeAdapter && provider !== "openrouter-api") { + if (shouldLogSuccessfulRoute && "ok" in result) { deps.log(` ✓ Inference route set: ${provider} / ${model}`); } return result; diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index ac0e20259af..1cfc35eadb6 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -56,6 +56,8 @@ export type ModelValidationResult = ModelValidationSuccess | ModelValidationFail export interface SandboxCreateIntent { /** Complete secret-free create plan resolved by the onboarding machine. */ readonly resolved?: import("./sandbox-create-intent-types").SandboxCreateIntent; + /** Defer provider, credential, and attachment effects until the created sandbox is verified. */ + readonly deferSandboxEffectsUntilPolicyVerification?: true; readonly recreate: boolean; readonly toolDisclosure: import("../tool-disclosure").ToolDisclosure; readonly observabilityEnabled: boolean; @@ -88,6 +90,42 @@ export interface SandboxCreateIntent { readonly rebuildPolicyPresets?: readonly string[]; } +/** Policy authority proved inside one exact post-create sandbox identity gate. */ +export type VerifiedSandboxPolicyRegistration = + | { + readonly policyAuthority: "nemoclaw-managed"; + readonly policyCreationReceipt: import("../policy/merge").NemoClawPolicyCreationReceipt; + readonly observedPolicyAuthority: "owner-unknown"; + } + | { + readonly policyAuthority: "externally-managed"; + readonly policyCreationReceipt: null; + /** Generic evidence seam; the default #10115 verifier produces only global authority. */ + readonly observedPolicyAuthority: "externally-managed" | "owner-unknown"; + readonly policyIdentity: import("../policy/merge").OpenShellPolicyIdentity; + }; + +/** Exact sandbox and policy result retained from the immediate create gate. */ +export interface VerifiedSandboxPolicyBoundary { + readonly registration: VerifiedSandboxPolicyRegistration; + readonly sandboxName: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly lifecycleGeneration: string; + readonly lifecycleLiveIdentityFingerprint: string; + readonly route: import("./docker-gpu-route").SelectedDockerGpuRoute; +} + +/** Exact context made available only after effective-policy verification. */ +export interface VerifiedSandboxCreateEffectsContext extends VerifiedSandboxPolicyBoundary { + readonly revalidatePolicyRequirements: (operation: string) => void; +} + +/** Ephemeral effects that may run only inside the exact post-create policy gate. */ +export type VerifiedSandboxCreateEffects = ( + context: VerifiedSandboxCreateEffectsContext, +) => Promise; + /** Durable onboarding-session identity that owns the pending inference route. */ export interface InferenceRouteReservationAuthority { readonly sessionId: string; diff --git a/src/lib/policy/baseline-exclusion-journal-integration.test.ts b/src/lib/policy/baseline-exclusion-journal-integration.test.ts index cfbea706568..41149da2223 100644 --- a/src/lib/policy/baseline-exclusion-journal-integration.test.ts +++ b/src/lib/policy/baseline-exclusion-journal-integration.test.ts @@ -6,12 +6,15 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + managedPolicyInspection, + managedSandboxEntry, + SANDBOX_IDENTITY, +} from "../../../test/helpers/managed-policy-receipt-fixture"; const harness = vi.hoisted(() => ({ - inspectSandboxPolicyAuthority: vi.fn(() => ({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - })), + inspectOpenShellSandboxIdentityFingerprint: vi.fn(() => SANDBOX_IDENTITY), + inspectSandboxPolicyAuthority: vi.fn(() => managedPolicyInspection()), livePolicy: "", run: vi.fn(), runCapture: vi.fn(), @@ -19,6 +22,7 @@ const harness = vi.hoisted(() => ({ vi.mock("../adapters/openshell/policy-authority", async (importOriginal) => ({ ...(await importOriginal()), + inspectOpenShellSandboxIdentityFingerprint: harness.inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority: harness.inspectSandboxPolicyAuthority, })); @@ -55,11 +59,7 @@ describe("baseline exclusion journal integration", () => { const registry = await import("../state/registry"); const baseline = await import("./baseline-exclusion"); const policy = await import("./index"); - registry.registerSandbox({ - name: "alpha", - agent: "hermes", - gatewayName: "nemoclaw", - }); + registry.registerSandbox(managedSandboxEntry("alpha", "hermes")); harness.livePolicy = `version: 1 network_policies: diff --git a/src/lib/policy/baseline-exclusion-persistence.test.ts b/src/lib/policy/baseline-exclusion-persistence.test.ts index 431e79caa22..725c85d8d15 100644 --- a/src/lib/policy/baseline-exclusion-persistence.test.ts +++ b/src/lib/policy/baseline-exclusion-persistence.test.ts @@ -5,6 +5,11 @@ import fs from "node:fs"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import YAML from "yaml"; +import { + managedPolicyInspection, + managedSandboxEntry, + SANDBOX_IDENTITY, +} from "../../../test/helpers/managed-policy-receipt-fixture"; const mocks = vi.hoisted(() => ({ addBaselineExclusion: vi.fn(), @@ -14,6 +19,7 @@ const mocks = vi.hoisted(() => ({ getBaselineExclusions: vi.fn(), getBaselineExclusionTransition: vi.fn(), getSandbox: vi.fn(), + inspectOpenShellSandboxIdentityFingerprint: vi.fn(), inspectSandboxPolicyAuthority: vi.fn(), removeBaselineExclusion: vi.fn(), run: vi.fn(), @@ -22,6 +28,7 @@ const mocks = vi.hoisted(() => ({ vi.mock("../adapters/openshell/policy-authority", async (importOriginal) => ({ ...(await importOriginal()), + inspectOpenShellSandboxIdentityFingerprint: mocks.inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority: mocks.inspectSandboxPolicyAuthority, })); @@ -97,15 +104,11 @@ describe("excludeBaselineEntry persistence boundary (#7178)", () => { vi.spyOn(console, "error").mockImplementation(() => undefined); mocks.runCapture.mockReturnValue(LIVE_POLICY); mocks.run.mockReturnValue({ status: 0 }); - mocks.inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }); + mocks.inspectOpenShellSandboxIdentityFingerprint.mockReturnValue(SANDBOX_IDENTITY); + mocks.inspectSandboxPolicyAuthority.mockReturnValue(managedPolicyInspection()); mocks.getSandbox.mockReturnValue({ - name: "alpha", - agent: "hermes", + ...managedSandboxEntry("alpha", "hermes"), agentVersion: "1.2.3", - policyAuthority: "nemoclaw-managed", }); mocks.getBaselineExclusions.mockReturnValue([]); mocks.getBaselineExclusionTransition.mockReturnValue(null); @@ -147,23 +150,23 @@ describe("excludeBaselineEntry persistence boundary (#7178)", () => { it("refuses exclusion journal changes when authority changes during the live read (#9833)", () => { mocks.inspectSandboxPolicyAuthority - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "externally-managed", effectivePolicy: {} }); + .mockReturnValueOnce(managedPolicyInspection()) + .mockReturnValueOnce({ ...managedPolicyInspection(), authority: "externally-managed" }); expect(excludeBaselineEntry("alpha", "nous_research", LIVE_DIGEST, { nonFatal: true })).toBe( false, ); expectNoBaselineMutation(); - expect(console.error).toHaveBeenCalledWith(expect.stringContaining("policy authority changed")); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("externally managed")); }); it("preserves a pending exclusion when authority changes at the policy-set edge (#9833)", () => { mocks.beginBaselineExclusionTransition.mockReturnValue(true); mocks.inspectSandboxPolicyAuthority - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValue({ authority: "externally-managed", effectivePolicy: {} }); + .mockReturnValueOnce(managedPolicyInspection()) + .mockReturnValueOnce(managedPolicyInspection()) + .mockReturnValue({ ...managedPolicyInspection(), authority: "externally-managed" }); expect(excludeBaselineEntry("alpha", "nous_research", LIVE_DIGEST, { nonFatal: true })).toBe( false, @@ -173,7 +176,7 @@ describe("excludeBaselineEntry persistence boundary (#7178)", () => { expect(mocks.run).not.toHaveBeenCalled(); expect(mocks.clearBaselineExclusionTransition).not.toHaveBeenCalled(); expect(mocks.commitBaselineExclusionTransition).not.toHaveBeenCalled(); - expect(console.error).toHaveBeenCalledWith(expect.stringContaining("policy authority changed")); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("externally managed")); }); it("clears a fresh transaction when live narrowing fails", () => { @@ -263,12 +266,11 @@ describe("excludeBaselineEntry persistence boundary (#7178)", () => { it("pins every live read and write to the sandbox's recorded gateway (#7178)", () => { vi.stubEnv("OPENSHELL_GATEWAY", "ambient-gateway"); mocks.getSandbox.mockReturnValue({ - name: "alpha", - agent: "hermes", + ...managedSandboxEntry("alpha", "hermes", { + gatewayName: "nemoclaw-18080", + gatewayPort: 18080, + }), agentVersion: "1.2.3", - gatewayName: "nemoclaw-18080", - gatewayPort: 18080, - policyAuthority: "nemoclaw-managed", }); mocks.beginBaselineExclusionTransition.mockReturnValue(true); mocks.runCapture @@ -412,15 +414,11 @@ describe("restoreBaselineEntry persistence boundary (#7178)", () => { vi.spyOn(console, "error").mockImplementation(() => undefined); mocks.runCapture.mockReturnValue("version: 1\nnetwork_policies: {}\n"); mocks.run.mockReturnValue({ status: 0 }); - mocks.inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }); + mocks.inspectOpenShellSandboxIdentityFingerprint.mockReturnValue(SANDBOX_IDENTITY); + mocks.inspectSandboxPolicyAuthority.mockReturnValue(managedPolicyInspection()); mocks.getSandbox.mockReturnValue({ - name: "alpha", - agent: "hermes", + ...managedSandboxEntry("alpha", "hermes"), agentVersion: "1.2.3", - policyAuthority: "nemoclaw-managed", }); mocks.getBaselineExclusions.mockReturnValue([RECORDED]); mocks.getBaselineExclusionTransition.mockReturnValue(null); @@ -471,13 +469,13 @@ describe("restoreBaselineEntry persistence boundary (#7178)", () => { it("refuses restore journal changes when authority changes during the live read (#9833)", () => { mocks.inspectSandboxPolicyAuthority - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "externally-managed", effectivePolicy: {} }); + .mockReturnValueOnce(managedPolicyInspection()) + .mockReturnValueOnce({ ...managedPolicyInspection(), authority: "externally-managed" }); expect(restoreBaselineEntry("alpha", "nous_research", { nonFatal: true })).toBe(false); expectNoBaselineMutation(); - expect(console.error).toHaveBeenCalledWith(expect.stringContaining("policy authority changed")); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("externally managed")); }); it("clears the restore transaction when live policy restoration fails", () => { @@ -511,10 +509,8 @@ describe("restoreBaselineEntry persistence boundary (#7178)", () => { digest: HERMES_MANAGED_INFERENCE_DIGEST, }; mocks.getSandbox.mockReturnValue({ - name: "alpha", - agent: "openclaw", + ...managedSandboxEntry("alpha", "openclaw"), agentVersion: "2.0.0", - policyAuthority: "nemoclaw-managed", }); mocks.getBaselineExclusions.mockReturnValue([staleExclusion]); mocks.runCapture @@ -625,10 +621,8 @@ describe("restoreBaselineEntry persistence boundary (#7178)", () => { it("keeps an interrupted restore pending when its agent baseline is unreadable (#7178)", () => { mocks.getSandbox.mockReturnValue({ - name: "alpha", - agent: "agent-without-a-readable-baseline", + ...managedSandboxEntry("alpha", "agent-without-a-readable-baseline"), agentVersion: "1.2.3", - policyAuthority: "nemoclaw-managed", }); mocks.runCapture.mockReturnValue(LIVE_POLICY); mocks.getBaselineExclusionTransition.mockReturnValue({ diff --git a/src/lib/policy/commands.test.ts b/src/lib/policy/commands.test.ts new file mode 100644 index 00000000000..8947080ae78 --- /dev/null +++ b/src/lib/policy/commands.test.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildGlobalPolicyGetFullJsonArgs, + buildGlobalPolicyListArgs, + buildPolicyGetArgs, + buildPolicyGetCommand, + buildPolicyGetFullCommand, + buildPolicyGetFullJsonArgs, + buildPolicyGetFullJsonCommand, + buildPolicySetCommand, +} from "./commands"; + +describe("OpenShell policy command builders", () => { + it("keeps every sandbox policy operation in an argv-only command", () => { + expect(buildPolicySetCommand("/tmp/policy.yaml", "alpha").slice(1)).toEqual([ + "policy", + "set", + "--policy", + "/tmp/policy.yaml", + "--wait", + "alpha", + ]); + expect(buildPolicyGetCommand("alpha").slice(1)).toEqual(["policy", "get", "--base", "alpha"]); + expect(buildPolicyGetFullCommand("alpha").slice(1)).toEqual([ + "policy", + "get", + "--full", + "alpha", + ]); + expect(buildPolicyGetFullJsonCommand("alpha").slice(1)).toEqual([ + "policy", + "get", + "--full", + "--output", + "json", + "alpha", + ]); + }); + + it("pins policy authority reads to the selected gateway", () => { + expect(buildPolicyGetArgs("alpha", "nemoclaw")).toEqual([ + "policy", + "get", + "-g", + "nemoclaw", + "--base", + "alpha", + ]); + expect(buildPolicyGetFullJsonArgs("alpha", "nemoclaw")).toEqual([ + "policy", + "get", + "-g", + "nemoclaw", + "--full", + "--output", + "json", + "alpha", + ]); + expect(buildGlobalPolicyGetFullJsonArgs("nemoclaw")).toEqual([ + "policy", + "get", + "-g", + "nemoclaw", + "--global", + "--full", + "--output", + "json", + ]); + expect(buildGlobalPolicyListArgs("nemoclaw")).toEqual([ + "policy", + "list", + "-g", + "nemoclaw", + "--global", + "--limit", + "1", + ]); + }); +}); diff --git a/src/lib/policy/commands.ts b/src/lib/policy/commands.ts index f3c182e9d7a..224ada198f1 100644 --- a/src/lib/policy/commands.ts +++ b/src/lib/policy/commands.ts @@ -21,13 +21,12 @@ export function buildPolicySetCommand( /** Read the round-trippable base policy before a mutation. */ export function buildPolicyGetCommand(sandboxName: string, gatewayName?: string): string[] { - return buildOpenshellCommand([ - "policy", - "get", - ...policyGatewayArgs(gatewayName), - "--base", - sandboxName, - ]); + return buildOpenshellCommand(buildPolicyGetArgs(sandboxName, gatewayName)); +} + +/** Read the round-trippable base policy before a mutation without selecting a binary. */ +export function buildPolicyGetArgs(sandboxName: string, gatewayName?: string): string[] { + return ["policy", "get", ...policyGatewayArgs(gatewayName), "--base", sandboxName]; } /** Read the effective policy for status and other diagnostics. */ @@ -46,8 +45,8 @@ function policyGatewayArgs(gatewayName?: string): string[] { } /** Read effective sandbox policy and its authority metadata as JSON. */ -export function buildPolicyGetFullJsonCommand(sandboxName: string, gatewayName?: string): string[] { - return buildOpenshellCommand([ +export function buildPolicyGetFullJsonArgs(sandboxName: string, gatewayName?: string): string[] { + return [ "policy", "get", ...policyGatewayArgs(gatewayName), @@ -55,12 +54,12 @@ export function buildPolicyGetFullJsonCommand(sandboxName: string, gatewayName?: "--output", "json", sandboxName, - ]); + ]; } -/** Read the global policy and its authority metadata as JSON. */ -export function buildGlobalPolicyGetFullJsonCommand(gatewayName?: string): string[] { - return buildOpenshellCommand([ +/** Read the active global policy and its authority metadata as JSON. */ +export function buildGlobalPolicyGetFullJsonArgs(gatewayName?: string): string[] { + return [ "policy", "get", ...policyGatewayArgs(gatewayName), @@ -68,17 +67,14 @@ export function buildGlobalPolicyGetFullJsonCommand(gatewayName?: string): strin "--full", "--output", "json", - ]); + ]; } /** Check whether the global policy has revision history. */ -export function buildGlobalPolicyListCommand(gatewayName?: string): string[] { - return buildOpenshellCommand([ - "policy", - "list", - ...policyGatewayArgs(gatewayName), - "--global", - "--limit", - "1", - ]); +export function buildGlobalPolicyListArgs(gatewayName?: string): string[] { + return ["policy", "list", ...policyGatewayArgs(gatewayName), "--global", "--limit", "1"]; +} + +export function buildPolicyGetFullJsonCommand(sandboxName: string, gatewayName?: string): string[] { + return buildOpenshellCommand(buildPolicyGetFullJsonArgs(sandboxName, gatewayName)); } diff --git a/src/lib/policy/context-builder.ts b/src/lib/policy/context-builder.ts index b07894602ff..668ab4caaf4 100644 --- a/src/lib/policy/context-builder.ts +++ b/src/lib/policy/context-builder.ts @@ -6,6 +6,7 @@ import { getBaselineExclusionRuntimeStatus, getGatewayPresets, getPresetEndpoints, + inspectPolicyMutationAuthority, isAgentBasePreset, listCustomPresets, listPresets, @@ -59,7 +60,7 @@ export interface PolicyContextTier { export interface PolicyContextSupportBoundary { capability: string; - owner: "nemoclaw" | "openshell" | "agent" | "external"; + owner: "nemoclaw" | "openshell" | "agent" | "external" | "unknown"; note?: string; } @@ -119,6 +120,10 @@ const EXTERNAL_POLICY_RESTORE_PATH = "Ask the external policy authority to restore baseline policy entry ``."; const EXTERNAL_POLICY_EXCLUDE_PATH = "Run `nemoclaw policy exclude --dry-run`, then ask the external policy authority to remove baseline policy entry ``."; +const UNKNOWN_POLICY_MUTATION_PATH = + "NemoClaw cannot change this policy because policy ownership is not verified. Recreate the sandbox before requesting a NemoClaw policy change."; + +type PolicyContextAuthority = "nemoclaw-managed" | "externally-managed" | "owner-unknown"; function hostStemsFromContent(content: string | null | undefined): { public: string[]; @@ -269,65 +274,87 @@ function buildBaselineExclusions( function buildApprovalPath( sandboxName: string, - externallyManaged: boolean, + authority: PolicyContextAuthority, ): PolicyContextApprovalPath { + const externallyManaged = authority === "externally-managed"; + const ownerUnknown = authority === "owner-unknown"; return { inspect: `nemoclaw ${sandboxName} policy list`, - add: externallyManaged - ? EXTERNAL_POLICY_ADD_PATH - : `nemoclaw ${sandboxName} policy add `, - remove: externallyManaged - ? EXTERNAL_POLICY_REMOVE_PATH - : `nemoclaw ${sandboxName} policy remove `, - excludeBaseline: externallyManaged - ? EXTERNAL_POLICY_EXCLUDE_PATH.replace("", sandboxName) - : `nemoclaw ${sandboxName} policy exclude --dry-run`, - restoreBaseline: externallyManaged - ? EXTERNAL_POLICY_RESTORE_PATH - : `nemoclaw ${sandboxName} policy restore `, + add: ownerUnknown + ? UNKNOWN_POLICY_MUTATION_PATH + : externallyManaged + ? EXTERNAL_POLICY_ADD_PATH + : `nemoclaw ${sandboxName} policy add `, + remove: ownerUnknown + ? UNKNOWN_POLICY_MUTATION_PATH + : externallyManaged + ? EXTERNAL_POLICY_REMOVE_PATH + : `nemoclaw ${sandboxName} policy remove `, + excludeBaseline: ownerUnknown + ? UNKNOWN_POLICY_MUTATION_PATH + : externallyManaged + ? EXTERNAL_POLICY_EXCLUDE_PATH.replace("", sandboxName) + : `nemoclaw ${sandboxName} policy exclude --dry-run`, + restoreBaseline: ownerUnknown + ? UNKNOWN_POLICY_MUTATION_PATH + : externallyManaged + ? EXTERNAL_POLICY_RESTORE_PATH + : `nemoclaw ${sandboxName} policy restore `, documentation: POLICY_DOC_URL, }; } function buildSupportBoundaries( tier: PolicyContextTier | null, - externallyManaged: boolean, + authority: PolicyContextAuthority, ): PolicyContextSupportBoundary[] { + const externallyManaged = authority === "externally-managed"; + const ownerUnknown = authority === "owner-unknown"; return [ { capability: "policy requirement selection and verification", owner: "nemoclaw", - note: externallyManaged - ? "NemoClaw selects preset and baseline requirements and verifies the live policy" - : tier - ? `tier: ${tier.label}` - : "no tier recorded", + note: ownerUnknown + ? "NemoClaw cannot verify the component that owns the live policy" + : externallyManaged + ? "NemoClaw selects preset and baseline requirements and verifies the live policy" + : tier + ? `tier: ${tier.label}` + : "no tier recorded", }, { capability: "host allowlist enforcement", owner: "openshell", note: "policy is enforced by the OpenShell gateway", }, - ...(externallyManaged + ...(ownerUnknown ? [ { - capability: "policy mutation", - owner: "external" as const, - note: "the external policy authority applies each required add, remove, restore, or baseline exclusion to the live policy", - }, - { - capability: "Shields state and configuration lock", - owner: "nemoclaw" as const, - note: "NemoClaw retains Shields state and locks configuration after it verifies restrictive policy", + capability: "policy and Shields mutation", + owner: "unknown" as const, + note: "NemoClaw refuses policy and Shields changes until it verifies policy ownership", }, ] - : [ - { - capability: "Shields transition", - owner: "nemoclaw" as const, - note: "Shields up locks down mutable configuration", - }, - ]), + : externallyManaged + ? [ + { + capability: "policy mutation", + owner: "external" as const, + note: "the external policy authority applies each required add, remove, restore, or baseline exclusion to the live policy", + }, + { + capability: "Shields state and configuration lock", + owner: "nemoclaw" as const, + note: "NemoClaw retains Shields state and locks configuration after it verifies restrictive policy", + }, + ] + : [ + { + capability: "Shields transition", + owner: "nemoclaw" as const, + note: "Shields up locks down mutable configuration", + }, + ]), { capability: "credential storage", owner: "nemoclaw", @@ -341,6 +368,19 @@ function buildSupportBoundaries( ]; } +function inspectContextPolicyAuthority( + sandboxName: string, + options: BuildPolicyContextOptions, +): PolicyContextAuthority { + if (options.skipGatewayProbe) return "owner-unknown"; + try { + return inspectPolicyMutationAuthority(sandboxName, "build the sandbox policy context") + .authority; + } catch { + return "owner-unknown"; + } +} + export interface BuildPolicyContextOptions { /** * Inject a gateway-preset list (or null when the gateway is unreachable) @@ -408,8 +448,8 @@ export function buildPolicyContext( options: BuildPolicyContextOptions = {}, ): PolicyContext { const sandbox = registry.getSandbox(sandboxName); - const externallyManaged = sandbox?.policyAuthority === "externally-managed"; - const tierName = externallyManaged ? null : (sandbox?.policyTier ?? null); + const authority = inspectContextPolicyAuthority(sandboxName, options); + const tierName = authority === "nemoclaw-managed" ? (sandbox?.policyTier ?? null) : null; const tierDef = tierName ? getTier(tierName) : null; const tier: PolicyContextTier | null = tierDef ? { name: tierDef.name, label: tierDef.label, description: tierDef.description } @@ -432,8 +472,8 @@ export function buildPolicyContext( sandboxName, sandbox?.baselineExclusionTransition ?? null, ), - approvalPath: buildApprovalPath(sandboxName, externallyManaged), - supportBoundaries: buildSupportBoundaries(tier, externallyManaged), + approvalPath: buildApprovalPath(sandboxName, authority), + supportBoundaries: buildSupportBoundaries(tier, authority), generatedAt: new Date().toISOString(), }; } diff --git a/src/lib/policy/context.test.ts b/src/lib/policy/context.test.ts index 96a067fff42..1c9d988931c 100644 --- a/src/lib/policy/context.test.ts +++ b/src/lib/policy/context.test.ts @@ -13,6 +13,7 @@ vi.mock(".", () => ({ getBaselineExclusionRuntimeStatus: vi.fn(() => "excluded"), getPresetEndpoints: vi.fn(), getGatewayPresets: vi.fn(() => null), + inspectPolicyMutationAuthority: vi.fn(() => ({ authority: "nemoclaw-managed" })), isAgentBasePreset: vi.fn(() => false), listCustomPresets: vi.fn(), listPresets: vi.fn(), @@ -111,6 +112,10 @@ function resetMocks() { vi.mocked(policies.getPresetEndpoints).mockReset(); vi.mocked(policies.getGatewayPresets).mockReset(); vi.mocked(policies.getGatewayPresets).mockReturnValue(null); + vi.mocked(policies.inspectPolicyMutationAuthority).mockReset(); + vi.mocked(policies.inspectPolicyMutationAuthority).mockReturnValue({ + authority: "nemoclaw-managed", + } as ReturnType); vi.mocked(policies.isAgentBasePreset).mockReset(); vi.mocked(policies.isAgentBasePreset).mockReturnValue(false); vi.mocked(registry.getBaselineExclusions).mockReset(); @@ -167,6 +172,9 @@ describe("buildPolicyContext", () => { policyTier: "balanced", policyAuthority: "externally-managed", }); + vi.mocked(policies.inspectPolicyMutationAuthority).mockReturnValue({ + authority: "externally-managed", + } as ReturnType); vi.mocked(registry.getBaselineExclusions).mockReturnValue([ { version: 1, @@ -230,6 +238,48 @@ describe("buildPolicyContext", () => { expect(markdown).not.toMatch(/nemoclaw alpha policy (?:add|remove|restore)(?:\s|`)/u); }); + it("does not advertise mutation commands when policy ownership is unknown (#9833)", () => { + resetMocks(); + mockBuiltinPresets(); + stubTier(); + stubRegistry({ policies: ["slack"], policyTier: "balanced" }); + vi.mocked(policies.inspectPolicyMutationAuthority).mockImplementation(() => { + throw new Error("receipt unavailable"); + }); + + const context = buildPolicyContext(SANDBOX); + + expect(context.tier).toBeNull(); + expect(context.approvalPath.add).not.toContain("policy add"); + expect(context.approvalPath.remove).not.toContain("policy remove"); + expect(context.approvalPath.excludeBaseline).not.toContain("policy exclude"); + expect(context.approvalPath.restoreBaseline).not.toContain("policy restore"); + expect(context.supportBoundaries).toContainEqual({ + capability: "policy and Shields mutation", + owner: "unknown", + note: "NemoClaw refuses policy and Shields changes until it verifies policy ownership", + }); + }); + + it("does not inspect live policy authority when gateway probes are disabled (#9833)", () => { + resetMocks(); + mockBuiltinPresets(); + stubTier(); + stubRegistry({ policies: ["slack"], policyTier: "balanced" }); + + const context = buildPolicyContext(SANDBOX, { skipGatewayProbe: true }); + + expect(policies.inspectPolicyMutationAuthority).not.toHaveBeenCalled(); + expect(policies.getGatewayPresets).not.toHaveBeenCalled(); + expect(context.tier).toBeNull(); + expect(context.approvalPath.add).not.toContain("policy add"); + expect(context.supportBoundaries).toContainEqual({ + capability: "policy and Shields mutation", + owner: "unknown", + note: "NemoClaw refuses policy and Shields changes until it verifies policy ownership", + }); + }); + it("marks active presets as `verified` when the gateway agrees and `registry-only` when it disagrees", () => { resetMocks(); mockBuiltinPresets(); diff --git a/src/lib/policy/failure-classifier.test.ts b/src/lib/policy/failure-classifier.test.ts index 56f818e26ce..b2e53dda495 100644 --- a/src/lib/policy/failure-classifier.test.ts +++ b/src/lib/policy/failure-classifier.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; +import { managedSandboxEntry } from "../../../test/helpers/managed-policy-receipt-fixture"; vi.mock("../state/registry", () => ({ getSandbox: vi.fn(), @@ -13,6 +14,7 @@ vi.mock(".", () => ({ getPresetEndpoints: vi.fn(), getGatewayPresets: vi.fn(() => null), getSandboxBaselineEntryDigest: vi.fn(() => null), + inspectPolicyMutationAuthority: vi.fn(() => ({ authority: "nemoclaw-managed" })), isAgentBasePreset: vi.fn(() => false), listCustomPresets: vi.fn(), listPresets: vi.fn(), @@ -78,7 +80,7 @@ function mockBuiltinPresets() { function stubRegistry(entry: Partial<{ policies: string[]; policyTier: string }>) { vi.mocked(registry.getSandbox).mockReturnValue({ - name: SANDBOX, + ...managedSandboxEntry(SANDBOX), policies: entry.policies, policyTier: entry.policyTier ?? null, } as ReturnType); @@ -104,6 +106,10 @@ function resetMocks() { vi.mocked(policies.getPresetEndpoints).mockReset(); vi.mocked(policies.getGatewayPresets).mockReset(); vi.mocked(policies.getGatewayPresets).mockReturnValue(null); + vi.mocked(policies.inspectPolicyMutationAuthority).mockReset(); + vi.mocked(policies.inspectPolicyMutationAuthority).mockReturnValue({ + authority: "nemoclaw-managed", + } as ReturnType); vi.mocked(policies.isAgentBasePreset).mockReset(); vi.mocked(policies.isAgentBasePreset).mockReturnValue(false); vi.mocked(getTier).mockReset(); diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index 7e6165cea31..e3c5695a435 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -15,6 +15,8 @@ import YAML from "yaml"; import { assertExternalPolicyRequirements, assertRecordedPolicyAuthority, + captureSandboxBasePolicy, + inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority, isExternalPolicyAuthorityRefusalError as isExternalAuthorityRefusalError, isPolicyAuthorityRefusalError as isAuthorityRefusalError, @@ -35,10 +37,10 @@ import { loadMessagingChannelPolicyPreset, materializeMessagingPolicySandboxName, } from "../messaging/channels"; -import { resolveSandboxGatewayName } from "../onboard/gateway-binding"; +import { resolveGatewayPortFromName, resolveSandboxGatewayName } from "../onboard/gateway-binding"; import { assertNoOpenShellGatewayEndpointOverride } from "../openshell-gateway-endpoint-guard"; import { OPENSHELL_SANDBOX_HOST_BRIDGE } from "../private-networks"; -import { ROOT, run, runCapture, runCaptureEx } from "../runner"; +import { ROOT, run, runCapture } from "../runner"; import { diagnosticPreview, isValidName, NAME_ALLOWED_FORMAT } from "../sandbox-name-contract"; import { redact } from "../security/redact"; import * as registry from "../state/registry"; @@ -57,7 +59,10 @@ import { } from "./commands"; import { inspectGatewayPresetNames, inspectPresetContentGatewayState } from "./gateway-state"; import { + assertNemoClawPolicyCreationReceiptMatches, + type NemoClawPolicyCreationReceipt, parseOpenShellPolicy, + parseNemoClawPolicyCreationReceipt, stripProviderComposedPolicies, withoutProviderComposedPolicies, } from "./merge"; @@ -589,39 +594,48 @@ export interface PolicyMutationAuthority { readonly authorityRecordedNow: boolean; readonly gatewayName: string; readonly inspection: SandboxPolicyAuthorityInspection; + readonly policyCreationReceipt?: NemoClawPolicyCreationReceipt | null; } export const isPolicyAuthorityRefusalError = isAuthorityRefusalError; export const isExternalPolicyAuthorityRefusalError = isExternalAuthorityRefusalError; -function inspectLivePolicyAuthority( +interface LivePolicyBoundary { + readonly sandbox: NonNullable>; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly inspection: SandboxPolicyAuthorityInspection; +} + +function inspectLivePolicyBoundary( sandboxName: string, operation: string, requestedGatewayName?: string, -): { - sandbox: ReturnType; - authority: PolicyMutationAuthority; -} { +): LivePolicyBoundary { let sandbox: ReturnType; try { sandbox = registry.getSandbox(sandboxName); } catch { throw new PolicyAuthorityRefusalError( - `Refusing to ${operation}: sandbox policy authority is unavailable.`, + `Refusing to ${operation}: sandbox '${sandboxName}' policy authority is unavailable.`, + ); + } + if (!sandbox) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: sandbox '${sandboxName}' policy authority is unavailable.`, ); } let recordedGatewayName: string | null; try { - recordedGatewayName = sandbox ? resolveSandboxGatewayName(sandbox) : null; + recordedGatewayName = resolveSandboxGatewayName(sandbox); } catch { throw new PolicyAuthorityRefusalError( `Refusing to ${operation}: the recorded sandbox gateway is unavailable or invalid.`, ); } if (recordedGatewayName && requestedGatewayName && requestedGatewayName !== recordedGatewayName) { - throw new Error( - `Refusing to ${operation}: sandbox '${sandboxName}' is recorded on gateway ` + - `'${recordedGatewayName}', not '${requestedGatewayName}'.`, + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the requested gateway does not match the recorded sandbox gateway.`, ); } let gatewayName: string; @@ -636,102 +650,209 @@ function inspectLivePolicyAuthority( const inspection = inspectSandboxPolicyAuthority({ sandboxName, gatewayName, - runCaptureEx, }); - return { - sandbox, - authority: { - authority: inspection.authority, - authorityRecordedNow: false, + const gatewayPort = resolveGatewayPortFromName(gatewayName); + if (gatewayPort === null) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the sandbox gateway is unavailable or invalid.`, + ); + } + return { sandbox, gatewayName, gatewayPort, inspection }; +} + +function managedReceiptSandboxBoundary( + live: LivePolicyBoundary, + sandboxName: string, + operation: string, +): { + readonly liveIdentityFingerprint: string; + readonly receipt: NemoClawPolicyCreationReceipt; +} { + const { sandbox, gatewayName, gatewayPort } = live; + if ( + sandbox.policyAuthority !== "nemoclaw-managed" || + sandbox.gatewayName !== gatewayName || + sandbox.gatewayPort !== gatewayPort || + typeof sandbox.lifecycleGeneration !== "string" || + typeof sandbox.lifecycleLiveIdentityFingerprint !== "string" + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: NemoClaw policy ownership is unavailable or incomplete.`, + "owner-unknown", + ); + } + + let liveIdentityFingerprint: string; + try { + liveIdentityFingerprint = inspectOpenShellSandboxIdentityFingerprint({ + sandboxName, gatewayName, - inspection, - }, - }; + }); + } catch { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the live sandbox identity could not be verified.`, + "owner-unknown", + ); + } + if (liveIdentityFingerprint !== sandbox.lifecycleLiveIdentityFingerprint) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the live sandbox identity does not match the registered lifecycle.`, + "owner-unknown", + ); + } + + let confirmedSandbox: ReturnType; + try { + confirmedSandbox = registry.getSandbox(sandboxName); + } catch { + confirmedSandbox = null; + } + if ( + !confirmedSandbox || + confirmedSandbox.pendingRouteReservation === true || + confirmedSandbox.policyAuthority !== sandbox.policyAuthority || + confirmedSandbox.gatewayName !== sandbox.gatewayName || + confirmedSandbox.gatewayPort !== sandbox.gatewayPort || + confirmedSandbox.lifecycleGeneration !== sandbox.lifecycleGeneration || + confirmedSandbox.lifecycleLiveIdentityFingerprint !== + sandbox.lifecycleLiveIdentityFingerprint || + !isDeepStrictEqual(confirmedSandbox.policyCreationReceipt, sandbox.policyCreationReceipt) + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the recorded policy creation receipt changed during live verification.`, + "owner-unknown", + ); + } + + let receipt: NemoClawPolicyCreationReceipt; + try { + receipt = parseNemoClawPolicyCreationReceipt(sandbox.policyCreationReceipt); + } catch { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the NemoClaw policy creation receipt is unavailable or invalid.`, + "owner-unknown", + ); + } + if ( + receipt.gatewayName !== gatewayName || + receipt.gatewayPort !== gatewayPort || + receipt.sandboxName !== sandboxName || + receipt.lifecycleGeneration !== sandbox.lifecycleGeneration || + receipt.sandboxIdentityFingerprint !== liveIdentityFingerprint + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the NemoClaw policy creation receipt does not match the live sandbox identity.`, + "owner-unknown", + ); + } + return { liveIdentityFingerprint, receipt }; } -/** Read live authority for Shields recovery without changing its durable owner. */ -export function inspectPolicyRecoveryAuthority( +function managedReceiptBoundary( + live: LivePolicyBoundary, sandboxName: string, operation: string, - requestedGatewayName?: string, -): PolicyMutationAuthority { - const live = inspectLivePolicyAuthority(sandboxName, operation, requestedGatewayName); - if (!live.sandbox) { +): { + readonly liveIdentityFingerprint: string; + readonly receipt: NemoClawPolicyCreationReceipt; +} { + const boundary = managedReceiptSandboxBoundary(live, sandboxName, operation); + try { + assertNemoClawPolicyCreationReceiptMatches(boundary.receipt, { + origin: "sandbox-create", + gatewayName: live.gatewayName, + gatewayPort: live.gatewayPort, + sandboxName, + lifecycleGeneration: live.sandbox.lifecycleGeneration as string, + sandboxIdentityFingerprint: boundary.liveIdentityFingerprint, + policyHash: live.inspection.policyIdentity.hash, + policyVersion: live.inspection.policyIdentity.activeVersion, + }); + } catch { throw new PolicyAuthorityRefusalError( - `Refusing to ${operation}: sandbox policy authority is unavailable.`, + `Refusing to ${operation}: the NemoClaw policy creation receipt does not match the live sandbox policy.`, + "owner-unknown", ); } - return live.authority; + return boundary; } -/** Inspect and, when needed, persist the authority that owns one sandbox policy. */ -export function inspectPolicyMutationAuthority( +function resolvePolicyAuthority( + live: LivePolicyBoundary, sandboxName: string, operation: string, - requestedGatewayName?: string, - requireRecordedAuthority = false, ): PolicyMutationAuthority { - const live = inspectLivePolicyAuthority(sandboxName, operation, requestedGatewayName); - const { sandbox } = live; - const { gatewayName, inspection } = live.authority; - if (sandbox?.policyAuthority !== undefined) { - try { - assertRecordedPolicyAuthority(sandbox.policyAuthority, inspection.authority, operation); - } catch (error) { - if ( - sandbox.policyAuthority === "externally-managed" || - inspection.authority === "externally-managed" - ) { - throw new PolicyAuthorityRefusalError( - `${policyAuthorityError(error)} The external policy authority must perform the requested policy mutation.`, - inspection.authority, - ); - } - throw error; + if (live.inspection.authority === "externally-managed") { + if (live.sandbox.policyAuthority === "nemoclaw-managed") { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the live policy is externally managed but the sandbox registry records NemoClaw ownership. The external policy authority must perform the requested policy mutation.`, + "externally-managed", + ); } return { - authority: inspection.authority, + authority: "externally-managed", authorityRecordedNow: false, - gatewayName, - inspection, + gatewayName: live.gatewayName, + inspection: live.inspection, + policyCreationReceipt: null, }; } - if (requireRecordedAuthority) { - throw new Error( - `Refusing to ${operation}: policy authority is not recorded for sandbox '${sandboxName}'.`, - ); - } - let authorityRecorded: boolean; - try { - authorityRecorded = Boolean( - sandbox && - registry.updateSandbox(sandboxName, { - policyAuthority: inspection.authority, - }), - ); - } catch { - authorityRecorded = false; - } - if (!authorityRecorded) { - throw new Error( - `Refusing to ${operation}: NemoClaw could not record policy authority for sandbox '${sandboxName}'.`, + if (live.inspection.authority !== "owner-unknown") { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the live sandbox policy authority is invalid.`, + "owner-unknown", ); } + const { receipt } = managedReceiptBoundary(live, sandboxName, operation); return { - authority: inspection.authority, - authorityRecordedNow: true, - gatewayName, - inspection, + authority: "nemoclaw-managed", + authorityRecordedNow: false, + gatewayName: live.gatewayName, + inspection: { ...live.inspection, authority: "nemoclaw-managed" }, + policyCreationReceipt: receipt, }; } +/** Read live authority for Shields recovery without changing its durable owner. */ +export function inspectPolicyRecoveryAuthority( + sandboxName: string, + operation: string, + requestedGatewayName?: string, +): PolicyMutationAuthority { + return resolvePolicyAuthority( + inspectLivePolicyBoundary(sandboxName, operation, requestedGatewayName), + sandboxName, + operation, + ); +} + +/** Inspect the live policy owner without creating an ownership claim from observation. */ +export function inspectPolicyMutationAuthority( + sandboxName: string, + operation: string, + requestedGatewayName?: string, + _requireRecordedAuthority = false, +): PolicyMutationAuthority { + return resolvePolicyAuthority( + inspectLivePolicyBoundary(sandboxName, operation, requestedGatewayName), + sandboxName, + operation, + ); +} + /** Require NemoClaw ownership before a local policy mutation. */ export function assertNemoClawManagedPolicy( authority: PolicyMutationAuthority, operation: string, ): void { if (authority.authority === "nemoclaw-managed") return; + if (authority.authority === "owner-unknown") { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: NemoClaw cannot verify policy ownership. Recreate this sandbox before requesting a NemoClaw policy mutation.`, + authority.authority, + ); + } throw new PolicyAuthorityRefusalError( `Refusing to ${operation}: this sandbox policy is externally managed. ` + "The external policy authority must perform the requested policy mutation.", @@ -752,6 +873,22 @@ export function recheckPolicyMutationAuthority( true, ); assertRecordedPolicyAuthority(recorded.authority, observed.authority, operation); + if ( + recorded.policyCreationReceipt != null && + observed.policyCreationReceipt != null && + (recorded.policyCreationReceipt.gatewayName !== observed.policyCreationReceipt.gatewayName || + recorded.policyCreationReceipt.gatewayPort !== observed.policyCreationReceipt.gatewayPort || + recorded.policyCreationReceipt.sandboxName !== observed.policyCreationReceipt.sandboxName || + recorded.policyCreationReceipt.lifecycleGeneration !== + observed.policyCreationReceipt.lifecycleGeneration || + recorded.policyCreationReceipt.sandboxIdentityFingerprint !== + observed.policyCreationReceipt.sandboxIdentityFingerprint) + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to ${operation}: the NemoClaw policy creation receipt changed sandbox identity.`, + "owner-unknown", + ); + } assertNemoClawManagedPolicy(observed, operation); return observed; } @@ -883,6 +1020,100 @@ function policySetFailure( ); } +export function finalizePolicyMutationReceipt( + sandboxName: string, + desiredPolicyDocument: string, + previous: PolicyMutationAuthority, +): void { + const previousReceipt = previous.policyCreationReceipt; + if (previousReceipt == null) { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but no policy creation receipt was available. The policy update is incomplete.`, + "owner-unknown", + ); + } + + const operation = "complete the sandbox policy update"; + const live = inspectLivePolicyBoundary(sandboxName, operation, previous.gatewayName); + if (live.inspection.authority !== "owner-unknown") { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but OpenShell no longer reports a sandbox-scoped policy. The policy update is incomplete.`, + live.inspection.authority, + ); + } + const boundary = managedReceiptSandboxBoundary(live, sandboxName, operation); + if (!isDeepStrictEqual(boundary.receipt, previousReceipt)) { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but its policy creation receipt changed during the update. The policy update is incomplete.`, + "owner-unknown", + ); + } + + let observedBasePolicy: string; + try { + observedBasePolicy = captureSandboxBasePolicy(sandboxName, previous.gatewayName); + } catch { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but could not verify the resulting base policy. The policy update is incomplete.`, + "owner-unknown", + ); + } + if (!policyDocumentsMatch(observedBasePolicy, desiredPolicyDocument)) { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but the resulting base policy did not match the requested policy. The policy update is incomplete.`, + "owner-unknown", + ); + } + + const confirmed = inspectLivePolicyBoundary(sandboxName, operation, previous.gatewayName); + if (confirmed.inspection.authority !== "owner-unknown") { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but OpenShell no longer reports a sandbox-scoped policy. The policy update is incomplete.`, + confirmed.inspection.authority, + ); + } + const confirmedBoundary = managedReceiptSandboxBoundary(confirmed, sandboxName, operation); + if ( + !isDeepStrictEqual(confirmedBoundary.receipt, previousReceipt) || + confirmedBoundary.liveIdentityFingerprint !== boundary.liveIdentityFingerprint || + confirmed.inspection.policyIdentity.hash !== live.inspection.policyIdentity.hash || + confirmed.inspection.policyIdentity.activeVersion !== + live.inspection.policyIdentity.activeVersion + ) { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but its policy identity changed during verification. The policy update is incomplete.`, + "owner-unknown", + ); + } + + const nextReceipt: NemoClawPolicyCreationReceipt = { + ...previousReceipt, + policyHash: confirmed.inspection.policyIdentity.hash, + policyVersion: confirmed.inspection.policyIdentity.activeVersion, + }; + if ( + !registry.compareAndSetSandboxPolicyCreationReceipt(sandboxName, previousReceipt, nextReceipt) + ) { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but could not record the resulting policy identity. The policy update is incomplete.`, + "owner-unknown", + ); + } + + const completed = inspectPolicyMutationAuthority( + sandboxName, + operation, + previous.gatewayName, + true, + ); + if (!isDeepStrictEqual(completed.policyCreationReceipt, nextReceipt)) { + throw new PolicyAuthorityRefusalError( + `NemoClaw applied the sandbox policy for '${sandboxName}', but could not verify the recorded policy identity. The policy update is incomplete.`, + "owner-unknown", + ); + } +} + /** * Apply a composed policy document while optionally keeping control in the * caller on failure. Lifecycle code that owns compensating actions must use @@ -892,31 +1123,28 @@ function policySetFailure( * The submission owns the temp policy file, so the composed policy is already * deleted by the time this ends the process for a fatal caller (#9206). */ -function setPolicyDocument( +export function setReceiptBoundPolicyDocument( sandboxName: string, policyDocument: string, options: { nonFatal?: boolean; gatewayName?: string; + operation?: string; + authority?: PolicyMutationAuthority; } = {}, ): boolean { + const operation = options.operation ?? "set the sandbox policy"; let authority: PolicyMutationAuthority; try { + if (options.authority) { + recheckPolicyMutationAuthority(sandboxName, operation, options.authority); + } authority = inspectPolicyMutationAuthority( sandboxName, - "set the sandbox policy", - options.gatewayName, + operation, + options.authority?.gatewayName ?? options.gatewayName, ); - assertNemoClawManagedPolicy(authority, "set the sandbox policy"); - if (authority.authorityRecordedNow) { - authority = inspectPolicyMutationAuthority( - sandboxName, - "set the sandbox policy", - authority.gatewayName, - true, - ); - assertNemoClawManagedPolicy(authority, "set the sandbox policy"); - } + assertNemoClawManagedPolicy(authority, operation); } catch (error) { console.error(` ${policyAuthorityError(error)}`); if (options.nonFatal) return false; @@ -928,7 +1156,16 @@ function setPolicyDocument( policyDocument, authority.gatewayName, ); - if (outcome.kind === "applied") return true; + if (outcome.kind === "applied") { + try { + finalizePolicyMutationReceipt(sandboxName, policyDocument, authority); + return true; + } catch (error) { + console.error(` ${policyAuthorityError(error)}`); + if (options.nonFatal) return false; + process.exit(1); + } + } console.error(` ${policySetFailure(sandboxName, outcome).message}`); if (options.nonFatal) return false; @@ -1705,7 +1942,7 @@ function removePreset( if (!recheckNemoClawManagedPolicy(sandboxName, operation, authority)) return false; if ( - !setPolicyDocument(sandboxName, updated, { + !setReceiptBoundPolicyDocument(sandboxName, updated, { nonFatal: options.nonFatal, gatewayName: authority.gatewayName, }) @@ -1736,7 +1973,7 @@ function pushPolicyYaml( options: { nonFatal?: boolean; gatewayName?: string } = {}, ): boolean { if (!assertOpenshellResolvable(options)) return false; - return setPolicyDocument(sandboxName, updatedPolicy, options); + return setReceiptBoundPolicyDocument(sandboxName, updatedPolicy, options); } /** Round-trippable live policy body from `--base`, or null when unreadable. */ @@ -2727,7 +2964,7 @@ function applyPresetContent( if (policyChanged) { if (!recheckNemoClawManagedPolicy(sandboxName, operation, authority)) return false; if ( - !setPolicyDocument(sandboxName, merged, { + !setReceiptBoundPolicyDocument(sandboxName, merged, { nonFatal: options.nonFatal, gatewayName: authority.gatewayName, }) @@ -2971,7 +3208,9 @@ function applyPresets(sandboxName: string, presetNames: string[]): boolean { // temporary policy. Onboarding defers that exit until its recovery state // and outer cleanup have finished. if (!recheckNemoClawManagedPolicy(sandboxName, operation, authority)) return false; - setPolicyDocument(sandboxName, merged, { gatewayName: authority.gatewayName }); + setReceiptBoundPolicyDocument(sandboxName, merged, { + gatewayName: authority.gatewayName, + }); if (!recheckNemoClawManagedPolicy(sandboxName, operation, authority)) return false; } @@ -3375,7 +3614,9 @@ function applyPermissivePolicy(sandboxName: string): void { console.log(" Applying permissive policy..."); assertOpenshellResolvable(); recheckPolicyMutationAuthority(sandboxName, operation, authority); - setPolicyDocument(sandboxName, materializedPolicy, { gatewayName: authority.gatewayName }); + setReceiptBoundPolicyDocument(sandboxName, materializedPolicy, { + gatewayName: authority.gatewayName, + }); const observed = inspectPolicyMutationAuthority( sandboxName, operation, diff --git a/src/lib/policy/merge.ts b/src/lib/policy/merge.ts index 415eeeaf0c2..d8deb34eb3f 100644 --- a/src/lib/policy/merge.ts +++ b/src/lib/policy/merge.ts @@ -3,12 +3,19 @@ import { assertExternalPolicyRequirementContainment as assertCanonicalExternalPolicyRequirementContainment, + assertPolicyRequirementContainment as assertCanonicalPolicyRequirementContainment, assertMatchingPolicyAuthority as assertCanonicalMatchingPolicyAuthority, - parseGlobalPolicyAuthorityMetadata as parseCanonicalGlobalPolicyAuthorityMetadata, + assertNemoClawPolicyCreationReceiptMatches as assertCanonicalNemoClawPolicyCreationReceiptMatches, + parseActiveGlobalPolicyAuthorityMetadata as parseCanonicalActiveGlobalPolicyAuthorityMetadata, + parseNemoClawPolicyCreationReceipt as parseCanonicalNemoClawPolicyCreationReceipt, + openShellPolicyValuesEqual as canonicalOpenShellPolicyValuesEqual, parseOpenShellPolicy as parseCanonicalOpenShellPolicy, parseSandboxPolicyAuthorityMetadata as parseCanonicalSandboxPolicyAuthorityMetadata, stripProviderComposedPolicies as stripCanonicalProviderComposedPolicies, + type NemoClawPolicyCreationReceipt, + type ActiveGlobalPolicyInspection, type OpenShellPolicyAuthority, + type OpenShellPolicyIdentity, type SandboxPolicyAuthorityInspection, withoutProviderComposedPolicies as withoutCanonicalProviderComposedPolicies, } from "../../../nemoclaw/dist/shared/openshell-policy-boundary.cjs"; @@ -19,15 +26,25 @@ import type { JsonObject } from "../core/json-types"; // generatedBoundary: build:cli emits the canonical .cjs/.d.cts before this // CommonJS wrapper is compiled. Keep this file implementation-free. export const parseOpenShellPolicy = parseCanonicalOpenShellPolicy; +export const openShellPolicyValuesEqual = canonicalOpenShellPolicyValuesEqual; +export const parseNemoClawPolicyCreationReceipt = parseCanonicalNemoClawPolicyCreationReceipt; +export const parseActiveGlobalPolicyAuthorityMetadata = + parseCanonicalActiveGlobalPolicyAuthorityMetadata; +export const assertNemoClawPolicyCreationReceiptMatches = + assertCanonicalNemoClawPolicyCreationReceiptMatches; export const stripProviderComposedPolicies = stripCanonicalProviderComposedPolicies; -export const parseGlobalPolicyAuthorityMetadata = - parseCanonicalGlobalPolicyAuthorityMetadata; -export const parseSandboxPolicyAuthorityMetadata = - parseCanonicalSandboxPolicyAuthorityMetadata; +export const parseSandboxPolicyAuthorityMetadata = parseCanonicalSandboxPolicyAuthorityMetadata; export const assertMatchingPolicyAuthority = assertCanonicalMatchingPolicyAuthority; export const assertExternalPolicyRequirementContainment = assertCanonicalExternalPolicyRequirementContainment; -export type { OpenShellPolicyAuthority, SandboxPolicyAuthorityInspection }; +export const assertPolicyRequirementContainment = assertCanonicalPolicyRequirementContainment; +export type { + ActiveGlobalPolicyInspection, + NemoClawPolicyCreationReceipt, + OpenShellPolicyAuthority, + OpenShellPolicyIdentity, + SandboxPolicyAuthorityInspection, +}; export function withoutProviderComposedPolicies(policies: JsonObject): JsonObject { return withoutCanonicalProviderComposedPolicies(policies) as JsonObject; diff --git a/src/lib/policy/policy-apply-finality.test.ts b/src/lib/policy/policy-apply-finality.test.ts index 60a5207530b..5cae22f8984 100644 --- a/src/lib/policy/policy-apply-finality.test.ts +++ b/src/lib/policy/policy-apply-finality.test.ts @@ -6,10 +6,16 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + managedPolicyInspection, + managedSandboxEntry, + SANDBOX_IDENTITY, +} from "../../../test/helpers/managed-policy-receipt-fixture"; const { addCustomPolicy, getSandbox, + inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority, resolveOpenshell, run, @@ -18,6 +24,7 @@ const { } = vi.hoisted(() => ({ addCustomPolicy: vi.fn(), getSandbox: vi.fn(), + inspectOpenShellSandboxIdentityFingerprint: vi.fn(), inspectSandboxPolicyAuthority: vi.fn(), resolveOpenshell: vi.fn(), run: vi.fn(), @@ -27,6 +34,7 @@ const { vi.mock("../adapters/openshell/policy-authority", async (importOriginal) => ({ ...(await importOriginal()), + inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority, })); @@ -152,23 +160,17 @@ describe("applyPresets finality when openshell rejects the composed policy", () run.mockReset(); runCapture.mockReset(); getSandbox.mockReset(); + inspectOpenShellSandboxIdentityFingerprint.mockReset(); inspectSandboxPolicyAuthority.mockReset(); updateSandbox.mockReset(); addCustomPolicy.mockReset(); resolveOpenshell.mockReset(); resolveOpenshell.mockReturnValue("/usr/local/bin/openshell"); - inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }); + inspectOpenShellSandboxIdentityFingerprint.mockReturnValue(SANDBOX_IDENTITY); + inspectSandboxPolicyAuthority.mockReturnValue(managedPolicyInspection()); runCapture.mockReturnValue(BASE_POLICY); - getSandbox.mockReturnValue({ - name: SANDBOX, - agent: "openclaw", - policies: [], - policyAuthority: "nemoclaw-managed", - }); + getSandbox.mockReturnValue(managedSandboxEntry(SANDBOX)); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); }); @@ -235,22 +237,16 @@ describe("single-preset mutations when openshell rejects the composed policy", ( run.mockReset(); runCapture.mockReset(); getSandbox.mockReset(); + inspectOpenShellSandboxIdentityFingerprint.mockReset(); inspectSandboxPolicyAuthority.mockReset(); updateSandbox.mockReset(); addCustomPolicy.mockReset(); resolveOpenshell.mockReset(); resolveOpenshell.mockReturnValue("/usr/local/bin/openshell"); - inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }); - getSandbox.mockReturnValue({ - name: SANDBOX, - agent: "openclaw", - policies: ["weather"], - policyAuthority: "nemoclaw-managed", - }); + inspectOpenShellSandboxIdentityFingerprint.mockReturnValue(SANDBOX_IDENTITY); + inspectSandboxPolicyAuthority.mockReturnValue(managedPolicyInspection()); + getSandbox.mockReturnValue({ ...managedSandboxEntry(SANDBOX), policies: ["weather"] }); run.mockReturnValue(policySetResult(openshellRejection(REJECTION_MESSAGE))); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); @@ -294,22 +290,16 @@ describe("applyPresets temporary policy material under local I/O failure", () => run.mockReset(); runCapture.mockReset(); getSandbox.mockReset(); + inspectOpenShellSandboxIdentityFingerprint.mockReset(); inspectSandboxPolicyAuthority.mockReset(); updateSandbox.mockReset(); resolveOpenshell.mockReset(); resolveOpenshell.mockReturnValue("/usr/local/bin/openshell"); - inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }); + inspectOpenShellSandboxIdentityFingerprint.mockReturnValue(SANDBOX_IDENTITY); + inspectSandboxPolicyAuthority.mockReturnValue(managedPolicyInspection()); runCapture.mockReturnValue(BASE_POLICY); - getSandbox.mockReturnValue({ - name: SANDBOX, - agent: "openclaw", - policies: [], - policyAuthority: "nemoclaw-managed", - }); + getSandbox.mockReturnValue(managedSandboxEntry(SANDBOX)); run.mockReturnValue(policySetResult(openshellRejection("refused"))); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); diff --git a/src/lib/policy/policy-mutation-authority.test.ts b/src/lib/policy/policy-mutation-authority.test.ts index 276d3fe7692..4dd00301fdf 100644 --- a/src/lib/policy/policy-mutation-authority.test.ts +++ b/src/lib/policy/policy-mutation-authority.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import YAML from "yaml"; +import fs from "node:fs"; const mocks = vi.hoisted(() => ({ addCustomPolicy: vi.fn(), @@ -10,7 +11,10 @@ const mocks = vi.hoisted(() => ({ getBaselineExclusions: vi.fn(), getBaselineExclusionTransition: vi.fn(), getSandbox: vi.fn(), + captureSandboxBasePolicy: vi.fn(), inspectSandboxPolicyAuthority: vi.fn(), + inspectOpenShellSandboxIdentityFingerprint: vi.fn(), + compareAndSetSandboxPolicyCreationReceipt: vi.fn(), resolveOpenshell: vi.fn(), run: vi.fn(), runCapture: vi.fn(), @@ -19,6 +23,8 @@ const mocks = vi.hoisted(() => ({ vi.mock("../adapters/openshell/policy-authority", async (importOriginal) => ({ ...(await importOriginal()), + captureSandboxBasePolicy: mocks.captureSandboxBasePolicy, + inspectOpenShellSandboxIdentityFingerprint: mocks.inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority: mocks.inspectSandboxPolicyAuthority, })); @@ -37,6 +43,7 @@ vi.mock("../state/registry", async (importOriginal) => ({ ...(await importOriginal()), addCustomPolicy: mocks.addCustomPolicy, beginBaselineExclusionTransition: mocks.beginBaselineExclusionTransition, + compareAndSetSandboxPolicyCreationReceipt: mocks.compareAndSetSandboxPolicyCreationReceipt, getBaselineExclusions: mocks.getBaselineExclusions, getBaselineExclusionTransition: mocks.getBaselineExclusionTransition, getSandbox: mocks.getSandbox, @@ -54,6 +61,11 @@ import { import { PolicyAuthorityRefusalError } from "../adapters/openshell/policy-authority"; const SANDBOX = "authority-9833"; +const GATEWAY_PORT = 8080; +const LIFECYCLE_GENERATION = "00000000-0000-4000-8000-000000000001"; +const SANDBOX_IDENTITY = "a".repeat(64); +const INITIAL_POLICY_HASH = "policy-initial"; +const UPDATED_POLICY_HASH = "policy-updated"; const BASE_POLICY = `version: 1 network_policies: existing: @@ -87,37 +99,79 @@ function reportedErrors(): string { describe("PolicyMutationAuthority", () => { let sandbox: Record; + let livePolicyHash: string; + let liveBasePolicy: string; beforeEach(() => { for (const mock of Object.values(mocks)) mock.mockReset(); sandbox = { name: SANDBOX, gatewayName: "nemoclaw", + gatewayPort: GATEWAY_PORT, + lifecycleGeneration: LIFECYCLE_GENERATION, + lifecycleLiveIdentityFingerprint: SANDBOX_IDENTITY, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: "nemoclaw", + gatewayPort: GATEWAY_PORT, + sandboxName: SANDBOX, + lifecycleGeneration: LIFECYCLE_GENERATION, + sandboxIdentityFingerprint: SANDBOX_IDENTITY, + policyHash: INITIAL_POLICY_HASH, + policyVersion: 1, + }, policies: [], }; + livePolicyHash = INITIAL_POLICY_HASH; + liveBasePolicy = BASE_POLICY; mocks.getSandbox.mockImplementation(() => sandbox); mocks.getBaselineExclusions.mockReturnValue([]); mocks.getBaselineExclusionTransition.mockReturnValue(null); mocks.inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "nemoclaw-managed", + authority: "owner-unknown", effectivePolicy: {}, + policyIdentity: { hash: livePolicyHash, activeVersion: 1 }, }); + mocks.inspectSandboxPolicyAuthority.mockImplementation(() => ({ + authority: "owner-unknown", + effectivePolicy: {}, + policyIdentity: { hash: livePolicyHash, activeVersion: 1 }, + })); + mocks.inspectOpenShellSandboxIdentityFingerprint.mockReturnValue(SANDBOX_IDENTITY); + mocks.captureSandboxBasePolicy.mockImplementation(() => liveBasePolicy); mocks.resolveOpenshell.mockReturnValue("/usr/local/bin/openshell"); - mocks.runCapture.mockReturnValue(BASE_POLICY); - mocks.run.mockReturnValue({ status: 0 }); + mocks.runCapture.mockImplementation(() => liveBasePolicy); + mocks.run.mockImplementation((command: readonly string[]) => { + const policyFlag = command.indexOf("--policy"); + expect(policyFlag).toBeGreaterThanOrEqual(0); + liveBasePolicy = fs.readFileSync(command[policyFlag + 1] as string, "utf8"); + livePolicyHash = UPDATED_POLICY_HASH; + return { status: 0 }; + }); mocks.updateSandbox.mockImplementation((_name, updates) => { sandbox = { ...sandbox, ...updates }; return true; }); + mocks.compareAndSetSandboxPolicyCreationReceipt.mockImplementation( + (_name, expected, replacement) => { + expect(sandbox.policyCreationReceipt).toEqual(expected); + sandbox = { ...sandbox, policyCreationReceipt: replacement }; + return true; + }, + ); vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "warn").mockImplementation(() => undefined); }); - it("accepts an externally supplied custom preset without setting or attributing it (#9833)", () => { + it("verifies an externally supplied preset without recording authority (#9833)", () => { + sandbox = { ...sandbox, policyAuthority: "externally-managed" }; mocks.inspectSandboxPolicyAuthority.mockReturnValue({ authority: "externally-managed", effectivePolicy: { network_policies: { weather: WEATHER_POLICY } }, + policyIdentity: { hash: INITIAL_POLICY_HASH, activeVersion: 1 }, }); expect( @@ -129,37 +183,15 @@ describe("PolicyMutationAuthority", () => { expect(mocks.runCapture).not.toHaveBeenCalled(); expect(mocks.run).not.toHaveBeenCalled(); expect(mocks.addCustomPolicy).not.toHaveBeenCalled(); - expect(mocks.updateSandbox).toHaveBeenCalledTimes(1); - expect(mocks.updateSandbox).toHaveBeenCalledWith(SANDBOX, { - policyAuthority: "externally-managed", - }); - }); - - it("records external authority before refusing a missing preset requirement (#9833)", () => { - mocks.inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "externally-managed", - effectivePolicy: { network_policies: {} }, - }); - - expect(applyPresetContent(SANDBOX, "weather", WEATHER_PRESET)).toBe(false); - - expect(reportedErrors()).toContain("external policy authority"); - expect(reportedErrors()).toContain('"weather"'); - expect(reportedErrors()).not.toContain("wttr.in"); - expect(reportedErrors()).not.toContain("network_policies:"); - expect(mocks.runCapture).not.toHaveBeenCalled(); - expect(mocks.run).not.toHaveBeenCalled(); - expect(mocks.updateSandbox).toHaveBeenCalledOnce(); - expect(sandbox).toEqual( - expect.objectContaining({ policyAuthority: "externally-managed", policies: [] }), - ); + expect(mocks.updateSandbox).not.toHaveBeenCalled(); }); - it("reads external recovery authority without replacing the durable owner (#9833)", () => { - sandbox = { ...sandbox, policyAuthority: "nemoclaw-managed" }; + it("reads verified global authority without changing the registry (#9833)", () => { + sandbox = { ...sandbox, policyAuthority: "externally-managed" }; mocks.inspectSandboxPolicyAuthority.mockReturnValue({ authority: "externally-managed", effectivePolicy: { network_policies: { weather: WEATHER_POLICY } }, + policyIdentity: { hash: INITIAL_POLICY_HASH, activeVersion: 1 }, }); expect(inspectPolicyRecoveryAuthority(SANDBOX, "verify Shields recovery")).toMatchObject({ @@ -170,73 +202,119 @@ describe("PolicyMutationAuthority", () => { expect(mocks.updateSandbox).not.toHaveBeenCalled(); }); - it("rechecks authority before policy set and refuses an ownership change (#9833)", () => { - mocks.inspectSandboxPolicyAuthority - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "externally-managed", effectivePolicy: {} }); + it("rotates the receipt after a matching policy mutation (#9833)", () => { + expect( + applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { + custom: { sourcePath: "/tmp/weather.yaml" }, + }), + ).toBe(true); + + expect(mocks.run).toHaveBeenCalledOnce(); + expect(mocks.captureSandboxBasePolicy).toHaveBeenCalledWith(SANDBOX, "nemoclaw"); + expect(mocks.compareAndSetSandboxPolicyCreationReceipt).toHaveBeenCalledOnce(); + expect(sandbox.policyCreationReceipt).toEqual( + expect.objectContaining({ policyHash: UPDATED_POLICY_HASH, policyVersion: 1 }), + ); + expect(mocks.addCustomPolicy).toHaveBeenCalledOnce(); + expect(console.log).toHaveBeenCalledWith(" Applied preset: weather"); + }); + + it.each([ + ["missing", undefined], + ["pending", { schemaVersion: 1, origin: "pending-sandbox-create" }], + ["malformed", { schemaVersion: 1, origin: "sandbox-create" }], + ])("refuses a sandbox policy with a %s receipt before mutation (#9833)", (_label, receipt) => { + sandbox = { ...sandbox, policyCreationReceipt: receipt }; + + expect(applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { nonFatal: true })).toBe(false); + expect(mocks.runCapture).not.toHaveBeenCalled(); + expect(mocks.run).not.toHaveBeenCalled(); + expect(mocks.updateSandbox).not.toHaveBeenCalled(); + expect(reportedErrors()).toContain("policy creation receipt"); + }); + + it("refuses a replacement sandbox before mutation (#9833)", () => { + mocks.inspectOpenShellSandboxIdentityFingerprint.mockReturnValue("b".repeat(64)); expect(applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { nonFatal: true })).toBe(false); - expect(mocks.inspectSandboxPolicyAuthority).toHaveBeenCalledTimes(2); - expect(mocks.runCapture).toHaveBeenCalledTimes(1); + expect(mocks.runCapture).not.toHaveBeenCalled(); expect(mocks.run).not.toHaveBeenCalled(); - expect(mocks.updateSandbox).toHaveBeenCalledOnce(); - expect(sandbox).toEqual( - expect.objectContaining({ policyAuthority: "nemoclaw-managed", policies: [] }), - ); - expect(reportedErrors()).toContain("policy authority changed"); - expect(reportedErrors()).toContain("external policy authority"); + expect(reportedErrors()).toContain("live sandbox identity"); }); - it("withholds single-preset success when the final registry check changes authority (#9833)", () => { - sandbox = { ...sandbox, policyAuthority: "nemoclaw-managed" }; - mocks.inspectSandboxPolicyAuthority - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "externally-managed", effectivePolicy: {} }); + it.each([ + ["gateway", { gatewayName: "replacement-gateway" }], + ["lifecycle", { lifecycleGeneration: "00000000-0000-4000-8000-000000000099" }], + ])("refuses a receipt bound to a different %s (#9833)", (_label, replacement) => { + sandbox = { + ...sandbox, + policyCreationReceipt: { + ...(sandbox.policyCreationReceipt as Record), + ...replacement, + }, + }; - expect( - applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { - custom: { sourcePath: "/tmp/weather.yaml" }, - }), - ).toBe(false); + expect(applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { nonFatal: true })).toBe(false); - expect(mocks.run).toHaveBeenCalledOnce(); - expect(mocks.addCustomPolicy).not.toHaveBeenCalled(); - expect(mocks.updateSandbox).not.toHaveBeenCalled(); - expect(console.log).not.toHaveBeenCalledWith(" Applied preset: weather"); - expect(reportedErrors()).toContain("policy authority changed"); + expect(mocks.runCapture).not.toHaveBeenCalled(); + expect(mocks.run).not.toHaveBeenCalled(); + expect(reportedErrors()).toContain("policy creation receipt"); }); - it("refuses external removal before reading or changing policy state (#9833)", () => { - sandbox = { ...sandbox, policyAuthority: "externally-managed", policies: ["weather"] }; - mocks.inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "externally-managed", - effectivePolicy: { network_policies: { weather: WEATHER_POLICY } }, + it("refuses a stale policy identity before mutation (#9833)", () => { + livePolicyHash = "policy-external-change"; + + expect(applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { nonFatal: true })).toBe(false); + + expect(mocks.runCapture).not.toHaveBeenCalled(); + expect(mocks.run).not.toHaveBeenCalled(); + expect(reportedErrors()).toContain("does not match the live sandbox policy"); + }); + + it("refuses a registry receipt that changes during live verification (#9833)", () => { + mocks.getSandbox.mockReturnValueOnce(sandbox).mockReturnValueOnce({ + ...sandbox, + policyCreationReceipt: { + ...(sandbox.policyCreationReceipt as Record), + policyHash: "concurrent-registry-change", + }, }); - expect(removePreset(SANDBOX, "weather", { nonFatal: true })).toBe(false); + expect(applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { nonFatal: true })).toBe(false); expect(mocks.runCapture).not.toHaveBeenCalled(); expect(mocks.run).not.toHaveBeenCalled(); - expect(mocks.updateSandbox).not.toHaveBeenCalled(); - expect(reportedErrors()).toContain("external policy authority"); + expect(reportedErrors()).toContain("changed during live verification"); }); - it("refuses external baseline and permissive changes before side effects (#9833)", () => { + it("reports an incomplete update when receipt rotation fails (#9833)", () => { + mocks.compareAndSetSandboxPolicyCreationReceipt.mockReturnValue(false); + + expect(applyPresetContent(SANDBOX, "weather", WEATHER_PRESET, { nonFatal: true })).toBe(false); + + expect(mocks.run).toHaveBeenCalledOnce(); + expect(mocks.addCustomPolicy).not.toHaveBeenCalled(); + expect(sandbox.policyCreationReceipt).toEqual( + expect.objectContaining({ policyHash: INITIAL_POLICY_HASH }), + ); + expect(reportedErrors()).toContain("policy update is incomplete"); + }); + + it("refuses external and owner-unknown mutations before side effects (#9833)", () => { sandbox = { ...sandbox, policyAuthority: "externally-managed" }; mocks.inspectSandboxPolicyAuthority.mockReturnValue({ authority: "externally-managed", effectivePolicy: {}, + policyIdentity: { hash: INITIAL_POLICY_HASH, activeVersion: 1 }, }); + expect(removePreset(SANDBOX, "weather", { nonFatal: true })).toBe(false); expect(excludeBaselineEntry(SANDBOX, "existing", "reviewed-digest", { nonFatal: true })).toBe( false, ); expect(restoreBaselineEntry(SANDBOX, "existing", { nonFatal: true })).toBe(false); - expect(() => applyPermissivePolicy(SANDBOX)).toThrow(/external policy authority/); + expect(() => applyPermissivePolicy(SANDBOX)).toThrow(PolicyAuthorityRefusalError); expect(mocks.getBaselineExclusionTransition).not.toHaveBeenCalled(); expect(mocks.beginBaselineExclusionTransition).not.toHaveBeenCalled(); @@ -244,32 +322,4 @@ describe("PolicyMutationAuthority", () => { expect(mocks.run).not.toHaveBeenCalled(); expect(mocks.updateSandbox).not.toHaveBeenCalled(); }); - - it("throws when permissive policy authority changes after the policy set (#9833)", () => { - sandbox = { ...sandbox, policyAuthority: "nemoclaw-managed" }; - mocks.inspectSandboxPolicyAuthority - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "externally-managed", effectivePolicy: {} }); - - expect(() => applyPermissivePolicy(SANDBOX)).toThrow(/policy authority changed/u); - - expect(mocks.run).toHaveBeenCalledOnce(); - expect(console.log).not.toHaveBeenCalledWith(" Applied permissive policy."); - }); - - it("throws a typed refusal when permissive policy authority changes before submission (#9833)", () => { - sandbox = { ...sandbox, policyAuthority: "nemoclaw-managed" }; - mocks.inspectSandboxPolicyAuthority - .mockReturnValueOnce({ authority: "nemoclaw-managed", effectivePolicy: {} }) - .mockReturnValueOnce({ authority: "externally-managed", effectivePolicy: {} }); - - expect(() => applyPermissivePolicy(SANDBOX)).toThrow(PolicyAuthorityRefusalError); - - expect(mocks.inspectSandboxPolicyAuthority).toHaveBeenCalledTimes(2); - expect(mocks.run).not.toHaveBeenCalled(); - expect(console.log).not.toHaveBeenCalledWith(" Applied permissive policy."); - expect(reportedErrors()).toBe(""); - }); }); diff --git a/src/lib/readiness/gateway-production.ts b/src/lib/readiness/gateway-production.ts index afb91100f6b..78f0eff097d 100644 --- a/src/lib/readiness/gateway-production.ts +++ b/src/lib/readiness/gateway-production.ts @@ -6,6 +6,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { + classifyManagedGatewayEndpointBinding, + type ManagedGatewayEndpointBinding, type GatewayReuseState, type GatewayVersionCompatibility, type GatewayVersionSource, @@ -17,6 +19,7 @@ import { parseVersionFromText, stripAnsi, } from "../adapters/openshell/gateway-drift"; +export { classifyManagedGatewayEndpointBinding }; import { cliName as resolveCliName } from "../onboard/branding"; import { getConfiguredGatewayPort, @@ -305,39 +308,6 @@ function getTrustedHostProcessGatewayRuntime( }; } -type ManagedGatewayEndpointBinding = "match" | "mismatch" | "not-applicable" | "unknown"; - -export function classifyManagedGatewayEndpointBinding( - outputs: readonly string[], - expectedGatewayPort: number, -): Exclude { - let observedEndpoint = false; - for (const output of outputs) { - for (const match of stripAnsi(output).matchAll(/^\s*(?:Gateway endpoint|Server):(.*)$/gm)) { - observedEndpoint = true; - const endpointText = match[1]?.trim() ?? ""; - if (!endpointText || /\s/u.test(endpointText)) return "mismatch"; - try { - const endpoint = new URL(endpointText); - const localProtocol = endpoint.protocol === "https:" || endpoint.protocol === "http:"; - const localHost = - endpoint.hostname === "127.0.0.1" || - endpoint.hostname === "localhost" || - endpoint.hostname === "[::1]"; - const endpointPort = - endpoint.port || - (endpoint.protocol === "https:" ? "443" : endpoint.protocol === "http:" ? "80" : ""); - if (!localProtocol || !localHost || endpointPort !== String(expectedGatewayPort)) { - return "mismatch"; - } - } catch { - return "mismatch"; - } - } - } - return observedEndpoint ? "match" : "unknown"; -} - function observeReuseState( gatewayName: string, gatewayPort: number, diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 33923017e3b..d426be6ff51 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -127,6 +127,34 @@ describe("shields command flow", () => { ); }); + it("restores restrictive policy when receipt finalization fails (#9833)", () => { + const harness = createHarness(); + harness.policyReceiptFinalizeSpy.mockImplementationOnce(() => { + throw new Error("policy receipt persistence failed"); + }); + + expect(() => + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "receipt finalization coverage", + throwOnError: true, + }), + ).toThrow("policy receipt persistence failed"); + + expect(harness.policySetBodies.map((body) => YAML.parse(body).network_policies)).toEqual([ + {}, + { test: {} }, + ]); + expect(harness.getOpenClawPosture()).toBe("mutable"); + expect(harness.auditSpy).not.toHaveBeenCalled(); + const stateFiles = fs.readdirSync(path.join(tmpDir, ".nemoclaw", "state")); + expect( + stateFiles.filter((name) => + /^(shields-openclaw|shields-timer-|shields-transition-openclaw)/u.test(name), + ), + ).toEqual([]); + }); + it("shieldsDown preserves an exact managed MCP policy and records its snapshot key (#7952)", { timeout: 15_000, }, () => { diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index e652ca86629..baa421db862 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -655,7 +655,7 @@ describe("shields — unit logic", () => { deadlineAuthoritative: true, expiredTimerRecovery: true, }), - ).toThrow(/policy authority inspection failed/i); + ).toThrow(/policy authority is unavailable/i); expect(createTempDirectory).not.toHaveBeenCalled(); expect(run).not.toHaveBeenCalled(); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index f4031777dc4..1747f419645 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -41,6 +41,7 @@ const { const { buildPolicyGetCommand, buildPolicySetCommand, + finalizePolicyMutationReceipt, parseCurrentPolicy, resolvePermissivePolicyPath, assertNemoClawManagedPolicy, @@ -4672,6 +4673,11 @@ function applyShieldsPolicySnapshot( }, ); rejectFinalShieldsPolicySetResult(result, "restore the Shields policy snapshot"); + finalizePolicyMutationReceipt( + sandboxName, + fs.readFileSync(snapshotPath, "utf-8"), + policyAuthority, + ); return result; } else { assertLegacyMcpPolicyRestoreSafe( @@ -4766,6 +4772,11 @@ function applyShieldsPolicySnapshot( }, ); rejectFinalShieldsPolicySetResult(result, "restore the Shields policy snapshot"); + finalizePolicyMutationReceipt( + sandboxName, + fs.readFileSync(runtimePolicyPath, "utf-8"), + policyAuthority, + ); return managedMcpOmissions.length > 0 ? { ...result, managedMcpOmissions } : result; } finally { if (runtimePolicyIsTemp) { @@ -5269,6 +5280,7 @@ function applyRecoveredShieldsDownForwardPolicy( if (result.status !== 0) { throw new Error("Interrupted Shields down forward policy could not be reapplied"); } + finalizePolicyMutationReceipt(sandboxName, fs.readFileSync(policyPath, "utf-8"), policyAuthority); requireShieldsDownForwardPolicy(completion.authority); assertRecoveredShieldsDownAuthority(sandboxName, completion, completion.authority.phase); } @@ -5910,6 +5922,7 @@ function shieldsDownWithoutHostLock( } console.log(` Applying ${policyName} policy...`); + const appliedPolicyDocument = fs.readFileSync(policyPathForApply, "utf-8"); let policySetResult: ReturnType; try { assertShieldsPolicyMutationAuthority( @@ -5997,6 +6010,7 @@ function shieldsDownWithoutHostLock( console.log(` Unlocking ${target.agentName} config (${target.configPath})...`); let inferenceRouteConvergenceFailed = false; try { + finalizePolicyMutationReceipt(sandboxName, appliedPolicyDocument, policyAuthority); if (transition && timerAuthority) { assertFreshShieldsDownAuthority(sandboxName, timerAuthority, transition, "preparing"); } diff --git a/src/lib/shields/policy-transition.test.ts b/src/lib/shields/policy-transition.test.ts index 5f1904c63ed..3474819482a 100644 --- a/src/lib/shields/policy-transition.test.ts +++ b/src/lib/shields/policy-transition.test.ts @@ -47,6 +47,7 @@ function mockManagedPolicyAuthority(sandboxName: string): void { vi.spyOn(policyAuthority, "inspectSandboxPolicyAuthority").mockReturnValue({ authority: "nemoclaw-managed", effectivePolicy: { version: 1, network_policies: {} }, + policyIdentity: { hash: "sha256:managed", activeVersion: 1 }, }); const receipt = { authority: "nemoclaw-managed" as const, @@ -55,10 +56,13 @@ function mockManagedPolicyAuthority(sandboxName: string): void { inspection: { authority: "nemoclaw-managed" as const, effectivePolicy: { version: 1, network_policies: {} }, + policyIdentity: { hash: "sha256:managed", activeVersion: 1 }, }, }; vi.spyOn(policy, "inspectPolicyMutationAuthority").mockReturnValue(receipt); + vi.spyOn(policy, "inspectPolicyRecoveryAuthority").mockReturnValue(receipt); vi.spyOn(policy, "recheckPolicyMutationAuthority").mockReturnValue(receipt); + vi.spyOn(policy, "finalizePolicyMutationReceipt").mockImplementation(() => undefined); } describe("shields policy transition", () => { @@ -153,6 +157,7 @@ describe("shields policy transition", () => { inspection: { authority: "externally-managed", effectivePolicy: { version: 1, network_policies: {} }, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, }, }); @@ -168,10 +173,11 @@ describe("shields policy transition", () => { inspection: { authority: "externally-managed" as const, effectivePolicy: { version: 1, network_policies: { test: {} } }, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, }, }; vi.mocked(policy.inspectPolicyMutationAuthority).mockReturnValue(matchingExternalAuthority); - vi.spyOn(policy, "inspectPolicyRecoveryAuthority") + vi.mocked(policy.inspectPolicyRecoveryAuthority) .mockReturnValueOnce(matchingExternalAuthority) .mockReturnValue({ ...matchingExternalAuthority, @@ -274,6 +280,11 @@ describe("shields down policy rejection", () => { .filter((command) => Array.isArray(command) && command.includes("policy")); expect(policyCommands.length).toBeGreaterThan(0); expect(policyCommands.every((command) => command.includes(gatewayName))).toBe(true); + expect(harness.policyReceiptFinalizeSpy).toHaveBeenCalledWith( + "openclaw", + expect.stringContaining("network_policies"), + expect.objectContaining({ gatewayName }), + ); }); it("keeps `shields status` at `UP` when OpenShell rejects the permissive policy (#8198)", () => { @@ -1010,6 +1021,7 @@ describe("managed MCP policy deadline restoration (#7952)", () => { vi.spyOn(policyAuthority, "inspectSandboxPolicyAuthority").mockReturnValue({ authority: "nemoclaw-managed", effectivePolicy: { version: 1, network_policies: {} }, + policyIdentity: { hash: "sha256:managed", activeVersion: 1 }, }); const authorityReceipt = { authority: "nemoclaw-managed" as const, @@ -1018,10 +1030,12 @@ describe("managed MCP policy deadline restoration (#7952)", () => { inspection: { authority: "nemoclaw-managed" as const, effectivePolicy: { version: 1, network_policies: {} }, + policyIdentity: { hash: "sha256:managed", activeVersion: 1 }, }, }; vi.spyOn(policy, "inspectPolicyMutationAuthority").mockReturnValue(authorityReceipt); vi.spyOn(policy, "recheckPolicyMutationAuthority").mockReturnValue(authorityReceipt); + vi.spyOn(policy, "finalizePolicyMutationReceipt").mockImplementation(() => undefined); const shields = requireSource(SHIELDS_MODULE) as typeof import("./index.js"); return { applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot, policySetBodies }; diff --git a/src/lib/state/config-io.test.ts b/src/lib/state/config-io.test.ts index bb91c32e335..1cc1b549917 100644 --- a/src/lib/state/config-io.test.ts +++ b/src/lib/state/config-io.test.ts @@ -28,6 +28,7 @@ function writeFileWithMode(filePath: string, contents: string, mode: number) { } afterEach(() => { + vi.restoreAllMocks(); for (const dir of tmpDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -221,6 +222,121 @@ describe("config-io", () => { expect(fs.readdirSync(dir).filter((name) => name.includes(".tmp."))).toEqual([]); }); + it("synchronizes replacement bytes before rename and the directory after rename", () => { + const dir = makeTempDir(); + const file = path.join(dir, "config.json"); + fs.writeFileSync(file, JSON.stringify({ durable: "prior" }), { mode: 0o600 }); + const openSpy = vi.spyOn(fs, "openSync"); + const syncSpy = vi.spyOn(fs, "fsyncSync"); + const linkSpy = vi.spyOn(fs, "linkSync"); + const renameSpy = vi.spyOn(fs, "renameSync"); + + writeConfigFile(file, { durable: true }); + + expect(openSpy).toHaveBeenCalledWith(`${file}.tmp.${String(process.pid)}`, "r"); + expect(openSpy).toHaveBeenCalledWith(dir, fs.constants.O_RDONLY); + expect(syncSpy).toHaveBeenCalledTimes(3); + expect(syncSpy.mock.invocationCallOrder[0]).toBeLessThan( + linkSpy.mock.invocationCallOrder[0]!, + ); + expect(linkSpy.mock.invocationCallOrder[0]).toBeLessThan( + syncSpy.mock.invocationCallOrder[1]!, + ); + expect(syncSpy.mock.invocationCallOrder[1]).toBeLessThan( + renameSpy.mock.invocationCallOrder[0]!, + ); + expect(renameSpy.mock.invocationCallOrder[0]).toBeLessThan( + syncSpy.mock.invocationCallOrder[2]!, + ); + }); + + it("replaces a stale rollback backup from a reused process ID", () => { + const dir = makeTempDir(); + const file = path.join(dir, "config.json"); + const backupFile = `${file}.rollback.${String(process.pid)}`; + fs.writeFileSync(file, JSON.stringify({ durable: "prior" }), { mode: 0o600 }); + fs.writeFileSync(backupFile, JSON.stringify({ durable: "stale" }), { mode: 0o600 }); + + writeConfigFile(file, { durable: "replacement" }); + + expect(readConfigFile(file, null)).toEqual({ durable: "replacement" }); + expect(fs.existsSync(backupFile)).toBe(false); + }); + + it("preserves the prior config when temporary-file synchronization fails", () => { + const dir = makeTempDir(); + const file = path.join(dir, "config.json"); + fs.writeFileSync(file, JSON.stringify({ durable: "prior" }), { mode: 0o600 }); + vi.spyOn(fs, "fsyncSync").mockImplementationOnce(() => { + throw Object.assign(new Error("storage synchronization failed"), { code: "EIO" }); + }); + + expect(() => writeConfigFile(file, { durable: "replacement" })).toThrow( + /storage synchronization failed/, + ); + expect(readConfigFile(file, null)).toEqual({ durable: "prior" }); + expect(fs.readdirSync(dir).filter((name) => name.includes(".tmp."))).toEqual([]); + }); + + it("removes an initial config when its directory synchronization fails", () => { + const dir = makeTempDir(); + const file = path.join(dir, "config.json"); + const syncSpy = vi.spyOn(fs, "fsyncSync"); + syncSpy + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw Object.assign(new Error("directory synchronization failed"), { code: "EIO" }); + }); + + expect(() => writeConfigFile(file, { durable: true })).toThrow( + /directory synchronization failed/, + ); + expect(syncSpy).toHaveBeenCalledTimes(3); + expect(readConfigFile(file, null)).toBeNull(); + }); + + it("restores the prior config when replacement directory synchronization fails", () => { + const dir = makeTempDir(); + const file = path.join(dir, "config.json"); + fs.writeFileSync(file, JSON.stringify({ durable: "prior" }), { mode: 0o600 }); + const syncSpy = vi.spyOn(fs, "fsyncSync"); + syncSpy + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw Object.assign(new Error("directory synchronization failed"), { code: "EIO" }); + }); + + expect(() => writeConfigFile(file, { durable: "replacement" })).toThrow( + /directory synchronization failed/, + ); + expect(readConfigFile(file, null)).toEqual({ durable: "prior" }); + expect(fs.readdirSync(dir).filter((name) => name.includes(".rollback."))).toEqual([]); + }); + + it("retains the prior config backup when durability rollback fails", () => { + const dir = makeTempDir(); + const file = path.join(dir, "config.json"); + const backupFile = `${file}.rollback.${String(process.pid)}`; + fs.writeFileSync(file, JSON.stringify({ durable: "prior" }), { mode: 0o600 }); + const syncSpy = vi.spyOn(fs, "fsyncSync"); + syncSpy + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw Object.assign(new Error("directory synchronization failed"), { code: "EIO" }); + }); + const renameSync = fs.renameSync; + vi.spyOn(fs, "renameSync") + .mockImplementationOnce(renameSync) + .mockImplementationOnce(() => { + throw Object.assign(new Error("rollback rename failed"), { code: "EIO" }); + }); + + expect(() => writeConfigFile(file, { durable: "replacement" })).toThrow(AggregateError); + expect(readConfigFile(backupFile, null)).toEqual({ durable: "prior" }); + }); + it("cleans up temp files when rename fails", () => { const dir = makeTempDir(); const file = path.join(dir, "config.json"); diff --git a/src/lib/state/config-io.ts b/src/lib/state/config-io.ts index 2a947d9122d..522ef0bbe21 100644 --- a/src/lib/state/config-io.ts +++ b/src/lib/state/config-io.ts @@ -60,6 +60,15 @@ function cleanupTempFile(filePath: string): void { } } +function fsyncDirectory(dirPath: string): void { + const directoryDescriptor = fs.openSync(dirPath, fs.constants.O_RDONLY); + try { + fs.fsyncSync(directoryDescriptor); + } finally { + fs.closeSync(directoryDescriptor); + } +} + function buildRemediation(): string { const home = process.env.HOME ?? os.homedir(); const nemoclawDir = nemoclawStateRoot(home, GATEWAY_PORT); @@ -368,11 +377,52 @@ export function writeConfigFile(filePath: string, data: SerializableConfig): voi ensureConfigDir(dirPath); const tmpFile = `${filePath}.tmp.${String(process.pid)}`; + const backupFile = `${filePath}.rollback.${String(process.pid)}`; + let backupCreated = false; + let replacementRenamed = false; try { fs.writeFileSync(tmpFile, JSON.stringify(data, null, 2), { mode: 0o600 }); + const fileDescriptor = fs.openSync(tmpFile, "r"); + try { + fs.fsyncSync(fileDescriptor); + } finally { + fs.closeSync(fileDescriptor); + } + fs.rmSync(backupFile, { force: true }); + try { + fs.linkSync(filePath, backupFile); + backupCreated = true; + fsyncDirectory(dirPath); + } catch (error) { + if (!isErrnoException(error) || error.code !== "ENOENT") throw error; + } fs.renameSync(tmpFile, filePath); + replacementRenamed = true; + try { + fsyncDirectory(dirPath); + } catch (commitError) { + try { + if (backupCreated) { + fs.renameSync(backupFile, filePath); + backupCreated = false; + } else { + fs.rmSync(filePath, { force: true }); + } + fsyncDirectory(dirPath); + } catch (rollbackError) { + throw new AggregateError( + [commitError, rollbackError], + `Could not make config replacement durable or restore '${filePath}'`, + ); + } + throw commitError; + } + if (backupCreated) { + cleanupTempFile(backupFile); + } } catch (error) { cleanupTempFile(tmpFile); + if (backupCreated && !replacementRenamed) cleanupTempFile(backupFile); const errnoError = error instanceof Error ? error : null; if (isPermissionError(errnoError)) { throw new ConfigPermissionError( diff --git a/src/lib/state/registry-normalization.test.ts b/src/lib/state/registry-normalization.test.ts index 3b70561b926..7d5b813d5e8 100644 --- a/src/lib/state/registry-normalization.test.ts +++ b/src/lib/state/registry-normalization.test.ts @@ -91,6 +91,32 @@ describe("sandbox registry normalization", () => { }; }; + const createManagedPolicyEntry = (name: string, policyVersion = 1) => { + const gatewayName = "nemoclaw"; + const gatewayPort = 8080; + const lifecycleGeneration = "123e4567-e89b-42d3-a456-426614174983"; + const lifecycleLiveIdentityFingerprint = "d".repeat(64); + return { + name, + gatewayName, + gatewayPort, + lifecycleGeneration, + lifecycleLiveIdentityFingerprint, + policyAuthority: "nemoclaw-managed" as const, + policyCreationReceipt: { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName, + gatewayPort, + sandboxName: name, + lifecycleGeneration, + sandboxIdentityFingerprint: lifecycleLiveIdentityFingerprint, + policyHash: `sha256:policy-${String(policyVersion)}`, + policyVersion, + }, + }; + }; + it.each([null, [], 42, "invalid"])( "treats a non-object top-level registry document as empty: %j", async (document) => { @@ -311,10 +337,15 @@ describe("sandbox registry normalization", () => { expect(entry).not.toHaveProperty("policyTier"); } - it("round-trips known policy authority while leaving legacy authority unknown (#9833)", async () => { + it("round-trips receipt-bound authority while removing a legacy managed claim (#9833)", async () => { const registry = await loadRegistryWith({ legacy: { name: "legacy" }, - managed: { name: "managed", policyAuthority: "nemoclaw-managed" }, + legacyManaged: { + name: "legacyManaged", + policyAuthority: "nemoclaw-managed", + policies: ["weather"], + }, + managed: createManagedPolicyEntry("managed"), external: { name: "external", policyAuthority: "externally-managed" }, }); registry.save(registry.load()); @@ -322,21 +353,122 @@ describe("sandbox registry normalization", () => { fs.readFileSync(path.join(process.env.HOME!, ".nemoclaw", "sandboxes.json"), "utf8"), ) as { sandboxes: Record> }; expect(registry.getSandbox("legacy")?.policyAuthority).toBeUndefined(); + expect(registry.getSandbox("legacyManaged")).toMatchObject({ policies: ["weather"] }); + expect(registry.getSandbox("legacyManaged")).not.toHaveProperty("policyAuthority"); + expect(registry.getSandbox("legacyManaged")).not.toHaveProperty("policyCreationReceipt"); expect(registry.getSandbox("managed")?.policyAuthority).toBe("nemoclaw-managed"); + expect(registry.getSandbox("managed")?.policyCreationReceipt).toEqual( + createManagedPolicyEntry("managed").policyCreationReceipt, + ); expect(registry.getSandbox("external")?.policyAuthority).toBe("externally-managed"); expect(persisted.sandboxes.legacy).not.toHaveProperty("policyAuthority"); + expect(persisted.sandboxes.legacyManaged).not.toHaveProperty("policyAuthority"); + expect(persisted.sandboxes.legacyManaged).not.toHaveProperty("policyCreationReceipt"); expect(persisted.sandboxes.managed?.policyAuthority).toBe("nemoclaw-managed"); expect(persisted.sandboxes.external?.policyAuthority).toBe("externally-managed"); }); + it("round-trips only complete non-authorizing create checkpoints (#9833)", async () => { + const managed = createManagedPolicyEntry("managed-pending"); + const managedCheckpoint = { + schemaVersion: 1 as const, + state: "verified-create" as const, + policyAuthority: "nemoclaw-managed" as const, + observedPolicyAuthority: "owner-unknown" as const, + gatewayName: managed.gatewayName, + gatewayPort: managed.gatewayPort, + sandboxName: managed.name, + lifecycleGeneration: managed.lifecycleGeneration, + sandboxIdentityFingerprint: managed.lifecycleLiveIdentityFingerprint, + route: "none" as const, + policyHash: managed.policyCreationReceipt.policyHash, + policyVersion: managed.policyCreationReceipt.policyVersion, + policyCreationReceipt: managed.policyCreationReceipt, + }; + const externalCheckpoint = { + ...managedCheckpoint, + policyAuthority: "externally-managed" as const, + observedPolicyAuthority: "externally-managed" as const, + sandboxName: "external-pending", + policyHash: "sha256:external", + policyCreationReceipt: undefined, + }; + const registry = await loadRegistryWith({ + "managed-pending": { + name: "managed-pending", + pendingRouteReservation: true, + reservationSessionId: "managed-session", + gatewayName: managed.gatewayName, + gatewayPort: managed.gatewayPort, + lifecycleGeneration: managed.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: managed.lifecycleLiveIdentityFingerprint, + pendingPolicyVerification: managedCheckpoint, + }, + "external-pending": { + name: "external-pending", + pendingRouteReservation: true, + reservationSessionId: "external-session", + gatewayName: managed.gatewayName, + gatewayPort: managed.gatewayPort, + lifecycleGeneration: managed.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: managed.lifecycleLiveIdentityFingerprint, + pendingPolicyVerification: externalCheckpoint, + }, + }); + + expect(registry.getSandbox("managed-pending")?.pendingPolicyVerification).toEqual( + managedCheckpoint, + ); + expect(registry.getSandbox("external-pending")?.pendingPolicyVerification).toEqual( + externalCheckpoint, + ); + expect(registry.getDefault()).toBeNull(); + }); + + it.each([ + ["ownerless", {}], + ["authorizing", { reservationSessionId: "session", policyAuthority: "externally-managed" }], + ["wrong-lifecycle", { reservationSessionId: "session", lifecycleGeneration: "changed" }], + ])("rejects a %s persisted create checkpoint (#9833)", async (_label, overrides) => { + const managed = createManagedPolicyEntry("pending"); + const checkpoint = { + schemaVersion: 1 as const, + state: "verified-create" as const, + policyAuthority: "nemoclaw-managed" as const, + observedPolicyAuthority: "owner-unknown" as const, + gatewayName: managed.gatewayName, + gatewayPort: managed.gatewayPort, + sandboxName: managed.name, + lifecycleGeneration: managed.lifecycleGeneration, + sandboxIdentityFingerprint: managed.lifecycleLiveIdentityFingerprint, + route: "none" as const, + policyHash: managed.policyCreationReceipt.policyHash, + policyVersion: managed.policyCreationReceipt.policyVersion, + policyCreationReceipt: managed.policyCreationReceipt, + }; + const registry = await loadRegistryWith({ + pending: { + name: "pending", + pendingRouteReservation: true, + gatewayName: managed.gatewayName, + gatewayPort: managed.gatewayPort, + lifecycleGeneration: managed.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: managed.lifecycleLiveIdentityFingerprint, + pendingPolicyVerification: checkpoint, + ...overrides, + }, + }); + + expect(() => registry.getSandbox("pending")).toThrow(/pending policy verification/u); + }); + it("clears NemoClaw policy attribution from externally managed rows (#9833)", async () => { const attribution = createPolicyAttribution(); const registry = await loadRegistryWith({ legacy: { name: "legacy", ...attribution, policyTier: "strict" }, managed: { - name: "managed", + ...createManagedPolicyEntry("managed"), ...attribution, - policyAuthority: "nemoclaw-managed", policyTier: "strict", }, external: { @@ -363,24 +495,163 @@ describe("sandbox registry normalization", () => { }, ); - it("backfills policy authority once without allowing later changes or removal (#9833)", async () => { - const registry = await loadRegistryWith({ legacy: { name: "legacy" } }); + it("does not backfill NemoClaw ownership and leaves external authority available (#9833)", async () => { + const registry = await loadRegistryWith({ + legacy: { name: "legacy", policyAuthority: "nemoclaw-managed" }, + }); expect(() => registry.updateSandbox("legacy", { policyAuthority: "global" as never })).toThrow( /invalid policy authority/i, ); - expect(registry.updateSandbox("legacy", { policyAuthority: "nemoclaw-managed" })).toBe(true); - expect(() => - registry.updateSandbox("legacy", { policyAuthority: "externally-managed" }), - ).toThrow(/policy authority changed/u); - expect(() => registry.updateSandbox("legacy", { policyAuthority: undefined })).toThrow( + expect(() => registry.updateSandbox("legacy", { policyAuthority: "nemoclaw-managed" })).toThrow( + /outside completed sandbox registration/u, + ); + expect(registry.updateSandbox("legacy", { policyAuthority: "externally-managed" })).toBe(true); + expect(registry.getSandbox("legacy")?.policyAuthority).toBe("externally-managed"); + expect(() => registry.registerSandbox(createManagedPolicyEntry("legacy"))).toThrow( /policy authority changed/u, ); - expect(registry.getSandbox("legacy")?.policyAuthority).toBe("nemoclaw-managed"); + }); + + it("publishes and rotates only an exact completed policy creation receipt (#9833)", async () => { + const registry = await loadRegistryWith({}); + const managed = createManagedPolicyEntry("managed"); + const registered = registry.registerSandbox(managed); + const replacement = createManagedPolicyEntry("managed", 2).policyCreationReceipt; + expect(registered.policyCreationReceipt).toEqual(managed.policyCreationReceipt); + expect(() => registry.updateSandbox("managed", { policyCreationReceipt: replacement })).toThrow( + /outside the receipt rotation transaction/u, + ); + expect( + registry.compareAndSetSandboxPolicyCreationReceipt( + "managed", + { ...managed.policyCreationReceipt, policyVersion: 9 }, + replacement, + ), + ).toBe(false); + expect( + registry.compareAndSetSandboxPolicyCreationReceipt( + "managed", + managed.policyCreationReceipt, + replacement, + ), + ).toBe(true); + expect(registry.getSandbox("managed")?.policyCreationReceipt).toEqual(replacement); expect(() => - registry.registerSandbox({ name: "legacy", policyAuthority: "externally-managed" }), - ).toThrow(/policy authority changed/u); + registry.compareAndSetSandboxPolicyCreationReceipt("managed", replacement, { + ...replacement, + lifecycleGeneration: "223e4567-e89b-42d3-a456-426614174983", + }), + ).toThrow(/gateway or sandbox identity/u); + }); + + it("rejects partial, mismatched, and pending policy creation receipts (#9833)", async () => { + const managed = createManagedPolicyEntry("managed"); + const malformedRegistry = await loadRegistryWith({ + malformed: { + ...createManagedPolicyEntry("malformed"), + policyCreationReceipt: { schemaVersion: 1 }, + }, + }); + expect(() => malformedRegistry.getSandbox("malformed")).toThrow( + /invalid policy creation receipt/u, + ); + + const mismatchedRegistry = await loadRegistryWith({ + managed: { + ...managed, + gatewayPort: 9090, + }, + }); + expect(() => mismatchedRegistry.getSandbox("managed")).toThrow( + /does not match its gateway and sandbox identity/u, + ); + + const registry = await loadRegistryWith({}); + expect(() => registry.registerSandbox(managed, undefined, { pending: true })).toThrow( + /pending sandbox registration/u, + ); + expect(() => + registry.registerSandbox({ name: "managed", policyAuthority: "nemoclaw-managed" }), + ).toThrow(/without a complete policy creation receipt/u); + + registry.registerSandbox(managed); + registry.reserveSandboxInferenceRoute("managed", { + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://api.example.test/v1", + credentialEnv: "CUSTOM_API_KEY", + preferredInferenceApi: "openai-responses", + gatewayName: "nemoclaw", + }); + expect(registry.getSandbox("managed")).toMatchObject({ + pendingRouteReservation: true, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: managed.policyCreationReceipt, + }); + registry.restoreSandboxEntry(managed); + expect(registry.getSandbox("managed")).toEqual(managed); + + const replacementSelection = { + provider: "compatible-endpoint", + model: "model-b", + endpointUrl: "https://api.example.test/v1", + endpointSource: null, + credentialEnv: "CUSTOM_API_KEY", + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + nimContainer: null, + gatewayName: "nemoclaw", + reservationSessionId: "session-owner", + } as const; + registry.reserveSandboxInferenceRoute("managed", replacementSelection); + const replacementLifecycle = { + ...createManagedPolicyEntry("managed", 2), + ...replacementSelection, + lifecycleGeneration: "223e4567-e89b-42d3-a456-426614174983", + lifecycleLiveIdentityFingerprint: "e".repeat(64), + }; + replacementLifecycle.policyCreationReceipt = { + ...replacementLifecycle.policyCreationReceipt, + lifecycleGeneration: replacementLifecycle.lifecycleGeneration, + sandboxIdentityFingerprint: replacementLifecycle.lifecycleLiveIdentityFingerprint, + }; + const createReservation = registry.qualifyPendingSandboxCreateReservation( + { + sandboxName: "managed", + gatewayName: "nemoclaw", + sessionId: "session-owner", + selection: replacementSelection, + }, + registry.getSandbox("managed"), + ); + const checkpoint = { + schemaVersion: 1 as const, + state: "verified-create" as const, + policyAuthority: "nemoclaw-managed" as const, + observedPolicyAuthority: "owner-unknown" as const, + gatewayName: "nemoclaw", + gatewayPort: 8080, + sandboxName: "managed", + lifecycleGeneration: replacementLifecycle.lifecycleGeneration, + sandboxIdentityFingerprint: replacementLifecycle.lifecycleLiveIdentityFingerprint, + route: "none" as const, + policyHash: replacementLifecycle.policyCreationReceipt.policyHash, + policyVersion: replacementLifecycle.policyCreationReceipt.policyVersion, + policyCreationReceipt: replacementLifecycle.policyCreationReceipt, + }; + registry.recordPendingSandboxPolicyVerification(createReservation, checkpoint); + expect( + registry.registerSandbox(replacementLifecycle, undefined, { + verifiedCreate: { reservation: createReservation, checkpoint }, + }), + ).toMatchObject({ + lifecycleGeneration: replacementLifecycle.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: replacementLifecycle.lifecycleLiveIdentityFingerprint, + policyCreationReceipt: replacementLifecycle.policyCreationReceipt, + }); }); it("sets a gateway port only while the complete qualified row remains current (#10056)", async () => { diff --git a/src/lib/state/registry-normalization.ts b/src/lib/state/registry-normalization.ts index 8888e5c62fb..5080eb226e1 100644 --- a/src/lib/state/registry-normalization.ts +++ b/src/lib/state/registry-normalization.ts @@ -1,13 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { SandboxPolicyAuthority } from "../adapters/openshell/policy-authority"; import { isObjectRecord } from "../core/json-types"; +import { + parseNemoClawPolicyCreationReceipt, + type NemoClawPolicyCreationReceipt, +} from "../policy/merge"; import { normalizeTrustedPrivatePolicyPinReceipt } from "../policy/trusted-private-endpoints"; import type { BaselineExclusionEntry, BaselineExclusionTransition, CustomPolicyEntry, + PendingSandboxPolicyVerification, + RecordedSandboxPolicyAuthority, SandboxEntry, } from "./registry/types"; @@ -15,11 +20,27 @@ const BASELINE_TRANSITION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const BASELINE_TRANSITION_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; const SHA256_DIGEST_PATTERN = /^[a-f0-9]{64}$/; +const RESERVATION_SESSION_CONTROL_CHARACTER = /[\u0000-\u001f\u007f]/u; +const PENDING_POLICY_VERIFICATION_KEYS = new Set([ + "schemaVersion", + "state", + "policyAuthority", + "observedPolicyAuthority", + "gatewayName", + "gatewayPort", + "sandboxName", + "lifecycleGeneration", + "sandboxIdentityFingerprint", + "route", + "policyHash", + "policyVersion", + "policyCreationReceipt", +]); /** Keep legacy absence unknown and reject every unrecognized authority value. */ export function normalizeSandboxPolicyAuthority( value: unknown, -): SandboxPolicyAuthority | undefined { +): RecordedSandboxPolicyAuthority | undefined { if (value === undefined) return undefined; if (value === "nemoclaw-managed" || value === "externally-managed") return value; throw new Error( @@ -27,9 +48,179 @@ export function normalizeSandboxPolicyAuthority( ); } +/** Clone one complete policy-creation receipt and reject every partial form. */ +export function cloneSandboxPolicyCreationReceipt( + value: unknown, +): NemoClawPolicyCreationReceipt | undefined { + if (value === undefined) return undefined; + try { + return parseNemoClawPolicyCreationReceipt(value); + } catch { + throw new Error( + "Sandbox registry contains an invalid policy creation receipt; repair the registry before continuing", + ); + } +} + +/** Clone one exact post-create policy checkpoint and reject partial or forged state. */ +export function normalizePendingSandboxPolicyVerification( + value: unknown, +): PendingSandboxPolicyVerification | undefined { + if (value === undefined) return undefined; + if ( + !isObjectRecord(value) || + Object.keys(value).some((key) => !PENDING_POLICY_VERIFICATION_KEYS.has(key)) || + value.schemaVersion !== 1 || + value.state !== "verified-create" || + (value.policyAuthority !== "nemoclaw-managed" && + value.policyAuthority !== "externally-managed") || + (value.observedPolicyAuthority !== "owner-unknown" && + value.observedPolicyAuthority !== "externally-managed") || + typeof value.gatewayName !== "string" || + value.gatewayName.length === 0 || + !Number.isSafeInteger(value.gatewayPort) || + Number(value.gatewayPort) < 1 || + Number(value.gatewayPort) > 65_535 || + typeof value.sandboxName !== "string" || + value.sandboxName.length === 0 || + typeof value.lifecycleGeneration !== "string" || + value.lifecycleGeneration.length === 0 || + typeof value.sandboxIdentityFingerprint !== "string" || + !SHA256_DIGEST_PATTERN.test(value.sandboxIdentityFingerprint) || + (value.route !== "none" && value.route !== "native" && value.route !== "compatibility") || + typeof value.policyHash !== "string" || + !Number.isSafeInteger(value.policyVersion) || + Number(value.policyVersion) < 1 + ) { + throw new Error( + "Sandbox registry contains an invalid pending policy verification; repair the registry before continuing", + ); + } + let boundary; + try { + boundary = parseNemoClawPolicyCreationReceipt({ + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: value.gatewayName, + gatewayPort: value.gatewayPort, + sandboxName: value.sandboxName, + lifecycleGeneration: value.lifecycleGeneration, + sandboxIdentityFingerprint: value.sandboxIdentityFingerprint, + policyHash: value.policyHash, + policyVersion: value.policyVersion, + }); + } catch { + throw new Error( + "Sandbox registry contains an invalid pending policy verification identity; repair the registry before continuing", + ); + } + if (value.policyAuthority === "nemoclaw-managed") { + if (value.observedPolicyAuthority !== "owner-unknown") { + throw new Error( + "Sandbox registry contains an invalid managed pending policy verification; repair the registry before continuing", + ); + } + const receipt = cloneSandboxPolicyCreationReceipt(value.policyCreationReceipt); + if ( + !receipt || + receipt.gatewayName !== boundary.gatewayName || + receipt.gatewayPort !== boundary.gatewayPort || + receipt.sandboxName !== boundary.sandboxName || + receipt.lifecycleGeneration !== boundary.lifecycleGeneration || + receipt.sandboxIdentityFingerprint !== boundary.sandboxIdentityFingerprint || + receipt.policyHash !== boundary.policyHash || + receipt.policyVersion !== boundary.policyVersion + ) { + throw new Error( + "Sandbox registry pending managed policy verification does not match its creation receipt", + ); + } + return { + schemaVersion: 1, + state: "verified-create", + policyAuthority: "nemoclaw-managed", + observedPolicyAuthority: "owner-unknown", + gatewayName: boundary.gatewayName, + gatewayPort: boundary.gatewayPort, + sandboxName: boundary.sandboxName, + lifecycleGeneration: boundary.lifecycleGeneration, + sandboxIdentityFingerprint: boundary.sandboxIdentityFingerprint, + route: value.route, + policyHash: boundary.policyHash, + policyVersion: boundary.policyVersion, + policyCreationReceipt: receipt, + }; + } + if (value.policyCreationReceipt !== undefined) { + throw new Error( + "Sandbox registry pending external policy verification cannot contain a creation receipt", + ); + } + return { + schemaVersion: 1, + state: "verified-create", + policyAuthority: "externally-managed", + observedPolicyAuthority: value.observedPolicyAuthority, + gatewayName: boundary.gatewayName, + gatewayPort: boundary.gatewayPort, + sandboxName: boundary.sandboxName, + lifecycleGeneration: boundary.lifecycleGeneration, + sandboxIdentityFingerprint: boundary.sandboxIdentityFingerprint, + route: value.route, + policyHash: boundary.policyHash, + policyVersion: boundary.policyVersion, + }; +} + /** Remove policy attribution that an external authority owns and normalize managed state. */ export function normalizeSandboxPolicyAttribution(entry: SandboxEntry): SandboxEntry { - const policyAuthority = normalizeSandboxPolicyAuthority(entry.policyAuthority); + const requestedPolicyAuthority = normalizeSandboxPolicyAuthority(entry.policyAuthority); + const parsedPolicyCreationReceipt = cloneSandboxPolicyCreationReceipt( + entry.policyCreationReceipt, + ); + if ( + parsedPolicyCreationReceipt && + (parsedPolicyCreationReceipt.sandboxName !== entry.name || + parsedPolicyCreationReceipt.gatewayName !== entry.gatewayName || + parsedPolicyCreationReceipt.gatewayPort !== entry.gatewayPort || + parsedPolicyCreationReceipt.lifecycleGeneration !== entry.lifecycleGeneration || + parsedPolicyCreationReceipt.sandboxIdentityFingerprint !== + entry.lifecycleLiveIdentityFingerprint) + ) { + throw new Error( + "Sandbox registry policy creation receipt does not match its gateway and sandbox identity", + ); + } + const hasManagedReceipt = + requestedPolicyAuthority === "nemoclaw-managed" && parsedPolicyCreationReceipt !== undefined; + const policyAuthority = + requestedPolicyAuthority === "nemoclaw-managed" && !hasManagedReceipt + ? undefined + : requestedPolicyAuthority; + const policyCreationReceipt = hasManagedReceipt ? parsedPolicyCreationReceipt : undefined; + const pendingPolicyVerification = normalizePendingSandboxPolicyVerification( + entry.pendingPolicyVerification, + ); + if ( + pendingPolicyVerification && + (entry.pendingRouteReservation !== true || + typeof entry.reservationSessionId !== "string" || + entry.reservationSessionId.length === 0 || + entry.reservationSessionId.length > 256 || + RESERVATION_SESSION_CONTROL_CHARACTER.test(entry.reservationSessionId) || + requestedPolicyAuthority !== undefined || + parsedPolicyCreationReceipt !== undefined || + pendingPolicyVerification.sandboxName !== entry.name || + pendingPolicyVerification.gatewayName !== entry.gatewayName || + pendingPolicyVerification.gatewayPort !== entry.gatewayPort || + pendingPolicyVerification.lifecycleGeneration !== entry.lifecycleGeneration || + pendingPolicyVerification.sandboxIdentityFingerprint !== + entry.lifecycleLiveIdentityFingerprint) + ) { + throw new Error( + "Sandbox registry pending policy verification does not match its route reservation", + ); + } const { policies: _policies, customPolicies: _customPolicies, @@ -38,10 +229,17 @@ export function normalizeSandboxPolicyAttribution(entry: SandboxEntry): SandboxE policyPresetsFinalized: _policyPresetsFinalized, policyTier: _policyTier, policyAuthority: _policyAuthority, + policyCreationReceipt: _policyCreationReceipt, + pendingPolicyVerification: _pendingPolicyVerification, ...rest } = entry; if (policyAuthority === "externally-managed") { - return { ...rest, policies: [], policyAuthority }; + return { + ...rest, + policies: [], + policyAuthority, + ...(pendingPolicyVerification ? { pendingPolicyVerification } : {}), + }; } const baselineExclusions = normalizeBaselineExclusions(entry.baselineExclusions); @@ -60,6 +258,8 @@ export function normalizeSandboxPolicyAttribution(entry: SandboxEntry): SandboxE : {}), ...(entry.policyTier !== undefined ? { policyTier: entry.policyTier } : {}), ...(policyAuthority !== undefined ? { policyAuthority } : {}), + ...(policyCreationReceipt ? { policyCreationReceipt } : {}), + ...(pendingPolicyVerification ? { pendingPolicyVerification } : {}), }; } diff --git a/src/lib/state/registry-route-reservation.test.ts b/src/lib/state/registry-route-reservation.test.ts index 8dcc60a3e87..3ab5e63053c 100644 --- a/src/lib/state/registry-route-reservation.test.ts +++ b/src/lib/state/registry-route-reservation.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { serializedHostLocalInferenceReceipt } from "../../../test/helpers/host-local-inference-receipt"; import type { InferenceSelection } from "../inference/selection"; import type { SandboxInferenceRouteReservationDisposition } from "./registry/route-reservation"; +import type { PendingSandboxPolicyVerification, SandboxEntry } from "./registry/types"; function ownedReservation(disposition: SandboxInferenceRouteReservationDisposition) { expect(disposition.kind).toBe("owned"); @@ -41,6 +42,75 @@ const EXACT_ROUTE_RESERVATION = { ...EXACT_ROUTE_SELECTION, }; +const LIFECYCLE_GENERATION = "123e4567-e89b-42d3-a456-426614174983"; +const LIVE_IDENTITY_FINGERPRINT = "a".repeat(64); + +function managedCheckpoint( + overrides: Partial< + Pick< + PendingSandboxPolicyVerification, + | "gatewayPort" + | "lifecycleGeneration" + | "sandboxIdentityFingerprint" + | "route" + | "policyHash" + | "policyVersion" + > + > = {}, +): PendingSandboxPolicyVerification { + const boundary = { + gatewayPort: 8080, + lifecycleGeneration: LIFECYCLE_GENERATION, + sandboxIdentityFingerprint: LIVE_IDENTITY_FINGERPRINT, + route: "none" as const, + policyHash: "sha256:policy-1", + policyVersion: 1, + ...overrides, + }; + return { + schemaVersion: 1, + state: "verified-create", + policyAuthority: "nemoclaw-managed", + observedPolicyAuthority: "owner-unknown", + gatewayName: EXACT_ROUTE_AUTHORITY.gatewayName, + sandboxName: EXACT_ROUTE_AUTHORITY.sandboxName, + ...boundary, + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: EXACT_ROUTE_AUTHORITY.gatewayName, + gatewayPort: boundary.gatewayPort, + sandboxName: EXACT_ROUTE_AUTHORITY.sandboxName, + lifecycleGeneration: boundary.lifecycleGeneration, + sandboxIdentityFingerprint: boundary.sandboxIdentityFingerprint, + policyHash: boundary.policyHash, + policyVersion: boundary.policyVersion, + }, + }; +} + +function externalCheckpoint( + overrides: Partial< + Pick + > = {}, +): PendingSandboxPolicyVerification { + return { + schemaVersion: 1, + state: "verified-create", + policyAuthority: "externally-managed", + observedPolicyAuthority: "externally-managed", + gatewayName: EXACT_ROUTE_AUTHORITY.gatewayName, + gatewayPort: 8080, + sandboxName: EXACT_ROUTE_AUTHORITY.sandboxName, + lifecycleGeneration: LIFECYCLE_GENERATION, + sandboxIdentityFingerprint: LIVE_IDENTITY_FINGERPRINT, + route: "none", + policyHash: "sha256:external-1", + policyVersion: 1, + ...overrides, + }; +} + function createdSandboxRegistrationInput( sandboxName: string, gatewayName: string, @@ -104,6 +174,32 @@ function reserveQualifiedRoute(registry: typeof import("./registry")) { ); } +function reserveQualifiedCreate(registry: typeof import("./registry")) { + const route = reserveQualifiedRoute(registry); + const create = registry.qualifyPendingSandboxCreateReservation( + EXACT_ROUTE_AUTHORITY, + registry.getSandbox(EXACT_ROUTE_AUTHORITY.sandboxName), + ); + return { route, create }; +} + +function completedEntry(checkpoint: PendingSandboxPolicyVerification): SandboxEntry { + return { + name: EXACT_ROUTE_AUTHORITY.sandboxName, + ...EXACT_ROUTE_SELECTION, + agent: "hermes", + openshellDriver: "docker", + gatewayName: EXACT_ROUTE_AUTHORITY.gatewayName, + gatewayPort: checkpoint.gatewayPort, + lifecycleGeneration: checkpoint.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: checkpoint.sandboxIdentityFingerprint, + policyAuthority: checkpoint.policyAuthority, + ...(checkpoint.policyAuthority === "nemoclaw-managed" + ? { policyCreationReceipt: checkpoint.policyCreationReceipt } + : {}), + }; +} + describe("sandbox inference route reservation", () => { afterEach(() => { vi.unstubAllEnvs(); @@ -383,6 +479,7 @@ describe("sandbox inference route reservation", () => { credentialEnv: "CUSTOM_API_KEY", preferredInferenceApi: "openai-responses", gatewayName: "nemoclaw-9090", + reservationSessionId: "session-owner", } as const; registry.reserveSandboxInferenceRoute("alpha", { @@ -392,6 +489,7 @@ describe("sandbox inference route reservation", () => { registry.reserveSandboxInferenceRoute("alpha", { ...route, model: "model-b" }); expect(registry.getSandbox("alpha")?.hostLocalInferenceReceipt).toBe(receipt); + expect(registry.finalizeSandboxRouteReservation("alpha", "session-owner")).toBe(true); const entry = registerCreatedSandbox({ sandboxName: "alpha", inferenceSelection: { @@ -884,18 +982,13 @@ describe("sandbox inference route reservation", () => { vi.resetModules(); try { const registry = await import("./registry"); - const qualified = reserveQualifiedRoute(registry); + const { route, create } = reserveQualifiedCreate(registry); + const checkpoint = managedCheckpoint(); + registry.recordPendingSandboxPolicyVerification(create, checkpoint); - const registered = registry.registerSandbox( - { - name: "alpha", - ...EXACT_ROUTE_SELECTION, - agent: "hermes", - openshellDriver: "docker", - gatewayName: "nemoclaw", - }, - qualified, - ); + const registered = registry.registerSandbox(completedEntry(checkpoint), route, { + verifiedCreate: { reservation: create, checkpoint }, + }); expect(registered).toMatchObject({ name: "alpha", @@ -904,12 +997,196 @@ describe("sandbox inference route reservation", () => { agent: "hermes", }); expect(registered.pendingRouteReservation).toBeUndefined(); + expect(registered.pendingPolicyVerification).toBeUndefined(); expect(registry.getSandbox("alpha")).toEqual(registered); } finally { await fs.rm(home, { recursive: true, force: true }); } }); + it("keeps a verified create checkpoint non-authorizing and hidden until final publication (#9833)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-policy-checkpoint-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + const { create } = reserveQualifiedCreate(registry); + const checkpoint = managedCheckpoint(); + + const pending = registry.recordPendingSandboxPolicyVerification(create, checkpoint); + + expect(pending).toMatchObject({ + pendingRouteReservation: true, + reservationSessionId: "session-owner", + pendingPolicyVerification: checkpoint, + lifecycleGeneration: LIFECYCLE_GENERATION, + lifecycleLiveIdentityFingerprint: LIVE_IDENTITY_FINGERPRINT, + }); + expect(pending).not.toHaveProperty("policyAuthority"); + expect(pending).not.toHaveProperty("policyCreationReceipt"); + expect(registry.getDefault()).toBeNull(); + expect(() => registry.updateSandbox("alpha", { agent: "hermes" })).toThrow( + /verified create checkpoint is incomplete/u, + ); + expect(registry.finalizeSandboxRouteReservation("alpha", "session-owner")).toBe(false); + expect(registry.finalizePendingSandboxRegistration("alpha")).toBe(false); + expect(registry.recordPendingSandboxPolicyVerification(create, checkpoint)).toEqual(pending); + expect( + registry.reserveSandboxInferenceRoute("alpha", { + ...EXACT_ROUTE_SELECTION, + gatewayName: "nemoclaw", + reservationSessionId: "session-owner", + }), + ).toBe(true); + expect(() => + registry.reserveSandboxInferenceRoute("alpha", { + ...EXACT_ROUTE_SELECTION, + gatewayName: "nemoclaw", + openshellDriver: "kubernetes", + reservationSessionId: "session-owner", + }), + ).toThrow(/verified create checkpoint is incomplete/u); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it("rotates a checkpoint only from the exact old evidence and supports an exact retry (#9833)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-policy-checkpoint-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + const { create } = reserveQualifiedCreate(registry); + const initial = managedCheckpoint({ route: "native" }); + const replacement = managedCheckpoint({ + route: "compatibility", + sandboxIdentityFingerprint: "b".repeat(64), + policyHash: "sha256:policy-2", + policyVersion: 2, + }); + registry.recordPendingSandboxPolicyVerification(create, initial); + + const rotated = registry.recordPendingSandboxPolicyVerification(create, replacement, { + expected: initial, + }); + expect(rotated.pendingPolicyVerification).toEqual(replacement); + expect( + registry.recordPendingSandboxPolicyVerification(create, replacement, { + expected: initial, + }), + ).toEqual(rotated); + expect(() => + registry.recordPendingSandboxPolicyVerification(create, initial, { + expected: initial, + }), + ).toThrow(/without exact authority/u); + expect(() => + registry.recordPendingSandboxPolicyVerification( + { + ...create, + authority: { ...create.authority, sessionId: "another-session" }, + }, + replacement, + ), + ).toThrow(/incomplete verified sandbox create checkpoint/u); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it("requires the exact durable checkpoint at final registration (#9833)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-policy-checkpoint-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + const { route, create } = reserveQualifiedCreate(registry); + const checkpoint = managedCheckpoint(); + + expect(() => registry.registerSandbox(completedEntry(checkpoint), route)).toThrow( + /verified policy checkpoint/u, + ); + registry.recordPendingSandboxPolicyVerification(create, checkpoint); + registry.removeSandbox("alpha"); + expect(() => + registry.registerSandbox(completedEntry(checkpoint), route, { + verifiedCreate: { reservation: create, checkpoint }, + }), + ).toThrow(/verified create checkpoint changed/u); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it("rejects a second route reservation authority at final registration (#9833)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-policy-checkpoint-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + const { route, create } = reserveQualifiedCreate(registry); + const checkpoint = managedCheckpoint(); + registry.recordPendingSandboxPolicyVerification(create, checkpoint); + + expect(() => + registry.registerSandbox( + completedEntry(checkpoint), + { + ...route, + authority: { ...route.authority, sessionId: "different-session" }, + }, + { verifiedCreate: { reservation: create, checkpoint } }, + ), + ).toThrow(/different route reservation authority/u); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it("rejects stale external identity and partial lifecycle evidence at final registration (#9833)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-policy-checkpoint-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + const { route, create } = reserveQualifiedCreate(registry); + const checkpoint = externalCheckpoint(); + registry.recordPendingSandboxPolicyVerification(create, checkpoint); + + expect(() => + registry.registerSandbox(completedEntry(checkpoint), route, { + verifiedCreate: { + reservation: create, + checkpoint: externalCheckpoint({ policyHash: "sha256:changed" }), + }, + }), + ).toThrow(/verified create checkpoint changed/u); + expect(() => + registry.registerSandbox( + { + ...completedEntry(checkpoint), + lifecycleGeneration: "223e4567-e89b-42d3-a456-426614174983", + }, + route, + { verifiedCreate: { reservation: create, checkpoint } }, + ), + ).toThrow(/verified create checkpoint/u); + expect(() => + registry.registerSandbox( + { + ...completedEntry(checkpoint), + lifecycleLiveIdentityFingerprint: "c".repeat(64), + }, + route, + { verifiedCreate: { reservation: create, checkpoint } }, + ), + ).toThrow(/verified create checkpoint/u); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + it("keeps an incomplete registration hidden until its caller commits it (#9733)", async () => { const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-pending-registration-")); vi.stubEnv("HOME", home); @@ -956,7 +1233,7 @@ describe("sandbox inference route reservation", () => { vi.resetModules(); try { const registry = await import("./registry"); - const qualified = reserveQualifiedRoute(registry); + const { create } = reserveQualifiedCreate(registry); registry.reserveSandboxInferenceRoute("alpha", { ...EXACT_ROUTE_SELECTION, model: "another-model", @@ -965,17 +1242,8 @@ describe("sandbox inference route reservation", () => { }); expect(() => - registry.registerSandbox( - { - name: "alpha", - ...EXACT_ROUTE_SELECTION, - agent: "hermes", - openshellDriver: "docker", - gatewayName: "nemoclaw", - }, - qualified, - ), - ).toThrow("Cannot register a sandbox after its inference route reservation changed"); + registry.recordPendingSandboxPolicyVerification(create, managedCheckpoint()), + ).toThrow("route reservation changed"); expect(registry.getSandbox("alpha")).toMatchObject({ pendingRouteReservation: true, reservationSessionId: "another-session", diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index 654c1f82bf6..e935960527c 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -19,28 +19,35 @@ import { withLock } from "./registry/lock"; import { load, save } from "./registry/persistence"; import { isCurrentSandboxInferenceRouteReservation, + isCurrentPendingSandboxCreateReservation, normalizeSandboxInferenceRouteSelection, sandboxRegistrationMatchesInferenceRouteReservation, + type QualifiedPendingSandboxCreateReservation, type QualifiedSandboxInferenceRouteReservation, } from "./registry/route-reservation"; export { classifySandboxInferenceRouteReservation, isCurrentSandboxInferenceRouteReservation, + isCurrentPendingSandboxCreateReservation, isPendingReservationForSession, isPublishedSandboxRegistration, isRouteOnlySandboxReservation, normalizeSandboxInferenceRouteSelection, + qualifyPendingSandboxCreateReservation, sandboxRegistrationMatchesInferenceRouteReservation, type QualifiedSandboxInferenceRouteReservation, + type QualifiedPendingSandboxCreateReservation, type SandboxInferenceRouteReservationAuthority, type SandboxInferenceRouteReservationDisposition, } from "./registry/route-reservation"; import { cloneSandboxWorkloadReceipt } from "./registry/workload"; import { normalizeSandboxMcpState } from "./registry-mcp"; import { + cloneSandboxPolicyCreationReceipt, normalizeBaselineExclusions, normalizeBaselineExclusionTransition, normalizeCustomPolicyEntries, + normalizePendingSandboxPolicyVerification, normalizeSandboxPolicyAttribution, normalizeSandboxPolicyAuthority, retainedDefaultSandbox, @@ -74,6 +81,7 @@ import type { BaselineExclusionEntry, BaselineExclusionTransition, CustomPolicyEntry, + PendingSandboxPolicyVerification, SandboxEntry, } from "./registry/types"; import { @@ -108,6 +116,7 @@ export type { SandboxGpuProofResult, SandboxGpuProofStatus, SandboxHostMount, + PendingSandboxPolicyVerification, SandboxRegistry, SandboxWorkloadReceipt, } from "./registry/types"; @@ -120,6 +129,7 @@ export { type SandboxMessagingState, } from "./registry-messaging"; export { + cloneSandboxPolicyCreationReceipt, hasUnsafeHostMountTerminalText, normalizeCustomPolicyEntries, normalizeSandboxPolicyAttribution, @@ -146,54 +156,319 @@ export function getDefault(): string | null { return names.length > 0 ? names[0] || null : null; } +function pendingVerifiedCreateEntry( + reservation: QualifiedPendingSandboxCreateReservation, + checkpoint: PendingSandboxPolicyVerification, +): SandboxEntry { + return normalizeSandboxPolicyAttribution({ + ...reservation.entry, + gatewayPort: checkpoint.gatewayPort, + lifecycleGeneration: checkpoint.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: checkpoint.sandboxIdentityFingerprint, + policyAuthority: undefined, + policyCreationReceipt: undefined, + pendingPolicyVerification: checkpoint, + }); +} + +function assertPendingPolicyVerificationMatchesRegistration( + recordedEntry: SandboxEntry | undefined, + requestedEntry: SandboxEntry, + authority: + | { + readonly reservation: QualifiedPendingSandboxCreateReservation; + readonly checkpoint: PendingSandboxPolicyVerification; + } + | undefined, +): void { + const checkpoint = normalizePendingSandboxPolicyVerification( + recordedEntry?.pendingPolicyVerification, + ); + const expectedCheckpoint = normalizePendingSandboxPolicyVerification(authority?.checkpoint); + if (!authority) { + if (checkpoint) { + throw new PolicyAuthorityRefusalError( + "Cannot publish a verified create checkpoint without exact transaction authority", + ); + } + return; + } + if (!checkpoint || !expectedCheckpoint || !isDeepStrictEqual(checkpoint, expectedCheckpoint)) { + throw new PolicyAuthorityRefusalError( + "Cannot publish a sandbox registration after its verified create checkpoint changed", + ); + } + const reservation = authority.reservation; + if ( + !isCurrentPendingSandboxCreateReservation(reservation, reservation.entry) || + !recordedEntry || + !isDeepStrictEqual(recordedEntry, pendingVerifiedCreateEntry(reservation, expectedCheckpoint)) + ) { + throw new PolicyAuthorityRefusalError( + "Cannot publish a sandbox registration after its verified create transaction changed", + ); + } + const commonMatches = + recordedEntry?.pendingRouteReservation === true && + recordedEntry.reservationSessionId === reservation.authority.sessionId && + recordedEntry.policyAuthority === undefined && + recordedEntry.policyCreationReceipt === undefined && + recordedEntry.lifecycleGeneration === checkpoint.lifecycleGeneration && + recordedEntry.lifecycleLiveIdentityFingerprint === checkpoint.sandboxIdentityFingerprint && + checkpoint.sandboxName === requestedEntry.name && + checkpoint.gatewayName === requestedEntry.gatewayName && + checkpoint.gatewayPort === requestedEntry.gatewayPort && + checkpoint.lifecycleGeneration === requestedEntry.lifecycleGeneration && + checkpoint.sandboxIdentityFingerprint === + requestedEntry.lifecycleLiveIdentityFingerprint && + checkpoint.policyAuthority === requestedEntry.policyAuthority && + reservation.authority.sandboxName === requestedEntry.name && + reservation.authority.gatewayName === requestedEntry.gatewayName && + isDeepStrictEqual( + normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(recordedEntry)), + normalizeSandboxInferenceRouteSelection(reservation.authority.selection), + ) && + isDeepStrictEqual( + normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(requestedEntry)), + normalizeSandboxInferenceRouteSelection(reservation.authority.selection), + ); + const authorityMatches = + checkpoint.policyAuthority === "nemoclaw-managed" + ? isDeepStrictEqual(checkpoint.policyCreationReceipt, requestedEntry.policyCreationReceipt) + : requestedEntry.policyCreationReceipt === undefined; + if (!commonMatches || !authorityMatches) { + throw new PolicyAuthorityRefusalError( + "Cannot publish a sandbox registration that differs from its verified create checkpoint", + ); + } +} + +/** Persist the exact verified create boundary before any unrelated post-create effect. */ +export function recordPendingSandboxPolicyVerification( + reservation: QualifiedPendingSandboxCreateReservation, + value: PendingSandboxPolicyVerification, + options: { readonly expected?: PendingSandboxPolicyVerification } = {}, +): SandboxEntry { + const checkpoint = normalizePendingSandboxPolicyVerification(value); + const expected = normalizePendingSandboxPolicyVerification(options.expected); + const { authority } = reservation; + const name = authority.sandboxName; + if ( + !checkpoint || + !authority.sessionId || + checkpoint.sandboxName !== name || + checkpoint.gatewayName !== authority.gatewayName || + !isCurrentPendingSandboxCreateReservation(reservation, reservation.entry) + ) { + throw new PolicyAuthorityRefusalError( + "Cannot record an incomplete verified sandbox create checkpoint", + ); + } + return withLock(() => { + const data = load(); + const current = data.sandboxes[name]; + const recordedCheckpoint = normalizePendingSandboxPolicyVerification( + current?.pendingPolicyVerification, + ); + if (!current) { + throw new PolicyAuthorityRefusalError( + `Cannot record sandbox '${name}' policy verification after its route reservation changed`, + ); + } + const desiredEntry = pendingVerifiedCreateEntry(reservation, checkpoint); + if (isDeepStrictEqual(current, desiredEntry)) { + return structuredClone(current); + } + if (expected === undefined) { + if ( + recordedCheckpoint !== undefined || + !isCurrentPendingSandboxCreateReservation(reservation, current) + ) { + throw new PolicyAuthorityRefusalError( + `Cannot record sandbox '${name}' policy verification after its route reservation changed`, + ); + } + } else { + const expectedEntry = pendingVerifiedCreateEntry(reservation, expected); + if ( + !recordedCheckpoint || + !isDeepStrictEqual(current, expectedEntry) || + checkpoint.lifecycleGeneration !== expected.lifecycleGeneration || + checkpoint.policyAuthority !== expected.policyAuthority || + checkpoint.gatewayName !== expected.gatewayName || + checkpoint.gatewayPort !== expected.gatewayPort || + checkpoint.sandboxName !== expected.sandboxName + ) { + throw new PolicyAuthorityRefusalError( + `Cannot replace sandbox '${name}' verified create checkpoint without exact authority`, + ); + } + } + data.sandboxes[name] = desiredEntry; + save(data); + return structuredClone(desiredEntry); + }); +} + +/** Re-read one durable verified create checkpoint before releasing an effect. */ +export function requireCurrentPendingSandboxPolicyVerification( + reservation: QualifiedPendingSandboxCreateReservation, + expected: PendingSandboxPolicyVerification, +): SandboxEntry { + const checkpoint = normalizePendingSandboxPolicyVerification(expected); + const { authority } = reservation; + const name = authority.sandboxName; + const current = load().sandboxes[name]; + if ( + !checkpoint || + !isCurrentPendingSandboxCreateReservation(reservation, reservation.entry) || + !current || + !isDeepStrictEqual(current, pendingVerifiedCreateEntry(reservation, checkpoint)) + ) { + throw new PolicyAuthorityRefusalError( + `Cannot continue sandbox '${name}' creation after its verified checkpoint changed`, + ); + } + return structuredClone(current); +} + export function registerSandbox( entry: SandboxEntry, routeReservation?: QualifiedSandboxInferenceRouteReservation, - options: { pending?: boolean; reservationSessionId?: string } = {}, + options: { + pending?: boolean; + reservationSessionId?: string; + verifiedCreate?: { + readonly reservation: QualifiedPendingSandboxCreateReservation; + readonly checkpoint: PendingSandboxPolicyVerification; + }; + } = {}, ): SandboxEntry { return withLock(() => { const data = load(); - const reserved = data.sandboxes[entry.name]; + const recordedEntry = data.sandboxes[entry.name]; + if (entry.pendingPolicyVerification !== undefined) { + throw new PolicyAuthorityRefusalError( + "Cannot publish a caller-supplied pending policy verification", + ); + } + if (routeReservation && options.pending !== true && !options.verifiedCreate) { + throw new PolicyAuthorityRefusalError( + "Cannot consume a create route reservation without its verified policy checkpoint", + ); + } + if ( + routeReservation && + options.verifiedCreate && + !isDeepStrictEqual(routeReservation, options.verifiedCreate.reservation) + ) { + throw new PolicyAuthorityRefusalError( + "Cannot publish a verified sandbox create with a different route reservation authority", + ); + } if ( routeReservation && - (!isCurrentSandboxInferenceRouteReservation(routeReservation, reserved ?? null) || + ((!options.verifiedCreate && + !isCurrentSandboxInferenceRouteReservation( + routeReservation, + data.sandboxes[entry.name] ?? null, + )) || !sandboxRegistrationMatchesInferenceRouteReservation(entry, routeReservation)) ) { throw new Error("Cannot register a sandbox after its inference route reservation changed"); } if ( !routeReservation && - reserved?.pendingRouteReservation === true && - typeof reserved.reservationSessionId === "string" && - reserved.reservationSessionId.length > 0 && + !options.verifiedCreate && + recordedEntry?.pendingRouteReservation === true && + typeof recordedEntry.reservationSessionId === "string" && + recordedEntry.reservationSessionId.length > 0 && (options.pending !== true || typeof options.reservationSessionId !== "string" || options.reservationSessionId.length === 0 || - options.reservationSessionId !== reserved.reservationSessionId) + options.reservationSessionId !== recordedEntry.reservationSessionId) ) { throw new Error("Cannot stage a sandbox after its inference route reservation changed"); } if (options.reservationSessionId) { if ( - reserved?.pendingRouteReservation !== true || - reserved.reservationSessionId !== options.reservationSessionId || - reserved.gatewayName !== entry.gatewayName || + recordedEntry?.pendingRouteReservation !== true || + recordedEntry.reservationSessionId !== options.reservationSessionId || + recordedEntry.gatewayName !== entry.gatewayName || !isDeepStrictEqual( - normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(reserved)), + normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(recordedEntry)), normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(entry)), ) ) { throw new Error("Cannot stage a sandbox after its inference route reservation changed"); } } + if ( + recordedEntry?.pendingRouteReservation === true && + options.pending !== true && + !options.verifiedCreate + ) { + throw new PolicyAuthorityRefusalError( + "Cannot publish a pending sandbox create without its verified policy checkpoint", + ); + } const servingProfileProvenance = parseServingProfileProvenance(entry.servingProfileProvenance); if (entry.servingProfileProvenance !== undefined && !servingProfileProvenance) { throw new Error("Cannot register a sandbox with invalid serving profile provenance"); } - const requestedPolicyAuthority = normalizeSandboxPolicyAuthority(entry.policyAuthority); - const recordedPolicyAuthority = normalizeSandboxPolicyAuthority( - data.sandboxes[entry.name]?.policyAuthority, + if ( + options.pending === true && + (entry.policyAuthority === "nemoclaw-managed" || entry.policyCreationReceipt !== undefined) + ) { + throw new PolicyAuthorityRefusalError( + "Cannot attach NemoClaw policy ownership to a pending sandbox registration", + ); + } + if (entry.policyAuthority === "nemoclaw-managed" && entry.policyCreationReceipt === undefined) { + throw new PolicyAuthorityRefusalError( + "Cannot register NemoClaw policy ownership without a complete policy creation receipt", + ); + } + if (entry.policyCreationReceipt !== undefined && entry.policyAuthority !== "nemoclaw-managed") { + throw new PolicyAuthorityRefusalError( + "Cannot register a policy creation receipt without NemoClaw policy ownership", + ); + } + const normalizedPolicyEntry = normalizeSandboxPolicyAttribution(entry); + const requestedPolicyAuthority = normalizeSandboxPolicyAuthority( + normalizedPolicyEntry.policyAuthority, ); + const requestedPolicyCreationReceipt = normalizedPolicyEntry.policyCreationReceipt; + if (requestedPolicyAuthority === "nemoclaw-managed") { + assertPolicyCreationReceiptMatchesSandboxEntry( + normalizedPolicyEntry, + requestedPolicyCreationReceipt, + ); + } + assertPendingPolicyVerificationMatchesRegistration( + recordedEntry, + normalizedPolicyEntry, + options.verifiedCreate, + ); + const recordedPolicyAuthority = normalizeSandboxPolicyAuthority(recordedEntry?.policyAuthority); + const recordedPolicyCreationReceipt = recordedEntry?.policyCreationReceipt; + const reservedGenerationChanged = + recordedEntry?.pendingRouteReservation === true && + recordedEntry.lifecycleGeneration !== normalizedPolicyEntry.lifecycleGeneration; + const reservedFingerprintChanged = + recordedEntry?.pendingRouteReservation === true && + recordedEntry.lifecycleLiveIdentityFingerprint !== + normalizedPolicyEntry.lifecycleLiveIdentityFingerprint; + if (reservedGenerationChanged !== reservedFingerprintChanged) { + throw new PolicyAuthorityRefusalError( + "Cannot register a sandbox after only part of its reserved lifecycle identity changed", + ); + } + const replacesReservedSandboxLifecycle = + recordedEntry?.pendingRouteReservation === true && + requestedPolicyCreationReceipt !== undefined && + reservedGenerationChanged && + reservedFingerprintChanged; if ( recordedPolicyAuthority !== undefined && requestedPolicyAuthority !== undefined && @@ -203,6 +478,15 @@ export function registerSandbox( "Cannot register a sandbox after its policy authority changed", ); } + if ( + recordedPolicyAuthority === "nemoclaw-managed" && + !isDeepStrictEqual(recordedPolicyCreationReceipt, requestedPolicyCreationReceipt) && + !replacesReservedSandboxLifecycle + ) { + throw new PolicyAuthorityRefusalError( + "Cannot register a sandbox after its policy creation receipt changed", + ); + } const policyAuthority = requestedPolicyAuthority ?? recordedPolicyAuthority; if (retainedDefaultSandbox(data.defaultSandbox, data.sandboxes) === null) { data.defaultSandbox = null; @@ -265,6 +549,9 @@ export function registerSandbox( openshellDriver: entry.openshellDriver || null, openshellVersion: entry.openshellVersion || null, ...(policyAuthority !== undefined ? { policyAuthority } : {}), + ...(policyAuthority === "nemoclaw-managed" && requestedPolicyCreationReceipt + ? { policyCreationReceipt: requestedPolicyCreationReceipt } + : {}), ...(policyAuthority === "externally-managed" ? { policies: [] } : { @@ -373,6 +660,33 @@ export function reserveSandboxInferenceRoute( const data = load(); const existing = data.sandboxes[name]; const normalized = normalizeInferenceSelection(route); + if (existing?.pendingPolicyVerification) { + const sameReservation = + existing.pendingRouteReservation === true && + Boolean(route.reservationSessionId) && + existing.reservationSessionId === route.reservationSessionId && + existing.gatewayName === route.gatewayName && + existing.gatewayPort === (route.gatewayPort ?? existing.gatewayPort) && + existing.openshellDriver === (route.openshellDriver ?? existing.openshellDriver) && + existing.hostLocalInferenceReceipt === + (route.hostLocalInferenceReceipt === undefined + ? existing.hostLocalInferenceReceipt + : route.hostLocalInferenceReceipt) && + isDeepStrictEqual( + existing.hostLocalInferenceProvenance, + route.hostLocalInferenceProvenance ?? existing.hostLocalInferenceProvenance, + ) && + isDeepStrictEqual( + normalizeInferenceSelection(existing), + normalizeInferenceSelection(route), + ); + if (!sameReservation) { + throw new PolicyAuthorityRefusalError( + `Cannot replace sandbox '${name}' while its verified create checkpoint is incomplete`, + ); + } + return true; + } const provenance = cloneSandboxHostLocalInferenceProvenance(route.hostLocalInferenceProvenance); if ( route.hostLocalInferenceProvenance !== undefined && @@ -413,8 +727,21 @@ export function reserveSandboxInferenceRoute( throw new Error("Cannot change an explicit host-local inference lifecycle reservation"); } } - const next: SandboxEntry = { - ...(existing ?? { name, pendingRouteReservation: true as const }), + const existingForReservation: SandboxEntry = existing + ? { ...existing } + : { name, pendingRouteReservation: true }; + if ( + existingForReservation.policyCreationReceipt !== undefined && + (route.gatewayName !== existingForReservation.gatewayName || + (route.gatewayPort !== undefined && + route.gatewayPort !== existingForReservation.gatewayPort)) + ) { + throw new PolicyAuthorityRefusalError( + "Cannot move a receipt-bound sandbox reservation to another gateway", + ); + } + const next = normalizeSandboxPolicyAttribution({ + ...existingForReservation, pendingRouteReservation: true, reservationSessionId: route.reservationSessionId ?? @@ -431,10 +758,12 @@ export function reserveSandboxInferenceRoute( ...(provenance ? { hostLocalInferenceProvenance: provenance } : {}), gatewayName: route.gatewayName, gatewayPort: - route.gatewayPort ?? - (existing?.gatewayName === route.gatewayName ? existing.gatewayPort : undefined), + existingForReservation.policyCreationReceipt === undefined + ? (route.gatewayPort ?? + (existing?.gatewayName === route.gatewayName ? existing.gatewayPort : undefined)) + : existingForReservation.gatewayPort, ...(route.openshellDriver === undefined ? {} : { openshellDriver: route.openshellDriver }), - }; + }); data.sandboxes[name] = next; save(data); return true; @@ -485,16 +814,65 @@ function assertRecordedPolicyAuthorityUnchanged( ); } +function assertPolicyCreationReceiptMatchesSandboxEntry( + entry: SandboxEntry, + receipt: SandboxEntry["policyCreationReceipt"], +): asserts receipt is NonNullable { + if ( + !receipt || + receipt.sandboxName !== entry.name || + receipt.gatewayName !== entry.gatewayName || + receipt.gatewayPort !== entry.gatewayPort || + receipt.lifecycleGeneration !== entry.lifecycleGeneration || + receipt.sandboxIdentityFingerprint !== entry.lifecycleLiveIdentityFingerprint + ) { + throw new PolicyAuthorityRefusalError( + `Cannot record NemoClaw policy ownership for sandbox '${entry.name}' without an exact gateway and sandbox identity receipt`, + ); + } +} + +function assertPolicyCreationReceiptUnchanged( + current: SandboxEntry, + updates: Partial, +): void { + if (!Object.prototype.hasOwnProperty.call(updates, "policyCreationReceipt")) return; + const requested = cloneSandboxPolicyCreationReceipt(updates.policyCreationReceipt); + if (isDeepStrictEqual(requested, current.policyCreationReceipt)) return; + throw new PolicyAuthorityRefusalError( + `Refusing to update sandbox '${current.name}' because its policy creation receipt changed outside the receipt rotation transaction.`, + ); +} + export function updateSandbox(name: string, updates: Partial): boolean { return withLock(() => { const data = load(); const current = data.sandboxes[name]; if (!current) return false; + if (Object.prototype.hasOwnProperty.call(updates, "pendingPolicyVerification")) { + throw new PolicyAuthorityRefusalError( + `Refusing to change sandbox '${name}' verified create checkpoint outside its transaction.`, + ); + } + if (current.pendingPolicyVerification) { + throw new PolicyAuthorityRefusalError( + `Refusing to update sandbox '${name}' while its verified create checkpoint is incomplete.`, + ); + } if (Object.prototype.hasOwnProperty.call(updates, "name") && updates.name !== name) { return false; } if (changesHostLocalInferenceLifecycleAuthority(current, updates)) return false; + if ( + updates.policyAuthority === "nemoclaw-managed" && + current.policyAuthority !== "nemoclaw-managed" + ) { + throw new PolicyAuthorityRefusalError( + `Refusing to assign NemoClaw policy ownership to sandbox '${current.name}' outside completed sandbox registration.`, + ); + } assertRecordedPolicyAuthorityUnchanged(current, updates); + assertPolicyCreationReceiptUnchanged(current, updates); data.sandboxes[name] = normalizeSandboxPolicyAttribution({ ...current, ...updates }); save(data); return true; @@ -540,6 +918,7 @@ export function finalizeSandboxRouteReservation(name: string, sessionId: string) const current = data.sandboxes[name]; if (!current || !sessionId || current.reservationSessionId !== sessionId) return false; if (current.pendingRouteReservation !== true) return true; + if (current.pendingPolicyVerification) return false; data.sandboxes[name] = { ...current, pendingRouteReservation: undefined, @@ -549,6 +928,52 @@ export function finalizeSandboxRouteReservation(name: string, sessionId: string) }); } +/** Replace only the policy identity in an exact, completed NemoClaw ownership receipt. */ +export function compareAndSetSandboxPolicyCreationReceipt( + name: string, + expected: NonNullable, + replacement: NonNullable, +): boolean { + const expectedReceipt = cloneSandboxPolicyCreationReceipt(expected); + const replacementReceipt = cloneSandboxPolicyCreationReceipt(replacement); + if (!expectedReceipt || !replacementReceipt) { + throw new PolicyAuthorityRefusalError( + "Cannot rotate an incomplete NemoClaw policy creation receipt", + ); + } + if ( + expectedReceipt.gatewayName !== replacementReceipt.gatewayName || + expectedReceipt.gatewayPort !== replacementReceipt.gatewayPort || + expectedReceipt.sandboxName !== replacementReceipt.sandboxName || + expectedReceipt.lifecycleGeneration !== replacementReceipt.lifecycleGeneration || + expectedReceipt.sandboxIdentityFingerprint !== replacementReceipt.sandboxIdentityFingerprint + ) { + throw new PolicyAuthorityRefusalError( + "Cannot rotate a policy creation receipt across a gateway or sandbox identity", + ); + } + + return withLock(() => { + const data = load(); + const current = data.sandboxes[name]; + if ( + !current || + current.pendingRouteReservation === true || + current.policyAuthority !== "nemoclaw-managed" || + !isDeepStrictEqual(current.policyCreationReceipt, expectedReceipt) + ) { + return false; + } + assertPolicyCreationReceiptMatchesSandboxEntry(current, replacementReceipt); + data.sandboxes[name] = { + ...current, + policyCreationReceipt: replacementReceipt, + }; + save(data); + return true; + }); +} + /** Atomically publish a pending registration and preserve its initial-default claim. */ export function finalizePendingSandboxRegistration(name: string): boolean { return withLock(() => { @@ -557,7 +982,8 @@ export function finalizePendingSandboxRegistration(name: string): boolean { if ( !current || current.pendingRouteReservation !== true || - current.reservationSessionId !== undefined + current.reservationSessionId !== undefined || + current.pendingPolicyVerification !== undefined ) { return false; } @@ -596,10 +1022,16 @@ export function restoreSandboxEntry( const data = load(); const normalizedEntry = normalizeSandboxPolicyAttribution(entry); const current = data.sandboxes[normalizedEntry.name]; + if (current?.pendingPolicyVerification && !isDeepStrictEqual(current, normalizedEntry)) { + throw new PolicyAuthorityRefusalError( + `Refusing to restore sandbox '${normalizedEntry.name}' while its verified create checkpoint is incomplete.`, + ); + } if ( current && - normalizeSandboxPolicyAuthority(current.policyAuthority) !== - normalizeSandboxPolicyAuthority(normalizedEntry.policyAuthority) + (normalizeSandboxPolicyAuthority(current.policyAuthority) !== + normalizeSandboxPolicyAuthority(normalizedEntry.policyAuthority) || + !isDeepStrictEqual(current.policyCreationReceipt, normalizedEntry.policyCreationReceipt)) ) { throw new PolicyAuthorityRefusalError( `Refusing to restore sandbox '${normalizedEntry.name}' because its policy authority changed during recovery.`, diff --git a/src/lib/state/registry/route-reservation.ts b/src/lib/state/registry/route-reservation.ts index 78b0799d9ef..35e82cd30c1 100644 --- a/src/lib/state/registry/route-reservation.ts +++ b/src/lib/state/registry/route-reservation.ts @@ -20,6 +20,7 @@ const ROUTE_RESERVATION_KEYS = new Set([ "name", "openshellDriver", "pendingRouteReservation", + "pendingPolicyVerification", "preferredInferenceApi", "policies", "policyPresetsFinalized", @@ -89,6 +90,12 @@ export interface QualifiedSandboxInferenceRouteReservation { readonly entry: SandboxEntry; } +/** Exact pending row admitted for one sandbox create transaction. */ +export interface QualifiedPendingSandboxCreateReservation { + readonly authority: SandboxInferenceRouteReservationAuthority; + readonly entry: SandboxEntry; +} + export type SandboxInferenceRouteReservationDisposition = | { readonly kind: "missing" } | { readonly kind: "owned"; readonly reservation: QualifiedSandboxInferenceRouteReservation } @@ -107,6 +114,42 @@ export function normalizeSandboxInferenceRouteSelection(input: InferenceSelectio }; } +/** Admit the full pending row used by a create, including preserved rebuild metadata. */ +export function qualifyPendingSandboxCreateReservation( + authority: SandboxInferenceRouteReservationAuthority, + entry: SandboxEntry | null, +): QualifiedPendingSandboxCreateReservation { + if ( + !entry || + entry.pendingRouteReservation !== true || + !authority.sessionId || + entry.reservationSessionId !== authority.sessionId || + entry.name !== authority.sandboxName || + entry.gatewayName !== authority.gatewayName || + !isDeepStrictEqual( + normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(entry)), + normalizeSandboxInferenceRouteSelection(authority.selection), + ) + ) { + throw new Error("The sandbox create route reservation is not owned by this onboarding session"); + } + return { authority: structuredClone(authority), entry: structuredClone(entry) }; +} + +/** Compare the complete admitted create reservation before its first checkpoint write. */ +export function isCurrentPendingSandboxCreateReservation( + reservation: QualifiedPendingSandboxCreateReservation, + entry: SandboxEntry | null, +): boolean { + if (!entry) return false; + try { + const qualified = qualifyPendingSandboxCreateReservation(reservation.authority, entry); + return isDeepStrictEqual(qualified.entry, reservation.entry); + } catch { + return false; + } +} + /** * True only for an inference route reserved before sandbox registration. * @@ -224,8 +267,14 @@ export function isCurrentSandboxInferenceRouteReservation( entry: SandboxEntry | null, ): boolean { const current = classifySandboxInferenceRouteReservation(reservation.authority, entry); + if (current.kind !== "owned") return false; + const { pendingPolicyVerification: currentCheckpoint, ...currentRoute } = + current.reservation.entry; + const { pendingPolicyVerification: reservedCheckpoint, ...reservedRoute } = reservation.entry; return ( - current.kind === "owned" && isDeepStrictEqual(current.reservation.entry, reservation.entry) + isDeepStrictEqual(currentRoute, reservedRoute) && + (reservedCheckpoint === undefined || + isDeepStrictEqual(currentCheckpoint, reservedCheckpoint)) ); } diff --git a/src/lib/state/registry/types.ts b/src/lib/state/registry/types.ts index d01c2a57654..f87932c3de6 100644 --- a/src/lib/state/registry/types.ts +++ b/src/lib/state/registry/types.ts @@ -7,12 +7,42 @@ import type { ServingProfileProvenance } from "../../inference/serving/types"; import type { WebSearchProvider } from "../../inference/web-search"; import type { DcodeAutoApprovalMode } from "../../onboard/dcode-auto-approval"; import type { NativeArtifactWorkloadReceiptV1 } from "../../onboard/workload/native-artifact"; +import type { NemoClawPolicyCreationReceipt } from "../../policy/merge"; import type { TrustedPrivatePolicyPinReceipt } from "../../policy/trusted-private-endpoints"; import type { ToolDisclosure } from "../../tool-disclosure"; import type { OpenClawImagePluginInstall } from "../openclaw-plugin-restore"; import type { SandboxMcpState } from "../registry-mcp"; import type { SandboxMessagingState } from "../registry-messaging"; +export type RecordedSandboxPolicyAuthority = Exclude; + +interface PendingSandboxPolicyVerificationBoundary { + readonly schemaVersion: 1; + readonly state: "verified-create"; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly sandboxName: string; + readonly lifecycleGeneration: string; + readonly sandboxIdentityFingerprint: string; + readonly route: "none" | "native" | "compatibility"; + readonly policyHash: string; + readonly policyVersion: number; +} + +/** Durable, incomplete create boundary recorded before post-create effects. */ +export type PendingSandboxPolicyVerification = PendingSandboxPolicyVerificationBoundary & + ( + | { + readonly policyAuthority: "nemoclaw-managed"; + readonly observedPolicyAuthority: "owner-unknown"; + readonly policyCreationReceipt: NemoClawPolicyCreationReceipt; + } + | { + readonly policyAuthority: "externally-managed"; + readonly observedPolicyAuthority: "externally-managed" | "owner-unknown"; + readonly policyCreationReceipt?: never; + } + ); export interface CustomPolicyEntry { name: string; content: string; @@ -121,7 +151,11 @@ export interface SandboxEntry extends Partial { openshellDriver?: string | null; openshellVersion?: string | null; /** Policy authority for a completed sandbox; absence means unknown. */ - policyAuthority?: SandboxPolicyAuthority; + policyAuthority?: RecordedSandboxPolicyAuthority; + /** Exact, secret-free proof that NemoClaw created this sandbox policy. */ + policyCreationReceipt?: NemoClawPolicyCreationReceipt; + /** Verified create boundary retained until final registration publishes atomically. */ + pendingPolicyVerification?: PendingSandboxPolicyVerification; policies?: string[]; customPolicies?: CustomPolicyEntry[]; /** Operator exclusions from the agent baseline policy, replayed on rebuild. */ diff --git a/test/cli/sandbox-mutations.test.ts b/test/cli/sandbox-mutations.test.ts index 9c4c1b9df6f..73930934b78 100644 --- a/test/cli/sandbox-mutations.test.ts +++ b/test/cli/sandbox-mutations.test.ts @@ -5,6 +5,11 @@ import fs from "node:fs"; import path from "node:path"; import { describe, expect, test as it } from "../helpers/owned-test-resources"; +import { + managedPolicyMetadata, + managedSandboxEntry, + SANDBOX_ID, +} from "../helpers/managed-policy-receipt-fixture"; import { runWithEnv, runWithInput, testTimeoutOptions, writeSandboxRegistry } from "./helpers"; @@ -23,17 +28,22 @@ function writePolicyMutationOpenshellStub(home: string): string { const localBin = path.join(home, "bin"); fs.mkdirSync(localBin, { recursive: true }); const openshell = path.join(localBin, "openshell"); + const appliedPolicy = path.join(home, "applied-policy.yaml"); fs.writeFileSync( openshell, [ "#!/usr/bin/env bash", "set -euo pipefail", + 'if [ "$1" = "sandbox" ] && [ "$2" = "get" ]; then', + ` printf 'Name: alpha\\nId: ${SANDBOX_ID}\\nPhase: Ready\\n'`, + " exit 0", + "fi", 'if [ "$1" = "policy" ] && [ "$2" = "get" ]; then', ' if [[ " $* " == *" --output json "* ]]; then', - ' sandbox="${@: -1}"', - ' printf \'{"scope":"sandbox","sandbox":"%s","status":"effective","policy_source":"sandbox","policy":{}}\\n\' "$sandbox"', + ` printf '%s\\n' ${JSON.stringify(managedPolicyMetadata("alpha"))}`, " exit 0", " fi", + ` if [ -f ${JSON.stringify(appliedPolicy)} ]; then cat ${JSON.stringify(appliedPolicy)}; exit 0; fi`, " cat <<'YAML'", "version: 1", "network_policies:", @@ -44,6 +54,13 @@ function writePolicyMutationOpenshellStub(home: string): string { " exit 0", "fi", 'if [ "$1" = "policy" ] && [ "$2" = "set" ]; then', + ' while [ "$#" -gt 0 ]; do', + ' if [ "$1" = "--policy" ]; then', + ` cp "$2" ${JSON.stringify(appliedPolicy)}`, + " break", + " fi", + " shift", + " done", " exit 0", "fi", 'printf "unexpected openshell args: %s\\n" "$*" >&2', @@ -121,7 +138,7 @@ describe("CLI dispatch", () => { it("keeps public policy-add/remove built-in mutation routes", ({ testHome }) => { const { home } = testHome; - writeSandboxRegistry(home); + writeSandboxRegistry(home, managedSandboxEntry("alpha")); const openshell = writePolicyMutationOpenshellStub(home); const add = runWithEnv( diff --git a/test/e2e-test.sh b/test/e2e-test.sh index 5a72d5a17b1..9824512ae2c 100755 --- a/test/e2e-test.sh +++ b/test/e2e-test.sh @@ -162,15 +162,53 @@ if [ "${1:-}" = "status" ]; then printf '%s\n' 'Gateway Status' ' Status: Connected' ' Gateway: fixture-gateway' exit 0 fi +if [ "${1:-} ${2:-}" = "gateway info" ]; then + printf '%s\n' 'Gateway endpoint: http://127.0.0.1:8080' + exit 0 +fi +if [ "${1:-} ${2:-}" = "sandbox get" ]; then + sandbox="${@: -1}" + printf 'Name: %s\nId: fixture-sandbox-id\nPhase: Ready\n' "$sandbox" + exit 0 +fi if [ "${1:-} ${2:-}" = "policy list" ]; then exit 0 fi +if [ "${1:-} ${2:-}" = "policy set" ]; then + if [ "$#" -ne 8 ] || [ "${5:-}" != "--policy" ] || [ -z "${6:-}" ]; then + echo "unexpected policy write: expected policy set -g fixture-gateway --policy --wait " >&2 + exit 64 + fi + cp "$6" "${BASH_SOURCE[0]%/*}/effective-policy.yaml" + exit 0 +fi if [ "${1:-} ${2:-}" = "policy get" ]; then printf '%s\n' "$*" >>"${BASH_SOURCE[0]%/*}/calls" fi if [ "${1:-} ${2:-}" = "policy get" ] && [[ " $* " == *" --output json "* ]]; then sandbox="${@: -1}" - printf '{"scope":"sandbox","sandbox":"%s","status":"effective","policy_source":"sandbox","policy":{}}\n' "$sandbox" + policy_file=/opt/nemoclaw-blueprint/policies/openclaw-sandbox.yaml + policy_hash=sha256:fixture-policy + policy_version=1 + if [ -f "${BASH_SOURCE[0]%/*}/effective-policy.yaml" ]; then + policy_file="${BASH_SOURCE[0]%/*}/effective-policy.yaml" + policy_hash=sha256:fixture-mutated-policy + policy_version=2 + fi + node -e ' + const fs = require("node:fs"); + const YAML = require("/opt/nemoclaw/node_modules/yaml"); + const policy = YAML.parse(fs.readFileSync(process.argv[2], "utf8")); + process.stdout.write(JSON.stringify({ + scope: "sandbox", + sandbox: process.argv[1], + status: "effective", + policy_source: "sandbox", + hash: process.argv[3], + active_version: Number(process.argv[4]), + policy, + }) + "\n"); + ' "$sandbox" "$policy_file" "$policy_hash" "$policy_version" exit 0 fi case "$*" in @@ -189,7 +227,10 @@ case "$*" in esac SH chmod 0755 "$FAKE_OPENSHELL_BIN/openshell" -PATH="$FAKE_OPENSHELL_BIN:$PATH" NEMOCLAW_BLUEPRINT_PATH=/opt/nemoclaw-blueprint node --input-type=module -e " +PATH="$FAKE_OPENSHELL_BIN:$PATH" \ + NEMOCLAW_BLUEPRINT_PATH=/opt/nemoclaw-blueprint \ + OPENSHELL_SANDBOX_POLICY=/opt/nemoclaw-blueprint/policies/openclaw-sandbox.yaml \ + node --input-type=module -e " const { main } = await import('/opt/nemoclaw/dist/blueprint/runner.js'); await main(['apply', '--profile', 'ncp']); " 2>&1 | tee "$APPLY_OUTPUT" diff --git a/test/helpers/hermes-portable-onboarding-fixture.ts b/test/helpers/hermes-portable-onboarding-fixture.ts index 8845279de4f..d8b94e35a0c 100644 --- a/test/helpers/hermes-portable-onboarding-fixture.ts +++ b/test/helpers/hermes-portable-onboarding-fixture.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { isDeepStrictEqual } from "node:util"; @@ -31,7 +32,9 @@ export const HERMES_PORTABLE_TEST_POLICY = "version: 1\nnetwork_policies: {}\n"; const CONTAINER_ID = "a".repeat(64); const IMAGE_ID = "b".repeat(64); const SANDBOX_ID = "sandbox-id-1"; -export const HERMES_PORTABLE_TEST_LIVE_IDENTITY = "live-identity-1"; +export const HERMES_PORTABLE_TEST_LIVE_IDENTITY = createHash("sha256") + .update(SANDBOX_ID) + .digest("hex"); const ROUTE_SESSION_ID = "session-alpha"; export function makeHermesPortableCheckoutPrivate(root: string): void { diff --git a/test/helpers/hermes-shields-provider-consumer-harness.ts b/test/helpers/hermes-shields-provider-consumer-harness.ts index b2dd802a704..9bcd6cab95d 100644 --- a/test/helpers/hermes-shields-provider-consumer-harness.ts +++ b/test/helpers/hermes-shields-provider-consumer-harness.ts @@ -290,9 +290,13 @@ export function createHermesShieldsProviderConsumerHarness( vi .spyOn(policy, "inspectPolicyMutationAuthority") .mockReturnValue(managedPolicyMutationAuthority), + vi + .spyOn(policy, "inspectPolicyRecoveryAuthority") + .mockReturnValue(managedPolicyMutationAuthority), vi .spyOn(policy, "recheckPolicyMutationAuthority") .mockReturnValue(managedPolicyMutationAuthority), + vi.spyOn(policy, "finalizePolicyMutationReceipt").mockImplementation(() => undefined), registrySpy, vi .spyOn(privilegedExec, "privilegedSandboxExecArgv") diff --git a/test/helpers/hermes-unsafe-config-shields-harness.ts b/test/helpers/hermes-unsafe-config-shields-harness.ts index 16989a94faa..47f5058b2b7 100644 --- a/test/helpers/hermes-unsafe-config-shields-harness.ts +++ b/test/helpers/hermes-unsafe-config-shields-harness.ts @@ -284,9 +284,13 @@ export function createHermesUnsafeConfigHarness( vi .spyOn(policy, "inspectPolicyMutationAuthority") .mockReturnValue(managedPolicyMutationAuthority); + vi + .spyOn(policy, "inspectPolicyRecoveryAuthority") + .mockReturnValue(managedPolicyMutationAuthority); vi .spyOn(policy, "recheckPolicyMutationAuthority") .mockReturnValue(managedPolicyMutationAuthority); + vi.spyOn(policy, "finalizePolicyMutationReceipt").mockImplementation(() => undefined); vi.spyOn(agentConfig, "resolveAgentConfig").mockReturnValue(hermesTarget); vi.spyOn(registry, "getSandbox").mockImplementation((name: unknown) => ({ name: String(name), diff --git a/test/helpers/managed-image-buildless-e2e.ts b/test/helpers/managed-image-buildless-e2e.ts index 8f570c35025..25978e9350b 100644 --- a/test/helpers/managed-image-buildless-e2e.ts +++ b/test/helpers/managed-image-buildless-e2e.ts @@ -191,8 +191,8 @@ const replace = (target, name, value) => { if (target[name] !== value) throw new Error("could not install test boundary for " + name); }; const childProcess = require("node:child_process"); -require(${source("test/helpers/onboard-script-mocks.cjs")}) - .mockStandaloneGatewayTeardownAuthority(); +const fixtureMocks = require(${source("test/helpers/onboard-script-mocks.cjs")}); +fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const coreVersion = require(${source("src/lib/core/version.ts")}); replace(coreVersion, "getVersion", () => catalogRelease); @@ -427,7 +427,7 @@ runner.run = (command, options = {}) => { } if (normalized.includes("sandbox get") && normalized.includes(sandboxName)) { return sandboxCreated - ? { status: 0, stdout: "Name: " + sandboxName + "\nId: sbx-managed-fixture\n", stderr: "" } + ? { status: 0, stdout: "Name: " + sandboxName + "\nId: fixture-managed-sandbox\n", stderr: "" } : { status: 1, stdout: "", stderr: "sandbox not found" }; } if (argv[0] === "docker" && argv[1] === "volume") { @@ -455,9 +455,28 @@ runner.runFile = (file, args = []) => runner.run([file, ...args]); runner.runCapture = (command) => { const normalized = normalize(command); runnerCommands.push(normalized); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { + sandboxName, + sandboxId: "fixture-managed-sandbox", + }); + if (createdIdentity !== null) return createdIdentity; + if (normalized.includes("policy get") && normalized.includes("--output json")) { + return JSON.stringify({ + scope: "sandbox", + sandbox: sandboxName, + status: "effective", + policy_source: "sandbox", + hash: "fixture-policy", + active_version: 1, + policy: {}, + }); + } + if (normalized.includes("gateway info")) { + return "Gateway endpoint: http://127.0.0.1:8080"; + } if (normalized.includes("sandbox get") && normalized.includes(sandboxName)) { return sandboxCreated - ? "Name: " + sandboxName + "\nId: " + sandboxName + "-id\nState: Ready" + ? "Name: " + sandboxName + "\nId: fixture-managed-sandbox\nState: Ready" : ""; } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; @@ -497,7 +516,7 @@ runner.runCaptureEx = (command) => { }; const registry = require(${source("src/lib/state/registry.ts")}); -const sourceEntry = recreate ? { +const sourceEntry = recreate ? fixtureMocks.managedSandboxPolicyReceiptFixture({ name: sandboxName, agent: "hermes", gpuEnabled: false, @@ -505,7 +524,6 @@ const sourceEntry = recreate ? { imageTag: catalogTemplate.hermes.reference, model, provider, - policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive", workload: { schemaVersion: 1, @@ -522,7 +540,7 @@ const sourceEntry = recreate ? { credentialProxyReplayRequired: true, shared: true, }, -} : null; +}, { sandboxName, sandboxId: "fixture-managed-sandbox" }) : null; registry.getSandbox = () => registeredSandbox ?? (existingEntryAvailable ? sourceEntry : null); registry.getDefault = () => null; registry.listExtraProviders = () => []; @@ -534,6 +552,17 @@ registry.registerSandbox = (entry) => { registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName, + provider, + model, + preferredInferenceApi: "openai-completions", + getSandbox: registry.getSandbox, + registerSandbox: (entry) => { + registerCalls.push(entry); + registeredSandbox = entry; + }, +}); const preflight = require(${source("src/lib/onboard/preflight.ts")}); preflight.checkPortAvailable = async () => ({ ok: true }); @@ -567,15 +596,24 @@ const { createSandbox } = require(${source("src/lib/onboard.ts")}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; await createSandbox( - null, - model, - provider, - "openai-completions", - sandboxName, - null, - [], - null, - loadAgent(agentName), + ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [ + null, + model, + provider, + "openai-completions", + sandboxName, + null, + [], + null, + loadAgent(agentName), + null, + null, + null, + [], + ], + createFixture, + ), ); console.log(JSON.stringify({ agent: agentName, diff --git a/test/helpers/managed-policy-receipt-fixture.ts b/test/helpers/managed-policy-receipt-fixture.ts new file mode 100644 index 00000000000..cc81b77d3be --- /dev/null +++ b/test/helpers/managed-policy-receipt-fixture.ts @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +export const POLICY_HASH = "policy-alpha"; +export const POLICY_VERSION = 7; +export const LIFECYCLE_GENERATION = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +export const SANDBOX_ID = "sandbox-id"; +export const SANDBOX_IDENTITY = createHash("sha256").update(SANDBOX_ID).digest("hex"); + +interface ManagedSandboxEntryOptions { + readonly gatewayName?: string; + readonly gatewayPort?: number; + readonly lifecycleGeneration?: string; + readonly policyHash?: string; + readonly policyVersion?: number; +} + +export function managedSandboxEntry( + name: string, + agent = "openclaw", + options: ManagedSandboxEntryOptions = {}, +) { + const gatewayName = options.gatewayName ?? "nemoclaw"; + const gatewayPort = options.gatewayPort ?? 8080; + const lifecycleGeneration = options.lifecycleGeneration ?? LIFECYCLE_GENERATION; + const policyHash = options.policyHash ?? POLICY_HASH; + const policyVersion = options.policyVersion ?? POLICY_VERSION; + return { + name, + agent, + policies: [], + openshellDriver: "docker", + gatewayName, + gatewayPort, + lifecycleGeneration, + lifecycleLiveIdentityFingerprint: SANDBOX_IDENTITY, + policyAuthority: "nemoclaw-managed" as const, + policyCreationReceipt: { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName, + gatewayPort, + sandboxName: name, + lifecycleGeneration, + sandboxIdentityFingerprint: SANDBOX_IDENTITY, + policyHash, + policyVersion, + }, + }; +} + +export function managedPolicyInspection() { + return { + authority: "owner-unknown" as const, + effectivePolicy: {}, + policyIdentity: { + hash: POLICY_HASH, + activeVersion: POLICY_VERSION, + }, + }; +} + +export function managedRegistrationSource(name: string, agent = "openclaw"): string { + return `registry.registerSandbox(${JSON.stringify(managedSandboxEntry(name, agent))});`; +} + +export function managedPolicyMetadata(sandboxName: string): string { + return JSON.stringify({ + scope: "sandbox", + sandbox: sandboxName, + status: "effective", + policy_source: "sandbox", + hash: POLICY_HASH, + active_version: POLICY_VERSION, + policy: { version: 1, network_policies: {} }, + }); +} + +export function parseResultPayload(stdout: string): any { + const marker = "__RESULT__"; + const markerIndex = stdout.indexOf(marker); + if (markerIndex < 0) throw new Error("Expected the result marker"); + return JSON.parse(stdout.slice(markerIndex + marker.length)); +} diff --git a/test/helpers/onboard-openshell-fixture.ts b/test/helpers/onboard-openshell-fixture.ts index 8179a9fcd42..5888d5beff3 100644 --- a/test/helpers/onboard-openshell-fixture.ts +++ b/test/helpers/onboard-openshell-fixture.ts @@ -10,14 +10,15 @@ function writeExecutable(target: string, contents: string): void { export function writeOkOpenshell( fakeBin: string, - options: { readySandboxGet?: boolean } = {}, + options: { gatewayPort?: number; readySandboxGet?: boolean } = {}, ): void { + const gatewayPort = options.gatewayPort ?? 8080; const sandboxGet = options.readySandboxGet ? 'if [ "${1:-}" = sandbox ] && [ "${2:-}" = get ]; then printf "Sandbox:\\n\\n Id: fixture-created-sandbox\\n Name: %s\\n Phase: Ready\\n" "${!#}"; fi\n' : ""; writeExecutable( path.join(fakeBin, "openshell"), - `#!/usr/bin/env bash\n${sandboxGet}if [ "\${1:-}" = policy ] && [ "\${2:-}" = get ]; then printf '{"scope":"sandbox","sandbox":"%s","status":"effective","policy_source":"sandbox","policy":{}}\\n' "\${!#}"; fi\nif [ "\${1:-}" = sandbox ] && [ "\${2:-}" = ssh-config ]; then printf "Host openshell-%s.default\\n HostName 127.0.0.1\\n User sandbox\\n" "\${3:-sandbox}"; fi\nexit 0\n`, + `#!/usr/bin/env bash\n${sandboxGet}if [ "\${1:-}" = policy ] && [ "\${2:-}" = get ] && [[ " $* " = *" --output json "* ]]; then printf '{"scope":"sandbox","sandbox":"%s","status":"effective","policy_source":"sandbox","hash":"fixture-policy","active_version":1,"policy":{}}\\n' "\${!#}"; fi\nif [ "\${1:-}" = policy ] && [ "\${2:-}" = get ] && [[ " $* " = *" --base "* ]]; then printf 'version: 1\\n'; fi\nif [ "\${1:-}" = gateway ] && [ "\${2:-}" = info ]; then printf 'Gateway endpoint: http://127.0.0.1:${gatewayPort}\\n'; fi\nif [ "\${1:-}" = sandbox ] && [ "\${2:-}" = ssh-config ]; then printf "Host openshell-%s.default\\n HostName 127.0.0.1\\n User sandbox\\n" "\${3:-sandbox}"; fi\nexit 0\n`, ); writeExecutable( path.join(fakeBin, "ssh"), diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index a725fa666fa..c1566c643af 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -123,9 +123,7 @@ function createStatefulMessagingProviderRunner({ if (providerAction === "profile") { const profileActionIndex = providerIndex + 2; const profileAction = - args[profileActionIndex] === "-g" - ? args[profileActionIndex + 2] - : args[profileActionIndex]; + args[profileActionIndex] === "-g" ? args[profileActionIndex + 2] : args[profileActionIndex]; if (profileAction === "export") { return messagingProfileImported ? { status: 0, stdout: messagingProfile, stderr: "" } @@ -274,15 +272,14 @@ const ONBOARD_SANDBOX_INSPECT = { function isOpenClawSecurityInventoryProbe(command) { const commandArgs = Array.isArray(command) ? command.map(String) : []; const dockerArgs = commandArgs[0] === "docker" ? commandArgs.slice(1) : commandArgs; - const matches = ( + const matches = dockerArgs.length === 14 && OPENCLAW_SECURITY_INVENTORY_PROBE_PREFIX.every( (expected, index) => dockerArgs[index] === expected, ) && dockerArgs[11].length > 0 && dockerArgs[12] === "-c" && - dockerArgs[13] === OPENCLAW_SECURITY_INVENTORY_PROBE - ); + dockerArgs[13] === OPENCLAW_SECURITY_INVENTORY_PROBE; return matches; } @@ -319,10 +316,7 @@ function mockOnboardRunCapture(command, options = {}) { ) { return `${ONBOARD_SANDBOX_OLD_CONTAINER_ID}\n${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`; } - if ( - normalized === - `docker inspect --type container ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}` - ) { + if (normalized === `docker inspect --type container ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`) { return JSON.stringify([ONBOARD_SANDBOX_INSPECT]); } if (isOpenClawSecurityInventoryProbe(command)) { @@ -338,11 +332,233 @@ function mockOnboardRunCapture(command, options = {}) { return mockSandboxExecCurl(command, options); } +function mockCreatedSandboxIdentityList(command, options = {}) { + const args = Array.isArray(command) ? command.map(String) : []; + const sandboxIndex = args.indexOf("sandbox"); + if ( + sandboxIndex < 0 || + args[sandboxIndex + 1] !== "list" || + !args.includes("--output") || + args[args.indexOf("--output") + 1] !== "json" + ) { + return null; + } + const selectorIndex = args.indexOf("--selector"); + const selector = selectorIndex >= 0 ? args[selectorIndex + 1] || "" : ""; + const prefix = "ai.nvidia.nemoclaw.create-attempt="; + if (!selector.startsWith(prefix)) return null; + const nonce = selector.slice(prefix.length); + return JSON.stringify([ + { + id: options.sandboxId || "fixture-created-sandbox", + name: options.sandboxName || "my-assistant", + labels: { "ai.nvidia.nemoclaw.create-attempt": nonce }, + resource_version: 1, + created_at: "2026-08-25T00:00:00Z", + phase: "Ready", + current_policy_version: 1, + }, + ]); +} + +function installVerifiedSandboxCreateFixture(registry, options) { + const sandboxName = options.sandboxName; + const gatewayName = options.gatewayName || "nemoclaw"; + const sessionId = options.sessionId || "integration-fixture-session"; + const selection = { + provider: options.provider, + model: options.model, + endpointUrl: options.endpointUrl || null, + endpointSource: options.endpointSource || null, + credentialEnv: options.credentialEnv || null, + preferredInferenceApi: options.preferredInferenceApi || null, + }; + const reservationEntry = { + name: sandboxName, + gatewayName, + pendingRouteReservation: true, + reservationSessionId: sessionId, + ...selection, + }; + let pendingCheckpoint = null; + let pendingEntry = null; + let publishedEntry = null; + let sourceEntry = options.getSandbox ? options.getSandbox(sandboxName) : null; + const qualifyPendingSandboxCreateReservation = (authority) => { + const selectionMatches = [ + "provider", + "model", + "endpointUrl", + "endpointSource", + "credentialEnv", + "preferredInferenceApi", + ].every((key) => (authority.selection[key] ?? null) === (selection[key] ?? null)); + if ( + authority.sandboxName !== sandboxName || + authority.gatewayName !== gatewayName || + authority.sessionId !== sessionId || + !selectionMatches + ) { + throw new Error("integration fixture received unexpected create reservation authority"); + } + return { + authority: structuredClone(authority), + entry: structuredClone(reservationEntry), + }; + }; + const recordPendingSandboxPolicyVerification = (reservation, checkpoint) => { + pendingCheckpoint = structuredClone(checkpoint); + pendingEntry = { + ...structuredClone(reservation.entry), + lifecycleGeneration: checkpoint.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: checkpoint.sandboxIdentityFingerprint, + pendingPolicyVerification: structuredClone(checkpoint), + }; + return structuredClone(pendingEntry); + }; + const requireCurrentPendingSandboxPolicyVerification = (reservation, checkpoint) => { + if ( + reservation.authority.sessionId !== sessionId || + pendingCheckpoint === null || + JSON.stringify(checkpoint) !== JSON.stringify(pendingCheckpoint) + ) { + throw new Error("integration fixture verified create checkpoint changed"); + } + return structuredClone(pendingEntry); + }; + + const registryPath = require.resolve(path.resolve(__dirname, "../../src/lib/state/registry.ts")); + const registryFixture = { + ...registry, + qualifyPendingSandboxCreateReservation, + recordPendingSandboxPolicyVerification, + requireCurrentPendingSandboxPolicyVerification, + getSandbox: (name) => + name === sandboxName + ? structuredClone(publishedEntry || pendingEntry || sourceEntry) + : registry.getSandbox(name), + registerSandbox: (entry) => { + publishedEntry = structuredClone(entry); + pendingEntry = null; + pendingCheckpoint = null; + options.registerSandbox?.(structuredClone(entry)); + return structuredClone(entry); + }, + updateSandbox: (name, updates) => { + if (name === sandboxName && publishedEntry) { + publishedEntry = { ...publishedEntry, ...structuredClone(updates) }; + } + options.updateSandbox?.(name, updates); + return true; + }, + setDefault: (name) => { + options.setDefault?.(name); + return true; + }, + removeSandbox: (name) => { + if (name === sandboxName) { + pendingEntry = null; + publishedEntry = null; + sourceEntry = null; + } + options.removeSandbox?.(name); + return true; + }, + }; + for (const [name, value] of Object.entries(registryFixture)) { + Object.defineProperty(registry, name, { + value, + configurable: true, + enumerable: true, + writable: true, + }); + } + require.cache[registryPath].exports = registry; + + const receiptPath = require.resolve( + path.resolve(__dirname, "../../src/lib/onboard/sandbox-create/policy-creation-receipt.ts"), + ); + const receipt = require(receiptPath); + Object.defineProperties(receipt, { + verifyCreatedSandboxPolicyRegistration: { + configurable: true, + enumerable: true, + writable: true, + value: (input) => { + if (input.plannedAuthority !== "nemoclaw-managed") { + throw new Error("integration fixture supports only managed sandbox creation"); + } + return { + policyAuthority: "nemoclaw-managed", + observedPolicyAuthority: "owner-unknown", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + sandboxName: input.sandboxName, + lifecycleGeneration: input.lifecycleGeneration, + sandboxIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, + policyHash: "fixture-policy", + policyVersion: 1, + }, + }; + }, + }, + revalidateCreatedSandboxPolicyRegistration: { + configurable: true, + enumerable: true, + writable: true, + value: (input) => input.registration, + }, + }); + require.cache[receiptPath].exports = receipt; + return { sessionId }; +} + +function sandboxCreateArgsWithVerifiedReservation(args, fixture) { + const createArgs = [...args]; + while (createArgs.length < 15) createArgs.push(null); + createArgs[14] = { sessionId: fixture.sessionId }; + return createArgs; +} + +function managedSandboxPolicyReceiptFixture(entry, options = {}) { + const sandboxName = options.sandboxName || entry.name; + const gatewayName = options.gatewayName || "nemoclaw"; + const gatewayPort = options.gatewayPort || 8080; + const lifecycleGeneration = options.lifecycleGeneration || "123e4567-e89b-42d3-a456-426614174983"; + const sandboxId = options.sandboxId || "fixture-created-sandbox"; + const sandboxIdentityFingerprint = require("node:crypto") + .createHash("sha256") + .update(sandboxId) + .digest("hex"); + const policyHash = options.policyHash || "fixture-policy"; + const policyVersion = options.policyVersion || 1; + return { + ...entry, + gatewayName, + gatewayPort, + lifecycleGeneration, + lifecycleLiveIdentityFingerprint: sandboxIdentityFingerprint, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt: { + schemaVersion: 1, + origin: "sandbox-create", + gatewayName, + gatewayPort, + sandboxName, + lifecycleGeneration, + sandboxIdentityFingerprint, + policyHash, + policyVersion, + }, + }; +} + function mockStructuredOpenShellCaptureFromRunner() { const runner = require(path.resolve(__dirname, "../../src/lib/runner.ts")); - const client = require( - path.resolve(__dirname, "../../src/lib/adapters/openshell/client.ts"), - ); + const client = require(path.resolve(__dirname, "../../src/lib/adapters/openshell/client.ts")); client.captureOpenshellCommand = (binary, args, options = {}) => { const stdout = String( runner.runCapture([binary, ...args], { @@ -493,8 +709,7 @@ function mockManagedImageCatalog() { release, cohort: "ghrun-9068-1", }, - startupProfileContractVersion: - contract.MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + startupProfileContractVersion: contract.MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, capabilityContractVersion: contract.MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, }, ]; @@ -511,10 +726,7 @@ function mockManagedImageBootstrap() { path.resolve(__dirname, "../../src/lib/onboard/managed-bootstrap/docker.ts"), ); const authorityStore = require( - path.resolve( - __dirname, - "../../src/lib/onboard/managed-bootstrap/docker-authority-store.ts", - ), + path.resolve(__dirname, "../../src/lib/onboard/managed-bootstrap/docker-authority-store.ts"), ); const sandboxIdentity = require( path.resolve(__dirname, "../../src/lib/adapters/openshell/sandbox-identity.ts"), @@ -676,7 +888,11 @@ module.exports = { isOpenClawSecurityInventoryProbe, mockDockerSandboxLifecycleReleaseFromRunner, mockFreshOpenClawPluginDiscovery, + mockCreatedSandboxIdentityList, + installVerifiedSandboxCreateFixture, + managedSandboxPolicyReceiptFixture, mockOnboardRunCapture, mockStandaloneGatewayTeardownAuthority, normalizeCommand, + sandboxCreateArgsWithVerifiedReservation, }; diff --git a/test/helpers/shields-flow-harness.ts b/test/helpers/shields-flow-harness.ts index 71b3cfcb83e..fc88567262c 100644 --- a/test/helpers/shields-flow-harness.ts +++ b/test/helpers/shields-flow-harness.ts @@ -16,6 +16,19 @@ const shieldsModulePath = "./index.js"; export const externalPolicyAuthorityInspection = { authority: "externally-managed" as const, effectivePolicy: { version: 1, network_policies: {} }, + policyIdentity: { hash: "external-policy-hash", activeVersion: 1 }, +}; + +const managedPolicyCreationReceipt = { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName: "nemoclaw", + gatewayPort: 50_065, + sandboxName: "openclaw", + lifecycleGeneration: "11111111-1111-4111-8111-111111111111", + sandboxIdentityFingerprint: "a".repeat(64), + policyHash: "managed-policy-hash", + policyVersion: 1, }; export const managedPolicyMutationAuthority = { @@ -25,7 +38,9 @@ export const managedPolicyMutationAuthority = { inspection: { authority: "nemoclaw-managed" as const, effectivePolicy: { version: 1, network_policies: {} }, + policyIdentity: { hash: "managed-policy-hash", activeVersion: 1 }, }, + policyCreationReceipt: managedPolicyCreationReceipt, }; export function bindManagedPolicyMutationAuthority( @@ -35,9 +50,13 @@ export function bindManagedPolicyMutationAuthority( vi .spyOn(policy, "inspectPolicyMutationAuthority") .mockReturnValue(managedPolicyMutationAuthority), + vi + .spyOn(policy, "inspectPolicyRecoveryAuthority") + .mockReturnValue(managedPolicyMutationAuthority), vi .spyOn(policy, "recheckPolicyMutationAuthority") .mockReturnValue(managedPolicyMutationAuthority), + vi.spyOn(policy, "finalizePolicyMutationReceipt").mockImplementation(() => undefined), ]; } @@ -52,6 +71,7 @@ export type ShieldsFlowHarness = { getOpenClawPosture: () => "locked" | "mutable"; logSpy: MockInstance; policyAuthoritySpy: MockInstance; + policyReceiptFinalizeSpy: MockInstance; policyRecoveryAuthoritySpy: MockInstance; policySetBodies: string[]; runCaptureSpy: MockInstance; @@ -371,6 +391,7 @@ export function createShieldsFlowHarness( effectivePolicy: YAML.parse( options.livePolicyYaml ?? "version: 1\nnetwork_policies:\n test: {}\n", ) as Record, + policyIdentity: { hash: "managed-policy-hash", activeVersion: 1 }, }; vi.spyOn(policyAuthority, "inspectSandboxPolicyAuthority").mockReturnValue( policyAuthorityInspection, @@ -380,6 +401,10 @@ export function createShieldsFlowHarness( authorityRecordedNow: false, gatewayName: options.sandboxEntry?.gatewayName ?? "nemoclaw", inspection: policyAuthorityInspection, + policyCreationReceipt: + policyAuthorityInspection.authority === "nemoclaw-managed" + ? managedPolicyCreationReceipt + : null, }; const policyAuthoritySpy = vi .spyOn(policy, "inspectPolicyMutationAuthority") @@ -388,6 +413,9 @@ export function createShieldsFlowHarness( .spyOn(policy, "inspectPolicyRecoveryAuthority") .mockReturnValue(policyMutationAuthority); vi.spyOn(policy, "recheckPolicyMutationAuthority").mockReturnValue(policyMutationAuthority); + const policyReceiptFinalizeSpy = vi + .spyOn(policy, "finalizePolicyMutationReceipt") + .mockImplementation(() => undefined); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [{ name: options.sandboxName ?? "openclaw", agent: resolvedAgentConfig.agentName }], }); @@ -659,6 +687,7 @@ export function createShieldsFlowHarness( getOpenClawPosture: () => openClawPosture, logSpy, policyAuthoritySpy, + policyReceiptFinalizeSpy, policyRecoveryAuthoritySpy, policySetBodies, runCaptureSpy, diff --git a/test/installer-integration/install-hermes-portable-active.test.ts b/test/installer-integration/install-hermes-portable-active.test.ts index 8e7ef989bfa..280af47b462 100644 --- a/test/installer-integration/install-hermes-portable-active.test.ts +++ b/test/installer-integration/install-hermes-portable-active.test.ts @@ -20,6 +20,7 @@ import { runHermesPortableOnboardingTransaction } from "../../src/lib/onboard/ex import { getHermesPortableSandboxRuntimeRegistryFields } from "../../src/lib/onboard/sandbox-registry-metadata"; import { resolveSandboxGpuConfig } from "../../src/lib/onboard/sandbox-gpu-mode"; import { completeHermesPortableSandboxRegistration } from "../../src/lib/onboard/sandbox-create/orchestration"; +import { pendingSandboxPolicyVerificationForBoundary } from "../../src/lib/onboard/sandbox-create/policy-creation-receipt"; import { materializeHermesPortableCreatePlan } from "../../src/lib/onboard/sandbox-create-plan-materialization"; import { resolveSandboxCreateIntent } from "../../src/lib/onboard/sandbox-create-intent"; import { createPortableOnboardEnvironmentScope } from "../../src/lib/onboard/session-bootstrap"; @@ -189,6 +190,7 @@ describe("Hermes portable installer admission", testTimeoutOptions(60_000), () = vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "portable"); const session = createSession(); + const lifecycleGeneration = "11111111-1111-4111-8111-111111111111"; expect( registry.reserveSandboxInferenceRoute(sandboxName, { provider: BUILD_SETTINGS.provider, @@ -248,7 +250,7 @@ describe("Hermes portable installer admission", testTimeoutOptions(60_000), () = const transactionInput = { sandboxName, gatewayName, - lifecycleGeneration: "generation-1", + lifecycleGeneration, runtimeAuthority, openshellExecutableAuthority: hermesPortableTestOpenShellAuthority(), stateDir, @@ -278,14 +280,62 @@ describe("Hermes portable installer admission", testTimeoutOptions(60_000), () = expect(argv[argv.indexOf("--policy") + 1]).not.toBe(basePolicyPath); return { ready: true }; }, - registerSandbox: (_created, receipt, liveIdentityFingerprint, revalidate, reservation) => - completeHermesPortableSandboxRegistration({ + registerSandbox: async ( + _created, + receipt, + liveIdentityFingerprint, + revalidate, + reservation, + ) => { + expect(revalidate()).toBe(liveIdentityFingerprint); + const createReservation = registry.qualifyPendingSandboxCreateReservation( + { + sandboxName, + gatewayName, + sessionId: session.sessionId, + selection, + }, + registry.getSandbox(sandboxName), + ); + expect(reservation.authority).toEqual(createReservation.authority); + const policyCreationReceipt = { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName, + gatewayPort: 8080, + sandboxName, + lifecycleGeneration: receipt.lifecycleGeneration, + sandboxIdentityFingerprint: liveIdentityFingerprint, + policyHash: "sha256:portable-installer-policy", + policyVersion: 1, + }; + const verifiedPolicyBoundary = { + registration: { + policyAuthority: "nemoclaw-managed" as const, + policyCreationReceipt, + observedPolicyAuthority: "owner-unknown" as const, + }, + sandboxName, + gatewayName, + gatewayPort: 8080, + lifecycleGeneration: receipt.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: liveIdentityFingerprint, + route: "native" as const, + }; + const checkpoint = pendingSandboxPolicyVerificationForBoundary( + verifiedPolicyBoundary, + ); + registry.recordPendingSandboxPolicyVerification(createReservation, checkpoint); + registry.requireCurrentPendingSandboxPolicyVerification( + createReservation, + checkpoint, + ); + return completeHermesPortableSandboxRegistration({ sandboxName, completeRegistration: async () => { - expect(revalidate()).toBe(liveIdentityFingerprint); registerCreatedSandbox({ sandboxName, - inferenceSelection: reservation.authority.selection, + inferenceSelection: createReservation.authority.selection, runtimeFields: getHermesPortableSandboxRuntimeRegistryFields( sandboxGpuConfig, receipt.openshellExecutableAuthority.version, @@ -303,11 +353,15 @@ describe("Hermes portable installer admission", testTimeoutOptions(60_000), () = lifecycleLiveIdentityFingerprint: liveIdentityFingerprint, gatewayName, gatewayPort: 8080, - inferenceRouteReservation: reservation, + policyAuthority: "nemoclaw-managed", + policyCreationReceipt, + inferenceRouteReservation: createReservation, + verifiedCreate: { reservation: createReservation, checkpoint }, }); }, readRegistry: registry.getSandbox, - }), + }); + }, }); const completed = await runHermesPortableOnboardingTransaction( @@ -323,7 +377,7 @@ describe("Hermes portable installer admission", testTimeoutOptions(60_000), () = agent: "hermes", endpointSource: null, gatewayName, - lifecycleGeneration: "generation-1", + lifecycleGeneration, lifecycleLiveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY, }); expect(registered).not.toHaveProperty("pendingRouteReservation"); diff --git a/test/mcp/mcp-policy-key-ownership.test.ts b/test/mcp/mcp-policy-key-ownership.test.ts index b06ecae6c3b..0e714e1ad94 100644 --- a/test/mcp/mcp-policy-key-ownership.test.ts +++ b/test/mcp/mcp-policy-key-ownership.test.ts @@ -8,14 +8,23 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { + managedPolicyMetadata, + managedRegistrationSource, + managedSandboxEntry, + SANDBOX_ID, +} from "../helpers/managed-policy-receipt-fixture"; + const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.106"); const MATCHING_OPENSHELL_VERSION_CLAUSE = `if [ "$1" = "--version" ]; then printf '%s\\n' 'openshell 0.0.106'; exit 0; fi`; -const MANAGED_POLICY_AUTHORITY_CLAUSE = `if [ "$1 $2" = "policy get" ]; then +const MANAGED_POLICY_AUTHORITY_CLAUSE = `if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: alpha\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi +if [ "$1 $2" = "policy get" ]; then case " $* " in *" --output json "*) - sandbox="" - for sandbox in "$@"; do :; done - printf '{"scope":"sandbox","sandbox":"%s","status":"effective","policy_source":"sandbox","policy":{}}\\n' "$sandbox" + printf '%s\n' ${JSON.stringify(managedPolicyMetadata("alpha"))} exit 0 ;; esac @@ -34,6 +43,7 @@ function runApply( const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-owner-")); const binDir = path.join(home, ".local", "bin"); const callsPath = path.join(home, "calls.log"); + const appliedPolicyPath = path.join(home, "applied-policy.yaml"); fs.mkdirSync(binDir, { recursive: true }); fs.writeFileSync( path.join(binDir, "openshell"), @@ -42,12 +52,25 @@ ${MATCHING_OPENSHELL_VERSION_CLAUSE} ${MANAGED_POLICY_AUTHORITY_CLAUSE} printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} if [ "$1 $2" = "policy get" ]; then + if [ -f ${JSON.stringify(appliedPolicyPath)} ]; then + cat ${JSON.stringify(appliedPolicyPath)} + exit 0 + fi printf 'Version: 1\nHash: test\n---\nversion: 1\n${ liveName === null ? "network_policies: {}" : `network_policies:\n example:\n name: ${liveName}\n endpoints: []` }\n' fi +if [ "$1 $2" = "policy set" ]; then + while [ "$#" -gt 0 ]; do + if [ "$1" = "--policy" ]; then + cp "$2" ${JSON.stringify(appliedPolicyPath)} + break + fi + shift + done +fi exit 0 `, { mode: 0o755 }, @@ -55,7 +78,7 @@ exit 0 const script = ` const registry = require("./src/lib/state/registry.js"); const policies = require("./src/lib/policy/index.js"); -registry.registerSandbox({ name: "alpha" }); +${managedRegistrationSource("alpha")} const result = policies.applyPresetContent( "alpha", "mcp-bridge-example", @@ -136,7 +159,7 @@ exit 0 const script = ` const registry = require("./src/lib/state/registry.js"); const policies = require("./src/lib/policy/index.js"); -registry.registerSandbox({ name: "alpha" }); +${managedRegistrationSource("alpha")} ${ operation === "remove" ? `registry.addCustomPolicy("alpha", { @@ -182,6 +205,7 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ function runSuccessfulPolicyRemoval(skipRegistryUpdate: boolean) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-remove-success-")); const binDir = path.join(home, ".local", "bin"); + const appliedPolicyPath = path.join(home, "applied-policy.yaml"); fs.mkdirSync(binDir, { recursive: true }); fs.writeFileSync( path.join(binDir, "openshell"), @@ -189,8 +213,21 @@ function runSuccessfulPolicyRemoval(skipRegistryUpdate: boolean) { ${MATCHING_OPENSHELL_VERSION_CLAUSE} ${MANAGED_POLICY_AUTHORITY_CLAUSE} if [ "$1 $2" = "policy get" ]; then + if [ -f ${JSON.stringify(appliedPolicyPath)} ]; then + cat ${JSON.stringify(appliedPolicyPath)} + exit 0 + fi printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n example:\n name: generated-policy\n endpoints: []\n' fi +if [ "$1 $2" = "policy set" ]; then + while [ "$#" -gt 0 ]; do + if [ "$1" = "--policy" ]; then + cp "$2" ${JSON.stringify(appliedPolicyPath)} + break + fi + shift + done +fi exit 0 `, { mode: 0o755 }, @@ -198,7 +235,7 @@ exit 0 const script = ` const registry = require("./src/lib/state/registry.js"); const policies = require("./src/lib/policy/index.js"); -registry.registerSandbox({ name: "alpha" }); +${managedRegistrationSource("alpha")} registry.addCustomPolicy("alpha", { name: "mcp-bridge-example", content: ${JSON.stringify(PRESET)}, @@ -360,7 +397,7 @@ processRecovery.executeSandboxExecCommand = () => ({ stdout: "", stderr: "", }); -registry.registerSandbox({ name: "alpha", agent: "openclaw" }); +${managedRegistrationSource("alpha")} const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); bridge.addMcpBridge("alpha", { server: "example", @@ -471,7 +508,7 @@ processRecovery.executeSandboxExecCommand = () => ({ stdout: "", stderr: "", }); -registry.registerSandbox({ name: "alpha", agent: "openclaw" }); +${managedRegistrationSource("alpha")} registry.addCustomPolicy = () => { throw new Error("injected registry write failure"); }; const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); bridge.addMcpBridge("alpha", { @@ -570,9 +607,7 @@ const entry = { addedAt: "2026-06-01T00:00:00.000Z", }; registry.registerSandbox({ - name: "alpha", - agent: "openclaw", - gatewayName: "nemoclaw", + ...${JSON.stringify(managedSandboxEntry("alpha"))}, mcp: { bridges: { example: entry } }, }); registry.addCustomPolicy("alpha", { diff --git a/test/networking/registry-host-local-inference.test.ts b/test/networking/registry-host-local-inference.test.ts index 21e7b807256..8b5fe6283d6 100644 --- a/test/networking/registry-host-local-inference.test.ts +++ b/test/networking/registry-host-local-inference.test.ts @@ -17,6 +17,85 @@ process.env.HOME = tmpDir; const registry = await import("../../src/lib/state/registry"); const regFile = path.join(tmpDir, ".nemoclaw", "sandboxes.json"); +const CREATE_SESSION_ID = "host-local-registry-fixture"; +const LIFECYCLE_GENERATION = "123e4567-e89b-42d3-a456-426614174983"; +const SANDBOX_IDENTITY_FINGERPRINT = "a".repeat(64); + +function prepareVerifiedCreate( + name: string, + route: { + provider: string; + model: string; + endpointUrl: string | null; + endpointSource: "inference-set"; + credentialEnv: string | null; + preferredInferenceApi: string | null; + gatewayName: string; + gatewayPort: number; + }, +) { + registry.reserveSandboxInferenceRoute(name, { + ...route, + reservationSessionId: CREATE_SESSION_ID, + }); + const authority = { + sandboxName: name, + gatewayName: route.gatewayName, + sessionId: CREATE_SESSION_ID, + selection: { + provider: route.provider, + model: route.model, + endpointUrl: route.endpointUrl, + endpointSource: route.endpointSource, + credentialEnv: route.credentialEnv, + preferredInferenceApi: route.preferredInferenceApi, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + nimContainer: null, + }, + }; + const reservation = registry.qualifyPendingSandboxCreateReservation( + authority, + registry.getSandbox(name), + ); + const policyCreationReceipt = { + schemaVersion: 1 as const, + origin: "sandbox-create" as const, + gatewayName: route.gatewayName, + gatewayPort: route.gatewayPort, + sandboxName: name, + lifecycleGeneration: LIFECYCLE_GENERATION, + sandboxIdentityFingerprint: SANDBOX_IDENTITY_FINGERPRINT, + policyHash: "sha256:host-local-fixture", + policyVersion: 1, + }; + const checkpoint = { + schemaVersion: 1 as const, + state: "verified-create" as const, + policyAuthority: "nemoclaw-managed" as const, + observedPolicyAuthority: "owner-unknown" as const, + gatewayName: route.gatewayName, + gatewayPort: route.gatewayPort, + sandboxName: name, + lifecycleGeneration: LIFECYCLE_GENERATION, + sandboxIdentityFingerprint: SANDBOX_IDENTITY_FINGERPRINT, + route: "none" as const, + policyHash: policyCreationReceipt.policyHash, + policyVersion: policyCreationReceipt.policyVersion, + policyCreationReceipt, + }; + registry.recordPendingSandboxPolicyVerification(reservation, checkpoint); + return { + checkpoint, + registration: { + lifecycleGeneration: LIFECYCLE_GENERATION, + lifecycleLiveIdentityFingerprint: SANDBOX_IDENTITY_FINGERPRINT, + policyAuthority: "nemoclaw-managed" as const, + policyCreationReceipt, + }, + reservation, + }; +} beforeEach(() => { fs.rmSync(regFile, { force: true }); @@ -63,7 +142,7 @@ describe("registry host-local inference authority", () => { it("round-trips explicit llama.cpp lifecycle provenance only through its exact reservation", () => { const receipt = serializedLlamaCppHostLocalInferenceReceipt(); const provenance = createSandboxHostLocalInferenceProvenance("llama-owner", receipt); - registry.reserveSandboxInferenceRoute("llama-clone", { + const route = { provider: "llama-cpp-local", model: "nemotron-llama-cpp", endpointUrl: "https://inference.local/v1", @@ -75,21 +154,17 @@ describe("registry host-local inference authority", () => { openshellDriver: "docker", hostLocalInferenceReceipt: receipt, hostLocalInferenceProvenance: provenance, - }); - registry.registerSandbox({ - name: "llama-clone", - openshellDriver: "docker", - provider: "llama-cpp-local", - model: "nemotron-llama-cpp", - endpointUrl: "https://inference.local/v1", - endpointSource: "inference-set" as const, - credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", - preferredInferenceApi: "openai-completions", - gatewayName: "nemoclaw", - gatewayPort: 8080, - hostLocalInferenceReceipt: receipt, - hostLocalInferenceProvenance: provenance, - }); + } as const; + const verified = prepareVerifiedCreate("llama-clone", route); + registry.registerSandbox( + { + name: "llama-clone", + ...route, + ...verified.registration, + }, + verified.reservation, + { verifiedCreate: verified }, + ); expect(registry.getSandbox("llama-clone")).toMatchObject({ hostLocalInferenceReceipt: receipt, @@ -107,7 +182,7 @@ describe("registry host-local inference authority", () => { ["inference API", { preferredInferenceApi: "openai-responses" }], ["gateway", { gatewayName: "nemoclaw-8090" }], ["gateway port", { gatewayPort: 8090 }], - ] as const)("rejects %s drift at explicit llama.cpp registration CAS", (_label, drift) => { + ] as const)("rejects %s drift at explicit llama.cpp registration CAS", (label, drift) => { const receipt = serializedLlamaCppHostLocalInferenceReceipt(); const provenance = createSandboxHostLocalInferenceProvenance("llama-owner", receipt); const route = { @@ -123,15 +198,22 @@ describe("registry host-local inference authority", () => { hostLocalInferenceReceipt: receipt, hostLocalInferenceProvenance: provenance, }; - registry.reserveSandboxInferenceRoute("llama-drift", route); + const verified = prepareVerifiedCreate("llama-drift", route); expect(() => - registry.registerSandbox({ - name: "llama-drift", - ...route, - ...drift, - }), - ).toThrow(/reservation changed/); + registry.registerSandbox( + { + name: "llama-drift", + ...route, + ...verified.registration, + ...drift, + }, + verified.reservation, + { verifiedCreate: verified }, + ), + ).toThrow( + label === "gateway port" ? /policy creation receipt does not match/u : /reservation changed/u, + ); }); it("rejects receipt or original-owner drift at explicit llama.cpp registration CAS", () => { @@ -150,31 +232,41 @@ describe("registry host-local inference authority", () => { hostLocalInferenceReceipt: receipt, hostLocalInferenceProvenance: provenance, }; - registry.reserveSandboxInferenceRoute("llama-owner-drift", route); + const ownerVerified = prepareVerifiedCreate("llama-owner-drift", route); expect(() => - registry.registerSandbox({ - name: "llama-owner-drift", - ...route, - hostLocalInferenceProvenance: createSandboxHostLocalInferenceProvenance( - "different-owner", - receipt, - ), - }), + registry.registerSandbox( + { + name: "llama-owner-drift", + ...route, + ...ownerVerified.registration, + hostLocalInferenceProvenance: createSandboxHostLocalInferenceProvenance( + "different-owner", + receipt, + ), + }, + ownerVerified.reservation, + { verifiedCreate: ownerVerified }, + ), ).toThrow(/reservation changed/); fs.rmSync(regFile, { force: true }); - registry.reserveSandboxInferenceRoute("llama-receipt-drift", route); + const receiptVerified = prepareVerifiedCreate("llama-receipt-drift", route); const changedReceipt = serializedLlamaCppHostLocalInferenceReceipt("mxc"); expect(() => - registry.registerSandbox({ - name: "llama-receipt-drift", - ...route, - hostLocalInferenceReceipt: changedReceipt, - hostLocalInferenceProvenance: createSandboxHostLocalInferenceProvenance( - "llama-owner", - changedReceipt, - ), - }), + registry.registerSandbox( + { + name: "llama-receipt-drift", + ...route, + ...receiptVerified.registration, + hostLocalInferenceReceipt: changedReceipt, + hostLocalInferenceProvenance: createSandboxHostLocalInferenceProvenance( + "llama-owner", + changedReceipt, + ), + }, + receiptVerified.reservation, + { verifiedCreate: receiptVerified }, + ), ).toThrow(/reservation changed/); }); diff --git a/test/onboard-external-policy-authority-composition.test.ts b/test/onboard-external-policy-authority-composition.test.ts index 5e01b9df23c..4219b1d50b7 100644 --- a/test/onboard-external-policy-authority-composition.test.ts +++ b/test/onboard-external-policy-authority-composition.test.ts @@ -77,8 +77,12 @@ describe("external policy authority onboarding composition", () => { const externalInspection = { authority: "externally-managed" as const, effectivePolicy, + policyIdentity: { hash: "sha256:external", activeVersion: 1 }, }; - const inspectGlobalPolicyAuthority = vi.fn(() => externalInspection); + const inspectActiveGlobalPolicy = vi.fn(() => ({ + state: "active" as const, + inspection: externalInspection, + })); const inspectSandboxPolicyAuthority = vi.fn(() => externalInspection); let liveExists = false; let existingEntry: SandboxEntry | null = null; @@ -87,12 +91,13 @@ describe("external policy authority onboarding composition", () => { mutator(durableSession); return durableSession; }); + const getAgentPolicyPath = vi.fn(() => basePolicyPath); const bindings = createOnboardPolicyAuthorityBindings( { GATEWAY_NAME: gatewayName, ROOT: REPO_ROOT, agentDefs: { loadAgent }, - agentOnboard: { getAgentPolicyPath: () => basePolicyPath }, + agentOnboard: { getAgentPolicyPath }, inspectSandboxForCreate: () => ({ existingEntry, liveExists }), onboardSession: { loadSession: () => durableSession, @@ -100,7 +105,7 @@ describe("external policy authority onboarding composition", () => { }, }, policyTier, - { inspectGlobalPolicyAuthority, inspectSandboxPolicyAuthority }, + { inspectActiveGlobalPolicy, inspectSandboxPolicyAuthority }, ); durableSession = await bindings.bindPolicyAuthority(gatewayName, durableSession); @@ -120,8 +125,12 @@ describe("external policy authority onboarding composition", () => { operation: `prepare sandbox '${sandboxName}'`, }; bindings.preflightPolicyRequirements(policyRequirements); - expect(inspectGlobalPolicyAuthority).toHaveBeenCalledTimes(2); + bindings.preflightPolicyRequirements({ ...policyRequirements, agent: null }); + expect(inspectActiveGlobalPolicy).toHaveBeenCalledTimes(3); expect(inspectSandboxPolicyAuthority).not.toHaveBeenCalled(); + expect(getAgentPolicyPath).toHaveBeenLastCalledWith( + expect.objectContaining({ name: "openclaw" }), + ); const intent = resolveSandboxCreateIntent({ basePolicyPath, @@ -179,7 +188,7 @@ describe("external policy authority onboarding composition", () => { sandboxName, revalidate: (_sandboxIsLive, operation) => bindings.preflightPolicyRequirements({ ...policyRequirements, operation }), - create: async () => { + create: async (verifyCreatedSandbox) => { liveExists = true; existingEntry = { name: sandboxName, @@ -187,13 +196,14 @@ describe("external policy authority onboarding composition", () => { policyTier, policies: [], } as SandboxEntry; + await verifyCreatedSandbox(launch); return launch; }, - captureCreatedSandboxLiveIdentity: () => "a".repeat(64), - observeCreatedSandbox: () => ({ - state: "ready" as const, - liveIdentityFingerprint: "a".repeat(64), - }), + captureCreatedSandboxIdentity: () => "a".repeat(64), + revalidateCreatedSandboxIdentity: vi.fn(), + verifyCreatedPolicy: () => "verified", + persistVerifiedPolicy: vi.fn(), + revalidateVerifiedPolicy: vi.fn(), cleanupTemporarySources: vi.fn(), }), ).resolves.toBe(launch); diff --git a/test/onboarding/onboard-custom-dockerfile.test.ts b/test/onboarding/onboard-custom-dockerfile.test.ts index c3d850aea1f..b1d970dc600 100644 --- a/test/onboarding/onboard-custom-dockerfile.test.ts +++ b/test/onboarding/onboard-custom-dockerfile.test.ts @@ -191,6 +191,7 @@ describe("onboard custom Dockerfile", () => { const script = String.raw` const runner = require(${runnerPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -230,28 +231,36 @@ fs.statSync = (target, ...rest) => { runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); + const profileResult = fixtureMocks.mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); +fixtureMocks.mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + const normalized = _n(command); + if (normalized.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (normalized.includes("policy get") && normalized.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { + sandboxName: "my-assistant", + }); + if (createdIdentity !== null) return createdIdentity; + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; + if (normalized.includes("sandbox list")) return "my-assistant Ready"; { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (normalized.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "openai-api", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -297,7 +306,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "openai-api", null, "my-assistant", null, null, ${customDockerfilePath}); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "openai-api", null, "my-assistant", null, null, ${customDockerfilePath}, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, hasExtraFile: hasExtraFileAtSpawn, stagedIgnoredFiles: stagedIgnoredFilesAtSpawn, stagedDockerignoreFiles: stagedDockerignoreFilesAtSpawn })); })().catch((error) => { console.error(error); diff --git a/test/onboarding/onboard-extra-provider-reconciliation.test.ts b/test/onboarding/onboard-extra-provider-reconciliation.test.ts index 6ea7cd36068..91a384920f3 100644 --- a/test/onboarding/onboard-extra-provider-reconciliation.test.ts +++ b/test/onboarding/onboard-extra-provider-reconciliation.test.ts @@ -17,34 +17,47 @@ const onboardScriptMocksPath = JSON.stringify( ); describe("onboard extra-provider reconciliation", () => { - it("attaches live user extras, prunes stale names, and converges registry state (#6501)", { - timeout: 90_000, - }, () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-extra-provider-reconcile-")); - try { - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "reconcile-provider-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const sandboxBaseImagePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "sandbox-base-image.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin, { readySandboxGet: true }); - - const script = String.raw` -const runner = require(${runnerPath}); + it( + "attaches live user extras, prunes stale names, and converges registry state (#6501)", + { + timeout: 90_000, + }, + () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-extra-provider-reconcile-")); + try { + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "reconcile-provider-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const sandboxBaseImagePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "sandbox-base-image.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); + + const script = String.raw` const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +registry.addExtraProvider("tavily-search"); +registry.addExtraProvider("brave-search"); +registry.addExtraProvider("custom-provider"); +registry.addExtraProvider("my-slack-bridge"); +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); +const runner = require(${runnerPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const sandboxBaseImage = require(${sandboxBaseImagePath}); @@ -53,10 +66,6 @@ const { EventEmitter } = require("node:events"); const _n = (command) => (Array.isArray(command) ? command.join(" ") : String(command)).replace(/'/g, ""); const commands = []; -registry.addExtraProvider("tavily-search"); -registry.addExtraProvider("brave-search"); -registry.addExtraProvider("custom-provider"); -registry.addExtraProvider("my-slack-bridge"); runner.run = (command, opts = {}) => { const normalized = _n(command); @@ -83,6 +92,8 @@ runner.run = (command, opts = {}) => { require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { const normalized = _n(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; if (normalized.includes("sandbox list")) return "my-assistant Ready"; const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -92,10 +103,6 @@ runner.runCapture = (command) => { } return ""; }; -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; sandboxBaseImage.resolveSandboxBaseImage = () => ({ @@ -124,11 +131,29 @@ childProcess.spawn = (...args) => { const { createSandbox } = require(${onboardPath}); +const createReservedSandbox = () => createSandbox( + null, + "gpt-5.4", + "nvidia-prod", + null, + null, + null, + null, + null, + null, + null, + null, + null, + [], + null, + { sessionId: createFixture.sessionId }, +); + (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; const sandboxNames = [ - await createSandbox(null, "gpt-5.4", "nvidia-prod"), - await createSandbox(null, "gpt-5.4", "nvidia-prod"), + await createReservedSandbox(), + await createReservedSandbox(), ]; console.log(JSON.stringify({ sandboxNames, @@ -140,83 +165,84 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - timeout: 30_000, - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", - NEMOCLAW_SANDBOX_PREBUILD: "1", - OPENSHELL_GATEWAY_ENDPOINT: undefined, - }, - }); - - assert.equal(result.status, 0, result.stderr || result.error?.message); - const payloadLine = result.stdout - .trim() - .split("\n") - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - assert.deepEqual(payload.sandboxNames, ["my-assistant", "my-assistant"]); - - const createCommands = payload.commands.filter((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.equal(createCommands.length, 2, "expected one sandbox create command per attempt"); - const [createCommand, retryCreateCommand] = createCommands; - assert.match(createCommand.command, /--provider brave-search/); - assert.match(createCommand.command, /--provider custom-provider/); - assert.match(createCommand.command, /--provider my-slack-bridge/); - assert.doesNotMatch(createCommand.command, /--provider tavily-search/); - assert.deepEqual( - createCommand.command.match(/--provider\s+\S+/g), - retryCreateCommand.command.match(/--provider\s+\S+/g), - "retry must preserve the exact filtered provider arguments", - ); - - const providerProbes = payload.commands.filter((entry: CommandEntry) => - entry.command.includes("provider get -g nemoclaw "), - ); - assert.deepEqual( - providerProbes - .map((entry: CommandEntry) => - entry.command.slice(entry.command.indexOf("provider get -g nemoclaw ")), - ) - .sort(), - [ - "provider get -g nemoclaw brave-search", - "provider get -g nemoclaw brave-search", - "provider get -g nemoclaw custom-provider", - "provider get -g nemoclaw custom-provider", - "provider get -g nemoclaw my-slack-bridge", - "provider get -g nemoclaw my-slack-bridge", - "provider get -g nemoclaw nvidia-prod", - "provider get -g nemoclaw nvidia-prod", - "provider get -g nemoclaw tavily-search", - ].sort(), - ); - assert.equal( - payload.commands.some((entry: CommandEntry) => - entry.command.includes("provider list -g nemoclaw --names"), - ), - false, - "provider-list snapshots must not control extra-provider attachment", - ); - assert.deepEqual(payload.extraProviders, [ - "brave-search", - "custom-provider", - "my-slack-bridge", - ]); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + timeout: 30_000, + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", + NEMOCLAW_SANDBOX_PREBUILD: "1", + OPENSHELL_GATEWAY_ENDPOINT: undefined, + }, + }); + + assert.equal(result.status, 0, result.stderr || result.error?.message); + const payloadLine = result.stdout + .trim() + .split("\n") + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + assert.deepEqual(payload.sandboxNames, ["my-assistant", "my-assistant"]); + + const createCommands = payload.commands.filter((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.equal(createCommands.length, 2, "expected one sandbox create command per attempt"); + const [createCommand, retryCreateCommand] = createCommands; + assert.match(createCommand.command, /--provider brave-search/); + assert.match(createCommand.command, /--provider custom-provider/); + assert.match(createCommand.command, /--provider my-slack-bridge/); + assert.doesNotMatch(createCommand.command, /--provider tavily-search/); + assert.deepEqual( + createCommand.command.match(/--provider\s+\S+/g), + retryCreateCommand.command.match(/--provider\s+\S+/g), + "retry must preserve the exact filtered provider arguments", + ); + + const providerProbes = payload.commands.filter((entry: CommandEntry) => + entry.command.includes("provider get -g nemoclaw "), + ); + assert.deepEqual( + providerProbes + .map((entry: CommandEntry) => + entry.command.slice(entry.command.indexOf("provider get -g nemoclaw ")), + ) + .sort(), + [ + "provider get -g nemoclaw brave-search", + "provider get -g nemoclaw brave-search", + "provider get -g nemoclaw custom-provider", + "provider get -g nemoclaw custom-provider", + "provider get -g nemoclaw my-slack-bridge", + "provider get -g nemoclaw my-slack-bridge", + "provider get -g nemoclaw nvidia-prod", + "provider get -g nemoclaw nvidia-prod", + "provider get -g nemoclaw tavily-search", + ].sort(), + ); + assert.equal( + payload.commands.some((entry: CommandEntry) => + entry.command.includes("provider list -g nemoclaw --names"), + ), + false, + "provider-list snapshots must not control extra-provider attachment", + ); + assert.deepEqual(payload.extraProviders, [ + "brave-search", + "custom-provider", + "my-slack-bridge", + ]); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, + ); }); diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts new file mode 100644 index 00000000000..42ba68b31e7 --- /dev/null +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -0,0 +1,247 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { beforeEach, describe, it, vi } from "vitest"; +import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; +import { type CommandEntry, onboardScriptMocksPath } from "../helpers/onboard-split-context"; + +beforeEach(() => { + vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); + vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); +}); + +describe("fresh create identity", () => { + it( + "registers a fresh create only after owner-scoped identity confirmation (#8942)", + { + timeout: 45000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-ready-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-ready-check.js"); + const payloadPath = path.join(tmpDir, "payload.json"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const dockerExecPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "adapters", "docker", "exec.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); +const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; +const registry = require(${registryPath}); +const preflight = require(${preflightPath}); +const credentials = require(${credentialsPath}); +const childProcess = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const dockerExec = require(${dockerExecPath}); +dockerExec.dockerSpawn = () => { + const child = new EventEmitter(); + process.nextTick(() => child.emit("close", 0)); + return child; +}; +const fs = require("node:fs"); + +const commands = []; +const lifecycleObservationCommands = []; +let sandboxListCalls = 0; +let dockerPsCalls = 0; +let sandboxCreated = false; +let registeredSandbox = null; +const keepAlive = setInterval(() => {}, 1000); +runner.run = (command, opts = {}) => { + const cmd = _n(command); + const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); + if (profileResult !== null) return profileResult; + _deleted = _deleted || cmd.includes("sandbox delete"); + commands.push({ command: cmd, env: opts.env || null }); + if (cmd.includes("sandbox list")) { + return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; + } + return cmd.includes("sandbox get") && cmd.includes("my-assistant") && sandboxCreated + ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-fresh-create\n"), stderr: Buffer.alloc(0) } + : { status: 0 }; +}; + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + if (cmd.includes("sandbox get") || cmd.includes("sandbox list")) { + lifecycleObservationCommands.push(cmd); + } + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { + sandboxName: "my-assistant", + sandboxId: "sbx-fresh-create", + }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.startsWith("docker ps -a --no-trunc ")) { + dockerPsCalls += 1; + if (dockerPsCalls === 1) return "a".repeat(64); + } + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) { + return sandboxCreated ? ["my-assistant", "Id: sbx-fresh-create"].join(String.fromCharCode(10)) : ""; + } + if (cmd.includes("sandbox list")) { + sandboxListCalls += 1; + return sandboxListCalls >= 2 ? "my-assistant Ready" : "my-assistant Pending"; + } + { + const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + if (mockedCapture !== null) return mockedCapture; + } + if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + return ""; +}; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + registerSandbox: (entry) => { registeredSandbox = entry; }, + }); +preflight.checkPortAvailable = async () => ({ ok: true }); +credentials.prompt = async () => ""; + +const groupKillCalls = []; +const realProcessKill = process.kill.bind(process); +process.kill = (pid, signal) => { + if (pid < 0) { + groupKillCalls.push({ pid, signal }); + const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); + process.nextTick(() => createCommand.child.emit("close", signal === "SIGTERM" ? 0 : 1)); + return true; + } + return realProcessKill(pid, signal); +}; + +childProcess.spawn = (...args) => { + sandboxCreated = true; + _deleted = false; + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.unref = () => {}; + child.pid = 4242; + child.killCalls = []; + child.unrefCalls = 0; + child.stdout.destroyCalls = 0; + child.stderr.destroyCalls = 0; + child.stdout.destroy = () => { + child.stdout.destroyCalls += 1; + }; + child.stderr.destroy = () => { + child.stderr.destroyCalls += 1; + }; + child.unref = () => { + child.unrefCalls += 1; + }; + child.kill = (signal) => { + child.killCalls.push(signal); + process.nextTick(() => child.emit("close", signal === "SIGTERM" ? 0 : 1)); + return true; + }; + commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null, child }); + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); + child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); + }); + return child; +}; + +const { createSandbox } = require(${onboardPath}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], + createFixture, + )); + const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); + fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ + sandboxName, + sandboxListCalls, + killCalls: createCommand.child.killCalls, + groupKillCalls, + unrefCalls: createCommand.child.unrefCalls, + stdoutDestroyCalls: createCommand.child.stdout.destroyCalls, + stderrDestroyCalls: createCommand.child.stderr.destroyCalls, + lifecycleObservationCommands, + registeredSandbox, + })); + clearInterval(keepAlive); +})().catch((error) => { + clearInterval(keepAlive); + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + OPENSHELL_DRIVERS: "docker", + }, + timeout: 30000, + }); + + assert.equal(result.status, 0, result.stderr); + const payload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); + assert.equal(payload.sandboxName, "my-assistant"); + assert.ok(payload.sandboxListCalls >= 2); + assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); + assert.deepEqual(payload.killCalls, []); + assert.equal(payload.unrefCalls, 1); + assert.equal(payload.stdoutDestroyCalls, 1); + assert.equal(payload.stderrDestroyCalls, 1); + assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); + assert.equal( + payload.registeredSandbox.lifecycleLiveIdentityFingerprint, + createHash("sha256").update("sbx-fresh-create").digest("hex"), + ); + const ownerScopedObservations = payload.lifecycleObservationCommands.filter( + (command: string) => command.includes("-g nemoclaw"), + ); + assert.ok( + ownerScopedObservations.length >= 6, + "expected owner-scoped sandbox identity observations", + ); + assert.ok( + ownerScopedObservations.every( + (command: string) => + command.includes("sandbox get -g nemoclaw my-assistant") || + command.includes("sandbox list -g nemoclaw"), + ), + `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, + ); + }, + ); +}); diff --git a/test/onboarding/onboard-installer-restore-intent.test.ts b/test/onboarding/onboard-installer-restore-intent.test.ts index db79ddf3edc..308eded935a 100644 --- a/test/onboarding/onboard-installer-restore-intent.test.ts +++ b/test/onboarding/onboard-installer-restore-intent.test.ts @@ -29,28 +29,34 @@ describe("createSandbox installer restore intent", () => { } }); - it("non-interactive not-ready sandbox with installer restore intent skips the fresh backup, restores the pre-upgrade backup, and stays exec-usable for a workspace marker (#6114)", { - timeout: 60_000, - }, async () => { - const tmpDir = makeTmpDir("nemoclaw-onboard-installer-restore-"); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "installer-restore.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const sandboxStatePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), - ); - const execActionPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "actions", "sandbox", "exec.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` + it( + "non-interactive not-ready sandbox with installer restore intent skips the fresh backup, restores the pre-upgrade backup, and stays exec-usable for a workspace marker (#6114)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = makeTmpDir("nemoclaw-onboard-installer-restore-"); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "installer-restore.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const sandboxStatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), + ); + const execActionPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "actions", "sandbox", "exec.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); + + const script = String.raw` const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); +const fixtureMocks = require(${onboardScriptMocksPath}); +fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); @@ -64,7 +70,7 @@ let sandboxRecreated = false; runner.run = (command) => { const cmd = _n(command); events.push({ kind: "run", cmd }); - const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); + const profileResult = fixtureMocks.mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; if (cmd.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; if (cmd.includes("sandbox delete")) sandboxDeleted = true; @@ -72,34 +78,42 @@ runner.run = (command) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("my-assistant\nId: fixture-created-sandbox\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); +fixtureMocks.mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + if (sandboxRecreated) { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); + if (createdIdentity !== null) return createdIdentity; + } + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command, { defaultCurlOutput: "ok", }); if (mockedCapture !== null) return mockedCapture; } return ""; }; -registry.getSandbox = () => ({ +const sourceEntry = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", - gpuEnabled: false, policyAuthority: "nemoclaw-managed", - policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive", + gpuEnabled: false, + toolDisclosure: "progressive", +}); +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: () => sourceEntry, }); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; sandboxState.getLatestBackup = (name) => { events.push({ kind: "getLatestBackup", name }); @@ -155,7 +169,10 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 process.env.OPENSHELL_GATEWAY = "nemoclaw"; delete process.env.NEMOCLAW_RECREATE_SANDBOX; process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); // Prove the recreated + restored sandbox is reachable through the real // "nemoclaw exec" boundary and can read a preserved workspace marker. @@ -185,100 +202,102 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const env: Record = { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", - NEMOCLAW_SANDBOX_PREBUILD: "1", - }; - delete env["NEMOCLAW_RECREATE_SANDBOX"]; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env, - timeout: 30_000, - }); + fs.writeFileSync(scriptPath, script); - assert.equal(result.status, 0, result.stderr || result.error?.message); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - - assert.equal( - payload.sandboxName, - "my-assistant", - "should recreate and return the sandbox name", - ); - - const events = payload.events as Array<{ - kind: string; - cmd?: string; - name?: string; - backupPath?: string; - marker?: string | null; - }>; - const getLatestIndex = events.findIndex((e) => e.kind === "getLatestBackup"); - const deleteIndex = events.findIndex( - (e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete"), - ); - const restoreIndex = events.findIndex((e) => e.kind === "restore"); - - assert.ok(getLatestIndex >= 0, "should consult the latest pre-upgrade backup"); - assert.ok( - !events.some((e) => e.kind === "backup"), - "should skip the fresh pre-recreate backup when a pre-upgrade backup is being restored", - ); - assert.ok(deleteIndex >= 0, "should delete the not-ready sandbox before recreating"); - assert.ok(restoreIndex > deleteIndex, "restore must happen after sandbox recreate"); - assert.equal( - events[restoreIndex]?.backupPath, - "/tmp/fake-pre-upgrade-backup", - "should restore from the selected pre-upgrade backup rather than a fresh backup", - ); - - const execIndex = events.findIndex((e) => e.kind === "exec"); - assert.ok(execIndex > restoreIndex, "exec marker read must happen after restore"); - assert.equal( - events[execIndex]?.marker, - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "nemoclaw exec should read the preserved workspace marker after restore", - ); - assert.equal(payload.execCode, 0, "nemoclaw exec of the workspace marker should succeed"); - }); + const env: Record = { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", + NEMOCLAW_SANDBOX_PREBUILD: "1", + }; + delete env["NEMOCLAW_RECREATE_SANDBOX"]; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env, + timeout: 30_000, + }); + + assert.equal(result.status, 0, result.stderr || result.error?.message); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + + assert.equal( + payload.sandboxName, + "my-assistant", + "should recreate and return the sandbox name", + ); + + const events = payload.events as Array<{ + kind: string; + cmd?: string; + name?: string; + backupPath?: string; + marker?: string | null; + }>; + const getLatestIndex = events.findIndex((e) => e.kind === "getLatestBackup"); + const deleteIndex = events.findIndex( + (e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete"), + ); + const restoreIndex = events.findIndex((e) => e.kind === "restore"); + + assert.ok(getLatestIndex >= 0, "should consult the latest pre-upgrade backup"); + assert.ok( + !events.some((e) => e.kind === "backup"), + "should skip the fresh pre-recreate backup when a pre-upgrade backup is being restored", + ); + assert.ok(deleteIndex >= 0, "should delete the not-ready sandbox before recreating"); + assert.ok(restoreIndex > deleteIndex, "restore must happen after sandbox recreate"); + assert.equal( + events[restoreIndex]?.backupPath, + "/tmp/fake-pre-upgrade-backup", + "should restore from the selected pre-upgrade backup rather than a fresh backup", + ); + + const execIndex = events.findIndex((e) => e.kind === "exec"); + assert.ok(execIndex > restoreIndex, "exec marker read must happen after restore"); + assert.equal( + events[execIndex]?.marker, + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "nemoclaw exec should read the preserved workspace marker after restore", + ); + assert.equal(payload.execCode, 0, "nemoclaw exec of the workspace marker should succeed"); + }, + ); it.each([ { change: "changes", race: "changed", error: /source registry row changed/u }, { change: "is removed", race: "removed", error: /source registry row is absent/u }, - ])("rejects installer restore when the source registry row $change after journal capture (#7736)", async ({ - race, - error, - }) => { - const tmpDir = makeTmpDir("nemoclaw-onboard-registry-race-"); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "registry-race.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const sandboxStatePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), - ); - const onboardSessionPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` + ])( + "rejects installer restore when the source registry row $change after journal capture (#7736)", + async ({ race, error }) => { + const tmpDir = makeTmpDir("nemoclaw-onboard-registry-race-"); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "registry-race.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const sandboxStatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), + ); + const onboardSessionPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` const runner = require(${runnerPath}); require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const normalize = (command) => @@ -350,56 +369,63 @@ const { createSandbox } = require(${onboardPath}); } })(); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", - NEMOCLAW_TEST_REGISTRY_RACE: race, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, "expected the registry-race fixture to report its result"); - const payload = JSON.parse(payloadLine) as { error: string | null; mutations: string[] }; - assert.match(payload.error ?? "", error); - assert.deepEqual( - payload.mutations, - [], - "registry drift must stop before backup lookup or an external mutation", - ); - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE: "1", + NEMOCLAW_TEST_REGISTRY_RACE: race, + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, "expected the registry-race fixture to report its result"); + const payload = JSON.parse(payloadLine) as { error: string | null; mutations: string[] }; + assert.match(payload.error ?? "", error); + assert.deepEqual( + payload.mutations, + [], + "registry drift must stop before backup lookup or an external mutation", + ); + }, + ); - it("non-interactive not-ready sandbox without installer restore intent exits before any sandbox delete (#6114)", { - timeout: 60_000, - }, async () => { - const tmpDir = makeTmpDir("nemoclaw-onboard-no-restore-intent-"); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "no-restore-intent.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const sandboxStatePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` + it( + "non-interactive not-ready sandbox without installer restore intent exits before any sandbox delete (#6114)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = makeTmpDir("nemoclaw-onboard-no-restore-intent-"); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "no-restore-intent.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const sandboxStatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); + + const script = String.raw` const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); +const fixtureMocks = require(${onboardScriptMocksPath}); +fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); @@ -412,19 +438,21 @@ runner.run = (command) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); - if (_n(command).includes("sandbox list")) return "my-assistant NotReady"; + const normalized = _n(command); + if (normalized.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (normalized.includes("policy get") && normalized.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); + if (normalized.includes("sandbox list")) return "my-assistant NotReady"; // Keep dashboard allocation inside this restore-intent fixture; host port // occupancy is unrelated to the not-ready decision under test. - if (_n(command).includes("forward list")) { + if (normalized.includes("forward list")) { return "my-assistant 127.0.0.1 18789 12345 running"; } return ""; }; -registry.getSandbox = () => ({ +registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", gpuEnabled: false, - policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive", }); sandboxState.getLatestBackup = () => { @@ -447,44 +475,45 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const env: Record = { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", - }; - delete env["NEMOCLAW_RECREATE_SANDBOX"]; - delete env["NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE"]; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env, - }); + fs.writeFileSync(scriptPath, script); - assert.notEqual( - result.status, - 0, - "expected non-zero exit when installer restore intent is unset", - ); - assert.ok( - !result.stdout.includes("ERROR_DID_NOT_EXIT"), - "should have exited before reaching sandbox create", - ); - const output = (result.stdout || "") + (result.stderr || ""); - assert.ok( - !output.includes("unexpected sandbox delete"), - "should exit before attempting sandbox delete", - ); - assert.ok( - !output.includes("unexpected getLatestBackup"), - "should not consult a pre-upgrade backup without installer restore intent", - ); - assert.ok( - output.includes("--recreate-sandbox") || output.includes("NEMOCLAW_RECREATE_SANDBOX"), - `should hint about --recreate-sandbox flag; output:\n${output}`, - ); - }); + const env: Record = { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", + }; + delete env["NEMOCLAW_RECREATE_SANDBOX"]; + delete env["NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE"]; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env, + }); + + assert.notEqual( + result.status, + 0, + "expected non-zero exit when installer restore intent is unset", + ); + assert.ok( + !result.stdout.includes("ERROR_DID_NOT_EXIT"), + "should have exited before reaching sandbox create", + ); + const output = (result.stdout || "") + (result.stderr || ""); + assert.ok( + !output.includes("unexpected sandbox delete"), + "should exit before attempting sandbox delete", + ); + assert.ok( + !output.includes("unexpected getLatestBackup"), + "should not consult a pre-upgrade backup without installer restore intent", + ); + assert.ok( + output.includes("--recreate-sandbox") || output.includes("NEMOCLAW_RECREATE_SANDBOX"), + `should hint about --recreate-sandbox flag; output:\n${output}`, + ); + }, + ); }); diff --git a/test/onboarding/onboard-mcp-observability-redirect.test.ts b/test/onboarding/onboard-mcp-observability-redirect.test.ts index bc4968cca41..a06f9f4ce95 100644 --- a/test/onboarding/onboard-mcp-observability-redirect.test.ts +++ b/test/onboarding/onboard-mcp-observability-redirect.test.ts @@ -17,7 +17,7 @@ describe("onboard managed MCP recreation redirect", () => { const fakeBin = path.join(tmpDir, "bin"); const scriptPath = path.join(tmpDir, "redirect.js"); fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); @@ -31,6 +31,7 @@ describe("onboard managed MCP recreation redirect", () => { const script = String.raw` const runner = require(${runnerPath}); const registry = require(${registryPath}); +const fixtureMocks = require(${mocksPath}); const normalize = (command) => (Array.isArray(command) ? command.join(" ") : String(command)).replace(/'/g, ""); runner.run = () => ({ status: 0 }); runner.runCapture = (command) => { @@ -43,7 +44,7 @@ runner.runCapture = (command) => { const mocked = require(${mocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok" }); return mocked === null ? "" : mocked; }; -registry.getSandbox = () => ({ +registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "alpha", agent: "langchain-deepagents-code", model: "model", diff --git a/test/onboarding/onboard-messaging-credential-validation.test.ts b/test/onboarding/onboard-messaging-credential-validation.test.ts new file mode 100644 index 00000000000..7392c9f76d5 --- /dev/null +++ b/test/onboarding/onboard-messaging-credential-validation.test.ts @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { beforeEach, describe, it, vi } from "vitest"; + +const repoRoot = path.join(import.meta.dirname, "../.."); + +beforeEach(() => { + vi.stubEnv("NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG", "1"); + vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); +}); + +describe("onboard messaging credential validation", () => { + it( + "interactive setupMessagingChannels drops slack when app token fails appTokenFormat check (#1912)", + { + timeout: 60_000, + }, + async () => { + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-slack-app-format-reject-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "slack-app-format-reject.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + // Subscript: mocks prompt to return a VALID bot token but a bogus app + // token. Expected behavior: bot token passes the regex and persists, + // app token fails the regex, channel is dropped from the enabled set, + // and SLACK_APP_TOKEN is never saved. + const script = String.raw` +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); + +const saveCalls = []; +credentials.saveCredential = (key, value) => { saveCalls.push({ key, value }); }; +credentials.getCredential = () => null; +credentials.prompt = async (message) => { + if (message.includes("Slack Bot Token")) return "xoxb-test-valid-bot-token"; + if (message.includes("Slack App Token")) return "abcd"; + return ""; +}; + +runner.run = () => ({ status: 0 }); +runner.runCapture = () => ""; + +const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); + +(async () => { + delete process.env.TELEGRAM_BOT_TOKEN; + delete process.env.DISCORD_BOT_TOKEN; + delete process.env.SLACK_BOT_TOKEN; + delete process.env.SLACK_APP_TOKEN; + + const result = await setupMessagingChannels(); + console.log(JSON.stringify({ + result, + saveCalls, + slackIndex1Based: MESSAGING_CHANNELS.findIndex((c) => c.name === "slack") + 1, + })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + // Dry run with Enter only to introspect Slack's 1-based digit. + const introspect = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, + input: "\n", + }); + assert.equal(introspect.status, 0, introspect.stderr); + const slackIdx = JSON.parse(introspect.stdout.trim().split("\n").pop()!).slackIndex1Based; + assert.ok(slackIdx >= 1, `unexpected slack index: ${slackIdx}`); + + // Real run: toggle Slack on, exit UI, bot prompt returns valid, app + // prompt returns "abcd", app-token check rejects, channel dropped. + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, + input: `${slackIdx}\n`, + }); + + assert.equal(result.status, 0, result.stderr); + const out = JSON.parse(result.stdout.trim().split("\n").pop()!); + + assert.ok( + !out.result.includes("slack"), + `slack should have been dropped after invalid app token; got ${JSON.stringify(out.result)}`, + ); + assert.ok( + !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_BOT_TOKEN"), + `SLACK_BOT_TOKEN should NOT be persisted until the app token also passes; saveCalls=${JSON.stringify(out.saveCalls)}`, + ); + assert.ok( + !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_APP_TOKEN"), + `SLACK_APP_TOKEN should NOT have been persisted (invalid format); saveCalls=${JSON.stringify(out.saveCalls)}`, + ); + assert.ok( + result.stderr.includes("Invalid format") || result.stdout.includes("Invalid format"), + `expected 'Invalid format' warning; stderr=${result.stderr} stdout=${result.stdout}`, + ); + }, + ); +}); diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index 3a1de449a42..8222c02132d 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -74,6 +74,7 @@ describe("onboard messaging", () => { const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -85,6 +86,8 @@ runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderR readySandboxName: "my-assistant", }); runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("provider get")) return "Provider: discord-bridge"; @@ -101,6 +104,11 @@ registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; childProcess.spawn = (...args) => { @@ -139,7 +147,7 @@ const { createSandbox, setupMessagingChannels } = require(${onboardPath}); process.env.KUBECONFIG = "/tmp/host-kubeconfig"; process.env.SSH_AUTH_SOCK = "/tmp/host-ssh-agent.sock"; await setupMessagingChannels(null, null, "my-assistant"); - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], createFixture)); console.log(JSON.stringify({ sandboxName, commands, @@ -334,6 +342,7 @@ const { createSandbox, setupMessagingChannels } = require(${onboardPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -361,6 +370,8 @@ runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderR readySandboxName: "my-assistant", }); runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; @@ -379,6 +390,12 @@ registry.registerSandbox = (entry) => { registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + registerSandbox: registry.registerSandbox, +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -415,17 +432,7 @@ const { createSandbox } = require(${onboardPath}); process.env.NEMOCLAW_AGENT = "hermes"; process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.SLACK_APP_TOKEN = "xapp-test-slack-app-token-value"; - const sandboxName = await createSandbox( - null, - "gpt-5.4", - "nvidia-prod", - null, - "my-assistant", - null, - null, - ${customDockerfileArg}, - loadAgent("hermes"), - ); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, ${customDockerfileArg}, loadAgent("hermes"), null, null, null, []], createFixture)); const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); const parsed = YAML.parse(createCommand?.policyContent || "") || {}; const slack = parsed.network_policies?.slack || {}; @@ -517,6 +524,7 @@ const { createSandbox } = require(${onboardPath}); writeOkOpenshell(fakeBin, { readySandboxGet: true }); const script = String.raw` const runner = require(${runnerPath}), registry = require(${registryPath}), preflight = require(${preflightPath}), credentials = require(${credentialsPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const childProcess = require("node:child_process"), { EventEmitter } = require("node:events"); const commands = [], credentialKeys = ${JSON.stringify(providerCredentialKeys)}; let registered = null; @@ -535,6 +543,8 @@ runner.run = (command, opts = {}) => { return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -543,6 +553,13 @@ runner.runCapture = (command) => { return ""; }; registry.registerSandbox = (entry) => { registered = entry; return true; }; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "compatible-endpoint", + model: "custom/model", + getSandbox: registry.getSandbox, + registerSandbox: registry.registerSandbox, +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; childProcess.spawn = (...args) => { const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.unref = () => {}; child.pid = 4242; @@ -556,7 +573,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_EXTRA_PLACEHOLDER_KEYS = "TELEGRAM_BOT_TOKEN_AGENT_A,TELEGRAM_BOT_TOKEN_AGENT_B,GITHUB_TOKEN"; process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["slack", "telegram", "whatsapp"])})).toString("base64"); Object.values(credentialKeys).forEach((key) => delete process.env[key]); delete process.env.GITHUB_TOKEN; - const sandboxName = await createSandbox(null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"]); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "custom/model", "compatible-endpoint", null, "my-assistant", null, ["slack", "telegram", "whatsapp"], null, null, null, null, null, []], createFixture)); console.log(JSON.stringify({ sandboxName, commands, registered })); })().catch((error) => { const temporaryCreateSources = require("node:fs").readdirSync(process.env.TMPDIR).filter((entry) => entry.startsWith("nemoclaw-initial-policy-") || entry.startsWith("nemoclaw-build-")); console.log(JSON.stringify({ commands, registered, error: String(error), providerRevisions: Object.fromEntries(revisions), temporaryCreateSources })); console.error(error); process.exit(1); }); `; @@ -682,6 +699,7 @@ const { createSandbox } = require(${onboardPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -702,6 +720,8 @@ runner.run = (command, opts = {}) => { return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { @@ -718,6 +738,13 @@ registry.registerSandbox = (entry) => { registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + registerSandbox: registry.registerSandbox, +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -752,9 +779,7 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; delete process.env.TELEGRAM_BOT_TOKEN; process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["telegram"], ["telegram"])})).toString("base64"); - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${customDockerfileArg}, - ); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -841,6 +866,7 @@ const { createSandbox } = require(${onboardPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -856,6 +882,8 @@ runner.run = (command, opts = {}) => { return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { @@ -872,6 +900,12 @@ registry.registerSandbox = (entry) => { registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + registerSandbox: registry.registerSandbox, +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -910,9 +944,7 @@ const { createSandbox } = require(${onboardPath}); } } process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, - ); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -999,6 +1031,7 @@ const { createSandbox } = require(${onboardPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1019,6 +1052,8 @@ runner.run = (command, opts = {}) => { return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { @@ -1035,6 +1070,13 @@ registry.registerSandbox = (entry) => { registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + registerSandbox: registry.registerSandbox, +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -1073,9 +1115,7 @@ const { createSandbox } = require(${onboardPath}); } } process.env.NEMOCLAW_MESSAGING_PLAN_B64 = Buffer.from(JSON.stringify(${messagingPlanLiteral(["whatsapp"], ["whatsapp"])})).toString("base64"); - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, - ); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["whatsapp"], ${customDockerfileArg}, null, null, null, null, []], createFixture)); console.log(JSON.stringify({ sandboxName, commands, registerCalls })); })().catch((error) => { console.error(error); @@ -1223,12 +1263,13 @@ const { createSandbox } = require(${onboardPath}); ); fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const commands = []; runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderRunner({ @@ -1241,14 +1282,18 @@ runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderR }); runner.runCapture = (command) => { // Existing sandbox that is ready - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) { + return "Name: my-assistant\nId: fixture-created-sandbox\n"; + } if (_n(command).includes("sandbox list")) return "my-assistant Ready"; // All messaging providers already exist in gateway if (_n(command).includes("provider get")) return "Provider: exists"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive" }); +registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture( + { name: "my-assistant", toolDisclosure: "progressive" }, +); const { createSandbox } = require(${onboardPath}); (async () => { @@ -1331,6 +1376,7 @@ const { createSandbox } = require(${onboardPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1342,6 +1388,8 @@ runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderR readySandboxName: "my-assistant", }); runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { @@ -1355,6 +1403,11 @@ registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -1380,9 +1433,7 @@ const { createSandbox } = require(${onboardPath}); process.env.SLACK_BOT_TOKEN = "xoxb-test-slack-token-value"; process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // Only enable telegram — discord and slack should be filtered out - const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], - ); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation([null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, ["telegram"], null, null, null, null, null, []], createFixture)); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); @@ -1466,6 +1517,7 @@ const { createSandbox } = require(${onboardPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1477,6 +1529,8 @@ runner.run = (command, opts = {}) => { return _n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { @@ -1490,6 +1544,11 @@ registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -1516,7 +1575,24 @@ const { createSandbox } = require(${onboardPath}); process.env.TELEGRAM_BOT_TOKEN = "123456:ABC-test-telegram-token"; // No selected messaging plan — ambient repository credentials are unrelated to this request. const sandboxName = await createSandbox( - null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, + ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [ + null, + "gpt-5.4", + "nvidia-prod", + null, + "my-assistant", + null, + null, + null, + null, + null, + null, + null, + [], + ], + createFixture, + ), ); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { @@ -1900,117 +1976,4 @@ const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); ); }, ); - - it( - "interactive setupMessagingChannels drops slack when app token fails appTokenFormat check (#1912)", - { - timeout: 60_000, - }, - async () => { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-slack-app-format-reject-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "slack-app-format-reject.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - // Subscript: mocks prompt to return a VALID bot token but a bogus app - // token. Expected behavior: bot token passes the regex and persists, - // app token fails the regex, channel is dropped from the enabled set, - // and SLACK_APP_TOKEN is never saved. - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const saveCalls = []; -credentials.saveCredential = (key, value) => { saveCalls.push({ key, value }); }; -credentials.getCredential = () => null; -credentials.prompt = async (message) => { - if (message.includes("Slack Bot Token")) return "xoxb-test-valid-bot-token"; - if (message.includes("Slack App Token")) return "abcd"; - return ""; -}; - -runner.run = () => ({ status: 0 }); -runner.runCapture = () => ""; - -const { setupMessagingChannels, MESSAGING_CHANNELS } = require(${onboardPath}); - -(async () => { - delete process.env.TELEGRAM_BOT_TOKEN; - delete process.env.DISCORD_BOT_TOKEN; - delete process.env.SLACK_BOT_TOKEN; - delete process.env.SLACK_APP_TOKEN; - - const result = await setupMessagingChannels(); - console.log(JSON.stringify({ - result, - saveCalls, - slackIndex1Based: MESSAGING_CHANNELS.findIndex((c) => c.name === "slack") + 1, - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - // Dry run with Enter only to introspect Slack's 1-based digit. - const introspect = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - input: "\n", - }); - assert.equal(introspect.status, 0, introspect.stderr); - const slackIdx = JSON.parse(introspect.stdout.trim().split("\n").pop()!).slackIndex1Based; - assert.ok(slackIdx >= 1, `unexpected slack index: ${slackIdx}`); - - // Real run: toggle Slack on, exit UI, bot prompt returns valid, app - // prompt returns "abcd", app-token check rejects, channel dropped. - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - input: `${slackIdx}\n`, - }); - - assert.equal(result.status, 0, result.stderr); - const out = JSON.parse(result.stdout.trim().split("\n").pop()!); - - assert.ok( - !out.result.includes("slack"), - `slack should have been dropped after invalid app token; got ${JSON.stringify(out.result)}`, - ); - assert.ok( - !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_BOT_TOKEN"), - `SLACK_BOT_TOKEN should NOT be persisted until the app token also passes; saveCalls=${JSON.stringify(out.saveCalls)}`, - ); - assert.ok( - !out.saveCalls.some((c: { key: string }) => c.key === "SLACK_APP_TOKEN"), - `SLACK_APP_TOKEN should NOT have been persisted (invalid format); saveCalls=${JSON.stringify(out.saveCalls)}`, - ); - assert.ok( - result.stderr.includes("Invalid format") || result.stdout.includes("Invalid format"), - `expected 'Invalid format' warning; stderr=${result.stderr} stdout=${result.stdout}`, - ); - }, - ); }); diff --git a/test/onboarding/onboard-prepared-build-context.test.ts b/test/onboarding/onboard-prepared-build-context.test.ts index c1022e80ba6..79276d03ab1 100644 --- a/test/onboarding/onboard-prepared-build-context.test.ts +++ b/test/onboarding/onboard-prepared-build-context.test.ts @@ -76,8 +76,21 @@ function runPreparedContextScenario(scenario: PreparedContextScenario): Prepared const fs = require("node:fs"); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); -const runner = require(${runnerPath}); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); +const scenario = ${JSON.stringify(scenario)}; +const buildCtx = ${JSON.stringify(preparedBuildCtx)}; +const buildId = ${JSON.stringify(buildId)}; +const sandboxName = "prepared-dcode"; +const commands = []; +const registerCalls = []; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName, + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + registerSandbox: (entry) => registerCalls.push(entry), +}); +const runner = require(${runnerPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const buildContextStage = require(${buildContextStagePath}); @@ -87,19 +100,13 @@ const imageTag = require(${imageTagPath}); const dockerGpuSandboxCreate = require(${dockerGpuSandboxCreatePath}); const wait = require(${waitPath}); const { loadAgent } = require(${agentDefsPath}); - -const scenario = ${JSON.stringify(scenario)}; -const buildCtx = ${JSON.stringify(preparedBuildCtx)}; -const buildId = ${JSON.stringify(buildId)}; -const sandboxName = "prepared-dcode"; -const commands = []; -const registerCalls = []; const planFromRefs = []; const resolvedBuildIds = []; let cleanupCalls = 0; let patchCalls = 0; let patchSleepUsesSeconds = null; let stageCalls = 0; +let sandboxCreated = false; dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = (options) => { patchSleepUsesSeconds = options.deps.sleep === wait.sleepSeconds; @@ -155,6 +162,8 @@ runner.runFile = (file, args = []) => { }; runner.runCapture = (command) => { const normalized = normalize(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName }); + if (createdIdentity !== null) return createdIdentity; if ( normalized.includes( "sandbox exec --name " + @@ -169,20 +178,14 @@ runner.runCapture = (command) => { "Endpoint: https://inference.local/v1", ].join("\n"); } - if (normalized.includes("sandbox get")) return ""; + if (normalized.includes("sandbox get")) { + return sandboxCreated ? sandboxName + "\nId: fixture-created-sandbox\n" : ""; + } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; return ""; }; -registry.getSandbox = () => null; registry.getDefault = () => null; registry.listExtraProviders = () => []; -registry.registerSandbox = (entry) => { - registerCalls.push(entry); - return true; -}; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -194,6 +197,7 @@ childProcess.spawn = (...args) => { child.pid = 6195; commands.push(normalize([args[0], ...(Array.isArray(args[1]) ? args[1] : [])])); process.nextTick(() => { + sandboxCreated = true; child.stdout.emit("data", Buffer.from("Created sandbox: " + sandboxName + "\n")); child.emit("close", 0); }); @@ -234,6 +238,7 @@ const { createSandbox } = require(${onboardPath}); null, [], null, + { sessionId: createFixture.sessionId }, null, null, preparedBuildContext, @@ -286,59 +291,70 @@ const { createSandbox } = require(${onboardPath}); } describe("onboard prepared DCode build context", () => { - it("creates from the supplied context without restaging or repatching it (#6195)", { - timeout: 90_000, - }, () => { - const result = runPreparedContextScenario("create"); - - assert.equal(result.errorMessage, null); - assert.equal(result.stageCalls, 0); - assert.equal(result.patchCalls, 0); - assert.deepEqual(result.planFromRefs, [`${result.buildCtx}/Dockerfile`]); - assert.deepEqual(result.resolvedBuildIds, [result.buildId]); - assert.equal(result.cleanupCalls, 1); - assert.ok( - result.commands.some((command) => - command.includes(`sandbox create --from ${result.buildCtx}/Dockerfile`), - ), - `expected create command to use prepared context; commands:\n${result.commands.join("\n")}`, - ); - assert.ok( - result.registerCalls.some( - (entry) => entry.imageTag === `openshell/sandbox-from:${result.buildId}`, - ), - "expected the prepared build ID to determine the registered image tag", - ); - }); + it( + "creates from the supplied context without restaging or repatching it (#6195)", + { + timeout: 90_000, + }, + () => { + const result = runPreparedContextScenario("create"); + assert.equal(result.errorMessage, null); + assert.equal(result.stageCalls, 0); + assert.equal(result.patchCalls, 0); + assert.deepEqual(result.planFromRefs, [`${result.buildCtx}/Dockerfile`]); + assert.deepEqual(result.resolvedBuildIds, [result.buildId]); + assert.equal(result.cleanupCalls, 1); + assert.ok( + result.commands.some((command) => + command.includes(`sandbox create --from ${result.buildCtx}/Dockerfile`), + ), + `expected create command to use prepared context; commands:\n${result.commands.join("\n")}`, + ); + assert.ok( + result.registerCalls.some( + (entry) => entry.imageTag === `openshell/sandbox-from:${result.buildId}`, + ), + "expected the prepared build ID to determine the registered image tag", + ); + }, + ); - it("passes the seconds-based sleep helper to the Docker GPU patch during prepared-context onboarding (#9218)", { - timeout: 90_000, - }, () => { - const result = runPreparedContextScenario("create"); + it( + "passes the seconds-based sleep helper to the Docker GPU patch during prepared-context onboarding (#9218)", + { + timeout: 90_000, + }, + () => { + const result = runPreparedContextScenario("create"); - assert.equal(result.errorMessage, null); - assert.equal(result.patchSleepUsesSeconds, true); - }); + assert.equal(result.errorMessage, null); + assert.equal(result.patchSleepUsesSeconds, true); + }, + ); - it("rejects a prepared context combined with a custom Dockerfile (#6195)", { - timeout: 90_000, - }, () => { - const result = runPreparedContextScenario("custom-dockerfile"); + it( + "rejects a prepared context combined with a custom Dockerfile (#6195)", + { + timeout: 90_000, + }, + () => { + const result = runPreparedContextScenario("custom-dockerfile"); - assert.match( - result.errorMessage ?? "", - /prepared DCode build context cannot be used for this sandbox target/i, - ); - assert.equal(result.stageCalls, 0); - assert.equal(result.patchCalls, 0); - assert.deepEqual(result.planFromRefs, []); - assert.deepEqual(result.resolvedBuildIds, []); - assert.equal(result.cleanupCalls, 0); - assert.equal( - result.commands.some((command) => command.includes("sandbox create")), - false, - ); - assert.deepEqual(result.registerCalls, []); - }); + assert.match( + result.errorMessage ?? "", + /prepared DCode build context cannot be used for this sandbox target/i, + ); + assert.equal(result.stageCalls, 0); + assert.equal(result.patchCalls, 0); + assert.deepEqual(result.planFromRefs, []); + assert.deepEqual(result.resolvedBuildIds, []); + assert.equal(result.cleanupCalls, 0); + assert.equal( + result.commands.some((command) => command.includes("sandbox create")), + false, + ); + assert.deepEqual(result.registerCalls, []); + }, + ); }); diff --git a/test/onboarding/onboard-remote-recreate-credential-reuse.test.ts b/test/onboarding/onboard-remote-recreate-credential-reuse.test.ts index 5b9ccd17278..bede29e7861 100644 --- a/test/onboarding/onboard-remote-recreate-credential-reuse.test.ts +++ b/test/onboarding/onboard-remote-recreate-credential-reuse.test.ts @@ -93,6 +93,7 @@ const registryRoute = { preferredInferenceApi: "openai-completions", source: "registry", }; +registry.removeSandbox("recovered-custom"); registry.registerSandbox({ name: "recovered-custom", ...registryRoute, diff --git a/test/onboarding/onboard-reservation-recreate.test.ts b/test/onboarding/onboard-reservation-recreate.test.ts index 5c7549591af..6ebb4d147a4 100644 --- a/test/onboarding/onboard-reservation-recreate.test.ts +++ b/test/onboarding/onboard-reservation-recreate.test.ts @@ -35,27 +35,30 @@ describe("onboard sandbox recreate reservation safety", () => { reservationSessionId: null, expectedRemoval: true, }, - ] as const)("$name (#6562)", { timeout: 60_000 }, async ({ - reservationSessionId, - expectedRemoval, - }) => { - const workspace = createOnboardProcessWorkspace("nemoclaw-onboard-reservation-survives-"); - onTestFinished(() => workspace.remove()); - const scriptPath = workspace.path("reservation-survives.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const onboardSessionPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), - ); + ] as const)( + "$name (#6562)", + { timeout: 60_000 }, + async ({ reservationSessionId, expectedRemoval }) => { + const workspace = createOnboardProcessWorkspace("nemoclaw-onboard-reservation-survives-"); + onTestFinished(() => workspace.remove()); + const scriptPath = workspace.path("reservation-survives.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const onboardSessionPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), + ); - writeOkOpenshell(workspace.binDir); + writeOkOpenshell(workspace.binDir); - const script = String.raw` + const script = String.raw` const runner = require(${runnerPath}); require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const onboardSession = require(${onboardSessionPath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); @@ -80,7 +83,9 @@ runner.run = (command) => { require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } @@ -97,22 +102,38 @@ runner.runCapture = (command) => { onboardSession.loadSession = () => ({ sessionId: "session-owner" }); const reservationSessionId = ${JSON.stringify(reservationSessionId)}; -registry.getSandbox = () => ({ +let sourceEntry = { name: "my-assistant", gpuEnabled: false, pendingRouteReservation: true, ...(reservationSessionId === null ? {} : { reservationSessionId }), -}); +}; +registry.getSandbox = () => sourceEntry; registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; -registry.removeSandbox = (name) => { +const removeSandbox = (name) => { events.push({ kind: "removeSandbox", name }); + sourceEntry = null; return true; }; +registry.removeSandbox = removeSandbox; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + removeSandbox, +}); const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); +const policyAuthorityPreflight = require(${JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "policy-authority", "preflight.ts"), + )}); +policyAuthorityPreflight.qualifySandboxPolicyAuthority = () => ({ + authority: "nemoclaw-managed", +}); childProcess.spawn = (...args) => { sandboxRecreated = true; @@ -135,45 +156,65 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [ + null, + "gpt-5.4", + "nvidia-prod", + null, + "my-assistant", + null, + null, + null, + null, + null, + null, + null, + [], + ], + createFixture, + ), + ); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); + fs.writeFileSync(scriptPath, script); - const result = runOnboardProcess([scriptPath], { - env: workspaceEnv(workspace, { - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", - NEMOCLAW_SANDBOX_PREBUILD: "1", - }), - timeoutMs: 30_000, - }); + const result = runOnboardProcess([scriptPath], { + env: workspaceEnv(workspace, { + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", + NEMOCLAW_SANDBOX_PREBUILD: "1", + }), + timeoutMs: 30_000, + }); - assert.equal(result.status, 0, result.stderr || result.error?.message); - const payload = trailingJsonPayload<{ - sandboxName: string; - events: Array<{ kind: string; cmd?: string; name?: string }>; - }>(result.stdout); - assert.equal(payload.sandboxName, "my-assistant"); + assert.equal(result.status, 0, result.stderr || result.error?.message); + const payload = trailingJsonPayload<{ + sandboxName: string; + events: Array<{ kind: string; cmd?: string; name?: string }>; + }>(result.stdout); + assert.equal(payload.sandboxName, "my-assistant"); - const events = payload.events; - const removedReservation = events.some( - (e) => e.kind === "removeSandbox" && e.name === "my-assistant", - ); - assert.equal( - removedReservation, - expectedRemoval, - expectedRemoval - ? "must delete abandoned pending route reservations during recreate" - : "must not delete the current session's pending route reservation during recreate", - ); - assert.ok( - events.some((e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete")), - "should still delete the not-ready gateway sandbox before rebuilding", - ); - }); + const events = payload.events; + const removedReservation = events.some( + (e) => e.kind === "removeSandbox" && e.name === "my-assistant", + ); + assert.equal( + removedReservation, + expectedRemoval, + expectedRemoval + ? "must delete abandoned pending route reservations during recreate" + : "must not delete the current session's pending route reservation during recreate", + ); + assert.ok( + events.some((e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete")), + "should still delete the not-ready gateway sandbox before rebuilding", + ); + }, + ); }); diff --git a/test/onboarding/onboard-sandbox-build.test.ts b/test/onboarding/onboard-sandbox-build.test.ts index 70c10f59311..6ddbe1bdb0c 100644 --- a/test/onboarding/onboard-sandbox-build.test.ts +++ b/test/onboarding/onboard-sandbox-build.test.ts @@ -22,28 +22,34 @@ beforeEach(() => { }); describe("onboard helpers", () => { - it("creates the stock managed sandbox without uploading an external OpenClaw config file", { - timeout: 90_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-sandbox-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "create-sandbox-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin, { readySandboxGet: true }); - - const script = String.raw` + it( + "creates the stock managed sandbox without uploading an external OpenClaw config file", + { + timeout: 90_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-sandbox-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "create-sandbox-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); + + const script = String.raw` const runner = require(${runnerPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -58,7 +64,7 @@ const defaultCalls = []; runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); + const profileResult = fixtureMocks.mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; if (normalized.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; @@ -68,28 +74,26 @@ runner.run = (command, opts = {}) => { : { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + const normalized = _n(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); + if (createdIdentity !== null) return createdIdentity; + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; + if (normalized.includes("sandbox list")) return "my-assistant Ready"; { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (normalized.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; -registry.registerSandbox = (entry) => { - registerCalls.push(entry); - return true; -}; -registry.updateSandbox = (name, updates) => { - updateCalls.push({ name, updates }); - return true; -}; -registry.setDefault = (name) => { - defaultCalls.push(name); - return true; -}; -registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + registerSandbox: (entry) => registerCalls.push(entry), + updateSandbox: (name, updates) => updateCalls.push({ name, updates }), + setDefault: (name) => defaultCalls.push(name), +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -111,79 +115,83 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, commands, registerCalls, updateCalls, defaultCalls })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - assert.equal(payload.sandboxName, "my-assistant"); - // createSandbox no longer marks the sandbox default — that is deferred to the - // finalization step so a cancel at policy presets can't leave an unconfigured - // sandbox as default (#4614). - assert.deepEqual(payload.defaultCalls, []); - assert.ok( - payload.registerCalls.some( - (entry: Record) => - entry.name === "my-assistant" && - entry.model === "gpt-5.4" && - Object.prototype.hasOwnProperty.call(entry, "agentVersion"), - ), - "expected registry metadata for created sandbox", - ); - assert.ok( - payload.updateCalls.every( - (call: { name: string; updates: Record }) => - call.name === "my-assistant" && call.updates, - ), - "expected any registry metadata updates to target the created sandbox", - ); - const createCommand = payload.commands.find((entry: CommandEntry) => - entry.command.includes("sandbox create"), - ); - assert.ok(createCommand, "expected sandbox create command"); - assert.match(createCommand.command, /nemoclaw-managed-startup-hold/); - assert.match( - createCommand.command, - /--from ghcr\.io\/nvidia\/nemoclaw\/openclaw-sandbox@sha256:[0-9a-f]{64}/, - ); - assert.doesNotMatch(createCommand.command, /--upload/); - assert.doesNotMatch(createCommand.command, /OPENCLAW_CONFIG_PATH/); - assert.doesNotMatch(createCommand.command, /NVIDIA_INFERENCE_API_KEY=/); - assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); - assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); - assert.ok( - payload.commands.some( - (entry: CommandEntry) => - entry.command.includes("forward start --background 18789 my-assistant") || - entry.command.includes("forward start --background 0.0.0.0:18789 my-assistant"), - ), - "expected dashboard forward (loopback or WSL 0.0.0.0)", - ); - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + assert.equal(payload.sandboxName, "my-assistant"); + // createSandbox no longer marks the sandbox default — that is deferred to the + // finalization step so a cancel at policy presets can't leave an unconfigured + // sandbox as default (#4614). + assert.deepEqual(payload.defaultCalls, []); + assert.ok( + payload.registerCalls.some( + (entry: Record) => + entry.name === "my-assistant" && + entry.model === "gpt-5.4" && + Object.prototype.hasOwnProperty.call(entry, "agentVersion"), + ), + "expected registry metadata for created sandbox", + ); + assert.ok( + payload.updateCalls.every( + (call: { name: string; updates: Record }) => + call.name === "my-assistant" && call.updates, + ), + "expected any registry metadata updates to target the created sandbox", + ); + const createCommand = payload.commands.find((entry: CommandEntry) => + entry.command.includes("sandbox create"), + ); + assert.ok(createCommand, "expected sandbox create command"); + assert.match(createCommand.command, /nemoclaw-managed-startup-hold/); + assert.match( + createCommand.command, + /--from ghcr\.io\/nvidia\/nemoclaw\/openclaw-sandbox@sha256:[0-9a-f]{64}/, + ); + assert.doesNotMatch(createCommand.command, /--upload/); + assert.doesNotMatch(createCommand.command, /OPENCLAW_CONFIG_PATH/); + assert.doesNotMatch(createCommand.command, /NVIDIA_INFERENCE_API_KEY=/); + assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); + assert.ok( + payload.commands.some( + (entry: CommandEntry) => + entry.command.includes("forward start --background 18789 my-assistant") || + entry.command.includes("forward start --background 0.0.0.0:18789 my-assistant"), + ), + "expected dashboard forward (loopback or WSL 0.0.0.0)", + ); + }, + ); it("resolves the sandbox base for an explicit custom Dockerfile", async () => { const repoRoot = path.join(import.meta.dirname, "../.."); @@ -217,6 +225,7 @@ const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); const runner = require(${runnerPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -282,7 +291,7 @@ agentOnboard.createAgentSandbox = () => { runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); + const profileResult = fixtureMocks.mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; if (normalized.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; @@ -294,20 +303,23 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get hermes-sandbox")) return ""; - if (_n(command).includes("sandbox list")) return "hermes-sandbox Ready"; + const normalized = _n(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "hermes-sandbox" }); + if (createdIdentity !== null) return createdIdentity; + if (normalized.includes("sandbox get hermes-sandbox")) return ""; + if (normalized.includes("sandbox list")) return "hermes-sandbox Ready"; { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "hermes-sandbox 127.0.0.1 18789 12345 running\nhermes-sandbox 127.0.0.1 8642 12346 running"; + if (normalized.includes("forward list")) return "hermes-sandbox 127.0.0.1 18789 12345 running\nhermes-sandbox 127.0.0.1 8642 12346 running"; return ""; }; -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -registry.getSandbox = () => null; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "hermes-sandbox", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -340,21 +352,24 @@ const { createSandbox } = require(${onboardPath}); policyAdditionsPath: ${hermesPolicyPath}, }; const customDockerfilePath = agentOnboard.createAgentSandbox().stagedDockerfile; - await createSandbox( - null, - "gpt-5.4", - "nvidia-prod", - null, - "hermes-sandbox", - null, - [], - customDockerfilePath, - agent, - null, - null, - null, - ["nous-web"], - ); + await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [ + null, + "gpt-5.4", + "nvidia-prod", + null, + "hermes-sandbox", + null, + [], + customDockerfilePath, + agent, + null, + null, + null, + ["nous-web"], + ], + createFixture, + )); console.log(JSON.stringify({ commands, logs, warnings, baseResolutionCalls })); })().catch((error) => { console.error(error); @@ -383,11 +398,11 @@ const { createSandbox } = require(${onboardPath}); baseResolutionCalls: Array<{ imageName?: string; dockerfilePath?: string }>; }>(result.stdout); assert.equal(payload.baseResolutionCalls.length, 1); + assert.equal(payload.baseResolutionCalls[0]?.imageName, "ghcr.io/nvidia/nemoclaw/sandbox-base"); assert.equal( - payload.baseResolutionCalls[0]?.imageName, - "ghcr.io/nvidia/nemoclaw/sandbox-base", + payload.baseResolutionCalls[0]?.dockerfilePath, + path.join(repoRoot, "Dockerfile.base"), ); - assert.equal(payload.baseResolutionCalls[0]?.dockerfilePath, path.join(repoRoot, "Dockerfile.base")); const createCommand = payload.commands.find((entry) => entry.command.includes("sandbox create"), ); @@ -435,6 +450,7 @@ const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); const runner = require(${runnerPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -493,7 +509,7 @@ buildContext.stageOptimizedSandboxBuildContext = () => { runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); + const profileResult = fixtureMocks.mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; if (normalized.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; @@ -507,22 +523,25 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) { - return sandboxCreated ? "Name: my-assistant\\nPhase: Ready\\n" : ""; + const normalized = _n(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); + if (createdIdentity !== null) return createdIdentity; + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) { + return sandboxCreated ? "Name: my-assistant\\nId: fixture-created-sandbox\\nPhase: Ready\\n" : ""; } - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (normalized.includes("sandbox list")) return "my-assistant Ready"; { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (normalized.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -registry.getSandbox = () => null; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -545,7 +564,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ commands, logs, baseResolutionCalls })); })().catch((error) => { console.error(error); @@ -598,6 +620,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -609,7 +632,7 @@ const commands = []; runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); + const profileResult = fixtureMocks.mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; if (normalized.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; @@ -619,19 +642,23 @@ runner.run = (command, opts = {}) => { : { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + const normalized = _n(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); + if (createdIdentity !== null) return createdIdentity; + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; + if (normalized.includes("sandbox list")) return "my-assistant Ready"; { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + if (normalized.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -654,7 +681,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.CHAT_UI_URL = "https://chat.example.com"; - await createSandbox(null, "gpt-5.4", "nvidia-prod"); + await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify(commands)); })().catch((error) => { console.error(error); @@ -704,6 +734,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const preflight = require(${preflightPath}); @@ -715,7 +746,7 @@ const commands = []; runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); + const profileResult = fixtureMocks.mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; if (normalized.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; @@ -729,20 +760,24 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + const normalized = _n(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); + if (createdIdentity !== null) return createdIdentity; + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; + if (normalized.includes("sandbox list")) return "my-assistant Ready"; // Custom port: dashboard readiness curl uses 19000 (DASHBOARD_PORT from env) { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 19000 12345 running"; + if (normalized.includes("forward list")) return "my-assistant 127.0.0.1 19000 12345 running"; return ""; }; -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; @@ -764,7 +799,10 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, null, null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); diff --git a/test/onboarding/onboard-sandbox-recreation.test.ts b/test/onboarding/onboard-sandbox-recreation.test.ts index 3c5d42dfa2a..738b37435b5 100644 --- a/test/onboarding/onboard-sandbox-recreation.test.ts +++ b/test/onboarding/onboard-sandbox-recreation.test.ts @@ -18,25 +18,31 @@ beforeEach(() => { }); describe("onboard helpers", () => { - it("non-interactive exits with error when existing sandbox is not ready", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-noninteractive-notready-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "noninteractive-notready.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); + it( + "non-interactive exits with error when existing sandbox is not ready", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-noninteractive-notready-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "noninteractive-notready.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -50,14 +56,20 @@ runner.run = (command) => { if (_n(command).includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; return { status: 0 }; }; -runner.runCapture = (command) => { - // Existing sandbox that is NOT ready - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); - if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant NotReady"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; - return ""; -}; -registry.getSandbox = () => ({ name: "my-assistant", policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive" }); + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + // Existing sandbox that is NOT ready + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant NotReady"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + return ""; + }; + registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + toolDisclosure: "progressive", + }, { sandboxId: "sbx-4f2a91c0d7" }); childProcess.spawn = () => { throw new Error("unexpected sandbox create"); }; @@ -73,53 +85,57 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const env: Record = { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }; - delete env["NEMOCLAW_RECREATE_SANDBOX"]; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env, - }); + fs.writeFileSync(scriptPath, script); - assert.notEqual(result.status, 0, "expected non-zero exit for not-ready sandbox"); - assert.ok( - !result.stdout.includes("ERROR_DID_NOT_EXIT"), - "should have exited before reaching sandbox create", - ); - const output = (result.stdout || "") + (result.stderr || ""); - assert.ok( - output.includes("--recreate-sandbox") || output.includes("NEMOCLAW_RECREATE_SANDBOX"), - "should hint about --recreate-sandbox flag", - ); - }); + const env: Record = { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }; + delete env["NEMOCLAW_RECREATE_SANDBOX"]; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env, + }); + + assert.notEqual(result.status, 0, "expected non-zero exit for not-ready sandbox"); + assert.ok( + !result.stdout.includes("ERROR_DID_NOT_EXIT"), + "should have exited before reaching sandbox create", + ); + const output = (result.stdout || "") + (result.stderr || ""); + assert.ok( + output.includes("--recreate-sandbox") || output.includes("NEMOCLAW_RECREATE_SANDBOX"), + "should hint about --recreate-sandbox flag", + ); + }, + ); + + it.each(["balanced", "restricted"])( + "recreate-sandbox records the %s policy tier and late replacement identity", + { + timeout: 60_000, + }, + async (policyTier) => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-flag-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "recreate-flag.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); - it.each([ - "balanced", - "restricted", - ])("recreate-sandbox records the %s policy tier and late replacement identity", { - timeout: 60_000, - }, async (policyTier) => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-flag-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "recreate-flag.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; const registry = require(${registryPath}); @@ -127,10 +143,10 @@ const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); const commands = []; let registeredSandbox = null; -const sourceSandbox = { - name: "my-assistant", - gpuEnabled: false, policyAuthority: "nemoclaw-managed", - openshellDriver: "docker", + const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + gpuEnabled: false, + openshellDriver: "docker", imageTag: "openshell/sandbox-from:source", workload: { schemaVersion: 1, @@ -138,7 +154,7 @@ const sourceSandbox = { reference: "openshell/sandbox-from:source", shared: false, }, -}; + }, { sandboxId: "sbx-4f2a91c0d7" }); runner.run = (command, opts = {}) => { const cmd = _n(command); const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); @@ -152,10 +168,15 @@ runner.run = (command, opts = {}) => { ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + _sandboxId + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); - if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant", sandboxId: _sandboxId }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -164,11 +185,15 @@ runner.runCapture = (command) => { } return ""; }; -registry.getSandbox = () => registeredSandbox || sourceSandbox; -registry.registerSandbox = (entry) => { registeredSandbox = entry; return true; }; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; + const getSandbox = () => registeredSandbox || sourceSandbox; + registry.getSandbox = getSandbox; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox, + registerSandbox: (entry) => { registeredSandbox = entry; }, + }); const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); @@ -195,83 +220,96 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, commands, registeredSandbox })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_POLICY_TIER: policyTier, + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + assert.ok( + payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox delete")), + "should delete existing sandbox when --recreate-sandbox is set", + ); + assert.ok( + payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox create")) && + payload.registeredSandbox?.policyTier === policyTier, + "should create a sandbox and persist its tier before policy finalization", + ); + assert.ok( + !payload.commands.some((entry: CommandEntry) => + entry.command.includes("docker rmi openshell/sandbox-from:source"), + ), + "must defer source image retirement until replacement registration is proven", + ); + const sourceFingerprint = createHash("sha256").update("sbx-4f2a91c0d7").digest("hex"); + const replacementFingerprint = createHash("sha256").update("sbx-8e6b10fd33").digest("hex"); + assert.match(payload.registeredSandbox?.lifecycleGeneration ?? "", /^[0-9a-f-]{36}$/); + assert.equal( + payload.registeredSandbox?.lifecycleLiveIdentityFingerprint, + replacementFingerprint, + "replacement registration must read the live identity after creation", + ); + assert.notEqual( + payload.registeredSandbox?.lifecycleLiveIdentityFingerprint, + sourceFingerprint, + ); + }, + ); + it( + "recreate-sandbox flag backs up and restores workspace state", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-backup-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "recreate-backup.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const sandboxStatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), + ); - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_POLICY_TIER: policyTier, - }, - }); + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - assert.ok( - payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox delete")), - "should delete existing sandbox when --recreate-sandbox is set", - ); - assert.ok( - payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox create")) && - payload.registeredSandbox?.policyTier === policyTier, - "should create a sandbox and persist its tier before policy finalization", - ); - assert.ok( - !payload.commands.some((entry: CommandEntry) => - entry.command.includes("docker rmi openshell/sandbox-from:source"), - ), - "must defer source image retirement until replacement registration is proven", - ); - const sourceFingerprint = createHash("sha256").update("sbx-4f2a91c0d7").digest("hex"); - const replacementFingerprint = createHash("sha256").update("sbx-8e6b10fd33").digest("hex"); - assert.match(payload.registeredSandbox?.lifecycleGeneration ?? "", /^[0-9a-f-]{36}$/); - assert.equal( - payload.registeredSandbox?.lifecycleLiveIdentityFingerprint, - replacementFingerprint, - "replacement registration must read the live identity after creation", - ); - assert.notEqual(payload.registeredSandbox?.lifecycleLiveIdentityFingerprint, sourceFingerprint); - }); - it("recreate-sandbox flag backs up and restores workspace state", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-backup-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "recreate-backup.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const sandboxStatePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); + const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); + let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -287,13 +325,17 @@ runner.run = (command) => { if (cmd.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } - return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + return cmd.includes("sandbox get") && cmd.includes("my-assistant") + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + _sandboxId + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant", sandboxId: _sandboxId }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -304,11 +346,17 @@ runner.runCapture = (command) => { } return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", gpuEnabled: false, policyAuthority: "nemoclaw-managed" }); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; + const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + gpuEnabled: false, + }, { sandboxId: "sbx-4f2a91c0d7" }); + registry.getSandbox = () => sourceSandbox; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + }); sandboxState.backupSandboxState = (name) => { events.push({ kind: "backup", name }); @@ -335,8 +383,9 @@ sandboxState.restoreRecreatedSandboxState = (name, backupPath, options) => { const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); -childProcess.spawn = (...args) => { - _deleted = false; + childProcess.spawn = (...args) => { + _deleted = false; + _sandboxId = "sbx-8e6b10fd34"; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -355,82 +404,98 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + + const events = payload.events as Array<{ + kind: string; + cmd?: string; + name?: string; + backupPath?: string; + options?: { targetAgentType?: string; freshOpenClawImagePluginInstalls?: unknown[] }; + }>; + const backupIndex = events.findIndex((e) => e.kind === "backup"); + const deleteIndex = events.findIndex( + (e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete"), + ); + const restoreIndex = events.findIndex((e) => e.kind === "restore"); + + assert.ok(backupIndex >= 0, "should call backupSandboxState before delete"); + assert.ok(deleteIndex > backupIndex, "backup must happen before sandbox delete"); + assert.ok(restoreIndex > deleteIndex, "restore must happen after sandbox recreate"); + const backupEvent = events[backupIndex]; + assert.equal(backupEvent?.name, "my-assistant", "backup target must match sandbox name"); + const restoreEvent = events[restoreIndex]; + assert.equal( + restoreEvent?.backupPath, + "/tmp/fake-backup-path", + "restore must use backup path", + ); + assert.equal(restoreEvent?.options?.targetAgentType, "openclaw"); + assert.equal(restoreEvent?.options?.freshOpenClawImagePluginInstalls, undefined); + }, + ); + + it( + "recreate-sandbox with NEMOCLAW_RECREATE_WITHOUT_BACKUP=1 skips backup", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-recreate-skip-backup-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "recreate-skip-backup.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const sandboxStatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), + ); - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - - const events = payload.events as Array<{ - kind: string; - cmd?: string; - name?: string; - backupPath?: string; - options?: { targetAgentType?: string; freshOpenClawImagePluginInstalls?: unknown[] }; - }>; - const backupIndex = events.findIndex((e) => e.kind === "backup"); - const deleteIndex = events.findIndex( - (e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete"), - ); - const restoreIndex = events.findIndex((e) => e.kind === "restore"); - - assert.ok(backupIndex >= 0, "should call backupSandboxState before delete"); - assert.ok(deleteIndex > backupIndex, "backup must happen before sandbox delete"); - assert.ok(restoreIndex > deleteIndex, "restore must happen after sandbox recreate"); - const backupEvent = events[backupIndex]; - assert.equal(backupEvent?.name, "my-assistant", "backup target must match sandbox name"); - const restoreEvent = events[restoreIndex]; - assert.equal(restoreEvent?.backupPath, "/tmp/fake-backup-path", "restore must use backup path"); - assert.equal(restoreEvent?.options?.targetAgentType, "openclaw"); - assert.equal(restoreEvent?.options?.freshOpenClawImagePluginInstalls, undefined); - }); + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); - it("recreate-sandbox with NEMOCLAW_RECREATE_WITHOUT_BACKUP=1 skips backup", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-skip-backup-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "recreate-skip-backup.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const sandboxStatePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); + const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); + let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -446,13 +511,17 @@ runner.run = (command) => { if (cmd.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } - return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + return cmd.includes("sandbox get") && cmd.includes("my-assistant") + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + _sandboxId + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant", sandboxId: _sandboxId }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -463,11 +532,17 @@ runner.runCapture = (command) => { } return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", gpuEnabled: false, policyAuthority: "nemoclaw-managed" }); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; + const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + gpuEnabled: false, + }, { sandboxId: "sbx-4f2a91c0d7" }); + registry.getSandbox = () => sourceSandbox; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + }); sandboxState.backupSandboxState = () => { events.push({ kind: "backup" }); @@ -481,8 +556,9 @@ sandboxState.restoreRecreatedSandboxState = () => { const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); -childProcess.spawn = (...args) => { - _deleted = false; + childProcess.spawn = (...args) => { + _deleted = false; + _sandboxId = "sbx-8e6b10fd35"; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -502,68 +578,78 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + const events = payload.events as Array<{ kind: string }>; + assert.ok( + !events.some((e) => e.kind === "backup"), + "should not call backupSandboxState when NEMOCLAW_RECREATE_WITHOUT_BACKUP=1", + ); + assert.ok( + !events.some((e) => e.kind === "restore"), + "should not call restoreRecreatedSandboxState when no backup occurred", + ); + }, + ); + + it( + "recreate-sandbox flag backs up and restores when existing sandbox is not ready", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-notready-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "recreate-notready.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const sandboxStatePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), + ); - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - const events = payload.events as Array<{ kind: string }>; - assert.ok( - !events.some((e) => e.kind === "backup"), - "should not call backupSandboxState when NEMOCLAW_RECREATE_WITHOUT_BACKUP=1", - ); - assert.ok( - !events.some((e) => e.kind === "restore"), - "should not call restoreRecreatedSandboxState when no backup occurred", - ); - }); - - it("recreate-sandbox flag backs up and restores when existing sandbox is not ready", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-notready-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "recreate-notready.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const sandboxStatePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "sandbox.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); + const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); + let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -582,13 +668,17 @@ runner.run = (command) => { if (cmd.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } - return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + return cmd.includes("sandbox get") && cmd.includes("my-assistant") + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + _sandboxId + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant", sandboxId: _sandboxId }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } @@ -601,11 +691,17 @@ runner.runCapture = (command) => { } return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", gpuEnabled: false, policyAuthority: "nemoclaw-managed" }); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; + const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + gpuEnabled: false, + }, { sandboxId: "sbx-4f2a91c0d7" }); + registry.getSandbox = () => sourceSandbox; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + }); sandboxState.backupSandboxState = (name) => { events.push({ kind: "backup", name }); @@ -632,8 +728,9 @@ sandboxState.restoreRecreatedSandboxState = (name, backupPath) => { const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); -childProcess.spawn = (...args) => { - _deleted = false; + childProcess.spawn = (...args) => { + _deleted = false; + _sandboxId = "sbx-8e6b10fd36"; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -652,75 +749,85 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, events })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + + const events = payload.events as Array<{ + kind: string; + cmd?: string; + name?: string; + backupPath?: string; + }>; + const backupIndex = events.findIndex((e) => e.kind === "backup"); + const deleteIndex = events.findIndex( + (e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete"), + ); + const restoreIndex = events.findIndex((e) => e.kind === "restore"); + + assert.ok(backupIndex >= 0, "should call backupSandboxState for not-ready sandbox"); + assert.ok(deleteIndex > backupIndex, "backup must happen before sandbox delete"); + assert.ok(restoreIndex > deleteIndex, "restore must happen after sandbox recreate"); + }, + ); + + it( + "recreating a sandbox preserves the user's policy preset selections", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-preserves-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "recreate-preserves.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const sessionModulePath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), + ); - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - - const events = payload.events as Array<{ - kind: string; - cmd?: string; - name?: string; - backupPath?: string; - }>; - const backupIndex = events.findIndex((e) => e.kind === "backup"); - const deleteIndex = events.findIndex( - (e) => e.kind === "run" && (e.cmd || "").includes("sandbox delete"), - ); - const restoreIndex = events.findIndex((e) => e.kind === "restore"); - - assert.ok(backupIndex >= 0, "should call backupSandboxState for not-ready sandbox"); - assert.ok(deleteIndex > backupIndex, "backup must happen before sandbox delete"); - assert.ok(restoreIndex > deleteIndex, "restore must happen after sandbox recreate"); - }); + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); - it("recreating a sandbox preserves the user's policy preset selections", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-recreate-preserves-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "recreate-preserves.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const sessionModulePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "onboard-session.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); + const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); + let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; const registry = require(${registryPath}); const onboardSession = require(${sessionModulePath}); const childProcess = require("node:child_process"); @@ -736,14 +843,19 @@ runner.run = (command, opts = {}) => { if (cmd.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } - return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + return cmd.includes("sandbox get") && cmd.includes("my-assistant") + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + _sandboxId + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); - if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant", sandboxId: _sandboxId }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -756,22 +868,26 @@ runner.runCapture = (command) => { // Existing sandbox has a custom preset selection: only "npm" (not the // full "balanced" tier). Recreating the sandbox must preserve this // customisation rather than reverting to the tier defaults. -registry.getSandbox = () => ({ - name: "my-assistant", - gpuEnabled: false, policyAuthority: "nemoclaw-managed", - policies: ["npm"], - policyTier: "balanced", -}); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; + const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + gpuEnabled: false, + policies: ["npm"], + policyTier: "balanced", + }, { sandboxId: "sbx-4f2a91c0d7" }); + registry.getSandbox = () => sourceSandbox; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + }); const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); -childProcess.spawn = (...args) => { - _deleted = false; + childProcess.spawn = (...args) => { + _deleted = false; + _sandboxId = "sbx-8e6b10fd37"; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -791,7 +907,10 @@ const { createSandbox } = require(${onboardPath}); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.NEMOCLAW_RECREATE_SANDBOX = "1"; process.env.NEMOCLAW_RECREATE_WITHOUT_BACKUP = "1"; - await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); const session = onboardSession.loadSession(); console.log(JSON.stringify({ policyPresets: session && session.policyPresets })); })().catch((error) => { @@ -799,56 +918,63 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - }, - }); + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + + assert.deepEqual( + payload.policyPresets, + ["npm"], + "createSandbox should write the previous sandbox's policy presets to the onboard session before destroying it so they can be reapplied after recreation", + ); + }, + ); + + it( + "interactive mode prompts before reusing an existing ready sandbox", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-interactive-reuse-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "interactive-reuse.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - - assert.deepEqual( - payload.policyPresets, - ["npm"], - "createSandbox should write the previous sandbox's policy presets to the onboard session before destroying it so they can be reapplied after recreation", - ); - }); + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); - it("interactive mode prompts before reusing an existing ready sandbox", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-interactive-reuse-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "interactive-reuse.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -892,13 +1018,19 @@ runner.runFile = (file, args = [], opts = {}) => { commands.push({ type: "runFile", command: _n([file, ...args]), file, args, env: opts.env || null }); return { status: 0 }; }; -runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); - if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; - return ""; -}; -registry.getSandbox = () => ({ name: "my-assistant", policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive" }); + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + return ""; + }; + registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + toolDisclosure: "progressive", + }, { sandboxId: "sbx-4f2a91c0d7" }); // Mock prompt to return "y" (reuse) credentials.prompt = async () => "y"; @@ -929,69 +1061,78 @@ const { createSandbox } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - // Run WITHOUT NEMOCLAW_NON_INTERACTIVE to exercise interactive path - const env: Record = { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }; - delete env["NEMOCLAW_NON_INTERACTIVE"]; - delete env["NEMOCLAW_RECREATE_SANDBOX"]; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env, - }); + fs.writeFileSync(scriptPath, script); - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - - assert.equal(payload.sandboxName, "my-assistant", "should reuse when user answers y"); - assert.ok( - payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox create")), - "should NOT recreate sandbox when user chooses to reuse", - ); - assert.ok( - payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox delete")), - "should NOT delete sandbox when user chooses to reuse", - ); - assert.ok( - result.stdout.includes("already exists"), - "should show 'already exists' message in interactive mode", - ); - }); + // Run WITHOUT NEMOCLAW_NON_INTERACTIVE to exercise interactive path + const env: Record = { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }; + delete env["NEMOCLAW_NON_INTERACTIVE"]; + delete env["NEMOCLAW_RECREATE_SANDBOX"]; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + + assert.equal(payload.sandboxName, "my-assistant", "should reuse when user answers y"); + assert.ok( + payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox create")), + "should NOT recreate sandbox when user chooses to reuse", + ); + assert.ok( + payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox delete")), + "should NOT delete sandbox when user chooses to reuse", + ); + assert.ok( + result.stdout.includes("already exists"), + "should show 'already exists' message in interactive mode", + ); + }, + ); + + it( + "interactive mode deletes and recreates sandbox when user confirms drift recreate", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-interactive-decline-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "interactive-decline.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); - it("interactive mode deletes and recreates sandbox when user confirms drift recreate", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-interactive-decline-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "interactive-decline.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); + const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); + let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1025,18 +1166,23 @@ runner.run = (command, opts = {}) => { if (cmd.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } - return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + return cmd.includes("sandbox get") && cmd.includes("my-assistant") + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + _sandboxId + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { commands.push({ type: "runFile", command: _n([file, ...args]), file, args, env: opts.env || null }); return { status: 0 }; }; -runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); - if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + runner.runCapture = (command) => { + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant", sandboxId: _sandboxId }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -1045,11 +1191,17 @@ runner.runCapture = (command) => { } return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive" }); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; + const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + toolDisclosure: "progressive", + }, { sandboxId: "sbx-4f2a91c0d7" }); + registry.getSandbox = () => sourceSandbox; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + }); const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); @@ -1057,8 +1209,9 @@ preflight.checkPortAvailable = async () => ({ ok: true }); // Mock prompt to return "y" (confirm recreate) credentials.prompt = async () => "y"; -childProcess.spawn = (...args) => { - _deleted = false; + childProcess.spawn = (...args) => { + _deleted = false; + _sandboxId = "sbx-8e6b10fd38"; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -1076,76 +1229,92 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - // Run WITHOUT NEMOCLAW_NON_INTERACTIVE to exercise interactive path - const env: Record = { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_RECREATE_WITHOUT_BACKUP: "1", - }; - delete env["NEMOCLAW_NON_INTERACTIVE"]; - delete env["NEMOCLAW_RECREATE_SANDBOX"]; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env, - }); + fs.writeFileSync(scriptPath, script); - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - - assert.ok( - payload.commands.some((entry: CommandEntry) => /sandbox.*delete/.test(String(entry.command))), - "should delete existing sandbox when user confirms recreate", - ); - assert.ok( - payload.commands.some((entry: CommandEntry) => /sandbox.*create/.test(String(entry.command))), - "should create a new sandbox when user confirms recreate", - ); - assert.ok( - result.stdout.includes("requested inference selection changed"), - "should show drift warning before prompting", - ); - }); + // Run WITHOUT NEMOCLAW_NON_INTERACTIVE to exercise interactive path + const env: Record = { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_RECREATE_WITHOUT_BACKUP: "1", + }; + delete env["NEMOCLAW_NON_INTERACTIVE"]; + delete env["NEMOCLAW_RECREATE_SANDBOX"]; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + + assert.ok( + payload.commands.some((entry: CommandEntry) => + /sandbox.*delete/.test(String(entry.command)), + ), + "should delete existing sandbox when user confirms recreate", + ); + assert.ok( + payload.commands.some((entry: CommandEntry) => + /sandbox.*create/.test(String(entry.command)), + ), + "should create a new sandbox when user confirms recreate", + ); + assert.ok( + result.stdout.includes("requested inference selection changed"), + "should show drift warning before prompting", + ); + }, + ); + + it( + "interactive mode auto-recreates when existing sandbox is not ready", + { + timeout: 60_000, + }, + async () => { + const repoRoot = path.join(import.meta.dirname, "../.."); + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-interactive-notready-"), + ); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "interactive-notready.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const registryPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "state", "registry.ts"), + ); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); - it("interactive mode auto-recreates when existing sandbox is not ready", { - timeout: 60_000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-interactive-notready-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "interactive-notready.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; + fs.mkdirSync(fakeBin, { recursive: true }); + writeOkOpenshell(fakeBin); + + const script = String.raw` + const runner = require(${runnerPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + fixtureMocks.mockStandaloneGatewayTeardownAuthority(); + const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); + let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1164,17 +1333,22 @@ runner.run = (command, opts = {}) => { if (cmd.includes("sandbox list")) { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } - return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + return cmd.includes("sandbox get") && cmd.includes("my-assistant") + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + _sandboxId + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -runner.runCapture = (command) => { - // Existing sandbox that is NOT ready initially, becomes Ready after recreation - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); - if (_n(command).includes("sandbox list")) { - return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; - } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; + runner.runCapture = (command) => { + // Existing sandbox that is NOT ready initially, becomes Ready after recreation + const cmd = _n(command); + if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; + if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant", sandboxId: _sandboxId }); + if (createdIdentity !== null) return createdIdentity; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + _sandboxId].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) { + return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + } + if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { defaultCurlOutput: "ok", @@ -1183,11 +1357,17 @@ runner.runCapture = (command) => { } return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive" }); -registry.registerSandbox = () => true; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; + const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + toolDisclosure: "progressive", + }, { sandboxId: "sbx-4f2a91c0d7" }); + registry.getSandbox = () => sourceSandbox; + const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", + getSandbox: registry.getSandbox, + }); const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); preflight.checkPortAvailable = async () => ({ ok: true }); @@ -1208,7 +1388,11 @@ const fakeSpawn = (...args) => { }); return child; }; -childProcess.spawn = (...args) => { _deleted = false; return fakeSpawn(...args); }; + childProcess.spawn = (...args) => { + _deleted = false; + _sandboxId = "sbx-8e6b10fd39"; + return fakeSpawn(...args); + }; // Also patch spawn inside the compiled sandbox-create-stream module. // It imports spawn at load time from "node:child_process", so patching the @@ -1225,251 +1409,52 @@ const { createSandbox } = require(${onboardPath}); (async () => { process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox(...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [null, "gpt-5.4", "nvidia-prod", null, "my-assistant", null, null, null, null, null, null, null, []], + createFixture, + )); console.log(JSON.stringify({ sandboxName, commands })); })().catch((error) => { console.error(error); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); - - // Run WITHOUT NEMOCLAW_NON_INTERACTIVE to exercise interactive path - const env: Record = { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_RECREATE_WITHOUT_BACKUP: "1", - }; - delete env["NEMOCLAW_NON_INTERACTIVE"]; - delete env["NEMOCLAW_RECREATE_SANDBOX"]; - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env, - }); - - assert.equal(result.status, 0, result.stderr); - const payloadLine = result.stdout - .trim() - .split("\n") - .slice() - .reverse() - .find((line) => line.startsWith("{") && line.endsWith("}")); - assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); - const payload = JSON.parse(payloadLine); - - assert.ok( - payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox delete")), - "should delete not-ready sandbox after user confirms", - ); - assert.ok( - payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox create")), - "should recreate sandbox when existing one is not ready", - ); - assert.ok(result.stdout.includes("not ready"), "should mention sandbox is not ready"); - }); - it("registers a fresh create only after owner-scoped identity confirmation (#8942)", { - timeout: 45000, - }, async () => { - const repoRoot = path.join(import.meta.dirname, "../.."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-ready-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "create-sandbox-ready-check.js"); - const payloadPath = path.join(tmpDir, "payload.json"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const preflightPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), - ); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const dockerExecPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "adapters", "docker", "exec.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; -const registry = require(${registryPath}); -const preflight = require(${preflightPath}); -const credentials = require(${credentialsPath}); -const childProcess = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const dockerExec = require(${dockerExecPath}); -dockerExec.dockerSpawn = () => { - const child = new EventEmitter(); - process.nextTick(() => child.emit("close", 0)); - return child; -}; -const fs = require("node:fs"); - -const commands = []; -const lifecycleObservationCommands = []; -let sandboxListCalls = 0; -let dockerPsCalls = 0; -let sandboxCreated = false; -let registeredSandbox = null; -const keepAlive = setInterval(() => {}, 1000); -runner.run = (command, opts = {}) => { - const cmd = _n(command); - const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); - if (profileResult !== null) return profileResult; - _deleted = _deleted || cmd.includes("sandbox delete"); - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("sandbox list")) { - return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; - } - return cmd.includes("sandbox get") && cmd.includes("my-assistant") && sandboxCreated - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-fresh-create\n"), stderr: Buffer.alloc(0) } - : { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") || _n(command).includes("sandbox list")) { - lifecycleObservationCommands.push(_n(command)); - } - if (_n(command).startsWith("docker ps -a --no-trunc ")) { - dockerPsCalls += 1; - if (dockerPsCalls === 1) return "a".repeat(64); - } - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) { - return sandboxCreated ? ["my-assistant", "Id: sbx-fresh-create"].join(String.fromCharCode(10)) : ""; - } - if (_n(command).includes("sandbox list")) { - sandboxListCalls += 1; - return sandboxListCalls >= 2 ? "my-assistant Ready" : "my-assistant Pending"; - } - { - const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); - if (mockedCapture !== null) return mockedCapture; - } - if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; - return ""; -}; -registry.registerSandbox = (entry) => { registeredSandbox = entry; return true; }; -registry.updateSandbox = () => true; -registry.setDefault = () => true; -registry.removeSandbox = () => true; -preflight.checkPortAvailable = async () => ({ ok: true }); -credentials.prompt = async () => ""; - -const groupKillCalls = []; -const realProcessKill = process.kill.bind(process); -process.kill = (pid, signal) => { - if (pid < 0) { - groupKillCalls.push({ pid, signal }); - const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); - process.nextTick(() => createCommand.child.emit("close", signal === "SIGTERM" ? 0 : 1)); - return true; - } - return realProcessKill(pid, signal); -}; - -childProcess.spawn = (...args) => { - sandboxCreated = true; - _deleted = false; - const child = new EventEmitter(); - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); - child.unref = () => {}; - child.pid = 4242; - child.killCalls = []; - child.unrefCalls = 0; - child.stdout.destroyCalls = 0; - child.stderr.destroyCalls = 0; - child.stdout.destroy = () => { - child.stdout.destroyCalls += 1; - }; - child.stderr.destroy = () => { - child.stderr.destroyCalls += 1; - }; - child.unref = () => { - child.unrefCalls += 1; - }; - child.kill = (signal) => { - child.killCalls.push(signal); - process.nextTick(() => child.emit("close", signal === "SIGTERM" ? 0 : 1)); - return true; - }; - commands.push({ command: _n([args[0], ...(Array.isArray(args[1]) ? args[1] : [])]), env: args[2]?.env || null, child }); - process.nextTick(() => { - child.stdout.emit("data", Buffer.from("Created sandbox: my-assistant\n")); - child.stderr.emit("data", Buffer.from("Setting up NemoClaw...\n")); - }); - return child; -}; - -const { createSandbox } = require(${onboardPath}); - -(async () => { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod"); - const createCommand = commands.find((entry) => entry.command.includes("sandbox create")); - fs.writeFileSync(${JSON.stringify(payloadPath)}, JSON.stringify({ - sandboxName, - sandboxListCalls, - killCalls: createCommand.child.killCalls, - groupKillCalls, - unrefCalls: createCommand.child.unrefCalls, - stdoutDestroyCalls: createCommand.child.stdout.destroyCalls, - stderrDestroyCalls: createCommand.child.stderr.destroyCalls, - lifecycleObservationCommands, - registeredSandbox, - })); - clearInterval(keepAlive); -})().catch((error) => { - clearInterval(keepAlive); - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + fs.writeFileSync(scriptPath, script); - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { + // Run WITHOUT NEMOCLAW_NON_INTERACTIVE to exercise interactive path + const env: Record = { ...process.env, HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - OPENSHELL_DRIVERS: "docker", - }, - timeout: 30000, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(fs.readFileSync(payloadPath, "utf8")); - assert.equal(payload.sandboxName, "my-assistant"); - assert.ok(payload.sandboxListCalls >= 2); - assert.deepEqual(payload.groupKillCalls, [{ pid: -4242, signal: "SIGTERM" }]); - assert.deepEqual(payload.killCalls, []); - assert.equal(payload.unrefCalls, 1); - assert.equal(payload.stdoutDestroyCalls, 1); - assert.equal(payload.stderrDestroyCalls, 1); - assert.match(payload.registeredSandbox.lifecycleGeneration, /^[0-9a-f-]{36}$/u); - assert.equal( - payload.registeredSandbox.lifecycleLiveIdentityFingerprint, - createHash("sha256").update("sbx-fresh-create").digest("hex"), - ); - const ownerScopedObservations = payload.lifecycleObservationCommands.filter( - (command: string) => command.includes("-g nemoclaw"), - ); - assert.equal(ownerScopedObservations.length, 6); - assert.ok( - ownerScopedObservations.every( - (command: string) => - command.includes("sandbox get -g nemoclaw my-assistant") || - command.includes("sandbox list -g nemoclaw"), - ), - `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, - ); - }); + NEMOCLAW_RECREATE_WITHOUT_BACKUP: "1", + }; + delete env["NEMOCLAW_NON_INTERACTIVE"]; + delete env["NEMOCLAW_RECREATE_SANDBOX"]; + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env, + }); + + assert.equal(result.status, 0, result.stderr); + const payloadLine = result.stdout + .trim() + .split("\n") + .slice() + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert.ok(payloadLine, `expected JSON payload in stdout:\n${result.stdout}`); + const payload = JSON.parse(payloadLine); + + assert.ok( + payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox delete")), + "should delete not-ready sandbox after user confirms", + ); + assert.ok( + payload.commands.some((entry: CommandEntry) => entry.command.includes("sandbox create")), + "should recreate sandbox when existing one is not ready", + ); + assert.ok(result.stdout.includes("not ready"), "should mention sandbox is not ready"); + }, + ); }); diff --git a/test/onboarding/onboard-terminal-dashboard.test.ts b/test/onboarding/onboard-terminal-dashboard.test.ts index 1cb1b50d152..4b145aaa767 100644 --- a/test/onboarding/onboard-terminal-dashboard.test.ts +++ b/test/onboarding/onboard-terminal-dashboard.test.ts @@ -55,6 +55,7 @@ const os = require("node:os"); const path = require("node:path"); const runner = require(${runnerPath}); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const agentDefs = require(${agentDefsPath}); const agentOnboard = require(${agentOnboardPath}); const dockerGpuSandboxCreate = require(${dockerGpuSandboxCreatePath}); @@ -107,6 +108,10 @@ runner.runFile = (file, args = [], opts = {}) => { }; runner.runCapture = (command) => { const normalized = _n(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { + sandboxName, + }); + if (createdIdentity !== null) return createdIdentity; commands.push({ command: normalized, env: null }); if ( normalized.includes( @@ -124,7 +129,7 @@ runner.runCapture = (command) => { } if (normalized.includes("sandbox get") && normalized.includes(sandboxName)) { return scenario === "reuse" - ? [sandboxName, "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) + ? [sandboxName, "Id: fixture-created-sandbox"].join(String.fromCharCode(10)) : ""; } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; @@ -134,15 +139,14 @@ runner.runCapture = (command) => { registry.getSandbox = () => scenario === "reuse" - ? { + ? fixtureMocks.managedSandboxPolicyReceiptFixture({ name: sandboxName, gpuEnabled: false, agent: "langchain-deepagents-code", dashboardPort: 18789, observabilityEnabled: false, - policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive", - } + }) : null; registry.registerSandbox = (entry) => { registerCalls.push(entry); @@ -154,6 +158,16 @@ registry.updateSandbox = (name, updates) => { }; registry.setDefault = () => true; registry.removeSandbox = () => true; +const createFixture = + scenario === "create" + ? fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName, + provider: "nvidia-prod", + model: "gpt-5.4", + registerSandbox: (entry) => registerCalls.push(entry), + updateSandbox: (name, updates) => updateCalls.push({ name, updates }), + }) + : null; childProcess.spawn = (...args) => { if (scenario === "reuse") throw new Error("unexpected sandbox create"); @@ -177,7 +191,26 @@ const agent = agentDefs.loadAgent("langchain-deepagents-code"); process.env.OPENSHELL_GATEWAY = "nemoclaw"; process.env.CHAT_UI_URL = "https://chat.example.test:19000"; process.env.NEMOCLAW_DASHBOARD_PORT = "19000"; - const resultName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, sandboxName, null, null, null, agent); + const createArgs = [ + null, + "gpt-5.4", + "nvidia-prod", + null, + sandboxName, + null, + null, + null, + agent, + null, + null, + null, + [], + ]; + const resultName = await createSandbox( + ...(createFixture + ? fixtureMocks.sandboxCreateArgsWithVerifiedReservation(createArgs, createFixture) + : createArgs), + ); console.log(JSON.stringify({ resultName, commands, registerCalls, updateCalls })); clearInterval(keepAlive); })().catch((error) => { diff --git a/test/onboarding/onboard.test.ts b/test/onboarding/onboard.test.ts index 150df5e0887..9ec4a8a9e68 100644 --- a/test/onboarding/onboard.test.ts +++ b/test/onboarding/onboard.test.ts @@ -678,29 +678,33 @@ startGateway(null).catch((error) => { ); fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const childProcess = require("node:child_process"); + const runner = require(${runnerPath}); + const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); + const registry = require(${registryPath}); + const fixtureMocks = require(${onboardScriptMocksPath}); + const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); const commands = []; runner.run = (command, opts = {}) => { commands.push({ command: _n(command), env: opts.env || null }); - const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); + const profileResult = fixtureMocks.mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); if (profileResult !== null) return profileResult; return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: fixture-created-sandbox"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; -registry.getSandbox = () => ({ name: "my-assistant", policyAuthority: "nemoclaw-managed", toolDisclosure: "progressive" }); + registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture({ + name: "my-assistant", + toolDisclosure: "progressive", + }); childProcess.spawn = (...args) => { const child = new EventEmitter(); @@ -754,7 +758,10 @@ const { createSandbox } = require(${onboardPath}); payload.commands.every((entry: CommandEntry) => !entry.command.includes("sandbox create")), "did not expect sandbox create when reusing existing sandbox", ); - assert.match(result.stdout, /\[reuse\] Skipping sandbox \(my-assistant\)/); + assert.match( + result.stdout, + /Existing provider\/model selection is unreadable; reusing sandbox\./, + ); }); it("accepts gateway inference when system inference is separately not configured", async () => { diff --git a/test/package-contract/openshell-policy-boundary.test.ts b/test/package-contract/openshell-policy-boundary.test.ts index 26897e88e5d..77b69f3c9dd 100644 --- a/test/package-contract/openshell-policy-boundary.test.ts +++ b/test/package-contract/openshell-policy-boundary.test.ts @@ -70,9 +70,9 @@ describe("OpenShell policy boundary package contract", () => { yamlBody: string; policy: Record; }; - parseGlobalPolicyAuthorityMetadata: ( + parseActiveGlobalPolicyAuthorityMetadata: ( raw: string, - ) => { authority: string; effectivePolicy: Record }; + ) => { state: string; inspection?: { authority: string } }; parseSandboxPolicyAuthorityMetadata: ( raw: string, sandboxName: string, @@ -97,7 +97,7 @@ describe("OpenShell policy boundary package contract", () => { yamlBody: string; policy: Record; }; - parseGlobalPolicyAuthorityMetadata: typeof cliPolicy.parseGlobalPolicyAuthorityMetadata; + parseActiveGlobalPolicyAuthorityMetadata: typeof cliPolicy.parseActiveGlobalPolicyAuthorityMetadata; parseSandboxPolicyAuthorityMetadata: typeof cliPolicy.parseSandboxPolicyAuthorityMetadata; withoutProviderComposedPolicies: ( policies: Record, @@ -108,7 +108,7 @@ describe("OpenShell policy boundary package contract", () => { require("../../nemoclaw/dist/shared/openshell-policy-boundary.cjs") as { assertExternalPolicyRequirementContainment: typeof cliPolicy.assertExternalPolicyRequirementContainment; assertMatchingPolicyAuthority: typeof cliPolicy.assertMatchingPolicyAuthority; - parseGlobalPolicyAuthorityMetadata: typeof cliPolicy.parseGlobalPolicyAuthorityMetadata; + parseActiveGlobalPolicyAuthorityMetadata: typeof cliPolicy.parseActiveGlobalPolicyAuthorityMetadata; parseOpenShellPolicy: typeof cliPolicy.parseOpenShellPolicy; parseSandboxPolicyAuthorityMetadata: typeof cliPolicy.parseSandboxPolicyAuthorityMetadata; stripProviderComposedPolicies: typeof cliPolicy.stripProviderComposedPolicies; @@ -136,8 +136,8 @@ describe("OpenShell policy boundary package contract", () => { expect(cliPolicy.stripProviderComposedPolicies).toBe( canonicalBoundary.stripProviderComposedPolicies, ); - expect(cliPolicy.parseGlobalPolicyAuthorityMetadata).toBe( - canonicalBoundary.parseGlobalPolicyAuthorityMetadata, + expect(cliPolicy.parseActiveGlobalPolicyAuthorityMetadata).toBe( + canonicalBoundary.parseActiveGlobalPolicyAuthorityMetadata, ); expect(cliPolicy.parseSandboxPolicyAuthorityMetadata).toBe( canonicalBoundary.parseSandboxPolicyAuthorityMetadata, @@ -153,6 +153,8 @@ describe("OpenShell policy boundary package contract", () => { sandbox: "alpha", status: "effective", policy_source: "global", + hash: "sha256:sandbox", + active_version: 1, policy: { version: 1, network_policies: {} }, }); expect(pluginBoundary.parseSandboxPolicyAuthorityMetadata(sandboxMetadata, "alpha")).toEqual( @@ -162,10 +164,12 @@ describe("OpenShell policy boundary package contract", () => { scope: "global", status: "loaded", policy_source: "global", + hash: "sha256:global", + active_version: 1, policy: { version: 1, network_policies: {} }, }); - expect(pluginBoundary.parseGlobalPolicyAuthorityMetadata(globalMetadata)).toEqual( - canonicalBoundary.parseGlobalPolicyAuthorityMetadata(globalMetadata), + expect(pluginBoundary.parseActiveGlobalPolicyAuthorityMetadata(globalMetadata)).toEqual( + canonicalBoundary.parseActiveGlobalPolicyAuthorityMetadata(globalMetadata), ); const pluginRunner = await import( diff --git a/test/runtime/policy/managed-policy-receipt-fixture.ts b/test/runtime/policy/managed-policy-receipt-fixture.ts new file mode 100644 index 00000000000..11e9ed20c4b --- /dev/null +++ b/test/runtime/policy/managed-policy-receipt-fixture.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export * from "../../helpers/managed-policy-receipt-fixture"; diff --git a/test/runtime/policy/policies-permissive-policy.test.ts b/test/runtime/policy/policies-permissive-policy.test.ts index 58b5138ed7e..42a759660f0 100644 --- a/test/runtime/policy/policies-permissive-policy.test.ts +++ b/test/runtime/policy/policies-permissive-policy.test.ts @@ -9,6 +9,12 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import YAML from "yaml"; +import { + managedPolicyMetadata, + managedRegistrationSource, + SANDBOX_ID, +} from "./managed-policy-receipt-fixture"; + const REPO_ROOT = path.join(import.meta.dirname, "../../.."); const POLICIES_PATH = JSON.stringify(path.join(REPO_ROOT, "src", "lib", "policy", "index.ts")); const REGISTRY_PATH = JSON.stringify(path.join(REPO_ROOT, "src", "lib", "state", "registry.ts")); @@ -35,15 +41,27 @@ function runHermesPermissivePolicy(policySetStatus: number): { const script = String.raw` const registry = require(${REGISTRY_PATH}); const policies = require(${POLICIES_PATH}); -registry.registerSandbox({ name: "hermes-sandbox", agent: "hermes", policies: [] }); +${managedRegistrationSource("hermes-sandbox", "hermes")} policies.applyPermissivePolicy("hermes-sandbox"); `; fs.writeFileSync( fakeOpenshell, `#!/usr/bin/env bash set -euo pipefail +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: hermes-sandbox\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi if [ "$1 $2" = "policy get" ]; then - printf '%s\n' '{"scope":"sandbox","sandbox":"hermes-sandbox","status":"effective","policy_source":"sandbox","policy":{}}' + if [[ " $* " == *" --output json "* ]]; then + printf '%s\n' ${JSON.stringify(managedPolicyMetadata("hermes-sandbox"))} + exit 0 + fi + if [ -f ${JSON.stringify(policyOut)} ]; then + cat ${JSON.stringify(policyOut)} + else + printf 'Version: 1\nHash: fixture-policy\n---\nversion: 1\n\nnetwork_policies: {}\n' + fi exit 0 fi if [ "$1 $2" = "policy set" ]; then diff --git a/test/runtime/policy/policies-teams.test.ts b/test/runtime/policy/policies-teams.test.ts index 4bc61c004f8..5d7063e2d7e 100644 --- a/test/runtime/policy/policies-teams.test.ts +++ b/test/runtime/policy/policies-teams.test.ts @@ -9,6 +9,11 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import * as policies from "../../../src/lib/policy"; +import { + managedPolicyMetadata, + managedRegistrationSource, + SANDBOX_ID, +} from "../../helpers/managed-policy-receipt-fixture"; const requireForTest = createRequire(import.meta.url); const YAML = requireForTest("yaml"); @@ -87,7 +92,7 @@ describe("Teams policy preset", () => { const fs = require("node:fs"); const registry = require(${REGISTRY_PATH}); const policies = require(${POLICIES_PATH}); -registry.registerSandbox({ name: "hermes-sandbox", agent: "hermes", policies: [] }); +${managedRegistrationSource("hermes-sandbox", "hermes")} const result = policies.applyPresets("hermes-sandbox", ["teams"]); process.stdout.write("\n__RESULT__" + JSON.stringify({ result, @@ -99,9 +104,17 @@ process.stdout.write("\n__RESULT__" + JSON.stringify({ fakeOpenshell, `#!/usr/bin/env bash set -euo pipefail +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: hermes-sandbox\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi if [ "$1 $2" = "policy get" ]; then if [[ " $* " == *" --output json "* ]]; then - printf '%s\n' '{"scope":"sandbox","sandbox":"hermes-sandbox","status":"effective","policy_source":"sandbox","policy":{}}' + printf '%s\n' ${JSON.stringify(managedPolicyMetadata("hermes-sandbox"))} + exit 0 + fi + if [ -f ${JSON.stringify(policyOut)} ]; then + cat ${JSON.stringify(policyOut)} exit 0 fi printf 'Version: 1\nHash: test\n---\nversion: 1\n\nnetwork_policies: {}\n' diff --git a/test/runtime/policy/policies.test.ts b/test/runtime/policy/policies.test.ts index a83ec901372..6fca5c4b412 100644 --- a/test/runtime/policy/policies.test.ts +++ b/test/runtime/policy/policies.test.ts @@ -8,6 +8,17 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + managedPolicyMetadata, + managedRegistrationSource, + managedSandboxEntry, + parseResultPayload, + POLICY_HASH, + POLICY_VERSION, + SANDBOX_ID, + SANDBOX_IDENTITY, +} from "./managed-policy-receipt-fixture"; + const requireForTest = createRequire(import.meta.url); const YAML = requireForTest("yaml"); const REPO_ROOT = path.join(import.meta.dirname, "../../.."); @@ -35,29 +46,17 @@ function requirePresetContent(content: string | null): string { return content; } -function parseResultPayload(stdout: string): any { - const marker = "__RESULT__"; - const markerIndex = stdout.indexOf(marker); - expect(markerIndex).toBeGreaterThanOrEqual(0); - return JSON.parse(stdout.slice(markerIndex + marker.length)); -} - -function managedPolicyMetadata(sandboxName: string): string { - return JSON.stringify({ - scope: "sandbox", - sandbox: sandboxName, - status: "effective", - policy_source: "sandbox", - policy: { version: 1, network_policies: {} }, - }); -} - describe("policies", () => { beforeEach(() => { vi.spyOn(policyAuthorityModule, "inspectSandboxPolicyAuthority").mockReturnValue({ - authority: "nemoclaw-managed", + authority: "owner-unknown", effectivePolicy: {}, + policyIdentity: { hash: POLICY_HASH, activeVersion: POLICY_VERSION }, }); + vi.spyOn(policyAuthorityModule, "inspectOpenShellSandboxIdentityFingerprint").mockReturnValue( + SANDBOX_IDENTITY, + ); + vi.spyOn(registryForTest, "compareAndSetSandboxPolicyCreationReceipt").mockReturnValue(true); }); afterEach(() => { @@ -174,7 +173,7 @@ describe("policies", () => { const fs = require("node:fs"); const registry = require(${REGISTRY_PATH}); const policies = require(${POLICIES_PATH}); -registry.registerSandbox({ name: "test-sandbox", policies: [] }); +${managedRegistrationSource("test-sandbox")} const result = policies.applyPresets("test-sandbox", ["npm", "pypi"]); process.stdout.write("\n__RESULT__" + JSON.stringify({ result, @@ -188,12 +187,20 @@ process.stdout.write("\n__RESULT__" + JSON.stringify({ `#!/usr/bin/env bash set -euo pipefail printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: test-sandbox\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi if [ "$1 $2" = "policy get" ]; then if [[ " $* " == *" --output json "* ]]; then printf '%s\n' ${JSON.stringify(managedPolicyMetadata("test-sandbox"))} exit 0 fi - printf 'Version: 1\nHash: test\n---\nversion: 1\n\nnetwork_policies: {}\n' + if [ -f ${JSON.stringify(policyOut)} ]; then + cat ${JSON.stringify(policyOut)} + else + printf 'Version: 1\nHash: test\n---\nversion: 1\n\nnetwork_policies: {}\n' + fi exit 0 fi if [ "$1 $2" = "policy set" ]; then @@ -252,7 +259,7 @@ exit 1 const fs = require("node:fs"); const registry = require(${REGISTRY_PATH}); const policies = require(${POLICIES_PATH}); -registry.registerSandbox({ name: "hermes-sandbox", agent: "hermes", policies: [] }); +${managedRegistrationSource("hermes-sandbox", "hermes")} const result = policies.applyPresets("hermes-sandbox", ["discord"]); process.stdout.write("\n__RESULT__" + JSON.stringify({ result, @@ -264,12 +271,20 @@ process.stdout.write("\n__RESULT__" + JSON.stringify({ fakeOpenshell, `#!/usr/bin/env bash set -euo pipefail +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: hermes-sandbox\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi if [ "$1 $2" = "policy get" ]; then if [[ " $* " == *" --output json "* ]]; then printf '%s\n' ${JSON.stringify(managedPolicyMetadata("hermes-sandbox"))} exit 0 fi - printf 'Version: 1\nHash: test\n---\nversion: 1\n\nnetwork_policies: {}\n' + if [ -f ${JSON.stringify(policyOut)} ]; then + cat ${JSON.stringify(policyOut)} + else + printf 'Version: 1\nHash: test\n---\nversion: 1\n\nnetwork_policies: {}\n' + fi exit 0 fi if [ "$1 $2" = "policy set" ]; then @@ -336,7 +351,7 @@ exit 1 const fs = require("node:fs"); const registry = require(${REGISTRY_PATH}); const policies = require(${POLICIES_PATH}); -registry.registerSandbox({ name: "hermes-sandbox", agent: "hermes", policies: [] }); +${managedRegistrationSource("hermes-sandbox", "hermes")} const result = policies.applyPresets("hermes-sandbox", ["wechat"]); process.stdout.write("\n__RESULT__" + JSON.stringify({ result, @@ -348,12 +363,20 @@ process.stdout.write("\n__RESULT__" + JSON.stringify({ fakeOpenshell, `#!/usr/bin/env bash set -euo pipefail +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: hermes-sandbox\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi if [ "$1 $2" = "policy get" ]; then if [[ " $* " == *" --output json "* ]]; then printf '%s\n' ${JSON.stringify(managedPolicyMetadata("hermes-sandbox"))} exit 0 fi - printf 'Version: 1\nHash: test\n---\nversion: 1\n\nnetwork_policies: {}\n' + if [ -f ${JSON.stringify(policyOut)} ]; then + cat ${JSON.stringify(policyOut)} + else + printf 'Version: 1\nHash: test\n---\nversion: 1\n\nnetwork_policies: {}\n' + fi exit 0 fi if [ "$1 $2" = "policy set" ]; then @@ -415,12 +438,9 @@ exit 1 ); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const sandboxSpy = vi.spyOn(registryForTest, "getSandbox").mockReturnValue({ - name: "test-sandbox", - agent: "openclaw", - policies: [], - policyAuthority: "nemoclaw-managed", - }); + const sandboxSpy = vi + .spyOn(registryForTest, "getSandbox") + .mockReturnValue(managedSandboxEntry("test-sandbox")); vi.stubEnv("NEMOCLAW_OPENSHELL_BIN", fakeOpenshell); try { try { @@ -705,12 +725,9 @@ exit 1 const mkdtempSpy = vi.spyOn(fs, "mkdtempSync"); const errSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); - const sandboxSpy = vi.spyOn(registryForTest, "getSandbox").mockReturnValue({ - name: "my-assistant", - agent: "openclaw", - policies: [], - policyAuthority: "nemoclaw-managed", - }); + const sandboxSpy = vi + .spyOn(registryForTest, "getSandbox") + .mockReturnValue(managedSandboxEntry("my-assistant")); const exitSpy = vi.spyOn(process, "exit").mockImplementation(((_code?: number) => { throw new Error("__test_exit__"); }) as never); @@ -761,10 +778,7 @@ exit 1 .mockReturnValue(fakeOpenshell); savedGetSandbox = registryModule.getSandbox; savedAddCustomPolicy = registryModule.addCustomPolicy; - registryModule.getSandbox = (name: string) => ({ - name, - policyAuthority: "nemoclaw-managed", - }); + registryModule.getSandbox = (name: string) => managedSandboxEntry(name); registryModule.addCustomPolicy = () => true; }); @@ -845,9 +859,24 @@ network_policies: const localBin = path.join(tmpHome, ".local", "bin"); fs.mkdirSync(localBin, { recursive: true }); fakeOpenshell = path.join(localBin, "openshell"); + const appliedPolicyPath = path.join(tmpHome, "applied-policy.yaml"); fs.writeFileSync( fakeOpenshell, - "#!/bin/sh\nprintf 'version: 1\\nnetwork_policies: {}\\n'\nexit 0\n", + `#!/bin/sh +if [ "$1 $2" = "policy set" ]; then + while [ "$#" -gt 0 ]; do + if [ "$1" = "--policy" ]; then cp "$2" ${JSON.stringify(appliedPolicyPath)}; break; fi + shift + done + exit 0 +fi +if [ -f ${JSON.stringify(appliedPolicyPath)} ]; then + cat ${JSON.stringify(appliedPolicyPath)} +else + printf 'version: 1\nnetwork_policies: {}\n' +fi +exit 0 +`, { mode: 0o755 }, ); origHome = process.env.HOME; @@ -892,7 +921,7 @@ network_policies: expect(addSpy).not.toHaveBeenCalled(); const combined = errors.join("\n"); expect(combined).toContain("my-assistant"); - expect(combined).toContain("could not record policy authority"); + expect(combined).toContain("policy authority is unavailable"); } finally { errSpy.mockRestore(); logSpy.mockRestore(); @@ -914,7 +943,7 @@ network_policies: expect(updateSpy).not.toHaveBeenCalled(); const combined = errors.join("\n"); expect(combined).toContain("my-assistant"); - expect(combined).toContain("could not record policy authority"); + expect(combined).toContain("policy authority is unavailable"); } finally { errSpy.mockRestore(); logSpy.mockRestore(); @@ -922,7 +951,7 @@ network_policies: }); it("applies a well-formed custom preset and records it verbatim (#9406)", () => { - let sandbox: Record = { name: "my-assistant" }; + let sandbox: Record = managedSandboxEntry("my-assistant"); registryModule.getSandbox = () => sandbox; registryModule.updateSandbox = (_name: string, updates: Record) => { sandbox = { ...sandbox, ...updates }; diff --git a/test/runtime/policy/policy-explain-cli.test.ts b/test/runtime/policy/policy-explain-cli.test.ts index cda2b07cf91..aac129948bc 100644 --- a/test/runtime/policy/policy-explain-cli.test.ts +++ b/test/runtime/policy/policy-explain-cli.test.ts @@ -7,6 +7,13 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + managedSandboxEntry, + POLICY_HASH, + POLICY_VERSION, + SANDBOX_ID, +} from "./managed-policy-receipt-fixture"; + const CLI = path.join(import.meta.dirname, "../../..", "bin", "nemoclaw.js"); type CliResult = { @@ -58,9 +65,28 @@ function writeRegistry(home: string, sandboxes: Record): void { } let scratchHome: string; +let fakeOpenshell: string; beforeEach(() => { scratchHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-policy-explain-")); + fakeOpenshell = path.join(scratchHome, "openshell"); + fs.writeFileSync( + fakeOpenshell, + `#!/usr/bin/env bash +set -euo pipefail +sandbox_name="\${!#}" +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: %s\nId: ${SANDBOX_ID}\nPhase: Ready\n' "$sandbox_name" + exit 0 +fi +if [ "$1 $2" = "policy get" ]; then + printf '{"scope":"sandbox","sandbox":"%s","status":"effective","policy_source":"sandbox","hash":"${POLICY_HASH}","active_version":${POLICY_VERSION},"policy":{"version":1,"network_policies":{}}}\n' "$sandbox_name" + exit 0 +fi +exit 1 +`, + { mode: 0o755 }, + ); }); afterEach(() => { @@ -87,7 +113,7 @@ describe("nemoclaw policy-explain (E2E)", () => { it("emits a redacted markdown summary for a sandbox with applied presets", () => { writeRegistry(scratchHome, { "policy-explain-e2e": { - name: "policy-explain-e2e", + ...managedSandboxEntry("policy-explain-e2e"), createdAt: "2026-06-07T00:00:00.000Z", policies: ["slack"], policyTier: "balanced", @@ -95,7 +121,10 @@ describe("nemoclaw policy-explain (E2E)", () => { }, }); - const result = runCli({ HOME: scratchHome }, ["policy-explain-e2e", "policy-explain"]); + const result = runCli({ HOME: scratchHome, NEMOCLAW_OPENSHELL_BIN: fakeOpenshell }, [ + "policy-explain-e2e", + "policy-explain", + ]); expect(result.status).toBe(0); expect(result.stdout).toContain("# Sandbox policy context: policy-explain-e2e"); @@ -111,7 +140,7 @@ describe("nemoclaw policy-explain (E2E)", () => { it("emits a structured JSON object when --json is set", () => { writeRegistry(scratchHome, { "policy-explain-json": { - name: "policy-explain-json", + ...managedSandboxEntry("policy-explain-json"), createdAt: "2026-06-07T00:00:00.000Z", policies: ["github"], policyTier: "balanced", @@ -119,7 +148,7 @@ describe("nemoclaw policy-explain (E2E)", () => { }, }); - const result = runCli({ HOME: scratchHome }, [ + const result = runCli({ HOME: scratchHome, NEMOCLAW_OPENSHELL_BIN: fakeOpenshell }, [ "policy-explain-json", "policy-explain", "--json", diff --git a/test/runtime/policy/policy-mutation-read-failure.test.ts b/test/runtime/policy/policy-mutation-read-failure.test.ts index 19b5cd05ecc..59b2b8004f3 100644 --- a/test/runtime/policy/policy-mutation-read-failure.test.ts +++ b/test/runtime/policy/policy-mutation-read-failure.test.ts @@ -7,6 +7,13 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + managedSandboxEntry, + POLICY_HASH, + POLICY_VERSION, + SANDBOX_IDENTITY, +} from "./managed-policy-receipt-fixture"; + const requireForTest = createRequire(import.meta.url); const policies = requireForTest( path.join(import.meta.dirname, "..", "../..", "src", "lib", "policy", "index.ts"), @@ -45,14 +52,14 @@ describe("OpenShell policy mutation read failures", () => { beforeEach(() => { vi.spyOn(policyAuthority, "inspectSandboxPolicyAuthority").mockReturnValue({ - authority: "nemoclaw-managed", + authority: "owner-unknown", effectivePolicy: {}, + policyIdentity: { hash: POLICY_HASH, activeVersion: POLICY_VERSION }, }); - vi.spyOn(registry, "getSandbox").mockReturnValue({ - name: "alpha", - agent: "openclaw", - policyAuthority: "nemoclaw-managed", - }); + vi.spyOn(policyAuthority, "inspectOpenShellSandboxIdentityFingerprint").mockReturnValue( + SANDBOX_IDENTITY, + ); + vi.spyOn(registry, "getSandbox").mockReturnValue(managedSandboxEntry("alpha")); }); afterEach(() => { diff --git a/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts b/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts index cc2c63b5422..b24498f46ea 100644 --- a/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts +++ b/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts @@ -8,6 +8,13 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { + managedPolicyMetadata, + managedSandboxEntry, + parseResultPayload, + SANDBOX_ID, +} from "../../helpers/managed-policy-receipt-fixture"; + const requireForTest = createRequire(import.meta.url); const YAML = requireForTest("yaml"); const REPO_ROOT = path.join(import.meta.dirname, "../../.."); @@ -109,20 +116,21 @@ function olderNpmEntry() { return entry; } -function parseResultPayload(stdout: string): any { - const marker = "__RESULT__"; - const markerIndex = stdout.indexOf(marker); - expect(markerIndex).toBeGreaterThanOrEqual(0); - return JSON.parse(stdout.slice(markerIndex + marker.length)); -} - type LiveScenario = { + sandboxName: string; + policies: string[]; initialPolicy: string; childScript: string; setMode?: "success" | "fail" | "fail-once"; }; -function runLiveScenario({ initialPolicy, childScript, setMode = "success" }: LiveScenario): { +function runLiveScenario({ + sandboxName, + policies: registeredPolicies, + initialPolicy, + childScript, + setMode = "success", +}: LiveScenario): { calls: string[]; payload: any; stderr: string; @@ -139,15 +147,16 @@ function runLiveScenario({ initialPolicy, childScript, setMode = "success" }: Li fakeOpenshell, `#!/usr/bin/env bash set -euo pipefail +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: %s\nId: ${SANDBOX_ID}\nPhase: Ready\n' ${JSON.stringify(sandboxName)} + exit 0 +fi if [ "$1 $2" = "policy get" ] && [[ " $* " == *" --output json "* ]]; then - sandbox="" - for sandbox in "$@"; do :; done - printf '{"scope":"sandbox","sandbox":"%s","status":"effective","policy_source":"sandbox","policy":{}}\n' "$sandbox" + printf '%s\n' ${JSON.stringify(managedPolicyMetadata(sandboxName))} exit 0 fi printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} if [ "$1 $2" = "policy get" ]; then - printf 'Version: 1\nHash: test\n---\n' cat ${JSON.stringify(currentPolicyPath)} exit 0 fi @@ -180,6 +189,10 @@ const fs = require("node:fs"); const YAML = require("yaml"); const registry = require(${REGISTRY_PATH}); const policies = require(${POLICIES_PATH}); +registry.registerSandbox(${JSON.stringify({ + ...managedSandboxEntry(sandboxName), + policies: registeredPolicies, + })}); ${childScript} `; @@ -225,13 +238,10 @@ describe("OpenClaw npm compatibility policy lifecycle", () => { personal_open_internet: structuredClone(REVIEWED_PERSONAL_ENTRY), }); const { calls, payload } = runLiveScenario({ + sandboxName: "personal-owner", + policies: ["personal-open-internet", "npm"], initialPolicy, childScript: ` -registry.registerSandbox({ - name: "personal-owner", - agent: "openclaw", - policies: ["personal-open-internet", "npm"], -}); const removed = policies.removePreset("personal-owner", "npm"); process.stdout.write("\\n__RESULT__" + JSON.stringify({ removed, @@ -252,13 +262,10 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ personal_open_internet: structuredClone(REVIEWED_PERSONAL_ENTRY), }); const { calls, payload, stderr } = runLiveScenario({ + sandboxName: "personal-pending", + policies: ["personal-open-internet", "npm"], initialPolicy, childScript: ` -registry.registerSandbox({ - name: "personal-pending", - agent: "openclaw", - policies: ["personal-open-internet", "npm"], -}); registry.beginBaselineExclusionTransition("personal-pending", { id: "123e4567-e89b-42d3-a456-426614174920", operation: "exclude", @@ -295,13 +302,10 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ npm_yarn: structuredClone(REVIEWED_NPM_ENTRY), }); const { calls, payload } = runLiveScenario({ + sandboxName: "personal-npm", + policies: ["personal-open-internet", "npm"], initialPolicy, childScript: ` -registry.registerSandbox({ - name: "personal-npm", - agent: "openclaw", - policies: ["personal-open-internet", "npm"], -}); const removed = policies.removePreset("personal-npm", "npm"); process.stdout.write("\\n__RESULT__" + JSON.stringify({ removed, @@ -320,9 +324,10 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ it("repairs an active npm preset and restores the reviewed baseline on removal (#8497)", () => { const { calls, payload, stdout } = runLiveScenario({ + sandboxName: "npm-lifecycle", + policies: ["npm"], initialPolicy: unoverlaidActivePolicy(), childScript: ` -registry.registerSandbox({ name: "npm-lifecycle", agent: "openclaw", policies: ["npm"] }); const beforeApplyState = policies.getOpenClawNpmCompatibilityState("npm-lifecycle"); const applied = policies.applyPresets("npm-lifecycle", ["npm"]); const afterApply = YAML.parse(fs.readFileSync(process.env.CURRENT_POLICY, "utf-8")); @@ -352,7 +357,7 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ expect(stdout).toContain("Effective egress scope that would replace the current preset policy"); expect(stdout).toContain("OpenClaw npm compatibility"); expect(stdout).not.toContain("already effective; no new egress would be opened"); - expect(calls.filter((call) => call.startsWith("policy get "))).toHaveLength(4); + expect(calls.filter((call) => call.startsWith("policy get "))).toHaveLength(6); expect(calls.filter((call) => call.startsWith("policy set "))).toHaveLength(2); }); @@ -360,10 +365,11 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ const oldNpmEntry = olderNpmEntry(); const oldActivePolicy = YAML.parse(activePolicy(oldNpmEntry)); const { calls, payload, stderr } = runLiveScenario({ + sandboxName: "npm-old-overlay", + policies: ["npm"], initialPolicy: YAML.stringify(oldActivePolicy), setMode: "fail-once", childScript: ` -registry.registerSandbox({ name: "npm-old-overlay", agent: "openclaw", policies: ["npm"] }); const failedRemoval = policies.removePreset("npm-old-overlay", "npm", { nonFatal: true }); const afterFailedPolicy = YAML.parse(fs.readFileSync(process.env.CURRENT_POLICY, "utf-8")); const afterFailedRegistry = structuredClone(registry.getSandbox("npm-old-overlay")); @@ -386,7 +392,7 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ expect(payload.afterRemove.network_policies.npm_yarn).toBeUndefined(); expect(payload.afterRemove.network_policies.npm_registry).toEqual(REVIEWED_BASELINE_ENTRY); expect(payload.registry.policies).toEqual([]); - expect(calls.filter((call) => call.startsWith("policy get "))).toHaveLength(2); + expect(calls.filter((call) => call.startsWith("policy get "))).toHaveLength(3); expect(calls.filter((call) => call.startsWith("policy set "))).toHaveLength(2); }); @@ -394,9 +400,10 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ const drifted = YAML.parse(activePolicy()); drifted.network_policies.npm_registry.endpoints[0].tls = "auto"; const { calls, payload, stderr } = runLiveScenario({ + sandboxName: "npm-drift", + policies: ["npm"], initialPolicy: YAML.stringify(drifted), childScript: ` -registry.registerSandbox({ name: "npm-drift", agent: "openclaw", policies: ["npm"] }); const removed = policies.removePreset("npm-drift", "npm", { nonFatal: true }); process.stdout.write("\\n__RESULT__" + JSON.stringify({ removed, @@ -414,9 +421,10 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ it("keeps an approved baseline exclusion absent through apply and removal (#8497)", () => { const { calls, payload, stderr } = runLiveScenario({ + sandboxName: "npm-excluded", + policies: [], initialPolicy: excludedBaselinePolicy(), childScript: ` -registry.registerSandbox({ name: "npm-excluded", agent: "openclaw", policies: [] }); ${exclusionRegistration("npm-excluded", "a")} const applied = policies.applyPresets("npm-excluded", ["npm"]); const afterApply = YAML.parse(fs.readFileSync(process.env.CURRENT_POLICY, "utf-8")); @@ -445,9 +453,10 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ it("refuses batch apply when an excluded baseline drifted back into the live policy (#8497)", () => { const initialPolicy = reviewedBaselinePolicy(); const { calls, payload, stderr } = runLiveScenario({ + sandboxName: "npm-excl-batch", + policies: [], initialPolicy, childScript: ` -registry.registerSandbox({ name: "npm-excl-batch", agent: "openclaw", policies: [] }); ${exclusionRegistration("npm-excl-batch", "b")} const applied = policies.applyPresets("npm-excl-batch", ["npm"]); process.stdout.write("\\n__RESULT__" + JSON.stringify({ @@ -467,9 +476,10 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ it("refuses direct apply when an excluded baseline drifted back into the live policy (#8497)", () => { const initialPolicy = reviewedBaselinePolicy(); const { calls, payload, stderr } = runLiveScenario({ + sandboxName: "npm-excl-direct", + policies: [], initialPolicy, childScript: ` -registry.registerSandbox({ name: "npm-excl-direct", agent: "openclaw", policies: [] }); ${exclusionRegistration("npm-excl-direct", "c")} const applied = policies.applyPreset("npm-excl-direct", "npm"); process.stdout.write("\\n__RESULT__" + JSON.stringify({ diff --git a/test/runtime/policy/policy-preset-noop-disclosure.test.ts b/test/runtime/policy/policy-preset-noop-disclosure.test.ts index 1522fb5812e..67067d00573 100644 --- a/test/runtime/policy/policy-preset-noop-disclosure.test.ts +++ b/test/runtime/policy/policy-preset-noop-disclosure.test.ts @@ -9,6 +9,12 @@ import { afterEach, describe, expect, it } from "vitest"; import YAML from "yaml"; import * as policies from "../../../src/lib/policy"; +import { + managedRegistrationSource, + POLICY_HASH, + POLICY_VERSION, + SANDBOX_ID, +} from "../../helpers/managed-policy-receipt-fixture"; const REPO_ROOT = path.join(import.meta.dirname, "../../.."); const POLICY_MODULE = JSON.stringify(path.join(REPO_ROOT, "src/lib/policy/index.ts")); @@ -44,19 +50,28 @@ function runScenario({ sandbox: "alpha", status: "effective", policy_source: "sandbox", + hash: POLICY_HASH, + active_version: POLICY_VERSION, policy: YAML.parse(currentPolicy), }); fs.writeFileSync( openshell, `#!/usr/bin/env bash set -euo pipefail +if [ "$1 $2" = "sandbox get" ]; then + printf 'Name: alpha\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi if [ "$1 $2" = "policy get" ]; then if [[ " $* " == *" --output json "* ]]; then printf '%s\n' ${JSON.stringify(policyMetadata)} exit 0 fi - printf 'Version: 1\nHash: test\n---\n' - cat ${JSON.stringify(currentPolicyPath)} + if [ -f ${JSON.stringify(appliedPolicyPath)} ]; then + cat ${JSON.stringify(appliedPolicyPath)} + else + cat ${JSON.stringify(currentPolicyPath)} + fi exit 0 fi if [ "$1 $2" = "policy set" ]; then @@ -82,7 +97,7 @@ exit 1 const fs = require("node:fs"); const policies = require(${POLICY_MODULE}); const registry = require(${REGISTRY_MODULE}); -registry.registerSandbox({ name: "alpha", policies: [] }); +${managedRegistrationSource("alpha")} const result = ${invocation}; process.stdout.write("\\n__RESULT__" + JSON.stringify({ result, diff --git a/test/runtime/policy/policy-semantic-validation-runtime.test.ts b/test/runtime/policy/policy-semantic-validation-runtime.test.ts index 784977e84cd..32ff887c481 100644 --- a/test/runtime/policy/policy-semantic-validation-runtime.test.ts +++ b/test/runtime/policy/policy-semantic-validation-runtime.test.ts @@ -6,8 +6,20 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const { getSandbox, inspectSandboxPolicyAuthority, runCapture } = vi.hoisted(() => ({ +import { + managedPolicyInspection, + managedSandboxEntry, + SANDBOX_IDENTITY, +} from "../../helpers/managed-policy-receipt-fixture"; + +const { + getSandbox, + inspectOpenShellSandboxIdentityFingerprint, + inspectSandboxPolicyAuthority, + runCapture, +} = vi.hoisted(() => ({ getSandbox: vi.fn(), + inspectOpenShellSandboxIdentityFingerprint: vi.fn(), inspectSandboxPolicyAuthority: vi.fn(), runCapture: vi.fn(), })); @@ -16,6 +28,7 @@ vi.mock("../../../src/lib/adapters/openshell/policy-authority", async (importOri ...(await importOriginal< typeof import("../../../src/lib/adapters/openshell/policy-authority") >()), + inspectOpenShellSandboxIdentityFingerprint, inspectSandboxPolicyAuthority, })); @@ -45,17 +58,11 @@ network_policies: beforeEach(() => { getSandbox.mockReset(); - getSandbox.mockImplementation((name: string) => ({ - name, - agent: "openclaw", - policies: [], - policyAuthority: "nemoclaw-managed", - })); + getSandbox.mockImplementation((name: string) => managedSandboxEntry(name)); inspectSandboxPolicyAuthority.mockReset(); - inspectSandboxPolicyAuthority.mockReturnValue({ - authority: "nemoclaw-managed", - effectivePolicy: {}, - }); + inspectSandboxPolicyAuthority.mockReturnValue(managedPolicyInspection()); + inspectOpenShellSandboxIdentityFingerprint.mockReset(); + inspectOpenShellSandboxIdentityFingerprint.mockReturnValue(SANDBOX_IDENTITY); runCapture.mockReset(); }); diff --git a/test/runtime/policy/portable-policy-failure-finality.test.ts b/test/runtime/policy/portable-policy-failure-finality.test.ts index 9da609af194..69f123e2396 100644 --- a/test/runtime/policy/portable-policy-failure-finality.test.ts +++ b/test/runtime/policy/portable-policy-failure-finality.test.ts @@ -8,6 +8,12 @@ import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import YAML from "yaml"; +import { + managedPolicyMetadata, + managedRegistrationSource, + SANDBOX_ID, +} from "../../helpers/managed-policy-receipt-fixture"; + const repoRoot = path.join(import.meta.dirname, "../../.."); const policyModulePath = path.join(repoRoot, "src", "lib", "policy", "index.ts"); const registryModulePath = path.join(repoRoot, "src", "lib", "state", "registry.ts"); @@ -97,30 +103,50 @@ interface PolicySetBehavior { * round-trippable base policy so the driven mutation reaches its submission; * the `policy set --wait` result is what each scenario varies. */ -function buildOpenshellStub(policySet: PolicySetBehavior, basePolicy: string): string { - const policyMetadata = JSON.stringify({ - scope: "sandbox", - sandbox: SANDBOX_NAME, - status: "effective", - policy_source: "sandbox", +function buildOpenshellStub( + policySet: PolicySetBehavior, + basePolicy: string, + appliedPolicyPath: string, +): string { + const policyMetadata = { + ...JSON.parse(managedPolicyMetadata(SANDBOX_NAME)), policy: YAML.parse(basePolicy), - }); + }; return `#!/bin/sh +if [ "$1" = "sandbox" ] && [ "$2" = "get" ]; then + printf 'Name: ${SANDBOX_NAME}\nId: ${SANDBOX_ID}\nPhase: Ready\n' + exit 0 +fi if [ "$1" = "policy" ] && [ "$2" = "get" ]; then case " $* " in *" --output json "*) cat <<'JSON' -${policyMetadata} +${JSON.stringify(policyMetadata)} JSON exit 0 ;; esac + if [ -f ${JSON.stringify(appliedPolicyPath)} ]; then + cat ${JSON.stringify(appliedPolicyPath)} + exit 0 + fi cat <<'YAML' ${basePolicy} YAML exit 0 fi if [ "$1" = "policy" ] && [ "$2" = "set" ]; then + ${ + policySet.exitCode === 0 + ? `while [ "$#" -gt 0 ]; do + if [ "$1" = "--policy" ]; then + cp "$2" ${JSON.stringify(appliedPolicyPath)} + break + fi + shift + done` + : ":" + } cat >&2 <<'POLICY_SET_STDERR' ${policySet.stderr} POLICY_SET_STDERR @@ -145,7 +171,7 @@ console.log(${JSON.stringify(RETURN_MARKER)} + String(returned)); const APPLY_PRESETS_DRIVER = `const registry = require(${JSON.stringify(registryModulePath)});\n` + - `registry.registerSandbox({ name: ${JSON.stringify(SANDBOX_NAME)}, agent: "openclaw", policies: [] });\n` + + `${managedRegistrationSource(SANDBOX_NAME)}\n` + buildDriver(`applyPresets(${JSON.stringify(SANDBOX_NAME)}, [${JSON.stringify(PRESET_NAME)}])`); /** @@ -154,11 +180,12 @@ const APPLY_PRESETS_DRIVER = */ const REMOVE_PRESET_DRIVER = `const registry = require(${JSON.stringify(registryModulePath)});\n` + - `registry.registerSandbox({ name: ${JSON.stringify(SANDBOX_NAME)}, agent: "openclaw", policies: [${JSON.stringify(PRESET_NAME)}] });\n` + + `${managedRegistrationSource(SANDBOX_NAME)}\n` + + `registry.updateSandbox(${JSON.stringify(SANDBOX_NAME)}, { policies: [${JSON.stringify(PRESET_NAME)}] });\n` + buildDriver(`removePreset(${JSON.stringify(SANDBOX_NAME)}, ${JSON.stringify(PRESET_NAME)})`); const APPLY_PRESET_DRIVER = `const registry = require(${JSON.stringify(registryModulePath)});\n` + - `registry.registerSandbox({ name: ${JSON.stringify(SANDBOX_NAME)}, agent: "openclaw", policies: [] });\n` + + `${managedRegistrationSource(SANDBOX_NAME)}\n` + buildDriver(`applyPreset(${JSON.stringify(SANDBOX_NAME)}, ${JSON.stringify(PRESET_NAME)})`); interface ChildRun { @@ -217,10 +244,15 @@ function runPolicyMutation({ driver, policySet, basePolicy }: PolicyMutationRun) fs.mkdirSync(homeDir); const openshellStubPath = path.join(scratchDir, "openshell"); - fs.writeFileSync(openshellStubPath, buildOpenshellStub(policySet, basePolicy), { - encoding: "utf-8", - mode: 0o755, - }); + const appliedPolicyPath = path.join(scratchDir, "applied-policy.yaml"); + fs.writeFileSync( + openshellStubPath, + buildOpenshellStub(policySet, basePolicy, appliedPolicyPath), + { + encoding: "utf-8", + mode: 0o755, + }, + ); const driverPath = path.join(scratchDir, "driver.js"); fs.writeFileSync(driverPath, driver, "utf-8"); diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index c9fff1dc750..4bde66e0631 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -66,6 +66,9 @@ describe("sandboxName command hardening in onboard.js", () => { const preflightPath = sourceModule("onboard", "preflight.ts"); const credentialsPath = sourceModule("credentials", "store.ts"); const streamPath = sourceModule("sandbox", "create-stream.ts"); + const onboardScriptMocksPath = JSON.stringify( + path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), + ); fs.mkdirSync(fakeBin, { recursive: true }); writeOkOpenshell(fakeBin, { readySandboxGet: true }); @@ -74,6 +77,7 @@ describe("sandboxName command hardening in onboard.js", () => { String.raw` const runner = require(${runnerPath}); const registry = require(${registryPath}); +const fixtureMocks = require(${onboardScriptMocksPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); const sandboxCreateStream = require(${streamPath}); @@ -116,14 +120,14 @@ runner.runFile = (file, args = [], opts = {}) => { }; runner.runCapture = (command) => { const text = asText(command); + const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); + if (createdIdentity !== null) return createdIdentity; if (text.includes("sandbox get") && text.includes("my-assistant")) return ""; if (text.includes("sandbox list")) return "my-assistant Ready"; if (text.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; if (text.includes("sandbox exec") && text.includes("http://localhost:") && text.includes("/health")) return "200"; if (text === "uname -r") return "6.8.0"; - const mockedCapture = require(${JSON.stringify( - path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), - )}).mockOnboardRunCapture(command); + const mockedCapture = fixtureMocks.mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; return ""; }; @@ -132,6 +136,11 @@ registry.getDisabledChannels = () => []; registry.registerSandbox = () => true; registry.removeSandbox = () => true; registry.updateSandbox = () => true; +const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { + sandboxName: "my-assistant", + provider: "nvidia-prod", + model: "gpt-5.4", +}); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; sandboxCreateStream.streamSandboxCreate = async () => ({ @@ -147,7 +156,26 @@ try { process.env.NEMOCLAW_HEALTH_POLL_COUNT = "1"; Object.defineProperty(process, "platform", { value: "darwin" }); Object.defineProperty(process, "arch", { value: "x64" }); - const sandboxName = await createSandbox(null, "gpt-5.4", "nvidia-prod", null, "my-assistant"); + const sandboxName = await createSandbox( + ...fixtureMocks.sandboxCreateArgsWithVerifiedReservation( + [ + null, + "gpt-5.4", + "nvidia-prod", + null, + "my-assistant", + null, + null, + null, + null, + null, + null, + null, + [], + ], + createFixture, + ), + ); console.log(JSON.stringify({ sandboxName, commands })); } catch (error) { console.error(error && error.stack ? error.stack : String(error)); diff --git a/test/support/setup-inference-test-harness.ts b/test/support/setup-inference-test-harness.ts index 05ff9f048c3..2510d4a1eef 100644 --- a/test/support/setup-inference-test-harness.ts +++ b/test/support/setup-inference-test-harness.ts @@ -384,6 +384,7 @@ export function createDirectSetupInferenceHarnessFactory( withOllamaModelOwnershipLock: (operation) => operation(), ...options.overrides, }); + const revalidatePolicyRequirements = vi.fn(); const setupInference: SetupInference = ( sandboxName, model, @@ -403,8 +404,9 @@ export function createDirectSetupInferenceHarnessFactory( hermesAuthMethod, hermesToolGateways, { - revalidatePolicyRequirements: () => {}, ...inferenceOptions, + revalidatePolicyRequirements: + inferenceOptions.revalidatePolicyRequirements ?? revalidatePolicyRequirements, }, ); return { @@ -413,6 +415,7 @@ export function createDirectSetupInferenceHarnessFactory( logs, runOpenshell, setupInference, + revalidatePolicyRequirements, unloadOllamaModels, updateSandbox, verifyInferenceRoute,