From 176e29ecf63ca9ed6ca723b4c26f95d95e1893fb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Sun, 2 Aug 2026 22:33:11 -0400 Subject: [PATCH 1/8] fix(shields): serialize deadline recovery Signed-off-by: Julie Yaunches --- ci/env-var-doc-allowlist.json | 8 + ci/source-shape-test-budget.json | 5 + docs/manage-sandboxes/backup-restore.mdx | 17 +- docs/manage-sandboxes/runtime-controls.mdx | 13 +- docs/reference/commands.mdx | 27 +- src/commands/sandbox/shields/status.ts | 3 +- src/commands/sandbox/shields/up.ts | 5 +- src/lib/actions/maintenance.test.ts | 225 ++- src/lib/actions/maintenance.ts | 273 +++- src/lib/actions/sandbox/snapshot.ts | 12 +- src/lib/shields/flow.test.ts | 710 ++++----- src/lib/shields/index.test.ts | 53 +- src/lib/shields/index.ts | 715 ++++++--- src/lib/shields/timer-bound-lock.ts | 69 +- src/lib/shields/timer-control.ts | 157 +- src/lib/shields/timer.test.ts | 331 ++++- src/lib/shields/timer.ts | 155 +- src/lib/shields/transition-lock.test.ts | 126 +- src/lib/shields/transition-lock.ts | 42 +- .../mcp-lifecycle-lock-acquisition.test.ts | 759 ++++++++++ .../state/mcp-lifecycle-lock-acquisition.ts | 1322 ++++++++++++++++- src/lib/state/mcp-lifecycle-lock-identity.ts | 7 + src/lib/state/mcp-lifecycle-lock-storage.ts | 127 ++ src/lib/state/mcp-lifecycle-lock.ts | 10 + .../shields-timer-authority.ts | 121 ++ src/lib/state/paths.test.ts | 26 +- src/lib/state/paths.ts | 13 +- test/config-set-nested-ssrf.test.ts | 2 + test/helpers/isolate-test-state.ts | 33 + test/mcp-lifecycle-lock.test.ts | 710 +++++++-- test/vitest-temp-root.test.ts | 34 + vitest.config.ts | 23 +- 32 files changed, 4955 insertions(+), 1178 deletions(-) create mode 100644 src/lib/state/mcp-lifecycle-lock-acquisition.test.ts create mode 100644 src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts create mode 100644 test/helpers/isolate-test-state.ts diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index eaac8c08b27..d2a20ba1e6e 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -59,6 +59,14 @@ "name": "NEMOCLAW_TEST_NO_SLEEP", "reason": "Test sentinel that bypasses real-time sleep() calls in onboard inference probes. Set to '1' only by Vitest tests; never user-set." }, + { + "name": "NEMOCLAW_TEST_BASE_HOME", + "reason": "Internal Vitest-only baseline used with NEMOCLAW_TEST_STATE_DIR so tests that explicitly replace HOME retain their fixture paths. Never user-set in production." + }, + { + "name": "NEMOCLAW_TEST_STATE_DIR", + "reason": "Internal Vitest-only state root that keeps lifecycle locks and Shields artifacts out of the caller's real NemoClaw state. The production resolver honors it only while Vitest is active." + }, { "name": "NEMOCLAW_TELEGRAM_STARTUP_GRACE_MS", "reason": "Internal Vitest-only override that shortens the Telegram diagnostics startup-grace timer. Production uses the built-in default." diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 8e473b6d06a..2c975a39324 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -716,6 +716,11 @@ "test": "wires cleanup into root and standalone plugin test runs", "category": "compatibility" }, + { + "file": "test/vitest-temp-root.test.ts", + "test": "isolates stateful non-live projects without redirecting live E2E state", + "category": "security" + }, { "file": "test/vitest-watch-triggers.test.ts", "test": "registers the focused mappings at the root configuration boundary (#6692)", diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 10f2d02d2d4..9713d405ac2 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -91,8 +91,11 @@ NemoClaw computes versions (`v1`, `v2`, through `vN`) from timestamp order, so ` `snapshot create` requires shields to be down. Snapshot creation and restore share the per-sandbox transition lock with the shields auto-restore timer. -If a timed shields-down window expires during snapshot work, auto-restore can interrupt the operation and restore lockdown instead of allowing state or policy changes to continue past the deadline. -Retry the snapshot in a new shields-down window if the deadline interrupts it. +If a timed shields-down window expires during snapshot work, auto-restore closes the per-sandbox lifecycle deadline gate. +The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation. +NemoClaw does not signal the snapshot process. +If ownership becomes ambiguous, NemoClaw records durable containment and reports exact-generation recovery guidance. +Stop all NemoClaw processes for the sandbox, then follow that guidance before you retry the snapshot. Tag a snapshot with a human-readable label: @@ -207,11 +210,13 @@ If a registered docker-driver sandbox's container is stopped, `backup-all` start If the container cannot be returned to the stopped state, the backup run fails and reports that the container was left running. If a sandbox is not running and its container cannot be started this way, start the sandbox or its container and rerun `$$nemoclaw backup-all`. -When Shields are UP for an eligible sandbox, `backup-all` opens a 30-minute shields-down window before it creates that sandbox's snapshot. +When an eligible sandbox starts with Shields up, `backup-all` acquires the lifecycle lock and opens a 30-minute shields-down window. A sandbox that starts with Shields down remains down. -An unlock failure marks that sandbox as failed, and `backup-all` continues with the next sandbox. -Because the timer does not defer to the backup process, it can restore lockdown when the 30-minute deadline expires. -NemoClaw always attempts to restore Shields lockdown before it processes the next sandbox, including when the backup fails. +`backup-all` reacquires the lock under that exact timer generation while it copies sandbox state. +After the copy, it uses a third lock interval with a 30-second acquisition budget to restore the previous Shields posture. +If the timer expires during the copy, the deadline gate blocks new mutations and waits for the exact backup owner to finish without signaling it. +An initial lock or unlock failure marks that sandbox as failed, and `backup-all` continues with the next sandbox. +NemoClaw attempts to restore the previous Shields posture before it processes the next sandbox, including when the backup fails. If lockdown cannot be restored, `backup-all` stops and does not process the remaining sandboxes. Correct the reported issue, run the printed `$$nemoclaw shields up` command, and rerun `$$nemoclaw backup-all`. diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 4aca95e49cf..0a233eb1385 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -108,10 +108,19 @@ Run `$$nemoclaw shields down` before the change, then restore lockdown wi NemoClaw serializes host-side config and inference writes, snapshot mutation, sandbox destruction, and shields transitions for each sandbox. When `shields down --timeout` is active, each mutation binds to that exact timer generation so a replaced or expired timer cannot race a later command or a new sandbox that reuses the same name. -If the timeout expires while a mutation is still changing sandbox state, auto-restore can stop that exact process tree, reclaim the transition, and restore the restrictive policy and config posture. +If the timeout expires while a mutation is changing sandbox state, auto-restore closes the per-sandbox lifecycle deadline gate. +The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation. +NemoClaw does not signal that process because portable process inspection cannot prove that all descendants stopped. +After the owner releases the lock, auto-restore restores the restrictive policy and config posture. The ownership check includes both the process ID and process start identity so PID reuse does not grant control over an unrelated process. -Retry a command that the auto-restore deadline interrupts after you open a new shields-down window. +When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window. +The deadline gate remains closed during those attempts. +If restoration cannot commit, NemoClaw records durable containment before the command returns an error. +NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. +Durable containment keeps new mutations blocked until you complete exact-generation operator recovery. +Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. +Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. ## Related Topics diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index b7d6dbb3fcd..7d63c6a98ce 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1161,8 +1161,16 @@ If `shields up` reports that the config remains unlocked or drifted, confirm tha If the retry still fails, rebuild a known-good baseline with `$$nemoclaw rebuild --yes`. Host-side config and inference writes, snapshot mutation, sandbox destruction, and shields transitions serialize per sandbox. -When a timed shields-down window reaches its deadline, auto-restore can interrupt the exact process tree holding that transition and restore lockdown. -Retry an interrupted command in a new shields-down window. +When a timed shields-down window reaches its deadline, auto-restore closes the per-sandbox lifecycle deadline gate. +The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation before auto-restore restores lockdown. +NemoClaw does not signal that process because portable process inspection cannot prove that all descendants stopped. +When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window. +The deadline gate remains closed during those attempts. +If restoration cannot commit, NemoClaw records durable containment before the command returns an error. +NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. +Durable containment blocks new mutations until you complete exact-generation operator recovery. +Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. +Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. @@ -2902,6 +2910,13 @@ A registered docker-driver sandbox whose container is stopped is started for the If the container cannot be returned to the stopped state, the command fails and reports that the container was left running. Sandboxes that are not running and cannot be started this way are skipped with remediation guidance. +For each eligible sandbox, `backup-all` acquires the lifecycle lock to open a 30-minute shields-down window when needed. +It reacquires the lock under that exact timer generation while it copies sandbox state. +After the copy, it uses a third lock interval with a 30-second acquisition budget to restore the previous Shields posture. +If the timer expires during the copy, the deadline gate blocks new mutations and waits for the exact backup owner to finish without signaling it. +An initial lock or unlock failure marks that sandbox as failed, and `backup-all` continues with the next sandbox. +A failure to restore the previous Shields posture stops `backup-all` before it processes another sandbox. + ```bash $$nemoclaw backup-all ``` @@ -2921,7 +2936,9 @@ A skipped sandbox's uncommitted state is not included in its last successful bac Create a timestamped snapshot of sandbox state. Snapshots are stored in `~/.nemoclaw/rebuild-backups//`. The command requires shields to be down and keeps the shields check and backup under one per-sandbox transition. -An expired auto-restore timer can interrupt a long-running backup and restore lockdown. +If the timer expires during a long-running backup, the deadline gate blocks new mutations and waits for the exact backup owner to finish. +Auto-restore does not signal the backup process. +If ownership becomes ambiguous, NemoClaw records durable containment and reports exact-generation recovery guidance. When the sandbox has active baseline exclusions, successful output lists their keys and repeats that excluded egress leaves dependent agent features unsupported for that sandbox. ```bash @@ -2955,7 +2972,9 @@ If no selector is provided, the latest snapshot is used. Restore removes files added after the snapshot only from state directories selected for cleanup. It preserves directories that exist only in the target manifest or whose backup failed. The state replacement, mutable-config permission repair, and policy reconciliation run under the same per-sandbox transition. -An expired auto-restore timer can interrupt that work and restore lockdown. +If the timer expires during that work, the deadline gate blocks new mutations and waits for the exact restore owner to finish. +Auto-restore does not signal the restore process. +If ownership becomes ambiguous, NemoClaw records durable containment and reports exact-generation recovery guidance. The selector accepts any of: diff --git a/src/commands/sandbox/shields/status.ts b/src/commands/sandbox/shields/status.ts index 62ea0f8b490..0d6fe51261e 100644 --- a/src/commands/sandbox/shields/status.ts +++ b/src/commands/sandbox/shields/status.ts @@ -4,7 +4,6 @@ import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; import { sandboxNameArg } from "../../../lib/sandbox/command-support"; import * as shields from "../../../lib/shields/index"; -import { withSandboxMutationLock } from "../../../lib/state/mcp-lifecycle-lock"; export default class ShieldsStatusCommand extends NemoClawCommand { static id = "sandbox:shields:status"; @@ -18,6 +17,6 @@ export default class ShieldsStatusCommand extends NemoClawCommand { public async run(): Promise { const { args } = await this.parse(ShieldsStatusCommand); - await withSandboxMutationLock(args.sandboxName, () => shields.shieldsStatus(args.sandboxName)); + shields.shieldsStatus(args.sandboxName); } } diff --git a/src/commands/sandbox/shields/up.ts b/src/commands/sandbox/shields/up.ts index f4cf4111978..166927dd296 100644 --- a/src/commands/sandbox/shields/up.ts +++ b/src/commands/sandbox/shields/up.ts @@ -4,7 +4,6 @@ import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; import { sandboxNameArg } from "../../../lib/sandbox/command-support"; import * as shields from "../../../lib/shields/index"; -import { withSandboxMutationLock } from "../../../lib/state/mcp-lifecycle-lock"; export default class ShieldsUpCommand extends NemoClawCommand { static id = "sandbox:shields:up"; @@ -18,8 +17,6 @@ export default class ShieldsUpCommand extends NemoClawCommand { public async run(): Promise { const { args } = await this.parse(ShieldsUpCommand); - await withSandboxMutationLock(args.sandboxName, () => - shields.shieldsUp(args.sandboxName, { throwOnError: true }), - ); + shields.shieldsUp(args.sandboxName, { throwOnError: true }); } } diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index 515ef28c6d1..f95952d7cec 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -22,6 +22,14 @@ const mocks = vi.hoisted(() => ({ withSandboxMutationLock: vi.fn(), })); +async function runSandboxMutationAction( + _sandboxName: string, + action: () => unknown, + _options?: { timeoutMs?: number }, +): Promise { + return action(); +} + vi.mock("../state/registry", () => ({ isRouteOnlySandboxReservation: (entry: { pendingRouteReservation?: true; createdAt?: string }) => entry.pendingRouteReservation === true && entry.createdAt === undefined, @@ -109,7 +117,7 @@ describe("backupAll", () => { wasLocked: false, })); mocks.relockBackupShieldsWindow.mockReturnValue(true); - mocks.withSandboxMutationLock.mockImplementation((_name, callback) => callback()); + mocks.withSandboxMutationLock.mockImplementation(runSandboxMutationAction); }); afterEach(() => { @@ -235,7 +243,7 @@ describe("backupAll", () => { mocks.parseReadySandboxNames.mockReturnValue(new Set(["alpha", "beta"])); mocks.withSandboxMutationLock .mockRejectedValueOnce(new Error("Timed out waiting for the sandbox mutation lock")) - .mockImplementationOnce((_name, callback) => callback()); + .mockImplementation(runSandboxMutationAction); mocks.backupSandboxState.mockReturnValue({ success: true, backedUpDirs: ["workspace"], @@ -255,6 +263,8 @@ describe("backupAll", () => { expect(mocks.withSandboxMutationLock.mock.calls.map(([name]) => name)).toEqual([ "alpha", "beta", + "beta", + "beta", ]); expect(mocks.backupSandboxState).toHaveBeenCalledOnce(); expect(mocks.backupSandboxState).toHaveBeenCalledWith("beta"); @@ -264,6 +274,33 @@ describe("backupAll", () => { ); }); + it("does not start a stopped container when the first mutation lock cannot be acquired", async () => { + mocks.listSandboxes.mockReturnValue({ + sandboxes: [{ name: "sb-stopped" }], + defaultSandbox: "sb-stopped", + }); + mocks.parseReadySandboxNames.mockReturnValue(new Set()); + mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ + containerName: "openshell-sb-stopped-abc", + }); + mocks.withSandboxMutationLock.mockRejectedValueOnce( + new Error("Timed out waiting for the sandbox mutation lock"), + ); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`exit:${code}`); + }) as never); + + await expect(backupAll()).rejects.toThrow("exit:1"); + + expect(mocks.withSandboxMutationLock).toHaveBeenCalledOnce(); + expect(mocks.startStoppedSandboxContainerForBackup).not.toHaveBeenCalled(); + expect(mocks.openBackupShieldsWindow).not.toHaveBeenCalled(); + expect(mocks.backupStartedSandboxState).not.toHaveBeenCalled(); + expect(mocks.returnSandboxContainerToStopped).not.toHaveBeenCalled(); + }); + it("does not back up when gateway preflight exits", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "sb-good" }], @@ -322,13 +359,23 @@ describe("backupAll", () => { logSpy.mockRestore(); }); - it("closes each shields window before backing up the next sandbox (#6455)", async () => { + it("serializes each Shields window, backup, and relock in separate intervals (#7952)", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "alpha" }, { name: "beta" }], defaultSandbox: "alpha", }); mocks.parseReadySandboxNames.mockReturnValue(new Set(["alpha", "beta"])); const events: string[] = []; + mocks.withSandboxMutationLock.mockImplementation( + async (name: string, action: () => unknown) => { + events.push(`lock:start:${name}`); + try { + return await action(); + } finally { + events.push(`lock:end:${name}`); + } + }, + ); mocks.openBackupShieldsWindow.mockImplementation( ( name: string, @@ -368,13 +415,34 @@ describe("backupAll", () => { await backupAll(); expect(events).toEqual([ + "lock:start:alpha", "open:alpha", + "lock:end:alpha", + "lock:start:alpha", "backup:alpha", + "lock:end:alpha", + "lock:start:alpha", "relock:alpha", + "lock:end:alpha", + "lock:start:beta", "open:beta", + "lock:end:beta", + "lock:start:beta", "backup:beta", + "lock:end:beta", + "lock:start:beta", "relock:beta", + "lock:end:beta", ]); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( + 3, + "alpha", + expect.any(Function), + { timeoutMs: 30_000 }, + ); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith(6, "beta", expect.any(Function), { + timeoutMs: 30_000, + }); }); it("relocks shields after a credential permission failure and keeps the failure hard (#6455)", async () => { @@ -478,11 +546,21 @@ describe("backupAll", () => { mocks.backupSandboxState.mockImplementation(() => { throw backupError; }); - mocks.relockBackupShieldsWindow.mockReturnValue(false); + const relockLockError = new Error("mutation lock timed out"); + mocks.withSandboxMutationLock + .mockImplementationOnce(runSandboxMutationAction) + .mockImplementationOnce(runSandboxMutationAction) + .mockRejectedValueOnce(relockLockError); vi.spyOn(console, "log").mockImplementation(() => undefined); const failure = await backupAll().catch((error: unknown) => error); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( + 3, + "alpha", + expect.any(Function), + { timeoutMs: 30_000 }, + ); expect(failure).toBeInstanceOf(AggregateError); expect((failure as AggregateError).message).toContain( "Backup for 'alpha' failed and Shields lockdown could not be restored", @@ -490,11 +568,13 @@ describe("backupAll", () => { expect((failure as AggregateError).errors).toEqual([ backupError, expect.objectContaining({ + cause: relockLockError, message: expect.stringContaining( "Shields lockdown could not be restored for 'alpha' after backup-all", ), }), ]); + expect(mocks.relockBackupShieldsWindow).not.toHaveBeenCalled(); }); it("preserves an orphan-manifest error when shields restoration also fails (#6455)", async () => { @@ -607,6 +687,86 @@ describe("backupAll", () => { expect(logOutput).not.toContain("Skipping 'sb-stopped'"); }); + it("keeps the stopped-container lifecycle inside the three backup lock intervals (#7952)", async () => { + mocks.listSandboxes.mockReturnValue({ + sandboxes: [{ name: "sb-stopped" }], + defaultSandbox: "sb-stopped", + }); + mocks.parseReadySandboxNames.mockReturnValue(new Set()); + const events: string[] = []; + let lockActive = false; + mocks.withSandboxMutationLock.mockImplementation( + async (name: string, action: () => unknown) => { + expect(lockActive).toBe(false); + events.push(`lock:start:${name}`); + lockActive = true; + try { + return await action(); + } finally { + lockActive = false; + events.push(`lock:end:${name}`); + } + }, + ); + mocks.startStoppedSandboxContainerForBackup.mockImplementation((name: string) => { + expect(lockActive).toBe(true); + events.push(`start:${name}`); + return { containerName: "openshell-sb-stopped-abc" }; + }); + mocks.openBackupShieldsWindow.mockImplementation((name: string) => { + expect(lockActive).toBe(true); + events.push(`open:${name}`); + return { relocked: false, wasLocked: true }; + }); + mocks.backupStartedSandboxState.mockImplementation(async (name: string) => { + expect(lockActive).toBe(true); + events.push(`backup:${name}`); + return { + success: true, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + manifest: { backupPath: "/backups/sb-stopped/timestamp" }, + }; + }); + mocks.relockBackupShieldsWindow.mockImplementation((name: string) => { + expect(lockActive).toBe(true); + events.push(`relock:${name}`); + return true; + }); + mocks.returnSandboxContainerToStopped.mockImplementation(() => { + expect(lockActive).toBe(true); + events.push("stop:sb-stopped"); + return true; + }); + vi.spyOn(console, "log").mockImplementation(() => undefined); + + await backupAll(); + + expect(events).toEqual([ + "lock:start:sb-stopped", + "start:sb-stopped", + "open:sb-stopped", + "lock:end:sb-stopped", + "lock:start:sb-stopped", + "backup:sb-stopped", + "lock:end:sb-stopped", + "lock:start:sb-stopped", + "relock:sb-stopped", + "stop:sb-stopped", + "lock:end:sb-stopped", + ]); + expect(lockActive).toBe(false); + expect(mocks.withSandboxMutationLock).toHaveBeenCalledTimes(3); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( + 3, + "sb-stopped", + expect.any(Function), + { timeoutMs: 30_000 }, + ); + }); + it("returns the container to stopped and counts a failure when the started backup fails (#6500)", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "sb-stopped" }], @@ -658,21 +818,64 @@ describe("backupAll", () => { manifest: { backupPath: "/backups/sb-stopped/timestamp" }, }); mocks.returnSandboxContainerToStopped.mockReturnValue(false); - process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1"; - const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`exit:${code}`); - }) as never); - await expect(backupAll()).rejects.toThrow("exit:1"); + await expect(backupAll()).rejects.toThrow( + "could not return its container to the stopped state", + ); - expect(logSpy.mock.calls.flat().join("\n")).toContain("0 backed up, 1 failed, 0 skipped"); expect(errorSpy.mock.calls.flat().join("\n")).toContain( "backup cleanup failed (could not return its container to the stopped state", ); }); + it("does not stop outside the lock when the cleanup interval cannot be acquired (#7952)", async () => { + mocks.listSandboxes.mockReturnValue({ + sandboxes: [{ name: "sb-stopped" }, { name: "beta" }], + defaultSandbox: "sb-stopped", + }); + mocks.parseReadySandboxNames.mockReturnValue(new Set(["beta"])); + mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ + containerName: "openshell-sb-stopped-abc", + }); + mocks.openBackupShieldsWindow.mockReturnValue({ relocked: false, wasLocked: true }); + mocks.backupStartedSandboxState.mockResolvedValue({ + success: true, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + manifest: { backupPath: "/backups/sb-stopped/timestamp" }, + }); + const cleanupLockError = new Error("Timed out waiting for the cleanup mutation lock"); + mocks.withSandboxMutationLock + .mockImplementationOnce(runSandboxMutationAction) + .mockImplementationOnce(runSandboxMutationAction) + .mockRejectedValueOnce(cleanupLockError); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const failure = await backupAll().catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(AggregateError); + expect((failure as AggregateError).errors).toEqual([ + expect.objectContaining({ + cause: cleanupLockError, + message: expect.stringContaining("Shields lockdown could not be restored"), + }), + expect.objectContaining({ + cause: cleanupLockError, + message: expect.stringContaining("container was left running"), + }), + ]); + expect(mocks.withSandboxMutationLock).toHaveBeenCalledTimes(3); + expect(mocks.relockBackupShieldsWindow).not.toHaveBeenCalled(); + expect(mocks.returnSandboxContainerToStopped).not.toHaveBeenCalled(); + expect(mocks.openBackupShieldsWindow).toHaveBeenCalledOnce(); + expect(errorSpy.mock.calls.flat().join("\n")).toContain("container was left running"); + }); + it("returns a started container to stopped when an orphan manifest skips backup (#6500)", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "sb-stopped" }], diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 1025b327fd7..17a1eb2d19b 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -27,6 +27,7 @@ import * as registry from "../state/registry"; import * as sandboxState from "../state/sandbox"; import { nemoclawStateRoot, resolveHome } from "../state/state-root"; import { + type BackupShieldsWindow, type BackupShieldsWindowOptions, openBackupShieldsWindow, relockBackupShieldsWindow, @@ -74,29 +75,138 @@ interface BackupAllSandboxAttempt { result: sandboxState.BackupResult | null; orphanManifestMessage: string | null; shieldsWindowOpened: boolean; + stoppedContainerUnavailable: boolean; + mutationLockError?: unknown; +} + +interface BackupAllSandboxSetup { + window: BackupShieldsWindow | null; + startedForBackup: StartedForBackup | null; + stoppedContainerUnavailable: boolean; + stoppedContainerCleanupError: Error | null; +} + +function returnStartedSandboxToStopped( + sandboxName: string, + startedForBackup: StartedForBackup, + cause?: unknown, +): Error | null { + const failureDetail = + "could not return its container to the stopped state; the container was left running"; + const failureMessage = `Backup cleanup failed for '${sandboxName}': ${failureDetail}.`; + if (cause !== undefined) { + const error = new Error(failureMessage, { cause }); + console.error(` ${RD}✗${R} ${sandboxName}: backup cleanup failed (${failureDetail})`); + return error; + } + try { + if (returnSandboxContainerToStopped(startedForBackup.containerName)) { + console.log(` ${D}Returned '${sandboxName}' to its stopped state.${R}`); + return null; + } + const error = new Error(failureMessage); + console.error(` ${RD}✗${R} ${sandboxName}: backup cleanup failed (${failureDetail})`); + return error; + } catch (error) { + const cleanupError = new Error(failureMessage, { cause: error }); + console.error(` ${RD}✗${R} ${sandboxName}: backup cleanup failed (${failureDetail})`); + return cleanupError; + } +} + +function shieldsRelockError(sandboxName: string, cause?: unknown): Error { + const message = `Shields lockdown could not be restored for '${sandboxName}' after backup-all; aborting remaining backups.`; + return cause === undefined ? new Error(message) : new Error(message, { cause }); } async function backupSandboxWithinShieldsWindow( sandboxName: string, - backup: () => sandboxState.BackupResult | Promise, + shouldStartStoppedContainer: boolean, + backup: ( + startedForBackup: StartedForBackup | null, + ) => sandboxState.BackupResult | Promise, ): Promise { const shieldsWindowOptions = backupAllShieldsWindowOptions(sandboxName); - const window = openBackupShieldsWindow(sandboxName, shieldsWindowOptions); - if (!window) { + let enteredOpenLock = false; + let setup: BackupAllSandboxSetup; + try { + setup = await withSandboxMutationLock(sandboxName, () => { + enteredOpenLock = true; + const startedForBackup = shouldStartStoppedContainer + ? startStoppedSandboxContainerForBackup(sandboxName) + : null; + if (shouldStartStoppedContainer && !startedForBackup) { + return { + window: null, + startedForBackup: null, + stoppedContainerUnavailable: true, + stoppedContainerCleanupError: null, + }; + } + if (startedForBackup) { + console.log(` Starting stopped sandbox '${sandboxName}' to back it up...`); + } + try { + const window = openBackupShieldsWindow(sandboxName, shieldsWindowOptions); + return { + window, + startedForBackup, + stoppedContainerUnavailable: false, + stoppedContainerCleanupError: + !window && startedForBackup + ? returnStartedSandboxToStopped(sandboxName, startedForBackup) + : null, + }; + } catch (error) { + if (!startedForBackup) throw error; + const cleanupError = returnStartedSandboxToStopped(sandboxName, startedForBackup); + if (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `Backup setup for '${sandboxName}' failed and its started container could not be returned to the stopped state.`, + ); + } + throw error; + } + }); + } catch (error) { + if (enteredOpenLock) throw error; + return { + result: null, + orphanManifestMessage: null, + shieldsWindowOpened: false, + stoppedContainerUnavailable: false, + mutationLockError: error, + }; + } + if (setup.stoppedContainerUnavailable) { return { result: null, orphanManifestMessage: null, shieldsWindowOpened: false, + stoppedContainerUnavailable: true, }; } + if (!setup.window) { + if (setup.stoppedContainerCleanupError) throw setup.stoppedContainerCleanupError; + return { + result: null, + orphanManifestMessage: null, + shieldsWindowOpened: false, + stoppedContainerUnavailable: false, + }; + } + const window = setup.window; + console.log(` Backing up '${sandboxName}'...`); let result: sandboxState.BackupResult | null = null; let orphanManifestMessage: string | null = null; let backupError: unknown; let hasBackupError = false; let relockError: Error | null = null; + let stoppedContainerCleanupError: Error | null = null; try { - result = await backup(); + result = await withSandboxMutationLock(sandboxName, () => backup(setup.startedForBackup)); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); // Preserve the narrow pre-upgrade orphan exception, but classify it inside @@ -109,30 +219,90 @@ async function backupSandboxWithinShieldsWindow( hasBackupError = true; } } finally { - if (!relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions)) { - relockError = new Error( - `Shields lockdown could not be restored for '${sandboxName}' after backup-all; aborting remaining backups.`, + let enteredRelockLock = false; + try { + await withSandboxMutationLock( + sandboxName, + () => { + enteredRelockLock = true; + try { + if (!relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions)) { + relockError = shieldsRelockError(sandboxName); + } + } catch (error) { + relockError = shieldsRelockError(sandboxName, error); + } finally { + if (setup.startedForBackup) { + stoppedContainerCleanupError = returnStartedSandboxToStopped( + sandboxName, + setup.startedForBackup, + ); + } + } + }, + { timeoutMs: 30_000 }, ); + } catch (error) { + relockError ??= shieldsRelockError(sandboxName, error); + if (!enteredRelockLock && setup.startedForBackup) { + stoppedContainerCleanupError = returnStartedSandboxToStopped( + sandboxName, + setup.startedForBackup, + error, + ); + } } } if (relockError) { if (hasBackupError) { throw new AggregateError( - [backupError, relockError], + [ + backupError, + relockError, + ...(stoppedContainerCleanupError ? [stoppedContainerCleanupError] : []), + ], `Backup for '${sandboxName}' failed and Shields lockdown could not be restored; aborting remaining backups.`, ); } if (orphanManifestMessage) { throw new AggregateError( - [new Error(orphanManifestMessage), relockError], + [ + new Error(orphanManifestMessage), + relockError, + ...(stoppedContainerCleanupError ? [stoppedContainerCleanupError] : []), + ], `Backup for '${sandboxName}' encountered an orphan manifest and Shields lockdown could not be restored; aborting remaining backups.`, ); } + if (stoppedContainerCleanupError) { + throw new AggregateError( + [relockError, stoppedContainerCleanupError], + `Shields lockdown could not be restored for '${sandboxName}' and its started container could not be returned to the stopped state; aborting remaining backups.`, + ); + } throw relockError; } + if (stoppedContainerCleanupError && hasBackupError) { + throw new AggregateError( + [backupError, stoppedContainerCleanupError], + `Backup for '${sandboxName}' failed and its started container could not be returned to the stopped state; aborting remaining backups.`, + ); + } + if (stoppedContainerCleanupError && orphanManifestMessage) { + throw new AggregateError( + [new Error(orphanManifestMessage), stoppedContainerCleanupError], + `Backup for '${sandboxName}' encountered an orphan manifest and its started container could not be returned to the stopped state; aborting remaining backups.`, + ); + } + if (stoppedContainerCleanupError) throw stoppedContainerCleanupError; if (hasBackupError) throw backupError; - return { result, orphanManifestMessage, shieldsWindowOpened: true }; + return { + result, + orphanManifestMessage, + shieldsWindowOpened: true, + stoppedContainerUnavailable: false, + }; } export async function backupAll(): Promise { @@ -205,30 +375,14 @@ export async function backupAll(): Promise { // backupable: start it for the duration of the backup and return it to // its stopped state after (#6500). Anything else that is not Ready keeps // the existing skip (and, under installer-strict mode, the #6114 gate). - let startedForBackup: StartedForBackup | null = null; - if (!readyNames.has(sb.name)) { - startedForBackup = startStoppedSandboxContainerForBackup(sb.name); - if (!startedForBackup) { - if (orphanNames.has(sb.name) && isSandboxContainerDefinitivelyAbsent(sb.name)) { - // Tracked separately from `skipped` so the strict gate stays - // untripped: there is nothing to back up and nothing to start. - strandedOrphans.push(sb.name); - return; - } - console.log(` ${D}${notRunningBackupSkipMessage(sb.name)}${R}`); - skipped++; - notRunningSkipped++; - return; - } - console.log(` Starting stopped sandbox '${sb.name}' to back it up...`); - } - console.log(` Backing up '${sb.name}'...`); let result: sandboxState.BackupResult | null = null; let orphanManifestMessage: string | null = null; - let shieldsWindowOpened = true; - let returnedToStopped = true; - try { - const attempt = await backupSandboxWithinShieldsWindow(sb.name, () => + let mutationLockError: unknown; + let mutationLockFailed = false; + const attempt = await backupSandboxWithinShieldsWindow( + sb.name, + !readyNames.has(sb.name), + (startedForBackup) => startedForBackup ? backupStartedSandboxState(sb.name) : snapshotBackup.backupSandboxStateWithManagedAuthority( @@ -238,27 +392,33 @@ export async function backupAll(): Promise { getSandbox: registry.getSandbox, }, ), - ); - result = attempt.result; - orphanManifestMessage = attempt.orphanManifestMessage; - shieldsWindowOpened = attempt.shieldsWindowOpened; - } finally { - if (startedForBackup) { - if (returnSandboxContainerToStopped(startedForBackup.containerName)) { - console.log(` ${D}Returned '${sb.name}' to its stopped state.${R}`); - } else { - returnedToStopped = false; - console.error( - ` ${RD}✗${R} ${sb.name}: backup cleanup failed (could not return its container to the stopped state; the container was left running)`, - ); - } + ); + if (attempt.stoppedContainerUnavailable) { + if (orphanNames.has(sb.name) && isSandboxContainerDefinitivelyAbsent(sb.name)) { + // Tracked separately from `skipped` so the strict gate stays + // untripped: there is nothing to back up and nothing to start. + strandedOrphans.push(sb.name); + return; } + console.log(` ${D}${notRunningBackupSkipMessage(sb.name)}${R}`); + skipped++; + notRunningSkipped++; + return; + } + result = attempt.result; + orphanManifestMessage = attempt.orphanManifestMessage; + if ("mutationLockError" in attempt) { + mutationLockError = attempt.mutationLockError; + mutationLockFailed = true; } - if (!returnedToStopped) { + if (mutationLockFailed) { + const detail = + mutationLockError instanceof Error ? mutationLockError.message : String(mutationLockError); + console.error(` ${RD}✗${R} ${sb.name}: backup failed (mutation lock: ${detail})`); failed++; return; } - if (!shieldsWindowOpened) { + if (!attempt.shieldsWindowOpened) { console.error(` ${RD}✗${R} ${sb.name}: backup failed (could not safely unlock shields)`); failed++; return; @@ -294,22 +454,7 @@ export async function backupAll(): Promise { } }; for (const sb of sandboxes) { - let enteredMutationLock = false; - try { - await withSandboxMutationLock(sb.name, () => { - enteredMutationLock = true; - return backupRegisteredSandbox(sb); - }); - } catch (error) { - // Callback failures retain the existing fail-fast behavior. A lock that - // could not be acquired is instead one failed sandbox attempt so the - // remaining backups, orphan confirmation, summary, and strict gate all - // still run. - if (enteredMutationLock) throw error; - const detail = error instanceof Error ? error.message : String(error); - console.error(` ${RD}✗${R} ${sb.name}: backup failed (mutation lock: ${detail})`); - failed++; - } + await backupRegisteredSandbox(sb); } // The classification above is only as fresh as the pre-loop listing, and // the backup loop can run for minutes. Confirm with a second pinned listing diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 2a1f1562a80..77595a0ab18 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -672,9 +672,9 @@ function runSnapshotCreate( snapshotExit(1); } return withTimerBoundShieldsMutationLock(sandboxName, "create sandbox snapshot", () => { - // Keep the shields check and backup in one timer-bound interval. Normal - // auto-restore waits; at the absolute deadline it may preempt this process - // and reclaim the token rather than changing policy/config mid-copy. + // Keep the shields check and backup in one timer-bound interval. At the + // absolute deadline, auto-restore closes the outer lifecycle gate and waits + // for this exact owner to finish before changing policy or config. if (!isSnapshotCreationAllowedByShields(sandboxName)) { console.error(" Cannot create snapshot while shields are up."); console.error(` Run \`${CLI_NAME} ${sandboxName} shields down\` first, then retry.`); @@ -1282,9 +1282,9 @@ async function runSnapshotRestoreUnlocked( } withTimerBoundShieldsMutationLock(targetSandbox, "restore sandbox snapshot", () => { // Serialize filesystem restore, mutable-permission repair, and policy - // reconciliation under the active timer generation. Normal auto-restore - // waits; the absolute deadline may preempt this process and reclaim the - // token, preventing policy/config mutation after lockdown resumes. + // reconciliation under the active timer generation. At the absolute + // deadline, auto-restore keeps the outer lifecycle gate closed and waits + // for this exact owner to finish before restoring lockdown. const validateManagedRestoreBeforeMutation = preparedRuntimeRestore ? () => { const currentTarget = registry.getSandbox(targetSandbox); diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 1fd6bc6f8aa..66cc34876ac 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -11,19 +11,6 @@ import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } fr const requireDist = createRequire(import.meta.url); const shieldsModulePath = "./index.js"; -const HUNG_FORWARD_OWNER_SOURCE = ` -const { spawn } = require("node:child_process"); -const childScriptPath = process.argv[2]; -const childReadyPath = process.argv[3]; -spawn(process.execPath, [childScriptPath, childReadyPath], { stdio: "ignore" }); -setInterval(() => {}, 60000); -`; -const WEAKENING_CHILD_SOURCE = ` -const fs = require("node:fs"); -const childReadyPath = process.argv[2]; -fs.writeFileSync(childReadyPath, String(process.pid)); -setInterval(() => {}, 60000); -`; type ShieldsHarness = { auditSpy: MockInstance; @@ -38,8 +25,12 @@ type ShieldsHarness = { }; let tmpDir: string; +const currentProcessStartIdentity = ( + requireDist("./timer-control.js") as typeof import("./timer-control.js") +).readProcessStartIdentity(process.pid); type HarnessOptions = { + beginContainment?: typeof import("../state/mcp-lifecycle-lock.js").beginCommittedMcpLifecycleContainmentSync; directSandboxUnavailable?: boolean; dockerExecFileSync?: (argv: unknown) => string; failOpenClawGuardActions?: Array<"lock" | "unlock">; @@ -75,6 +66,14 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { delete require.cache[requireDist.resolve("./transition-lock.js")]; delete require.cache[requireDist.resolve("../sandbox/privileged-exec.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; + const lifecycleLock = requireDist( + "../state/mcp-lifecycle-lock.js", + ) as typeof import("../state/mcp-lifecycle-lock.js"); + const beginContainment = + options.beginContainment ?? lifecycleLock.beginCommittedMcpLifecycleContainmentSync; + vi.spyOn(lifecycleLock, "beginCommittedMcpLifecycleContainmentSync").mockImplementation( + beginContainment, + ); const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(console, "warn").mockImplementation(() => undefined); @@ -240,6 +239,55 @@ function expectStagedDriverNeutralRecovery( return output; } +function writeExpiredShieldsFixture( + processToken: string, + reason: string, + ownerState: "dead" | "live", +) { + const liveOwner = ownerState === "live"; + const sandboxName = "openclaw"; + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, `snapshot-${processToken.slice(0, 8)}.yaml`); + const timerMarkerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const transitionLockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); + fs.writeFileSync( + path.join(stateDir, `shields-${sandboxName}.json`), + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), + shieldsDownTimeout: 60, + shieldsDownReason: reason, + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }), + ); + fs.writeFileSync( + timerMarkerPath, + JSON.stringify({ + pid: liveOwner ? 2_147_483_647 : 4242, + sandboxName, + snapshotPath, + restoreAt: new Date(Date.now() - 60_000).toISOString(), + processToken, + }), + ); + fs.writeFileSync( + transitionLockPath, + JSON.stringify({ + version: 1, + sandboxName, + pid: liveOwner ? process.pid : 4242, + processStartIdentity: liveOwner ? currentProcessStartIdentity : "dead-timer", + command: liveOwner ? "shields down" : "shields auto-restore", + acquiredAtMs: Date.now() - 60_000, + takeoverToken: processToken, + }), + ); + return { stateDir, timerMarkerPath, transitionLockPath }; +} + describe("shields command flow", () => { beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-flow-")); @@ -346,42 +394,6 @@ describe("shields command flow", () => { }); }); - it("never selects the detached recovery timer or its children for owner-tree takeover", () => { - const shields = requireDist(shieldsModulePath) as { - excludeRecoveryProcessTree: ( - descendants: Array<{ pid: number; startIdentity: string; depth: number }>, - recovery: { pid: number; startIdentity: string }, - recoveryDescendants: Array<{ pid: number; startIdentity: string; depth: number }>, - ) => Array<{ pid: number; startIdentity: string; depth: number }>; - }; - const recovery = { pid: 200, startIdentity: "timer", depth: 1 }; - const recoveryChild = { pid: 201, startIdentity: "timer-child", depth: 2 }; - const weakeningChild = { pid: 300, startIdentity: "policy-set", depth: 1 }; - - expect( - shields.excludeRecoveryProcessTree([recovery, recoveryChild, weakeningChild], recovery, [ - recoveryChild, - ]), - ).toEqual([weakeningChild]); - }); - - it("does not exclude a weakening child that reused a recovery PID", () => { - const shields = requireDist(shieldsModulePath) as { - excludeRecoveryProcessTree: ( - descendants: Array<{ pid: number; startIdentity: string; depth: number }>, - recovery: { pid: number; startIdentity: string }, - recoveryDescendants: Array<{ pid: number; startIdentity: string; depth: number }>, - ) => Array<{ pid: number; startIdentity: string; depth: number }>; - }; - const recovery = { pid: 200, startIdentity: "timer", depth: 1 }; - const sampledRecoveryChild = { pid: 201, startIdentity: "timer-child", depth: 2 }; - const reusedPidChild = { pid: 201, startIdentity: "policy-set", depth: 1 }; - - expect( - shields.excludeRecoveryProcessTree([reusedPidChild], recovery, [sampledRecoveryChild]), - ).toEqual([reusedPidChild]); - }); - it("auto-restore waits for the forward shields-down commit before reclaiming policy", () => { const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -451,154 +463,19 @@ describe("shields command flow", () => { } expect(Date.now() - startedAt).toBeGreaterThanOrEqual(100); - expect(fs.existsSync(transitionPath)).toBe(false); + expect(fs.existsSync(transitionPath)).toBe(true); expect(harness.runSpy).toHaveBeenCalledWith( ["openshell", "policy", "set"], expect.objectContaining({ ignoreError: true }), ); }); - it("preempts a hung forward owner and its weakening subprocess before restoring", { - timeout: 20_000, - }, async () => { + it("preserves a live transition owner instead of attempting portable process-tree takeover", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "openclaw"; + const sandboxName = "live-transition-owner"; const processToken = "b".repeat(32); - const snapshotPath = path.join(stateDir, "policy-snapshot-hung.yaml"); - const childReadyPath = path.join(stateDir, "weakening-child-ready"); - const transitionPath = path.join( - stateDir, - `shields-transition-${sandboxName}-${processToken}.json`, - ); - const ownerScriptPath = path.join(stateDir, "hung-forward-owner.cjs"); - const childScriptPath = path.join(stateDir, "weakening-child.cjs"); - fs.writeFileSync(ownerScriptPath, HUNG_FORWARD_OWNER_SOURCE, { mode: 0o600 }); - fs.writeFileSync(childScriptPath, WEAKENING_CHILD_SOURCE, { mode: 0o600 }); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); - fs.writeFileSync( - path.join(stateDir, `shields-timer-${sandboxName}.json`), - JSON.stringify({ - pid: process.pid, - sandboxName, - snapshotPath, - restoreAt: new Date(Date.now() - 1_000).toISOString(), - processToken, - }), - ); - - const owner = spawn(process.execPath, [ownerScriptPath, childScriptPath, childReadyPath], { - stdio: "ignore", - }); - expect(owner.pid).toBeTypeOf("number"); - await vi.waitFor(() => expect(fs.existsSync(childReadyPath)).toBe(true), { - timeout: 5_000, - interval: 10, - }); - const childPid = Number(fs.readFileSync(childReadyPath, "utf-8")); - expect(Number.isInteger(childPid) && childPid > 0).toBe(true); - const timerControl = requireDist("./timer-control.js"); - const ownerStartIdentity = timerControl.readProcessStartIdentity(owner.pid); - expect(ownerStartIdentity).toBeTypeOf("string"); - const childStartIdentity = timerControl.readProcessStartIdentity(childPid); - expect(childStartIdentity).toBeTypeOf("string"); - const initialDescendants = timerControl.listDescendantProcessIdentities(owner.pid); - expect(initialDescendants).not.toBeNull(); - expect(initialDescendants.some(({ pid }: { pid: number }) => pid === childPid)).toBe(true); - const takeoverEvents: string[] = []; - const readProcessStartIdentity = timerControl.readProcessStartIdentity; - let unreadableOwnerIdentityReads = 2; - vi.spyOn(timerControl, "readProcessStartIdentity").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - const unreadable = pid === owner.pid && unreadableOwnerIdentityReads > 0; - unreadableOwnerIdentityReads -= unreadable ? 1 : 0; - return unreadable ? null : readProcessStartIdentity(pid, deadline); - }); - const readProcessState = timerControl.readProcessState; - vi.spyOn(timerControl, "readProcessState").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - const state = readProcessState(pid, deadline); - pid === owner.pid && /^[Tt]/.test(state ?? "") && takeoverEvents.push("owner-stopped"); - return state; - }); - const listDescendantProcessIdentities = timerControl.listDescendantProcessIdentities; - vi.spyOn(timerControl, "listDescendantProcessIdentities").mockImplementation( - (...args: unknown[]) => { - const [rootPid, deadline] = args as [number, number?]; - rootPid === owner.pid && takeoverEvents.push("owner-enumerated"); - return listDescendantProcessIdentities(rootPid, deadline); - }, - ); - const harness = createHarness({ - run: (cmd) => { - expect(cmd).toEqual(["openshell", "policy", "set"]); - const observedChildIdentity = readProcessStartIdentity(childPid); - const observedChildState = readProcessState(childPid); - let childCanBeSignaled = true; - try { - process.kill(childPid, 0); - } catch (error) { - childCanBeSignaled = (error as NodeJS.ErrnoException).code === "EPERM"; - } - const exactChildIsGone = - !childCanBeSignaled || - (observedChildIdentity !== null && observedChildIdentity !== childStartIdentity); - const childIsZombie = observedChildState?.startsWith("Z") === true; - expect(exactChildIsGone || childIsZombie).toBe(true); - takeoverEvents.push("policy-restored"); - return { status: 0 }; - }, - }); - fs.writeFileSync( - transitionPath, - JSON.stringify({ - version: 1, - phase: "preparing", - ownerPid: owner.pid, - ownerStartIdentity, - processToken, - sandboxName, - snapshotPath, - }), - { mode: 0o600 }, - ); - - try { - harness.synchronizeAutoRestoreWithShieldsDown(sandboxName); - } finally { - owner.kill("SIGKILL"); - try { - timerControl.readProcessStartIdentity(childPid) === childStartIdentity && - process.kill(childPid, "SIGKILL"); - } catch { - // The takeover already killed the exact child. - } - } - - expect(fs.existsSync(transitionPath)).toBe(false); - expect(takeoverEvents.indexOf("owner-stopped")).toBeGreaterThanOrEqual(0); - expect(takeoverEvents.indexOf("owner-enumerated")).toBeGreaterThan( - takeoverEvents.indexOf("owner-stopped"), - ); - expect(takeoverEvents).toContain("owner-enumerated"); - expect(takeoverEvents.indexOf("policy-restored")).toBeGreaterThan( - takeoverEvents.indexOf("owner-enumerated"), - ); - expect(harness.runSpy).toHaveBeenCalledWith( - ["openshell", "policy", "set"], - expect.objectContaining({ ignoreError: true }), - ); - }); - - it("fails closed when the weakening subprocess set never reaches quiescence", { - timeout: 10_000, - }, () => { - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "non-quiescent"; - const processToken = "c".repeat(32); const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); - const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 5000)"], { stdio: "ignore", }); @@ -619,38 +496,6 @@ describe("shields command flow", () => { }), { mode: 0o600 }, ); - - const syntheticPidBase = 2_000_000_000; - const readProcessStartIdentity = timerControl.readProcessStartIdentity; - vi.spyOn(timerControl, "readProcessStartIdentity").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - return pid === owner.pid - ? ownerStartIdentity - : pid >= syntheticPidBase - ? `synthetic:${String(pid)}` - : readProcessStartIdentity(pid, deadline); - }); - const readProcessState = timerControl.readProcessState; - vi.spyOn(timerControl, "readProcessState").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - return pid === owner.pid ? "T" : readProcessState(pid, deadline); - }); - const listDescendantProcessIdentities = timerControl.listDescendantProcessIdentities; - let ownerEnumerationPass = 0; - vi.spyOn(timerControl, "listDescendantProcessIdentities").mockImplementation( - (...args: unknown[]) => { - const [rootPid, deadline] = args as [number, number?]; - const ownerEnumeration = rootPid === owner.pid; - ownerEnumerationPass += ownerEnumeration ? 1 : 0; - const syntheticPid = syntheticPidBase + ownerEnumerationPass; - return ownerEnumeration - ? [{ pid: syntheticPid, startIdentity: `synthetic:${String(syntheticPid)}`, depth: 1 }] - : rootPid === process.pid - ? [] - : listDescendantProcessIdentities(rootPid, deadline); - }, - ); - vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); const processKillSpy = vi.spyOn(process, "kill"); createHarness(); const shields = requireDist(shieldsModulePath) as { @@ -668,88 +513,46 @@ describe("shields command flow", () => { processToken, path.join(stateDir, "unused-snapshot.yaml"), ), - ).toThrow("Timed-out shields-down process tree could not be frozen safely"); - expect(processKillSpy).toHaveBeenCalledWith(owner.pid, "SIGSTOP"); + ).toThrow("still active"); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); + expect(fs.existsSync(lockPath)).toBe(true); } finally { - owner.kill("SIGCONT"); owner.kill("SIGKILL"); } - - expect(ownerEnumerationPass).toBe(8); - expect(fs.existsSync(lockPath)).toBe(true); }); - it("does not signal a replacement that reuses the owner PID during final verification", () => { + it.each([ + ["matching", "c".repeat(32)], + ["different", "d".repeat(32)], + ])("enters durable containment for a %s-token transition whose owner exited in the recovery gap", (_tokenRelationship, transitionOwnerToken) => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "reused-owner"; - const processToken = "d".repeat(32); - const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); - const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 5000)"], { - stdio: "ignore", - }); - expect(owner.pid).toBeTypeOf("number"); - const timerControl = requireDist("./timer-control.js"); - const ownerStartIdentity = timerControl.readProcessStartIdentity(owner.pid); - expect(ownerStartIdentity).toBeTypeOf("string"); + const sandboxName = "dead-transition-owner"; + const processToken = "c".repeat(32); + const transitionLockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); fs.writeFileSync( - lockPath, + transitionLockPath, JSON.stringify({ version: 1, sandboxName, - pid: owner.pid, - processStartIdentity: ownerStartIdentity, + pid: 2_147_483_647, + processStartIdentity: "dead-owner", command: "config set write", acquiredAtMs: Date.now(), - takeoverToken: processToken, + takeoverToken: transitionOwnerToken, }), { mode: 0o600 }, ); - - const processKill = process.kill; - let ownerLivenessChecks = 0; - let replacementVisible = false; - const processKillSpy = vi.spyOn(process, "kill").mockImplementation((...args: unknown[]) => { - const [pid, signal] = args as [number, NodeJS.Signals | 0 | undefined]; - const ownerLivenessCheck = pid === owner.pid && signal === 0; - ownerLivenessChecks += ownerLivenessCheck ? 1 : 0; - replacementVisible ||= ownerLivenessCheck && ownerLivenessChecks === 2; - return processKill(pid, signal); - }); - const readProcessStartIdentity = timerControl.readProcessStartIdentity; - vi.spyOn(timerControl, "readProcessStartIdentity").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - return pid === owner.pid && replacementVisible - ? "replacement-process-start" - : readProcessStartIdentity(pid, deadline); - }); createHarness(); - const shields = requireDist(shieldsModulePath) as { - prepareAutoRestoreTransitionTakeover: ( + const transitionLock = requireDist("./transition-lock.js") as { + withShieldsTransitionLock: ( sandboxName: string, - processToken: string, - snapshotPath: string, + command: string, + fn: () => void, + options: { recoverStaleOwner: boolean; waitTimeoutMs: number }, ) => void; }; - - try { - shields.prepareAutoRestoreTransitionTakeover( - sandboxName, - processToken, - path.join(stateDir, "unused-snapshot.yaml"), - ); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); - } finally { - owner.kill("SIGCONT"); - owner.kill("SIGKILL"); - } - - expect(ownerLivenessChecks).toBeGreaterThanOrEqual(2); - }); - - it("preempts timer-token config and inference mutations at the restore deadline", async () => { const shields = requireDist(shieldsModulePath) as { prepareAutoRestoreTransitionTakeover: ( sandboxName: string, @@ -757,54 +560,37 @@ describe("shields command flow", () => { snapshotPath: string, ) => void; }; - const transitionLockPath = path.join(import.meta.dirname, "transition-lock.ts"); - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - fs.mkdirSync(stateDir, { recursive: true }); - - for (const [index, command] of ["config set write", "inference set"].entries()) { - const sandboxName = `deadline-${String(index)}`; - const processToken = String(index + 1).repeat(32); - const readyPath = path.join(stateDir, `${sandboxName}.ready`); - const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); - const owner = spawn( - process.execPath, - [ - "--import", - "tsx", - "-e", - [ - `const {withShieldsTransitionLock}=require(${JSON.stringify(transitionLockPath)})`, - "const fs=require('fs')", - "const [name,command,token,ready]=process.argv.slice(1)", - "withShieldsTransitionLock(name,command,()=>{fs.writeFileSync(ready,'ready');Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,10000)},{takeoverToken:token})", - ].join(";"), - sandboxName, - command, - processToken, - readyPath, - ], - { env: { ...process.env, HOME: tmpDir }, stdio: "ignore" }, - ); - - try { - const deadline = Date.now() + 5_000; - while ((!fs.existsSync(readyPath) || !fs.existsSync(lockPath)) && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(fs.existsSync(readyPath)).toBe(true); - expect(fs.existsSync(lockPath)).toBe(true); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js") as { + getMcpLifecycleLockPath: (sandboxName: string, stateDir: string) => string; + }; + const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath( + sandboxName, + stateDir, + )}.containment`; - shields.prepareAutoRestoreTransitionTakeover( - sandboxName, - processToken, - path.join(stateDir, `${sandboxName}.snapshot.yaml`), - ); + expect(() => + transitionLock.withShieldsTransitionLock( + sandboxName, + "shields auto-restore contender", + () => undefined, + { + recoverStaleOwner: false, + waitTimeoutMs: 0, + }, + ), + ).toThrow("recorded owner PID"); + expect(fs.existsSync(transitionLockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(false); - expect(fs.existsSync(lockPath)).toBe(false); - } finally { - owner.kill("SIGKILL"); - } - } + expect(() => + shields.prepareAutoRestoreTransitionTakeover( + sandboxName, + processToken, + path.join(stateDir, "unused-snapshot.yaml"), + ), + ).toThrow("durable containment"); + expect(fs.existsSync(transitionLockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); it("publishes preparing recovery ownership before weakening and active only after unlock", () => { @@ -874,6 +660,53 @@ describe("shields command flow", () => { expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(true); }); + it.skipIf(process.platform === "win32")( + "atomically replaces a timer marker symlink without modifying its target", + () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const markerPath = path.join(stateDir, "shields-timer-openclaw.json"); + const markerTargetPath = path.join(stateDir, "operator-owned-marker.json"); + const markerTarget = "operator-owned marker contents"; + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(markerTargetPath, markerTarget); + const originalRename = fs.renameSync.bind(fs); + const plantMarkerSymlink = () => fs.symlinkSync(markerTargetPath, markerPath); + const publicationRoutes = new Map void>([[markerPath, plantMarkerSymlink]]); + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => { + (publicationRoutes.get(String(destination)) ?? (() => undefined))(); + originalRename(source, destination); + }); + const harness = createHarness({ + fork: () => ({ + pid: 4242, + disconnect: vi.fn(), + unref: vi.fn(), + send: vi.fn(() => true), + kill: vi.fn(() => true), + }), + }); + + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "marker publication coverage", + throwOnError: true, + }); + + expect(renameSpy).toHaveBeenCalledWith(expect.stringContaining(".tmp"), markerPath); + const markerFd = fs.openSync(markerPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + expect(fs.fstatSync(markerFd).isFile()).toBe(true); + expect(JSON.parse(fs.readFileSync(markerFd, "utf-8"))).toMatchObject({ + pid: 4242, + sandboxName: "openclaw", + }); + } finally { + fs.closeSync(markerFd); + } + expect(fs.readFileSync(markerTargetPath, "utf-8")).toBe(markerTarget); + }, + ); + it("shieldsUp refuses to mark lockdown active when the saved restrictive policy snapshot is missing", () => { const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -1127,76 +960,53 @@ describe("shields command flow", () => { ).toBe(true); }); - it("shieldsStatus restores an expired dead timer under the shared sandbox lock", async () => { - const configPath = "/sandbox/.openclaw/openclaw.json"; - const configDir = "/sandbox/.openclaw"; - const hashPath = `${configDir}/.config-hash`; - const configHash = "a".repeat(64); - const hashHash = "b".repeat(64); + it("shieldsStatus contains an expired timer whose transition owner exited", () => { const processToken = "7".repeat(32); - const execCalls: string[] = []; - const execResponses = new Map([ - [` stat -c %a %U:%G ${hashPath}`, "444 root:root\n"], - [` stat -c %a %U:%G ${configPath}`, "444 root:root\n"], - [` stat -c %a %U:%G ${configDir}`, "755 root:root\n"], - [" stat -c %a %U:%G /sandbox", "1775 root:sandbox\n"], - [` lsattr -d ${hashPath}`, `----i---------e----- ${hashPath}\n`], - [` lsattr -d ${configPath}`, `----i---------e----- ${configPath}\n`], - [` sha256sum ${hashPath}`, `${hashHash} ${hashPath}\n`], - [` sha256sum ${configPath}`, `${configHash} ${configPath}\n`], - ]); const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); const sandboxMutationLockPath = lifecycleLock.getMcpLifecycleLockPath("openclaw"); - let policySetSawSandboxLock = false; - const harness = createHarness({ - run: () => { - policySetSawSandboxLock = fs.existsSync(sandboxMutationLockPath); - return { status: 0 }; - }, - dockerExecFileSync: (argv: unknown) => { - const args = Array.isArray(argv) ? argv.map(String) : []; - const cmd = args.join(" "); - execCalls.push(cmd); - return [...execResponses].find(([needle]) => cmd.includes(needle))?.[1] ?? ""; - }, + const containmentPath = `${sandboxMutationLockPath}.containment`; + const harness = createHarness(); + const { + stateDir, + timerMarkerPath, + transitionLockPath: lockPath, + } = writeExpiredShieldsFixture(processToken, "coverage", "dead"); + vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { + const failDeadTimerProbe = () => { + const error = new Error("timer is gone") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + }; + const deadTimerProbe = `${pid}:${signal}` === "4242:0" ? failDeadTimerProbe : undefined; + deadTimerProbe?.(); + return true; }); - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const lockPath = path.join(stateDir, "shields-transition-lock-openclaw.json"); - fs.mkdirSync(stateDir, { recursive: true }); - const snapshotPath = path.join(stateDir, "policy-snapshot-expired.yaml"); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), - shieldsDownTimeout: 60, - shieldsDownReason: "coverage", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, - }), + + expect(() => harness.shieldsStatus("openclaw")).toThrow("durable containment"); + + const state = JSON.parse( + fs.readFileSync(path.join(stateDir, "shields-openclaw.json"), "utf-8"), ); - fs.writeFileSync( - path.join(stateDir, "shields-timer-openclaw.json"), - JSON.stringify({ - pid: 4242, - sandboxName: "openclaw", - snapshotPath, - restoreAt: new Date(Date.now() - 30_000).toISOString(), - processToken, - }), + expect(state.shieldsDown).toBe(true); + expect(fs.existsSync(timerMarkerPath)).toBe(true); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); + expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); + expect(harness.runSpy).not.toHaveBeenCalledWith( + ["openshell", "policy", "set"], + expect.anything(), ); - fs.writeFileSync( - lockPath, - JSON.stringify({ - version: 1, - sandboxName: "openclaw", - pid: 4242, - processStartIdentity: "dead-timer", - command: "shields auto-restore", - acquiredAtMs: Date.now() - 60_000, - takeoverToken: processToken, - }), + }); + + it("retains the timer-bound lifecycle generation when a caller handles a failed containment write", () => { + const processToken = "a".repeat(32); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const mainLockPath = lifecycleLock.getMcpLifecycleLockPath("openclaw"); + const containmentPath = `${mainLockPath}.containment`; + const { timerMarkerPath, transitionLockPath } = writeExpiredShieldsFixture( + processToken, + "containment write failure coverage", + "dead", ); vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { const failDeadTimerProbe = () => { @@ -1208,32 +1018,106 @@ describe("shields command flow", () => { deadTimerProbe?.(); return true; }); + const harness = createHarness({ + beginContainment: () => { + throw new Error("state directory is read-only"); + }, + }); + let containmentFailure: unknown; - await lifecycleLock.withSandboxMutationLock("openclaw", () => - harness.shieldsStatus("openclaw"), - ); + let result: string | undefined; + try { + harness.shieldsStatus("openclaw"); + } catch (error) { + containmentFailure = error; + result = "handled"; + } - const state = JSON.parse( - fs.readFileSync(path.join(stateDir, "shields-openclaw.json"), "utf-8"), - ); - expect(harness.logSpy).toHaveBeenCalledWith(" Shields: UP (lockdown active)"); - expect(state.shieldsDown).toBe(false); - expect(state.fileHashes).toMatchObject({ - [configPath]: configHash, - [hashPath]: hashHash, + expect(result).toBe("handled"); + expect(containmentFailure).toMatchObject({ + code: "NEMOCLAW_DURABLE_CONTAINMENT", }); - expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(false); - expect(fs.existsSync(lockPath)).toBe(false); - expect(policySetSawSandboxLock).toBe(true); - expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); - expect(harness.auditSpy).toHaveBeenCalledWith( - expect.objectContaining({ - action: "shields_auto_restore", - policy_snapshot: snapshotPath, - restored_by: "auto_timer", - sandbox: "openclaw", - }), + expect(String(containmentFailure)).toContain("state directory is read-only"); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(JSON.parse(fs.readFileSync(mainLockPath, "utf8"))).toMatchObject({ + sandboxName: "openclaw", + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(timerMarkerPath)).toBe(true); + expect(fs.existsSync(transitionLockPath)).toBe(true); + expect(harness.runSpy).not.toHaveBeenCalledWith( + ["openshell", "policy", "set"], + expect.anything(), ); - expect(execCalls.some((cmd) => cmd.includes(` sha256sum ${hashPath}`))).toBe(true); }); + + it.skipIf(currentProcessStartIdentity === null)( + "bounds live transition takeover before committing durable containment", + () => { + const sandboxName = "openclaw"; + const processToken = "8".repeat(32); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath(sandboxName)}.containment`; + writeExpiredShieldsFixture(processToken, "takeover exhaustion coverage", "live"); + const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + const harness = createHarness(); + + expect(() => harness.shieldsStatus(sandboxName)).toThrow( + "Auto-restore transition takeover exhausted 7 attempts", + ); + + expect(waitSpy.mock.calls.map((call) => call[3])).toEqual([ + 5_000, 5_000, 5_000, 5_000, 5_000, 5_000, + ]); + expect(fs.existsSync(containmentPath)).toBe(true); + expect(harness.auditSpy).toHaveBeenCalledTimes(1); + expect(harness.auditSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: "shields_up_failed", + sandbox: sandboxName, + error: + "Shields transition owner is still active; automatic recovery is waiting behind the deadline gate", + }), + ); + }, + ); + + it.skipIf(currentProcessStartIdentity === null)( + "returns after bounded containment commit failures without reopening the deadline gate", + () => { + const sandboxName = "openclaw"; + const processToken = "9".repeat(32); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const mainLockPath = lifecycleLock.getMcpLifecycleLockPath(sandboxName); + const containmentPath = `${mainLockPath}.containment`; + writeExpiredShieldsFixture(processToken, "containment write failure coverage", "live"); + const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + let containmentAttempts = 0; + const harness = createHarness({ + beginContainment: () => { + containmentAttempts += 1; + throw new Error("state directory is read-only"); + }, + }); + + expect(() => harness.shieldsStatus(sandboxName)).toThrow( + /Durable containment could not be committed after 11 attempts: state directory is read-only.*Correct the state-directory write failure/, + ); + + expect(containmentAttempts).toBe(11); + expect(waitSpy.mock.calls.filter((call) => call[3] === 5_000)).toHaveLength(6); + expect(waitSpy.mock.calls.filter((call) => call[3] === 50)).toHaveLength(10); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(fs.existsSync(mainLockPath)).toBe(true); + expect(fs.existsSync(`${mainLockPath}.deadline`)).toBe(true); + expect(harness.auditSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: "shields_up_failed", + sandbox: sandboxName, + error: + "Durable containment commit failed; retrying behind the deadline gate: state directory is read-only", + }), + ); + }, + ); }); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index 2912f73d972..ad3bdbbd457 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -103,6 +103,19 @@ function withDefaultNodeExecFileSync( return defaultNodeExecFileSync(file, argv) || fallback(); } +function throwProcessNotRunning(): never { + throw Object.assign(new Error("not running"), { code: "ESRCH" }); +} + +function reportProcessRunning(): true { + return true; +} + +function routeProcessKill(pid: number, signal?: string | number): true { + const processActions = new Map true>([["2147483647:0", throwProcessNotRunning]]); + return (processActions.get(`${pid}:${signal}`) ?? reportProcessRunning)(); +} + beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-test-")); vi.stubEnv("HOME", tmpDir); @@ -527,6 +540,40 @@ describe("shields — unit logic", () => { expect(fs.existsSync(path.join(stateDir(), `shields-timer-${sandboxName}.json`))).toBe(true); }); + it("bounds current-generation inline recovery when the snapshot is missing (#7952)", async () => { + const sandboxName = "openclaw"; + const processToken = "c".repeat(32); + const missingSnapshotPath = path.join(stateDir(), "missing-current-snapshot.yaml"); + writeState(sandboxName, { + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 60_000).toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "testing", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: missingSnapshotPath, + updatedAt: new Date().toISOString(), + }); + writeMarker(sandboxName, { + pid: 2_147_483_647, + sandboxName, + snapshotPath: missingSnapshotPath, + restoreAt: new Date(Date.now() - 30_000).toISOString(), + processToken, + }); + vi.spyOn(process, "kill").mockImplementation(routeProcessKill); + vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + const { shieldsStatus } = await loadShieldsModule(); + + expect(() => shieldsStatus(sandboxName)).toThrow("Inline auto-restore exhausted 7 attempts"); + const { getMcpLifecycleLockPath } = await import("../state/mcp-lifecycle-lock"); + expect(fs.existsSync(`${getMcpLifecycleLockPath(sandboxName, stateDir())}.containment`)).toBe( + true, + ); + expect(fs.existsSync(path.join(stateDir(), `shields-timer-${sandboxName}.json`))).toBe(true); + }); + it("shieldsStatus attempts inline recovery when expired marker PID is alive but cmdline does not match recorded timer", async () => { const sandboxName = "openclaw"; const snapshotPath = path.join(stateDir(), "policy-snapshot-test.yaml"); @@ -958,7 +1005,7 @@ describe("NC-2227-05: shields timer marker behavior", () => { expect(readTimerMarker("openclaw")).toBeNull(); }); - it("killTimer terminates verified live timer process and clears marker", async () => { + it("killTimer cooperatively revokes a verified live timer without signaling it", async () => { const sourceModulePath = path.join(process.cwd(), "src", "lib", "shields", "timer-control.ts"); const { killTimer } = await import(sourceModulePath); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -1004,11 +1051,11 @@ describe("NC-2227-05: shields timer marker behavior", () => { markerFound: true, markerPid: 7331, wasAlive: true, - terminated: true, + terminated: false, warnings: [], }); expect(processKillSpy).toHaveBeenCalledWith(7331, 0); - expect(processKillSpy).toHaveBeenCalledWith(7331, "SIGTERM"); + expect(processKillSpy).toHaveBeenCalledTimes(1); expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(false); }); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index cd3110b8d3f..81437c29cc4 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -46,9 +46,7 @@ const { readTimerMarker, clearTimerMarker, isProcessAlive, - readProcessState, readProcessStartIdentity, - listDescendantProcessIdentities, processInspectionDeadlineAfter, processInspectionDeadlineReached, verifyTimerMarkerIdentity, @@ -64,11 +62,18 @@ const { cleanupTempDir } = require("../onboard/temp-files"); const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock"); const { relockAndReconfirm }: typeof import("./relock-reconfirm") = require("./relock-reconfirm"); const { - inspectShieldsTransitionLockOwner, - takeoverShieldsTransitionLock, + inspectAnyShieldsTransitionLockOwner, withShieldsTransitionLock, }: typeof import("./transition-lock") = require("./transition-lock"); const { + beginCommittedMcpLifecycleContainmentSync, + getMcpLifecycleLockPath, + isMcpLifecycleLockHeld, + durableMcpLifecycleContainmentFailure, + readMcpLockProcessIdentity, + withMcpLifecycleDeadlineFenceSync, + withMcpLifecycleLockSync, + withTimerBoundAutoRestoreLock, withTimerBoundShieldsMutationLock, }: typeof import("./timer-bound-lock") = require("./timer-bound-lock"); const { @@ -97,13 +102,19 @@ const { }: typeof import("./mutable-config-repair") = require("./mutable-config-repair"); type MutableConfigPermsInspection = import("./mutable-config-perms").MutableConfigPermsInspection; type MutableConfigRepairResult = import("./mutable-config-perms").MutableConfigRepairResult; -type ProcessIdentity = import("./timer-control").ProcessIdentity; - +type TimerMarker = import("./timer-control").TimerMarker; const STATE_DIR = resolveNemoclawStateDir(); const SHIELDS_TRANSITION_POLL_MS = 50; const SHIELDS_TRANSITION_HANDOFF_GRACE_MS = 500; const SHIELDS_TRANSITION_TERMINATE_GRACE_MS = 1000; +const INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS = + Math.floor(SHIELDS_TRANSITION_HANDOFF_GRACE_MS / SHIELDS_TRANSITION_POLL_MS) + 1; const AUTO_RESTORE_COMPLETION_GRACE_MS = 30_000; +// Retry on the detached timer's cadence for one additional completion-grace +// window before converting the live deadline fence into durable containment. +const INTERACTIVE_AUTO_RESTORE_RETRY_MS = 5_000; +const INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS = + Math.floor(AUTO_RESTORE_COMPLETION_GRACE_MS / INTERACTIVE_AUTO_RESTORE_RETRY_MS) + 1; const HERMES_RUNTIME_CONFIG_GUARD = "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py"; const HERMES_PYTHON = "/opt/hermes/.venv/bin/python"; const HERMES_RESTART_SEAL_STATE = "/run/nemoclaw/hermes-restart-seal.json"; @@ -117,6 +128,7 @@ type ShieldsDownTransition = { phase: "preparing" | "active"; ownerPid: number; ownerStartIdentity: string; + ownerMcpProcessIdentity?: string; processToken: string; sandboxName: string; snapshotPath: string; @@ -124,6 +136,33 @@ type ShieldsDownTransition = { const transitionPollBuffer = new Int32Array(new SharedArrayBuffer(4)); +function sameTimerMarkerGeneration(current: TimerMarker | null, expected: TimerMarker): boolean { + return ( + current?.pid === expected.pid && + current.sandboxName === expected.sandboxName && + current.snapshotPath === expected.snapshotPath && + current.restoreAt === expected.restoreAt && + current.processToken === expected.processToken && + current.allowLegacyHermesProtocol === expected.allowLegacyHermesProtocol && + current.leaseOwnerPid === expected.leaseOwnerPid && + current.leaseOwnerStartIdentity === expected.leaseOwnerStartIdentity + ); +} + +function assertTimerMarkerGeneration(sandboxName: string, expected: TimerMarker): void { + if (!sameTimerMarkerGeneration(readTimerMarker(sandboxName), expected)) { + throw new Error("Auto-restore authority changed before Shields transition takeover"); + } +} + +function appendAuditEntryBestEffort(entry: Parameters[0]): void { + try { + appendAuditEntry(entry); + } catch { + // A failed diagnostic write must not release an active recovery gate. + } +} + function shieldsDownTransitionPath(sandboxName: string, processToken: string): string { return path.join(STATE_DIR, `shields-transition-${sandboxName}-${processToken}.json`); } @@ -138,6 +177,9 @@ function isShieldsDownTransition(value: unknown): value is ShieldsDownTransition value.ownerPid > 0 && typeof value.ownerStartIdentity === "string" && value.ownerStartIdentity.length > 0 && + (value.ownerMcpProcessIdentity === undefined || + (typeof value.ownerMcpProcessIdentity === "string" && + value.ownerMcpProcessIdentity.length > 0)) && typeof value.processToken === "string" && /^[0-9a-f]{32}$/.test(value.processToken) && typeof value.sandboxName === "string" && @@ -172,7 +214,8 @@ function writeShieldsDownTransition( !current || current.phase !== expectedPhase || current.ownerPid !== transition.ownerPid || - current.snapshotPath !== transition.snapshotPath + current.snapshotPath !== transition.snapshotPath || + current.ownerMcpProcessIdentity !== transition.ownerMcpProcessIdentity ) { throw new Error("Shields-down recovery ownership changed during the transition"); } @@ -191,6 +234,22 @@ function writeShieldsDownTransition( } } +function writeTimerMarkerAtomic(sandboxName: string, marker: TimerMarker): void { + const markerPath = timerMarkerPath(sandboxName); + fs.mkdirSync(path.dirname(markerPath), { recursive: true, mode: 0o700 }); + const tempPath = `${markerPath}.${String(process.pid)}.${randomBytes(8).toString("hex")}.tmp`; + try { + fs.writeFileSync(tempPath, JSON.stringify(marker), { flag: "wx", mode: 0o600 }); + fs.renameSync(tempPath, markerPath); + } finally { + try { + fs.rmSync(tempPath, { force: true }); + } catch { + // Best effort. The authoritative path was either atomically replaced or unchanged. + } + } +} + function clearShieldsDownTransition(sandboxName: string, processToken: string): void { try { fs.rmSync(shieldsDownTransitionPath(sandboxName, processToken), { force: true }); @@ -223,9 +282,25 @@ function readExactProcessStatus( return alive ? "current" : "gone"; } +function persistUnresolvedShieldsContainment( + sandboxName: string, + processToken: string, + reason: string, +): void { + const containmentPath = `${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`; + if (fs.existsSync(containmentPath)) return; + try { + beginCommittedMcpLifecycleContainmentSync(sandboxName, processToken, reason, STATE_DIR); + } catch (error) { + if (fs.existsSync(containmentPath)) return; + throw error; + } +} + function waitForShieldsDownForwardCommit( sandboxName: string, processToken: string, + assertTakeoverAuthority?: () => void, ): ShieldsDownTransition | null { let observed = readShieldsDownTransition(sandboxName, processToken); if (!observed) return null; @@ -245,6 +320,7 @@ function waitForShieldsDownForwardCommit( if ( next.ownerPid !== observed.ownerPid || next.ownerStartIdentity !== observed.ownerStartIdentity || + next.ownerMcpProcessIdentity !== observed.ownerMcpProcessIdentity || next.snapshotPath !== observed.snapshotPath || next.processToken !== observed.processToken ) { @@ -254,150 +330,38 @@ function waitForShieldsDownForwardCommit( } if (observed.phase === "preparing") { - // The absolute shields-down deadline has expired while the forward owner - // is still able to weaken policy/config. Preempt that exact process - // instance, then restore from the captured snapshot. Waiting forever would - // turn the requested timeout into an unbounded mutable window. - stopTimedOutShieldsDownTree(observed.ownerPid, observed.ownerStartIdentity); - } - return observed; -} - -function excludeRecoveryProcessTree( - descendants: ProcessIdentity[], - recovery: Pick, - recoveryDescendants: ProcessIdentity[], -): ProcessIdentity[] { - const identityKey = ({ pid, startIdentity }: Pick) => - `${String(pid)}\0${startIdentity}`; - const excludedIdentities = new Set([recovery, ...recoveryDescendants].map(identityKey)); - return descendants.filter((descendant) => !excludedIdentities.has(identityKey(descendant))); -} - -function stopTimedOutShieldsDownTree(ownerPid: number, ownerStartIdentity: string): void { - let freezeDeadline = processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS); - const waitForKnownExactProcess = ( - pid: number, - startIdentity: string, - deadline: number, - ): Exclude => { - while (true) { - const status = readExactProcessStatus(pid, startIdentity, deadline); - if (status !== "unknown") return status; - if (processInspectionDeadlineReached(deadline)) { - throw new Error("Timed-out shields-down process identity could not be verified safely"); - } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); - } - }; - const signalExact = ( - pid: number, - startIdentity: string, - signal: NodeJS.Signals, - deadline: number, - ): void => { - if (waitForKnownExactProcess(pid, startIdentity, deadline) === "gone") return; - try { - process.kill(pid, signal); - } catch (error) { - const errno = error as NodeJS.ErrnoException; - if (errno.code !== "ESRCH") throw error; - } - }; - const waitForExactStop = (pid: number, startIdentity: string): "gone" | "stopped" => { - while (true) { - const state = readProcessState(pid, freezeDeadline); - const status = readExactProcessStatus(pid, startIdentity, freezeDeadline); - if (status === "gone" || state?.startsWith("Z")) return "gone"; - if (status === "current" && /^[Tt]/.test(state ?? "")) return "stopped"; - if (processInspectionDeadlineReached(freezeDeadline)) { - throw new Error("Timed-out shields-down process tree could not be frozen safely"); - } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); - } - }; - if (waitForKnownExactProcess(ownerPid, ownerStartIdentity, freezeDeadline) === "gone") return; - // Stop the exact owner before enumerating its descendants so it cannot launch - // another weakening subprocess while takeover is being established. - signalExact(ownerPid, ownerStartIdentity, "SIGSTOP", freezeDeadline); - if (waitForExactStop(ownerPid, ownerStartIdentity) === "gone") return; - freezeDeadline = processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS); - const recoveryStartIdentity = readProcessStartIdentity(process.pid, freezeDeadline); - if (recoveryStartIdentity === null) { - throw new Error("Cannot identify the auto-restore recovery process safely"); - } - const recoveryTree = listDescendantProcessIdentities(process.pid, freezeDeadline); - if (recoveryTree === null) { - throw new Error("Cannot identify the auto-restore recovery process tree safely"); - } - const tracked = new Map(); - let observedQuiescentPass = false; - for (let pass = 0; pass < 8; pass += 1) { - if (waitForExactStop(ownerPid, ownerStartIdentity) === "gone") { - throw new Error("Timed-out shields-down process tree could not be frozen safely"); - } - const descendants = listDescendantProcessIdentities(ownerPid, freezeDeadline); - if (descendants === null) { - throw new Error("Cannot enumerate timed-out shields-down subprocesses safely"); - } - let added = false; - const recoveryIsInsideOwnerTree = descendants.some( - ({ pid }: { pid: number }) => pid === process.pid, + const ownerStatus = readExactProcessStatus( + observed.ownerPid, + observed.ownerStartIdentity, + processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS), ); - const passDescendants = excludeRecoveryProcessTree( - descendants, - { pid: process.pid, startIdentity: recoveryStartIdentity }, - recoveryIsInsideOwnerTree ? recoveryTree : [], - ); - for (const descendant of passDescendants) { - const previous = tracked.get(descendant.pid); - if (!previous || previous.startIdentity !== descendant.startIdentity) added = true; - tracked.set(descendant.pid, { - startIdentity: descendant.startIdentity, - depth: descendant.depth, - }); - signalExact(descendant.pid, descendant.startIdentity, "SIGSTOP", freezeDeadline); - } - for (const descendant of passDescendants) { - waitForExactStop(descendant.pid, descendant.startIdentity); - } - if (!added) { - observedQuiescentPass = true; - break; + if (ownerStatus === "gone") { + assertTakeoverAuthority?.(); + try { + persistUnresolvedShieldsContainment( + sandboxName, + processToken, + `Shields recovery owner PID ${String( + observed.ownerPid, + )} exited without descendant-containment proof`, + ); + } catch (error) { + throw durableMcpLifecycleContainmentFailure( + error, + getMcpLifecycleLockPath(sandboxName, STATE_DIR), + ); + } + throw new Error( + "Shields-down forward owner exited before committing its final mutation; durable containment requires operator resolution", + ); } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); - } - if (!observedQuiescentPass) { - throw new Error("Timed-out shields-down process tree could not be frozen safely"); - } - - const deepestFirst = [...tracked.entries()].sort((a, b) => b[1].depth - a[1].depth); - const killDeadline = processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS); - for (const [pid, identity] of deepestFirst) { - signalExact(pid, identity.startIdentity, "SIGKILL", killDeadline); - } - signalExact(ownerPid, ownerStartIdentity, "SIGKILL", killDeadline); - - const exactProcessIsGone = (pid: number, startIdentity: string): boolean => { - const state = readProcessState(pid, killDeadline); - return ( - state?.startsWith("Z") === true || - readExactProcessStatus(pid, startIdentity, killDeadline) === "gone" - ); - }; - while (!processInspectionDeadlineReached(killDeadline)) { - const survivor = deepestFirst.some( - ([pid, identity]) => !exactProcessIsGone(pid, identity.startIdentity), + throw new Error( + "Shields-down forward owner is still active; automatic recovery is waiting behind the deadline gate", ); - if (!survivor && exactProcessIsGone(ownerPid, ownerStartIdentity)) { - return; - } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); } - throw new Error("Timed-out shields-down process tree could not be stopped safely"); + return observed; } -// --------------------------------------------------------------------------- // privileged sandbox exec — bypasses the sandbox's Landlock context // // openshell sandbox exec runs commands INSIDE the Landlock domain, so it @@ -877,30 +841,257 @@ function getShieldsPostureWithoutHostLock( return { ...describeShieldsMode(mode), state }; } -function prepareExpiredAutoRestoreHostLockTakeover(sandboxName: string): void { +type ExpiredAutoRestoreTakeover = { + marker: TimerMarker & { processToken: string }; +}; + +function inspectExpiredAutoRestoreMarker(sandboxName: string): TimerMarker | null { const state = loadShieldsState(sandboxName); - if (state._isCorrupt || state.shieldsDown !== true) return; + if (state._isCorrupt || state.shieldsDown !== true) return null; const marker = readTimerMarker(sandboxName); - if (!marker?.processToken || !/^[0-9a-f]{32}$/.test(marker.processToken)) return; + if (!marker) return null; const restoreAtMs = new Date(marker.restoreAt).getTime(); const now = Date.now(); - if (!Number.isFinite(restoreAtMs) || restoreAtMs > now) return; + if (!Number.isFinite(restoreAtMs) || restoreAtMs > now) return null; if ( isProcessAlive(marker.pid) && verifyTimerMarkerIdentity(marker).verified && now <= restoreAtMs + AUTO_RESTORE_COMPLETION_GRACE_MS ) { - return; + return null; + } + return marker; +} + +function inspectExpiredAutoRestoreTakeover( + sandboxName: string, + marker = inspectExpiredAutoRestoreMarker(sandboxName), +): ExpiredAutoRestoreTakeover | null { + if (!marker?.processToken || !/^[0-9a-f]{32}$/.test(marker.processToken)) return null; + return { + marker: marker as TimerMarker & { processToken: string }, + }; +} + +function failInteractiveAutoRestoreClosed( + sandboxName: string, + marker: TimerMarker & { processToken: string }, + message: string, +): never { + const containmentPath = `${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`; + let notifiedError: string | null = null; + let lastContainmentError: string | null = null; + // Retry a durable containment commit for one normal transition-handoff + // window. A persistent state-directory failure returns to the operator only + // through the coded failure that keeps the owned lifecycle gates. + for (let attempt = 0; attempt < INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS; attempt += 1) { + assertTimerMarkerGeneration(sandboxName, marker); + try { + persistUnresolvedShieldsContainment( + sandboxName, + marker.processToken, + `Interactive auto-restore could not complete safely: ${message}`, + ); + break; + } catch (error) { + if (fs.existsSync(containmentPath)) break; + assertTimerMarkerGeneration(sandboxName, marker); + const containmentError = error instanceof Error ? error.message : String(error); + lastContainmentError = containmentError; + if (containmentError !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: `Durable containment commit failed; retrying behind the deadline gate: ${containmentError}`, + }); + notifiedError = containmentError; + } + if (attempt + 1 < INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS) { + Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); + } + } + } + if (!fs.existsSync(containmentPath)) { + throw durableMcpLifecycleContainmentFailure( + new Error( + `${message}. Durable containment could not be committed after ${String( + INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS, + )} attempts: ${lastContainmentError ?? "unknown state-directory failure"}. Correct the state-directory write failure and retry the command before running another sandbox mutation`, + ), + getMcpLifecycleLockPath(sandboxName, STATE_DIR), + ); + } + throw new Error(`${message}. Durable sandbox mutation containment requires operator resolution`); +} + +function isDurableContainmentFailure(error: unknown): boolean { + return ( + error instanceof Error && + (error as Error & { code?: string }).code === "NEMOCLAW_DURABLE_CONTAINMENT" + ); +} + +function retryInlineAutoRestore( + sandboxName: string, + marker: TimerMarker & { processToken: string }, +): void { + let notifiedError: string | null = null; + for (let attempt = 0; attempt < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS; attempt += 1) { + try { + const recoveredState = recoverExpiredAutoRestoreGate(sandboxName, true); + if (!recoveredState._isCorrupt && recoveredState.shieldsDown !== true) { + return; + } + assertTimerMarkerGeneration(sandboxName, marker); + const message = "Inline auto-restore did not complete; retrying under the lifecycle gate"; + if (message !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: message, + }); + notifiedError = message; + } + } catch (error) { + assertTimerMarkerGeneration(sandboxName, marker); + const message = error instanceof Error ? error.message : String(error); + if (message !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: message, + }); + notifiedError = message; + } + } + if (attempt + 1 < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS) { + Atomics.wait(transitionPollBuffer, 0, 0, INTERACTIVE_AUTO_RESTORE_RETRY_MS); + } + } + failInteractiveAutoRestoreClosed( + sandboxName, + marker, + `Inline auto-restore exhausted ${String( + INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS, + )} attempts: ${notifiedError ?? "recovery did not complete"}`, + ); +} + +function withExpiredAutoRestoreDeadlineFence( + sandboxName: string, + command: string, + operation: (allowInlineRecovery: boolean) => T, +): T { + const expiredMarker = inspectExpiredAutoRestoreMarker(sandboxName); + const takeover = inspectExpiredAutoRestoreTakeover(sandboxName, expiredMarker); + const runWithHostLock = (callback: () => T) => + withTimerBoundShieldsMutationLock(sandboxName, command, callback); + const recoverThenRun = () => + withTimerBoundAutoRestoreLock(sandboxName, command, () => { + if (takeover) retryInlineAutoRestore(sandboxName, takeover.marker); + return operation(false); + }); + if (isMcpLifecycleLockHeld(sandboxName, STATE_DIR)) { + if (!expiredMarker || !takeover) { + return runWithHostLock(() => operation(true)); + } + const { marker } = takeover; + prepareAutoRestoreTransitionTakeover( + sandboxName, + marker.processToken, + marker.snapshotPath, + () => assertTimerMarkerGeneration(sandboxName, marker), + ); + return recoverThenRun(); + } + if (!takeover) { + return withMcpLifecycleLockSync(sandboxName, () => runWithHostLock(() => operation(true)), { + stateDir: STATE_DIR, + }); } - prepareAutoRestoreTransitionTakeover(sandboxName, marker.processToken, marker.snapshotPath); + + const { marker } = takeover; + const assertTakeoverAuthority = () => assertTimerMarkerGeneration(sandboxName, marker); + return withMcpLifecycleDeadlineFenceSync( + sandboxName, + marker.processToken, + () => { + let notifiedError: string | null = null; + for (let attempt = 0; attempt < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS; attempt += 1) { + try { + prepareAutoRestoreTransitionTakeover( + sandboxName, + marker.processToken, + marker.snapshotPath, + assertTakeoverAuthority, + ); + return recoverThenRun(); + } catch (error) { + assertTakeoverAuthority(); + if ( + isDurableContainmentFailure(error) || + fs.existsSync(`${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`) + ) { + throw error; + } + const message = error instanceof Error ? error.message : String(error); + if (message !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: message, + }); + notifiedError = message; + } + if (attempt + 1 < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS) { + Atomics.wait(transitionPollBuffer, 0, 0, INTERACTIVE_AUTO_RESTORE_RETRY_MS); + } + } + } + return failInteractiveAutoRestoreClosed( + sandboxName, + marker, + `Auto-restore transition takeover exhausted ${String( + INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS, + )} attempts: ${notifiedError ?? "transition ownership did not become available"}`, + ); + }, + { + stateDir: STATE_DIR, + throwOnCommittedContainment: true, + onContainment: ({ ownerPid, reason }) => { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: `${reason}${ownerPid ? ` Contained owner PID: ${String(ownerPid)}.` : ""}`, + }); + }, + }, + ); } function getShieldsPosture(sandboxName: string, allowInlineRecovery = false): ShieldsPosture { if (!allowInlineRecovery) return getShieldsPostureWithoutHostLock(sandboxName, false); validateName(sandboxName, "sandbox name"); - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock(sandboxName, "recover expired shields posture", () => - getShieldsPostureWithoutHostLock(sandboxName, true), + return withExpiredAutoRestoreDeadlineFence( + sandboxName, + "recover expired shields posture", + (allowInlineRecovery) => getShieldsPostureWithoutHostLock(sandboxName, allowInlineRecovery), ); } @@ -1936,15 +2127,14 @@ function unlockAgentConfig( function inspectMutableConfigPerms(sandboxName: string): MutableConfigPermsInspection { validateName(sandboxName, "sandbox name"); - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock( + return withExpiredAutoRestoreDeadlineFence( sandboxName, "inspect mutable config permissions", - () => { + (allowInlineRecovery) => { const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); return inspectMutableConfigPermsCore( target, - getShieldsPostureWithoutHostLock(sandboxName, true).mode, + getShieldsPostureWithoutHostLock(sandboxName, allowInlineRecovery).mode, (p) => privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", p]), ); }, @@ -1953,15 +2143,18 @@ function inspectMutableConfigPerms(sandboxName: string): MutableConfigPermsInspe function repairMutableConfigPerms(sandboxName: string): MutableConfigRepairResult { validateName(sandboxName, "sandbox name"); - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock(sandboxName, "repair mutable config permissions", () => { - const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); - return repairMutableConfigPermsCore( - target, - getShieldsPostureWithoutHostLock(sandboxName, true).mode, - () => normalizeMutableOpenClawConfig(sandboxName, target.configDir), - ); - }); + return withExpiredAutoRestoreDeadlineFence( + sandboxName, + "repair mutable config permissions", + (allowInlineRecovery) => { + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); + return repairMutableConfigPermsCore( + target, + getShieldsPostureWithoutHostLock(sandboxName, allowInlineRecovery).mode, + () => normalizeMutableOpenClawConfig(sandboxName, target.configDir), + ); + }, + ); } // --------------------------------------------------------------------------- @@ -2241,8 +2434,16 @@ function synchronizeAutoRestoreTransition( sandboxName: string, processToken: string, snapshotPath: string, + options: { + retainTransition?: boolean; + assertTakeoverAuthority?: () => void; + } = {}, ): void { - const transition = waitForShieldsDownForwardCommit(sandboxName, processToken); + const transition = waitForShieldsDownForwardCommit( + sandboxName, + processToken, + options.assertTakeoverAuthority, + ); if (!transition) return; if (transition.snapshotPath !== snapshotPath) { throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); @@ -2263,49 +2464,75 @@ function synchronizeAutoRestoreTransition( `Policy restore after shields-down handoff exited with status ${String(status)}`, ); } - clearShieldsDownTransition(sandboxName, processToken); + if (!options.retainTransition) { + clearShieldsDownTransition(sandboxName, processToken); + } } -function prepareAutoRestoreTransitionTakeover( +function inspectAutoRestoreTransitionTakeoverOwner( sandboxName: string, processToken: string, snapshotPath: string, -): void { +): { pid: number; processIdentity: string } | null { if (!/^[0-9a-f]{32}$/.test(processToken)) { throw new Error("Invalid auto-restore transition takeover token"); } + const transition = readShieldsDownTransition(sandboxName, processToken); + if (transition && transition.snapshotPath !== snapshotPath) { + throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); + } + return transition?.ownerMcpProcessIdentity !== undefined + ? { pid: transition.ownerPid, processIdentity: transition.ownerMcpProcessIdentity } + : null; +} +function prepareAutoRestoreTransitionTakeover( + sandboxName: string, + processToken: string, + snapshotPath: string, + assertTakeoverAuthority?: () => void, +): { pid: number; processIdentity: string } | null { + const initialTransitionOwner = inspectAutoRestoreTransitionTakeoverOwner( + sandboxName, + processToken, + snapshotPath, + ); const transition = readShieldsDownTransition(sandboxName, processToken); if (transition && transition.snapshotPath !== snapshotPath) { throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); } if (transition) { - // This waits briefly for the forward commit and stops its exact process - // tree if the deadline fired while it was still weakening the sandbox. - waitForShieldsDownForwardCommit(sandboxName, processToken); - } - - const owner = inspectShieldsTransitionLockOwner(sandboxName, processToken); - if (!owner) return; - // The same timer token is also propagated to config/inference/restart - // mutations made during the mutable window. At expiry those operations - // are weaker than restoring lockdown and may be preempted safely. The stop - // helper pins the exact identity and fails closed if it cannot be read. - stopTimedOutShieldsDownTree(owner.pid, owner.processStartIdentity); - const takeover = takeoverShieldsTransitionLock( - sandboxName, + waitForShieldsDownForwardCommit(sandboxName, processToken, assertTakeoverAuthority); + } + + const owner = inspectAnyShieldsTransitionLockOwner(sandboxName); + if (!owner) return initialTransitionOwner; + const ownerStatus = readExactProcessStatus( owner.pid, owner.processStartIdentity, - processToken, + processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS), ); - if ( - !takeover.removed && - takeover.reason !== "missing" && - takeover.reason !== "path-changed" && - takeover.reason !== "owner-mismatch" - ) { - throw new Error(`Cannot take over expired shields transition lock: ${takeover.reason}`); + if (ownerStatus === "gone") { + assertTakeoverAuthority?.(); + try { + persistUnresolvedShieldsContainment( + sandboxName, + processToken, + `Shields recovery owner PID ${String(owner.pid)} exited without descendant-containment proof`, + ); + } catch (error) { + throw durableMcpLifecycleContainmentFailure( + error, + getMcpLifecycleLockPath(sandboxName, STATE_DIR), + ); + } + throw new Error( + "Shields transition owner exited without descendant-containment proof; durable containment requires operator resolution", + ); } + throw new Error( + "Shields transition owner is still active; automatic recovery is waiting behind the deadline gate", + ); } function synchronizeAutoRestoreWithShieldsDown(sandboxName: string): void { @@ -2318,7 +2545,36 @@ function synchronizeAutoRestoreWithShieldsDown(sandboxName: string): void { ) { return; } - synchronizeAutoRestoreTransition(sandboxName, timerMarker.processToken, timerMarker.snapshotPath); + synchronizeAutoRestoreTransition( + sandboxName, + timerMarker.processToken, + timerMarker.snapshotPath, + { + retainTransition: true, + assertTakeoverAuthority: () => assertTimerMarkerGeneration(sandboxName, timerMarker), + }, + ); +} + +function completeAutoRestoreTransition( + sandboxName: string, + processToken: string, + snapshotPath: string, +): boolean { + const marker = readTimerMarker(sandboxName); + if ( + marker?.pid !== process.pid || + marker.processToken !== processToken || + marker.snapshotPath !== snapshotPath + ) { + return false; + } + const transition = readShieldsDownTransition(sandboxName, processToken); + if (transition && transition.snapshotPath !== snapshotPath) { + throw new Error("Auto-restore completion does not match shields-down transition ownership"); + } + clearShieldsDownTransition(sandboxName, processToken); + return true; } function lockAgentConfigWithoutHostLock( @@ -2495,20 +2751,10 @@ function recoverExpiredAutoRestoreInline( if (Date.now() <= restoreAtMs + AUTO_RESTORE_COMPLETION_GRACE_MS) { return { attempted: false, restored: false }; } - const timerStartIdentity = readProcessStartIdentity(marker.pid); - if (!timerStartIdentity) { - console.error( - " Recovery warning: expired auto-restore timer identity cannot be pinned safely.", - ); - return { attempted: true, restored: false }; - } - try { - stopTimedOutShieldsDownTree(marker.pid, timerStartIdentity); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Recovery warning: ${message}`); - return { attempted: true, restored: false }; - } + console.error( + " Recovery warning: the expired auto-restore timer is still active; refusing portable process-tree preemption and waiting for it to exit.", + ); + return { attempted: true, restored: false }; } console.error( @@ -2517,7 +2763,10 @@ function recoverExpiredAutoRestoreInline( if (marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken)) { try { - synchronizeAutoRestoreTransition(sandboxName, marker.processToken, marker.snapshotPath); + synchronizeAutoRestoreTransition(sandboxName, marker.processToken, marker.snapshotPath, { + retainTransition: true, + assertTakeoverAuthority: () => assertTimerMarkerGeneration(sandboxName, marker), + }); } catch (error) { const message = error instanceof Error ? error.message : String(error); appendAuditEntry({ @@ -2727,6 +2976,11 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = (() => { throw new Error("Cannot identify shields-down owner process"); })(), + ownerMcpProcessIdentity: + readMcpLockProcessIdentity(process.pid, true) ?? + (() => { + throw new Error("Cannot identify shields-down lifecycle owner process"); + })(), processToken, sandboxName, snapshotPath, @@ -2762,32 +3016,23 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = }, ); if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); - fs.writeFileSync( - timerMarkerPath(sandboxName), - JSON.stringify({ - pid: timerChild.pid, - sandboxName, - snapshotPath, - restoreAt: restoreAt.toISOString(), - processToken, - allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, - ...(leaseOwnerPid !== null && leaseOwnerStartIdentity - ? { leaseOwnerPid, leaseOwnerStartIdentity } - : {}), - }), - { mode: 0o600 }, - ); + writeTimerMarkerAtomic(sandboxName, { + pid: timerChild.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAt.toISOString(), + processToken, + allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, + ...(leaseOwnerPid !== null && leaseOwnerStartIdentity + ? { leaseOwnerPid, leaseOwnerStartIdentity } + : {}), + }); if (!timerChild.send({ type: "authorize", processToken })) { throw new Error("auto-restore timer authorization channel closed early"); } timerChild.disconnect(); timerChild.unref(); } catch (err) { - try { - timerChild?.kill("SIGTERM"); - } catch { - // Best effort; without a matching marker the child has no authority. - } clearTimerMarker(sandboxName); clearShieldsDownTransition(sandboxName, processToken); const message = err instanceof Error ? err.message : String(err); @@ -3213,7 +3458,7 @@ function shieldsUp( ): void { validateName(sandboxName, "sandbox name"); try { - return withTimerBoundShieldsMutationLock(sandboxName, "shields up", () => + return withExpiredAutoRestoreDeadlineFence(sandboxName, "shields up", () => shieldsUpWithoutHostLock(sandboxName, opts), ); } catch (error) { @@ -3368,9 +3613,10 @@ function shieldsStatus( shieldsStatusWithoutHostLock(sandboxName, false, deps), ); } - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock(sandboxName, "shields status", () => - shieldsStatusWithoutHostLock(sandboxName, true, deps), + return withExpiredAutoRestoreDeadlineFence( + sandboxName, + "shields status", + (allowInlineRecovery) => shieldsStatusWithoutHostLock(sandboxName, allowInlineRecovery, deps), ); } catch (error) { return completeDeferredShieldsExit(error); @@ -3431,10 +3677,11 @@ function clearShieldsState(sandboxName: string): void { export { clearShieldsState, + completeAutoRestoreTransition, DEFAULT_TIMEOUT_SECONDS, deriveShieldsMode, - excludeRecoveryProcessTree, getShieldsPosture, + inspectAutoRestoreTransitionTakeoverOwner, inspectMutableConfigPerms, isShieldsDown, killTimer, diff --git a/src/lib/shields/timer-bound-lock.ts b/src/lib/shields/timer-bound-lock.ts index 4514cefe55f..d27b6e00420 100644 --- a/src/lib/shields/timer-bound-lock.ts +++ b/src/lib/shields/timer-bound-lock.ts @@ -2,7 +2,21 @@ // SPDX-License-Identifier: Apache-2.0 import { readAutoRestoreTakeoverToken } from "./timer-control"; -import { withShieldsTransitionLock, withShieldsTransitionLockAsync } from "./transition-lock"; +import { + type ShieldsTransitionLockOptions, + withShieldsTransitionLock, + withShieldsTransitionLockAsync, +} from "./transition-lock"; + +export { + beginCommittedMcpLifecycleContainmentSync, + durableMcpLifecycleContainmentFailure, + getMcpLifecycleLockPath, + isMcpLifecycleLockHeld, + readMcpLockProcessIdentity, + withMcpLifecycleDeadlineFenceSync, + withMcpLifecycleLockSync, +} from "../state/mcp-lifecycle-lock"; const MAX_TIMER_GENERATION_RETRIES = 3; @@ -20,18 +34,12 @@ const defaultDeps: TimerBoundLockDeps = { type Attempt = { retry: true } | { retry: false; value: T }; -/** - * Serialize a mutation and bind its lock owner to the exact active restore - * timer generation. If a timer is replaced while this operation waits for the - * lock, release without mutating and retry with the new token. This prevents a - * command that observed timer A from becoming non-preemptible inside timer B's - * mutable window. - */ -export function withTimerBoundShieldsMutationLock( +function withTimerBoundShieldsMutationLockOptions( sandboxName: string, command: string, fn: () => T, - deps: TimerBoundLockDeps = defaultDeps, + lockOptions: ShieldsTransitionLockOptions, + deps: TimerBoundLockDeps, ): T { for (let attempt = 0; attempt < MAX_TIMER_GENERATION_RETRIES; attempt += 1) { const token = deps.readToken(sandboxName); @@ -42,13 +50,52 @@ export function withTimerBoundShieldsMutationLock( if (deps.readToken(sandboxName) !== token) return { retry: true }; return { retry: false, value: fn() }; }, - token ? { takeoverToken: token } : {}, + { + ...lockOptions, + ...(token ? { takeoverToken: token } : {}), + }, ); if (!result.retry) return result.value; } throw new Error(`Auto-restore timer generation kept changing while acquiring '${command}'`); } +/** + * Serialize a mutation and bind its lock owner to the exact active restore + * timer generation. If a timer is replaced while this operation waits for the + * lock, release without mutating and retry with the new token. This prevents a + * command that observed timer A from becoming non-preemptible inside timer B's + * mutable window. + */ +export function withTimerBoundShieldsMutationLock( + sandboxName: string, + command: string, + fn: () => T, + deps: TimerBoundLockDeps = defaultDeps, +): T { + return withTimerBoundShieldsMutationLockOptions(sandboxName, command, fn, {}, deps); +} + +/** + * Auto-restore uses a stronger stale-owner protocol than ordinary commands. + * A stale transition owner is preserved so the recovery coordinator can + * publish durable containment instead of deleting a generation whose + * descendants cannot be ruled out. + */ +export function withTimerBoundAutoRestoreLock( + sandboxName: string, + command: string, + fn: () => T, +): T { + return withTimerBoundShieldsMutationLockOptions( + sandboxName, + command, + fn, + { recoverStaleOwner: false, waitTimeoutMs: 0 }, + defaultDeps, + ); +} + export async function withTimerBoundShieldsMutationLockAsync( sandboxName: string, command: string, diff --git a/src/lib/shields/timer-control.ts b/src/lib/shields/timer-control.ts index 727a89b7347..9055eab296a 100644 --- a/src/lib/shields/timer-control.ts +++ b/src/lib/shields/timer-control.ts @@ -3,11 +3,14 @@ import { execFileSync } from "node:child_process"; import fs from "node:fs"; -import path from "node:path"; import { performance } from "node:perf_hooks"; -import { isObjectRecord } from "../core/json-types"; -import { resolveNemoclawStateDir } from "../state/paths"; +import { + readShieldsTimerMarker, + readShieldsTimerTakeoverToken, + type ShieldsTimerMarker, + shieldsTimerMarkerPath, +} from "../state/mcp-lifecycle-lock/shields-timer-authority"; const DEFAULT_PROCESS_INSPECTION_TIMEOUT_MS = 5_000; @@ -28,68 +31,16 @@ function processInspectionDeadlineReached(deadline: number): boolean { return performance.now() >= deadline; } -interface TimerMarker { - pid: number; - sandboxName: string; - snapshotPath: string; - restoreAt: string; - processToken?: string; - allowLegacyHermesProtocol?: boolean; - leaseOwnerPid?: number; - leaseOwnerStartIdentity?: string; -} - -function isTimerMarker(value: unknown): value is TimerMarker { - if (!isObjectRecord(value)) return false; - const pid = value.pid; - return ( - typeof pid === "number" && - Number.isInteger(pid) && - pid > 0 && - typeof value.sandboxName === "string" && - typeof value.snapshotPath === "string" && - typeof value.restoreAt === "string" && - (value.processToken === undefined || typeof value.processToken === "string") && - (value.allowLegacyHermesProtocol === undefined || - typeof value.allowLegacyHermesProtocol === "boolean") && - (value.leaseOwnerPid === undefined || - (typeof value.leaseOwnerPid === "number" && - Number.isInteger(value.leaseOwnerPid) && - value.leaseOwnerPid > 0)) && - (value.leaseOwnerStartIdentity === undefined || - typeof value.leaseOwnerStartIdentity === "string") && - ((value.leaseOwnerPid === undefined && value.leaseOwnerStartIdentity === undefined) || - (typeof value.leaseOwnerPid === "number" && - typeof value.leaseOwnerStartIdentity === "string" && - value.leaseOwnerStartIdentity.length > 0)) - ); -} - function timerMarkerPath(sandboxName: string): string { - return path.join(resolveNemoclawStateDir(), `shields-timer-${sandboxName}.json`); + return shieldsTimerMarkerPath(sandboxName); } -function readTimerMarker(sandboxName: string): TimerMarker | null { - const p = timerMarkerPath(sandboxName); - if (!fs.existsSync(p)) return null; - try { - const parsed = JSON.parse(fs.readFileSync(p, "utf-8")); - return isTimerMarker(parsed) ? parsed : null; - } catch { - return null; - } +function readTimerMarker(sandboxName: string): ShieldsTimerMarker | null { + return readShieldsTimerMarker(sandboxName); } function readAutoRestoreTakeoverToken(sandboxName: string): string | undefined { - const marker = readTimerMarker(sandboxName); - if ( - marker?.sandboxName !== sandboxName || - typeof marker.processToken !== "string" || - !/^[0-9a-f]{32}$/.test(marker.processToken) - ) { - return undefined; - } - return marker.processToken; + return readShieldsTimerTakeoverToken(sandboxName); } interface ClearTimerMarkerResult { @@ -193,66 +144,6 @@ function readProcessStartIdentity( } } -interface ProcessIdentity { - pid: number; - startIdentity: string; - depth: number; -} - -function listDescendantProcessIdentities( - rootPid: number, - deadline = processInspectionDeadline(), -): ProcessIdentity[] | null { - if (!Number.isInteger(rootPid) || rootPid <= 0) return null; - let rows: Array<{ pid: number; ppid: number }> = []; - try { - const timeout = remainingProcessInspectionTimeout(deadline); - if (timeout === null) return null; - rows = execFileSync("ps", ["-e", "-o", "pid=,ppid="], { - stdio: ["ignore", "pipe", "ignore"], - timeout, - }) - .toString() - .split("\n") - .map((line) => line.trim().split(/\s+/)) - .filter((parts) => parts.length >= 2) - .map(([pid, ppid]) => ({ pid: Number(pid), ppid: Number(ppid) })) - .filter((row) => Number.isInteger(row.pid) && Number.isInteger(row.ppid)); - } catch { - return null; - } - - const descendants: Array<{ pid: number; depth: number }> = []; - let frontier = [{ pid: rootPid, depth: 0 }]; - const seen = new Set([rootPid]); - while (frontier.length > 0) { - const next: Array<{ pid: number; depth: number }> = []; - for (const parent of frontier) { - for (const row of rows) { - if (row.ppid !== parent.pid || seen.has(row.pid)) continue; - seen.add(row.pid); - const child = { pid: row.pid, depth: parent.depth + 1 }; - descendants.push(child); - next.push(child); - } - } - frontier = next; - } - - const identities: ProcessIdentity[] = []; - for (const { pid, depth } of descendants) { - const startIdentity = readProcessStartIdentity(pid, deadline); - if (startIdentity) { - identities.push({ pid, startIdentity, depth }); - } else if (isProcessAlive(pid, deadline)) { - // A live descendant that cannot be identity-pinned must not be signaled; - // callers fail closed instead of risking PID-reuse collateral damage. - return null; - } - } - return identities.sort((a, b) => b.depth - a.depth); -} - function readProcessCommandLine( pid: number, deadline = processInspectionDeadline(), @@ -284,7 +175,10 @@ function readProcessCommandLine( } } -function verifyTimerMarkerIdentity(marker: TimerMarker): { verified: boolean; warning?: string } { +function verifyTimerMarkerIdentity(marker: ShieldsTimerMarker): { + verified: boolean; + warning?: string; +} { const commandLine = readProcessCommandLine(marker.pid); if (!commandLine) { return { @@ -325,7 +219,6 @@ interface KillTimerResult { function killTimer(sandboxName: string): KillTimerResult { const marker = readTimerMarker(sandboxName); let wasAlive = false; - let terminated = false; const warnings: string[] = []; if (marker) { @@ -336,22 +229,15 @@ function killTimer(sandboxName: string): KillTimerResult { if (verification.warning) { warnings.push(verification.warning); } - } else { - try { - process.kill(marker.pid, "SIGTERM"); - terminated = true; - } catch (error) { - const errno = error as NodeJS.ErrnoException; - if (errno.code !== "ESRCH") { - warnings.push( - `Failed to terminate shields timer PID ${String(marker.pid)} for sandbox '${sandboxName}': ${errno.message}`, - ); - } - } } } } + // Marker removal is cooperative cancellation and revokes the timer's exact + // recovery generation. Do not signal a verified live timer: it may own the + // lifecycle deadline fence, and an unhandled signal could bypass its finally + // cleanup and strand the fence. Recovery loops re-check marker authority and + // unwind their locks after this revocation. const markerClear = clearTimerMarker(sandboxName); if (markerClear.warning) { warnings.push(markerClear.warning); @@ -361,17 +247,16 @@ function killTimer(sandboxName: string): KillTimerResult { markerFound: marker !== null, markerPid: marker?.pid ?? null, wasAlive, - terminated, + terminated: false, warnings, }; } -export type { ClearTimerMarkerResult, KillTimerResult, ProcessIdentity, TimerMarker }; +export type { ClearTimerMarkerResult, KillTimerResult, ShieldsTimerMarker as TimerMarker }; export { clearTimerMarker, isProcessAlive, killTimer, - listDescendantProcessIdentities, processInspectionDeadlineAfter, processInspectionDeadlineReached, readAutoRestoreTakeoverToken, diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index bdad08cf883..251dca46f78 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -6,9 +6,10 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { getMcpLifecycleLockPath } from "../state/mcp-lifecycle-lock"; +import { getMcpLifecycleLockPath, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ + completeAutoRestoreTransition: vi.fn(() => true), lockAgentConfig: vi.fn() as unknown, prepareAutoRestoreTransitionTakeover: vi.fn(), })); @@ -43,6 +44,7 @@ vi.mock("../sandbox/agent-config", () => ({ })); vi.mock("./index", () => ({ + completeAutoRestoreTransition: shieldsIndexMock.completeAutoRestoreTransition, get lockAgentConfig() { return shieldsIndexMock.lockAgentConfig; }, @@ -56,6 +58,7 @@ describe("shields timer authorization", () => { tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-timer-")); vi.stubEnv("HOME", tmpHome); shieldsIndexMock.lockAgentConfig = vi.fn(); + runMock.mockImplementation(() => ({ status: 0 })); vi.resetModules(); vi.clearAllMocks(); }); @@ -66,7 +69,7 @@ describe("shields timer authorization", () => { }); async function invokeTimerAndCaptureExit( - runRestoreTimer: (args: any) => Promise, + runRestoreTimer: (args: any, options?: { retryDelayMs?: number }) => Promise, args: unknown, ): Promise { const exitSpy = vi.spyOn(process, "exit").mockImplementation((code?: any) => { @@ -74,7 +77,7 @@ describe("shields timer authorization", () => { }); try { - await runRestoreTimer(args); + await runRestoreTimer(args, { retryDelayMs: 1 }); throw new Error("Expected runRestoreTimer to exit"); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -86,21 +89,45 @@ describe("shields timer authorization", () => { } } + async function waitForRetryBoundary(deadlinePath: string, auditPath: string): Promise { + await vi.waitFor( + () => { + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(auditPath)).toBe(true); + }, + { interval: 1, timeout: 200 }, + ); + } + async function invokeTimerAndExpectRetry( - runRestoreTimer: (args: any) => Promise, + runRestoreTimer: (args: any, options?: { retryDelayMs?: number }) => Promise, args: unknown, ): Promise { - vi.useFakeTimers(); const exitSpy = vi .spyOn(process, "exit") .mockImplementation((() => undefined) as typeof process.exit); + const { markerPath, sandboxName } = args as { + markerPath: string; + sandboxName: string; + }; + const markerContents = fs.readFileSync(markerPath); + const deadlinePath = `${getMcpLifecycleLockPath( + sandboxName, + path.join(tmpHome, ".nemoclaw", "state"), + )}.deadline`; + const auditPath = path.join(path.dirname(markerPath), "shields-audit.jsonl"); try { - await runRestoreTimer(args); + const pending = runRestoreTimer(args, { retryDelayMs: 50 }); + await waitForRetryBoundary(deadlinePath, auditPath); expect(exitSpy).not.toHaveBeenCalled(); - expect(vi.getTimerCount()).toBe(1); + expect(fs.existsSync(deadlinePath)).toBe(true); + const policyApplicationsBeforeRevocation = runMock.mock.calls.length; + fs.rmSync(markerPath, { force: true }); + await pending; + expect(runMock).toHaveBeenCalledTimes(policyApplicationsBeforeRevocation); } finally { + fs.writeFileSync(markerPath, markerContents); exitSpy.mockRestore(); - vi.useRealTimers(); } } @@ -171,6 +198,52 @@ describe("shields timer authorization", () => { expect(fs.existsSync(markerPath)).toBe(true); }); + it.skipIf(process.platform === "win32")( + "does not restore or rewrite state through a symlinked timer marker", + async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + + const sandboxName = "alpha"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); + const stateFile = path.join(stateDir, `shields-${sandboxName}.json`); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const markerTargetPath = path.join(stateDir, "operator-owned-marker.json"); + const initialState = { shieldsDown: true, updatedAt: "2026-01-01T00:00:00.000Z" }; + const markerTarget = JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + }); + + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync(stateFile, JSON.stringify(initialState, null, 2)); + fs.writeFileSync(markerTargetPath, markerTarget); + fs.symlinkSync(markerTargetPath, markerPath); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + ]); + expect(args).not.toBeNull(); + + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + + expect(exitCode).toBe(0); + expect(runMock).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); + expect(fs.lstatSync(markerPath).isSymbolicLink()).toBe(true); + expect(fs.readFileSync(markerTargetPath, "utf-8")).toBe(markerTarget); + }, + ); + it("binds rebuild-only legacy authorization to both argv and the root-owned marker", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); @@ -273,61 +346,73 @@ describe("shields timer authorization", () => { } }); - it("retains a dead rebuild owner's timer and retries a transient restore failure", async () => { - vi.useFakeTimers(); - const exitSpy = vi - .spyOn(process, "exit") - .mockImplementation((() => undefined) as typeof process.exit); - try { - const timer = await import("./timer"); - const stateDir = path.join(tmpHome, ".nemoclaw", "state"); - fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "rebuild-dead"; - const snapshotPath = path.join(stateDir, "snapshot.yaml"); - const restoreAtIso = new Date().toISOString(); - const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); - const sandboxMutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync( - markerPath, - JSON.stringify({ - pid: process.pid, - sandboxName, - snapshotPath, - restoreAt: restoreAtIso, - processToken: PROCESS_TOKEN, - leaseOwnerPid: 2_147_483_000, - leaseOwnerStartIdentity: "proc:dead-owner", - }), - ); - runMock.mockImplementationOnce(() => { - expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); - return { status: 17 }; - }); - const args = timer.parseTimerArgs([ + it("audits a successful restore retry while retaining deadline ownership", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const sandboxName = "rebuild-dead"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const sandboxMutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const deadlinePath = `${sandboxMutationLockPath}.deadline`; + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, sandboxName, snapshotPath, - restoreAtIso, - "", - "", - PROCESS_TOKEN, - "0", - "2147483000", - "proc:dead-owner", - ]); - expect(args).not.toBeNull(); + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + leaseOwnerPid: 2_147_483_000, + leaseOwnerStartIdentity: "proc:dead-owner", + }), + ); + runMock.mockImplementationOnce(() => { + expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); + expect(fs.existsSync(deadlinePath)).toBe(true); + return { status: 17 }; + }); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + "0", + "2147483000", + "proc:dead-owner", + ]); + expect(args).not.toBeNull(); - await timer.runRestoreTimer(args!); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); - expect(runMock).toHaveBeenCalledTimes(1); - expect(exitSpy).not.toHaveBeenCalled(); - expect(vi.getTimerCount()).toBe(1); - expect(fs.existsSync(markerPath)).toBe(true); - expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); - } finally { - exitSpy.mockRestore(); - vi.useRealTimers(); - } + expect(exitCode).toBe(0); + expect(runMock).toHaveBeenCalledTimes(2); + expect(shieldsIndexMock.completeAutoRestoreTransition).toHaveBeenCalledWith( + sandboxName, + PROCESS_TOKEN, + snapshotPath, + ); + expect(fs.existsSync(markerPath)).toBe(false); + expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + const audits = fs + .readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + expect(audits).toContainEqual( + expect.objectContaining({ + action: "shields_up_failed", + error: "Policy restore exited with status 17", + }), + ); + expect(audits).toContainEqual( + expect.objectContaining({ action: "shields_auto_restore", sandbox: sandboxName }), + ); }); it("does not restore or rewrite state when marker pid mismatches", async () => { @@ -419,6 +504,59 @@ describe("shields timer authorization", () => { expect(JSON.parse(fs.readFileSync(markerPath, "utf-8"))).toEqual(replacementMarker); }); + it("does not preempt a transition owner after timer authority is revoked", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const sandboxName = "revoked-takeover"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const mutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const deadlinePath = `${mutationLockPath}.deadline`; + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + }), + ); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + ]); + expect(args).not.toBeNull(); + shieldsIndexMock.prepareAutoRestoreTransitionTakeover.mockImplementationOnce( + ( + _sandboxName: string, + _processToken: string, + _snapshotPath: string, + assertTakeoverAuthority: () => void, + ) => { + expect(fs.existsSync(deadlinePath)).toBe(true); + fs.rmSync(markerPath); + assertTakeoverAuthority(); + }, + ); + + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + + expect(exitCode).toBe(1); + expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.completeAutoRestoreTransition).not.toHaveBeenCalled(); + expect(fs.existsSync(mutationLockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + expect(fs.existsSync(`${mutationLockPath}.containment`)).toBe(false); + }); + it("restores and updates state when marker matches current timer invocation", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); @@ -453,8 +591,10 @@ describe("shields timer authorization", () => { expect(args).not.toBeNull(); const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); + const deadlinePath = `${sandboxMutationLockPath}.deadline`; runMock.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); + expect(fs.existsSync(deadlinePath)).toBe(true); expect(JSON.parse(fs.readFileSync(lockPath, "utf-8"))).toMatchObject({ sandboxName, command: "shields auto-restore", @@ -473,6 +613,79 @@ describe("shields timer authorization", () => { expect(updatedState.shieldsDownAt).toBeNull(); expect(fs.existsSync(markerPath)).toBe(false); expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + expect(shieldsIndexMock.completeAutoRestoreTransition).toHaveBeenCalledWith( + sandboxName, + PROCESS_TOKEN, + snapshotPath, + ); + }); + + it("keeps the deadline gate closed while a failed restore retries", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const sandboxName = "retry-gate"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const mutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + }), + ); + runMock.mockReturnValueOnce({ status: 1 }).mockReturnValue({ status: 0 }); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + ]); + expect(args).not.toBeNull(); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + let contenderEntered = false; + + try { + const restore = timer.runRestoreTimer(args!, { retryDelayMs: 100 }); + await vi.waitFor(() => expect(runMock).toHaveBeenCalledTimes(1), { + interval: 1, + timeout: 200, + }); + expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); + + const contender = withMcpLifecycleLock( + sandboxName, + () => { + contenderEntered = true; + }, + { stateDir, pollIntervalMs: 5, timeoutMs: 2_000 }, + ); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(contenderEntered).toBe(false); + expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); + + await Promise.all([restore, contender]); + expect(runMock).toHaveBeenCalledTimes(2); + expect(contenderEntered).toBe(true); + expect(shieldsIndexMock.completeAutoRestoreTransition).toHaveBeenCalledWith( + sandboxName, + PROCESS_TOKEN, + snapshotPath, + ); + } finally { + exitSpy.mockRestore(); + } }); it("retains recovery authority when the locked-state commit cannot be persisted", async () => { diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index 1b3ff66938f..2737d62f17f 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -14,7 +14,11 @@ import { isObjectRecord, type UnknownRecord } from "../core/json-types"; import { buildPolicySetCommand } from "../policy"; import { run } from "../runner"; import { resolveAgentConfig } from "../sandbox/agent-config"; -import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; +import { withMcpLifecycleDeadlineFence } from "../state/mcp-lifecycle-lock"; +import { + readShieldsTimerMarkerFile, + type ShieldsTimerMarker, +} from "../state/mcp-lifecycle-lock/shields-timer-authority"; import { resolveNemoclawStateDir } from "../state/paths"; import { appendAuditEntry, type ShieldsAuditEntry } from "./audit"; import * as shields from "./index"; @@ -48,7 +52,12 @@ interface TimerArgs { leaseOwnerStartIdentity?: string; } +interface TimerRuntimeOptions { + retryDelayMs?: number; +} + type LockAgentConfig = typeof shields.lockAgentConfig; +type RestoreAttemptOutcome = "complete" | "retry" | "revoked"; const STATE_DIR = resolveNemoclawStateDir(); const AUTO_RESTORE_RETRY_MS = 5_000; @@ -150,19 +159,14 @@ function updateState(stateFile: string, patch: ShieldsStatePatch): void { } } -function readTimerMarker(markerPath: string): UnknownRecord | null { - try { - if (!fs.existsSync(markerPath)) { - return null; - } - const parsed = JSON.parse(fs.readFileSync(markerPath, "utf-8")); - return isObjectRecord(parsed) ? parsed : null; - } catch { - return null; - } +function readTimerMarker(markerPath: string): ShieldsTimerMarker | null { + return readShieldsTimerMarkerFile(markerPath); } -function markerRecordMatchesCurrentTimer(marker: UnknownRecord | null, args: TimerArgs): boolean { +function markerRecordMatchesCurrentTimer( + marker: ShieldsTimerMarker | null, + args: TimerArgs, +): boolean { if (!marker) return false; return ( marker.pid === process.pid && @@ -227,16 +231,22 @@ function rebuildLeaseOwnerIsCurrent(args: TimerArgs): boolean { ); } -async function runRestoreTimer(args: TimerArgs): Promise { - const now = new Date().toISOString(); +async function runRestoreTimer( + args: TimerArgs, + runtimeOptions: TimerRuntimeOptions = {}, +): Promise { + const retryDelayMs = + Number.isFinite(runtimeOptions.retryDelayMs) && (runtimeOptions.retryDelayMs ?? 0) >= 0 + ? Math.floor(runtimeOptions.retryDelayMs!) + : AUTO_RESTORE_RETRY_MS; let exitCode = 0; let retryScheduled = false; const scheduleRetry = (): boolean => { if (!markerMatchesCurrentTimer(args)) return false; retryScheduled = true; setTimeout(() => { - void runRestoreTimer(args); - }, AUTO_RESTORE_RETRY_MS); + void runRestoreTimer(args, runtimeOptions); + }, retryDelayMs); return true; }; @@ -261,13 +271,12 @@ async function runRestoreTimer(args: TimerArgs): Promise { if (!args.processToken || !/^[0-9a-f]{32}$/.test(args.processToken)) { throw new Error("Auto-restore timer has no valid transition takeover token"); } - shields.prepareAutoRestoreTransitionTakeover( - args.sandboxName, - args.processToken, - args.snapshotPath, - ); - - await withSandboxMutationLock(args.sandboxName, () => + const assertTakeoverAuthority = (): void => { + if (!markerMatchesCurrentTimer(args)) { + throw new Error("Auto-restore authority changed before Shields transition takeover"); + } + }; + const restoreUnderDeadlineFence = (): RestoreAttemptOutcome => withShieldsTransitionLock( args.sandboxName, "shields auto-restore", @@ -275,19 +284,18 @@ async function runRestoreTimer(args: TimerArgs): Promise { // A manual hardening command may have completed while this timer waited // for the host mutation lock. The marker is the timer's authority, so // re-check it only after serialization is established. - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; if (!fs.existsSync(args.snapshotPath)) { appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: "Policy snapshot file missing", }); exitCode = 1; - scheduleRetry(); - return; + return "retry"; } // Restore policy (slow — openshell policy set --wait blocks) @@ -300,19 +308,18 @@ async function runRestoreTimer(args: TimerArgs): Promise { appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: `Policy restore exited with status ${String(status)}`, }); exitCode = 1; - scheduleRetry(); - return; + return "retry"; } // Destroy and force-restore can revoke this marker while a slow // policy restore is already in flight. Stop before the next sandbox // mutation if this timer generation no longer owns recovery. - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; // Re-lock config file using the shared lockAgentConfig from shields.ts. // lockAgentConfig runs each operation independently and verifies the @@ -354,7 +361,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { appendAudit({ action: "shields_auto_restore_lock_warning", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", warning: "Missing config directory for auto-restore re-lock verification", lock_verified: false, @@ -363,7 +370,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { } if (lockTarget) { try { - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; const lockAgentConfig = resolveLockAgentConfig(); // #4663: a single instantaneous lock+verify cannot prove an // in-sandbox reconciler didn't re-permission .config-hash after the @@ -387,7 +394,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { appendAudit({ action: "shields_auto_restore_lock_warning", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", warning: relock.error ?? "Config re-lock did not re-confirm after settle window", @@ -399,7 +406,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { appendAudit({ action: "shields_auto_restore_lock_warning", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", warning: error instanceof Error ? error.message : String(error), lock_verified: false, @@ -410,7 +417,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { // Re-lock verification includes a settle window. Do not rewrite state // or remove a replacement marker if authority changed while it ran. - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; // Only mark shields as UP if the lock was verified (or no config path). if (lockVerified) { @@ -424,17 +431,27 @@ async function runRestoreTimer(args: TimerArgs): Promise { if (lockedChattr !== null) patch.chattrApplied = lockedChattr; if (lockedHashes !== null) patch.fileHashes = lockedHashes; updateState(args.stateFile, patch); + if ( + !shields.completeAutoRestoreTransition( + args.sandboxName, + args.processToken!, + args.snapshotPath, + ) + ) { + return "revoked"; + } appendAudit({ action: "shields_auto_restore", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", policy_snapshot: args.snapshotPath, scheduled_restore_at: args.restoreAtIso, }); cleanupOwnedTimerMarker(args); - return; + exitCode = 0; + return "complete"; } // Explicitly ensure state reflects shields are still DOWN. @@ -444,21 +461,73 @@ async function runRestoreTimer(args: TimerArgs): Promise { appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: "Config re-lock verification failed — shields remain DOWN", }); exitCode = 1; - scheduleRetry(); + return "retry"; + }, + { + takeoverToken: args.processToken, + recoverStaleOwner: false, + waitTimeoutMs: 0, + }, + ); + const restoreWhileDeadlineOwned = async (): Promise => { + for (;;) { + let outcome: RestoreAttemptOutcome; + try { + shields.prepareAutoRestoreTransitionTakeover( + args.sandboxName, + args.processToken!, + args.snapshotPath, + assertTakeoverAuthority, + ); + outcome = restoreUnderDeadlineFence(); + } catch (error) { + appendAudit({ + action: "shields_up_failed", + sandbox: args.sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: args.snapshotPath, + error: error instanceof Error ? error.message : String(error), + }); + exitCode = 1; + outcome = "retry"; + } + if (outcome !== "retry") return; + if (!markerMatchesCurrentTimer(args)) return; + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + if (!markerMatchesCurrentTimer(args)) return; + } + }; + await withMcpLifecycleDeadlineFence( + args.sandboxName, + args.processToken, + restoreWhileDeadlineOwned, + { + stateDir: STATE_DIR, + pollIntervalMs: 50, + timeoutMs: 5_000, + onContainment: ({ ownerPid, reason }) => { + appendAudit({ + action: "shields_up_failed", + sandbox: args.sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: args.snapshotPath, + error: `${reason}${ownerPid ? ` Contained owner PID: ${String(ownerPid)}.` : ""}`, + }); }, - { takeoverToken: args.processToken }, - ), + }, ); } catch (error: unknown) { appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: error instanceof Error ? error.message : String(error), }); diff --git a/src/lib/shields/transition-lock.test.ts b/src/lib/shields/transition-lock.test.ts index 5d966815b98..1fbf625584f 100644 --- a/src/lib/shields/transition-lock.test.ts +++ b/src/lib/shields/transition-lock.test.ts @@ -289,6 +289,11 @@ describe("host shields transition lock", () => { processStartIdentity: "proc:holder", command: "shields down", }); + expect(locker.inspectAnyShieldsTransitionLockOwner("alpha")).toEqual({ + pid: 202, + processStartIdentity: "proc:holder", + command: "shields down", + }); }); it("returns no inspected owner when the canonical path changes identity before open", () => { @@ -491,6 +496,39 @@ describe("host shields transition lock", () => { expect(fs.readdirSync(stateDir)).toEqual([]); }); + it("preserves a stale owner when the containment protocol owns recovery", () => { + const recorded = owner("alpha", 202, "proc:dead-holder", "shields down", TAKEOVER_TOKEN); + const lockPath = writeOwner("alpha", recorded); + + expect(() => + manager().withShieldsTransitionLock("alpha", "timer restore", () => undefined, { + takeoverToken: TAKEOVER_TOKEN, + recoverStaleOwner: false, + waitTimeoutMs: 0, + }), + ).toThrow(/recorded owner PID 202 is not running/); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toEqual(recorded); + }); + + it("preserves a reused-PID owner when the containment protocol owns recovery", () => { + const holderPid = 202; + const recorded = owner("alpha", holderPid, "proc:original"); + const lockPath = writeOwner("alpha", recorded); + const locker = manager({ + isProcessAlive: (pid) => pid === holderPid || pid === SELF_PID, + readProcessStartIdentity: (pid) => + pid === SELF_PID ? SELF_IDENTITY : pid === holderPid ? "proc:reused" : null, + }); + + expect(() => + locker.withShieldsTransitionLock("alpha", "gateway restart", () => undefined, { + recoverStaleOwner: false, + waitTimeoutMs: 0, + }), + ).toThrow(/PID 202 now has process-start identity/); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toEqual(recorded); + }); + it("recovers a stale lock when a live PID has been reused", () => { const holderPid = 202; const recorded = owner("alpha", holderPid, "proc:original"); @@ -502,16 +540,21 @@ describe("host shields transition lock", () => { }); expect( - locker.withShieldsTransitionLock("alpha", "timer restore", () => { - const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); - expect(replacement).toMatchObject({ - sandboxName: "alpha", - pid: SELF_PID, - processStartIdentity: SELF_IDENTITY, - command: "timer restore", - }); - return "acquired"; - }), + locker.withShieldsTransitionLock( + "alpha", + "timer restore", + () => { + const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); + expect(replacement).toMatchObject({ + sandboxName: "alpha", + pid: SELF_PID, + processStartIdentity: SELF_IDENTITY, + command: "timer restore", + }); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); }); @@ -535,6 +578,7 @@ describe("host shields transition lock", () => { expect(() => locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => undefined, { + recoverStaleOwner: true, waitTimeoutMs: 2, pollIntervalMs: 1, }), @@ -566,6 +610,7 @@ describe("host shields transition lock", () => { throw new Error("should not acquire after timeout"); }, { + recoverStaleOwner: true, waitTimeoutMs: 2, pollIntervalMs: 1, }, @@ -608,16 +653,21 @@ describe("host shields transition lock", () => { }); await expect( - locker.withShieldsTransitionLockAsync("alpha", "timer restore", async () => { - const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); - expect(replacement).toMatchObject({ - sandboxName: "alpha", - pid: SELF_PID, - processStartIdentity: SELF_IDENTITY, - command: "timer restore", - }); - return "acquired"; - }), + locker.withShieldsTransitionLockAsync( + "alpha", + "timer restore", + async () => { + const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); + expect(replacement).toMatchObject({ + sandboxName: "alpha", + pid: SELF_PID, + processStartIdentity: SELF_IDENTITY, + command: "timer restore", + }); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).resolves.toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); }); @@ -677,6 +727,7 @@ describe("host shields transition lock", () => { expect(() => locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => undefined, { + recoverStaleOwner: true, waitTimeoutMs: 2, pollIntervalMs: 1, }), @@ -728,7 +779,9 @@ describe("host shields transition lock", () => { const locker = manager(); expect( - locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => "acquired"), + locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => "acquired", { + recoverStaleOwner: true, + }), ).toBe("acquired"); expect(thirdAcquired).toBe(false); @@ -776,6 +829,7 @@ describe("host shields transition lock", () => { "alpha", "nemoclaw alpha shields up", async () => "acquired", + { recoverStaleOwner: true }, ), ).resolves.toBe("acquired"); @@ -793,11 +847,16 @@ describe("host shields transition lock", () => { const locker = manager(); expect( - locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => { - expect(fs.existsSync(lockPath)).toBe(true); - expect(fs.existsSync(guardPath)).toBe(false); - return "acquired"; - }), + locker.withShieldsTransitionLock( + "alpha", + "nemoclaw alpha shields up", + () => { + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(guardPath)).toBe(false); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); @@ -813,11 +872,16 @@ describe("host shields transition lock", () => { const locker = manager(); await expect( - locker.withShieldsTransitionLockAsync("alpha", "nemoclaw alpha shields up", async () => { - expect(fs.existsSync(lockPath)).toBe(true); - expect(fs.existsSync(guardPath)).toBe(false); - return "acquired"; - }), + locker.withShieldsTransitionLockAsync( + "alpha", + "nemoclaw alpha shields up", + async () => { + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(guardPath)).toBe(false); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).resolves.toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); diff --git a/src/lib/shields/transition-lock.ts b/src/lib/shields/transition-lock.ts index b31eb7a635c..4575f87e18e 100644 --- a/src/lib/shields/transition-lock.ts +++ b/src/lib/shields/transition-lock.ts @@ -35,6 +35,8 @@ export interface ShieldsTransitionLockOptions { pollIntervalMs?: number; malformedStaleMs?: number; takeoverToken?: string; + /** Preserve stale owners for a caller that applies a stronger containment protocol. */ + recoverStaleOwner?: boolean; } export interface ShieldsTransitionLockDependencies { @@ -488,6 +490,26 @@ export class ShieldsTransitionLockManager { } } + inspectAnyShieldsTransitionLockOwner( + sandboxName: string, + ): InspectedShieldsTransitionOwner | null { + const validName = validateSandboxName(sandboxName); + const lockPath = shieldsTransitionLockPath(validName, this.stateDir); + const snapshot = readExistingLock(lockPath, validName); + if (!snapshot) return null; + try { + const owner = snapshot.owner; + if (!owner) return null; + return { + pid: owner.pid, + processStartIdentity: owner.processStartIdentity, + command: owner.command, + }; + } finally { + closeSnapshot(snapshot); + } + } + takeoverShieldsTransitionLock( sandboxName: string, expectedOwnerPid: number, @@ -816,7 +838,12 @@ export class ShieldsTransitionLockManager { ); if (!observed) continue; lastWaitReason = observed; - if (this.recoveredObservedStaleOwner(sandboxName, observed)) continue; + if ( + options.recoverStaleOwner !== false && + this.recoveredObservedStaleOwner(sandboxName, observed) + ) { + continue; + } } this.sleep(this.waitDuration(state, lastWaitReason)); } @@ -847,7 +874,12 @@ export class ShieldsTransitionLockManager { ); if (!observed) continue; lastWaitReason = observed; - if (this.recoveredObservedStaleOwner(sandboxName, observed)) continue; + if ( + options.recoverStaleOwner !== false && + this.recoveredObservedStaleOwner(sandboxName, observed) + ) { + continue; + } } await this.sleepAsync(this.waitDuration(state, lastWaitReason)); } @@ -1140,6 +1172,12 @@ export function inspectShieldsTransitionLockOwner( return defaultManager.inspectShieldsTransitionLockOwner(sandboxName, takeoverToken); } +export function inspectAnyShieldsTransitionLockOwner( + sandboxName: string, +): InspectedShieldsTransitionOwner | null { + return defaultManager.inspectAnyShieldsTransitionLockOwner(sandboxName); +} + export function takeoverShieldsTransitionLock( sandboxName: string, expectedOwnerPid: number, diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts new file mode 100644 index 00000000000..71b03bb046f --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts @@ -0,0 +1,759 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + beginCommittedMcpLifecycleContainmentSync, + durableMcpLifecycleContainmentFailure, + isMcpLifecycleLockHeld, + withMcpLifecycleDeadlineFence, + withMcpLifecycleDeadlineFenceSync, + withMcpLifecycleLock, + withMcpLifecycleLockSync, +} from "./mcp-lifecycle-lock-acquisition"; +import { + createMcpLifecycleLockOwner, + readMcpLockHostIdentity, + readMcpLockPidNamespaceIdentity, +} from "./mcp-lifecycle-lock-identity"; +import { getMcpLifecycleLockPath } from "./mcp-lifecycle-lock-storage"; + +const SANDBOX_NAME = "alpha"; +let stateDir: string; + +function options() { + return { + stateDir, + pollIntervalMs: 1, + timeoutMs: 20, + corruptLockGraceMs: 1, + }; +} + +function writeTimerMarker( + processToken: string | undefined, + restoreAt = new Date(Date.now() + 60_000).toISOString(), + pid = process.pid, +): void { + fs.writeFileSync( + path.join(stateDir, `shields-timer-${SANDBOX_NAME}.json`), + JSON.stringify({ + pid, + sandboxName: SANDBOX_NAME, + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt, + ...(processToken ? { processToken } : {}), + }), + ); +} + +function writeStaleMainOwner(shieldsTakeoverToken?: string): string { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: SANDBOX_NAME, + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: readMcpLockHostIdentity(), + pidNamespaceIdentity: readMcpLockPidNamespaceIdentity(), + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), + token: "stale-main-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + return lockPath; +} + +function publishTimerWhenStaleOwnerIsObserved(processToken: string): ReturnType { + const realProcessKill = process.kill.bind(process); + const processKill = vi.fn((pid: number, signal?: string | number) => { + if (pid === 2_147_483_647 && signal === 0) { + writeTimerMarker(processToken); + const error = new Error("stale owner exited") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + } + return realProcessKill(pid, signal as never); + }); + vi.spyOn(process, "kill").mockImplementation(processKill); + return processKill; +} + +beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-lock-acquisition-")); +}); + +afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(stateDir, { recursive: true, force: true }); +}); + +describe("MCP lifecycle lock acquisition", () => { + it("keeps asynchronous ordinary acquisition closed after an unpublished timer deadline", async () => { + const operation = vi.fn(() => "must not enter"); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker("1".repeat(32), new Date(Date.now() - 1_000).toISOString()); + + await expect(withMcpLifecycleLock(SANDBOX_NAME, operation, options())).rejects.toThrow( + "Timed out waiting for the sandbox mutation lock", + ); + + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + }); + + it("keeps synchronous ordinary acquisition closed after a dead timer misses its deadline", () => { + const operation = vi.fn(() => "must not enter"); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker("2".repeat(32), new Date(Date.now() - 1_000).toISOString(), 2_147_483_647); + + expect(() => withMcpLifecycleLockSync(SANDBOX_NAME, operation, options())).toThrow( + "Timed out waiting for sandbox mutation lock", + ); + + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + }); + + it("does not strand asynchronous recovery behind an expired legacy marker", async () => { + writeTimerMarker(undefined, new Date(Date.now() - 1_000).toISOString()); + + await expect(withMcpLifecycleLock(SANDBOX_NAME, () => "entered", options())).resolves.toBe( + "entered", + ); + }); + + it("does not strand synchronous recovery behind an expired legacy short-token marker", () => { + writeTimerMarker("legacy-token", new Date(Date.now() - 1_000).toISOString()); + + expect(withMcpLifecycleLockSync(SANDBOX_NAME, () => "entered", options())).toBe("entered"); + }); + + it("rejects asynchronous admission when restoreAt passes after main publication", async () => { + const operation = vi.fn(() => "must not enter"); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const beforeDeadline = Date.now(); + writeTimerMarker("3".repeat(32), new Date(beforeDeadline + 500).toISOString()); + const linkSpy = vi.spyOn(fs.promises, "link"); + vi.spyOn(Date, "now") + .mockReturnValueOnce(beforeDeadline) + .mockReturnValueOnce(beforeDeadline) + .mockReturnValue(beforeDeadline + 1_000); + + await expect(withMcpLifecycleLock(SANDBOX_NAME, operation, options())).rejects.toThrow( + "Timed out waiting for the sandbox mutation lock", + ); + + expect(linkSpy).toHaveBeenCalled(); + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("releases a synchronous lock after nested work completes", () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const events: string[] = []; + + const result = withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + expect(isMcpLifecycleLockHeld(SANDBOX_NAME, stateDir)).toBe(true); + events.push("outer"); + return withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + expect(isMcpLifecycleLockHeld(SANDBOX_NAME, stateDir)).toBe(true); + events.push("nested"); + return "complete"; + }, + options(), + ); + }, + options(), + ); + + expect(result).toBe("complete"); + expect(events).toEqual(["outer", "nested"]); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("allows a nested synchronous lock during deadline recovery and releases the main lock and deadline gate afterward", () => { + const processToken = "a".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + + const result = withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + expect(isMcpLifecycleLockHeld(SANDBOX_NAME, stateDir)).toBe(true); + return withMcpLifecycleLockSync(SANDBOX_NAME, () => "restored", options()); + }, + options(), + ); + + expect(result).toBe("restored"); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + }); + + it("releases the synchronous deadline and main generations after an ordinary error", () => { + const processToken = "e".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + throw new Error("ordinary recovery failure"); + }, + options(), + ), + ).toThrow("ordinary recovery failure"); + + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + }); + + it("retains exact synchronous deadline and main generations after an uncommitted durable-containment failure", () => { + const processToken = "f".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + throw durableMcpLifecycleContainmentFailure( + new Error("containment state is read-only"), + lockPath, + ); + }, + options(), + ), + ).toThrow("containment state is read-only"); + + const mainOwner = JSON.parse(fs.readFileSync(lockPath, "utf8")); + const deadlineOwner = JSON.parse(fs.readFileSync(deadlinePath, "utf8")); + expect(mainOwner).toMatchObject({ + sandboxName: SANDBOX_NAME, + pid: process.pid, + shieldsTakeoverToken: processToken, + }); + expect(deadlineOwner).toMatchObject({ + sandboxName: SANDBOX_NAME, + pid: process.pid, + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + + it("releases owned generations when committed containment is proven present", () => { + const processToken = "1".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + beginCommittedMcpLifecycleContainmentSync( + SANDBOX_NAME, + processToken, + "test containment", + stateDir, + ); + throw durableMcpLifecycleContainmentFailure( + new Error("containment reporting stopped"), + lockPath, + ); + }, + options(), + ), + ).toThrow("containment reporting stopped"); + + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("retains owned generations when committed containment cannot be inspected", () => { + const processToken = "2".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const containmentPath = `${lockPath}.containment`; + const realLstatSync = fs.lstatSync.bind(fs); + let denyContainmentInspection = false; + const rejectContainmentInspection = (): never => { + const error = new Error("containment inspection denied") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + }; + vi.spyOn(fs, "lstatSync").mockImplementation((target, options) => { + return denyContainmentInspection && String(target) === containmentPath + ? rejectContainmentInspection() + : realLstatSync(target, options as never); + }); + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + denyContainmentInspection = true; + throw durableMcpLifecycleContainmentFailure( + new Error("containment commit could not be verified"), + lockPath, + ); + }, + options(), + ), + ).toThrow("containment commit could not be verified"); + + denyContainmentInspection = false; + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(true); + }); + + it("retains an async lifecycle generation only for an uncommitted coded failure", async () => { + const retainedLockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker("8".repeat(32)); + + await expect( + withMcpLifecycleLock( + SANDBOX_NAME, + () => { + throw durableMcpLifecycleContainmentFailure( + new Error("nested containment commit failed"), + retainedLockPath, + ); + }, + options(), + ), + ).rejects.toThrow("nested containment commit failed"); + expect(fs.existsSync(retainedLockPath)).toBe(true); + + fs.rmSync(retainedLockPath, { force: true }); + await expect( + withMcpLifecycleLock( + SANDBOX_NAME, + () => { + throw new Error("ordinary nested failure"); + }, + options(), + ), + ).rejects.toThrow("ordinary nested failure"); + expect(fs.existsSync(retainedLockPath)).toBe(false); + }); + + it("retains a synchronous lifecycle generation only for an uncommitted coded failure", () => { + const retainedLockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker("9".repeat(32)); + + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw durableMcpLifecycleContainmentFailure( + new Error("synchronous nested containment commit failed"), + retainedLockPath, + ); + }, + options(), + ), + ).toThrow("synchronous nested containment commit failed"); + expect(fs.existsSync(retainedLockPath)).toBe(true); + + fs.rmSync(retainedLockPath, { force: true }); + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw new Error("ordinary synchronous nested failure"); + }, + options(), + ), + ).toThrow("ordinary synchronous nested failure"); + expect(fs.existsSync(retainedLockPath)).toBe(false); + }); + + it("retains a timer-bound lifecycle generation when nested code handles the containment failure", () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const processToken = "4".repeat(32); + writeTimerMarker(processToken); + + const result = withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + try { + throw durableMcpLifecycleContainmentFailure( + new Error("nested containment commit failed"), + lockPath, + ); + } catch { + return "handled"; + } + }, + options(), + ); + + expect(result).toBe("handled"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toMatchObject({ + sandboxName: SANDBOX_NAME, + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + + it("retains an async timer-bound lifecycle generation when nested code handles the containment failure", async () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const processToken = "5".repeat(32); + writeTimerMarker(processToken); + + const result = await withMcpLifecycleLock( + SANDBOX_NAME, + async () => { + try { + throw durableMcpLifecycleContainmentFailure( + new Error("nested async containment commit failed"), + lockPath, + ); + } catch { + await Promise.resolve(); + return "handled"; + } + }, + options(), + ); + + expect(result).toBe("handled"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toMatchObject({ + sandboxName: SANDBOX_NAME, + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + + it("releases a non-timer-bound lifecycle generation after a coded containment failure", () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw durableMcpLifecycleContainmentFailure( + new Error("non-timer containment failure"), + lockPath, + ); + }, + options(), + ), + ).toThrow("non-timer containment failure"); + + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("contains a retained timer-bound main generation after its owner exits", () => { + const processToken = "a".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw durableMcpLifecycleContainmentFailure( + new Error("retain this timer-bound generation"), + lockPath, + ); + }, + options(), + ), + ).toThrow("retain this timer-bound generation"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toMatchObject({ + pid: process.pid, + shieldsTakeoverToken: processToken, + }); + + const realProcessKill = process.kill.bind(process); + vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { + const failRetainedOwnerProbe = () => { + const error = new Error("retained owner exited") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + }; + const retainedOwnerProbe = + pid === process.pid && signal === 0 ? failRetainedOwnerProbe : undefined; + retainedOwnerProbe?.(); + return realProcessKill(pid, signal as never); + }); + const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + + expect(() => withMcpLifecycleLockSync(SANDBOX_NAME, () => "must not enter", options())).toThrow( + "Sandbox mutation containment is active", + ); + expect(waitSpy).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("returns operator guidance after recording durable containment for a stale deadline generation", () => { + const processToken = "6".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + const operation = vi.fn(); + const containmentReasons: string[] = []; + writeTimerMarker(processToken); + fs.mkdirSync(path.dirname(deadlinePath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + JSON.stringify({ + ...createMcpLifecycleLockOwner(SANDBOX_NAME, "stale-deadline-token", processToken), + pid: 2_147_483_647, + processIdentity: "dead-process", + }), + ); + + let failure: unknown; + try { + withMcpLifecycleDeadlineFenceSync(SANDBOX_NAME, processToken, operation, { + ...options(), + throwOnCommittedContainment: true, + onContainment: ({ reason }) => containmentReasons.push(reason), + }); + } catch (error) { + failure = error; + } + + expect(failure).toMatchObject({ code: "NEMOCLAW_DURABLE_CONTAINMENT" }); + expect(String(failure)).toContain( + "A committed process-tree containment requires operator resolution", + ); + expect(containmentReasons).toEqual([ + expect.stringContaining("remove only the exact stale owner generations"), + ]); + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("returns operator guidance after recording durable containment for a stale timer-bound main generation", () => { + const processToken = "b".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const operation = vi.fn(); + const containmentReasons: string[] = []; + writeTimerMarker(processToken); + writeStaleMainOwner(processToken); + + let failure: unknown; + try { + withMcpLifecycleDeadlineFenceSync(SANDBOX_NAME, processToken, operation, { + ...options(), + throwOnCommittedContainment: true, + onContainment: ({ reason }) => containmentReasons.push(reason), + }); + } catch (error) { + failure = error; + } + + expect(failure).toMatchObject({ code: "NEMOCLAW_DURABLE_CONTAINMENT" }); + expect(String(failure)).toContain("remove only the exact stale owner generations"); + expect(containmentReasons).toEqual([ + expect.stringContaining("remove only the exact stale owner generations"), + ]); + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("retains async deadline generations only for an uncommitted coded failure", async () => { + const processToken = "7".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker(processToken); + + await expect( + withMcpLifecycleDeadlineFence( + SANDBOX_NAME, + processToken, + () => { + throw durableMcpLifecycleContainmentFailure( + new Error("async deadline containment commit failed"), + lockPath, + ); + }, + options(), + ), + ).rejects.toThrow("async deadline containment commit failed"); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(deadlinePath)).toBe(true); + + fs.rmSync(lockPath, { force: true }); + fs.rmSync(deadlinePath, { force: true }); + await expect( + withMcpLifecycleDeadlineFence( + SANDBOX_NAME, + processToken, + () => { + throw new Error("ordinary async deadline failure"); + }, + options(), + ), + ).rejects.toThrow("ordinary async deadline failure"); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + }); + + it("contains a stale async main generation that records a rotated Shields timer token", async () => { + const ownerToken = "3".repeat(32); + const currentToken = "4".repeat(32); + const lockPath = writeStaleMainOwner(ownerToken); + writeTimerMarker(currentToken); + let entered = false; + + await expect( + withMcpLifecycleLock( + SANDBOX_NAME, + () => { + entered = true; + }, + options(), + ), + ).rejects.toThrow("Sandbox mutation containment is active"); + + expect(entered).toBe(false); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("contains a stale synchronous main generation that records the current Shields timer token", () => { + const processToken = "5".repeat(32); + const lockPath = writeStaleMainOwner(processToken); + writeTimerMarker(processToken); + + expect(() => withMcpLifecycleLockSync(SANDBOX_NAME, () => "entered", options())).toThrow( + "Sandbox mutation containment is active", + ); + + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("contains a stale async owner acquired before its Shields timer marker appears", async () => { + const processToken = "6".repeat(32); + const lockPath = writeStaleMainOwner(); + const operation = vi.fn(); + const processKill = publishTimerWhenStaleOwnerIsObserved(processToken); + + await expect( + withMcpLifecycleLock(SANDBOX_NAME, operation, { + ...options(), + timeoutMs: 2_000, + }), + ).rejects.toThrow("Sandbox mutation containment is active"); + + expect(operation).not.toHaveBeenCalled(); + expect(processKill).toHaveBeenCalledWith(2_147_483_647, 0); + expect(fs.existsSync(lockPath)).toBe(true); + expect(JSON.parse(fs.readFileSync(`${lockPath}.containment`, "utf8"))).toMatchObject({ + sandboxName: SANDBOX_NAME, + shieldsTakeoverToken: processToken, + containmentReason: expect.stringContaining("timer-bound sandbox mutation owner exited"), + }); + }); + + it("contains a stale synchronous owner acquired before its Shields timer marker appears", () => { + const processToken = "7".repeat(32); + const lockPath = writeStaleMainOwner(); + const operation = vi.fn(); + const processKill = publishTimerWhenStaleOwnerIsObserved(processToken); + + expect(() => + withMcpLifecycleLockSync(SANDBOX_NAME, operation, { + ...options(), + timeoutMs: 2_000, + }), + ).toThrow("Sandbox mutation containment is active"); + + expect(operation).not.toHaveBeenCalled(); + expect(processKill).toHaveBeenCalledWith(2_147_483_647, 0); + expect(fs.existsSync(lockPath)).toBe(true); + expect(JSON.parse(fs.readFileSync(`${lockPath}.containment`, "utf8"))).toMatchObject({ + sandboxName: SANDBOX_NAME, + shieldsTakeoverToken: processToken, + containmentReason: expect.stringContaining("timer-bound sandbox mutation owner exited"), + }); + }); + + it("reclaims a stale main generation with no Shields timer authority", () => { + const lockPath = writeStaleMainOwner(); + + expect( + withMcpLifecycleLockSync(SANDBOX_NAME, () => "entered", { + ...options(), + timeoutMs: 2_000, + }), + ).toBe("entered"); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + + it("blocks synchronous mutation while committed containment is active", () => { + const processToken = "b".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + beginCommittedMcpLifecycleContainmentSync( + SANDBOX_NAME, + processToken, + "test containment", + stateDir, + ); + + expect(() => withMcpLifecycleLockSync(SANDBOX_NAME, () => "entered", options())).toThrow( + "Sandbox mutation containment is active", + ); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("keeps an active deadline gate closed when containment reporting fails", async () => { + const processToken = "c".repeat(32); + const replacementToken = "d".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker(processToken); + fs.mkdirSync(path.dirname(deadlinePath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + `${JSON.stringify( + createMcpLifecycleLockOwner(SANDBOX_NAME, "active-deadline-owner", processToken), + )}\n`, + ); + const onContainment = vi.fn(() => { + writeTimerMarker(replacementToken); + throw new Error("audit unavailable"); + }); + + await expect( + withMcpLifecycleDeadlineFence(SANDBOX_NAME, processToken, () => "entered", { + ...options(), + onContainment, + }), + ).rejects.toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalledOnce(); + expect(fs.existsSync(deadlinePath)).toBe(true); + }); +}); diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index b1606c16667..ac898f55e83 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -7,19 +7,32 @@ import fs from "node:fs"; import path from "node:path"; import { performance } from "node:perf_hooks"; +import { + isShieldsTimerDeadlineExpired, + readShieldsTimerTakeoverToken, +} from "./mcp-lifecycle-lock/shields-timer-authority"; import { classifyMcpLifecycleLock, createMcpLifecycleLockOwner, type LockObservation, type McpLifecycleLockDisposition, + type McpLifecycleLockOwner, + readMcpLockHostIdentity, + readMcpLockPidNamespaceIdentity, + readMcpLockProcessIdentity, } from "./mcp-lifecycle-lock-identity"; import { getMcpLifecycleLockPath, mcpLifecycleLockPathExists, + mcpLifecycleLockPathExistsSync, readMcpLifecycleLockObservation, + readMcpLifecycleLockObservationSync, reclaimStaleMcpLifecycleLockGeneration, + reclaimStaleMcpLifecycleLockGenerationSync, safelyReleaseMcpLifecycleLock, + safelyReleaseMcpLifecycleLockSync, writeMcpLifecycleLockCandidateAndLink, + writeMcpLifecycleLockCandidateAndLinkSync, } from "./mcp-lifecycle-lock-storage"; import { resolveNemoclawStateDir } from "./paths"; @@ -35,6 +48,24 @@ interface CorruptGenerationTracker { interface AcquiredMcpLifecycleLock { lockPath: string; token: string; + shieldsTakeoverToken?: string; +} + +export interface McpLifecycleDeadlineFenceOptions extends McpLifecycleLockOptions { + /** Audit an owner that keeps the deadline gate closed while it exits naturally. */ + onContainment?: (details: McpLifecycleDeadlineContainment) => Promise | void; +} + +export interface McpLifecycleDeadlineFenceSyncOptions extends McpLifecycleLockOptions { + /** Audit an owner that keeps the deadline gate closed while it exits naturally. */ + onContainment?: (details: McpLifecycleDeadlineContainment) => void; + /** Return operator guidance instead of waiting when durable containment already exists. */ + throwOnCommittedContainment?: boolean; +} + +export interface McpLifecycleDeadlineContainment { + ownerPid: number | null; + reason: string; } export interface McpLifecycleLockOptions { @@ -47,6 +78,7 @@ export interface McpLifecycleLockOptions { interface HeldLockLease { active: boolean; + retainForDurableContainment: boolean; } type HeldLockContext = ReadonlyMap; @@ -61,6 +93,93 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)); + +function sleepSync(ms: number): void { + Atomics.wait(sleepBuffer, 0, 0, ms); +} + +function committedContainmentPath(lockPath: string): string { + return `${lockPath}.containment`; +} + +function fsyncLockDirectorySync(lockPath: string): void { + const directoryFd = fs.openSync(path.dirname(lockPath), fs.constants.O_RDONLY); + try { + fs.fsyncSync(directoryFd); + } finally { + fs.closeSync(directoryFd); + } +} + +function beginCommittedContainmentAtPathSync( + lockPath: string, + sandboxName: string, + takeoverToken: string | undefined, + reason: string, +): void { + const containmentPath = committedContainmentPath(lockPath); + const token = crypto.randomUUID(); + const owner = { + ...createMcpLifecycleLockOwner(sandboxName, token, takeoverToken), + containmentReason: reason, + }; + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + if (!writeMcpLifecycleLockCandidateAndLinkSync(containmentPath, owner)) { + throw new Error( + `A committed process-tree containment already exists for sandbox '${sandboxName}'`, + ); + } + try { + fsyncLockDirectorySync(containmentPath); + } catch (error) { + safelyReleaseMcpLifecycleLockSync(containmentPath, token); + throw error; + } +} + +function ensureDurableContainmentForStaleGenerationSync( + lockPath: string, + sandboxName: string, + stateDir: string, + observation: LockObservation, + reason: string, +): void { + const containmentPath = committedContainmentPath(lockPath); + if (mcpLifecycleLockPathExistsSync(containmentPath)) return; + const generation = `${String(observation.dev)}:${String(observation.ino)}:${ + observation.owner?.token ?? "invalid" + }`; + try { + beginCommittedContainmentAtPathSync( + lockPath, + sandboxName, + readShieldsTimerTakeoverToken(sandboxName, stateDir), + `${reason}; contained generation ${generation}`, + ); + } catch (error) { + if (mcpLifecycleLockPathExistsSync(containmentPath)) return; + throw error; + } +} + +export function beginCommittedMcpLifecycleContainmentSync( + sandboxName: string, + takeoverToken: string, + reason: string, + stateDir = resolveNemoclawStateDir(), +): void { + if (!/^[0-9a-f]{32}$/.test(takeoverToken)) { + throw new Error("Auto-restore takeover token must be 32 lowercase hexadecimal characters"); + } + beginCommittedContainmentAtPathSync( + getMcpLifecycleLockPath(sandboxName, stateDir), + sandboxName, + takeoverToken, + reason, + ); +} + function resetCorruptGenerationTracker(tracker: CorruptGenerationTracker): void { tracker.generation = null; tracker.firstSeenAt = 0; @@ -93,33 +212,145 @@ function classifyObservedMcpLifecycleLock( ); } -async function tryReapStaleLock( +function isValidMainOwnerForSandbox(observation: LockObservation, sandboxName: string): boolean { + return observation.owner?.sandboxName === sandboxName; +} + +function committedContainmentActiveError( + sandboxName: string, + lockPath: string, + containment: LockObservation, +): Error { + const containmentPath = committedContainmentPath(lockPath); + return new Error( + `Sandbox mutation containment is active for '${sandboxName}' at '${containmentPath}' (generation token '${containment.owner?.token ?? "invalid"}'). A previous owner or stale-lock reaper exited without proof that every descendant stopped. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', and '${lockPath}.deadline'; record each target's file kind, device/inode, and owner token when present; verify those identities and this containment token are unchanged; remove only those exact stale owner generations first and this exact containment generation last before retrying.`, + ); +} + +async function tryReapStaleMainLock( lockPath: string, sandboxName: string, + stateDir: string, corruptLockGraceMs: number, corruptTracker: CorruptGenerationTracker, ): Promise { + const containmentPath = committedContainmentPath(lockPath); + const deadlinePath = `${lockPath}.deadline`; + if ( + (await mcpLifecycleLockPathExists(containmentPath)) || + (await mcpLifecycleLockPathExists(deadlinePath)) + ) { + return false; + } + + const takeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); const reaperPath = `${lockPath}.reaper`; const reaperToken = crypto.randomUUID(); - const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken); + const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken, takeoverToken); if (!(await writeMcpLifecycleLockCandidateAndLink(reaperPath, reaperOwner))) return false; try { + if ( + (await mcpLifecycleLockPathExists(containmentPath)) || + (await mcpLifecycleLockPathExists(deadlinePath)) || + readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken + ) { + return false; + } const latest = await readMcpLifecycleLockObservation(lockPath); if (!latest) return true; if ( + !isValidMainOwnerForSandbox(latest, sandboxName) || classifyObservedMcpLifecycleLock(latest, sandboxName, corruptLockGraceMs, corruptTracker) !== - "stale" + "stale" ) { return false; } - + const currentTakeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); + if (currentTakeoverToken !== takeoverToken) { + // Shields-down acquires the main generation before publishing its timer + // marker. Never reclaim across that authority transition; a subsequent + // reaper will inspect the generation under the now-current token. + return false; + } + if (latest.owner?.shieldsTakeoverToken || currentTakeoverToken) { + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + latest, + "A timer-bound sandbox mutation owner exited before durable containment was committed", + ); + return false; + } return reclaimStaleMcpLifecycleLockGeneration(lockPath, latest); } finally { await safelyReleaseMcpLifecycleLock(reaperPath, reaperToken); } } +function tryReapStaleMainLockSync( + lockPath: string, + sandboxName: string, + stateDir: string, + corruptLockGraceMs: number, + corruptTracker: CorruptGenerationTracker, +): boolean { + const containmentPath = committedContainmentPath(lockPath); + const deadlinePath = `${lockPath}.deadline`; + if ( + mcpLifecycleLockPathExistsSync(containmentPath) || + mcpLifecycleLockPathExistsSync(deadlinePath) + ) { + return false; + } + + const takeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); + const reaperPath = `${lockPath}.reaper`; + const reaperToken = crypto.randomUUID(); + const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken, takeoverToken); + if (!writeMcpLifecycleLockCandidateAndLinkSync(reaperPath, reaperOwner)) return false; + + try { + if ( + mcpLifecycleLockPathExistsSync(containmentPath) || + mcpLifecycleLockPathExistsSync(deadlinePath) || + readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken + ) { + return false; + } + const latest = readMcpLifecycleLockObservationSync(lockPath); + if (!latest) return true; + if ( + !isValidMainOwnerForSandbox(latest, sandboxName) || + classifyObservedMcpLifecycleLock(latest, sandboxName, corruptLockGraceMs, corruptTracker) !== + "stale" + ) { + return false; + } + const currentTakeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); + if (currentTakeoverToken !== takeoverToken) { + // Shields-down acquires the main generation before publishing its timer + // marker. Never reclaim across that authority transition; a subsequent + // reaper will inspect the generation under the now-current token. + return false; + } + if (latest.owner?.shieldsTakeoverToken || currentTakeoverToken) { + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + latest, + "A timer-bound sandbox mutation owner exited before durable containment was committed", + ); + return false; + } + return reclaimStaleMcpLifecycleLockGenerationSync(lockPath, latest); + } finally { + safelyReleaseMcpLifecycleLockSync(reaperPath, reaperToken); + } +} + async function acquireMcpLifecycleLock( sandboxName: string, options: McpLifecycleLockOptions, @@ -130,7 +361,8 @@ async function acquireMcpLifecycleLock( options.corruptLockGraceMs, DEFAULT_CORRUPT_LOCK_GRACE_MS, ); - const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); await fs.promises.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700, @@ -139,8 +371,14 @@ async function acquireMcpLifecycleLock( const startedAt = performance.now(); const corruptMainTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; const corruptReaperTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + const corruptDeadlineTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; let lastOwnerPid: number | null = null; for (;;) { + const containmentPath = committedContainmentPath(lockPath); + const containment = await readMcpLifecycleLockObservation(containmentPath); + if (containment) { + throw committedContainmentActiveError(sandboxName, lockPath, containment); + } if (performance.now() - startedAt >= timeoutMs) { const ownerSuffix = lastOwnerPid ? ` (owner pid ${lastOwnerPid})` : ""; throw new Error( @@ -148,6 +386,30 @@ async function acquireMcpLifecycleLock( ); } + const deadlinePath = `${lockPath}.deadline`; + const deadlineObservation = await readMcpLifecycleLockObservation(deadlinePath); + if (deadlineObservation) { + const deadlineDisposition = classifyObservedMcpLifecycleLock( + deadlineObservation, + sandboxName, + corruptLockGraceMs, + corruptDeadlineTracker, + ); + if (deadlineDisposition === "stale") { + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + deadlineObservation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + await sleep(pollIntervalMs); + continue; + } + resetCorruptGenerationTracker(corruptDeadlineTracker); + const reaperPath = `${lockPath}.reaper`; const reaperObservation = await readMcpLifecycleLockObservation(reaperPath); if (reaperObservation) { @@ -158,10 +420,13 @@ async function acquireMcpLifecycleLock( corruptReaperTracker, ); if (reaperDisposition === "stale") { - // The reaper has the same atomic, PID-identified owner format as the - // main lock. A SIGKILL at any point in stale-lock cleanup is therefore - // recoverable without age-expiring a legitimate long operation. - await reclaimStaleMcpLifecycleLockGeneration(reaperPath, reaperObservation); + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + reaperObservation, + "A stale-lock reaper exited before cleanup completed", + ); continue; } await sleep(pollIntervalMs); @@ -169,14 +434,41 @@ async function acquireMcpLifecycleLock( } resetCorruptGenerationTracker(corruptReaperTracker); - if (!(await mcpLifecycleLockPathExists(reaperPath))) { + // The detached timer publishes its deadline fence only after restoreAt. + // Close the ordinary-acquisition gate directly from the durable marker so + // no new top-level mutation can enter in that publication window, or if + // the timer exits before it publishes the fence. Timer/interactive + // recovery uses the dedicated deadline-fence API and does not pass here. + if (isShieldsTimerDeadlineExpired(sandboxName, stateDir)) { + await sleep(pollIntervalMs); + continue; + } + + if ( + !(await mcpLifecycleLockPathExists(deadlinePath)) && + !(await mcpLifecycleLockPathExists(reaperPath)) && + !isShieldsTimerDeadlineExpired(sandboxName, stateDir) + ) { const token = crypto.randomUUID(); - const owner = createMcpLifecycleLockOwner(sandboxName, token); + const shieldsTakeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); + const owner = createMcpLifecycleLockOwner(sandboxName, token, shieldsTakeoverToken); if (await writeMcpLifecycleLockCandidateAndLink(lockPath, owner)) { // A stale-lock reaper may have appeared between our pre-check and the // atomic link. Do not enter the critical section until that generation // gate has gone away. - if (!(await mcpLifecycleLockPathExists(reaperPath))) return { lockPath, token }; + if ( + !(await mcpLifecycleLockPathExists(containmentPath)) && + !(await mcpLifecycleLockPathExists(deadlinePath)) && + !(await mcpLifecycleLockPathExists(reaperPath)) && + !isShieldsTimerDeadlineExpired(sandboxName, stateDir) && + readShieldsTimerTakeoverToken(sandboxName, stateDir) === shieldsTakeoverToken + ) { + return { + lockPath, + token, + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), + }; + } await safelyReleaseMcpLifecycleLock(lockPath, token); } } @@ -192,17 +484,1006 @@ async function acquireMcpLifecycleLock( corruptMainTracker, ) === "stale" ) { - if (await tryReapStaleLock(lockPath, sandboxName, corruptLockGraceMs, corruptMainTracker)) { + if (isValidMainOwnerForSandbox(observation, sandboxName)) { + await tryReapStaleMainLock( + lockPath, + sandboxName, + stateDir, + corruptLockGraceMs, + corruptMainTracker, + ); + continue; + } + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + observation, + "A sandbox mutation owner exited before its descendants could be proven contained", + ); + continue; + } + } else { + resetCorruptGenerationTracker(corruptMainTracker); + } + await sleep(pollIntervalMs); + } +} + +function acquireMcpLifecycleLockSync( + sandboxName: string, + options: McpLifecycleLockOptions & { stateDir: string }, +): AcquiredMcpLifecycleLock { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const timeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptLockGraceMs = positiveInteger( + options.corruptLockGraceMs, + DEFAULT_CORRUPT_LOCK_GRACE_MS, + ); + const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + + const startedAt = performance.now(); + const corruptMainTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + const corruptReaperTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + const corruptDeadlineTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + let lastOwnerPid: number | null = null; + for (;;) { + const containmentPath = committedContainmentPath(lockPath); + const containment = readMcpLifecycleLockObservationSync(containmentPath); + if (containment) { + throw committedContainmentActiveError(sandboxName, lockPath, containment); + } + if (performance.now() - startedAt >= timeoutMs) { + throw new Error( + `Timed out waiting for sandbox mutation lock for '${sandboxName}'${ + lastOwnerPid ? ` (owner PID ${lastOwnerPid})` : "" + }`, + ); + } + + const deadlinePath = `${lockPath}.deadline`; + const deadlineObservation = readMcpLifecycleLockObservationSync(deadlinePath); + if (deadlineObservation) { + const deadlineDisposition = classifyObservedMcpLifecycleLock( + deadlineObservation, + sandboxName, + corruptLockGraceMs, + corruptDeadlineTracker, + ); + if (deadlineDisposition === "stale") { + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + deadlineObservation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + sleepSync(pollIntervalMs); + continue; + } + resetCorruptGenerationTracker(corruptDeadlineTracker); + + const reaperPath = `${lockPath}.reaper`; + const reaperObservation = readMcpLifecycleLockObservationSync(reaperPath); + if (reaperObservation) { + const reaperDisposition = classifyObservedMcpLifecycleLock( + reaperObservation, + sandboxName, + corruptLockGraceMs, + corruptReaperTracker, + ); + if (reaperDisposition === "stale") { + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + reaperObservation, + "A stale-lock reaper exited before cleanup completed", + ); + continue; + } + sleepSync(pollIntervalMs); + continue; + } + resetCorruptGenerationTracker(corruptReaperTracker); + + if (isShieldsTimerDeadlineExpired(sandboxName, options.stateDir)) { + sleepSync(pollIntervalMs); + continue; + } + + if ( + !mcpLifecycleLockPathExistsSync(deadlinePath) && + !mcpLifecycleLockPathExistsSync(reaperPath) && + !isShieldsTimerDeadlineExpired(sandboxName, options.stateDir) + ) { + const token = crypto.randomUUID(); + const shieldsTakeoverToken = readShieldsTimerTakeoverToken(sandboxName, options.stateDir); + const owner = createMcpLifecycleLockOwner(sandboxName, token, shieldsTakeoverToken); + if (writeMcpLifecycleLockCandidateAndLinkSync(lockPath, owner)) { + if ( + !mcpLifecycleLockPathExistsSync(containmentPath) && + !mcpLifecycleLockPathExistsSync(deadlinePath) && + !mcpLifecycleLockPathExistsSync(reaperPath) && + !isShieldsTimerDeadlineExpired(sandboxName, options.stateDir) && + readShieldsTimerTakeoverToken(sandboxName, options.stateDir) === shieldsTakeoverToken + ) { + return { + lockPath, + token, + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), + }; + } + safelyReleaseMcpLifecycleLockSync(lockPath, token); + } + } + + const observation = readMcpLifecycleLockObservationSync(lockPath); + if (observation) { + lastOwnerPid = observation.owner?.pid ?? null; + if ( + classifyObservedMcpLifecycleLock( + observation, + sandboxName, + corruptLockGraceMs, + corruptMainTracker, + ) === "stale" + ) { + if (isValidMainOwnerForSandbox(observation, sandboxName)) { + tryReapStaleMainLockSync( + lockPath, + sandboxName, + options.stateDir, + corruptLockGraceMs, + corruptMainTracker, + ); continue; } + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + observation, + "A sandbox mutation owner exited before its descendants could be proven contained", + ); + continue; } } else { + lastOwnerPid = null; resetCorruptGenerationTracker(corruptMainTracker); } + sleepSync(pollIntervalMs); + } +} + +function sameLockGeneration(left: LockObservation, right: LockObservation | null): boolean { + if (!right || left.dev !== right.dev || left.ino !== right.ino) return false; + const leftToken = left.owner?.token; + const rightToken = right.owner?.token; + return leftToken === rightToken; +} + +function selfOwnedDeadlineMainToken( + observation: LockObservation | null, + sandboxName: string, + takeoverToken: string, + expectedToken: string, +): string | null { + const owner = observation?.owner; + const processIdentity = readMcpLockProcessIdentity(process.pid); + return owner?.sandboxName === sandboxName && + owner.pid === process.pid && + Boolean(processIdentity) && + owner.processIdentity === processIdentity && + owner.hostIdentity === readMcpLockHostIdentity() && + owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity() && + owner.shieldsTakeoverToken === takeoverToken && + owner.token === expectedToken + ? owner.token + : null; +} + +function isDurableContainmentError(error: unknown): boolean { + return ( + error instanceof Error && + (error as Error & { code?: string }).code === "NEMOCLAW_DURABLE_CONTAINMENT" + ); +} + +export function durableMcpLifecycleContainmentFailure( + error: unknown, + lockPath: string, +): Error & { code: string } { + const lease = heldLocks.getStore()?.get(lockPath); + if (lease?.active) lease.retainForDurableContainment = true; + if (isDurableContainmentError(error)) { + return error as Error & { code: string }; + } + const failure = new Error( + `Durable sandbox mutation containment requires operator resolution: ${ + error instanceof Error ? error.message : String(error) + }`, + ) as Error & { code: string }; + failure.code = "NEMOCLAW_DURABLE_CONTAINMENT"; + return failure; +} + +async function ownedLifecycleGateMustRemainClosed(lockPath: string): Promise { + try { + return !(await mcpLifecycleLockPathExists(committedContainmentPath(lockPath))); + } catch { + // If committed containment cannot be inspected, retaining the exact owned + // generation is the only fail-closed outcome. + return true; + } +} + +function ownedLifecycleGateMustRemainClosedSync(lockPath: string): boolean { + try { + return !mcpLifecycleLockPathExistsSync(committedContainmentPath(lockPath)); + } catch { + // If committed containment cannot be inspected, retaining the exact + // owned generation is the only fail-closed outcome. + return true; + } +} + +async function retainOwnedLifecycleGateAfterFailure( + error: unknown, + lockPath: string, +): Promise { + if (!isDurableContainmentError(error)) return false; + return await ownedLifecycleGateMustRemainClosed(lockPath); +} + +function retainOwnedLifecycleGateAfterFailureSync(error: unknown, lockPath: string): boolean { + if (!isDurableContainmentError(error)) return false; + return ownedLifecycleGateMustRemainClosedSync(lockPath); +} + +async function deadlineMainStillPresent(lockPath: string): Promise { + try { + return (await readMcpLifecycleLockObservation(lockPath)) !== null; + } catch { + return true; + } +} + +function deadlineMainStillPresentSync(lockPath: string): boolean { + try { + return readMcpLifecycleLockObservationSync(lockPath) !== null; + } catch { + return true; + } +} + +async function reportDeadlineContainment( + options: McpLifecycleDeadlineFenceOptions, + details: McpLifecycleDeadlineContainment, +): Promise { + try { + await options.onContainment?.(details); + } catch { + // Reporting must not release the security gate it is describing. + } +} + +function reportDeadlineContainmentSync( + options: McpLifecycleDeadlineFenceSyncOptions, + details: McpLifecycleDeadlineContainment, +): void { + try { + options.onContainment?.(details); + } catch { + // Reporting must not release the security gate it is describing. + } +} + +async function acquireDeadlineFence( + sandboxName: string, + takeoverToken: string, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const timeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptLockGraceMs = positiveInteger( + options.corruptLockGraceMs, + DEFAULT_CORRUPT_LOCK_GRACE_MS, + ); + const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + const deadlinePath = `${lockPath}.deadline`; + await fs.promises.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + if (await mcpLifecycleLockPathExists(committedContainmentPath(lockPath))) { + if ( + notifiedGeneration !== "committed-containment" && + performance.now() - blockedAt >= timeoutMs + ) { + await reportDeadlineContainment(options, { + ownerPid: null, + reason: + "A committed process-tree containment requires operator resolution before auto-restore can continue.", + }); + notifiedGeneration = "committed-containment"; + } + await sleep(pollIntervalMs); + continue; + } + + const token = crypto.randomUUID(); + const owner = createMcpLifecycleLockOwner(sandboxName, token, takeoverToken); + if (await writeMcpLifecycleLockCandidateAndLink(deadlinePath, owner)) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) === takeoverToken) { + return { lockPath: deadlinePath, token }; + } + await safelyReleaseMcpLifecycleLock(deadlinePath, token); + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + + const observation = await readMcpLifecycleLockObservation(deadlinePath); + if ( + observation && + classifyObservedMcpLifecycleLock( + observation, + sandboxName, + corruptLockGraceMs, + corruptTracker, + ) === "stale" + ) { + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir ?? resolveNemoclawStateDir(), + observation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + if (observation) { + const generation = `${String(observation.dev)}:${String(observation.ino)}:${ + observation.owner?.token ?? "invalid" + }`; + if (generation !== notifiedGeneration && performance.now() - blockedAt >= timeoutMs) { + await reportDeadlineContainment(options, { + ownerPid: observation.owner?.pid ?? null, + reason: + "Another auto-restore deadline owner is active or cannot be verified; the deadline gate remains closed pending operator or distributed-lease resolution.", + }); + notifiedGeneration = generation; + } + } else { + blockedAt = performance.now(); + notifiedGeneration = null; + } + await sleep(pollIntervalMs); + } +} + +function acquireDeadlineFenceSync( + sandboxName: string, + takeoverToken: string, + options: McpLifecycleDeadlineFenceSyncOptions & { stateDir: string }, +): AcquiredMcpLifecycleLock { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const timeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptLockGraceMs = positiveInteger( + options.corruptLockGraceMs, + DEFAULT_CORRUPT_LOCK_GRACE_MS, + ); + const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + const deadlinePath = `${lockPath}.deadline`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const containmentPath = committedContainmentPath(lockPath); + if (mcpLifecycleLockPathExistsSync(containmentPath)) { + if (options.throwOnCommittedContainment) { + const reason = `A committed process-tree containment requires operator resolution before auto-restore can continue. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${deadlinePath}', and '${containmentPath}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason, + }); + throw durableMcpLifecycleContainmentFailure(new Error(reason), lockPath); + } + if ( + notifiedGeneration !== "committed-containment" && + performance.now() - blockedAt >= timeoutMs + ) { + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason: + "A committed process-tree containment requires operator resolution before auto-restore can continue.", + }); + notifiedGeneration = "committed-containment"; + } + sleepSync(pollIntervalMs); + continue; + } + + const token = crypto.randomUUID(); + const owner = createMcpLifecycleLockOwner(sandboxName, token, takeoverToken); + if (writeMcpLifecycleLockCandidateAndLinkSync(deadlinePath, owner)) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) === takeoverToken) { + return { lockPath: deadlinePath, token }; + } + safelyReleaseMcpLifecycleLockSync(deadlinePath, token); + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + + const observation = readMcpLifecycleLockObservationSync(deadlinePath); + if ( + observation && + classifyObservedMcpLifecycleLock( + observation, + sandboxName, + corruptLockGraceMs, + corruptTracker, + ) === "stale" + ) { + ensureDurableContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + observation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + if (observation) { + const generation = `${String(observation.dev)}:${String(observation.ino)}:${ + observation.owner?.token ?? "invalid" + }`; + if (generation !== notifiedGeneration && performance.now() - blockedAt >= timeoutMs) { + reportDeadlineContainmentSync(options, { + ownerPid: observation.owner?.pid ?? null, + reason: + "Another auto-restore deadline owner is active or cannot be verified; the deadline gate remains closed pending operator or distributed-lease resolution.", + }); + notifiedGeneration = generation; + } + } else { + blockedAt = performance.now(); + notifiedGeneration = null; + } + sleepSync(pollIntervalMs); + } +} + +async function clearDeadlineProtectedPath( + targetPath: string, + targetLabel: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const containmentTimeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const observed = await readMcpLifecycleLockObservation(targetPath); + if (!observed) return; + + const disposition = classifyObservedMcpLifecycleLock(observed, sandboxName, 0, corruptTracker); + const owner = observed.owner; + const exactLocalOwner = + owner?.sandboxName === sandboxName && + Boolean(owner.processIdentity) && + owner.hostIdentity === readMcpLockHostIdentity() && + owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity(); + if (disposition === "stale" && exactLocalOwner) { + const confirmed = await readMcpLifecycleLockObservation(targetPath); + if (!sameLockGeneration(observed, confirmed)) continue; + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const lifecyclePath = targetPath.endsWith(".reaper") + ? targetPath.slice(0, -".reaper".length) + : targetPath; + ensureDurableContainmentForStaleGenerationSync( + lifecyclePath, + sandboxName, + stateDir, + observed, + `The ${targetLabel} owner PID ${String(owner?.pid)} was already gone before takeover`, + ); + throw durableMcpLifecycleContainmentFailure( + new Error( + `The exact ${targetLabel} owner was already gone, so surviving descendants cannot be ruled out`, + ), + lifecyclePath, + ); + } + + const generation = `${String(observed.dev)}:${String(observed.ino)}:${ + owner?.token ?? "invalid" + }`; + if ( + generation !== notifiedGeneration && + performance.now() - blockedAt >= containmentTimeoutMs + ) { + await reportDeadlineContainment(options, { + ownerPid: owner?.pid ?? null, + reason: `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`, + }); + notifiedGeneration = generation; + } + await sleep(pollIntervalMs); + } +} + +function clearDeadlineProtectedPathSync( + targetPath: string, + targetLabel: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceSyncOptions, +): void { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const containmentTimeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const observed = readMcpLifecycleLockObservationSync(targetPath); + if (!observed) return; + + const disposition = classifyObservedMcpLifecycleLock(observed, sandboxName, 0, corruptTracker); + const owner = observed.owner; + const exactLocalOwner = + owner?.sandboxName === sandboxName && + Boolean(owner.processIdentity) && + owner.hostIdentity === readMcpLockHostIdentity() && + owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity(); + const currentProcessIdentity = + exactLocalOwner && owner?.pid === process.pid + ? readMcpLockProcessIdentity(process.pid, true) + : null; + if ( + exactLocalOwner && + owner?.pid === process.pid && + currentProcessIdentity !== null && + owner.processIdentity === currentProcessIdentity + ) { + const error = new Error( + "Synchronous auto-restore cannot wait behind a sibling lifecycle operation in this process", + ) as Error & { code: string }; + error.code = "NEMOCLAW_SYNC_REENTRANT_OWNER"; + throw error; + } + if (disposition === "stale" && exactLocalOwner) { + const confirmed = readMcpLifecycleLockObservationSync(targetPath); + if (!sameLockGeneration(observed, confirmed)) continue; + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const lifecyclePath = targetPath.endsWith(".reaper") + ? targetPath.slice(0, -".reaper".length) + : targetPath; + ensureDurableContainmentForStaleGenerationSync( + lifecyclePath, + sandboxName, + stateDir, + observed, + `The ${targetLabel} owner PID ${String(owner?.pid)} was already gone before takeover`, + ); + throw durableMcpLifecycleContainmentFailure( + new Error( + `The exact ${targetLabel} owner was already gone, so surviving descendants cannot be ruled out`, + ), + lifecyclePath, + ); + } + if (exactLocalOwner && owner?.pid === process.pid && currentProcessIdentity === null) { + const error = new Error( + "Synchronous auto-restore cannot verify the process identity of a same-PID lifecycle owner", + ) as Error & { code: string }; + error.code = "NEMOCLAW_SYNC_REENTRANT_OWNER"; + throw error; + } + + const generation = `${String(observed.dev)}:${String(observed.ino)}:${ + owner?.token ?? "invalid" + }`; + if ( + generation !== notifiedGeneration && + performance.now() - blockedAt >= containmentTimeoutMs + ) { + reportDeadlineContainmentSync(options, { + ownerPid: owner?.pid ?? null, + reason: `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`, + }); + notifiedGeneration = generation; + } + sleepSync(pollIntervalMs); + } +} + +async function publishDeadlineMainOwner( + lockPath: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + let notifiedError: string | null = null; + let pendingCandidateToken: string | null = null; + for (;;) { + try { + if (pendingCandidateToken) { + const existingSelfToken = selfOwnedDeadlineMainToken( + await readMcpLifecycleLockObservation(lockPath), + sandboxName, + takeoverToken, + pendingCandidateToken, + ); + if (existingSelfToken) return existingSelfToken; + pendingCandidateToken = null; + } + await clearDeadlineProtectedPath( + `${lockPath}.reaper`, + "stale-lock reaper", + sandboxName, + takeoverToken, + stateDir, + options, + ); + await clearDeadlineProtectedPath( + lockPath, + "sandbox mutation", + sandboxName, + takeoverToken, + stateDir, + options, + ); + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const candidateToken = crypto.randomUUID(); + pendingCandidateToken = candidateToken; + const timerOwner = createMcpLifecycleLockOwner(sandboxName, candidateToken, takeoverToken); + if (await writeMcpLifecycleLockCandidateAndLink(lockPath, timerOwner)) { + return candidateToken; + } + pendingCandidateToken = null; + notifiedError = null; + } catch (error) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const message = error instanceof Error ? error.message : String(error); + if (isDurableContainmentError(error)) { + if (message !== notifiedError) { + await reportDeadlineContainment(options, { + ownerPid: null, + reason: `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`, + }); + notifiedError = message; + } + while ( + readShieldsTimerTakeoverToken(sandboxName, stateDir) === takeoverToken && + ((await mcpLifecycleLockPathExists(committedContainmentPath(lockPath))) || + (await deadlineMainStillPresent(lockPath))) + ) { + await sleep(pollIntervalMs); + } + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + continue; + } + if (message !== notifiedError) { + await reportDeadlineContainment(options, { + ownerPid: null, + reason: `Auto-restore deadline setup is retrying while the gate remains closed: ${message}`, + }); + notifiedError = message; + } + } await sleep(pollIntervalMs); } } +function publishDeadlineMainOwnerSync( + lockPath: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceSyncOptions, +): string { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + let notifiedError: string | null = null; + let pendingCandidateToken: string | null = null; + for (;;) { + try { + if (pendingCandidateToken) { + const existingSelfToken = selfOwnedDeadlineMainToken( + readMcpLifecycleLockObservationSync(lockPath), + sandboxName, + takeoverToken, + pendingCandidateToken, + ); + if (existingSelfToken) return existingSelfToken; + pendingCandidateToken = null; + } + clearDeadlineProtectedPathSync( + `${lockPath}.reaper`, + "stale-lock reaper", + sandboxName, + takeoverToken, + stateDir, + options, + ); + clearDeadlineProtectedPathSync( + lockPath, + "sandbox mutation", + sandboxName, + takeoverToken, + stateDir, + options, + ); + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const candidateToken = crypto.randomUUID(); + pendingCandidateToken = candidateToken; + const timerOwner = createMcpLifecycleLockOwner(sandboxName, candidateToken, takeoverToken); + if (writeMcpLifecycleLockCandidateAndLinkSync(lockPath, timerOwner)) { + return candidateToken; + } + pendingCandidateToken = null; + notifiedError = null; + } catch (error) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + if ( + error instanceof Error && + (error as Error & { code?: string }).code === "NEMOCLAW_SYNC_REENTRANT_OWNER" + ) { + throw error; + } + const message = error instanceof Error ? error.message : String(error); + if (isDurableContainmentError(error)) { + const resolutionReason = `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; + if (message !== notifiedError) { + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason: resolutionReason, + }); + notifiedError = message; + } + if (options.throwOnCommittedContainment) { + const failure = durableMcpLifecycleContainmentFailure(error, lockPath); + failure.message = resolutionReason; + throw failure; + } + while ( + readShieldsTimerTakeoverToken(sandboxName, stateDir) === takeoverToken && + (mcpLifecycleLockPathExistsSync(committedContainmentPath(lockPath)) || + deadlineMainStillPresentSync(lockPath)) + ) { + sleepSync(pollIntervalMs); + } + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + continue; + } + if (message !== notifiedError) { + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason: `Auto-restore deadline setup is retrying while the gate remains closed: ${message}`, + }); + notifiedError = message; + } + } + sleepSync(pollIntervalMs); + } +} + +/** + * Establish the auto-restore deadline as a generation-pinned exclusion fence. + * + * Ordinary acquisitions refuse to enter while `.deadline` exists. + * The timer keeps that gate through policy restoration and configuration re-locking, + * and waits for an active owner to release naturally. Portable PID inspection + * cannot prove that force-killing a process also contained every descendant. + */ +export async function withMcpLifecycleDeadlineFence( + sandboxName: string, + takeoverToken: string, + operation: () => Promise | T, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + if (!/^[0-9a-f]{32}$/.test(takeoverToken)) { + throw new Error("Auto-restore takeover token must be 32 lowercase hexadecimal characters"); + } + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const inherited = heldLocks.getStore(); + if (inherited?.get(lockPath)?.active) return await operation(); + + const fence = await acquireDeadlineFence(sandboxName, takeoverToken, { + ...options, + stateDir, + }); + let mainToken: string | null = null; + let retainOwnedGate = false; + let lease: HeldLockLease | null = null; + try { + // An ordinary acquirer can pass its pre-publication deadline check before + // this fence exists, then link the main path after an earlier clear. Keep + // the fence and repeat takeover until this timer owns the main generation. + mainToken = await publishDeadlineMainOwner( + lockPath, + sandboxName, + takeoverToken, + stateDir, + options, + ); + + const activeLease: HeldLockLease = { + active: true, + retainForDurableContainment: false, + }; + lease = activeLease; + const context = new Map(inherited ?? []); + context.set(lockPath, activeLease); + return await heldLocks.run(context, async () => { + try { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + return await operation(); + } finally { + activeLease.active = false; + } + }); + } catch (error) { + retainOwnedGate = await retainOwnedLifecycleGateAfterFailure(error, lockPath); + throw error; + } finally { + if (!retainOwnedGate && lease?.retainForDurableContainment) { + retainOwnedGate = await ownedLifecycleGateMustRemainClosed(lockPath); + } + if (!retainOwnedGate) { + if (mainToken) await safelyReleaseMcpLifecycleLock(lockPath, mainToken); + await safelyReleaseMcpLifecycleLock(fence.lockPath, fence.token); + } + } +} + +/** + * Synchronous deadline fence for inline recovery from synchronous status and + * permission-inspection paths. + */ +export function withMcpLifecycleDeadlineFenceSync( + sandboxName: string, + takeoverToken: string, + operation: () => T, + options: McpLifecycleDeadlineFenceSyncOptions, +): T { + if (!/^[0-9a-f]{32}$/.test(takeoverToken)) { + throw new Error("Auto-restore takeover token must be 32 lowercase hexadecimal characters"); + } + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const inherited = heldLocks.getStore(); + if (inherited?.get(lockPath)?.active) return operation(); + const fence = acquireDeadlineFenceSync(sandboxName, takeoverToken, { + ...options, + stateDir, + }); + let mainToken: string | null = null; + let retainOwnedGate = false; + let lease: HeldLockLease | null = null; + try { + mainToken = publishDeadlineMainOwnerSync( + lockPath, + sandboxName, + takeoverToken, + stateDir, + options, + ); + + const activeLease: HeldLockLease = { + active: true, + retainForDurableContainment: false, + }; + lease = activeLease; + const context = new Map(inherited ?? []); + context.set(lockPath, activeLease); + try { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + return heldLocks.run(context, operation); + } finally { + activeLease.active = false; + } + } catch (error) { + retainOwnedGate = retainOwnedLifecycleGateAfterFailureSync(error, lockPath); + throw error; + } finally { + if (!retainOwnedGate && lease?.retainForDurableContainment) { + retainOwnedGate = ownedLifecycleGateMustRemainClosedSync(lockPath); + } + if (!retainOwnedGate) { + if (mainToken) safelyReleaseMcpLifecycleLockSync(lockPath, mainToken); + safelyReleaseMcpLifecycleLockSync(fence.lockPath, fence.token); + } + } +} + +export function isMcpLifecycleLockHeld( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): boolean { + return heldLocks.getStore()?.get(getMcpLifecycleLockPath(sandboxName, stateDir))?.active === true; +} + +export function withMcpLifecycleLockSync( + sandboxName: string, + operation: () => T, + options: McpLifecycleLockOptions = {}, +): T { + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const inherited = heldLocks.getStore(); + if (inherited?.get(lockPath)?.active) return operation(); + + const acquired = acquireMcpLifecycleLockSync(sandboxName, { ...options, stateDir }); + const lease: HeldLockLease = { active: true, retainForDurableContainment: false }; + const context = new Map(inherited ?? []); + context.set(lockPath, lease); + let retainOwnedGate = false; + try { + return heldLocks.run(context, operation); + } catch (error) { + retainOwnedGate = + Boolean(acquired.shieldsTakeoverToken) && + retainOwnedLifecycleGateAfterFailureSync(error, lockPath); + throw error; + } finally { + lease.active = false; + if (!retainOwnedGate && acquired.shieldsTakeoverToken && lease.retainForDurableContainment) { + retainOwnedGate = ownedLifecycleGateMustRemainClosedSync(lockPath); + } + if (!retainOwnedGate) { + safelyReleaseMcpLifecycleLockSync(acquired.lockPath, acquired.token); + } + } +} + /** * Serializes the complete MCP lifecycle for one sandbox across processes. * AsyncLocalStorage makes nested calls in the same lifecycle operation @@ -230,18 +1511,29 @@ export async function withMcpLifecycleLock( ...options, stateDir, }); - const lease: HeldLockLease = { active: true }; + const lease: HeldLockLease = { active: true, retainForDurableContainment: false }; const context = new Map(inherited ?? []); context.set(lockKey, lease); return heldLocks.run(context, async () => { + let retainOwnedGate = false; try { return await operation(); + } catch (error) { + retainOwnedGate = + Boolean(acquired.shieldsTakeoverToken) && + (await retainOwnedLifecycleGateAfterFailure(error, lockKey)); + throw error; } finally { // Async resources created by the callback retain their ALS store. Mark // the lease inactive before releasing so a detached/later promise cannot // mistake an ended parent operation for a still-held reentrant lock. lease.active = false; - await safelyReleaseMcpLifecycleLock(acquired.lockPath, acquired.token); + if (!retainOwnedGate && acquired.shieldsTakeoverToken && lease.retainForDurableContainment) { + retainOwnedGate = await ownedLifecycleGateMustRemainClosed(lockKey); + } + if (!retainOwnedGate) { + await safelyReleaseMcpLifecycleLock(acquired.lockPath, acquired.token); + } } }); } diff --git a/src/lib/state/mcp-lifecycle-lock-identity.ts b/src/lib/state/mcp-lifecycle-lock-identity.ts index b8e864ea4aa..5b0690efcb5 100644 --- a/src/lib/state/mcp-lifecycle-lock-identity.ts +++ b/src/lib/state/mcp-lifecycle-lock-identity.ts @@ -21,6 +21,8 @@ export interface McpLifecycleLockOwner { hostIdentity?: string | null; /** Linux PID namespace identity. Cross-namespace owners fail closed. */ pidNamespaceIdentity?: string | null; + /** Exact Shields timer generation correlated with this mutable-window operation. */ + shieldsTakeoverToken?: string; token: string; acquiredAt: string; } @@ -59,6 +61,9 @@ export function isMcpLifecycleLockOwner(value: unknown): value is McpLifecycleLo (candidate.pidNamespaceIdentity === undefined || candidate.pidNamespaceIdentity === null || typeof candidate.pidNamespaceIdentity === "string") && + (candidate.shieldsTakeoverToken === undefined || + (typeof candidate.shieldsTakeoverToken === "string" && + /^[0-9a-f]{32}$/.test(candidate.shieldsTakeoverToken))) && typeof candidate.token === "string" && candidate.token.length > 0 && typeof candidate.acquiredAt === "string" @@ -175,6 +180,7 @@ const LOCAL_IDENTITY_PROBES: McpLifecycleLockIdentityProbes = { export function createMcpLifecycleLockOwner( sandboxName: string, token: string, + shieldsTakeoverToken?: string, ): McpLifecycleLockOwner { return { version: LOCK_SCHEMA_VERSION, @@ -183,6 +189,7 @@ export function createMcpLifecycleLockOwner( processIdentity: readMcpLockProcessIdentity(process.pid), hostIdentity: LOCAL_HOST_IDENTITY, pidNamespaceIdentity: LOCAL_PID_NAMESPACE_IDENTITY, + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), token, acquiredAt: new Date().toISOString(), }; diff --git a/src/lib/state/mcp-lifecycle-lock-storage.ts b/src/lib/state/mcp-lifecycle-lock-storage.ts index ad5d7c0b5c7..d75caf4ac4e 100644 --- a/src/lib/state/mcp-lifecycle-lock-storage.ts +++ b/src/lib/state/mcp-lifecycle-lock-storage.ts @@ -76,6 +76,48 @@ export async function readMcpLifecycleLockObservation( } } +export function readMcpLifecycleLockObservationSync(lockPath: string): LockObservation | null { + let fd: number; + try { + fd = fs.openSync( + lockPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK, + ); + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return null; + try { + const stat = fs.lstatSync(lockPath); + if (!stat.isFile() || stat.isSymbolicLink()) { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + } catch (statError) { + if (isErrnoException(statError) && statError.code === "ENOENT") return null; + throw statError; + } + throw error; + } + + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile()) { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + try { + const parsed: unknown = JSON.parse(fs.readFileSync(fd, "utf8")); + return { + owner: isMcpLifecycleLockOwner(parsed) ? parsed : null, + mtimeMs: stat.mtimeMs, + dev: stat.dev, + ino: stat.ino, + }; + } catch { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + } finally { + fs.closeSync(fd); + } +} + export async function mcpLifecycleLockPathExists(targetPath: string): Promise { try { await fs.promises.lstat(targetPath); @@ -86,6 +128,16 @@ export async function mcpLifecycleLockPathExists(targetPath: string): Promise= 2 && published?.owner?.token === owner.token) { + return true; + } + if (isErrnoException(error) && error.code === "EEXIST") return false; + throw error; + } + } finally { + try { + fs.rmSync(candidatePath, { force: true }); + } catch { + // Publication is decided only by LINK plus owner-token reconciliation. + } + } +} diff --git a/src/lib/state/mcp-lifecycle-lock.ts b/src/lib/state/mcp-lifecycle-lock.ts index 4d6bc2934c6..e7db7296202 100644 --- a/src/lib/state/mcp-lifecycle-lock.ts +++ b/src/lib/state/mcp-lifecycle-lock.ts @@ -2,13 +2,23 @@ // SPDX-License-Identifier: Apache-2.0 export { + beginCommittedMcpLifecycleContainmentSync, + durableMcpLifecycleContainmentFailure, + isMcpLifecycleLockHeld, + type McpLifecycleDeadlineContainment, + type McpLifecycleDeadlineFenceOptions, + type McpLifecycleDeadlineFenceSyncOptions, type McpLifecycleLockOptions, + withMcpLifecycleDeadlineFence, + withMcpLifecycleDeadlineFenceSync, withMcpLifecycleLock, withMcpLifecycleLock as withSandboxMutationLock, + withMcpLifecycleLockSync, } from "./mcp-lifecycle-lock-acquisition"; export { classifyMcpLifecycleLock, type McpLifecycleLockDisposition, + type McpLifecycleLockOwner, readMcpLockHostIdentity, readMcpLockPidNamespaceIdentity, readMcpLockProcessIdentity, diff --git a/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts new file mode 100644 index 00000000000..da96c93d3b2 --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { isObjectRecord } from "../../core/json-types"; +import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../../name-validation"; +import { resolveNemoclawStateDir } from "../paths"; + +export interface ShieldsTimerMarker { + pid: number; + sandboxName: string; + snapshotPath: string; + restoreAt: string; + processToken?: string; + allowLegacyHermesProtocol?: boolean; + leaseOwnerPid?: number; + leaseOwnerStartIdentity?: string; +} + +function isShieldsTimerMarker(value: unknown): value is ShieldsTimerMarker { + if (!isObjectRecord(value)) return false; + const pid = value.pid; + return ( + typeof pid === "number" && + Number.isInteger(pid) && + pid > 0 && + typeof value.sandboxName === "string" && + typeof value.snapshotPath === "string" && + typeof value.restoreAt === "string" && + (value.processToken === undefined || typeof value.processToken === "string") && + (value.allowLegacyHermesProtocol === undefined || + typeof value.allowLegacyHermesProtocol === "boolean") && + (value.leaseOwnerPid === undefined || + (typeof value.leaseOwnerPid === "number" && + Number.isInteger(value.leaseOwnerPid) && + value.leaseOwnerPid > 0)) && + (value.leaseOwnerStartIdentity === undefined || + typeof value.leaseOwnerStartIdentity === "string") && + ((value.leaseOwnerPid === undefined && value.leaseOwnerStartIdentity === undefined) || + (typeof value.leaseOwnerPid === "number" && + typeof value.leaseOwnerStartIdentity === "string" && + value.leaseOwnerStartIdentity.length > 0)) + ); +} + +export function shieldsTimerMarkerPath( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): string { + if ( + sandboxName.length === 0 || + sandboxName.length > NAME_MAX_LENGTH || + !NAME_VALID_PATTERN.test(sandboxName) + ) { + throw new Error("Cannot resolve a Shields timer marker for an invalid sandbox name"); + } + return path.join(stateDir, `shields-timer-${sandboxName}.json`); +} + +export function readShieldsTimerMarker( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): ShieldsTimerMarker | null { + try { + return readShieldsTimerMarkerFile(shieldsTimerMarkerPath(sandboxName, stateDir)); + } catch { + return null; + } +} + +export function readShieldsTimerMarkerFile(markerPath: string): ShieldsTimerMarker | null { + try { + const markerFd = fs.openSync( + markerPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK, + ); + try { + if (!fs.fstatSync(markerFd).isFile()) return null; + const parsed = JSON.parse(fs.readFileSync(markerFd, "utf-8")); + return isShieldsTimerMarker(parsed) ? parsed : null; + } finally { + fs.closeSync(markerFd); + } + } catch { + return null; + } +} + +export function readShieldsTimerTakeoverToken( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): string | undefined { + const marker = readShieldsTimerMarker(sandboxName, stateDir); + if ( + marker?.sandboxName !== sandboxName || + typeof marker.processToken !== "string" || + !/^[0-9a-f]{32}$/.test(marker.processToken) + ) { + return undefined; + } + return marker.processToken; +} + +export function isShieldsTimerDeadlineExpired( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), + now = Date.now(), +): boolean { + const marker = readShieldsTimerMarker(sandboxName, stateDir); + if ( + marker?.sandboxName !== sandboxName || + typeof marker.processToken !== "string" || + !/^[0-9a-f]{32}$/.test(marker.processToken) + ) { + return false; + } + const restoreAtMs = new Date(marker.restoreAt).getTime(); + return Number.isFinite(restoreAtMs) && restoreAtMs <= now; +} diff --git a/src/lib/state/paths.test.ts b/src/lib/state/paths.test.ts index a43b016c656..2124a9e9a99 100644 --- a/src/lib/state/paths.test.ts +++ b/src/lib/state/paths.test.ts @@ -3,9 +3,9 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { ROOT, SCRIPTS } from "./paths"; +import { ROOT, resolveNemoclawHomeDir, resolveNemoclawStateDir, SCRIPTS } from "./paths"; describe("paths", () => { it("resolves the repo root", () => { @@ -17,4 +17,26 @@ describe("paths", () => { expect(SCRIPTS).toBe(join(ROOT, "scripts")); expect(existsSync(join(SCRIPTS, "debug.sh"))).toBe(true); }); + + it("isolates default state during Vitest without changing explicit home resolution", () => { + const isolatedState = process.env.NEMOCLAW_TEST_STATE_DIR; + + expect(isolatedState).toBeDefined(); + expect(resolveNemoclawStateDir()).toBe(isolatedState); + expect(resolveNemoclawStateDir("/explicit-home")).toBe( + join("/explicit-home", ".nemoclaw", "state"), + ); + }); + + it("honors a test fixture that explicitly changes HOME", () => { + vi.stubEnv("HOME", "/fixture-home"); + + expect(resolveNemoclawStateDir()).toBe(join("/fixture-home", ".nemoclaw", "state")); + }); + + it("does not honor the internal state override outside Vitest", () => { + vi.stubEnv("VITEST", "false"); + + expect(resolveNemoclawStateDir()).toBe(join(resolveNemoclawHomeDir(), "state")); + }); }); diff --git a/src/lib/state/paths.ts b/src/lib/state/paths.ts index ffd5d9d2856..72b9e0b2e7e 100644 --- a/src/lib/state/paths.ts +++ b/src/lib/state/paths.ts @@ -14,8 +14,15 @@ export function resolveNemoclawHomeDir(homeDir: string = process.env.HOME ?? os. return nemoclawStateRoot(homeDir, GATEWAY_PORT); } -export function resolveNemoclawStateDir( - homeDir: string = process.env.HOME ?? os.homedir(), -): string { +export function resolveNemoclawStateDir(homeDir?: string): string { + if ( + homeDir === undefined && + process.env.VITEST === "true" && + (process.env.HOME ?? "") === process.env.NEMOCLAW_TEST_BASE_HOME && + process.env.NEMOCLAW_TEST_STATE_DIR && + path.isAbsolute(process.env.NEMOCLAW_TEST_STATE_DIR) + ) { + return process.env.NEMOCLAW_TEST_STATE_DIR; + } return path.join(resolveNemoclawHomeDir(homeDir), "state"); } diff --git a/test/config-set-nested-ssrf.test.ts b/test/config-set-nested-ssrf.test.ts index 14325ce61e6..7bec805e8fe 100644 --- a/test/config-set-nested-ssrf.test.ts +++ b/test/config-set-nested-ssrf.test.ts @@ -45,6 +45,8 @@ function installMockPrivilegedExec( exports: { // Transition-lock behavior has dedicated coverage. Keep this SSRF suite // independent from host process-identity discovery while it mocks ps. + // configSet holds the lifecycle lock before entering this boundary. + isMcpLifecycleLockHeld: () => true, withTimerBoundShieldsMutationLock: ( _sandboxName: string, _command: string, diff --git a/test/helpers/isolate-test-state.ts b/test/helpers/isolate-test-state.ts new file mode 100644 index 00000000000..b0e80b287ef --- /dev/null +++ b/test/helpers/isolate-test-state.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll } from "vitest"; + +const tempRoot = process.env.TMPDIR; +if (!tempRoot || !path.isAbsolute(tempRoot)) { + throw new Error("Vitest state isolation requires the shared absolute temporary root"); +} + +const previousStateDir = process.env.NEMOCLAW_TEST_STATE_DIR; +const previousBaseHome = process.env.NEMOCLAW_TEST_BASE_HOME; +process.env.NEMOCLAW_TEST_BASE_HOME = process.env.HOME ?? ""; +process.env.NEMOCLAW_TEST_STATE_DIR = fs.mkdtempSync(path.join(tempRoot, `state-${process.pid}-`), { + encoding: "utf8", +}); +fs.chmodSync(process.env.NEMOCLAW_TEST_STATE_DIR, 0o700); + +afterAll(() => { + if (previousBaseHome === undefined) { + delete process.env.NEMOCLAW_TEST_BASE_HOME; + } else { + process.env.NEMOCLAW_TEST_BASE_HOME = previousBaseHome; + } + if (previousStateDir === undefined) { + delete process.env.NEMOCLAW_TEST_STATE_DIR; + } else { + process.env.NEMOCLAW_TEST_STATE_DIR = previousStateDir; + } +}); diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index 79599b94ed5..39036c8f6d4 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -10,13 +10,16 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as lifecycleLock from "../src/lib/state/mcp-lifecycle-lock"; import "./helpers/mcp-lifecycle-lock-properties"; -type LifecycleLockModule = typeof import("../src/lib/state/mcp-lifecycle-lock"); - const requireDist = createRequire(import.meta.url); const lockModulePath = requireDist.resolve("../src/lib/state/mcp-lifecycle-lock.js"); -const lifecycleLock = requireDist(lockModulePath) as LifecycleLockModule; +// Keep one CommonJS instance for the macOS probe spy. Behavior tests use the +// static source import so Vitest attributes their coverage to the split modules. +const requiredLifecycleLock = requireDist( + lockModulePath, +) as typeof import("../src/lib/state/mcp-lifecycle-lock"); const currentProcessIdentity = lifecycleLock.readMcpLockProcessIdentity(process.pid); const currentHostIdentity = lifecycleLock.readMcpLockHostIdentity(); const currentPidNamespaceIdentity = lifecycleLock.readMcpLockPidNamespaceIdentity(); @@ -42,6 +45,28 @@ function deferred(): { promise: Promise; resolve: () => void } { return { promise, resolve }; } +function writeTimerMarker(sandboxName: string, processToken: string): void { + fs.writeFileSync( + path.join(stateDir, `shields-timer-${sandboxName}.json`), + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + ); +} + +function routeLinkToPath( + targetPath: string, + targetLink: typeof fs.promises.link, + fallback: typeof fs.promises.link, +): typeof fs.promises.link { + const routes = new Map([[targetPath, targetLink]]); + return (from, to) => (routes.get(String(to)) ?? fallback)(from, to); +} + function waitForLine(child: ChildProcess, expected: string): Promise { return new Promise((resolve, reject) => { let output = ""; @@ -85,7 +110,7 @@ describe("MCP lifecycle lock", () => { process.env.OPENSHELL_GATEWAY = "nemoclaw-19080"; try { - expect(lifecycleLock.readMcpLockProcessIdentity(4242, true)).toBe( + expect(requiredLifecycleLock.readMcpLockProcessIdentity(4242, true)).toBe( "darwin:Mon Jun 30 12:00:00 2026", ); const options = spawnSync.mock.calls[0]?.[2] as { env?: NodeJS.ProcessEnv }; @@ -104,9 +129,10 @@ describe("MCP lifecycle lock", () => { }); it.skipIf(process.platform === "win32")( - "does not follow a symlink when observing lock ownership", + "contains a symlink generation without following or deleting it", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; const targetPath = path.join(stateDir, "operator-owned-target"); const target = `${JSON.stringify({ version: 1, @@ -121,18 +147,21 @@ describe("MCP lifecycle lock", () => { fs.symlinkSync(targetPath, lockPath); await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", options()), - ).resolves.toBe("acquired"); + lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", options({ timeoutMs: 50 })), + ).rejects.toThrow(/containment is active/); expect(fs.readFileSync(targetPath, "utf8")).toBe(target); + expect(fs.lstatSync(lockPath).isSymbolicLink()).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }, ); it.skipIf(process.platform === "win32")( - "reaps a non-regular Unix socket found at the lock path", + "contains a non-regular Unix socket generation without deleting it", async () => { const shortStateDir = path.join("/tmp", `m${process.pid}`); fs.rmSync(shortStateDir, { recursive: true, force: true }); const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", shortStateDir); + const containmentPath = `${lockPath}.containment`; expect(Buffer.byteLength(lockPath)).toBeLessThan(104); fs.mkdirSync(path.dirname(lockPath), { recursive: true }); const server = createServer(); @@ -147,8 +176,11 @@ describe("MCP lifecycle lock", () => { lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", { ...options(), stateDir: shortStateDir, + timeoutMs: 50, }), - ).resolves.toBe("acquired"); + ).rejects.toThrow(/containment is active/); + expect(fs.lstatSync(lockPath).isSocket()).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); } finally { await new Promise((resolve) => server.close(() => resolve())); fs.rmSync(shortStateDir, { recursive: true, force: true }); @@ -279,6 +311,7 @@ const releasePath = process.argv[3]; it("recovers an atomic lock left by a dead owner", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync( lockPath, @@ -304,6 +337,139 @@ const releasePath = process.argv[3]; ); expect(entered).toBe(true); expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + }); + + it("recovers an atomic lock left by a dead owner for synchronous callers", () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "stale-sync-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + + expect(lifecycleLock.withMcpLifecycleLockSync("alpha", () => "acquired", options())).toBe( + "acquired", + ); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + }); + + it("preserves a replacement main lock published during stale recovery", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "observed-stale-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const replacement = { + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: currentProcessIdentity, + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "replacement-main-token", + acquiredAt: new Date().toISOString(), + }; + const rename = fs.promises.rename.bind(fs.promises); + let injectedReplacement = false; + const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { + switch (!injectedReplacement && String(from) === lockPath) { + case true: + injectedReplacement = true; + fs.unlinkSync(lockPath); + fs.writeFileSync(lockPath, `${JSON.stringify(replacement)}\n`); + } + return rename(from, to); + }); + + try { + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + } finally { + renameSpy.mockRestore(); + } + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-main-token"); + }); + + it("commits durable containment for a stale deadline generation before an ordinary mutation", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const deadlinePath = `${lockPath}.deadline`; + const containmentPath = `${lockPath}.containment`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-deadline", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "stale-deadline-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); + }); + + it("commits durable containment for a stale deadline generation before deadline recovery", async () => { + const processToken = "9".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const deadlinePath = `${lockPath}.deadline`; + const containmentPath = `${lockPath}.containment`; + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-deadline", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: processToken, + token: "stale-deadline-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const onContainment = vi.fn(() => writeTimerMarker("alpha", "8".repeat(32))); + await expect( + lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { + ...options({ timeoutMs: 40 }), + onContainment, + }), + ).rejects.toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalledOnce(); + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); it("waits for a foreign-host owner instead of reaping it with local PID checks", async () => { @@ -421,8 +587,9 @@ const releasePath = process.argv[3]; expect(fs.existsSync(lockPath)).toBe(false); }); - it("waits for grace then recovers a stable truncated owner record", async () => { + it("waits for grace then commits durable containment for a stable truncated owner record", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync(lockPath, '{"version":1,"sandboxName":"alpha"'); @@ -441,16 +608,18 @@ const releasePath = process.argv[3]; await expect( lifecycleLock.withMcpLifecycleLock( "alpha", - () => "acquired", - options({ corruptLockGraceMs: 20 }), + () => undefined, + options({ timeoutMs: 50, corruptLockGraceMs: 20 }), ), - ).resolves.toBe("acquired"); - expect(fs.existsSync(lockPath)).toBe(false); + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); - it("recovers a reaper whose owner was killed during stale-lock cleanup", async () => { + it("commits durable containment for a reaper whose owner died during stale-lock cleanup", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); const reaperPath = `${lockPath}.reaper`; + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync( reaperPath, @@ -466,122 +635,43 @@ const releasePath = process.argv[3]; })}\n`, ); - let entered = false; - await lifecycleLock.withMcpLifecycleLock( - "alpha", - () => { - entered = true; - }, - options(), - ); - expect(entered).toBe(true); - expect(fs.existsSync(lockPath)).toBe(false); - expect(fs.existsSync(reaperPath)).toBe(false); - }); - - it("does not unlink a replacement reaper published during stale recovery", async () => { - const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); - const reaperPath = `${lockPath}.reaper`; - fs.mkdirSync(path.dirname(lockPath), { recursive: true }); - fs.writeFileSync( - reaperPath, - `${JSON.stringify({ - version: 1, - sandboxName: "alpha", - pid: 2_147_483_647, - processIdentity: "dead-reaper", - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "observed-stale-token", - acquiredAt: "2026-01-01T00:00:00.000Z", - })}\n`, - ); - const replacement = { - version: 1, - sandboxName: "alpha", - pid: process.pid, - processIdentity: lifecycleLock.readMcpLockProcessIdentity(process.pid), - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "replacement-reaper-token", - acquiredAt: new Date().toISOString(), - }; - const rename = fs.promises.rename.bind(fs.promises); - let injectedReplacement = false; - const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { - const shouldInject = !injectedReplacement && String(from) === reaperPath; - switch (shouldInject) { - case true: - injectedReplacement = true; - fs.unlinkSync(reaperPath); - fs.writeFileSync(reaperPath, `${JSON.stringify(replacement)}\n`); - } - return rename(from, to); - }); - - try { - await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), - ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); - } finally { - renameSpy.mockRestore(); - } - expect(JSON.parse(fs.readFileSync(reaperPath, "utf8")).token).toBe("replacement-reaper-token"); + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(reaperPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); - it("does not delete a replacement main lock during stale recovery", async () => { - const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); - fs.mkdirSync(path.dirname(lockPath), { recursive: true }); - fs.writeFileSync( - lockPath, - `${JSON.stringify({ - version: 1, - sandboxName: "alpha", - pid: 2_147_483_647, - processIdentity: "dead-process", - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "observed-stale-token", - acquiredAt: "2026-01-01T00:00:00.000Z", - })}\n`, + it("never overwrites an existing durable-containment generation", () => { + const processToken = "a".repeat(32); + const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath( + "alpha", + stateDir, + )}.containment`; + lifecycleLock.beginCommittedMcpLifecycleContainmentSync( + "alpha", + processToken, + "first containment", + stateDir, ); - const replacement = { - version: 1, - sandboxName: "alpha", - pid: process.pid, - processIdentity: lifecycleLock.readMcpLockProcessIdentity(process.pid), - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "replacement-main-token", - acquiredAt: new Date().toISOString(), - }; - const rename = fs.promises.rename.bind(fs.promises); - let injectedReplacement = false; - const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { - const shouldInject = !injectedReplacement && String(from) === lockPath; - switch (shouldInject) { - case true: - injectedReplacement = true; - fs.unlinkSync(lockPath); - fs.writeFileSync(lockPath, `${JSON.stringify(replacement)}\n`); - } - return rename(from, to); - }); + const firstGeneration = fs.readFileSync(containmentPath, "utf8"); - try { - await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), - ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); - } finally { - renameSpy.mockRestore(); - } - expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-main-token"); + expect(() => + lifecycleLock.beginCommittedMcpLifecycleContainmentSync( + "alpha", + processToken, + "replacement containment", + stateDir, + ), + ).toThrow("already exists"); + expect(fs.readFileSync(containmentPath, "utf8")).toBe(firstGeneration); }); it.skipIf(currentProcessIdentity === null)( - "recovers a recycled PID by comparing process-start identity", + "recovers a recycled PID after confirming a fresh process-start mismatch", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync( lockPath, @@ -601,6 +691,41 @@ const releasePath = process.argv[3]; lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options()), ).resolves.toBeUndefined(); expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + }, + ); + + it.skipIf(currentProcessIdentity === null)( + "does not treat a recycled same PID as synchronous reentrancy", + () => { + const processToken = "6".repeat(32); + const replacementProcessToken = "7".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: `${String(currentProcessIdentity)}-different-start`, + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: processToken, + token: "recycled-sync-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const onContainment = vi.fn(() => writeTimerMarker("alpha", replacementProcessToken)); + + expect(() => + lifecycleLock.withMcpLifecycleDeadlineFenceSync("alpha", processToken, () => undefined, { + ...options({ timeoutMs: 10 }), + onContainment, + }), + ).toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalled(); }, ); @@ -642,4 +767,345 @@ const releasePath = process.argv[3]; expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-token"); }); + + it("binds an ordinary lifecycle owner to the active Shields timer generation", async () => { + const processToken = "a".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", processToken); + + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).shieldsTakeoverToken).toBe( + processToken, + ); + }, + options(), + ); + }); + + it("does not read a timer marker through a traversal-shaped lifecycle key", async () => { + const sandboxName = `a/../../escaped-${path.basename(stateDir)}`; + const escapedMarkerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const processToken = "a".repeat(32); + fs.writeFileSync( + escapedMarkerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + ); + + try { + const lockPath = lifecycleLock.getMcpLifecycleLockPath(sandboxName, stateDir); + await lifecycleLock.withMcpLifecycleLock( + sandboxName, + () => { + expect( + JSON.parse(fs.readFileSync(lockPath, "utf8")).shieldsTakeoverToken, + ).toBeUndefined(); + }, + options(), + ); + } finally { + fs.rmSync(escapedMarkerPath, { force: true }); + } + }); + + it.skipIf(process.platform === "win32")( + "does not derive Shields authority from a symlinked timer marker", + async () => { + const processToken = "8".repeat(32); + const targetPath = path.join(stateDir, "operator-owned-timer-marker.json"); + const markerPath = path.join(stateDir, "shields-timer-alpha.json"); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.writeFileSync( + targetPath, + JSON.stringify({ + pid: process.pid, + sandboxName: "alpha", + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + ); + fs.symlinkSync(targetPath, markerPath); + let observedTakeoverToken: string | undefined; + + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + observedTakeoverToken = JSON.parse( + fs.readFileSync(lockPath, "utf8"), + ).shieldsTakeoverToken; + }, + options(), + ); + + expect(observedTakeoverToken).toBeUndefined(); + expect(JSON.parse(fs.readFileSync(targetPath, "utf8")).processToken).toBe(processToken); + }, + ); + + it("retries without entering when the timer generation changes during lock publication", async () => { + const firstProcessToken = "a".repeat(32); + const replacementProcessToken = "b".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", firstProcessToken); + const link = fs.promises.link.bind(fs.promises); + const lockPathLink = vi.fn(link); + lockPathLink.mockImplementationOnce(async (from, to) => { + await link(from, to); + writeTimerMarker("alpha", replacementProcessToken); + }); + const linkSpy = vi + .spyOn(fs.promises, "link") + .mockImplementation(routeLinkToPath(lockPath, lockPathLink, link)); + + try { + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).shieldsTakeoverToken).toBe( + replacementProcessToken, + ); + }, + options(), + ); + } finally { + linkSpy.mockRestore(); + } + expect(lockPathLink).toHaveBeenCalled(); + }); + + it("keeps the deadline fence closed through restore and configuration relock", async () => { + const processToken = "c".repeat(32); + writeTimerMarker("alpha", processToken); + const entered = deferred(); + const release = deferred(); + let contenderEntered = false; + + const deadline = lifecycleLock.withMcpLifecycleDeadlineFence( + "alpha", + processToken, + async () => { + entered.resolve(); + await release.promise; + }, + options(), + ); + await entered.promise; + const contender = lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + contenderEntered = true; + }, + options({ timeoutMs: 2_000 }), + ); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(contenderEntered).toBe(false); + + release.resolve(); + await Promise.all([deadline, contender]); + expect(contenderEntered).toBe(true); + }); + + it("keeps the deadline fence when an in-flight ordinary publication wins the main link", async () => { + const processToken = "0".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker("alpha", processToken); + + const ordinaryLinkStarted = deferred(); + const allowOrdinaryPublication = deferred(); + const ordinaryPublished = deferred(); + const allowOrdinaryLinkReturn = deferred(); + const timerMainLinkStarted = deferred(); + const timerEntered = deferred(); + const releaseTimer = deferred(); + const link = fs.promises.link.bind(fs.promises); + let ordinaryEntered = false; + const lockPathLink = vi.fn(link); + lockPathLink.mockImplementationOnce(async (from, to) => { + ordinaryLinkStarted.resolve(); + await allowOrdinaryPublication.promise; + await link(from, to); + ordinaryPublished.resolve(); + await allowOrdinaryLinkReturn.promise; + }); + lockPathLink.mockImplementationOnce(async (from, to) => { + timerMainLinkStarted.resolve(); + await ordinaryPublished.promise; + await link(from, to); + }); + const linkSpy = vi + .spyOn(fs.promises, "link") + .mockImplementation(routeLinkToPath(lockPath, lockPathLink, link)); + + try { + const ordinary = lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + ordinaryEntered = true; + }, + options({ timeoutMs: 2_000 }), + ); + await ordinaryLinkStarted.promise; + + const deadline = lifecycleLock.withMcpLifecycleDeadlineFence( + "alpha", + processToken, + async () => { + timerEntered.resolve(); + await releaseTimer.promise; + }, + options({ timeoutMs: 2_000 }), + ); + await timerMainLinkStarted.promise; + expect(fs.existsSync(deadlinePath)).toBe(true); + + allowOrdinaryPublication.resolve(); + await ordinaryPublished.promise; + allowOrdinaryLinkReturn.resolve(); + await timerEntered.promise; + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(ordinaryEntered).toBe(false); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(ordinaryEntered).toBe(false); + + releaseTimer.resolve(); + await Promise.all([deadline, ordinary]); + expect(ordinaryEntered).toBe(true); + expect(lockPathLink.mock.calls.length).toBeGreaterThanOrEqual(3); + } finally { + allowOrdinaryPublication.resolve(); + allowOrdinaryLinkReturn.resolve(); + releaseTimer.resolve(); + linkSpy.mockRestore(); + } + }); + + it("waits for a live same-generation owner to release naturally", async () => { + const processToken = "d".repeat(32); + writeTimerMarker("alpha", processToken); + const releasePath = path.join(stateDir, "release-owner"); + const script = String.raw` +const fs = require("node:fs"); +const lock = require(process.argv[1]); +const stateDir = process.argv[2]; +const releasePath = process.argv[3]; +(async () => { + await lock.withMcpLifecycleLock("alpha", async () => { + process.stdout.write("READY\n"); + while (!fs.existsSync(releasePath)) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }, { stateDir, pollIntervalMs: 5, timeoutMs: 2000 }); +})().then(() => process.exit(0), () => process.exit(1)); +`; + const child = spawn(process.execPath, ["-e", script, lockModulePath, stateDir, releasePath], { + stdio: ["ignore", "pipe", "pipe"], + }); + children.add(child); + await waitForLine(child, "READY"); + const childExit = new Promise((resolve) => child.once("exit", () => resolve())); + const containmentReported = deferred(); + let entered = false; + let reportedOwnerPid: number | null = null; + const deadline = lifecycleLock.withMcpLifecycleDeadlineFence( + "alpha", + processToken, + () => { + entered = true; + }, + { + ...options({ timeoutMs: 10 }), + onContainment: ({ ownerPid }) => { + reportedOwnerPid = ownerPid; + containmentReported.resolve(); + }, + }, + ); + await containmentReported.promise; + expect(reportedOwnerPid).toBe(child.pid); + expect(entered).toBe(false); + expect(() => process.kill(child.pid!, 0)).not.toThrow(); + + fs.writeFileSync(releasePath, "release\n"); + await Promise.all([deadline, childExit]); + expect(entered).toBe(true); + children.delete(child); + }); + + it("preserves an active owner from a different timer generation", async () => { + const processToken = "e".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: currentProcessIdentity, + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: "f".repeat(32), + token: "replacement-generation", + acquiredAt: new Date().toISOString(), + })}\n`, + ); + const deadlinePath = `${lockPath}.deadline`; + const deadlineObservations: boolean[] = []; + const onContainment = vi.fn(() => { + deadlineObservations.push(fs.existsSync(deadlinePath)); + writeTimerMarker("alpha", "3".repeat(32)); + }); + await expect( + lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { + ...options({ timeoutMs: 10 }), + onContainment, + }), + ).rejects.toThrow("Auto-restore authority changed"); + expect(deadlineObservations).not.toHaveLength(0); + expect(deadlineObservations.every(Boolean)).toBe(true); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-generation"); + }); + + it("contains an already-dead local owner because surviving descendants cannot be ruled out", async () => { + const processToken = "4".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-owner", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: "5".repeat(32), + token: "dead-foreign-generation", + acquiredAt: new Date().toISOString(), + })}\n`, + ); + const onContainment = vi.fn(() => writeTimerMarker("alpha", "a".repeat(32))); + await expect( + lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { + ...options(), + onContainment, + }), + ).rejects.toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalledOnce(); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); + }); }); diff --git a/test/vitest-temp-root.test.ts b/test/vitest-temp-root.test.ts index bfeafce5e40..eb781c21a8c 100644 --- a/test/vitest-temp-root.test.ts +++ b/test/vitest-temp-root.test.ts @@ -13,6 +13,7 @@ import { setupVitestTempRoot } from "./helpers/vitest-temp-root"; const TEMP_ENV_KEYS = ["TMPDIR", "TMP", "TEMP"] as const; const ROOT_SETUP = "test/helpers/vitest-temp-root.ts"; +const STATE_SETUP = "test/helpers/isolate-test-state.ts"; type TempEnv = Record<(typeof TEMP_ENV_KEYS)[number], string | undefined>; @@ -51,6 +52,17 @@ describe("Vitest temp root", () => { expect(fs.statSync(root).isDirectory()).toBe(true); }); + it("isolates each test file's NemoClaw state inside the run root", () => { + const root = process.env.TMPDIR as string; + const stateDir = process.env.NEMOCLAW_TEST_STATE_DIR as string; + const relativeStateDir = path.relative(root, stateDir); + + expect(relativeStateDir).toMatch(/^state-\d+-/); + expect(relativeStateDir.startsWith(`..${path.sep}`)).toBe(false); + expect(path.isAbsolute(stateDir)).toBe(true); + expect(fs.statSync(stateDir).mode & 0o777).toBe(0o700); + }); + it("removes run artifacts and restores the caller temp environment", () => { const outerEnv = readTempEnv(); const previousKeep = process.env.NEMOCLAW_TEST_KEEP_TEMP; @@ -209,4 +221,26 @@ describe("Vitest temp root", () => { path.resolve(import.meta.dirname, "..", ROOT_SETUP), ); }); + + // source-shape-contract: security -- Non-live state isolation must never redirect credential-bearing live E2E state + it("isolates stateful non-live projects without redirecting live E2E state", () => { + const projects = (rootVitestConfig.test?.projects ?? []) as Array<{ + test?: { name?: string; setupFiles?: string[] }; + }>; + const setupFilesByProject = new Map( + projects.map((project) => [project.test?.name, project.test?.setupFiles ?? []]), + ); + + for (const name of [ + "cli", + "integration", + "installer-integration", + "package-contract", + "e2e-support", + ]) { + expect(setupFilesByProject.get(name), name).toContain(STATE_SETUP); + } + expect(setupFilesByProject.get("plugin")).not.toContain(STATE_SETUP); + expect(setupFilesByProject.get("e2e-live")).not.toContain(STATE_SETUP); + }); }); diff --git a/vitest.config.ts b/vitest.config.ts index b21fbd483a5..9dcbc2ebbe8 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -74,6 +74,7 @@ const controlledNonLiveEnv = { // intentionally excluded below and keep their own stricter umask handling. See // test/helpers/normalize-fixture-umask.ts (#6448). const fixtureUmaskSetup = "test/helpers/normalize-fixture-umask.ts"; +const isolatedTestStateSetup = "test/helpers/isolate-test-state.ts"; const pluginVitestProject = defineProject(pluginVitestProjectOptions); const integrationProjectScheduling = resolveIntegrationProjectScheduling({ isCi, @@ -107,7 +108,11 @@ export default defineConfig({ alias: canonicalSourceAliases, env: controlledNonLiveEnv, testTimeout: testTimeout(), - setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], + setupFiles: [ + fixtureUmaskSetup, + isolatedTestStateSetup, + "test/helpers/onboard-script-mocks.cjs", + ], include: ["src/**/*.test.ts"], exclude: ["**/node_modules/**", "**/.claude/**"], }, @@ -121,7 +126,11 @@ export default defineConfig({ // Source-backed process fixtures can exceed the unit-test budget // when several coverage shards transpile and spawn them concurrently. testTimeout: testTimeout(15_000), - setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], + setupFiles: [ + fixtureUmaskSetup, + isolatedTestStateSetup, + "test/helpers/onboard-script-mocks.cjs", + ], // Integration fixtures often spawn short Node programs. Coverage // stays serial because concurrent source-loader forks exhaust the // 7 GiB CI runner. The canonical local full suite instead runs this @@ -171,7 +180,7 @@ export default defineConfig({ name: "installer-integration", alias: canonicalSourceAliases, env: controlledNonLiveEnv, - setupFiles: [fixtureUmaskSetup], + setupFiles: [fixtureUmaskSetup, isolatedTestStateSetup], include: [ "test/install-express-prompt.test.ts", "test/install-express-wsl-ollama.test.ts", @@ -202,7 +211,7 @@ export default defineConfig({ name: "package-contract", alias: canonicalSourceAliases, env: controlledNonLiveEnv, - setupFiles: [fixtureUmaskSetup], + setupFiles: [fixtureUmaskSetup, isolatedTestStateSetup], include: ["test/package-contract/**/*.test.ts"], }, }, @@ -217,7 +226,11 @@ export default defineConfig({ alias: canonicalSourceAliases, env: controlledNonLiveEnv, testTimeout: testTimeout(), - setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], + setupFiles: [ + fixtureUmaskSetup, + isolatedTestStateSetup, + "test/helpers/onboard-script-mocks.cjs", + ], include: ["test/e2e/support/**/*.test.ts"], }, }, From c023a4b23aa9e760c495a68ddc000afc83af4f6b Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 3 Aug 2026 09:12:17 -0400 Subject: [PATCH 2/8] test(shields): simplify stale-owner fixture Signed-off-by: Julie Yaunches --- src/lib/state/mcp-lifecycle-lock-acquisition.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts index 71b03bb046f..56459cea218 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts @@ -74,15 +74,16 @@ function writeStaleMainOwner(shieldsTakeoverToken?: string): string { function publishTimerWhenStaleOwnerIsObserved(processToken: string): ReturnType { const realProcessKill = process.kill.bind(process); - const processKill = vi.fn((pid: number, signal?: string | number) => { - if (pid === 2_147_483_647 && signal === 0) { + const processKill = vi + .fn((pid: number, signal?: string | number) => realProcessKill(pid, signal as never)) + .mockImplementationOnce((pid: number, signal?: string | number) => { + expect(pid).toBe(2_147_483_647); + expect(signal).toBe(0); writeTimerMarker(processToken); const error = new Error("stale owner exited") as NodeJS.ErrnoException; error.code = "ESRCH"; throw error; - } - return realProcessKill(pid, signal as never); - }); + }); vi.spyOn(process, "kill").mockImplementation(processKill); return processKill; } From 0f6d8018106c7f2dd4d9363a9a73277b84e35639 Mon Sep 17 00:00:00 2001 From: "J. Yaunches" Date: Mon, 3 Aug 2026 19:37:32 -0400 Subject: [PATCH 3/8] fix(shields): preserve managed MCP policies (#8141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `nemoclaw shields down` replaced the complete live OpenShell policy and dropped generated policy entries for registered Model Context Protocol (MCP) servers. This change reconciles only exact NemoClaw-managed MCP entries during Shields transitions, so a surviving server remains reachable while removed servers stay removed. Stacked on prerequisite #8130, which makes Shields deadline recovery serialize with lifecycle mutations without signaling the lock owner, this focused fix supersedes the MCP portion of #7980. ## Related Issue Fixes #7952 ## Changes - Prove managed MCP policy ownership from exact agreement between the sandbox registry, committed generated-policy record, and live gateway policy. - Save the owned MCP key manifest with the Shields snapshot, remove snapshot-time managed entries during restoration, and overlay only current exact entries. - Fail closed on ambiguous, stale, incomplete, malformed, or legacy ownership during manual transitions. At an expired deadline, omit unproven managed MCP entries and audit the omission instead of extending the Shields-down window. - Preserve current managed MCP entries when building the permissive runtime policy, while rejecting an unreadable or ambiguous live policy. - Clean staged runtime policy files across early failure paths. - Restore the Hermes live regression assertions at the actual failure boundary and around the unrelated server lifecycle. - Document MCP policy reconciliation for manual and automatic restoration. ## Failure Timing and Hermes Upgrade Context The original journey had a hidden Shields lifecycle between the first successful call to server A and the later lifecycle for server B: 1. Run `shields up`. 2. Restart the Hermes gateway. 3. Run `shields down`. 4. Exercise the configuration rollback path. 5. Add and remove B. 6. Call A. Boundary instrumentation recorded in #7952 showed that A remained healthy through Shields up and the gateway restart. It became unusable immediately after Shields down, which dropped A's generated MCP policy. The later failure after B was removed was only where the test noticed the already-broken route; B removal was a misleading correlation. This surfaced during the Hermes upgrade work because new coverage and upgrade repairs landed nearly back-to-back: - #7761 added the Hermes MCP helper containing Shields up, gateway restart, Shields down, and rollback. Its verification collected and imported the live target but did not run the complete live E2E. - #7771 upgraded Hermes the next day, but its selected E2Es skipped the `mcp-bridge` target. - #7849 repaired Hermes 0.19 migrations and updated MCP tool naming, allowing the live test to progress far enough to expose the later failure. - #7866 moved the explicit `mcp restart A` before the first post-removal call. Restart reapplied A's generated policy and masked the missing-policy state. The corrected regression order is: 1. Run `shields up`. 2. Restart the Hermes gateway. 3. Run `shields down`. 4. Call A immediately. 5. Exercise the configuration rollback path. 6. Add B, prove the DNS-rebinding connection is denied, remove B, and verify that A's managed policy is unchanged while B's policy is gone. 7. Call A before the later explicit restart. 8. Capture the authenticated rediscovery offset. 9. Run `mcp restart A` without resupplying the secret. 10. Call A and verify authenticated rediscovery. Whole-policy Shields replacement and the filesystem-only runtime merge predate the Hermes upgrade. This is a latent NemoClaw Shields policy-composition defect detected by expanded Hermes regression coverage, not a Hermes upgrade regression. ## Type of Change - [ ] Code change (feature, bug fix, or refactor) - [x] Code change with doc updates - [ ] Doc only (prose changes, no code sample modifications) - [ ] Doc only (includes code sample changes) ## Quality Gates - [x] Tests added or updated for changed behavior - [ ] Existing tests cover changed behavior — justification: - [ ] Tests not applicable — justification: - [x] Docs updated for user-facing behavior changes - [ ] Docs not applicable — justification: - [x] Sensitive paths changed (security, policy, credentials, preflight, onboarding, inference, runner, sandbox, or messaging) - [x] Sensitive-path review completed or maintainer-approved waiver recorded — reviewer/approval link/justification: Independent exact-head Codex security review passed all nine categories at `18039569796d6ac7604de032edb7abf84f2c73c4`; no findings. - [ ] Non-success, skipped, or missing CI check accepted by maintainer — check name, approval link, and follow-up issue: ## Documentation Writer Review - [x] Documentation writer subagent reviewed the completed changes - Result: `docs-updated` - Evidence: Reviewed `docs/manage-sandboxes/runtime-controls.mdx` and `docs/reference/commands.mdx`, all rendered guide variants, changed operator-facing text, comments, test titles, and the Hermes E2E chronology. Verified claims against source, issue #7952, and PRs #7761, #7771, #7849, and #7866. `npm run docs` completed with 0 errors and 2 existing Fern warnings. - Agent: Codex Desktop ## DGX Station Hardware Evidence - [ ] Tested on DGX Station - Tested commit: Not applicable; `scripts/prepare-dgx-station-host.sh` is unchanged. - Station profile/scenario: Not applicable. - Result: Not applicable. - Supporting evidence: Not applicable. ## Verification - [x] PR description includes a `Signed-off-by:` line and every commit appears as `Verified` in GitHub - [x] Normal `pre-commit`, `commit-msg`, and `pre-push` hooks passed, or `npm run validate:pr` passed after refreshing `origin/main` when hooks were skipped or unavailable - [x] Targeted behavior tests pass for the current change set, or tests are marked not applicable above — command/result or justification: Focused CLI 123/123, integration 11/11, E2E support 13/13, `npm run typecheck:cli`, `npm run checks:repository`, test-size guardrail, E2E semantic phase plans, and serial `npm run test:changed` 674/674 passed. - [x] Applicable broad gate passed — `npm test` for broad runtime/test-harness changes; `npm run check` for repo-wide validation/coverage changes — command/result: [Standard PR CI run 30824992396](https://github.com/NVIDIA/NemoClaw/actions/runs/30824992396) passed. One inherited 50 ms lifecycle-lock assertion timing flake passed on the failed-job rerun without a code change. - [x] Quality Gates section completed with required justifications or waivers - [x] No secrets, API keys, or credentials committed - [ ] `npm run docs` builds without warnings (doc changes only) - [x] Doc pages follow the [style guide](https://github.com/NVIDIA/NemoClaw/blob/main/docs/CONTRIBUTING.md) (doc changes only) - [ ] New doc pages include SPDX header and frontmatter (new pages only) `npm run docs` passed with 0 errors and 2 existing Fern warnings, so the warning-free checkbox remains unchecked. No new documentation pages were added. Trusted E2E [run 30826792180](https://github.com/NVIDIA/NemoClaw/actions/runs/30826792180) passed all 10 selected checks: cloud inference, cloud onboard, security posture, inference routing, MCP bridge, MCP bridge dev, network policy, onboard repair, onboard resume, and OpenShell credential-generation window. The primary review advisor reported no findings. Nemotron completed after retrying a protocol-only failure; its one test warning requested the exact transition/state ownership-mismatch deadline regression already present in `src/lib/shields/policy-transition.test.ts`, which passed. --- Signed-off-by: Julie Yaunches Signed-off-by: Julie Yaunches --- ci/source-architecture-budget.json | 2 +- docs/manage-sandboxes/runtime-controls.mdx | 8 + docs/reference/commands.mdx | 8 + .../checks/openshell-policy-mutation-read.mts | 2 +- src/lib/actions/sandbox/mcp-bridge-policy.ts | 463 ++++++++++++ src/lib/shields/flow.test.ts | 363 +++++++++- src/lib/shields/index.test.ts | 92 +++ src/lib/shields/index.ts | 554 +++++++++++---- src/lib/shields/mcp-policy-transition.test.ts | 666 ++++++++++++++++++ src/lib/shields/mcp-policy-transition.ts | 182 +++++ src/lib/shields/permissive-runtime.ts | 150 +++- src/lib/shields/policy-transition.test.ts | 207 ++++++ src/lib/shields/timer.test.ts | 127 ++-- src/lib/shields/timer.ts | 14 +- test/e2e/live/mcp-bridge-hermes-lifecycle.ts | 1 - test/e2e/live/mcp-bridge-sandbox.ts | 76 ++ test/e2e/live/mcp-bridge.test.ts | 84 ++- test/e2e/support/mcp-bridge-sandbox.test.ts | 66 +- test/permissive-runtime.test.ts | 69 +- 19 files changed, 2864 insertions(+), 270 deletions(-) create mode 100644 src/lib/shields/mcp-policy-transition.test.ts create mode 100644 src/lib/shields/mcp-policy-transition.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 8d1def477b4..825a7032186 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -24,7 +24,7 @@ "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 26, "src/lib/onboard/gateway-binding.ts": 48, - "src/lib/runner.ts": 89, + "src/lib/runner.ts": 88, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 36, "src/lib/state/registry.ts": 101, diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 0a233eb1385..ccfc887339f 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -114,6 +114,14 @@ NemoClaw does not signal that process because portable process inspection cannot After the owner releases the lock, auto-restore restores the restrictive policy and config posture. The ownership check includes both the process ID and process start identity so PID reuse does not grant control over an unrelated process. +Before a manual Shields transition replaces a policy, NemoClaw requires exact Model Context Protocol (MCP) agreement among the sandbox registry, generated-policy record, and live gateway policy. +`shields down` carries the proven managed MCP policy entries into the relaxed policy. +Restoration removes snapshot-time managed MCP entries before it overlays current exact entries. +If exact agreement is absent, a manual Shields transition refuses the replacement policy. +At an expired deadline, auto-restore omits unproven managed MCP policy entries, restores lockdown, and records the omission count in its audit entry. +An MCP server removed during the shields-down window stays removed. +A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. + When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window. The deadline gate remains closed during those attempts. If restoration cannot commit, NemoClaw records durable containment before the command returns an error. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 7d63c6a98ce..ea5a909bc92 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1172,6 +1172,14 @@ Durable containment blocks new mutations until you complete exact-generation ope Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. +Before a manual Shields transition replaces a policy, NemoClaw requires exact agreement among the sandbox registry, generated-policy record, and live gateway policy. +`shields down` carries the proven managed MCP policy entries into the relaxed policy. +Restoration removes snapshot-time managed MCP entries before it overlays current exact entries. +If exact agreement is absent, a manual Shields transition refuses the replacement policy. +At an expired deadline, auto-restore omits unproven managed MCP policy entries, restores lockdown, and records the omission count in its audit entry. +An MCP server removed during the shields-down window stays removed. +A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. + ### `$$nemoclaw recover` diff --git a/scripts/checks/openshell-policy-mutation-read.mts b/scripts/checks/openshell-policy-mutation-read.mts index 133a981dc62..63fc1e5db7b 100644 --- a/scripts/checks/openshell-policy-mutation-read.mts +++ b/scripts/checks/openshell-policy-mutation-read.mts @@ -58,7 +58,7 @@ export const MUTATION_READS: readonly AuditedMutationRead[] = [ }, { relativePath: "src/lib/shields/index.ts", - expectedReadCalls: 1, + expectedReadCalls: 3, baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", unsafeBaseCommand: "runCapture(buildPolicyGetCommand(sandboxName), {", fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName))", diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index 88a5e024655..b951485b690 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -1,8 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isIP } from "node:net"; +import { isDeepStrictEqual } from "node:util"; +import YAML from "yaml"; + import type { AgentMcpAdapter } from "../../agent/defs"; import * as policies from "../../policy"; +import { isBlockedMcpUrlTargetHost } from "../../security/mcp-url-target"; import type { McpBridgeEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { @@ -16,6 +21,7 @@ import { buildMcpBridgePolicyYaml, } from "./mcp-bridge-policy-render"; +export { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; export { buildMcpBridgePolicyKey, buildMcpBridgePolicyName, @@ -24,6 +30,463 @@ export { MCP_BRIDGE_POLICY_MAX_BODY_BYTES, } from "./mcp-bridge-policy-render"; +export interface ExactManagedMcpPolicy { + key: string; + networkPolicy: unknown; + policyName: string; + server: string; +} + +export interface ManagedMcpPolicyOmission { + key?: string; + policyName?: string; + server?: string; + reason: string; +} + +export interface ProvableManagedMcpPolicies { + policies: ExactManagedMcpPolicy[]; + omissions: ManagedMcpPolicyOmission[]; +} + +type ManagedMcpPolicyInspectionDeps = { + getSandbox: typeof registry.getSandbox; +}; + +const managedMcpPolicyInspectionDeps: ManagedMcpPolicyInspectionDeps = { + getSandbox: registry.getSandbox, +}; + +function parseManagedPolicyDocument(source: string, label: string): Record { + let parsed: unknown; + try { + parsed = YAML.parse(source); + } catch { + throw new Error(`${label} is not valid YAML`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`${label} must be a YAML mapping`); + } + return parsed as Record; +} + +function readManagedNetworkPolicies( + document: Record, + label: string, +): Record { + const networkPolicies = document.network_policies; + if (networkPolicies === undefined || networkPolicies === null) return {}; + if (typeof networkPolicies !== "object" || Array.isArray(networkPolicies)) { + throw new Error(`${label} network_policies must be a mapping`); + } + return networkPolicies as Record; +} + +function requireCanonicalAllowedIps(networkPolicy: unknown, policyName: string): readonly string[] { + if (!networkPolicy || typeof networkPolicy !== "object" || Array.isArray(networkPolicy)) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + const endpoints = (networkPolicy as Record).endpoints; + if (!Array.isArray(endpoints) || endpoints.length !== 1) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + const endpoint = endpoints[0]; + if (!endpoint || typeof endpoint !== "object" || Array.isArray(endpoint)) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + const allowedIps = (endpoint as Record).allowed_ips; + if (!Array.isArray(allowedIps) || allowedIps.length === 0) { + throw new Error(`Managed MCP policy '${policyName}' has no exact public address pins`); + } + if ( + allowedIps.some( + (address) => + typeof address !== "string" || + address !== address.toLowerCase() || + address.includes("%") || + isIP(address) === 0 || + isBlockedMcpUrlTargetHost(address), + ) + ) { + throw new Error(`Managed MCP policy '${policyName}' has invalid public address pins`); + } + const pins = allowedIps as string[]; + if (new Set(pins).size !== pins.length || !isDeepStrictEqual(pins, [...pins].sort())) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical public address pins`); + } + return pins; +} + +function resolveCanonicalManagedMcpAdapter( + sandbox: registry.SandboxEntry, + bridge: McpBridgeEntry, +): AgentMcpAdapter { + if (isAgentMcpAdapter(bridge.adapter)) return bridge.adapter; + switch (sandbox.agent || "openclaw") { + case "openclaw": + return "mcporter"; + case "hermes": + return "hermes-config"; + case "langchain-deepagents-code": + return "deepagents-config"; + default: + throw new Error("Managed MCP bridge has no canonical adapter"); + } +} + +function requireCanonicalManagedPolicy( + sandbox: registry.SandboxEntry, + server: string, + livePolicies: Record, +): ExactManagedMcpPolicy { + const bridge = sandbox.mcp?.bridges[server]; + if (!bridge || bridge.addState || bridge.server !== server) { + throw new Error(`Managed MCP bridge '${server}' has an incomplete lifecycle transition`); + } + + const policyName = buildMcpBridgePolicyName(server); + const policyKey = buildMcpBridgePolicyKey(server); + if (bridge.policyName !== policyName) { + throw new Error(`Managed MCP bridge '${server}' has a non-canonical policy name`); + } + + const registrations = (sandbox.customPolicies ?? []).filter( + (policy) => policy.name === policyName, + ); + if (registrations.length !== 1) { + throw new Error( + `Managed MCP bridge '${server}' does not have one exact policy ownership record`, + ); + } + const [registration] = registrations; + if (registration?.sourcePath !== MCP_BRIDGE_POLICY_SOURCE) { + throw new Error(`Managed MCP bridge '${server}' has no NemoClaw-owned policy record`); + } + if (registration.pendingContent !== undefined) { + throw new Error(`Managed MCP bridge '${server}' has an incomplete policy transition`); + } + + const registeredDocument = parseManagedPolicyDocument( + registration.content, + `Managed MCP policy '${policyName}'`, + ); + const preset = registeredDocument.preset; + if ( + !preset || + typeof preset !== "object" || + Array.isArray(preset) || + (preset as Record).name !== policyName + ) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical preset metadata`); + } + const registeredPolicies = readManagedNetworkPolicies( + registeredDocument, + `Managed MCP policy '${policyName}'`, + ); + const registeredKeys = Object.keys(registeredPolicies); + if (registeredKeys.length !== 1 || registeredKeys[0] !== policyKey) { + throw new Error(`Managed MCP policy '${policyName}' has a non-canonical network policy key`); + } + + const registeredNetworkPolicy = registeredPolicies[policyKey]; + const allowedIps = requireCanonicalAllowedIps(registeredNetworkPolicy, policyName); + let expectedDocument: Record; + try { + expectedDocument = parseManagedPolicyDocument( + buildMcpBridgePolicyYaml( + bridge.server, + bridge.url, + resolveCanonicalManagedMcpAdapter(sandbox, bridge), + allowedIps, + ), + `Canonical managed MCP policy '${policyName}'`, + ); + } catch { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + if (!isDeepStrictEqual(registeredDocument, expectedDocument)) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + + if (!Object.hasOwn(livePolicies, policyKey)) { + throw new Error(`Managed MCP policy '${policyName}' is absent from the live gateway policy`); + } + if (!isDeepStrictEqual(livePolicies[policyKey], registeredNetworkPolicy)) { + throw new Error(`Managed MCP policy '${policyName}' has drifted from its ownership record`); + } + + return { + key: policyKey, + networkPolicy: registeredNetworkPolicy, + policyName, + server, + }; +} + +/** + * Resolve the exact generated MCP entries that NemoClaw currently owns. + * + * The registry is an ownership claim, not sufficient authority to overwrite + * the gateway. Every committed bridge must have one canonical, fully + * committed custom-policy record whose sole network entry exactly matches the + * live base policy. + */ +export function inspectExactManagedMcpPolicies( + sandboxName: string, + livePolicyYaml: string, + deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, +): ExactManagedMcpPolicy[] { + const liveDocument = parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"); + const livePolicies = readManagedNetworkPolicies(liveDocument, "Live gateway policy"); + const sandbox = deps.getSandbox(sandboxName); + if (!sandbox) { + const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + ); + } + return []; + } + const generatedRegistrations = (sandbox.customPolicies ?? []).filter( + (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, + ); + if (!sandbox.mcp) { + const orphaned = generatedRegistrations[0]; + if (orphaned) { + throw new Error( + `Generated MCP policy '${orphaned.name}' has no committed managed bridge ownership`, + ); + } + const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + ); + } + return []; + } + if (sandbox.mcp.destroyPreparedAt || sandbox.mcp.destroyPendingAt) { + throw new Error("Managed MCP sandbox destruction is incomplete"); + } + + const bridgeEntries = Object.entries(sandbox.mcp.bridges); + if (bridgeEntries.some(([, bridge]) => bridge.addState !== undefined)) { + throw new Error("A managed MCP bridge lifecycle transition is incomplete"); + } + const exact = bridgeEntries.map(([server]) => + requireCanonicalManagedPolicy(sandbox, server, livePolicies), + ); + + const committedPolicyNames = new Set(exact.map((entry) => entry.policyName)); + const orphaned = generatedRegistrations.find( + (registration) => !committedPolicyNames.has(registration.name), + ); + if (orphaned) { + throw new Error( + `Generated MCP policy '${orphaned.name}' has no committed managed bridge ownership`, + ); + } + + const keys = new Set(); + for (const entry of exact) { + if (keys.has(entry.key)) { + throw new Error(`Managed MCP policy key '${entry.key}' has ambiguous bridge ownership`); + } + keys.add(entry.key); + } + const unclassifiedKey = Object.keys(livePolicies).find( + (key) => key.startsWith("mcp_bridge_") && !keys.has(key), + ); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + ); + } + return exact.sort((left, right) => left.key.localeCompare(right.key)); +} + +/** + * Deadline-only inspection for automatic Shields restoration. + * + * Each entry is admitted independently through the same exact committed/live + * proof as the strict path. Incomplete, drifted, orphaned, or ambiguous claims + * are omitted instead of extending the mutable window; registry state is never + * reconciled or rewritten here. + */ +export function inspectProvableManagedMcpPoliciesForDeadline( + sandboxName: string, + livePolicyYaml: string, + deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, +): ProvableManagedMcpPolicies { + const derivedIdentity = (server: string): { key?: string; policyName?: string } => { + try { + return { + key: buildMcpBridgePolicyKey(server), + policyName: buildMcpBridgePolicyName(server), + }; + } catch { + return {}; + } + }; + const omit = (reason: string, server?: string, policyName?: string): ManagedMcpPolicyOmission => { + const identity = server ? derivedIdentity(server) : {}; + return { + ...(server ? { server } : {}), + ...identity, + ...(policyName ? { policyName } : {}), + reason, + }; + }; + const sandbox = deps.getSandbox(sandboxName); + const generatedRegistrations = (sandbox?.customPolicies ?? []).filter( + (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, + ); + const bridgeEntries = Object.entries(sandbox?.mcp?.bridges ?? {}); + + if (sandbox?.mcp?.destroyPreparedAt || sandbox?.mcp?.destroyPendingAt) { + const reason = "Managed MCP sandbox destruction is incomplete"; + const omissions = bridgeEntries.map(([server]) => omit(reason, server)); + for (const registration of generatedRegistrations) { + if (!omissions.some((entry) => entry.policyName === registration.name)) { + omissions.push(omit(reason, undefined, registration.name)); + } + } + if (omissions.length === 0) omissions.push({ reason }); + return { policies: [], omissions }; + } + + let livePolicies: Record; + try { + livePolicies = readManagedNetworkPolicies( + parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"), + "Live gateway policy", + ); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + const omissions = bridgeEntries.map(([server]) => omit(reason, server)); + for (const registration of generatedRegistrations) { + if (!omissions.some((entry) => entry.policyName === registration.name)) { + omissions.push(omit(reason, undefined, registration.name)); + } + } + return { policies: [], omissions }; + } + + const policies: ExactManagedMcpPolicy[] = []; + const omissions: ManagedMcpPolicyOmission[] = []; + if (!sandbox) { + for (const key of Object.keys(livePolicies).filter((candidate) => + candidate.startsWith("mcp_bridge_"), + )) { + omissions.push({ + key, + reason: `Reserved MCP policy key '${key}' has no committed managed bridge ownership`, + }); + } + return { policies, omissions }; + } + const claimedServersByKey = new Map(); + const claimedServersByPolicyName = new Map(); + for (const [server] of bridgeEntries) { + const identity = derivedIdentity(server); + if (identity.key) { + const servers = claimedServersByKey.get(identity.key) ?? []; + servers.push(server); + claimedServersByKey.set(identity.key, servers); + } + if (identity.policyName) { + const servers = claimedServersByPolicyName.get(identity.policyName) ?? []; + servers.push(server); + claimedServersByPolicyName.set(identity.policyName, servers); + } + } + const ambiguousServers = new Set(); + for (const servers of [...claimedServersByKey.values(), ...claimedServersByPolicyName.values()]) { + if (servers.length <= 1) continue; + for (const server of servers) ambiguousServers.add(server); + } + for (const [server] of bridgeEntries) { + if (ambiguousServers.has(server)) { + omissions.push(omit("Managed MCP policy identity has ambiguous bridge ownership", server)); + continue; + } + try { + policies.push(requireCanonicalManagedPolicy(sandbox, server, livePolicies)); + } catch (error) { + omissions.push(omit(error instanceof Error ? error.message : String(error), server)); + } + } + + const bridgePolicyNames = new Set( + bridgeEntries + .map(([server]) => derivedIdentity(server).policyName) + .filter((name): name is string => name !== undefined), + ); + for (const registration of generatedRegistrations) { + if (!bridgePolicyNames.has(registration.name)) { + omissions.push( + omit( + `Generated MCP policy '${registration.name}' has no committed managed bridge ownership`, + undefined, + registration.name, + ), + ); + } + } + + const policiesByKey = new Map(); + for (const policy of policies) { + const entries = policiesByKey.get(policy.key) ?? []; + entries.push(policy); + policiesByKey.set(policy.key, entries); + } + const exact: ExactManagedMcpPolicy[] = []; + for (const entries of policiesByKey.values()) { + if (entries.length === 1) { + exact.push(entries[0]!); + continue; + } + for (const entry of entries) { + omissions.push( + omit(`Managed MCP policy key '${entry.key}' has ambiguous ownership`, entry.server), + ); + } + } + const exactKeys = new Set(exact.map((entry) => entry.key)); + for (const key of Object.keys(livePolicies).filter( + (candidate) => candidate.startsWith("mcp_bridge_") && !exactKeys.has(candidate), + )) { + if (omissions.some((entry) => entry.key === key)) continue; + omissions.push({ + key, + reason: `Reserved MCP policy key '${key}' has no exact committed managed bridge ownership`, + }); + } + return { + policies: exact.sort((left, right) => left.key.localeCompare(right.key)), + omissions, + }; +} + +export function hasManagedMcpPolicyClaims( + sandboxName: string, + deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, +): boolean { + const sandbox = deps.getSandbox(sandboxName); + if (!sandbox) return false; + return ( + Boolean( + sandbox.mcp && + (Object.keys(sandbox.mcp.bridges).length > 0 || + (sandbox.mcp.managedServerNames?.length ?? 0) > 0 || + sandbox.mcp.destroyPreparedAt || + sandbox.mcp.destroyPendingAt), + ) || + (sandbox.customPolicies ?? []).some((policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE) + ); +} + type GeneratedPolicyRegistrationState = { policy: registry.CustomPolicyEntry; state: "match" | "absent" | "drift" | null; diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 66cc34876ac..4390e7cde09 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -8,14 +8,20 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; +import YAML from "yaml"; +import { buildMcpBridgePolicyYaml } from "../actions/sandbox/mcp-bridge-policy-render"; +import type { SandboxEntry } from "../state/registry"; const requireDist = createRequire(import.meta.url); const shieldsModulePath = "./index.js"; type ShieldsHarness = { + applyShieldsPolicySnapshot: typeof import("./index.js").applyShieldsPolicySnapshot; auditSpy: MockInstance; + cleanupTempDirSpy: MockInstance; errorSpy: MockInstance; logSpy: MockInstance; + policySetBodies: string[]; runSpy: MockInstance; shieldsDown: typeof import("./index.js").shieldsDown; shieldsStatus: typeof import("./index.js").shieldsStatus; @@ -34,6 +40,7 @@ type HarnessOptions = { directSandboxUnavailable?: boolean; dockerExecFileSync?: (argv: unknown) => string; failOpenClawGuardActions?: Array<"lock" | "unlock">; + failStateSave?: boolean; invokedAs?: "nemoclaw" | "nemohermes"; openClawGuardFailure?: { code: string; @@ -52,18 +59,67 @@ type HarnessOptions = { send: () => boolean; kill: () => boolean; }; + livePolicy?: string; run?: (cmd: unknown) => { status: number }; + sandboxEntry?: SandboxEntry; }; +function managedMcpPolicy(server: string, address = "8.8.8.8") { + const content = buildMcpBridgePolicyYaml( + server, + `https://${server}.example.com/mcp`, + "hermes-config", + [address], + ); + const entries = Object.entries(YAML.parse(content).network_policies as Record); + expect(entries, `rendered MCP policies for ${server}`).toHaveLength(1); + const [key, networkPolicy] = entries[0]!; + return { content, key, networkPolicy, server }; +} + +function managedMcpSandbox(policies: Array>): SandboxEntry { + return { + name: "openclaw", + openshellDriver: "docker", + customPolicies: policies.map(({ content, server }) => ({ + name: `mcp-bridge-${server}`, + content, + sourcePath: "generated:nemoclaw-mcp-bridge", + })), + mcp: { + bridges: Object.fromEntries( + policies.map(({ server }) => [ + server, + { + server, + agent: "hermes", + adapter: "hermes-config", + url: `https://${server}.example.com/mcp`, + env: ["MCP_SECRET"], + policyName: `mcp-bridge-${server}`, + addedAt: "2026-07-30T00:00:00.000Z", + }, + ]), + ), + }, + }; +} + function throwHarnessError(error: Error): never { throw error; } +function recordPolicySetBody(policySetBodies: string[], file: unknown): void { + policySetBodies.push(fs.readFileSync(String(file), "utf-8")); +} + function createHarness(options: HarnessOptions = {}): ShieldsHarness { vi.stubEnv("NEMOCLAW_INVOKED_AS", options.invokedAs ?? "nemoclaw"); delete require.cache[requireDist.resolve(shieldsModulePath)]; delete require.cache[requireDist.resolve("./timer-bound-lock.js")]; delete require.cache[requireDist.resolve("./transition-lock.js")]; + delete require.cache[requireDist.resolve("./permissive-runtime.js")]; + delete require.cache[requireDist.resolve("../actions/sandbox/mcp-bridge-policy.js")]; delete require.cache[requireDist.resolve("../sandbox/privileged-exec.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; const lifecycleLock = requireDist( @@ -85,17 +141,24 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { const privilegedExec = requireDist("../sandbox/privileged-exec.js"); const dockerExec = requireDist("../adapters/docker/exec.js"); const audit = requireDist("./audit.js"); + const tempFiles = requireDist("../onboard/temp-files.js"); const childProcess = requireDist("node:child_process"); + const policySetBodies: string[] = []; let openClawPosture: "locked" | "mutable" = "mutable"; vi.spyOn(runner, "validateName").mockImplementation((name: unknown) => String(name)); - vi.spyOn(runner, "runCapture").mockReturnValue("version: 1\nnetwork_policies:\n test: {}\n"); + vi.spyOn(runner, "runCapture").mockReturnValue( + options.livePolicy ?? "version: 1\nnetwork_policies:\n test: {}\n", + ); const runSpy = vi.spyOn(runner, "run").mockImplementation((cmd: unknown) => { return options.run ? options.run(cmd) : { status: 0 }; }); options.fork && vi.spyOn(childProcess, "fork").mockImplementation(options.fork); vi.spyOn(policy, "buildPolicyGetCommand").mockReturnValue(["openshell", "policy", "get"]); - vi.spyOn(policy, "buildPolicySetCommand").mockReturnValue(["openshell", "policy", "set"]); + vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: unknown) => { + recordPolicySetBody(policySetBodies, file); + return ["openshell", "policy", "set"]; + }); vi.spyOn(policy, "parseCurrentPolicy").mockImplementation((raw: unknown) => String(raw)); vi.spyOn(policy, "resolvePermissivePolicyPath").mockReturnValue( path.join(tmpDir, "permissive.yaml"), @@ -108,8 +171,13 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { configPath: "/sandbox/.openclaw/openclaw.json", format: "json", }); - vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "openclaw", openshellDriver: "docker" }); + vi.spyOn(registry, "getSandbox").mockReturnValue( + options.sandboxEntry ?? { name: "openclaw", openshellDriver: "docker" }, + ); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [{ name: "openclaw" }] }); + const permissiveRuntime = requireDist( + "./permissive-runtime.js", + ) as typeof import("./permissive-runtime.js"); const directSandboxUnavailableError = new Error( "No running direct OpenShell sandbox container found for 'openclaw' (driver: docker). Expected a running container named openshell-openclaw or openshell-openclaw-*. Is the sandbox running?", ); @@ -205,15 +273,33 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { : ""; }); const auditSpy = vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined); + const cleanupTempDirSpy = vi.spyOn(tempFiles, "cleanupTempDir"); + const prepareStateSaveFailure = options.failStateSave + ? () => + fs.mkdirSync(path.join(tmpDir, ".nemoclaw", "state", "shields-openclaw.json"), { + recursive: true, + }) + : () => undefined; + const buildRuntimePermissivePolicy = permissiveRuntime.buildRuntimePermissivePolicy; + vi.spyOn(permissiveRuntime, "buildRuntimePermissivePolicy").mockImplementation( + (basePath, deps) => { + const runtimePolicy = buildRuntimePermissivePolicy(basePath, deps); + prepareStateSaveFailure(); + return runtimePolicy; + }, + ); const shields = requireDist(shieldsModulePath); logSpy.mockClear(); errorSpy.mockClear(); auditSpy.mockClear(); return { + applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot, auditSpy, + cleanupTempDirSpy, errorSpy, logSpy, + policySetBodies, runSpy, shieldsDown: shields.shieldsDown, shieldsStatus: shields.shieldsStatus, @@ -301,6 +387,8 @@ describe("shields command flow", () => { delete require.cache[requireDist.resolve(shieldsModulePath)]; delete require.cache[requireDist.resolve("./timer-bound-lock.js")]; delete require.cache[requireDist.resolve("./transition-lock.js")]; + delete require.cache[requireDist.resolve("./permissive-runtime.js")]; + delete require.cache[requireDist.resolve("../actions/sandbox/mcp-bridge-policy.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; }); @@ -331,6 +419,274 @@ describe("shields command flow", () => { ); }); + it("shieldsDown preserves an exact managed MCP policy and records its snapshot key (#7952)", { + timeout: 15_000, + }, () => { + const alpha = managedMcpPolicy("alpha"); + const harness = createHarness({ + livePolicy: YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: alpha.networkPolicy, + }, + }), + sandboxEntry: managedMcpSandbox([alpha]), + }); + + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "managed MCP transition coverage", + skipTimer: true, + throwOnError: true, + }); + + const state = JSON.parse( + fs.readFileSync(path.join(tmpDir, ".nemoclaw", "state", "shields-openclaw.json"), "utf-8"), + ); + expect(state.shieldsManagedMcpPolicyKeys).toEqual(["mcp_bridge_alpha"]); + const applied = YAML.parse(harness.policySetBodies.at(-1)!); + expect(applied.network_policies.mcp_bridge_alpha).toEqual(alpha.networkPolicy); + expect(applied.network_policies).not.toHaveProperty("restrictive_baseline"); + }); + + it("cleans the staged managed MCP policy when timer startup fails", () => { + const alpha = managedMcpPolicy("alpha"); + const harness = createHarness({ + fork: () => { + throw new Error("timer startup failed"); + }, + livePolicy: YAML.stringify({ + version: 1, + network_policies: { [alpha.key]: alpha.networkPolicy }, + }), + sandboxEntry: managedMcpSandbox([alpha]), + }); + + expect(() => + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "cleanup coverage", + throwOnError: true, + }), + ).toThrow("Cannot start auto-restore timer: timer startup failed"); + + expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( + expect.stringContaining("nemoclaw-permissive-runtime"), + "nemoclaw-permissive-runtime", + ); + expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); + const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); + expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); + }); + + it("cleans the staged managed MCP policy when state persistence fails", () => { + const alpha = managedMcpPolicy("alpha"); + const harness = createHarness({ + failStateSave: true, + livePolicy: YAML.stringify({ + version: 1, + network_policies: { [alpha.key]: alpha.networkPolicy }, + }), + sandboxEntry: managedMcpSandbox([alpha]), + }); + + expect(() => + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "cleanup coverage", + skipTimer: true, + throwOnError: true, + }), + ).toThrow(/EISDIR|directory/i); + + expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( + expect.stringContaining("nemoclaw-permissive-runtime"), + "nemoclaw-permissive-runtime", + ); + expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); + const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); + expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); + }); + + it("timer restore uses persisted MCP ownership after its transition marker clears (#7952)", () => { + const alpha = managedMcpPolicy("alpha", "8.8.8.8"); + const beta = managedMcpPolicy("beta", "1.1.1.1"); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "policy-snapshot-managed-restore.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: alpha.networkPolicy, + }, + }), + ); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], + }), + ); + const harness = createHarness({ + livePolicy: YAML.stringify({ + version: 1, + network_policies: { + permissive_baseline: { endpoints: [{ host: "*" }] }, + mcp_bridge_alpha: alpha.networkPolicy, + mcp_bridge_beta: beta.networkPolicy, + }, + }), + sandboxEntry: managedMcpSandbox([alpha, beta]), + }); + + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { + transitionProcessToken: "6".repeat(32), + }); + + expect(result.status).toBe(0); + const restored = YAML.parse(harness.policySetBodies.at(-1)!); + expect(Object.keys(restored.network_policies).sort()).toEqual([ + "mcp_bridge_alpha", + "mcp_bridge_beta", + "restrictive_baseline", + ]); + expect(restored.network_policies.mcp_bridge_beta).toEqual(beta.networkPolicy); + }); + + it("refuses manual restoration when persisted MCP ownership is malformed (#7952)", () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "policy-snapshot-corrupt-state.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["../not-a-managed-key"], + }), + ); + const harness = createHarness(); + + expect(() => harness.applyShieldsPolicySnapshot("openclaw", snapshotPath)).toThrow( + /Saved Shields MCP policy ownership is invalid/, + ); + expect(harness.policySetBodies).toHaveLength(0); + }); + + it("refuses a legacy restore whose persisted state names a different snapshot (#7952)", () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const expectedSnapshotPath = path.join(stateDir, "policy-snapshot-expected.yaml"); + const requestedSnapshotPath = path.join(stateDir, "policy-snapshot-requested.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(expectedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync(requestedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: expectedSnapshotPath, + }), + ); + const harness = createHarness(); + + expect(() => harness.applyShieldsPolicySnapshot("openclaw", requestedSnapshotPath)).toThrow( + /does not match the policy snapshot/, + ); + expect(harness.policySetBodies).toHaveLength(0); + }); + + it("uses token-bound transition ownership when the forward owner dies before state commit (#7952)", () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const processToken = "8".repeat(32); + const snapshotPath = path.join(stateDir, "policy-snapshot-new-cycle.yaml"); + const oldSnapshotPath = path.join(stateDir, "policy-snapshot-old-cycle.yaml"); + const alpha = managedMcpPolicy("alpha", "8.8.8.8"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ + version: 1, + network_policies: { mcp_bridge_alpha: alpha.networkPolicy }, + }), + ); + fs.writeFileSync(oldSnapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: false, + shieldsPolicySnapshotPath: oldSnapshotPath, + shieldsManagedMcpPolicyKeys: [], + }), + ); + fs.writeFileSync( + path.join(stateDir, `shields-transition-openclaw-${processToken}.json`), + JSON.stringify({ + version: 1, + phase: "preparing", + ownerPid: process.pid, + ownerStartIdentity: "forward-owner", + processToken, + sandboxName: "openclaw", + snapshotPath, + managedMcpPolicyKeys: ["mcp_bridge_alpha"], + }), + ); + const harness = createHarness({ + livePolicy: YAML.stringify({ + version: 1, + network_policies: { mcp_bridge_alpha: alpha.networkPolicy }, + }), + sandboxEntry: managedMcpSandbox([alpha]), + }); + + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { + transitionProcessToken: processToken, + }); + + expect(result.status).toBe(0); + expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies.mcp_bridge_alpha).toEqual( + alpha.networkPolicy, + ); + }); + + it("loads 257 managed keys recorded by Shields down (#7952)", { timeout: 15_000 }, () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "policy-snapshot-many-managed-keys.yaml"); + const policies = Array.from({ length: 257 }, (_, index) => managedMcpPolicy(`server${index}`)); + const keys = policies.map(({ key }) => key); + const networkPolicies = Object.fromEntries( + policies.map(({ key, networkPolicy }) => [key, networkPolicy]), + ); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, YAML.stringify({ network_policies: networkPolicies })); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: keys, + }), + ); + const harness = createHarness({ + livePolicy: YAML.stringify({ version: 1, network_policies: networkPolicies }), + sandboxEntry: managedMcpSandbox(policies), + }); + + expect(harness.applyShieldsPolicySnapshot("openclaw", snapshotPath).status).toBe(0); + const applied = YAML.parse(harness.policySetBodies.at(-1)!); + const appliedKeys = Object.keys(applied.network_policies); + expect([...appliedKeys].sort()).toEqual([...keys].sort()); + expect(appliedKeys).toHaveLength(257); + expect(appliedKeys).toContain("mcp_bridge_server256"); + }); + it("binds manual shields-up to the active auto-restore timer generation", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const sandboxName = "openclaw"; @@ -656,6 +1012,7 @@ describe("shields command flow", () => { ownerPid: process.pid, sandboxName: "openclaw", snapshotPath: expect.stringContaining("policy-snapshot-"), + managedMcpPolicyKeys: [], }); expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(true); }); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index ad3bdbbd457..2c2cc8dfade 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -103,6 +103,22 @@ function withDefaultNodeExecFileSync( return defaultNodeExecFileSync(file, argv) || fallback(); } +function throwRegistryPermissionDenied(): never { + throw Object.assign(new Error("registry permission denied"), { code: "EACCES" }); +} + +function readFileWithUnreadableRegistry( + originalReadFileSync: typeof fs.readFileSync, + file: fs.PathOrFileDescriptor, + options?: unknown, +): unknown { + const readers = new Map unknown>([ + [true, throwRegistryPermissionDenied], + [false, () => originalReadFileSync(file, options as never)], + ]); + return readers.get(String(file).endsWith(`${path.sep}sandboxes.json`))!(); +} + function throwProcessNotRunning(): never { throw Object.assign(new Error("not running"), { code: "ESRCH" }); } @@ -116,6 +132,23 @@ function routeProcessKill(pid: number, signal?: string | number): true { return (processActions.get(`${pid}:${signal}`) ?? reportProcessRunning)(); } +function readRuntimePolicyBeforeCleanup( + cleanupDir: string, + readFile: typeof fs.readFileSync, +): string | null { + switch ( + path.basename(cleanupDir).startsWith("nemoclaw-permissive-runtime-") && + fs.existsSync(cleanupDir) + ) { + case false: + return null; + case true: { + const policyFile = fs.readdirSync(cleanupDir).find((name) => name.endsWith(".yaml")); + return policyFile ? readFile(path.join(cleanupDir, policyFile), "utf-8") : null; + } + } +} + beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-test-")); vi.stubEnv("HOME", tmpDir); @@ -496,6 +529,65 @@ describe("shields — unit logic", () => { expect(logSpy).toHaveBeenCalledWith(" Shields: DOWN (temporarily unlocked)"); }); + it("deadline composition removes an unproven MCP add from the restrictive policy", async () => { + const snapshot = + "version: 1\nnetwork_policies:\n restrictive_baseline: {}\n mcp_bridge_beta: {}\n"; + const { composeDeadlineManagedMcpPolicies } = await import("./mcp-policy-transition"); + const composition = composeDeadlineManagedMcpPolicies(snapshot, [], ["mcp_bridge_beta"]); + + expect(composition.yaml).toContain("restrictive_baseline"); + expect(composition.yaml).not.toContain("mcp_bridge_beta"); + }); + + it("deadline restore removes saved MCP keys when the registry cannot be read", async () => { + const sandboxName = "openclaw"; + const processToken = "b".repeat(32); + const snapshotPath = path.join(stateDir(), "policy-snapshot-unreadable-registry.yaml"); + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync( + snapshotPath, + "version: 1\nnetwork_policies:\n restrictive_baseline: {}\n mcp_bridge_alpha: {}\n", + ); + writeState(sandboxName, { + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], + }); + writeMarker(sandboxName, { + pid: 2_147_483_647, + sandboxName, + snapshotPath, + restoreAt: new Date(Date.now() - 1_000).toISOString(), + processToken, + }); + const originalReadFileSync = fs.readFileSync.bind(fs); + vi.spyOn(fs, "readFileSync").mockImplementation((file, options) => { + return readFileWithUnreadableRegistry(originalReadFileSync, file, options) as never; + }); + vi.spyOn(process, "kill").mockImplementation(routeProcessKill); + const originalRmSync = fs.rmSync.bind(fs); + let appliedPolicy = ""; + vi.spyOn(fs, "rmSync").mockImplementation((target, options) => { + const cleanupDir = String(target); + appliedPolicy = + readRuntimePolicyBeforeCleanup(cleanupDir, originalReadFileSync) ?? appliedPolicy; + originalRmSync(target, options); + }); + const { applyShieldsPolicySnapshot } = await loadShieldsModule(); + + const result = applyShieldsPolicySnapshot(sandboxName, snapshotPath, { + transitionProcessToken: processToken, + deadlineAuthoritative: true, + expiredTimerRecovery: true, + }); + + expect(result.managedMcpOmissions).toEqual([ + expect.objectContaining({ reason: expect.stringMatching(/Cannot read config file:/) }), + ]); + expect(appliedPolicy).toContain("restrictive_baseline"); + expect(appliedPolicy).not.toContain("mcp_bridge_alpha"); + }); + it("shieldsStatus warns and stays DOWN when inline recovery fails", async () => { const sandboxName = "openclaw"; const missingSnapshotPath = path.join(stateDir(), "missing-snapshot.yaml"); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 81437c29cc4..975747f785d 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -56,7 +56,13 @@ const { resolveNemoclawStateDir } = require("../state/paths"); const { appendAuditEntry } = require("./audit"); const { resolveAgentConfig } = require("../sandbox/agent-config"); const { + assertLegacyMcpPolicyRestoreSafe, + buildDeadlineRuntimeManagedMcpPolicy, + buildRuntimeManagedMcpPolicy, buildRuntimePermissivePolicy, + hasManagedMcpPolicyClaims, + inspectExactManagedMcpPolicies, + inspectProvableManagedMcpPoliciesForDeadline, }: typeof import("./permissive-runtime") = require("./permissive-runtime"); const { cleanupTempDir } = require("../onboard/temp-files"); const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock"); @@ -102,6 +108,7 @@ const { }: typeof import("./mutable-config-repair") = require("./mutable-config-repair"); type MutableConfigPermsInspection = import("./mutable-config-perms").MutableConfigPermsInspection; type MutableConfigRepairResult = import("./mutable-config-perms").MutableConfigRepairResult; +type ManagedMcpPolicyOmission = import("./permissive-runtime").ManagedMcpPolicyOmission; type TimerMarker = import("./timer-control").TimerMarker; const STATE_DIR = resolveNemoclawStateDir(); const SHIELDS_TRANSITION_POLL_MS = 50; @@ -132,6 +139,8 @@ type ShieldsDownTransition = { processToken: string; sandboxName: string; snapshotPath: string; + /** Exact generated MCP keys owned when snapshotPath was captured. */ + managedMcpPolicyKeys?: string[]; }; const transitionPollBuffer = new Int32Array(new SharedArrayBuffer(4)); @@ -183,10 +192,19 @@ function isShieldsDownTransition(value: unknown): value is ShieldsDownTransition typeof value.processToken === "string" && /^[0-9a-f]{32}$/.test(value.processToken) && typeof value.sandboxName === "string" && - typeof value.snapshotPath === "string" + typeof value.snapshotPath === "string" && + isOptionalManagedMcpPolicyKeys(value.managedMcpPolicyKeys) ); } +function sameManagedMcpPolicyKeys( + left: readonly string[] | undefined, + right: readonly string[] | undefined, +): boolean { + if (left === undefined || right === undefined) return left === right; + return left.length === right.length && left.every((key, index) => key === right[index]); +} + function readShieldsDownTransition( sandboxName: string, processToken: string, @@ -215,7 +233,8 @@ function writeShieldsDownTransition( current.phase !== expectedPhase || current.ownerPid !== transition.ownerPid || current.snapshotPath !== transition.snapshotPath || - current.ownerMcpProcessIdentity !== transition.ownerMcpProcessIdentity + current.ownerMcpProcessIdentity !== transition.ownerMcpProcessIdentity || + !sameManagedMcpPolicyKeys(current.managedMcpPolicyKeys, transition.managedMcpPolicyKeys) ) { throw new Error("Shields-down recovery ownership changed during the transition"); } @@ -322,7 +341,8 @@ function waitForShieldsDownForwardCommit( next.ownerStartIdentity !== observed.ownerStartIdentity || next.ownerMcpProcessIdentity !== observed.ownerMcpProcessIdentity || next.snapshotPath !== observed.snapshotPath || - next.processToken !== observed.processToken + next.processToken !== observed.processToken || + !sameManagedMcpPolicyKeys(next.managedMcpPolicyKeys, observed.managedMcpPolicyKeys) ) { throw new Error("Shields-down recovery ownership changed while waiting for forward commit"); } @@ -681,6 +701,8 @@ interface ShieldsState { shieldsDownReason?: string | null; shieldsDownPolicy?: string | null; shieldsPolicySnapshotPath?: string | null; + /** Exact generated MCP keys owned in the restrictive snapshot. */ + shieldsManagedMcpPolicyKeys?: string[]; chattrApplied?: boolean; // SHA-256 seal of each locked file, captured by `shields up` after the // lock verification passes. `shields status` re-hashes the same files @@ -1147,6 +1169,14 @@ function isOptionalHashMap(value: unknown): value is { [path: string]: string } return true; } +function isOptionalManagedMcpPolicyKeys(value: unknown): value is string[] | undefined { + if (value === undefined) return true; + // Preserve string entries exactly so deadline recovery can strip and audit + // malformed or duplicate ownership without delaying restrictive lockdown. + // Manual restoration validates the same entries strictly during composition. + return Array.isArray(value) && value.every((key) => typeof key === "string"); +} + function isShieldsState(value: unknown): value is ShieldsState { return ( isObjectRecord(value) && @@ -1156,6 +1186,7 @@ function isShieldsState(value: unknown): value is ShieldsState { isOptionalNullableString(value.shieldsDownReason) && isOptionalNullableString(value.shieldsDownPolicy) && isOptionalNullableString(value.shieldsPolicySnapshotPath) && + isOptionalManagedMcpPolicyKeys(value.shieldsManagedMcpPolicyKeys) && isOptionalBoolean(value.chattrApplied) && isOptionalHashMap(value.fileHashes) && isOptionalString(value.updatedAt) @@ -2435,6 +2466,7 @@ function synchronizeAutoRestoreTransition( processToken: string, snapshotPath: string, options: { + expiredTimerRecovery?: boolean; retainTransition?: boolean; assertTakeoverAuthority?: () => void; } = {}, @@ -2455,8 +2487,16 @@ function synchronizeAutoRestoreTransition( // above waits until the forward path has either committed its last weakening // mutation or its owner has died; restore the restrictive snapshot again at // that stable boundary before locking config. - const restoreResult = run(buildPolicySetCommand(transition.snapshotPath, sandboxName), { - ignoreError: true, + const marker = readTimerMarker(sandboxName); + const timerOwnsRecovery = + marker?.pid === process.pid && + marker.processToken === processToken && + marker.snapshotPath === transition.snapshotPath; + const deadlineAuthoritative = timerOwnsRecovery || options.expiredTimerRecovery === true; + const restoreResult = applyShieldsPolicySnapshot(sandboxName, transition.snapshotPath, { + transitionProcessToken: processToken, + ...(deadlineAuthoritative ? { deadlineAuthoritative: true } : {}), + ...(options.expiredTimerRecovery ? { expiredTimerRecovery: true } : {}), }); const status = typeof restoreResult.status === "number" ? restoreResult.status : 1; if (status !== 0) { @@ -2614,6 +2654,199 @@ function lockAgentConfig( }); } +function resolveExactManagedMcpPolicies( + sandboxName: string, + livePolicyYaml?: string, +): ReturnType { + let effectiveLivePolicy = livePolicyYaml; + if (!effectiveLivePolicy) { + let rawPolicy: string; + try { + rawPolicy = runCapture(buildPolicyGetCommand(sandboxName)); + } catch (error) { + throw new Error("Cannot read the live gateway policy for managed MCP reconciliation", { + cause: error, + }); + } + effectiveLivePolicy = parseCurrentPolicy(rawPolicy); + } + if (!effectiveLivePolicy) { + throw new Error("Cannot parse the live gateway policy for managed MCP reconciliation"); + } + return inspectExactManagedMcpPolicies(sandboxName, effectiveLivePolicy); +} + +function resolveProvableManagedMcpPoliciesForDeadline( + sandboxName: string, +): ReturnType { + try { + let effectiveLivePolicy = ""; + try { + effectiveLivePolicy = parseCurrentPolicy(runCapture(buildPolicyGetCommand(sandboxName))); + } catch { + // The tolerant deadline inspector records exact omissions for every claim + // when the live policy cannot be parsed or read. + } + return inspectProvableManagedMcpPoliciesForDeadline(sandboxName, effectiveLivePolicy); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + policies: [], + omissions: [ + { + reason: `Managed MCP registry inspection failed at the auto-restore deadline: ${message}`, + }, + ], + }; + } +} + +/** + * Restore a saved complete policy while reconciling only exact generated MCP + * entries. Snapshot-time keys are removed before currently owned entries are + * overlaid, so changes made during the shields-down window survive both manual + * and timer restoration. + */ +interface ShieldsPolicySnapshotRestoreOptions { + transitionProcessToken?: string; + deadlineAuthoritative?: boolean; + expiredTimerRecovery?: boolean; +} + +type ShieldsPolicySnapshotRestoreResult = ReturnType & { + managedMcpOmissions?: ManagedMcpPolicyOmission[]; +}; + +function applyShieldsPolicySnapshot( + sandboxName: string, + snapshotPath: string, + options: ShieldsPolicySnapshotRestoreOptions = {}, +): ShieldsPolicySnapshotRestoreResult { + const state = loadShieldsState(sandboxName); + let transition: ShieldsDownTransition | null = null; + if (options.transitionProcessToken !== undefined) { + if (!/^[0-9a-f]{32}$/.test(options.transitionProcessToken)) { + throw new Error("Invalid Shields transition recovery token"); + } + transition = readShieldsDownTransition(sandboxName, options.transitionProcessToken); + if ( + !transition && + fs.existsSync(shieldsDownTransitionPath(sandboxName, options.transitionProcessToken)) + ) { + throw new Error("Shields transition recovery authority is invalid"); + } + if (transition && transition.snapshotPath !== snapshotPath) { + throw new Error("Shields transition does not authorize the policy snapshot being restored"); + } + } + if (options.deadlineAuthoritative) { + const marker = readTimerMarker(sandboxName); + const markerMatchesRecovery = + marker?.sandboxName === sandboxName && + marker.snapshotPath === snapshotPath && + marker.processToken === options.transitionProcessToken; + const restoreAtMs = marker ? new Date(marker.restoreAt).getTime() : Number.NaN; + const expiredTimerIsInactive = + options.expiredTimerRecovery === true && + markerMatchesRecovery && + Number.isFinite(restoreAtMs) && + restoreAtMs <= Date.now() && + (!isProcessAlive(marker!.pid) || !verifyTimerMarkerIdentity(marker!).verified); + if ( + options.transitionProcessToken === undefined || + !markerMatchesRecovery || + (marker!.pid !== process.pid && !expiredTimerIsInactive) + ) { + throw new Error("The active auto-restore timer does not authorize deadline restoration"); + } + } + + if (state._isCorrupt && !transition) { + throw new Error( + `Cannot restore a Shields policy while persisted state is corrupt: ${ + state._corruptError ?? "invalid state" + }`, + ); + } + // A preparing transition can outlive its owner before Shields state is + // committed; its token-bound marker is then the recovery authority. + // Every ordinary restore remains bound to the exact persisted snapshot. + if (!transition && state.shieldsPolicySnapshotPath !== snapshotPath) { + throw new Error("Shields state does not match the policy snapshot being restored"); + } + const persistedSnapshotMatches = state.shieldsPolicySnapshotPath === snapshotPath; + const ownershipOmissions: ManagedMcpPolicyOmission[] = []; + if ( + transition?.managedMcpPolicyKeys !== undefined && + persistedSnapshotMatches && + state.shieldsManagedMcpPolicyKeys !== undefined && + !sameManagedMcpPolicyKeys(transition.managedMcpPolicyKeys, state.shieldsManagedMcpPolicyKeys) + ) { + if (!options.deadlineAuthoritative) { + throw new Error("Shields transition ownership does not match persisted policy ownership"); + } + ownershipOmissions.push({ + reason: + "Shields transition ownership did not match persisted policy ownership at the auto-restore deadline", + }); + } + let snapshotManagedPolicyKeys = + transition?.managedMcpPolicyKeys ?? + (persistedSnapshotMatches ? state.shieldsManagedMcpPolicyKeys : undefined); + // Older Shields state has no exact snapshot-time ownership manifest. + // A manual restore preserves raw-snapshot behavior only when neither current + // state nor the snapshot can involve managed MCP. Deadline restoration + // instead strips every reserved key and overlays only independently proven + // current entries so legacy metadata cannot delay restrictive lockdown. + if (snapshotManagedPolicyKeys === undefined) { + if (options.deadlineAuthoritative) { + snapshotManagedPolicyKeys = []; + ownershipOmissions.push({ + reason: + "Legacy Shields state had no managed MCP ownership manifest at the auto-restore deadline", + }); + } else { + assertLegacyMcpPolicyRestoreSafe( + fs.readFileSync(snapshotPath, "utf-8"), + hasManagedMcpPolicyClaims(sandboxName), + ); + return run(buildPolicySetCommand(snapshotPath, sandboxName), { + ignoreError: true, + }); + } + } + let managedMcpOmissions: ManagedMcpPolicyOmission[] = []; + let runtimePolicyPath: string; + if (options.deadlineAuthoritative) { + const inspection = resolveProvableManagedMcpPoliciesForDeadline(sandboxName); + const runtime = buildDeadlineRuntimeManagedMcpPolicy(snapshotPath, { + managedMcpPolicies: inspection.policies, + snapshotManagedPolicyKeys, + readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), + }); + runtimePolicyPath = runtime.path; + managedMcpOmissions = [...ownershipOmissions, ...inspection.omissions, ...runtime.omissions]; + } else { + const managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName); + runtimePolicyPath = buildRuntimeManagedMcpPolicy(snapshotPath, { + managedMcpPolicies, + snapshotManagedPolicyKeys, + readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), + }); + } + const runtimePolicyIsTemp = runtimePolicyPath !== snapshotPath; + try { + const result = run(buildPolicySetCommand(runtimePolicyPath, sandboxName), { + ignoreError: true, + }); + return managedMcpOmissions.length > 0 ? { ...result, managedMcpOmissions } : result; + } finally { + if (runtimePolicyIsTemp) { + cleanupTempDir(runtimePolicyPath, "nemoclaw-permissive-runtime"); + } + } +} + function rollbackShieldsDown( sandboxName: string, target: AgentConfigTarget, @@ -2622,12 +2855,16 @@ function rollbackShieldsDown( cachedProtocol?: HermesShieldsProtocol, ): void { console.error(" Rolling back — restoring policy from snapshot..."); - const rollbackResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { - ignoreError: true, - }); + let rollbackResult: ReturnType | null = null; + try { + rollbackResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(` Warning: Policy restore preparation failed during rollback: ${message}`); + } let rollbackChattrApplied: boolean | null = null; let rollbackFileHashes: { [path: string]: string } | null = null; - if (rollbackResult.status === 0) { + if (rollbackResult?.status === 0) { // Re-confirm after the settle window so a reconciler revert cannot leave // the rolled-back config DRIFTED — same fail-closed treatment as the // auto-restore path. Leaves the hashes null (→ "manual intervention" @@ -2668,6 +2905,7 @@ interface LockdownActivationResult { error?: string; chattrApplied?: boolean; fileHashes?: { [path: string]: string }; + managedMcpOmissions?: ManagedMcpPolicyOmission[]; } function activateLockdownFromSnapshot( @@ -2676,14 +2914,23 @@ function activateLockdownFromSnapshot( allowLegacyHermesProtocol = false, cachedTarget?: AgentConfigTarget, cachedProtocol?: HermesShieldsProtocol, + restoreOptions: ShieldsPolicySnapshotRestoreOptions = {}, ): LockdownActivationResult { if (!snapshotPath || !fs.existsSync(snapshotPath)) { return { ok: false, error: "saved snapshot is missing" }; } - const restoreResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { - ignoreError: true, - }); + let restoreResult: ShieldsPolicySnapshotRestoreResult; + try { + restoreResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath, restoreOptions); + } catch (error) { + return { + ok: false, + error: `policy restore preparation failed: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } const restoreStatus = typeof restoreResult.status === "number" ? restoreResult.status : 1; if (restoreStatus !== 0) { return { @@ -2725,6 +2972,9 @@ function activateLockdownFromSnapshot( ok: true, chattrApplied: relock.lastResult.chattrApplied, fileHashes: relock.lastResult.fileHashes, + ...(restoreResult.managedMcpOmissions + ? { managedMcpOmissions: restoreResult.managedMcpOmissions } + : {}), }; } @@ -2764,6 +3014,7 @@ function recoverExpiredAutoRestoreInline( if (marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken)) { try { synchronizeAutoRestoreTransition(sandboxName, marker.processToken, marker.snapshotPath, { + expiredTimerRecovery: true, retainTransition: true, assertTakeoverAuthority: () => assertTimerMarkerGeneration(sandboxName, marker), }); @@ -2786,6 +3037,15 @@ function recoverExpiredAutoRestoreInline( sandboxName, marker.snapshotPath, marker.allowLegacyHermesProtocol === true, + undefined, + undefined, + marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken) + ? { + transitionProcessToken: marker.processToken, + deadlineAuthoritative: true, + expiredTimerRecovery: true, + } + : {}, ); const nowIso = new Date().toISOString(); if (!activation.ok) { @@ -2826,6 +3086,13 @@ function recoverExpiredAutoRestoreInline( restored_by: "auto_timer", policy_snapshot: marker.snapshotPath, restored_at: nowIso, + ...(activation.managedMcpOmissions?.length + ? { + warning: `Inline auto-restore omitted ${String( + activation.managedMcpOmissions.length, + )} unproven managed MCP policy entries`, + } + : {}), }); return { attempted: true, restored: true }; } @@ -2921,6 +3188,19 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = return failShieldsCommand("Cannot capture current policy", opts.throwOnError); } + let managedMcpPolicies: ReturnType; + try { + managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName, policyYaml); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(` Cannot preserve managed MCP policy state: ${message}`); + return failShieldsCommand( + `Cannot preserve managed MCP policy state: ${message}`, + opts.throwOnError, + ); + } + const snapshotManagedMcpPolicyKeys = managedMcpPolicies.map((policy) => policy.key); + const ts = Date.now(); const snapshotPath = path.join(STATE_DIR, `policy-snapshot-${ts}.yaml`); fs.mkdirSync(STATE_DIR, { recursive: true, mode: 0o700 }); @@ -2930,136 +3210,153 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = // 2. Determine and apply relaxed policy let policyFile: string; let policyFileIsTemp = false; - if (policyName === "permissive") { - const basePath = resolvePermissivePolicyPath(sandboxName); - // Union the live sandbox's filesystem_policy.read_only/read_write into - // the static permissive baseline. OpenShell rejects removal of those - // paths on a live sandbox, and runtime-injected entries (/proc on - // GPU, /opt/hermes on Hermes, /home/linuxbrew on post-#3913 OpenClaw, - // etc.) are not present in the static YAML. See #3942, #3957, #3168. - // policyYaml is the pre-parsed body we already captured for the - // snapshot above — reuse it instead of re-fetching. - policyFile = buildRuntimePermissivePolicy(basePath, { - livePolicyYaml: policyYaml, - readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), - }); - policyFileIsTemp = policyFile !== basePath; - } else if (fs.existsSync(policyName)) { - policyFile = path.resolve(policyName); - } else { - console.error(` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`); - return failShieldsCommand(`Unknown policy "${policyName}"`, opts.throwOnError); + try { + if (policyName === "permissive") { + const basePath = resolvePermissivePolicyPath(sandboxName); + // Union the live sandbox's filesystem_policy.read_only/read_write into + // the static permissive baseline. OpenShell rejects removal of those + // paths on a live sandbox, and runtime-injected entries (/proc on + // GPU, /opt/hermes on Hermes, /home/linuxbrew on post-#3913 OpenClaw, + // etc.) are not present in the static YAML. See #3942, #3957, #3168. + // policyYaml is the pre-parsed body we already captured for the + // snapshot above — reuse it instead of re-fetching. Exact generated MCP + // entries are overlaid without copying any unrelated live egress. + policyFile = buildRuntimePermissivePolicy(basePath, { + livePolicyYaml: policyYaml, + managedMcpPolicies, + readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), + }); + policyFileIsTemp = policyFile !== basePath; + } else if (fs.existsSync(policyName)) { + const basePath = path.resolve(policyName); + policyFile = buildRuntimeManagedMcpPolicy(basePath, { + managedMcpPolicies, + readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), + }); + policyFileIsTemp = policyFile !== basePath; + } else { + console.error(` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`); + fs.rmSync(snapshotPath, { force: true }); + return failShieldsCommand(`Unknown policy "${policyName}"`, opts.throwOnError); + } + } catch (error) { + fs.rmSync(snapshotPath, { force: true }); + const message = error instanceof Error ? error.message : String(error); + console.error(` Cannot compose Shields-down policy: ${message}`); + return failShieldsCommand(`Cannot compose Shields-down policy: ${message}`, opts.throwOnError); } const now = new Date().toISOString(); let transition: ShieldsDownTransition | null = null; - // Commit the host-side recovery authority before weakening policy or file - // permissions. If this process is killed later, the detached timer and its - // marker already exist and the persisted state honestly reports shields - // down. A crash can therefore never leave an untracked mutable window. - if (!opts.skipTimer) { - const restoreAt = new Date(Date.now() + timeoutSeconds * 1000); - const processToken = opts.processToken ?? randomBytes(16).toString("hex"); - if (!/^[0-9a-f]{32}$/.test(processToken)) { - throw new Error("Invalid shields-down recovery process token"); - } - const timerScript = path.join(__dirname, "timer.ts"); - const timerScriptJs = timerScript.replace(/\.ts$/, ".js"); - const actualScript = fs.existsSync(timerScriptJs) ? timerScriptJs : timerScript; - transition = { - version: 1, - phase: "preparing", - ownerPid: process.pid, - ownerStartIdentity: - readProcessStartIdentity(process.pid) ?? - (() => { - throw new Error("Cannot identify shields-down owner process"); - })(), - ownerMcpProcessIdentity: - readMcpLockProcessIdentity(process.pid, true) ?? - (() => { - throw new Error("Cannot identify shields-down lifecycle owner process"); - })(), - processToken, - sandboxName, - snapshotPath, - }; - const leaseOwnerPid = opts.deferAutoRestoreWhileOwnerAlive ? transition.ownerPid : null; - const leaseOwnerStartIdentity = opts.deferAutoRestoreWhileOwnerAlive - ? transition.ownerStartIdentity - : null; - let timerChild: ReturnType | null = null; + try { + // Commit the host-side recovery authority before weakening policy or file + // permissions. If this process is killed later, the detached timer and its + // marker already exist and the persisted state honestly reports shields + // down. A crash can therefore never leave an untracked mutable window. + if (!opts.skipTimer) { + const restoreAt = new Date(Date.now() + timeoutSeconds * 1000); + const processToken = opts.processToken ?? randomBytes(16).toString("hex"); + if (!/^[0-9a-f]{32}$/.test(processToken)) { + throw new Error("Invalid shields-down recovery process token"); + } + const timerScript = path.join(__dirname, "timer.ts"); + const timerScriptJs = timerScript.replace(/\.ts$/, ".js"); + const actualScript = fs.existsSync(timerScriptJs) ? timerScriptJs : timerScript; + transition = { + version: 1, + phase: "preparing", + ownerPid: process.pid, + ownerStartIdentity: + readProcessStartIdentity(process.pid) ?? + (() => { + throw new Error("Cannot identify shields-down owner process"); + })(), + ownerMcpProcessIdentity: + readMcpLockProcessIdentity(process.pid, true) ?? + (() => { + throw new Error("Cannot identify shields-down lifecycle owner process"); + })(), + processToken, + sandboxName, + snapshotPath, + managedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, + }; + const leaseOwnerPid = opts.deferAutoRestoreWhileOwnerAlive ? transition.ownerPid : null; + const leaseOwnerStartIdentity = opts.deferAutoRestoreWhileOwnerAlive + ? transition.ownerStartIdentity + : null; + let timerChild: ReturnType | null = null; - try { - // Publish the forward-transition ownership marker before authorizing the - // timer. If the timeout expires while this command is still weakening - // policy/config, the timer waits for phase=active or owner death instead - // of racing the forward mutations. - writeShieldsDownTransition(transition, null); - timerChild = fork( - actualScript, - [ + try { + // Publish the forward-transition ownership marker before authorizing the + // timer. If the timeout expires while this command is still weakening + // policy/config, the timer waits for phase=active or owner death instead + // of racing the forward mutations. + writeShieldsDownTransition(transition, null); + timerChild = fork( + actualScript, + [ + sandboxName, + snapshotPath, + restoreAt.toISOString(), + target.configPath, + target.configDir, + processToken, + opts.allowLegacyHermesProtocol === true ? "1" : "0", + leaseOwnerPid === null ? "" : String(leaseOwnerPid), + leaseOwnerStartIdentity ?? "", + ], + { + detached: true, + stdio: ["ignore", "ignore", "ignore", "ipc"], + }, + ); + if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); + writeTimerMarkerAtomic(sandboxName, { + pid: timerChild.pid, sandboxName, snapshotPath, - restoreAt.toISOString(), - target.configPath, - target.configDir, + restoreAt: restoreAt.toISOString(), processToken, - opts.allowLegacyHermesProtocol === true ? "1" : "0", - leaseOwnerPid === null ? "" : String(leaseOwnerPid), - leaseOwnerStartIdentity ?? "", - ], - { - detached: true, - stdio: ["ignore", "ignore", "ignore", "ipc"], - }, - ); - if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); - writeTimerMarkerAtomic(sandboxName, { - pid: timerChild.pid, - sandboxName, - snapshotPath, - restoreAt: restoreAt.toISOString(), - processToken, - allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, - ...(leaseOwnerPid !== null && leaseOwnerStartIdentity - ? { leaseOwnerPid, leaseOwnerStartIdentity } - : {}), - }); - if (!timerChild.send({ type: "authorize", processToken })) { - throw new Error("auto-restore timer authorization channel closed early"); + allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, + ...(leaseOwnerPid !== null && leaseOwnerStartIdentity + ? { leaseOwnerPid, leaseOwnerStartIdentity } + : {}), + }); + if (!timerChild.send({ type: "authorize", processToken })) { + throw new Error("auto-restore timer authorization channel closed early"); + } + timerChild.disconnect(); + timerChild.unref(); + } catch (err) { + clearTimerMarker(sandboxName); + clearShieldsDownTransition(sandboxName, processToken); + const message = err instanceof Error ? err.message : String(err); + console.error(` Cannot start auto-restore timer: ${message}`); + return failShieldsCommand(`Cannot start auto-restore timer: ${message}`, opts.throwOnError); } - timerChild.disconnect(); - timerChild.unref(); - } catch (err) { - clearTimerMarker(sandboxName); - clearShieldsDownTransition(sandboxName, processToken); - const message = err instanceof Error ? err.message : String(err); - console.error(` Cannot start auto-restore timer: ${message}`); - return failShieldsCommand(`Cannot start auto-restore timer: ${message}`, opts.throwOnError); } - } - try { - saveShieldsState(sandboxName, { - shieldsDown: true, - shieldsDownAt: now, - shieldsDownTimeout: timeoutSeconds, - shieldsDownReason: reason, - shieldsDownPolicy: policyName, - shieldsPolicySnapshotPath: snapshotPath, - }); - } catch (error) { - if (transition) { - clearShieldsDownTransition(sandboxName, transition.processToken); - killTimer(sandboxName); + try { + saveShieldsState(sandboxName, { + shieldsDown: true, + shieldsDownAt: now, + shieldsDownTimeout: timeoutSeconds, + shieldsDownReason: reason, + shieldsDownPolicy: policyName, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, + }); + } catch (error) { + if (transition) { + clearShieldsDownTransition(sandboxName, transition.processToken); + killTimer(sandboxName); + } + throw error; } - throw error; - } - console.log(` Applying ${policyName} policy...`); - try { + console.log(` Applying ${policyName} policy...`); run(buildPolicySetCommand(policyFile, sandboxName)); } finally { if (policyFileIsTemp) { @@ -3676,6 +3973,7 @@ function clearShieldsState(sandboxName: string): void { // --------------------------------------------------------------------------- export { + applyShieldsPolicySnapshot, clearShieldsState, completeAutoRestoreTransition, DEFAULT_TIMEOUT_SECONDS, diff --git a/src/lib/shields/mcp-policy-transition.test.ts b/src/lib/shields/mcp-policy-transition.test.ts new file mode 100644 index 00000000000..181a97bbac3 --- /dev/null +++ b/src/lib/shields/mcp-policy-transition.test.ts @@ -0,0 +1,666 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { + hasManagedMcpPolicyClaims, + inspectProvableManagedMcpPoliciesForDeadline, + inspectExactManagedMcpPolicies as inspectRegisteredManagedMcpPolicies, + MCP_BRIDGE_POLICY_SOURCE, +} from "../actions/sandbox/mcp-bridge-policy"; +import { + buildMcpBridgePolicyKey, + buildMcpBridgePolicyName, + buildMcpBridgePolicyYaml, +} from "../actions/sandbox/mcp-bridge-policy-render"; +import type { SandboxEntry } from "../state/registry"; +import { + assertLegacyMcpPolicyRestoreSafe, + composeDeadlineManagedMcpPolicies, + composeManagedMcpPolicies, +} from "./mcp-policy-transition"; + +const ADAPTER = "hermes-config"; + +function registeredPolicy( + server: string, + address: string, +): NonNullable[number] { + return { + name: buildMcpBridgePolicyName(server), + content: buildMcpBridgePolicyYaml(server, `https://${server}.example.com/mcp`, ADAPTER, [ + address, + ]), + sourcePath: MCP_BRIDGE_POLICY_SOURCE, + }; +} + +function bridge(server: string): NonNullable["bridges"]>[string] { + return { + server, + agent: "hermes", + adapter: ADAPTER, + url: `https://${server}.example.com/mcp`, + env: ["MCP_SECRET"], + providerName: `sandbox-mcp-${server}`, + providerId: `provider-${server}`, + policyName: buildMcpBridgePolicyName(server), + addedAt: "2026-07-30T00:00:00.000Z", + }; +} + +function sandboxWithPolicies( + policies: Array>, + bridgeServers = policies.map((policy) => policy.name.replace(/^mcp-bridge-/, "")), +): SandboxEntry { + return { + name: "alpha", + agent: "hermes", + customPolicies: policies, + mcp: { + bridges: Object.fromEntries(bridgeServers.map((server) => [server, bridge(server)])), + }, + }; +} + +function networkEntry(content: string, server: string): unknown { + return YAML.parse(content).network_policies[buildMcpBridgePolicyKey(server)]; +} + +function mutateRegisteredNetworkPolicy( + policy: ReturnType, + server: string, + mutate: (entry: Record) => void, +): void { + const document = YAML.parse(policy.content) as { + network_policies: Record>; + }; + mutate(document.network_policies[buildMcpBridgePolicyKey(server)]!); + policy.content = YAML.stringify(document); +} + +function livePolicy( + entries: Array<{ content: string; server: string }>, + extra: Record = {}, +): string { + return YAML.stringify({ + version: 1, + network_policies: { + ...extra, + ...Object.fromEntries( + entries.map(({ content, server }) => [ + buildMcpBridgePolicyKey(server), + networkEntry(content, server), + ]), + ), + }, + }); +} + +function inspectExactManagedMcpPolicies(sandbox: SandboxEntry, livePolicyYaml: string) { + return inspectRegisteredManagedMcpPolicies("alpha", livePolicyYaml, { + getSandbox: () => sandbox, + }); +} + +describe("managed MCP Shields policy transitions (#7952)", () => { + it("admits only canonical committed registrations that exactly match the live policy", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox = sandboxWithPolicies([alpha]); + + const exact = inspectExactManagedMcpPolicies( + sandbox, + livePolicy([{ content: alpha.content, server: "alpha" }], { + unrelated_live_entry: { endpoints: [{ host: "unrelated.example.com" }] }, + }), + ); + + expect(exact).toEqual([ + expect.objectContaining({ + key: "mcp_bridge_alpha", + policyName: "mcp-bridge-alpha", + server: "alpha", + }), + ]); + }); + + it.each([ + { + label: "pending policy content", + mutate: (sandbox: SandboxEntry) => { + sandbox.customPolicies![0]!.pendingContent = sandbox.customPolicies![0]!.content; + }, + expected: /incomplete policy transition/, + }, + { + label: "an orphaned generated registration", + mutate: (sandbox: SandboxEntry) => { + sandbox.customPolicies!.push(registeredPolicy("orphan", "1.1.1.1")); + }, + expected: /no committed managed bridge ownership/, + }, + { + label: "an incomplete bridge add", + mutate: (sandbox: SandboxEntry) => { + sandbox.mcp!.bridges.alpha!.addState = "prepared"; + }, + expected: /lifecycle transition is incomplete/, + }, + ])("fails closed on $label", ({ mutate, expected }) => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox = sandboxWithPolicies([alpha]); + mutate(sandbox); + + expect(() => + inspectExactManagedMcpPolicies( + sandbox, + livePolicy( + (sandbox.customPolicies ?? []).map((policy) => ({ + content: policy.content, + server: policy.name.replace(/^mcp-bridge-/, ""), + })), + ), + ), + ).toThrow(expected); + }); + + it("fails closed when the live policy differs from the ownership record", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const drifted = registeredPolicy("alpha", "1.1.1.1"); + + expect(() => + inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha]), + livePolicy([{ content: drifted.content, server: "alpha" }]), + ), + ).toThrow(/drifted from its ownership record/); + }); + + it("rejects matching registry and live documents with weakened generated semantics", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + mutateRegisteredNetworkPolicy(alpha, "alpha", (entry) => { + const endpoint = (entry.endpoints as Array>)[0]!; + endpoint.enforcement = "observe"; + }); + const sandbox = sandboxWithPolicies([alpha]); + const live = livePolicy([{ content: alpha.content, server: "alpha" }]); + + expect(() => inspectExactManagedMcpPolicies(sandbox, live)).toThrow( + /non-canonical generated content/, + ); + expect( + inspectProvableManagedMcpPoliciesForDeadline("alpha", live, { + getSandbox: () => sandbox, + }), + ).toEqual({ + policies: [], + omissions: [ + expect.objectContaining({ + server: "alpha", + reason: expect.stringMatching(/non-canonical generated content/), + }), + ], + }); + }); + + it.each([ + { + label: "a private literal", + pins: ["127.0.0.1"], + expected: /invalid public address pins/, + }, + { + label: "a scoped public IPv6 literal", + pins: ["2001:4860:4860::8888%lo0"], + expected: /invalid public address pins/, + }, + { + label: "duplicate literals", + pins: ["8.8.8.8", "8.8.8.8"], + expected: /non-canonical public address pins/, + }, + { + label: "unsorted literals", + pins: ["8.8.8.8", "1.1.1.1"], + expected: /non-canonical public address pins/, + }, + ])("rejects matching registry and live documents with $label", ({ pins, expected }) => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + mutateRegisteredNetworkPolicy(alpha, "alpha", (entry) => { + const endpoint = (entry.endpoints as Array>)[0]!; + endpoint.allowed_ips = pins; + }); + + expect(() => + inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha]), + livePolicy([{ content: alpha.content, server: "alpha" }]), + ), + ).toThrow(expected); + }); + + it("fails closed on a generated policy record without managed MCP state", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox: SandboxEntry = { + name: "alpha", + agent: "hermes", + customPolicies: [alpha], + }; + const deps = { getSandbox: () => sandbox }; + + expect(hasManagedMcpPolicyClaims("alpha", deps)).toBe(true); + expect(() => + inspectRegisteredManagedMcpPolicies( + "alpha", + livePolicy([{ content: alpha.content, server: "alpha" }]), + deps, + ), + ).toThrow(/no committed managed bridge ownership/); + }); + + it("treats residual managed server history as an ownership claim", () => { + const sandbox: SandboxEntry = { + name: "alpha", + agent: "hermes", + mcp: { bridges: {}, managedServerNames: ["retired"] }, + }; + const deps = { getSandbox: () => sandbox }; + + expect(hasManagedMcpPolicyClaims("alpha", deps)).toBe(true); + expect( + inspectRegisteredManagedMcpPolicies( + "alpha", + livePolicy([], { unrelated_live_entry: {} }), + deps, + ), + ).toEqual([]); + }); + + it.each([ + { + label: "no sandbox registry entry", + sandbox: undefined, + }, + { + label: "only residual ownership history", + sandbox: { + name: "alpha", + agent: "hermes", + mcp: { bridges: {}, managedServerNames: ["retired"] }, + } satisfies SandboxEntry, + }, + ])("rejects an unclassified reserved live key with $label", ({ sandbox }) => { + expect(() => + inspectRegisteredManagedMcpPolicies("alpha", livePolicy([], { mcp_bridge_retired: {} }), { + getSandbox: () => sandbox ?? null, + }), + ).toThrow( + /Reserved MCP policy key 'mcp_bridge_retired'.*no committed managed bridge ownership/, + ); + }); + + it("retains additions while restoring the restrictive snapshot", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha, beta]), + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: beta.content, server: "beta" }, + ]), + ); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), + }, + }); + + const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"])); + + expect(Object.keys(restored.network_policies).sort()).toEqual([ + "mcp_bridge_alpha", + "mcp_bridge_beta", + "restrictive_baseline", + ]); + }); + + it("does not restore a managed MCP policy removed during the shields-down window", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), + }, + }); + + const restored = YAML.parse(composeManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"])); + + expect(restored.network_policies).toEqual({ + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }); + }); + + it("replaces a stale snapshot entry with the current exact registration", () => { + const oldAlpha = registeredPolicy("alpha", "8.8.8.8"); + const currentAlpha = registeredPolicy("alpha", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([currentAlpha]), + livePolicy([{ content: currentAlpha.content, server: "alpha" }]), + ); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_alpha: networkEntry(oldAlpha.content, "alpha"), + }, + }); + + const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"])); + + expect(restored.network_policies.mcp_bridge_alpha).toEqual( + networkEntry(currentAlpha.content, "alpha"), + ); + }); + + it("rejects an unclassified reserved key in the restrictive snapshot", () => { + const currentAlpha = registeredPolicy("alpha", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([currentAlpha]), + livePolicy([{ content: currentAlpha.content, server: "alpha" }]), + ); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_alpha: { + name: "operator-owned-alpha", + endpoints: [{ host: "operator.example.com" }], + }, + }, + }); + + expect(() => composeManagedMcpPolicies(snapshot, current, [])).toThrow( + /Reserved MCP policy key 'mcp_bridge_alpha'.*absent from the saved ownership manifest/, + ); + }); + + it("accepts an empty ownership manifest when the snapshot has no reserved keys", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { restrictive_baseline: {} }, + }); + + expect(YAML.parse(composeManagedMcpPolicies(snapshot, [], [])).network_policies).toEqual({ + restrictive_baseline: {}, + }); + }); + + it("rejects a saved managed key that is absent from its policy snapshot", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }, + }); + + expect(() => composeManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"])).toThrow( + /absent from its policy snapshot/, + ); + }); + + it.each([ + { + label: "current managed MCP ownership", + hasCurrentManagedClaims: true, + networkPolicies: { restrictive_baseline: {} }, + }, + { + label: "a managed-shaped key in the snapshot", + hasCurrentManagedClaims: false, + networkPolicies: { mcp_bridge_alpha: {} }, + }, + ])("refuses legacy restore with $label", ({ hasCurrentManagedClaims, networkPolicies }) => { + expect(() => + assertLegacyMcpPolicyRestoreSafe( + YAML.stringify({ version: 1, network_policies: networkPolicies }), + hasCurrentManagedClaims, + ), + ).toThrow(/no managed MCP ownership manifest/); + }); + + it("allows a legacy restore with no current or snapshot MCP ownership", () => { + expect(() => + assertLegacyMcpPolicyRestoreSafe( + YAML.stringify({ + version: 1, + network_policies: { restrictive_baseline: {} }, + }), + false, + ), + ).not.toThrow(); + }); + + it("proves committed bridges independently while omitting an incomplete add at the deadline", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const sandbox = sandboxWithPolicies([alpha, beta]); + sandbox.mcp!.bridges.beta!.addState = "prepared"; + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: beta.content, server: "beta" }, + ]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies.map((policy) => policy.server)).toEqual(["alpha"]); + expect(result.omissions).toEqual([ + expect.objectContaining({ server: "beta", reason: expect.stringMatching(/incomplete/) }), + ]); + }); + + it("omits every deadline claimant whose canonical policy identity collides", () => { + const collidingPolicy = registeredPolicy("foo-bar", "8.8.8.8"); + const sandbox = sandboxWithPolicies([collidingPolicy], ["foo-bar", "foo_bar"]); + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([{ content: collidingPolicy.content, server: "foo-bar" }]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies).toEqual([]); + expect(result.omissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + server: "foo-bar", + reason: expect.stringMatching(/ambiguous bridge ownership/), + }), + expect.objectContaining({ + server: "foo_bar", + reason: expect.stringMatching(/ambiguous bridge ownership/), + }), + ]), + ); + }); + + it.each([ + "destroyPreparedAt", + "destroyPendingAt", + ] as const)("omits every generated policy while %s is present", (marker) => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox = sandboxWithPolicies([alpha]); + sandbox.mcp![marker] = "2026-07-30T01:00:00.000Z"; + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([{ content: alpha.content, server: "alpha" }]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies).toEqual([]); + expect(result.omissions).toEqual([ + expect.objectContaining({ server: "alpha", reason: expect.stringMatching(/destruction/) }), + ]); + }); + + it("omits drift and orphan claims without discarding another exact bridge", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const driftedBeta = registeredPolicy("beta", "9.9.9.9"); + const orphan = registeredPolicy("orphan", "4.4.4.4"); + const sandbox = sandboxWithPolicies([alpha, beta, orphan], ["alpha", "beta"]); + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: driftedBeta.content, server: "beta" }, + { content: orphan.content, server: "orphan" }, + ]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies.map((policy) => policy.server)).toEqual(["alpha"]); + expect(result.omissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ server: "beta", reason: expect.stringMatching(/drifted/) }), + expect.objectContaining({ + policyName: "mcp-bridge-orphan", + reason: expect.stringMatching(/no committed managed bridge ownership/), + }), + ]), + ); + }); + + it("deadline inspection reports an unclassified reserved live key", () => { + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([], { mcp_bridge_residual: {} }), + { getSandbox: () => null }, + ); + + expect(result).toEqual({ + policies: [], + omissions: [ + expect.objectContaining({ + key: "mcp_bridge_residual", + reason: expect.stringMatching(/no committed managed bridge ownership/), + }), + ], + }); + }); + + it("deadline composition strips unclassified reserved keys before overlaying proven entries", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha, beta]), + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: beta.content, server: "beta" }, + ]), + ); + const operatorEntry = { endpoints: [{ host: "operator.example.com" }] }; + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), + mcp_bridge_beta: operatorEntry, + restrictive_baseline: {}, + }, + }); + + const result = composeDeadlineManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"]); + const restored = YAML.parse(result.yaml); + + expect(restored.network_policies.mcp_bridge_alpha).toEqual( + networkEntry(alpha.content, "alpha"), + ); + expect(restored.network_policies.mcp_bridge_beta).toEqual(networkEntry(beta.content, "beta")); + expect(result.omissions).toEqual([ + expect.objectContaining({ + key: "mcp_bridge_beta", + reason: expect.stringMatching(/absent from the saved ownership manifest/), + }), + ]); + }); + + it("deadline composition strips every reserved shape with an empty manifest", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_: {}, + mcp_bridge_legacy_invalid_name: {}, + restrictive_baseline: {}, + }, + }); + + const result = composeDeadlineManagedMcpPolicies(snapshot, [], []); + + expect(YAML.parse(result.yaml).network_policies).toEqual({ restrictive_baseline: {} }); + expect(result.omissions.map((entry) => entry.key)).toEqual([ + "mcp_bridge_", + "mcp_bridge_legacy_invalid_name", + ]); + }); + + it("deadline composition omits malformed and duplicate manifest entries without delaying lockdown", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_: {}, + mcp_bridge_alpha: {}, + restrictive_baseline: {}, + }, + }); + + const result = composeDeadlineManagedMcpPolicies( + snapshot, + [], + ["mcp_bridge_", "restrictive_baseline", "mcp_bridge_alpha", "mcp_bridge_alpha"], + ); + + expect(YAML.parse(result.yaml).network_policies).toEqual({ restrictive_baseline: {} }); + expect(result.omissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: "mcp_bridge_", + reason: expect.stringMatching(/ownership key.*invalid/), + }), + expect.objectContaining({ + key: "restrictive_baseline", + reason: expect.stringMatching(/ownership key.*invalid/), + }), + expect.objectContaining({ + key: "mcp_bridge_alpha", + reason: expect.stringMatching(/more than once/), + }), + ]), + ); + }); + + it("deadline composition restores the restrictive baseline when a saved key is absent", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }, + }); + + const result = composeDeadlineManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"]); + const restored = YAML.parse(result.yaml); + + expect(restored.network_policies).toEqual({ + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }); + expect(result.omissions).toEqual([ + expect.objectContaining({ reason: expect.stringMatching(/already absent/) }), + ]); + }); +}); diff --git a/src/lib/shields/mcp-policy-transition.ts b/src/lib/shields/mcp-policy-transition.ts new file mode 100644 index 00000000000..858536bb790 --- /dev/null +++ b/src/lib/shields/mcp-policy-transition.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import YAML from "yaml"; + +import type { + ExactManagedMcpPolicy, + ManagedMcpPolicyOmission, +} from "../actions/sandbox/mcp-bridge-policy"; + +const CANONICAL_MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_[a-z][a-z0-9_]{0,63}$/; +const RESERVED_MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_/; + +function parsePolicyDocument(source: string, label: string): Record { + let parsed: unknown; + try { + parsed = YAML.parse(source); + } catch { + throw new Error(`${label} is not valid YAML`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`${label} must be a YAML mapping`); + } + return parsed as Record; +} + +function readNetworkPolicies( + document: Record, + label: string, +): Record { + const policies = document.network_policies; + if (policies === undefined || policies === null) return {}; + if (typeof policies !== "object" || Array.isArray(policies)) { + throw new Error(`${label} network_policies must be a mapping`); + } + return policies as Record; +} + +/** + * Reconcile generated MCP entries into a complete target policy. + * + * Snapshot-time keys are removed first so an MCP server deleted during the + * shields-down window cannot be restored. The current exact entries are then overlaid, + * retaining additions and replacing stale pins. Every non-MCP target entry + * remains authoritative; unrelated live entries are never copied. + */ +export function composeManagedMcpPolicies( + targetPolicyYaml: string, + currentPolicies: readonly ExactManagedMcpPolicy[], + snapshotManagedPolicyKeys: readonly string[] = [], +): string { + const target = parsePolicyDocument(targetPolicyYaml, "Target Shields policy"); + const targetPolicies = readNetworkPolicies(target, "Target Shields policy"); + + const snapshotKeys = new Set(); + for (const key of snapshotManagedPolicyKeys) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(key) || snapshotKeys.has(key)) { + throw new Error("Saved Shields MCP policy ownership is invalid"); + } + if (!Object.hasOwn(targetPolicies, key)) { + throw new Error(`Saved Shields MCP policy '${key}' is absent from its policy snapshot`); + } + snapshotKeys.add(key); + delete targetPolicies[key]; + } + const unclassifiedKey = Object.keys(targetPolicies).find((key) => + RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key), + ); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' is absent from the saved ownership manifest`, + ); + } + + const currentKeys = new Set(); + for (const policy of currentPolicies) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { + throw new Error(`Managed MCP policy key '${policy.key}' has ambiguous ownership`); + } + currentKeys.add(policy.key); + targetPolicies[policy.key] = policy.networkPolicy; + } + + target.network_policies = targetPolicies; + return YAML.stringify(target); +} + +export interface DeadlineManagedMcpPolicyComposition { + yaml: string; + omissions: ManagedMcpPolicyOmission[]; +} + +/** + * Security-authoritative deadline composition. + * + * Every reserved key is removed from the snapshot, including keys missing from + * an incomplete manifest. Only independently proven current entries are then + * overlaid. + */ +export function composeDeadlineManagedMcpPolicies( + targetPolicyYaml: string, + currentPolicies: readonly ExactManagedMcpPolicy[], + snapshotManagedPolicyKeys: readonly string[], +): DeadlineManagedMcpPolicyComposition { + const target = parsePolicyDocument(targetPolicyYaml, "Target Shields policy"); + const targetPolicies = readNetworkPolicies(target, "Target Shields policy"); + + const snapshotKeys = new Set(); + const omissions: ManagedMcpPolicyOmission[] = []; + for (const key of snapshotManagedPolicyKeys) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(key)) { + if (RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key)) { + delete targetPolicies[key]; + } + omissions.push({ + key, + reason: `Saved Shields MCP policy ownership key '${key}' is invalid`, + }); + continue; + } + if (snapshotKeys.has(key)) { + omissions.push({ + key, + reason: `Saved Shields MCP policy '${key}' appeared more than once in its ownership manifest`, + }); + continue; + } + if (!Object.hasOwn(targetPolicies, key)) { + omissions.push({ + reason: `Saved Shields MCP policy '${key}' was already absent from its policy snapshot`, + }); + } + snapshotKeys.add(key); + delete targetPolicies[key]; + } + for (const key of Object.keys(targetPolicies)) { + if (!RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key)) continue; + delete targetPolicies[key]; + omissions.push({ + key, + reason: `Reserved MCP policy key '${key}' was absent from the saved ownership manifest`, + }); + } + + const currentKeys = new Set(); + for (const policy of currentPolicies) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { + throw new Error(`Managed MCP policy key '${policy.key}' has ambiguous ownership`); + } + currentKeys.add(policy.key); + targetPolicies[policy.key] = policy.networkPolicy; + } + + target.network_policies = targetPolicies; + return { yaml: YAML.stringify(target), omissions }; +} + +export function isManagedMcpPolicyKey(value: unknown): value is string { + return typeof value === "string" && RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(value); +} + +/** + * Refuse to guess managed ownership for a Shields snapshot captured before the + * ownership manifest existed. Current claims prove reconciliation is needed; + * a managed-shaped snapshot key may be a removed bridge or an operator entry. + * Either case requires explicit recovery instead of a destructive raw apply. + */ +export function assertLegacyMcpPolicyRestoreSafe( + snapshotPolicyYaml: string, + hasCurrentManagedClaims: boolean, +): void { + const snapshot = parsePolicyDocument(snapshotPolicyYaml, "Legacy Shields policy snapshot"); + const snapshotPolicies = readNetworkPolicies(snapshot, "Legacy Shields policy snapshot"); + if ( + hasCurrentManagedClaims || + Object.keys(snapshotPolicies).some((key) => isManagedMcpPolicyKey(key)) + ) { + throw new Error( + "Legacy Shields state has no managed MCP ownership manifest; refusing policy restore", + ); + } +} diff --git a/src/lib/shields/permissive-runtime.ts b/src/lib/shields/permissive-runtime.ts index 46f523f60be..897ed4d52a9 100644 --- a/src/lib/shields/permissive-runtime.ts +++ b/src/lib/shields/permissive-runtime.ts @@ -4,8 +4,30 @@ import fs from "node:fs"; import YAML from "yaml"; +export { + type ExactManagedMcpPolicy, + hasManagedMcpPolicyClaims, + inspectExactManagedMcpPolicies, + inspectProvableManagedMcpPoliciesForDeadline, + type ManagedMcpPolicyOmission, +} from "../actions/sandbox/mcp-bridge-policy"; + +import type { + ExactManagedMcpPolicy, + ManagedMcpPolicyOmission, +} from "../actions/sandbox/mcp-bridge-policy"; import { cleanupTempDir, secureTempFile } from "../onboard/temp-files"; +export { + assertLegacyMcpPolicyRestoreSafe, + isManagedMcpPolicyKey, +} from "./mcp-policy-transition"; + +import { + composeDeadlineManagedMcpPolicies, + composeManagedMcpPolicies, +} from "./mcp-policy-transition"; + const TEMP_FILE_PREFIX = "nemoclaw-permissive-runtime"; /** @@ -60,6 +82,10 @@ export interface PermissiveRuntimeDeps { // secureTempFile when omitted. Exposed so tests can drive the // write-failure fallback path without monkey-patching node:fs. writeTempPolicy?: (yaml: string) => string; + // Exact, live-matching generated MCP policies resolved by the Shields + // coordinator. These entries remain active while the static policy replaces + // the rest of the complete gateway policy. + managedMcpPolicies?: readonly ExactManagedMcpPolicy[]; } export function buildRuntimePermissivePolicy( @@ -69,21 +95,31 @@ export function buildRuntimePermissivePolicy( const live = deps.livePolicyYaml ? safeYamlObject(deps.livePolicyYaml) : null; const liveRw = readStringList(live, "read_write"); const liveRo = readStringList(live, "read_only"); + const managedMcpPolicies = deps.managedMcpPolicies ?? []; // No live filesystem section to merge — keep the static path so the - // caller's apply path is unchanged. - if (liveRw.length === 0 && liveRo.length === 0) { + // caller's apply path is unchanged unless exact managed MCP entries must + // survive the complete-policy replacement. + if (liveRw.length === 0 && liveRo.length === 0 && managedMcpPolicies.length === 0) { return basePermissivePath; } let baseYaml: string; try { baseYaml = deps.readBasePolicy(); - } catch { + } catch (error) { + if (managedMcpPolicies.length > 0) { + throw new Error("Cannot read the Shields-down policy while managed MCP policies are active", { + cause: error, + }); + } return basePermissivePath; } const base = safeYamlObject(baseYaml); if (!base) { + if (managedMcpPolicies.length > 0) { + throw new Error("Cannot parse the Shields-down policy while managed MCP policies are active"); + } return basePermissivePath; } const fsPolicy = @@ -108,11 +144,17 @@ export function buildRuntimePermissivePolicy( fsPolicy.read_write = [...baseRw]; fsPolicy.read_only = [...baseRo]; - const yaml = YAML.stringify(base); + const yaml = composeManagedMcpPolicies(YAML.stringify(base), managedMcpPolicies); if (deps.writeTempPolicy) { try { return deps.writeTempPolicy(yaml); - } catch { + } catch (error) { + if (managedMcpPolicies.length > 0) { + throw new Error( + "Cannot stage the Shields-down policy while managed MCP policies are active", + { cause: error }, + ); + } return basePermissivePath; } } @@ -121,15 +163,111 @@ export function buildRuntimePermissivePolicy( tmpPath = secureTempFile(TEMP_FILE_PREFIX, ".yaml"); fs.writeFileSync(tmpPath, yaml, { mode: 0o600 }); return tmpPath; - } catch { + } catch (error) { // secureTempFile may have created an mkdtemp directory before // writeFileSync failed. Clean it up so we do not leak a 0700 dir // on /tmp every time the write path errors. if (tmpPath) cleanupTempDir(tmpPath, TEMP_FILE_PREFIX); + if (managedMcpPolicies.length > 0) { + throw new Error( + "Cannot stage the Shields-down policy while managed MCP policies are active", + { cause: error }, + ); + } return basePermissivePath; } } +export interface ManagedMcpRuntimePolicyDeps { + managedMcpPolicies: readonly ExactManagedMcpPolicy[]; + readBasePolicy: () => string; + snapshotManagedPolicyKeys?: readonly string[]; + writeTempPolicy?: (yaml: string) => string; +} + +/** + * Reconcile current generated MCP policies into a custom Shields-down policy + * or a saved restrictive snapshot. Unlike the legacy filesystem-only fallback, + * this path must fail closed: returning the unmodified base could silently + * discard a managed entry or restore one that was removed during the + * shields-down window. + */ +export function buildRuntimeManagedMcpPolicy( + _basePolicyPath: string, + deps: ManagedMcpRuntimePolicyDeps, +): string { + const snapshotManagedPolicyKeys = deps.snapshotManagedPolicyKeys ?? []; + + let baseYaml: string; + try { + baseYaml = deps.readBasePolicy(); + } catch (error) { + throw new Error("Cannot read the Shields policy for managed MCP reconciliation", { + cause: error, + }); + } + const yaml = composeManagedMcpPolicies( + baseYaml, + deps.managedMcpPolicies, + snapshotManagedPolicyKeys, + ); + if (deps.writeTempPolicy) { + try { + return deps.writeTempPolicy(yaml); + } catch (error) { + throw new Error("Cannot stage the Shields policy for managed MCP reconciliation", { + cause: error, + }); + } + } + + let tmpPath: string | null = null; + try { + tmpPath = secureTempFile(TEMP_FILE_PREFIX, ".yaml"); + fs.writeFileSync(tmpPath, yaml, { mode: 0o600 }); + return tmpPath; + } catch (error) { + if (tmpPath) cleanupTempDir(tmpPath, TEMP_FILE_PREFIX); + throw new Error("Cannot stage the Shields policy for managed MCP reconciliation", { + cause: error, + }); + } +} + +export interface DeadlineManagedMcpRuntimePolicy { + path: string; + omissions: ManagedMcpPolicyOmission[]; +} + +export function buildDeadlineRuntimeManagedMcpPolicy( + _basePolicyPath: string, + deps: ManagedMcpRuntimePolicyDeps, +): DeadlineManagedMcpRuntimePolicy { + const baseYaml = deps.readBasePolicy(); + const composition = composeDeadlineManagedMcpPolicies( + baseYaml, + deps.managedMcpPolicies, + deps.snapshotManagedPolicyKeys ?? [], + ); + let runtimePath: string | null = null; + try { + runtimePath = deps.writeTempPolicy + ? deps.writeTempPolicy(composition.yaml) + : secureTempFile(TEMP_FILE_PREFIX, ".yaml"); + if (!deps.writeTempPolicy) { + fs.writeFileSync(runtimePath, composition.yaml, { mode: 0o600 }); + } + return { path: runtimePath, omissions: composition.omissions }; + } catch (error) { + if (runtimePath && !deps.writeTempPolicy) { + cleanupTempDir(runtimePath, TEMP_FILE_PREFIX); + } + throw new Error("Cannot stage the deadline Shields policy for managed MCP reconciliation", { + cause: error, + }); + } +} + function safeYamlObject(text: string): Record | null { try { const parsed = YAML.parse(text); diff --git a/src/lib/shields/policy-transition.test.ts b/src/lib/shields/policy-transition.test.ts index 50df05618fb..43ea45c4601 100644 --- a/src/lib/shields/policy-transition.test.ts +++ b/src/lib/shields/policy-transition.test.ts @@ -7,6 +7,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; +import YAML from "yaml"; const requireSource = createRequire(import.meta.url); const SHIELDS_MODULE = "./index.js"; @@ -285,3 +286,209 @@ describe("shields config lock without a shipped config hash", () => { expect(entries.get(CONFIG_DIR)).toEqual({ mode: "2770", owner: "sandbox:sandbox" }); }); }); + +describe("managed MCP policy deadline restoration (#7952)", () => { + let homeDir: string; + + function createRestoreHarness() { + delete require.cache[requireSource.resolve(SHIELDS_MODULE)]; + delete require.cache[requireSource.resolve("./permissive-runtime.js")]; + delete require.cache[requireSource.resolve("../actions/sandbox/mcp-bridge-policy.js")]; + + const runner = requireSource("../runner.js") as typeof import("../runner.js"); + const policy = requireSource("../policy/index.js") as typeof import("../policy/index.js"); + const registry = requireSource("../state/registry.js") as typeof import("../state/registry.js"); + const policySetBodies: string[] = []; + + vi.spyOn(runner, "runCapture").mockReturnValue( + "version: 1\nnetwork_policies:\n live_baseline: {}\n", + ); + vi.spyOn(runner, "run").mockReturnValue({ status: 0 } as never); + vi.spyOn(policy, "buildPolicyGetCommand").mockReturnValue(["openshell", "policy", "get"]); + vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: unknown) => { + policySetBodies.push(fs.readFileSync(String(file), "utf-8")); + return ["openshell", "policy", "set"]; + }); + vi.spyOn(policy, "parseCurrentPolicy").mockImplementation((raw: unknown) => String(raw)); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "openclaw", + openshellDriver: "docker", + }); + + const shields = requireSource(SHIELDS_MODULE) as typeof import("./index.js"); + return { applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot, policySetBodies }; + } + + function writeCurrentProcessTimerMarker(snapshotPath: string, processToken: string): void { + fs.writeFileSync( + path.join(homeDir, ".nemoclaw", "state", "shields-timer-openclaw.json"), + JSON.stringify({ + pid: process.pid, + sandboxName: "openclaw", + snapshotPath, + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + { mode: 0o600 }, + ); + } + + beforeEach(() => { + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-mcp-deadline-flow-")); + vi.stubEnv("HOME", homeDir); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(homeDir, { recursive: true, force: true }); + delete require.cache[requireSource.resolve(SHIELDS_MODULE)]; + delete require.cache[requireSource.resolve("./permissive-runtime.js")]; + delete require.cache[requireSource.resolve("../actions/sandbox/mcp-bridge-policy.js")]; + }); + + it("restores lockdown with malformed and duplicate ownership", () => { + const stateDir = path.join(homeDir, ".nemoclaw", "state"); + const processToken = "a".repeat(32); + const snapshotPath = path.join(stateDir, "policy-snapshot-malformed-deadline.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: {}, + mcp_bridge_: {}, + mcp_bridge_alpha: {}, + }, + }), + ); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["mcp_bridge_", "mcp_bridge_alpha", "mcp_bridge_alpha"], + }), + ); + writeCurrentProcessTimerMarker(snapshotPath, processToken); + const harness = createRestoreHarness(); + + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { + transitionProcessToken: processToken, + deadlineAuthoritative: true, + }); + + expect(result.status).toBe(0); + expect(result.managedMcpOmissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: "mcp_bridge_", + reason: expect.stringMatching(/ownership key.*invalid/), + }), + expect.objectContaining({ + key: "mcp_bridge_alpha", + reason: expect.stringMatching(/more than once/), + }), + ]), + ); + expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies).toEqual({ + restrictive_baseline: {}, + }); + }); + + it("restores lockdown when transition and persisted ownership differ", () => { + const stateDir = path.join(homeDir, ".nemoclaw", "state"); + const processToken = "b".repeat(32); + const snapshotPath = path.join(stateDir, "policy-snapshot-mismatched-deadline.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: {}, + mcp_bridge_alpha: {}, + mcp_bridge_beta: {}, + }, + }), + ); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], + }), + ); + fs.writeFileSync( + path.join(stateDir, `shields-transition-openclaw-${processToken}.json`), + JSON.stringify({ + version: 1, + phase: "active", + ownerPid: process.pid, + ownerStartIdentity: "test-owner", + processToken, + sandboxName: "openclaw", + snapshotPath, + managedMcpPolicyKeys: ["mcp_bridge_beta"], + }), + { mode: 0o600 }, + ); + writeCurrentProcessTimerMarker(snapshotPath, processToken); + const harness = createRestoreHarness(); + + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { + transitionProcessToken: processToken, + deadlineAuthoritative: true, + }); + + expect(result.status).toBe(0); + expect(result.managedMcpOmissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + reason: expect.stringMatching(/did not match persisted policy ownership/), + }), + ]), + ); + expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies).toEqual({ + restrictive_baseline: {}, + }); + }); + + it("restores lockdown from a legacy snapshot without ownership metadata", () => { + const stateDir = path.join(homeDir, ".nemoclaw", "state"); + const processToken = "c".repeat(32); + const snapshotPath = path.join(stateDir, "policy-snapshot-legacy-deadline.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ + version: 1, + network_policies: { restrictive_baseline: {}, mcp_bridge_alpha: {} }, + }), + ); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ shieldsDown: true, shieldsPolicySnapshotPath: snapshotPath }), + ); + writeCurrentProcessTimerMarker(snapshotPath, processToken); + const harness = createRestoreHarness(); + + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { + transitionProcessToken: processToken, + deadlineAuthoritative: true, + }); + + expect(result.status).toBe(0); + expect(result.managedMcpOmissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ reason: expect.stringMatching(/no managed MCP ownership/) }), + expect.objectContaining({ key: "mcp_bridge_alpha" }), + ]), + ); + expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies).toEqual({ + restrictive_baseline: {}, + }); + }); +}); diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index 251dca46f78..82e91bf2ca8 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -9,6 +9,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getMcpLifecycleLockPath, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ + applyShieldsPolicySnapshot: vi.fn( + (): { + status: number; + managedMcpOmissions?: Array<{ server: string; reason: string }>; + } => ({ status: 0 }), + ), completeAutoRestoreTransition: vi.fn(() => true), lockAgentConfig: vi.fn() as unknown, prepareAutoRestoreTransitionTakeover: vi.fn(), @@ -16,25 +22,6 @@ const shieldsIndexMock = vi.hoisted(() => ({ const PROCESS_TOKEN = "a".repeat(32); -const runMock = vi.fn(() => ({ status: 0 })); - -vi.mock("../runner", async (importOriginal) => ({ - ...(await importOriginal()), - run: runMock, -})); - -vi.mock("../policy", () => ({ - buildPolicySetCommand: vi.fn((file: string, name: string) => [ - "openshell", - "policy", - "set", - "--policy", - file, - "--wait", - name, - ]), -})); - vi.mock("../sandbox/agent-config", () => ({ DEFAULT_AGENT_CONFIG: Symbol("DEFAULT_AGENT_CONFIG"), resolveAgentConfig: vi.fn(() => ({ @@ -44,6 +31,7 @@ vi.mock("../sandbox/agent-config", () => ({ })); vi.mock("./index", () => ({ + applyShieldsPolicySnapshot: shieldsIndexMock.applyShieldsPolicySnapshot, completeAutoRestoreTransition: shieldsIndexMock.completeAutoRestoreTransition, get lockAgentConfig() { return shieldsIndexMock.lockAgentConfig; @@ -57,8 +45,8 @@ describe("shields timer authorization", () => { beforeEach(() => { tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-timer-")); vi.stubEnv("HOME", tmpHome); + shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementation(() => ({ status: 0 })); shieldsIndexMock.lockAgentConfig = vi.fn(); - runMock.mockImplementation(() => ({ status: 0 })); vi.resetModules(); vi.clearAllMocks(); }); @@ -121,10 +109,13 @@ describe("shields timer authorization", () => { await waitForRetryBoundary(deadlinePath, auditPath); expect(exitSpy).not.toHaveBeenCalled(); expect(fs.existsSync(deadlinePath)).toBe(true); - const policyApplicationsBeforeRevocation = runMock.mock.calls.length; + const policyApplicationsBeforeRevocation = + shieldsIndexMock.applyShieldsPolicySnapshot.mock.calls.length; fs.rmSync(markerPath, { force: true }); await pending; - expect(runMock).toHaveBeenCalledTimes(policyApplicationsBeforeRevocation); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes( + policyApplicationsBeforeRevocation, + ); } finally { fs.writeFileSync(markerPath, markerContents); exitSpy.mockRestore(); @@ -151,7 +142,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); }); @@ -193,7 +184,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.existsSync(markerPath)).toBe(true); }); @@ -237,7 +228,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.lstatSync(markerPath).isSymbolicLink()).toBe(true); expect(fs.readFileSync(markerTargetPath, "utf-8")).toBe(markerTarget); @@ -336,7 +327,7 @@ describe("shields timer authorization", () => { await timer.runRestoreTimer(args!); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(exitSpy).not.toHaveBeenCalled(); expect(vi.getTimerCount()).toBe(1); expect(fs.existsSync(markerPath)).toBe(true); @@ -346,7 +337,7 @@ describe("shields timer authorization", () => { } }); - it("audits a successful restore retry while retaining deadline ownership", async () => { + it("audits a successful restore retry without stale MCP warnings or timestamps", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); @@ -369,10 +360,17 @@ describe("shields timer authorization", () => { leaseOwnerStartIdentity: "proc:dead-owner", }), ); - runMock.mockImplementationOnce(() => { + shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); expect(fs.existsSync(deadlinePath)).toBe(true); - return { status: 17 }; + return { + status: 17, + managedMcpOmissions: [{ server: "beta", reason: "incomplete add" }], + }; + }); + shieldsIndexMock.applyShieldsPolicySnapshot.mockReturnValueOnce({ + status: 0, + managedMcpOmissions: [], }); const args = timer.parseTimerArgs([ sandboxName, @@ -390,7 +388,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).toHaveBeenCalledTimes(2); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(2); expect(shieldsIndexMock.completeAutoRestoreTransition).toHaveBeenCalledWith( sandboxName, PROCESS_TOKEN, @@ -404,14 +402,22 @@ describe("shields timer authorization", () => { .split("\n") .filter(Boolean) .map((line) => JSON.parse(line)); - expect(audits).toContainEqual( + const successAudits = audits.filter((audit) => audit.action === "shields_auto_restore"); + expect(successAudits).toEqual([ expect.objectContaining({ - action: "shields_up_failed", - error: "Policy restore exited with status 17", + action: "shields_auto_restore", + sandbox: sandboxName, }), + ]); + expect(successAudits[0]).not.toHaveProperty("warning"); + const failedAudit = audits.find( + (audit) => + audit.action === "shields_up_failed" && + audit.error === "Policy restore exited with status 17", ); - expect(audits).toContainEqual( - expect.objectContaining({ action: "shields_auto_restore", sandbox: sandboxName }), + expect(failedAudit).toEqual(expect.objectContaining({ timestamp: expect.any(String) })); + expect(Date.parse(successAudits[0].timestamp)).toBeGreaterThan( + Date.parse(failedAudit.timestamp), ); }); @@ -453,7 +459,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.existsSync(markerPath)).toBe(true); }); @@ -550,7 +556,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(1); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(shieldsIndexMock.completeAutoRestoreTransition).not.toHaveBeenCalled(); expect(fs.existsSync(mutationLockPath)).toBe(false); expect(fs.existsSync(deadlinePath)).toBe(false); @@ -592,7 +598,7 @@ describe("shields timer authorization", () => { const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); const deadlinePath = `${sandboxMutationLockPath}.deadline`; - runMock.mockImplementationOnce(() => { + shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); expect(fs.existsSync(deadlinePath)).toBe(true); expect(JSON.parse(fs.readFileSync(lockPath, "utf-8"))).toMatchObject({ @@ -600,7 +606,10 @@ describe("shields timer authorization", () => { command: "shields auto-restore", takeoverToken: PROCESS_TOKEN, }); - return { status: 0 }; + return { + status: 0, + managedMcpOmissions: [{ server: "beta", reason: "incomplete add" }], + }; }); const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); @@ -608,7 +617,7 @@ describe("shields timer authorization", () => { const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); expect(exitCode).toBe(0); - expect(runMock).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); expect(updatedState.shieldsDown).toBe(false); expect(updatedState.shieldsDownAt).toBeNull(); expect(fs.existsSync(markerPath)).toBe(false); @@ -619,6 +628,18 @@ describe("shields timer authorization", () => { PROCESS_TOKEN, snapshotPath, ); + expect( + fs + .readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)), + ).toContainEqual( + expect.objectContaining({ + action: "shields_auto_restore", + warning: "Auto-restore omitted 1 unproven managed MCP policy entries", + }), + ); }); it("keeps the deadline gate closed while a failed restore retries", async () => { @@ -641,7 +662,9 @@ describe("shields timer authorization", () => { processToken: PROCESS_TOKEN, }), ); - runMock.mockReturnValueOnce({ status: 1 }).mockReturnValue({ status: 0 }); + shieldsIndexMock.applyShieldsPolicySnapshot + .mockReturnValueOnce({ status: 1 }) + .mockReturnValue({ status: 0 }); const args = timer.parseTimerArgs([ sandboxName, snapshotPath, @@ -658,10 +681,13 @@ describe("shields timer authorization", () => { try { const restore = timer.runRestoreTimer(args!, { retryDelayMs: 100 }); - await vi.waitFor(() => expect(runMock).toHaveBeenCalledTimes(1), { - interval: 1, - timeout: 200, - }); + await vi.waitFor( + () => expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1), + { + interval: 1, + timeout: 200, + }, + ); expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); const contender = withMcpLifecycleLock( @@ -676,7 +702,7 @@ describe("shields timer authorization", () => { expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); await Promise.all([restore, contender]); - expect(runMock).toHaveBeenCalledTimes(2); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(2); expect(contenderEntered).toBe(true); expect(shieldsIndexMock.completeAutoRestoreTransition).toHaveBeenCalledWith( sandboxName, @@ -800,7 +826,12 @@ describe("shields timer authorization", () => { const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); expect(exitCode).toBe(0); - expect(runMock).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledWith( + sandboxName, + snapshotPath, + { deadlineAuthoritative: true, transitionProcessToken: PROCESS_TOKEN }, + ); // #4663: relockAndReconfirm applies then re-confirms after the settle // window (0ms under test), so lockAgentConfig is invoked twice for a clean // lock. @@ -866,7 +897,7 @@ describe("shields timer authorization", () => { .split("\n") .map((line) => JSON.parse(line)); - expect(runMock).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); expect(updatedState.shieldsDown).toBe(true); expect(auditEntries).toContainEqual( expect.objectContaining({ diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index 2737d62f17f..6226b5244b4 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -11,8 +11,6 @@ import fs from "node:fs"; import path from "node:path"; import { isObjectRecord, type UnknownRecord } from "../core/json-types"; -import { buildPolicySetCommand } from "../policy"; -import { run } from "../runner"; import { resolveAgentConfig } from "../sandbox/agent-config"; import { withMcpLifecycleDeadlineFence } from "../state/mcp-lifecycle-lock"; import { @@ -241,6 +239,7 @@ async function runRestoreTimer( : AUTO_RESTORE_RETRY_MS; let exitCode = 0; let retryScheduled = false; + let managedMcpWarning: string | undefined; const scheduleRetry = (): boolean => { if (!markerMatchesCurrentTimer(args)) return false; retryScheduled = true; @@ -299,10 +298,16 @@ async function runRestoreTimer( } // Restore policy (slow — openshell policy set --wait blocks) - const result = run(buildPolicySetCommand(args.snapshotPath, args.sandboxName), { - ignoreError: true, + const result = shields.applyShieldsPolicySnapshot(args.sandboxName, args.snapshotPath, { + transitionProcessToken: args.processToken, + deadlineAuthoritative: true, }); const status = typeof result.status === "number" ? result.status : 1; + managedMcpWarning = result.managedMcpOmissions?.length + ? `Auto-restore omitted ${String( + result.managedMcpOmissions.length, + )} unproven managed MCP policy entries` + : undefined; if (status !== 0) { appendAudit({ @@ -448,6 +453,7 @@ async function runRestoreTimer( restored_by: "auto_timer", policy_snapshot: args.snapshotPath, scheduled_restore_at: args.restoreAtIso, + ...(managedMcpWarning ? { warning: managedMcpWarning } : {}), }); cleanupOwnedTimerMarker(args); exitCode = 0; diff --git a/test/e2e/live/mcp-bridge-hermes-lifecycle.ts b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts index 0f9b7a06881..5044cc5e31a 100644 --- a/test/e2e/live/mcp-bridge-hermes-lifecycle.ts +++ b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts @@ -187,7 +187,6 @@ export async function assertHermesManagedAddSurvivesLockedGatewayRestartAndState }, ); expectExitZero(shieldsDown, "unlock Hermes config for remaining managed MCP lifecycle"); - await assertHermesReloadRollback(sandbox, sandboxName, mcpUrl); } /** diff --git a/test/e2e/live/mcp-bridge-sandbox.ts b/test/e2e/live/mcp-bridge-sandbox.ts index 5d9f5b8927e..aa801ef4c28 100644 --- a/test/e2e/live/mcp-bridge-sandbox.ts +++ b/test/e2e/live/mcp-bridge-sandbox.ts @@ -1,15 +1,91 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import assert from "node:assert/strict"; +import YAML from "yaml"; import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const MCP_CURL_HTTP_CODE_MARKER = "NEMOCLAW_MCP_CURL_HTTP_CODE="; export type McpDnsRebindingAdapter = "mcporter" | "hermes-config" | "deepagents-config"; +export type CapturedManagedMcpPolicy = { + networkPolicies: Record; + policy: McpNetworkPolicy; +}; + +type McpNetworkPolicy = { + endpoints?: Array<{ + host?: string; + allowed_ips?: string[]; + [key: string]: unknown; + }>; + [key: string]: unknown; +}; + +export async function captureManagedMcpPolicy( + sandbox: SandboxClient, + options: { + artifactName: string; + label: string; + policyKey: string; + sandboxName: string; + url: string; + }, +): Promise { + const result = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { + artifactName: options.artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + assertExitZero(result, options.label); + const document = YAML.parse(parseOpenShellPolicy(resultText(result)).yamlBody) as { + network_policies?: Record; + }; + const networkPolicies = document.network_policies ?? {}; + const policy = networkPolicies[options.policyKey]; + if (!policy) { + throw new Error(`${options.label}: managed MCP policy '${options.policyKey}' is absent`); + } + const endpoint = policy.endpoints?.[0]; + const expectedHost = new URL(options.url).hostname; + if (endpoint?.host !== expectedHost) { + throw new Error(`${options.label}: expected managed MCP host '${expectedHost}'`); + } + if ( + !Array.isArray(endpoint.allowed_ips) || + endpoint.allowed_ips.length === 0 || + endpoint.allowed_ips.some((address) => typeof address !== "string") + ) { + throw new Error(`${options.label}: expected at least one managed MCP address pin`); + } + return { networkPolicies, policy }; +} + +export function assertManagedMcpPolicySurvivedRemoval( + before: McpNetworkPolicy, + after: CapturedManagedMcpPolicy, + removedPolicyKey: string, +): void { + assert.deepStrictEqual(after.policy, before); + assert.equal(after.networkPolicies[removedPolicyKey], undefined); +} + +export function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { + assert.notEqual( + result.exitCode, + 0, + `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + assert.match(resultText(result), pattern); +} + export async function hostAddressForSandbox(_host: HostCliClient): Promise { return "host.openshell.internal"; } diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 3cf1043cdea..84789a9161e 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -4,14 +4,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import YAML from "yaml"; import { buildDeepAgentsMcpStatusCommand, buildHermesMcpStatusCommand, buildOpenClawMcporterInspectCommand, } from "../../../src/lib/actions/sandbox/mcp-bridge-adapter-status"; import { shellQuote } from "../../../src/lib/core/shell-quote"; -import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import type { McpBridgeEntry } from "../../../src/lib/state/registry"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; @@ -27,13 +25,17 @@ import { assertHermesConfig, assertHermesInspectionRejectsUnmanagedFields, assertHermesManagedAddSurvivesLockedGatewayRestartAndStateLayout, + assertHermesReloadRollback, assertHermesRemovalSurvivesGatewayRestart, } from "./mcp-bridge-hermes-lifecycle.ts"; import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv } from "./mcp-bridge-onboard-env.ts"; import { MCP_BRIDGE_PHASES } from "./mcp-bridge-phases.ts"; import { retryAfterHermesRestartTransportFailure } from "./mcp-bridge-reliability.ts"; import { + assertManagedMcpPolicySurvivedRemoval, buildMcpDnsRebindingProbeScript, + captureManagedMcpPolicy, + expectExitNonZero, hostAddressForSandbox, hostPrivateAddressForSandbox, isExpectedMcpCurlPolicyDenial, @@ -75,12 +77,10 @@ const COMPATIBLE_MODEL = "mock/mcp-bridge"; const TOOL_CHALLENGE = "nemoclaw-authenticated-mcp-proof"; const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); const selectedMcpBridgeShard = resolveMcpBridgeShard(); - function mcpBridgeShardTest(shard: McpBridgeShard) { return selectedMcpBridgeShard === shard ? e2eTest : e2eTest.skip; } const test = mcpBridgeShardTest("openclaw"); - type McpAgent = "openclaw" | "hermes" | "langchain-deepagents-code"; type McpAdapter = "mcporter" | "hermes-config" | "deepagents-config"; const MCP_MUTATION_TIMEOUT_MS: Record = { @@ -90,19 +90,6 @@ const MCP_MUTATION_TIMEOUT_MS: Record = { }; const MCP_BRIDGE_ALREADY_ABSENT = /No MCP servers are registered|No MCP server '.+' is registered|MCP server '.+' not found/iu; - -function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { - expect( - result.exitCode, - `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, - ).not.toBe(0); - expect(resultText(result)).toMatch(pattern); -} - -function parseCurrentPolicy(raw: string): string { - return parseOpenShellPolicy(raw).yamlBody; -} - async function cleanupMcpBridge( host: HostCliClient, sandboxName: string, @@ -120,7 +107,6 @@ async function cleanupMcpBridge( `cleanup MCP bridge ${server} on sandbox ${sandboxName}`, ); } - async function onboardAgent( host: HostCliClient, cleanup: CleanupRegistry, @@ -158,7 +144,6 @@ async function onboardAgent( ); expectExitZero(result, `onboard ${options.agent} sandbox for MCP bridge`); } - async function assertSecretAbsentFromSandbox( sandbox: SandboxClient, sandboxName: string, @@ -189,6 +174,7 @@ async function assertAdapterDnsRebindingDenied( artifactPrefix: string; sandboxName: string; secretPaths: string[]; + survivingMcpUrl: string; }, ): Promise { const rebindMcp = await startFakeMcpHttpsServer({ secret: REBIND_HOST_SECRET }); @@ -207,6 +193,14 @@ async function assertAdapterDnsRebindingDenied( cleanup.add(`restore ${options.artifactPrefix} DNS rebinding hosts fixture`, () => restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture), ); + const survivingPolicyBeforeAddResult = await captureManagedMcpPolicy(sandbox, { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-before-add`, + label: `${options.artifactPrefix} captures the surviving MCP policy before adding the rebinding route`, + policyKey: SERVER_POLICY_KEY, + sandboxName: options.sandboxName, + url: options.survivingMcpUrl, + }); + const survivingPolicyBeforeAdd = survivingPolicyBeforeAddResult.policy; await remapDnsRebindingHostname( host, options.sandboxName, @@ -261,19 +255,14 @@ async function assertAdapterDnsRebindingDenied( policy: { gatewayPresent: true }, adapter: { registered: true }, }); - const policy = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { + const rebindingPolicy = await captureManagedMcpPolicy(sandbox, { artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-policy-pinned-public-ip`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, + label: `${options.artifactPrefix} validates the add-time DNS pin`, + policyKey: REBIND_POLICY_KEY, + sandboxName: options.sandboxName, + url: rebindMcpUrl, }); - expectExitZero(policy, `${options.artifactPrefix} inspects add-time DNS pin`); - const policyJson = YAML.parse(parseCurrentPolicy(resultText(policy))) as { - network_policies?: Record< - string, - { endpoints?: Array<{ host?: string; allowed_ips?: string[] }> } - >; - }; - expect(policyJson.network_policies?.[REBIND_POLICY_KEY]?.endpoints?.[0]).toMatchObject({ + expect(rebindingPolicy.policy.endpoints?.[0]).toMatchObject({ host: REBIND_HOSTNAME, allowed_ips: [REBIND_PUBLIC_IP], }); @@ -322,6 +311,18 @@ async function assertAdapterDnsRebindingDenied( timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], }); expectExitZero(remove, `${options.artifactPrefix} removes DNS rebinding route after proof`); + const survivingPolicyAfterRemoveResult = await captureManagedMcpPolicy(sandbox, { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-after-remove`, + label: `${options.artifactPrefix} inspects MCP policy after removing the rebinding route`, + policyKey: SERVER_POLICY_KEY, + sandboxName: options.sandboxName, + url: options.survivingMcpUrl, + }); + assertManagedMcpPolicySurvivedRemoval( + survivingPolicyBeforeAdd, + survivingPolicyAfterRemoveResult, + REBIND_POLICY_KEY, + ); } async function addBridgeAndReadStatus( host: HostCliClient, @@ -354,7 +355,6 @@ async function addBridgeAndReadStatus( }, ); expectExitZero(add, `${options.artifactPrefix} mcp add fake server`); - const status = await host.nemoclaw( [options.sandboxName, "mcp", "status", SERVER_NAME, "--json"], { @@ -981,6 +981,7 @@ test("mcp-bridge", { artifactPrefix: "openclaw", sandboxName: OPENCLAW_SANDBOX_NAME, secretPaths: ["/sandbox/.openclaw", "/sandbox/.mcp.json"], + survivingMcpUrl: mcpUrl, }); const requestCountBeforeAllowedNodeProof = fakeMcp.requests.length; @@ -1171,6 +1172,13 @@ mcpBridgeShardTest("hermes")( challenge: TOOL_CHALLENGE, resultToken: hermesResult, }); + const assertHermesToolCall = (artifactName: string) => + assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + resultToken: hermesResult, + artifactName, + }); cleanup.add("stop fake Hermes MCP HTTPS server", () => fakeMcp.close()); const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ cleanup, @@ -1190,7 +1198,6 @@ mcpBridgeShardTest("hermes")( cleanup.add("remove Hermes MCP bridge", () => cleanupMcpBridge(host, HERMES_SANDBOX_NAME, SERVER_NAME, "hermes-config"), ); - progress.phase("configure and inspect the Hermes MCP bridge"); await assertConcurrentAddSerialized(host, cleanup, { sandboxName: HERMES_SANDBOX_NAME, @@ -1198,7 +1205,6 @@ mcpBridgeShardTest("hermes")( expectedAdapter: "hermes-config", artifactPrefix: "hermes", }); - const initialDiscoveryOffset = fakeMcp.requests.length; const providerName = await addBridgeAndReadStatus(host, { sandboxName: HERMES_SANDBOX_NAME, @@ -1233,6 +1239,8 @@ mcpBridgeShardTest("hermes")( HERMES_SANDBOX_NAME, mcpUrl, ); + await assertHermesToolCall("hermes-real-mcp-tool-call-immediately-after-shields-down"); + await assertHermesReloadRollback(sandbox, HERMES_SANDBOX_NAME, mcpUrl); await assertSecretAbsentFromSandbox( sandbox, HERMES_SANDBOX_NAME, @@ -1251,15 +1259,12 @@ mcpBridgeShardTest("hermes")( artifactPrefix: "hermes", sandboxName: HERMES_SANDBOX_NAME, secretPaths: ["/sandbox/.hermes"], + survivingMcpUrl: mcpUrl, }); + await assertHermesToolCall("hermes-real-mcp-tool-call-after-dns-rebinding-remove"); const survivingDiscoveryOffset = fakeMcp.requests.length; await restartBridgeWithoutHostSecret(host, HERMES_SANDBOX_NAME, "hermes"); - await assertRealAdapterToolCall(sandbox, fakeMcp, { - agent: "hermes", - sandboxName: HERMES_SANDBOX_NAME, - resultToken: hermesResult, - artifactName: "hermes-real-mcp-tool-call-after-rediscovery-restart", - }); + await assertHermesToolCall("hermes-real-mcp-tool-call-after-rediscovery-restart"); await assertAuthenticatedMcpRediscovery(survivingMcp, survivingDiscoveryOffset); fakeMcp.setSecret(ROTATED_HOST_SECRET); await rotateBridgeCredential(host, HERMES_SANDBOX_NAME, "hermes"); @@ -1418,6 +1423,7 @@ mcpBridgeShardTest("deepagents")( artifactPrefix: "deepagents", sandboxName: DEEPAGENTS_SANDBOX_NAME, secretPaths: ["/sandbox/.deepagents"], + survivingMcpUrl: mcpUrl, }); progress.phase("exercise lifecycle and confirm Deep Agents bridge removal"); await assertRealAdapterToolCall(sandbox, fakeMcp, { diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts index 4346ae4b62b..c2e6017ee9f 100644 --- a/test/e2e/support/mcp-bridge-sandbox.test.ts +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -12,6 +12,7 @@ import YAML from "yaml"; import { testTimeout } from "../../helpers/timeouts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { + assertManagedMcpPolicySurvivedRemoval, buildMcpDnsRebindingProbeScript, hostAddressForSandbox, hostPrivateAddressForSandbox, @@ -302,45 +303,34 @@ network_policies: expect(contractSource).not.toContain("assertAdapterDnsRebindingDenied"); }); - it("runs the zero-upstream rebinding proof for all three adapters", () => { - const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); - - expect(source.match(/await assertAdapterDnsRebindingDenied/g)).toHaveLength(3); - for (const adapter of [ - 'adapter: "mcporter"', - 'adapter: "hermes-config"', - 'adapter: "deepagents-config"', - ]) { - expect(source).toContain(adapter); - } - expect(source).toContain("rebound request must not reach the upstream MCP server"); - expect(source).toContain(").toHaveLength(0);"); - }); + it("accepts an unchanged surviving policy only after the unrelated policy is absent", () => { + const survivingPolicy = { + endpoints: [{ host: "surviving.example.test", allowed_ips: ["203.0.113.10"] }], + }; - it("captures the Hermes rediscovery offset after route removal and before restart", () => { - const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); - const denialProof = source.indexOf("rebound request must not reach the upstream MCP server"); - const restore = source.indexOf("await restoreDnsRebindingHostsFixture", denialProof); - const remove = source.indexOf("const remove = await host.nemoclaw", denialProof); - const hermesTest = source.indexOf('mcpBridgeShardTest("hermes")'); - const rebinding = source.indexOf("await assertAdapterDnsRebindingDenied", hermesTest); - const offset = source.indexOf( - "const survivingDiscoveryOffset = fakeMcp.requests.length", - rebinding, - ); - const restart = source.indexOf("await restartBridgeWithoutHostSecret", offset); - const toolCall = source.indexOf("await assertRealAdapterToolCall", restart); - const rediscovery = source.indexOf("await assertAuthenticatedMcpRediscovery", toolCall); - - expect(denialProof).toBeGreaterThanOrEqual(0); - expect(restore).toBeGreaterThan(denialProof); - expect(remove).toBeGreaterThan(restore); - expect(rebinding).toBeGreaterThan(hermesTest); - expect(offset).toBeGreaterThan(rebinding); - expect(restart).toBeGreaterThan(offset); - expect(toolCall).toBeGreaterThan(restart); - expect(rediscovery).toBeGreaterThan(toolCall); - expect(source).toContain("Hermes MCP rediscovery after explicit restart"); + expect(() => + assertManagedMcpPolicySurvivedRemoval( + survivingPolicy, + { + networkPolicies: { mcp_bridge_surviving: survivingPolicy }, + policy: survivingPolicy, + }, + "mcp_bridge_rebinding", + ), + ).not.toThrow(); + expect(() => + assertManagedMcpPolicySurvivedRemoval( + survivingPolicy, + { + networkPolicies: { + mcp_bridge_rebinding: { endpoints: [] }, + mcp_bridge_surviving: survivingPolicy, + }, + policy: survivingPolicy, + }, + "mcp_bridge_rebinding", + ), + ).toThrow(); }); it("restores host DNS strictly while treating the ephemeral sandbox as best effort", async () => { diff --git a/test/permissive-runtime.test.ts b/test/permissive-runtime.test.ts index 504d9a2898d..01b3f91212b 100644 --- a/test/permissive-runtime.test.ts +++ b/test/permissive-runtime.test.ts @@ -7,7 +7,10 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import YAML from "yaml"; -import { buildRuntimePermissivePolicy } from "../src/lib/shields/permissive-runtime.js"; +import { + buildRuntimePermissivePolicy, + type ExactManagedMcpPolicy, +} from "../src/lib/shields/permissive-runtime.js"; const BASE_PERMISSIVE = YAML.stringify({ filesystem_policy: { @@ -46,6 +49,51 @@ afterEach(() => { }); describe("buildRuntimePermissivePolicy (#3942)", () => { + it("preserves exact managed MCP entries without copying unrelated live egress (#7952)", () => { + const managedPolicy: ExactManagedMcpPolicy = { + key: "mcp_bridge_alpha", + networkPolicy: { + endpoints: [{ host: "alpha.example.com", port: 443, protocol: "mcp" }], + binaries: [{ path: "/opt/hermes/.venv/bin/python*" }], + }, + policyName: "mcp-bridge-alpha", + server: "alpha", + }; + const liveYaml = YAML.stringify({ + filesystem_policy: { read_write: ["/proc"] }, + network_policies: { + mcp_bridge_alpha: managedPolicy.networkPolicy, + unrelated_live_entry: { + endpoints: [{ host: "unrelated.example.com", port: 443 }], + }, + }, + }); + + const out = buildRuntimePermissivePolicy("/unused-base.yaml", { + livePolicyYaml: liveYaml, + managedMcpPolicies: [managedPolicy], + readBasePolicy: () => + YAML.stringify({ + ...YAML.parse(BASE_PERMISSIVE), + network_policies: { + permissive_baseline: { + endpoints: [{ host: "*", port: 443 }], + }, + }, + }), + }); + trackTempForCleanup(out, "/unused-base.yaml"); + + const result = YAML.parse(fs.readFileSync(out, "utf-8")); + expect(result.network_policies).toMatchObject({ + mcp_bridge_alpha: managedPolicy.networkPolicy, + permissive_baseline: { + endpoints: [{ host: "*", port: 443 }], + }, + }); + expect(result.network_policies).not.toHaveProperty("unrelated_live_entry"); + }); + it("preserves /proc when the live GPU sandbox has it in read_write", () => { const liveYaml = YAML.stringify({ filesystem_policy: { @@ -167,6 +215,25 @@ describe("buildRuntimePermissivePolicy (#3942)", () => { expect(out).toBe(basePath); }); + it("fails closed when the base cannot be read with managed MCP policies active (#7952)", () => { + expect(() => + buildRuntimePermissivePolicy("/path/to/static.yaml", { + livePolicyYaml: "version: 1\nnetwork_policies: {}\n", + managedMcpPolicies: [ + { + key: "mcp_bridge_alpha", + networkPolicy: {}, + policyName: "mcp-bridge-alpha", + server: "alpha", + }, + ], + readBasePolicy: () => { + throw new Error("ENOENT"); + }, + }), + ).toThrow(/Cannot read the Shields-down policy/); + }); + it("returns the static base path when base YAML is unparseable", () => { const basePath = "/path/to/static.yaml"; const liveYaml = YAML.stringify({ From 05b1f381922e1f2f97b0a3d51c522aa99c988a63 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 3 Aug 2026 21:50:15 -0400 Subject: [PATCH 4/8] fix(shields): bound failed auto-restore recovery Stop detached restore retries after seven attempts. Preserve exact lifecycle authority through durable containment or retained gates. Sanitize MCP policy diagnostics and add race-focused regression coverage. Signed-off-by: Julie Yaunches --- docs/manage-sandboxes/runtime-controls.mdx | 10 +- docs/reference/commands.mdx | 10 +- src/lib/actions/sandbox/mcp-bridge-policy.ts | 11 +- ...snapshot-baseline-exclusion-output.test.ts | 10 +- src/lib/shields/flow.test.ts | 288 ++++++++---------- src/lib/shields/index.ts | 29 +- src/lib/shields/mcp-policy-transition.test.ts | 21 +- src/lib/shields/timer.test.ts | 255 +++++++++++++++- src/lib/shields/timer.ts | 66 +++- .../mcp-lifecycle-lock-acquisition.test.ts | 30 ++ .../state/mcp-lifecycle-lock-acquisition.ts | 54 +++- 11 files changed, 584 insertions(+), 200 deletions(-) diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index ccfc887339f..02e8f8f9e0e 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -122,11 +122,15 @@ At an expired deadline, auto-restore omits unproven managed MCP policy entries, An MCP server removed during the shields-down window stays removed. A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. -When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window. +An interactive command can take over an expired timer. +The interactive takeover and the detached auto-restore timer each make up to 7 restoration attempts, with 5 seconds between attempts and up to 30 seconds of retry delay. The deadline gate remains closed during those attempts. -If restoration cannot commit, NemoClaw records durable containment before the command returns an error. +If restoration cannot commit, NemoClaw attempts to record durable containment. +If that containment commit also fails, NemoClaw retains the exact lifecycle and deadline gates instead. +Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. +The interactive command returns an error, or the detached timer exits with a failure status. NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. -Durable containment keeps new mutations blocked until you complete exact-generation operator recovery. +Durable containment, or the retained exact gates after a failed containment commit, keeps new mutations blocked until you complete exact-generation operator recovery. Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index a0e77856800..bc64246d186 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1176,11 +1176,15 @@ Host-side config and inference writes, snapshot mutation, sandbox destruction, a When a timed shields-down window reaches its deadline, auto-restore closes the per-sandbox lifecycle deadline gate. The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation before auto-restore restores lockdown. NemoClaw does not signal that process because portable process inspection cannot prove that all descendants stopped. -When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window. +An interactive command can take over an expired timer. +The interactive takeover and the detached auto-restore timer each make up to 7 restoration attempts, with 5 seconds between attempts and up to 30 seconds of retry delay. The deadline gate remains closed during those attempts. -If restoration cannot commit, NemoClaw records durable containment before the command returns an error. +If restoration cannot commit, NemoClaw attempts to record durable containment. +If that containment commit also fails, NemoClaw retains the exact lifecycle and deadline gates instead. +Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. +The interactive command returns an error, or the detached timer exits with a failure status. NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. -Durable containment blocks new mutations until you complete exact-generation operator recovery. +Durable containment, or the retained exact gates after a failed containment commit, blocks new mutations until you complete exact-generation operator recovery. Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index b951485b690..5594dced93c 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -6,6 +6,7 @@ import { isDeepStrictEqual } from "node:util"; import YAML from "yaml"; import type { AgentMcpAdapter } from "../../agent/defs"; +import { diagnosticPreview } from "../../name-validation"; import * as policies from "../../policy"; import { isBlockedMcpUrlTargetHost } from "../../security/mcp-url-target"; import type { McpBridgeEntry } from "../../state/registry"; @@ -243,7 +244,7 @@ export function inspectExactManagedMcpPolicies( const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); if (unclassifiedKey) { throw new Error( - `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + `Reserved MCP policy key ${diagnosticPreview(unclassifiedKey)} has no committed managed bridge ownership`, ); } return []; @@ -255,13 +256,13 @@ export function inspectExactManagedMcpPolicies( const orphaned = generatedRegistrations[0]; if (orphaned) { throw new Error( - `Generated MCP policy '${orphaned.name}' has no committed managed bridge ownership`, + `Generated MCP policy ${diagnosticPreview(orphaned.name)} has no committed managed bridge ownership`, ); } const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); if (unclassifiedKey) { throw new Error( - `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + `Reserved MCP policy key ${diagnosticPreview(unclassifiedKey)} has no committed managed bridge ownership`, ); } return []; @@ -284,7 +285,7 @@ export function inspectExactManagedMcpPolicies( ); if (orphaned) { throw new Error( - `Generated MCP policy '${orphaned.name}' has no committed managed bridge ownership`, + `Generated MCP policy ${diagnosticPreview(orphaned.name)} has no committed managed bridge ownership`, ); } @@ -300,7 +301,7 @@ export function inspectExactManagedMcpPolicies( ); if (unclassifiedKey) { throw new Error( - `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + `Reserved MCP policy key ${diagnosticPreview(unclassifiedKey)} has no committed managed bridge ownership`, ); } return exact.sort((left, right) => left.key.localeCompare(right.key)); diff --git a/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts b/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts index c3017fcb8bd..16ca5aa73f2 100644 --- a/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts +++ b/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { runSandboxSnapshot } from "./snapshot"; const mocks = vi.hoisted(() => ({ backupSandboxState: vi.fn(), @@ -30,6 +31,10 @@ vi.mock("../../shields/timer-bound-lock", () => ({ ), })); +vi.mock("../../state/mcp-lifecycle-lock", () => ({ + withSandboxMutationLock: vi.fn((_sandboxName: string, operation: () => unknown) => operation()), +})); + vi.mock("../../state/registry", () => ({ getBaselineExclusions: mocks.getBaselineExclusions, getSandbox: vi.fn(() => ({ name: "alpha", agent: "hermes" })), @@ -40,6 +45,10 @@ vi.mock("../../state/sandbox", () => ({ findBackup: mocks.findBackup, })); +vi.mock("./snapshot/dependencies", () => ({ + backupSandboxStateWithManagedAuthority: mocks.backupSandboxState, +})); + vi.mock("./sandbox-gateway-routing", () => ({ probeGatewayRunning: vi.fn(() => true), selectSandboxGatewayIfRegistered: vi.fn(() => true), @@ -71,7 +80,6 @@ describe("snapshot baseline exclusion output", () => { it("reports active exclusions and support impact after a successful snapshot (#7178)", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); - const { runSandboxSnapshot } = await import("./snapshot"); await runSandboxSnapshot("alpha", { kind: "create" }); diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 3fa93ae6f6f..52370d755fd 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -326,6 +326,20 @@ function expectStagedDriverNeutralRecovery( return output; } +function expectManagedPolicyCleanup(harness: ShieldsHarness): void { + expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( + expect.stringContaining("nemoclaw-permissive-runtime"), + "nemoclaw-permissive-runtime", + ); + expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); + const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); + expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); +} + +function writeOpenClawShieldsState(stateDir: string, state: Record): void { + fs.writeFileSync(path.join(stateDir, "shields-openclaw.json"), JSON.stringify(state)); +} + function writeExpiredShieldsFixture( processToken: string, reason: string, @@ -472,13 +486,7 @@ describe("shields command flow", () => { }), ).toThrow("Cannot start auto-restore timer: timer startup failed"); - expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( - expect.stringContaining("nemoclaw-permissive-runtime"), - "nemoclaw-permissive-runtime", - ); - expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); - const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); - expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); + expectManagedPolicyCleanup(harness); }); it("cleans the staged managed MCP policy when state persistence fails", () => { @@ -501,13 +509,7 @@ describe("shields command flow", () => { }), ).toThrow(/EISDIR|directory/i); - expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( - expect.stringContaining("nemoclaw-permissive-runtime"), - "nemoclaw-permissive-runtime", - ); - expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); - const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); - expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); + expectManagedPolicyCleanup(harness); }); it("timer restore uses persisted MCP ownership after its transition marker clears (#7952)", () => { @@ -526,14 +528,11 @@ describe("shields command flow", () => { }, }), ); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], + }); const harness = createHarness({ livePolicy: YAML.stringify({ version: 1, @@ -565,14 +564,11 @@ describe("shields command flow", () => { const snapshotPath = path.join(stateDir, "policy-snapshot-corrupt-state.yaml"); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: ["../not-a-managed-key"], - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["../not-a-managed-key"], + }); const harness = createHarness(); expect(() => harness.applyShieldsPolicySnapshot("openclaw", snapshotPath)).toThrow( @@ -588,13 +584,10 @@ describe("shields command flow", () => { fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(expectedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); fs.writeFileSync(requestedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsPolicySnapshotPath: expectedSnapshotPath, - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: true, + shieldsPolicySnapshotPath: expectedSnapshotPath, + }); const harness = createHarness(); expect(() => harness.applyShieldsPolicySnapshot("openclaw", requestedSnapshotPath)).toThrow( @@ -618,14 +611,11 @@ describe("shields command flow", () => { }), ); fs.writeFileSync(oldSnapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: false, - shieldsPolicySnapshotPath: oldSnapshotPath, - shieldsManagedMcpPolicyKeys: [], - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: false, + shieldsPolicySnapshotPath: oldSnapshotPath, + shieldsManagedMcpPolicyKeys: [], + }); fs.writeFileSync( path.join(stateDir, `shields-transition-openclaw-${processToken}.json`), JSON.stringify({ @@ -667,14 +657,11 @@ describe("shields command flow", () => { ); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(snapshotPath, YAML.stringify({ network_policies: networkPolicies })); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: keys, - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: keys, + }); const harness = createHarness({ livePolicy: YAML.stringify({ version: 1, network_policies: networkPolicies }), sandboxEntry: managedMcpSandbox(policies), @@ -1122,17 +1109,14 @@ describe("shields command flow", () => { const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), - shieldsDownTimeout: 300, - shieldsDownReason: "coverage", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: path.join(stateDir, "missing-snapshot.yaml"), - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "coverage", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: path.join(stateDir, "missing-snapshot.yaml"), + }); expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( "Saved policy snapshot is missing", @@ -1162,17 +1146,14 @@ describe("shields command flow", () => { const snapshotPath = path.join(stateDir, "policy-snapshot-failed-restore.yaml"); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsDownAt: new Date().toISOString(), - shieldsDownTimeout: 300, - shieldsDownReason: "recovery-hint coverage", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: true, + shieldsDownAt: new Date().toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "recovery-hint coverage", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }); expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( "policy restore exited with status 1", @@ -1228,73 +1209,57 @@ describe("shields command flow", () => { expect(output).not.toContain("CRITICAL: OpenClaw lock rollback"); }); - it("retains critical recovery for non-transient OpenClaw rollback failures (#6126)", () => { - const harness = createHarness({ - failOpenClawGuardActions: ["lock"], - openClawGuardFailure: { - code: "unsafe-config-path", - path: "/sandbox/.openclaw/openclaw.json", - detail: "canonical config path is not a safe regular file", - }, - }); - - expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( - /unsafe-config-path/, - ); - - const output = harness.errorSpy.mock.calls.flat().map(String).join("\n"); - expect(output).toContain( - "CRITICAL: OpenClaw lock rollback could not restore the trusted posture. Restore from a trusted backup and recreate the sandbox.", - ); - expect(output).not.toContain( - "Warning: OpenClaw lock rollback could not restore the trusted posture", - ); - }); - - it("retains critical recovery for structural startup-not-ready diagnostics (#6126)", () => { - const harness = createHarness({ - failOpenClawGuardActions: ["lock"], - openClawGuardFailure: { - code: "startup-not-ready", - path: "/run/nemoclaw/openclaw-config-ready.json", - detail: "installed config guard requires NemoClaw PID 1", - }, - }); - - expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( - /requires NemoClaw PID 1/, - ); - - const output = harness.errorSpy.mock.calls.flat().map(String).join("\n"); - expect(output).toContain( - "CRITICAL: OpenClaw lock rollback could not restore the trusted posture. Restore from a trusted backup and recreate the sandbox.", - ); - expect(output).not.toContain( - "Warning: OpenClaw lock rollback could not restore the trusted posture", - ); - }); - - it("retains critical recovery when a transient diagnostic is followed by another issue (#6126)", () => { - const harness = createHarness({ - failOpenClawGuardActions: ["lock"], - openClawGuardFailures: [ - { - code: "startup-not-ready", - path: "/run/nemoclaw/openclaw-config-ready.json", - detail: "OpenClaw startup is not ready for host config mutations", - }, - { + it.each<{ + scenario: string; + options: HarnessOptions; + expectedError: RegExp; + }>([ + { + scenario: "non-transient OpenClaw rollback failures", + options: { + failOpenClawGuardActions: ["lock"], + openClawGuardFailure: { code: "unsafe-config-path", path: "/sandbox/.openclaw/openclaw.json", detail: "canonical config path is not a safe regular file", }, - ], - }); - - expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( - /unsafe-config-path/, - ); - + }, + expectedError: /unsafe-config-path/, + }, + { + scenario: "structural startup-not-ready diagnostics", + options: { + failOpenClawGuardActions: ["lock"], + openClawGuardFailure: { + code: "startup-not-ready", + path: "/run/nemoclaw/openclaw-config-ready.json", + detail: "installed config guard requires NemoClaw PID 1", + }, + }, + expectedError: /requires NemoClaw PID 1/, + }, + { + scenario: "a transient diagnostic followed by another issue", + options: { + failOpenClawGuardActions: ["lock"], + openClawGuardFailures: [ + { + code: "startup-not-ready", + path: "/run/nemoclaw/openclaw-config-ready.json", + detail: "OpenClaw startup is not ready for host config mutations", + }, + { + code: "unsafe-config-path", + path: "/sandbox/.openclaw/openclaw.json", + detail: "canonical config path is not a safe regular file", + }, + ], + }, + expectedError: /unsafe-config-path/, + }, + ])("retains critical recovery for $scenario (#6126)", ({ options, expectedError }) => { + const harness = createHarness(options); + expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow(expectedError); const output = harness.errorSpy.mock.calls.flat().map(String).join("\n"); expect(output).toContain( "CRITICAL: OpenClaw lock rollback could not restore the trusted posture. Restore from a trusted backup and recreate the sandbox.", @@ -1308,17 +1273,14 @@ describe("shields command flow", () => { const harness = createHarness({ failOpenClawGuardActions: ["lock"] }); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: false, - chattrApplied: false, - fileHashes: { - "/sandbox/.openclaw/openclaw.json": "a".repeat(64), - "/sandbox/.openclaw/.config-hash": "a".repeat(64), - }, - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: false, + chattrApplied: false, + fileHashes: { + "/sandbox/.openclaw/openclaw.json": "a".repeat(64), + "/sandbox/.openclaw/.config-hash": "a".repeat(64), + }, + }); expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( /startup-not-ready/, @@ -1335,17 +1297,14 @@ describe("shields command flow", () => { const markerPath = path.join(stateDir, "shields-timer-openclaw.json"); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsDownAt: new Date().toISOString(), - shieldsDownTimeout: 1800, - shieldsDownReason: "rebuild", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, - }), - ); + writeOpenClawShieldsState(stateDir, { + shieldsDown: true, + shieldsDownAt: new Date().toISOString(), + shieldsDownTimeout: 1800, + shieldsDownReason: "rebuild", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }); fs.writeFileSync( markerPath, JSON.stringify({ @@ -1431,7 +1390,13 @@ describe("shields command flow", () => { }); const harness = createHarness({ beginContainment: () => { - throw new Error("state directory is read-only"); + const error = new Error("state directory is read-only") as Error & { + code: string; + retainOwnedLifecycleGates: boolean; + }; + error.code = "NEMOCLAW_DURABLE_CONTAINMENT"; + error.retainOwnedLifecycleGates = true; + throw error; }, }); let containmentFailure: unknown; @@ -1447,6 +1412,7 @@ describe("shields command flow", () => { expect(result).toBe("handled"); expect(containmentFailure).toMatchObject({ code: "NEMOCLAW_DURABLE_CONTAINMENT", + retainOwnedLifecycleGates: true, }); expect(String(containmentFailure)).toContain("state directory is read-only"); expect(fs.existsSync(containmentPath)).toBe(false); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 883dc90d213..52f9f2a741f 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -307,12 +307,20 @@ function persistUnresolvedShieldsContainment( sandboxName: string, processToken: string, reason: string, + assertTakeoverAuthority?: () => void, ): void { const containmentPath = `${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`; if (fs.existsSync(containmentPath)) return; try { - beginCommittedMcpLifecycleContainmentSync(sandboxName, processToken, reason, STATE_DIR); + beginCommittedMcpLifecycleContainmentSync( + sandboxName, + processToken, + reason, + STATE_DIR, + assertTakeoverAuthority, + ); } catch (error) { + if (isDurableContainmentFailure(error)) throw error; if (fs.existsSync(containmentPath)) return; throw error; } @@ -366,8 +374,11 @@ function waitForShieldsDownForwardCommit( `Shields recovery owner PID ${String( observed.ownerPid, )} exited without descendant-containment proof`, + assertTakeoverAuthority, ); } catch (error) { + if (isDurableContainmentFailure(error)) throw error; + assertTakeoverAuthority?.(); throw durableMcpLifecycleContainmentFailure( error, getMcpLifecycleLockPath(sandboxName, STATE_DIR), @@ -972,9 +983,11 @@ function failInteractiveAutoRestoreClosed( sandboxName, marker.processToken, `Interactive auto-restore could not complete safely: ${message}`, + () => assertTimerMarkerGeneration(sandboxName, marker), ); break; } catch (error) { + if (isDurableContainmentFailure(error)) throw error; if (fs.existsSync(containmentPath)) break; assertTimerMarkerGeneration(sandboxName, marker); const containmentError = error instanceof Error ? error.message : String(error); @@ -1040,6 +1053,7 @@ function retryInlineAutoRestore( notifiedError = message; } } catch (error) { + if (isDurableContainmentFailure(error)) throw error; assertTimerMarkerGeneration(sandboxName, marker); const message = error instanceof Error ? error.message : String(error); if (message !== notifiedError) { @@ -1117,11 +1131,9 @@ function withExpiredAutoRestoreDeadlineFence( ); return recoverThenRun(); } catch (error) { + if (isDurableContainmentFailure(error)) throw error; assertTakeoverAuthority(); - if ( - isDurableContainmentFailure(error) || - fs.existsSync(`${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`) - ) { + if (fs.existsSync(`${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`)) { throw error; } const message = error instanceof Error ? error.message : String(error); @@ -2736,8 +2748,11 @@ function prepareAutoRestoreTransitionTakeover( sandboxName, processToken, `Shields recovery owner PID ${String(owner.pid)} exited without descendant-containment proof`, + assertTakeoverAuthority, ); } catch (error) { + if (isDurableContainmentFailure(error)) throw error; + assertTakeoverAuthority?.(); throw durableMcpLifecycleContainmentFailure( error, getMcpLifecycleLockPath(sandboxName, STATE_DIR), @@ -3196,6 +3211,7 @@ function recoverExpiredAutoRestoreInline( assertTakeoverAuthority: () => assertTimerMarkerGeneration(sandboxName, marker), }); } catch (error) { + if (isDurableContainmentFailure(error)) throw error; const message = error instanceof Error ? error.message : String(error); appendAuditEntry({ action: "shields_up_failed", @@ -3214,6 +3230,7 @@ function recoverExpiredAutoRestoreInline( const activation = activateLockdownFromSnapshot( sandboxName, marker.snapshotPath, + marker.allowLegacyHermesProtocol === true, cachedTarget, undefined, marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken) @@ -3529,7 +3546,6 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = } clearTimerMarker(sandboxName); clearShieldsDownTransition(sandboxName, processToken); - cleanupRuntimePolicyFile(); const message = err instanceof Error ? err.message : String(err); console.error(` Cannot start auto-restore timer: ${message}`); return failShieldsCommand(`Cannot start auto-restore timer: ${message}`, opts.throwOnError); @@ -3551,7 +3567,6 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = clearShieldsDownTransition(sandboxName, transition.processToken); killTimer(sandboxName); } - cleanupRuntimePolicyFile(); throw error; } diff --git a/src/lib/shields/mcp-policy-transition.test.ts b/src/lib/shields/mcp-policy-transition.test.ts index 181a97bbac3..f754e7e2452 100644 --- a/src/lib/shields/mcp-policy-transition.test.ts +++ b/src/lib/shields/mcp-policy-transition.test.ts @@ -297,10 +297,29 @@ describe("managed MCP Shields policy transitions (#7952)", () => { getSandbox: () => sandbox ?? null, }), ).toThrow( - /Reserved MCP policy key 'mcp_bridge_retired'.*no committed managed bridge ownership/, + /Reserved MCP policy key "mcp_bridge_retired".*no committed managed bridge ownership/, ); }); + it("escapes unclassified live policy keys in operator diagnostics", () => { + const maliciousKey = "mcp_bridge_\u001b[31mforged\nline\u0085"; + + let failure: unknown; + try { + inspectRegisteredManagedMcpPolicies("alpha", livePolicy([], { [maliciousKey]: {} }), { + getSandbox: () => null, + }); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain( + String.raw`"mcp_bridge_\u001b[31mforged\u000aline\u0085"`, + ); + expect((failure as Error).message).not.toMatch(/[\u0000-\u001f\u007f-\u009f]/u); + }); + it("retains additions while restoring the restrictive snapshot", () => { const alpha = registeredPolicy("alpha", "8.8.8.8"); const beta = registeredPolicy("beta", "1.1.1.1"); diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index f3b5c7f1098..a3e56a64d32 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -6,7 +6,11 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { getMcpLifecycleLockPath, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; +import { + beginCommittedMcpLifecycleContainmentSync, + getMcpLifecycleLockPath, + withMcpLifecycleLock, +} from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ applyShieldsPolicySnapshot: vi.fn( @@ -23,6 +27,11 @@ const shieldsIndexMock = vi.hoisted(() => ({ const PROCESS_TOKEN = "a".repeat(32); +interface TimerTestOptions { + retryDelayMs?: number; + maxRestoreAttempts?: number; +} + vi.mock("./index", () => ({ applyShieldsPolicySnapshot: shieldsIndexMock.applyShieldsPolicySnapshot, completeAutoRestoreTransition: shieldsIndexMock.completeAutoRestoreTransition, @@ -72,15 +81,16 @@ describe("shields timer authorization", () => { }); async function invokeTimerAndCaptureExit( - runRestoreTimer: (args: any, options?: { retryDelayMs?: number }) => Promise, + runRestoreTimer: (args: any, options?: TimerTestOptions) => Promise, args: unknown, + options: TimerTestOptions = { retryDelayMs: 1 }, ): Promise { const exitSpy = vi.spyOn(process, "exit").mockImplementation((code?: any) => { throw new Error(`process.exit:${String(code ?? 0)}`); }); try { - await runRestoreTimer(args, { retryDelayMs: 1 }); + await runRestoreTimer(args, options); throw new Error("Expected runRestoreTimer to exit"); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -98,12 +108,12 @@ describe("shields timer authorization", () => { expect(fs.existsSync(deadlinePath)).toBe(true); expect(fs.existsSync(auditPath)).toBe(true); }, - { interval: 1, timeout: 200 }, + { interval: 1, timeout: 2_000 }, ); } async function invokeTimerAndExpectRetry( - runRestoreTimer: (args: any, options?: { retryDelayMs?: number }) => Promise, + runRestoreTimer: (args: any, options?: TimerTestOptions) => Promise, args: unknown, ): Promise { const exitSpy = vi @@ -119,24 +129,65 @@ describe("shields timer authorization", () => { path.join(tmpHome, ".nemoclaw", "state"), )}.deadline`; const auditPath = path.join(path.dirname(markerPath), "shields-audit.jsonl"); + const pending = runRestoreTimer(args, { retryDelayMs: 50 }); + let policyApplicationsBeforeRevocation: number | undefined; try { - const pending = runRestoreTimer(args, { retryDelayMs: 50 }); await waitForRetryBoundary(deadlinePath, auditPath); expect(exitSpy).not.toHaveBeenCalled(); expect(fs.existsSync(deadlinePath)).toBe(true); - const policyApplicationsBeforeRevocation = + policyApplicationsBeforeRevocation = shieldsIndexMock.applyShieldsPolicySnapshot.mock.calls.length; + } finally { fs.rmSync(markerPath, { force: true }); await pending; + fs.writeFileSync(markerPath, markerContents); + exitSpy.mockRestore(); + } + if (policyApplicationsBeforeRevocation !== undefined) { expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes( policyApplicationsBeforeRevocation, ); - } finally { - fs.writeFileSync(markerPath, markerContents); - exitSpy.mockRestore(); } } + function createFailedRestoreFixture( + sandboxName: string, + parseTimerArgs: (argv: string[]) => unknown, + ) { + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const mutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + const writeMarker = (processToken: string) => + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken, + }), + ); + writeMarker(PROCESS_TOKEN); + shieldsIndexMock.applyShieldsPolicySnapshot.mockReturnValue({ status: 1 }); + const args = parseTimerArgs([sandboxName, snapshotPath, restoreAtIso, "", "", PROCESS_TOKEN]); + expect(args).not.toBeNull(); + return { + args, + containmentPath: `${mutationLockPath}.containment`, + deadlinePath: `${mutationLockPath}.deadline`, + markerPath, + mutationLockPath, + sandboxName, + stateDir, + writeMarker, + }; + } + it("does not restore or rewrite state when marker is missing", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); @@ -766,6 +817,190 @@ describe("shields timer authorization", () => { } }); + it("commits durable containment after the default seven-attempt auto-restore budget", async () => { + const timer = await import("./timer"); + const fixture = createFailedRestoreFixture("retry-containment", timer.parseTimerArgs); + const { + args, + containmentPath, + deadlinePath, + markerPath, + mutationLockPath, + sandboxName, + stateDir, + } = fixture; + const auditPath = path.join(stateDir, "shields-audit.jsonl"); + + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args, { + retryDelayMs: 0, + }); + + expect(exitCode).toBe(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(7); + expect(fs.existsSync(markerPath)).toBe(true); + expect(fs.existsSync(mutationLockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + expect(JSON.parse(fs.readFileSync(containmentPath, "utf-8"))).toMatchObject({ + sandboxName, + shieldsTakeoverToken: PROCESS_TOKEN, + containmentReason: expect.stringContaining("failed after 7 attempts"), + }); + await expect(withMcpLifecycleLock(sandboxName, () => undefined, { stateDir })).rejects.toThrow( + "Sandbox mutation containment is active", + ); + const auditEntries = fs + .readFileSync(auditPath, "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(auditEntries).toHaveLength(8); + expect(auditEntries.at(-1)).toMatchObject({ + action: "shields_up_failed", + sandbox: sandboxName, + error: expect.stringContaining("Durable containment now blocks sandbox mutations"), + }); + }); + + it("reclaims its exact containment generation when timer authority changes during publication", async () => { + const timer = await import("./timer"); + const fixture = createFailedRestoreFixture( + "containment-publication-revoked", + timer.parseTimerArgs, + ); + const { args, containmentPath, deadlinePath, markerPath, mutationLockPath, writeMarker } = + fixture; + const replacementToken = "f".repeat(32); + const originalLink = fs.linkSync.bind(fs); + const linkSpy = vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { + const result = originalLink(existingPath, newPath); + if (String(newPath) === containmentPath) writeMarker(replacementToken); + return result; + }); + + try { + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args, { + retryDelayMs: 0, + maxRestoreAttempts: 1, + }); + expect(exitCode).toBe(1); + } finally { + linkSpy.mockRestore(); + } + + expect(JSON.parse(fs.readFileSync(markerPath, "utf-8"))).toMatchObject({ + processToken: replacementToken, + }); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(fs.existsSync(mutationLockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + }); + + it("retains the exact deadline gates when revoked containment rollback cannot be proven", async () => { + const timer = await import("./timer"); + const fixture = createFailedRestoreFixture( + "containment-publication-rollback-failure", + timer.parseTimerArgs, + ); + const { args, containmentPath, deadlinePath, markerPath, mutationLockPath, writeMarker } = + fixture; + const replacementToken = "e".repeat(32); + const originalLink = fs.linkSync.bind(fs); + const linkSpy = vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { + const result = originalLink(existingPath, newPath); + if (String(newPath) === containmentPath) writeMarker(replacementToken); + return result; + }); + const originalRename = fs.renameSync.bind(fs); + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((oldPath, newPath) => { + if (String(oldPath) === containmentPath) { + const error = new Error("simulated exact rollback failure") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + } + return originalRename(oldPath, newPath); + }); + + try { + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args, { + retryDelayMs: 0, + maxRestoreAttempts: 1, + }); + expect(exitCode).toBe(1); + } finally { + renameSpy.mockRestore(); + linkSpy.mockRestore(); + } + + expect(JSON.parse(fs.readFileSync(markerPath, "utf-8"))).toMatchObject({ + processToken: replacementToken, + }); + expect(fs.existsSync(containmentPath)).toBe(true); + expect(fs.existsSync(mutationLockPath)).toBe(true); + expect(fs.existsSync(deadlinePath)).toBe(true); + }); + + it("retains the exact deadline gates when durable containment cannot be committed", async () => { + const timer = await import("./timer"); + const { args, containmentPath, deadlinePath, markerPath, mutationLockPath, stateDir } = + createFailedRestoreFixture("retry-containment-failure", timer.parseTimerArgs); + const originalLink = fs.linkSync.bind(fs); + const linkSpy = vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { + if (String(newPath) === containmentPath) { + const error = new Error("simulated containment commit failure") as NodeJS.ErrnoException; + error.code = "EROFS"; + throw error; + } + return originalLink(existingPath, newPath); + }); + + try { + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args, { + retryDelayMs: 1, + maxRestoreAttempts: 1, + }); + expect(exitCode).toBe(1); + } finally { + linkSpy.mockRestore(); + } + + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); + expect(fs.existsSync(markerPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(fs.existsSync(mutationLockPath)).toBe(true); + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8")).toContain( + "Correct the state-directory write failure", + ); + }); + + it("stops retrying when transition takeover commits durable containment", async () => { + const timer = await import("./timer"); + const { args, markerPath, mutationLockPath, sandboxName, stateDir } = + createFailedRestoreFixture("takeover-containment", timer.parseTimerArgs); + shieldsIndexMock.prepareAutoRestoreTransitionTakeover.mockImplementationOnce(() => { + beginCommittedMcpLifecycleContainmentSync( + sandboxName, + PROCESS_TOKEN, + "transition takeover requires operator recovery", + stateDir, + ); + throw new Error("transition takeover stopped"); + }); + + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args, { + retryDelayMs: 1, + maxRestoreAttempts: 3, + }); + + expect(exitCode).toBe(1); + expect(shieldsIndexMock.prepareAutoRestoreTransitionTakeover).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(fs.existsSync(markerPath)).toBe(true); + expect(fs.existsSync(mutationLockPath)).toBe(false); + expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(false); + expect(fs.existsSync(`${mutationLockPath}.containment`)).toBe(true); + }); + it("retains recovery authority when the locked-state commit cannot be persisted", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index c75b817325b..14e45eb4e13 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -11,7 +11,12 @@ import fs from "node:fs"; import path from "node:path"; import { isObjectRecord, type UnknownRecord } from "../core/json-types"; -import { withMcpLifecycleDeadlineFence } from "../state/mcp-lifecycle-lock"; +import { + beginCommittedMcpLifecycleContainmentSync, + durableMcpLifecycleContainmentFailure, + getMcpLifecycleLockPath, + withMcpLifecycleDeadlineFence, +} from "../state/mcp-lifecycle-lock"; import { readShieldsTimerMarkerFile, type ShieldsTimerMarker, @@ -52,6 +57,7 @@ interface TimerArgs { interface TimerRuntimeOptions { retryDelayMs?: number; + maxRestoreAttempts?: number; } type LockAgentConfig = typeof shields.lockAgentConfig; @@ -59,6 +65,7 @@ type RestoreAttemptOutcome = "complete" | "retry" | "revoked"; const STATE_DIR = resolveNemoclawStateDir(); const AUTO_RESTORE_RETRY_MS = 5_000; +const AUTO_RESTORE_MAX_ATTEMPTS = 7; function parseTimerArgs(argv: string[]): TimerArgs | null { const [ @@ -114,6 +121,13 @@ function appendAudit(entry: ShieldsAuditEntry): void { } } +function isDurableContainmentFailure(error: unknown): boolean { + return ( + error instanceof Error && + (error as Error & { code?: string }).code === "NEMOCLAW_DURABLE_CONTAINMENT" + ); +} + function readStateFile(stateFile: string): UnknownRecord { try { if (!fs.existsSync(stateFile)) { @@ -242,8 +256,14 @@ async function runRestoreTimer( Number.isFinite(runtimeOptions.retryDelayMs) && (runtimeOptions.retryDelayMs ?? 0) >= 0 ? Math.floor(runtimeOptions.retryDelayMs!) : AUTO_RESTORE_RETRY_MS; + const maxRestoreAttempts = + Number.isInteger(runtimeOptions.maxRestoreAttempts) && + (runtimeOptions.maxRestoreAttempts ?? 0) > 0 + ? runtimeOptions.maxRestoreAttempts! + : AUTO_RESTORE_MAX_ATTEMPTS; let exitCode = 0; let retryScheduled = false; + let terminalContainment = false; let managedMcpWarning: string | undefined; const scheduleRetry = (): boolean => { if (!markerMatchesCurrentTimer(args)) return false; @@ -465,7 +485,7 @@ async function runRestoreTimer( }, ); const restoreWhileDeadlineOwned = async (): Promise => { - for (;;) { + for (let attempt = 1; ; attempt += 1) { let outcome: RestoreAttemptOutcome; try { shields.prepareAutoRestoreTransitionTakeover( @@ -476,6 +496,11 @@ async function runRestoreTimer( ); outcome = restoreUnderDeadlineFence(); } catch (error) { + const lockPath = getMcpLifecycleLockPath(args.sandboxName, STATE_DIR); + if (isDurableContainmentFailure(error) || fs.existsSync(`${lockPath}.containment`)) { + terminalContainment = true; + throw durableMcpLifecycleContainmentFailure(error, lockPath); + } appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, @@ -489,6 +514,40 @@ async function runRestoreTimer( } if (outcome !== "retry") return; if (!markerMatchesCurrentTimer(args)) return; + if (attempt >= maxRestoreAttempts) { + const lockPath = getMcpLifecycleLockPath(args.sandboxName, STATE_DIR); + const reason = `Auto-restore failed after ${String(attempt)} attempts while the exact timer generation retained the lifecycle deadline`; + try { + beginCommittedMcpLifecycleContainmentSync( + args.sandboxName, + args.processToken!, + reason, + STATE_DIR, + assertTakeoverAuthority, + ); + } catch (error) { + if (isDurableContainmentFailure(error)) { + terminalContainment = true; + throw error; + } + if (!markerMatchesCurrentTimer(args)) throw error; + terminalContainment = true; + const message = error instanceof Error ? error.message : String(error); + throw durableMcpLifecycleContainmentFailure( + new Error( + `${reason}; durable containment could not be committed: ${message}. Correct the state-directory write failure, then retry a NemoClaw command for this sandbox to obtain exact-generation recovery guidance`, + ), + lockPath, + ); + } + terminalContainment = true; + throw durableMcpLifecycleContainmentFailure( + new Error( + `${reason}. Durable containment now blocks sandbox mutations. Stop all NemoClaw processes for this sandbox, then follow the exact-generation recovery guidance from the next NemoClaw command.`, + ), + lockPath, + ); + } await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); if (!markerMatchesCurrentTimer(args)) return; } @@ -514,6 +573,7 @@ async function runRestoreTimer( }, ); } catch (error: unknown) { + if (isDurableContainmentFailure(error)) terminalContainment = true; appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, @@ -522,7 +582,7 @@ async function runRestoreTimer( error: error instanceof Error ? error.message : String(error), }); exitCode = 1; - scheduleRetry(); + if (!terminalContainment) scheduleRetry(); } finally { if (!retryScheduled) process.exit(exitCode); } diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts index 56459cea218..d53edcec57c 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts @@ -292,6 +292,36 @@ describe("MCP lifecycle lock acquisition", () => { expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); }); + it("releases async owned generations when committed containment is proven present", async () => { + const processToken = "a".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + + await expect( + withMcpLifecycleDeadlineFence( + SANDBOX_NAME, + processToken, + () => { + beginCommittedMcpLifecycleContainmentSync( + SANDBOX_NAME, + processToken, + "test async containment", + stateDir, + ); + throw durableMcpLifecycleContainmentFailure( + new Error("async containment reporting stopped"), + lockPath, + ); + }, + options(), + ), + ).rejects.toThrow("async containment reporting stopped"); + + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + it("retains owned generations when committed containment cannot be inspected", () => { const processToken = "2".repeat(32); const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index ac898f55e83..00ae611ebd1 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -117,6 +117,7 @@ function beginCommittedContainmentAtPathSync( sandboxName: string, takeoverToken: string | undefined, reason: string, + assertAuthority?: () => void, ): void { const containmentPath = committedContainmentPath(lockPath); const token = crypto.randomUUID(); @@ -124,6 +125,7 @@ function beginCommittedContainmentAtPathSync( ...createMcpLifecycleLockOwner(sandboxName, token, takeoverToken), containmentReason: reason, }; + assertAuthority?.(); fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); if (!writeMcpLifecycleLockCandidateAndLinkSync(containmentPath, owner)) { throw new Error( @@ -132,8 +134,27 @@ function beginCommittedContainmentAtPathSync( } try { fsyncLockDirectorySync(containmentPath); + assertAuthority?.(); } catch (error) { - safelyReleaseMcpLifecycleLockSync(containmentPath, token); + let rolledBack = false; + try { + const published = readMcpLifecycleLockObservationSync(containmentPath); + if (published?.owner?.token === token) { + rolledBack = reclaimStaleMcpLifecycleLockGenerationSync(containmentPath, published); + if (rolledBack) fsyncLockDirectorySync(containmentPath); + } + } catch { + rolledBack = false; + } + if (!rolledBack) { + throw durableMcpLifecycleContainmentFailure( + new Error( + `Containment publication for sandbox '${sandboxName}' lost authority or durability, and its exact generation could not be rolled back`, + ), + lockPath, + { retainOwnedLifecycleGates: true }, + ); + } throw error; } } @@ -168,6 +189,7 @@ export function beginCommittedMcpLifecycleContainmentSync( takeoverToken: string, reason: string, stateDir = resolveNemoclawStateDir(), + assertAuthority?: () => void, ): void { if (!/^[0-9a-f]{32}$/.test(takeoverToken)) { throw new Error("Auto-restore takeover token must be 32 lowercase hexadecimal characters"); @@ -177,6 +199,7 @@ export function beginCommittedMcpLifecycleContainmentSync( sandboxName, takeoverToken, reason, + assertAuthority, ); } @@ -693,21 +716,32 @@ function isDurableContainmentError(error: unknown): boolean { ); } +function durableContainmentMustRetainOwnedLifecycleGates(error: unknown): boolean { + return ( + isDurableContainmentError(error) && + (error as Error & { retainOwnedLifecycleGates?: boolean }).retainOwnedLifecycleGates === true + ); +} + export function durableMcpLifecycleContainmentFailure( error: unknown, lockPath: string, -): Error & { code: string } { + options: { retainOwnedLifecycleGates?: boolean } = {}, +): Error & { code: string; retainOwnedLifecycleGates?: boolean } { const lease = heldLocks.getStore()?.get(lockPath); if (lease?.active) lease.retainForDurableContainment = true; if (isDurableContainmentError(error)) { - return error as Error & { code: string }; + const failure = error as Error & { code: string; retainOwnedLifecycleGates?: boolean }; + if (options.retainOwnedLifecycleGates) failure.retainOwnedLifecycleGates = true; + return failure; } const failure = new Error( `Durable sandbox mutation containment requires operator resolution: ${ error instanceof Error ? error.message : String(error) }`, - ) as Error & { code: string }; + ) as Error & { code: string; retainOwnedLifecycleGates?: boolean }; failure.code = "NEMOCLAW_DURABLE_CONTAINMENT"; + if (options.retainOwnedLifecycleGates) failure.retainOwnedLifecycleGates = true; return failure; } @@ -736,11 +770,13 @@ async function retainOwnedLifecycleGateAfterFailure( lockPath: string, ): Promise { if (!isDurableContainmentError(error)) return false; + if (durableContainmentMustRetainOwnedLifecycleGates(error)) return true; return await ownedLifecycleGateMustRemainClosed(lockPath); } function retainOwnedLifecycleGateAfterFailureSync(error: unknown, lockPath: string): boolean { if (!isDurableContainmentError(error)) return false; + if (durableContainmentMustRetainOwnedLifecycleGates(error)) return true; return ownedLifecycleGateMustRemainClosedSync(lockPath); } @@ -1462,7 +1498,10 @@ export function withMcpLifecycleLockSync( if (inherited?.get(lockPath)?.active) return operation(); const acquired = acquireMcpLifecycleLockSync(sandboxName, { ...options, stateDir }); - const lease: HeldLockLease = { active: true, retainForDurableContainment: false }; + const lease: HeldLockLease = { + active: true, + retainForDurableContainment: false, + }; const context = new Map(inherited ?? []); context.set(lockPath, lease); let retainOwnedGate = false; @@ -1511,7 +1550,10 @@ export async function withMcpLifecycleLock( ...options, stateDir, }); - const lease: HeldLockLease = { active: true, retainForDurableContainment: false }; + const lease: HeldLockLease = { + active: true, + retainForDurableContainment: false, + }; const context = new Map(inherited ?? []); context.set(lockKey, lease); return heldLocks.run(context, async () => { From f0f888bc8907e66d7623ae606d6336fe7a62d480 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 3 Aug 2026 22:27:26 -0400 Subject: [PATCH 5/8] fix(shields): share detached recovery budget Count deadline setup, publication, and restoration against one bounded retry budget. On exhaustion, commit durable containment. If that commit fails, retain exact owned gates and return actionable recovery guidance. Signed-off-by: Julie Yaunches --- docs/manage-sandboxes/backup-restore.mdx | 5 +- docs/manage-sandboxes/runtime-controls.mdx | 9 +- docs/reference/commands.mdx | 19 +- src/lib/shields/timer-recovery-budget.test.ts | 268 ++++++++++++++++++ src/lib/shields/timer.ts | 120 +++++--- .../state/mcp-lifecycle-lock-acquisition.ts | 82 ++++-- 6 files changed, 434 insertions(+), 69 deletions(-) create mode 100644 src/lib/shields/timer-recovery-budget.test.ts diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 9713d405ac2..ff8162d65d8 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -94,7 +94,10 @@ Snapshot creation and restore share the per-sandbox transition lock with the shi If a timed shields-down window expires during snapshot work, auto-restore closes the per-sandbox lifecycle deadline gate. The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation. NemoClaw does not signal the snapshot process. -If ownership becomes ambiguous, NemoClaw records durable containment and reports exact-generation recovery guidance. +If ownership becomes ambiguous, NemoClaw attempts to record durable containment and reports exact-generation recovery guidance. +If the containment commit fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. +A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. +Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. Stop all NemoClaw processes for the sandbox, then follow that guidance before you retry the snapshot. Tag a snapshot with a human-readable label: diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 02e8f8f9e0e..119d7125f75 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -123,14 +123,17 @@ An MCP server removed during the shields-down window stays removed. A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. An interactive command can take over an expired timer. -The interactive takeover and the detached auto-restore timer each make up to 7 restoration attempts, with 5 seconds between attempts and up to 30 seconds of retry delay. +The interactive takeover makes up to 7 restoration attempts. +The detached auto-restore timer makes up to 7 total recovery attempts across deadline setup and restoration. +Both paths wait 5 seconds between failed attempts, for up to 30 seconds of retry delay. The deadline gate remains closed during those attempts. If restoration cannot commit, NemoClaw attempts to record durable containment. -If that containment commit also fails, NemoClaw retains the exact lifecycle and deadline gates instead. +If that containment commit also fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. +A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. The interactive command returns an error, or the detached timer exits with a failure status. NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. -Durable containment, or the retained exact gates after a failed containment commit, keeps new mutations blocked until you complete exact-generation operator recovery. +Durable containment, retained exact gates, or the fail-closed state-directory error keeps new mutations blocked until you complete exact-generation operator recovery. Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index bc64246d186..c39d79753ba 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1177,14 +1177,17 @@ When a timed shields-down window reaches its deadline, auto-restore closes the p The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation before auto-restore restores lockdown. NemoClaw does not signal that process because portable process inspection cannot prove that all descendants stopped. An interactive command can take over an expired timer. -The interactive takeover and the detached auto-restore timer each make up to 7 restoration attempts, with 5 seconds between attempts and up to 30 seconds of retry delay. +The interactive takeover makes up to 7 restoration attempts. +The detached auto-restore timer makes up to 7 total recovery attempts across deadline setup and restoration. +Both paths wait 5 seconds between failed attempts, for up to 30 seconds of retry delay. The deadline gate remains closed during those attempts. If restoration cannot commit, NemoClaw attempts to record durable containment. -If that containment commit also fails, NemoClaw retains the exact lifecycle and deadline gates instead. +If that containment commit also fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. +A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. The interactive command returns an error, or the detached timer exits with a failure status. NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. -Durable containment, or the retained exact gates after a failed containment commit, blocks new mutations until you complete exact-generation operator recovery. +Durable containment, retained exact gates, or the fail-closed state-directory error blocks new mutations until you complete exact-generation operator recovery. Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. @@ -2967,7 +2970,10 @@ Snapshots are stored in `~/.nemoclaw/rebuild-backups//`. The command requires shields to be down and keeps the shields check and backup under one per-sandbox transition. If the timer expires during a long-running backup, the deadline gate blocks new mutations and waits for the exact backup owner to finish. Auto-restore does not signal the backup process. -If ownership becomes ambiguous, NemoClaw records durable containment and reports exact-generation recovery guidance. +If ownership becomes ambiguous, NemoClaw attempts to record durable containment and reports exact-generation recovery guidance. +If the containment commit fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. +A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. +Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. When the sandbox has active baseline exclusions, successful output lists their keys and repeats that excluded egress leaves dependent agent features unsupported for that sandbox. ```bash @@ -3003,7 +3009,10 @@ It preserves directories that exist only in the target manifest or whose backup The state replacement, mutable-config permission repair, and policy reconciliation run under the same per-sandbox transition. If the timer expires during that work, the deadline gate blocks new mutations and waits for the exact restore owner to finish. Auto-restore does not signal the restore process. -If ownership becomes ambiguous, NemoClaw records durable containment and reports exact-generation recovery guidance. +If ownership becomes ambiguous, NemoClaw attempts to record durable containment and reports exact-generation recovery guidance. +If the containment commit fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. +A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. +Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. The selector accepts any of: diff --git a/src/lib/shields/timer-recovery-budget.test.ts b/src/lib/shields/timer-recovery-budget.test.ts new file mode 100644 index 00000000000..1b2651d1983 --- /dev/null +++ b/src/lib/shields/timer-recovery-budget.test.ts @@ -0,0 +1,268 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + beginCommittedMcpLifecycleContainmentSync, + getMcpLifecycleLockPath, + withMcpLifecycleLock, +} from "../state/mcp-lifecycle-lock"; + +const shieldsIndexMock = vi.hoisted(() => ({ + applyShieldsPolicySnapshot: vi.fn(() => ({ status: 0 })), + completeAutoRestoreTransition: vi.fn(() => true), + lockAgentConfig: vi.fn(), + prepareAutoRestoreTransitionTakeover: vi.fn(), + resolvePersistedAutoRestoreTarget: vi.fn(), +})); + +vi.mock("./index", () => shieldsIndexMock); + +const PROCESS_TOKEN = "a".repeat(32); + +describe("detached Shields recovery budget", () => { + let tmpHome: string; + let stateDir: string; + + beforeEach(() => { + tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-recovery-budget-")); + stateDir = path.join(tmpHome, ".nemoclaw", "state"); + vi.stubEnv("HOME", tmpHome); + vi.resetModules(); + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(tmpHome, { recursive: true, force: true }); + }); + + async function createFixture(sandboxName: string) { + const timer = await import("./timer"); + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + }), + ); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + ]); + expect(args).not.toBeNull(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + return { args: args!, lockPath, markerPath, sandboxName, timer }; + } + + function readAuditEntries(): Array<{ error?: string }> { + return fs + .readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + } + + it("shares seven attempts across pre-fence retries and then stops scheduling", async () => { + const { args, lockPath, sandboxName, timer } = await createFixture("pre-fence-budget"); + fs.writeFileSync(path.dirname(lockPath), "blocks the lifecycle lock directory"); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + + await timer.runRestoreTimer(args, { retryDelayMs: 0, maxRestoreAttempts: 7 }); + await vi.waitFor(() => expect(exitSpy).toHaveBeenCalledWith(1), { + interval: 1, + timeout: 2_000, + }); + + const auditsAtExit = readAuditEntries(); + expect(auditsAtExit).toHaveLength(7); + expect( + auditsAtExit.filter((entry) => entry.error?.includes("recovery failed after 7 attempts")), + ).toHaveLength(1); + expect(auditsAtExit.at(-1)?.error).toContain("recovery failed after 7 attempts"); + expect(auditsAtExit.at(-1)?.error).toContain("Correct the state-directory write failure"); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + await expect( + withMcpLifecycleLock(sandboxName, () => undefined, { stateDir }), + ).rejects.toThrow(); + + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(exitSpy).toHaveBeenCalledTimes(1); + expect(readAuditEntries()).toHaveLength(7); + }); + + it("shares the budget across scheduled setup failures and restoration", async () => { + const { args, lockPath, timer } = await createFixture("cross-phase-budget"); + const containmentPath = `${lockPath}.containment`; + const lifecycleDirectory = path.dirname(lockPath); + const originalMkdir = fs.promises.mkdir.bind(fs.promises); + let setupFailuresRemaining = 2; + vi.spyOn(fs.promises, "mkdir").mockImplementation(async (targetPath, options) => { + if (String(targetPath) === lifecycleDirectory && setupFailuresRemaining > 0) { + setupFailuresRemaining -= 1; + const error = new Error("simulated pre-fence setup failure") as NodeJS.ErrnoException; + error.code = "EIO"; + throw error; + } + return await originalMkdir(targetPath, options); + }); + shieldsIndexMock.applyShieldsPolicySnapshot.mockReturnValue({ status: 1 }); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + + await timer.runRestoreTimer(args, { retryDelayMs: 0, maxRestoreAttempts: 7 }); + await vi.waitFor(() => expect(exitSpy).toHaveBeenCalledWith(1), { + interval: 1, + timeout: 2_000, + }); + + expect(setupFailuresRemaining).toBe(0); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(5); + expect(fs.existsSync(containmentPath)).toBe(true); + expect( + readAuditEntries().filter((entry) => + entry.error?.includes("recovery failed after 7 attempts"), + ), + ).toHaveLength(1); + }); + + it("charges deadline-main publication failures to the same bounded budget", async () => { + const { args, lockPath, sandboxName, timer } = await createFixture("publication-budget"); + const containmentPath = `${lockPath}.containment`; + const deadlinePath = `${lockPath}.deadline`; + const originalLink = fs.promises.link.bind(fs.promises); + let mainPublicationAttempts = 0; + vi.spyOn(fs.promises, "link").mockImplementation(async (existingPath, newPath) => { + if (String(newPath) === lockPath) { + mainPublicationAttempts += 1; + const error = new Error( + "simulated deadline-main publication failure", + ) as NodeJS.ErrnoException; + error.code = "EROFS"; + throw error; + } + return await originalLink(existingPath, newPath); + }); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + + await timer.runRestoreTimer(args, { retryDelayMs: 0, maxRestoreAttempts: 3 }); + + expect(exitSpy).toHaveBeenCalledTimes(1); + expect(exitSpy).toHaveBeenCalledWith(1); + expect(mainPublicationAttempts).toBe(3); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(fs.existsSync(deadlinePath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(true); + expect(readAuditEntries()).toHaveLength(2); + expect(readAuditEntries().at(-1)?.error).toContain("recovery failed after 3 attempts"); + await expect(withMcpLifecycleLock(sandboxName, () => undefined, { stateDir })).rejects.toThrow( + "Sandbox mutation containment is active", + ); + }); + + it("retains its exact deadline when publication and containment both fail", async () => { + const { args, lockPath, markerPath, sandboxName, timer } = await createFixture( + "publication-retained-gate", + ); + const containmentPath = `${lockPath}.containment`; + const deadlinePath = `${lockPath}.deadline`; + const originalAsyncLink = fs.promises.link.bind(fs.promises); + vi.spyOn(fs.promises, "link").mockImplementation(async (existingPath, newPath) => { + if (String(newPath) === lockPath) { + const error = new Error( + "simulated deadline-main publication failure", + ) as NodeJS.ErrnoException; + error.code = "EROFS"; + throw error; + } + return await originalAsyncLink(existingPath, newPath); + }); + const originalSyncLink = fs.linkSync.bind(fs); + vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { + if (String(newPath) === containmentPath) { + const error = new Error( + "simulated containment publication failure", + ) as NodeJS.ErrnoException; + error.code = "EROFS"; + throw error; + } + return originalSyncLink(existingPath, newPath); + }); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + let contenderEntered = false; + + await timer.runRestoreTimer(args, { retryDelayMs: 0, maxRestoreAttempts: 1 }); + + expect(exitSpy).toHaveBeenCalledTimes(1); + expect(exitSpy).toHaveBeenCalledWith(1); + expect(fs.existsSync(markerPath)).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(JSON.parse(fs.readFileSync(deadlinePath, "utf-8"))).toMatchObject({ + sandboxName, + shieldsTakeoverToken: PROCESS_TOKEN, + }); + const audits = readAuditEntries(); + expect(audits).toHaveLength(1); + expect(audits[0]?.error).toContain("recovery failed after 1 attempt"); + expect(audits[0]?.error).toContain("Correct the state-directory write failure"); + expect(audits[0]?.error).not.toContain("setup is retrying"); + await expect( + withMcpLifecycleLock( + sandboxName, + () => { + contenderEntered = true; + }, + { stateDir, pollIntervalMs: 1, timeoutMs: 10 }, + ), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + expect(contenderEntered).toBe(false); + }); + + it("exits immediately when durable containment already owns recovery", async () => { + const { args, lockPath, sandboxName, timer } = await createFixture("existing-containment"); + beginCommittedMcpLifecycleContainmentSync( + sandboxName, + PROCESS_TOKEN, + "existing exact-generation containment", + stateDir, + ); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + + await timer.runRestoreTimer(args, { retryDelayMs: 0, maxRestoreAttempts: 7 }); + + expect(exitSpy).toHaveBeenCalledTimes(1); + expect(exitSpy).toHaveBeenCalledWith(1); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + const audits = readAuditEntries(); + expect(audits).toHaveLength(1); + expect(audits[0]?.error).toContain("committed process-tree containment"); + }); +}); diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index 14e45eb4e13..920996533c6 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -60,6 +60,10 @@ interface TimerRuntimeOptions { maxRestoreAttempts?: number; } +interface RecoveryAttemptBudget { + attemptsUsed: number; +} + type LockAgentConfig = typeof shields.lockAgentConfig; type RestoreAttemptOutcome = "complete" | "retry" | "revoked"; @@ -248,9 +252,10 @@ function rebuildLeaseOwnerIsCurrent(args: TimerArgs): boolean { ); } -async function runRestoreTimer( +async function runRestoreTimerWithBudget( args: TimerArgs, - runtimeOptions: TimerRuntimeOptions = {}, + runtimeOptions: TimerRuntimeOptions, + recoveryBudget: RecoveryAttemptBudget, ): Promise { const retryDelayMs = Number.isFinite(runtimeOptions.retryDelayMs) && (runtimeOptions.retryDelayMs ?? 0) >= 0 @@ -269,10 +274,48 @@ async function runRestoreTimer( if (!markerMatchesCurrentTimer(args)) return false; retryScheduled = true; setTimeout(() => { - void runRestoreTimer(args, runtimeOptions); + void runRestoreTimerWithBudget(args, runtimeOptions, recoveryBudget); }, retryDelayMs); return true; }; + const assertTakeoverAuthority = (): void => { + if (!markerMatchesCurrentTimer(args)) { + throw new Error("Auto-restore authority changed before Shields transition takeover"); + } + }; + const terminalRecoveryFailure = (attempt: number): Error => { + const lockPath = getMcpLifecycleLockPath(args.sandboxName, STATE_DIR); + const reason = `Auto-restore recovery failed after ${String(attempt)} ${attempt === 1 ? "attempt" : "attempts"} while the exact timer generation still owned recovery authority`; + try { + beginCommittedMcpLifecycleContainmentSync( + args.sandboxName, + args.processToken!, + reason, + STATE_DIR, + assertTakeoverAuthority, + ); + } catch (error) { + if (isDurableContainmentFailure(error)) return error as Error; + if (!markerMatchesCurrentTimer(args)) { + return error instanceof Error ? error : new Error(String(error)); + } + const message = error instanceof Error ? error.message : String(error); + return durableMcpLifecycleContainmentFailure( + new Error( + `${reason}; durable containment could not be committed: ${message}. Correct the state-directory write failure, then retry a NemoClaw command for this sandbox to obtain exact-generation recovery guidance`, + ), + lockPath, + { retainOwnedLifecycleGates: true }, + ); + } + return durableMcpLifecycleContainmentFailure( + new Error( + `${reason}. Durable containment now blocks sandbox mutations. Stop all NemoClaw processes for this sandbox, then follow the exact-generation recovery guidance from the next NemoClaw command.`, + ), + lockPath, + ); + }; + const attemptsAtEntry = recoveryBudget.attemptsUsed; try { // Timer markers are the source of authority. If the marker was removed or @@ -295,11 +338,6 @@ async function runRestoreTimer( if (!args.processToken || !/^[0-9a-f]{32}$/.test(args.processToken)) { throw new Error("Auto-restore timer has no valid transition takeover token"); } - const assertTakeoverAuthority = (): void => { - if (!markerMatchesCurrentTimer(args)) { - throw new Error("Auto-restore authority changed before Shields transition takeover"); - } - }; const restoreUnderDeadlineFence = (): RestoreAttemptOutcome => withShieldsTransitionLock( args.sandboxName, @@ -485,7 +523,8 @@ async function runRestoreTimer( }, ); const restoreWhileDeadlineOwned = async (): Promise => { - for (let attempt = 1; ; attempt += 1) { + for (;;) { + const attempt = (recoveryBudget.attemptsUsed += 1); let outcome: RestoreAttemptOutcome; try { shields.prepareAutoRestoreTransitionTakeover( @@ -515,38 +554,8 @@ async function runRestoreTimer( if (outcome !== "retry") return; if (!markerMatchesCurrentTimer(args)) return; if (attempt >= maxRestoreAttempts) { - const lockPath = getMcpLifecycleLockPath(args.sandboxName, STATE_DIR); - const reason = `Auto-restore failed after ${String(attempt)} attempts while the exact timer generation retained the lifecycle deadline`; - try { - beginCommittedMcpLifecycleContainmentSync( - args.sandboxName, - args.processToken!, - reason, - STATE_DIR, - assertTakeoverAuthority, - ); - } catch (error) { - if (isDurableContainmentFailure(error)) { - terminalContainment = true; - throw error; - } - if (!markerMatchesCurrentTimer(args)) throw error; - terminalContainment = true; - const message = error instanceof Error ? error.message : String(error); - throw durableMcpLifecycleContainmentFailure( - new Error( - `${reason}; durable containment could not be committed: ${message}. Correct the state-directory write failure, then retry a NemoClaw command for this sandbox to obtain exact-generation recovery guidance`, - ), - lockPath, - ); - } terminalContainment = true; - throw durableMcpLifecycleContainmentFailure( - new Error( - `${reason}. Durable containment now blocks sandbox mutations. Stop all NemoClaw processes for this sandbox, then follow the exact-generation recovery guidance from the next NemoClaw command.`, - ), - lockPath, - ); + throw terminalRecoveryFailure(attempt); } await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); if (!markerMatchesCurrentTimer(args)) return; @@ -560,6 +569,13 @@ async function runRestoreTimer( stateDir: STATE_DIR, pollIntervalMs: 50, timeoutMs: 5_000, + throwOnCommittedContainment: true, + onSetupFailure: async () => { + const attempt = (recoveryBudget.attemptsUsed += 1); + if (attempt >= maxRestoreAttempts) throw terminalRecoveryFailure(attempt); + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + assertTakeoverAuthority(); + }, onContainment: ({ ownerPid, reason }) => { appendAudit({ action: "shields_up_failed", @@ -573,21 +589,39 @@ async function runRestoreTimer( }, ); } catch (error: unknown) { - if (isDurableContainmentFailure(error)) terminalContainment = true; + let reportedError = error; + if (isDurableContainmentFailure(reportedError)) { + terminalContainment = true; + } else if (markerMatchesCurrentTimer(args)) { + if (recoveryBudget.attemptsUsed === attemptsAtEntry) { + recoveryBudget.attemptsUsed += 1; + } + if (recoveryBudget.attemptsUsed >= maxRestoreAttempts) { + reportedError = terminalRecoveryFailure(recoveryBudget.attemptsUsed); + terminalContainment = isDurableContainmentFailure(reportedError); + } + } appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, timestamp: new Date().toISOString(), restored_by: "auto_timer", - error: error instanceof Error ? error.message : String(error), + error: reportedError instanceof Error ? reportedError.message : String(reportedError), }); exitCode = 1; - if (!terminalContainment) scheduleRetry(); + if (!terminalContainment && recoveryBudget.attemptsUsed < maxRestoreAttempts) scheduleRetry(); } finally { if (!retryScheduled) process.exit(exitCode); } } +async function runRestoreTimer( + args: TimerArgs, + runtimeOptions: TimerRuntimeOptions = {}, +): Promise { + return await runRestoreTimerWithBudget(args, runtimeOptions, { attemptsUsed: 0 }); +} + function main(): void { const args = parseTimerArgs(process.argv.slice(2)); if (!args) { diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index 00ae611ebd1..688575b849c 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -54,6 +54,10 @@ interface AcquiredMcpLifecycleLock { export interface McpLifecycleDeadlineFenceOptions extends McpLifecycleLockOptions { /** Audit an owner that keeps the deadline gate closed while it exits naturally. */ onContainment?: (details: McpLifecycleDeadlineContainment) => Promise | void; + /** Account for a recoverable deadline acquisition or main-publication failure. */ + onSetupFailure?: (error: unknown) => Promise | void; + /** Return operator guidance instead of waiting when durable containment already exists. */ + throwOnCommittedContainment?: boolean; } export interface McpLifecycleDeadlineFenceSyncOptions extends McpLifecycleLockOptions { @@ -841,6 +845,11 @@ async function acquireDeadlineFence( throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); } if (await mcpLifecycleLockPathExists(committedContainmentPath(lockPath))) { + if (options.throwOnCommittedContainment) { + const containmentPath = committedContainmentPath(lockPath); + const reason = `A committed process-tree containment requires operator resolution before auto-restore can continue. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${deadlinePath}', and '${containmentPath}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; + throw durableMcpLifecycleContainmentFailure(new Error(reason), lockPath); + } if ( notifiedGeneration !== "committed-containment" && performance.now() - blockedAt >= timeoutMs @@ -889,17 +898,33 @@ async function acquireDeadlineFence( const generation = `${String(observation.dev)}:${String(observation.ino)}:${ observation.owner?.token ?? "invalid" }`; - if (generation !== notifiedGeneration && performance.now() - blockedAt >= timeoutMs) { - await reportDeadlineContainment(options, { - ownerPid: observation.owner?.pid ?? null, - reason: - "Another auto-restore deadline owner is active or cannot be verified; the deadline gate remains closed pending operator or distributed-lease resolution.", - }); - notifiedGeneration = generation; + if (performance.now() - blockedAt >= timeoutMs) { + const reason = + "Another auto-restore deadline owner is active or cannot be verified; the deadline gate remains closed pending operator or distributed-lease resolution."; + if (options.onSetupFailure) { + blockedAt = performance.now(); + await options.onSetupFailure(new Error(reason)); + } + if (generation !== notifiedGeneration) { + await reportDeadlineContainment(options, { + ownerPid: observation.owner?.pid ?? null, + reason, + }); + notifiedGeneration = generation; + } + if (options.onSetupFailure) { + continue; + } } } else { blockedAt = performance.now(); notifiedGeneration = null; + if (options.onSetupFailure) { + await options.onSetupFailure( + new Error("Auto-restore could not publish or inspect its lifecycle deadline gate"), + ); + continue; + } } await sleep(pollIntervalMs); } @@ -1056,15 +1081,22 @@ async function clearDeadlineProtectedPath( const generation = `${String(observed.dev)}:${String(observed.ino)}:${ owner?.token ?? "invalid" }`; - if ( - generation !== notifiedGeneration && - performance.now() - blockedAt >= containmentTimeoutMs - ) { - await reportDeadlineContainment(options, { - ownerPid: owner?.pid ?? null, - reason: `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`, - }); - notifiedGeneration = generation; + if (performance.now() - blockedAt >= containmentTimeoutMs) { + const reason = `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`; + if (options.onSetupFailure) { + blockedAt = performance.now(); + await options.onSetupFailure(new Error(reason)); + } + if (generation !== notifiedGeneration) { + await reportDeadlineContainment(options, { + ownerPid: owner?.pid ?? null, + reason, + }); + notifiedGeneration = generation; + } + if (options.onSetupFailure) { + continue; + } } await sleep(pollIntervalMs); } @@ -1211,16 +1243,28 @@ async function publishDeadlineMainOwner( } pendingCandidateToken = null; notifiedError = null; + if (options.onSetupFailure) { + await options.onSetupFailure( + new Error("Auto-restore could not publish its deadline-owned mutation generation"), + ); + continue; + } } catch (error) { if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); } const message = error instanceof Error ? error.message : String(error); if (isDurableContainmentError(error)) { + const resolutionReason = `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; + if (options.throwOnCommittedContainment) { + const failure = durableMcpLifecycleContainmentFailure(error, lockPath); + failure.message = resolutionReason; + throw failure; + } if (message !== notifiedError) { await reportDeadlineContainment(options, { ownerPid: null, - reason: `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`, + reason: resolutionReason, }); notifiedError = message; } @@ -1236,6 +1280,7 @@ async function publishDeadlineMainOwner( } continue; } + if (options.onSetupFailure) await options.onSetupFailure(error); if (message !== notifiedError) { await reportDeadlineContainment(options, { ownerPid: null, @@ -1243,6 +1288,9 @@ async function publishDeadlineMainOwner( }); notifiedError = message; } + if (options.onSetupFailure) { + continue; + } } await sleep(pollIntervalMs); } From 6a43160378f051351973a320c2aaa6cb608f3e3d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 11:13:22 -0400 Subject: [PATCH 6/8] refactor(shields): isolate deadline prerequisite Signed-off-by: Julie Yaunches --- ci/source-architecture-budget.json | 4 +- docs/manage-sandboxes/backup-restore.mdx | 13 +- docs/manage-sandboxes/runtime-controls.mdx | 20 +- docs/reference/commands.mdx | 31 +- .../checks/openshell-policy-mutation-read.mts | 488 ++++++++++--- src/lib/actions/maintenance.test.ts | 110 ++- src/lib/actions/maintenance.ts | 269 +++---- src/lib/actions/sandbox/mcp-bridge-policy.ts | 464 ------------ ...snapshot-baseline-exclusion-output.test.ts | 10 +- src/lib/shields/flow.test.ts | 668 ++++++----------- src/lib/shields/index.test.ts | 92 --- src/lib/shields/index.ts | 638 ++++------------ src/lib/shields/mcp-policy-transition.test.ts | 685 ------------------ src/lib/shields/mcp-policy-transition.ts | 182 ----- src/lib/shields/permissive-runtime.ts | 150 +--- src/lib/shields/policy-transition.test.ts | 207 ------ src/lib/shields/timer-process.test.ts | 121 ++++ src/lib/shields/timer-recovery-budget.test.ts | 188 +++-- src/lib/shields/timer.test.ts | 188 ++--- src/lib/shields/timer.ts | 81 ++- .../state/mcp-lifecycle-lock-acquisition.ts | 32 +- .../shields-timer-authority.test.ts | 48 ++ .../shields-timer-authority.ts | 3 +- test/e2e/live/mcp-bridge-hermes-lifecycle.ts | 1 + test/e2e/live/mcp-bridge-sandbox.ts | 76 -- test/e2e/live/mcp-bridge.test.ts | 84 +-- test/e2e/support/mcp-bridge-sandbox.test.ts | 66 +- test/permissive-runtime.test.ts | 69 +- test/policy-mutation-read-discovery.test.ts | 288 +++++++- 29 files changed, 1793 insertions(+), 3483 deletions(-) delete mode 100644 src/lib/shields/mcp-policy-transition.test.ts delete mode 100644 src/lib/shields/mcp-policy-transition.ts create mode 100644 src/lib/shields/timer-process.test.ts create mode 100644 src/lib/state/mcp-lifecycle-lock/shields-timer-authority.test.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index fd0dc346a43..e5c1566b367 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -11,7 +11,7 @@ "src/lib/adapters/openshell/runtime.ts": 52, "src/lib/adapters/openshell/timeouts.ts": 37, "src/lib/agent/defs.ts": 32, - "src/lib/cli/branding.ts": 85, + "src/lib/cli/branding.ts": 86, "src/lib/cli/nemoclaw-oclif-command.ts": 103, "src/lib/cli/terminal-style.ts": 45, "src/lib/core/json-types.ts": 37, @@ -24,7 +24,7 @@ "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, "src/lib/onboard/gateway-binding.ts": 49, - "src/lib/runner.ts": 88, + "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 36, "src/lib/state/registry.ts": 101, diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index ff8162d65d8..8ed720bd6bb 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -97,8 +97,8 @@ NemoClaw does not signal the snapshot process. If ownership becomes ambiguous, NemoClaw attempts to record durable containment and reports exact-generation recovery guidance. If the containment commit fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. -Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. -Stop all NemoClaw processes for the sandbox, then follow that guidance before you retry the snapshot. +Correct the reported state-directory write failure, then run `$$nemoclaw shields status` to resume recovery or receive exact-generation recovery guidance. +If the command reports exact-generation recovery guidance, stop all NemoClaw processes for the sandbox, then follow that guidance before you retry the snapshot. Tag a snapshot with a human-readable label: @@ -213,13 +213,12 @@ If a registered docker-driver sandbox's container is stopped, `backup-all` start If the container cannot be returned to the stopped state, the backup run fails and reports that the container was left running. If a sandbox is not running and its container cannot be started this way, start the sandbox or its container and rerun `$$nemoclaw backup-all`. -When an eligible sandbox starts with Shields up, `backup-all` acquires the lifecycle lock and opens a 30-minute shields-down window. +For each eligible sandbox, `backup-all` holds one lifecycle transaction through the complete backup. +Within that transaction, it starts a stopped container when required, opens a 30-minute shields-down window when the sandbox starts with Shields up, copies sandbox state, restores the previous Shields state, and returns any container it started to the stopped state. A sandbox that starts with Shields down remains down. -`backup-all` reacquires the lock under that exact timer generation while it copies sandbox state. -After the copy, it uses a third lock interval with a 30-second acquisition budget to restore the previous Shields posture. -If the timer expires during the copy, the deadline gate blocks new mutations and waits for the exact backup owner to finish without signaling it. +If the timer expires during the transaction, the deadline gate blocks new mutations and waits for the exact backup owner to finish without signaling it. An initial lock or unlock failure marks that sandbox as failed, and `backup-all` continues with the next sandbox. -NemoClaw attempts to restore the previous Shields posture before it processes the next sandbox, including when the backup fails. +NemoClaw attempts to restore the previous Shields state before it processes the next sandbox, including when the backup fails. If lockdown cannot be restored, `backup-all` stops and does not process the remaining sandboxes. Correct the reported issue, run the printed `$$nemoclaw shields up` command, and rerun `$$nemoclaw backup-all`. diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 119d7125f75..b53392256e4 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -111,27 +111,19 @@ When `shields down --timeout` is active, each mutation binds to that exact timer If the timeout expires while a mutation is changing sandbox state, auto-restore closes the per-sandbox lifecycle deadline gate. The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation. NemoClaw does not signal that process because portable process inspection cannot prove that all descendants stopped. -After the owner releases the lock, auto-restore restores the restrictive policy and config posture. +After the owner releases the lock, auto-restore restores the restrictive policy and configuration posture. The ownership check includes both the process ID and process start identity so PID reuse does not grant control over an unrelated process. -Before a manual Shields transition replaces a policy, NemoClaw requires exact Model Context Protocol (MCP) agreement among the sandbox registry, generated-policy record, and live gateway policy. -`shields down` carries the proven managed MCP policy entries into the relaxed policy. -Restoration removes snapshot-time managed MCP entries before it overlays current exact entries. -If exact agreement is absent, a manual Shields transition refuses the replacement policy. -At an expired deadline, auto-restore omits unproven managed MCP policy entries, restores lockdown, and records the omission count in its audit entry. -An MCP server removed during the shields-down window stays removed. -A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. - An interactive command can take over an expired timer. -The interactive takeover makes up to 7 restoration attempts. -The detached auto-restore timer makes up to 7 total recovery attempts across deadline setup and restoration. -Both paths wait 5 seconds between failed attempts, for up to 30 seconds of retry delay. +Interactive recovery has separate transition-takeover and restoration phases. +Each phase makes up to 7 attempts and waits 5 seconds between failures, for up to 30 seconds of retry delay per phase. +Detached recovery uses one 7-attempt budget across deadline setup, main-generation publication, and restoration. The deadline gate remains closed during those attempts. If restoration cannot commit, NemoClaw attempts to record durable containment. If that containment commit also fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. -Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. -The interactive command returns an error, or the detached timer exits with a failure status. +Correct the reported state-directory write failure, then run `$$nemoclaw shields status` to resume recovery or receive exact-generation recovery guidance. +When recovery cannot complete, an interactive command returns an error, or the detached timer exits with a failure status. NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. Durable containment, retained exact gates, or the fail-closed state-directory error keeps new mutations blocked until you complete exact-generation operator recovery. Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 1ee4323d5fd..b1fad839081 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1177,28 +1177,20 @@ When a timed shields-down window reaches its deadline, auto-restore closes the p The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation before auto-restore restores lockdown. NemoClaw does not signal that process because portable process inspection cannot prove that all descendants stopped. An interactive command can take over an expired timer. -The interactive takeover makes up to 7 restoration attempts. -The detached auto-restore timer makes up to 7 total recovery attempts across deadline setup and restoration. -Both paths wait 5 seconds between failed attempts, for up to 30 seconds of retry delay. +Interactive recovery has separate transition-takeover and restoration phases. +Each phase makes up to 7 attempts and waits 5 seconds between failures, for up to 30 seconds of retry delay per phase. +Detached recovery uses one 7-attempt budget across deadline setup, main-generation publication, and restoration. The deadline gate remains closed during those attempts. If restoration cannot commit, NemoClaw attempts to record durable containment. If that containment commit also fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. -Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. -The interactive command returns an error, or the detached timer exits with a failure status. +Correct the reported state-directory write failure, then run `$$nemoclaw shields status` to resume recovery or receive exact-generation recovery guidance. +When recovery cannot complete, an interactive command returns an error, or the detached timer exits with a failure status. NemoClaw also records durable containment when an owner exits before it can prove that the owner's descendants stopped, or when ownership becomes ambiguous. Durable containment, retained exact gates, or the fail-closed state-directory error blocks new mutations until you complete exact-generation operator recovery. Stop all NemoClaw processes for the sandbox, then follow the paths, identities, tokens, and removal order in the reported error. Verify each recorded generation is unchanged, remove only the exact stale generations first, and remove the exact containment generation last. -Before a manual Shields transition replaces a policy, NemoClaw requires exact agreement among the sandbox registry, generated-policy record, and live gateway policy. -`shields down` carries the proven managed MCP policy entries into the relaxed policy. -Restoration removes snapshot-time managed MCP entries before it overlays current exact entries. -If exact agreement is absent, a manual Shields transition refuses the replacement policy. -At an expired deadline, auto-restore omits unproven managed MCP policy entries, restores lockdown, and records the omission count in its audit entry. -An MCP server removed during the shields-down window stays removed. -A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. - ### `$$nemoclaw recover` @@ -2961,12 +2953,11 @@ A registered docker-driver sandbox whose container is stopped is started for the If the container cannot be returned to the stopped state, the command fails and reports that the container was left running. Sandboxes that are not running and cannot be started this way are skipped with remediation guidance. -For each eligible sandbox, `backup-all` acquires the lifecycle lock to open a 30-minute shields-down window when needed. -It reacquires the lock under that exact timer generation while it copies sandbox state. -After the copy, it uses a third lock interval with a 30-second acquisition budget to restore the previous Shields posture. -If the timer expires during the copy, the deadline gate blocks new mutations and waits for the exact backup owner to finish without signaling it. +For each eligible sandbox, `backup-all` holds one lifecycle transaction through the complete backup. +Within that transaction, it starts a stopped container when required, opens a 30-minute shields-down window when the sandbox starts with Shields up, copies sandbox state, restores the previous Shields state, and returns any container it started to the stopped state. +If the timer expires during the transaction, the deadline gate blocks new mutations and waits for the exact backup owner to finish without signaling it. An initial lock or unlock failure marks that sandbox as failed, and `backup-all` continues with the next sandbox. -A failure to restore the previous Shields posture stops `backup-all` before it processes another sandbox. +A failure to restore the previous Shields state stops `backup-all` before it processes another sandbox. ```bash $$nemoclaw backup-all @@ -2992,7 +2983,7 @@ Auto-restore does not signal the backup process. If ownership becomes ambiguous, NemoClaw attempts to record durable containment and reports exact-generation recovery guidance. If the containment commit fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. -Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. +Correct the reported state-directory write failure, then run `$$nemoclaw shields status` to resume recovery or receive exact-generation recovery guidance. When the sandbox has active baseline exclusions, successful output lists their keys and repeats that excluded egress leaves dependent agent features unsupported for that sandbox. ```bash @@ -3031,7 +3022,7 @@ Auto-restore does not signal the restore process. If ownership becomes ambiguous, NemoClaw attempts to record durable containment and reports exact-generation recovery guidance. If the containment commit fails, NemoClaw retains any exact lifecycle and deadline gates it already owns. A state-directory failure that prevented gate publication also prevents normal mutation-lock acquisition. -Correct the reported state-directory write failure, then retry a NemoClaw command for the sandbox to obtain exact-generation recovery guidance. +Correct the reported state-directory write failure, then run `$$nemoclaw shields status` to resume recovery or receive exact-generation recovery guidance. The selector accepts any of: diff --git a/scripts/checks/openshell-policy-mutation-read.mts b/scripts/checks/openshell-policy-mutation-read.mts index 63fc1e5db7b..8fb536ba96e 100644 --- a/scripts/checks/openshell-policy-mutation-read.mts +++ b/scripts/checks/openshell-policy-mutation-read.mts @@ -11,10 +11,10 @@ * exhaustive discovery and classification of their production call sites. * whyNotSourceFix: TypeScript cannot distinguish a command array after it * crosses the process runner, so this defense-in-depth check intentionally uses - * deterministic source patterns plus repository-wide read-site discovery. + * deterministic AST classifications plus repository-wide read-site discovery. * regressionTest: test/policy-mutation-read-discovery.test.ts injects * unaccounted reads and requires this audit to fail. - * removalCondition: replace the source-pattern table when mutation and + * removalCondition: replace the AST classification table when mutation and * diagnostic commands carry enforced tagged types through the runner boundary. */ @@ -26,65 +26,101 @@ import ts from "typescript"; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); -interface AuditedMutationRead { +export type PolicyReadView = "base" | "full"; +export type PolicyReadFailureHandling = "error-preserving" | "ignore-error" | "unclassified"; + +export interface DiscoveredPolicyRead { + readonly site: string; + readonly view: PolicyReadView; + readonly failureHandling: PolicyReadFailureHandling; +} + +interface AuditedPolicyReadFile { readonly relativePath: string; - readonly expectedReadCalls: number; - readonly baseCommand: string; - readonly unsafeBaseCommand?: string; - readonly fullCommand: string; - readonly diagnosticFullRead?: string; + readonly expectedReads: readonly DiscoveredPolicyRead[]; +} + +function preservingBase(site: string): DiscoveredPolicyRead { + return { site, view: "base", failureHandling: "error-preserving" }; +} + +function ignoredBase(site: string): DiscoveredPolicyRead { + return { site, view: "base", failureHandling: "ignore-error" }; +} + +function unclassifiedBase(site: string): DiscoveredPolicyRead { + return { site, view: "base", failureHandling: "unclassified" }; } -export const MUTATION_READS: readonly AuditedMutationRead[] = [ +function ignoredFull(site: string): DiscoveredPolicyRead { + return { site, view: "full", failureHandling: "ignore-error" }; +} + +export const MUTATION_READS: readonly AuditedPolicyReadFile[] = [ { relativePath: "src/lib/actions/sandbox/policy-get.ts", - expectedReadCalls: 1, - baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", - fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName))", + expectedReads: [preservingBase("getSandboxPolicy")], }, { relativePath: "src/lib/policy/index.ts", - expectedReadCalls: 7, - baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", - unsafeBaseCommand: "runCapture(buildPolicyGetCommand(sandboxName), { ignoreError: true })", - fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName), { ignoreError: true })", - diagnosticFullRead: "runCapture(buildPolicyGetFullCommand(sandboxName), { ignoreError: true })", + expectedReads: [ + ignoredBase("removePreset"), + ignoredBase("readCurrentSandboxPolicy"), + ignoredBase("applyPresetContent"), + ignoredBase("applyPresets"), + preservingBase("customPresetOwnsNetworkPolicyKey"), + ignoredFull("getGatewayPresets/readPolicy"), + preservingBase("getPresetContentGatewayState/readPolicy"), + ], }, { relativePath: "nemoclaw/src/blueprint/runner.ts", - expectedReadCalls: 1, - baseCommand: '["openshell", "policy", "get", "--base", sandboxName]', - fullCommand: '["openshell", "policy", "get", "--full", sandboxName]', + expectedReads: [unclassifiedBase("actionApply")], }, { relativePath: "src/lib/shields/index.ts", - expectedReadCalls: 3, - baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", - unsafeBaseCommand: "runCapture(buildPolicyGetCommand(sandboxName), {", - fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName))", + expectedReads: [ignoredBase("shieldsDownWithoutHostLock")], }, ]; -const NON_MUTATION_POLICY_READS = [ +const NON_MUTATION_POLICY_READS: readonly AuditedPolicyReadFile[] = [ { relativePath: "src/lib/actions/sandbox/gateway-state.ts", - expectedReadCalls: 2, + expectedReads: [ + ignoredFull("getSandboxGatewayState"), + ignoredFull("getSandboxGatewayStateForStatus"), + ], }, { relativePath: "src/lib/policy/commands.ts", - expectedReadCalls: 2, + expectedReads: [ + { + site: "buildPolicyGetCommand", + view: "base", + failureHandling: "unclassified", + }, + { + site: "buildPolicyGetFullCommand", + view: "full", + failureHandling: "unclassified", + }, + ], }, ] as const; export interface DiscoveredPolicyReadSite { readonly relativePath: string; readonly readCalls: number; + readonly reads: readonly DiscoveredPolicyRead[]; } -const POLICY_GET_BUILDERS = new Set(["buildPolicyGetCommand", "buildPolicyGetFullCommand"]); +const POLICY_GET_BUILDERS = new Map([ + ["buildPolicyGetCommand", "base"], + ["buildPolicyGetFullCommand", "full"], +]); interface PolicyBuilderBindings { - readonly identifiers: ReadonlySet; + readonly identifiers: ReadonlyMap; readonly namespaces: ReadonlySet; } @@ -144,7 +180,7 @@ function collectRequiredPolicyBindings( fileName: string, repoRoot: string, checker: ts.TypeChecker, - identifiers: Set, + identifiers: Map, namespaces: Set, ): void { const moduleSpecifier = requireModuleSpecifier(declaration.initializer, checker); @@ -155,16 +191,26 @@ function collectRequiredPolicyBindings( return; } if (!ts.isObjectBindingPattern(declaration.name)) return; - for (const element of declaration.name.elements) { - if (element.dotDotDotToken || !ts.isIdentifier(element.name)) continue; - const importedName = element.propertyName ?? element.name; - if ( - (ts.isIdentifier(importedName) || ts.isStringLiteralLike(importedName)) && - POLICY_GET_BUILDERS.has(importedName.text) - ) { + collectPolicyBuilderObjectBindings(declaration.name, checker, identifiers, namespaces); +} + +function collectPolicyBuilderObjectBindings( + pattern: ts.ObjectBindingPattern, + checker: ts.TypeChecker, + identifiers: Map, + namespaces: Set, +): void { + for (const element of pattern.elements) { + if (!ts.isIdentifier(element.name)) continue; + if (element.dotDotDotToken) { const symbol = checker.getSymbolAtLocation(element.name); - if (symbol) identifiers.add(symbol); + if (symbol) namespaces.add(symbol); + continue; } + const importedName = element.propertyName ?? element.name; + const view = POLICY_GET_BUILDERS.get(propertyNameText(importedName) ?? ""); + const symbol = checker.getSymbolAtLocation(element.name); + if (symbol && view) identifiers.set(symbol, view); } } @@ -174,7 +220,7 @@ function collectPolicyBuilderBindings( repoRoot: string, checker: ts.TypeChecker, ): PolicyBuilderBindings { - const identifiers = new Set(); + const identifiers = new Map(); const namespaces = new Set(); for (const statement of sourceFile.statements) { if ( @@ -192,9 +238,10 @@ function collectPolicyBuilderBindings( for (const element of namedBindings.elements) { if (element.isTypeOnly) continue; const importedName = element.propertyName?.text ?? element.name.text; - if (!POLICY_GET_BUILDERS.has(importedName)) continue; + const view = POLICY_GET_BUILDERS.get(importedName); + if (!view) continue; const symbol = checker.getSymbolAtLocation(element.name); - if (symbol) identifiers.add(symbol); + if (symbol) identifiers.set(symbol, view); } } } else if (ts.isVariableStatement(statement)) { @@ -210,27 +257,84 @@ function collectPolicyBuilderBindings( } } } + + const aliasDeclarations: ts.VariableDeclaration[] = []; + function collectAliasDeclarations(node: ts.Node): void { + if (ts.isVariableDeclaration(node) && node.initializer) { + aliasDeclarations.push(node); + } + ts.forEachChild(node, collectAliasDeclarations); + } + collectAliasDeclarations(sourceFile); + + for (let pass = 0; pass < aliasDeclarations.length; pass += 1) { + const previousSize = identifiers.size + namespaces.size; + for (const declaration of aliasDeclarations) { + const { initializer } = declaration; + if (!initializer) continue; + const bindings = { identifiers, namespaces }; + if (ts.isIdentifier(declaration.name)) { + const symbol = checker.getSymbolAtLocation(declaration.name); + if (!symbol) continue; + if (isPolicyNamespaceReference(initializer, bindings, fileName, repoRoot, checker)) { + namespaces.add(symbol); + continue; + } + const view = policyBuilderReferenceView(initializer, bindings, checker); + if (view) identifiers.set(symbol, view); + continue; + } + if ( + ts.isObjectBindingPattern(declaration.name) && + isPolicyNamespaceReference(initializer, bindings, fileName, repoRoot, checker) + ) { + collectPolicyBuilderObjectBindings(declaration.name, checker, identifiers, namespaces); + } + } + if (identifiers.size + namespaces.size === previousSize) break; + } return { identifiers, namespaces }; } -function isPolicyBuilderCall( - expression: ts.LeftHandSideExpression, +function isPolicyNamespaceReference( + expression: ts.Expression, bindings: PolicyBuilderBindings, + fileName: string, + repoRoot: string, checker: ts.TypeChecker, ): boolean { + if (ts.isParenthesizedExpression(expression)) { + return isPolicyNamespaceReference(expression.expression, bindings, fileName, repoRoot, checker); + } + const moduleSpecifier = requireModuleSpecifier(expression, checker); + if (moduleSpecifier) return isPolicyBuilderModule(fileName, moduleSpecifier, repoRoot); + if (!ts.isIdentifier(expression)) return false; + const symbol = checker.getSymbolAtLocation(expression); + return !!symbol && bindings.namespaces.has(symbol); +} + +function policyBuilderReferenceView( + expression: ts.Expression, + bindings: PolicyBuilderBindings, + checker: ts.TypeChecker, +): PolicyReadView | null { + if (ts.isParenthesizedExpression(expression)) { + return policyBuilderReferenceView(expression.expression, bindings, checker); + } if (ts.isIdentifier(expression)) { const symbol = checker.getSymbolAtLocation(expression); - return !!symbol && bindings.identifiers.has(symbol); + return symbol ? (bindings.identifiers.get(symbol) ?? null) : null; + } + if (!ts.isPropertyAccessExpression(expression) && !ts.isElementAccessExpression(expression)) { + return null; } const memberName = calledName(expression); - if (!memberName || !POLICY_GET_BUILDERS.has(memberName)) return false; - const target = - ts.isPropertyAccessExpression(expression) || ts.isElementAccessExpression(expression) - ? expression.expression - : null; - if (!target || !ts.isIdentifier(target)) return false; + const view = memberName ? POLICY_GET_BUILDERS.get(memberName) : undefined; + if (!view) return null; + const target = expression.expression; + if (!ts.isIdentifier(target)) return null; const symbol = checker.getSymbolAtLocation(target); - return !!symbol && bindings.namespaces.has(symbol); + return symbol && bindings.namespaces.has(symbol) ? view : null; } function createBoundSourceFile( @@ -289,14 +393,14 @@ function isCanonicalOpenshellResolverCall( ); } -function isDirectPolicyRead( +function directPolicyReadView( expression: ts.ArrayLiteralExpression, fileName: string, repoRoot: string, checker: ts.TypeChecker, -): boolean { +): PolicyReadView | null { const first = expression.elements[0]; - if (!first || !ts.isExpression(first)) return false; + if (!first || !ts.isExpression(first)) return null; const firstText = literalText(first); const offset = firstText === "policy" @@ -305,43 +409,216 @@ function isDirectPolicyRead( isCanonicalOpenshellResolverCall(first, fileName, repoRoot, checker) ? 1 : -1; - if (offset < 0) return false; + if (offset < 0) return null; const values = expression.elements.map((element) => ts.isExpression(element) ? literalText(element) : null, ); + if (values[offset] !== "policy" || values[offset + 1] !== "get") return null; + if (values[offset + 2] === "--base") return "base"; + if (values[offset + 2] === "--full") return "full"; + return null; +} + +function declarationNameText(name: ts.DeclarationName | undefined): string | null { + if (!name) return null; + if (ts.isIdentifier(name) || ts.isPrivateIdentifier(name) || ts.isStringLiteralLike(name)) { + return name.text; + } + return name.getText(); +} + +function functionScopeName(node: ts.FunctionLikeDeclaration): string { + const ownName = "name" in node ? declarationNameText(node.name) : null; + if (ownName) return ownName; + const { parent } = node; + if (ts.isVariableDeclaration(parent)) return declarationNameText(parent.name) ?? ""; + if (ts.isPropertyAssignment(parent)) return declarationNameText(parent.name) ?? ""; + return ""; +} + +function isFunctionScope(node: ts.Node): node is ts.FunctionLikeDeclaration { return ( - values[offset] === "policy" && - values[offset + 1] === "get" && - (values[offset + 2] === "--base" || values[offset + 2] === "--full") + ts.isFunctionDeclaration(node) || + ts.isFunctionExpression(node) || + ts.isArrowFunction(node) || + ts.isMethodDeclaration(node) || + ts.isGetAccessorDeclaration(node) || + ts.isSetAccessorDeclaration(node) || + ts.isConstructorDeclaration(node) ); } -export function countPolicyReadCalls( +function policyReadSite(node: ts.Node): string { + const scopes: string[] = []; + for (let current = node.parent; current; current = current.parent) { + if (isFunctionScope(current)) scopes.unshift(functionScopeName(current)); + } + return scopes.join("/") || ""; +} + +type IgnoreErrorOption = "absent" | "present" | "unclassified"; + +function propertyNameText(name: ts.PropertyName): string | null { + if (ts.isIdentifier(name) || ts.isPrivateIdentifier(name) || ts.isStringLiteralLike(name)) { + return name.text; + } + if (ts.isComputedPropertyName(name) && ts.isStringLiteralLike(name.expression)) { + return name.expression.text; + } + return null; +} + +function mergeIgnoreErrorOptions( + left: IgnoreErrorOption, + right: IgnoreErrorOption, +): IgnoreErrorOption { + if (left === "unclassified" || right === "unclassified") return "unclassified"; + return left === "present" || right === "present" ? "present" : "absent"; +} + +function classifyIgnoreErrorOption(expression: ts.Expression): IgnoreErrorOption { + if (ts.isParenthesizedExpression(expression)) { + return classifyIgnoreErrorOption(expression.expression); + } + if (ts.isConditionalExpression(expression)) { + return mergeIgnoreErrorOptions( + classifyIgnoreErrorOption(expression.whenTrue), + classifyIgnoreErrorOption(expression.whenFalse), + ); + } + if (!ts.isObjectLiteralExpression(expression)) return "unclassified"; + + let result: IgnoreErrorOption = "absent"; + for (const property of expression.properties) { + if (ts.isSpreadAssignment(property)) { + result = mergeIgnoreErrorOptions(result, classifyIgnoreErrorOption(property.expression)); + continue; + } + if (!ts.isPropertyAssignment(property) && !ts.isShorthandPropertyAssignment(property)) { + const name = "name" in property ? propertyNameText(property.name) : null; + if (name === "ignoreError") result = "unclassified"; + continue; + } + const name = propertyNameText(property.name); + if (name !== "ignoreError") continue; + if (ts.isShorthandPropertyAssignment(property)) { + result = "unclassified"; + continue; + } + if (property.initializer.kind === ts.SyntaxKind.TrueKeyword) { + result = mergeIgnoreErrorOptions(result, "present"); + continue; + } + if (property.initializer.kind !== ts.SyntaxKind.FalseKeyword) result = "unclassified"; + } + return result; +} + +const POLICY_READ_RUNNERS = new Set([ + "captureOpenshell", + "captureOpenshellForStatus", + "runCapture", + "runCmd", +]); + +function isWithin(node: ts.Node, ancestor: ts.Node): boolean { + return node.pos >= ancestor.pos && node.end <= ancestor.end; +} + +function containsThrowOutsideNestedFunctions(node: ts.Node): boolean { + let containsThrow = false; + function visit(current: ts.Node): void { + if (ts.isThrowStatement(current)) { + containsThrow = true; + return; + } + if (current !== node && isFunctionScope(current)) return; + ts.forEachChild(current, visit); + } + visit(node); + return containsThrow; +} + +function catchGuaranteesDirectThrow(block: ts.Block): boolean { + const finalStatement = block.statements[block.statements.length - 1]; + if (!finalStatement || !ts.isThrowStatement(finalStatement)) return false; + return block.statements + .slice(0, -1) + .every((statement) => ts.isVariableStatement(statement) || ts.isEmptyStatement(statement)); +} + +function policyReadFailureHandling(node: ts.Node): PolicyReadFailureHandling { + let runnerHandling: PolicyReadFailureHandling = "unclassified"; + for (let current = node.parent; current && !isFunctionScope(current); current = current.parent) { + if (!ts.isCallExpression(current) || !ts.isIdentifier(current.expression)) continue; + if (!POLICY_READ_RUNNERS.has(current.expression.text)) continue; + const command = current.arguments[0]; + if (!command || !isWithin(node, command)) return "unclassified"; + if (current.arguments.length === 1) { + runnerHandling = "error-preserving"; + break; + } + if (current.arguments.length !== 2) return "unclassified"; + const option = classifyIgnoreErrorOption(current.arguments[1]); + runnerHandling = + option === "present" + ? "ignore-error" + : option === "absent" + ? "error-preserving" + : "unclassified"; + break; + } + if (runnerHandling !== "error-preserving") return runnerHandling; + + for (let current = node.parent; current && !isFunctionScope(current); current = current.parent) { + if (!ts.isTryStatement(current) || !current.catchClause || !isWithin(node, current.tryBlock)) { + continue; + } + if (catchGuaranteesDirectThrow(current.catchClause.block)) continue; + if (containsThrowOutsideNestedFunctions(current.catchClause.block)) return "unclassified"; + return "ignore-error"; + } + return runnerHandling; +} + +export function classifyPolicyReadCalls( source: string, fileName: string, repoRoot = REPO_ROOT, -): number { +): DiscoveredPolicyRead[] { const { sourceFile, checker } = createBoundSourceFile(source, fileName); const builderBindings = collectPolicyBuilderBindings(sourceFile, fileName, repoRoot, checker); - let readCalls = 0; + const reads: DiscoveredPolicyRead[] = []; + + function record(node: ts.Node, view: PolicyReadView): void { + reads.push({ + site: policyReadSite(node), + view, + failureHandling: policyReadFailureHandling(node), + }); + } function visit(node: ts.Node): void { - if ( - ts.isCallExpression(node) && - isPolicyBuilderCall(node.expression, builderBindings, checker) - ) { - readCalls += 1; - } else if ( - ts.isArrayLiteralExpression(node) && - isDirectPolicyRead(node, fileName, repoRoot, checker) - ) { - readCalls += 1; + if (ts.isCallExpression(node)) { + const view = policyBuilderReferenceView(node.expression, builderBindings, checker); + if (view) record(node, view); + } else if (ts.isArrayLiteralExpression(node)) { + const view = directPolicyReadView(node, fileName, repoRoot, checker); + if (view) record(node, view); } ts.forEachChild(node, visit); } visit(sourceFile); - return readCalls; + return reads; +} + +export function countPolicyReadCalls( + source: string, + fileName: string, + repoRoot = REPO_ROOT, +): number { + return classifyPolicyReadCalls(source, fileName, repoRoot).length; } function productionTypeScriptFiles(directory: string): string[] { @@ -365,12 +642,13 @@ export function discoverPolicyReadSites(repoRoot: string): DiscoveredPolicyReadS .flatMap((sourceRoot) => productionTypeScriptFiles(path.join(repoRoot, sourceRoot))) .flatMap((sourcePath) => { const source = readFileSync(sourcePath, "utf8"); - const readCalls = countPolicyReadCalls(source, sourcePath, repoRoot); - return readCalls > 0 + const reads = classifyPolicyReadCalls(source, sourcePath, repoRoot); + return reads.length > 0 ? [ { relativePath: path.relative(repoRoot, sourcePath).split(path.sep).join("/"), - readCalls, + readCalls: reads.length, + reads, }, ] : []; @@ -378,59 +656,39 @@ export function discoverPolicyReadSites(repoRoot: string): DiscoveredPolicyReadS .sort((left, right) => left.relativePath.localeCompare(right.relativePath)); } +function policyReadDescription(read: DiscoveredPolicyRead): string { + return `${read.site} (${read.view}, ${read.failureHandling})`; +} + +function policyReadDescriptions(reads: readonly DiscoveredPolicyRead[]): string[] { + return reads.map(policyReadDescription).sort(); +} + export function auditOpenShellPolicyMutationReads(repoRoot = REPO_ROOT): string[] { const violations: string[] = []; - for (const { - relativePath, - baseCommand, - unsafeBaseCommand, - fullCommand, - diagnosticFullRead, - } of MUTATION_READS) { + const discoveredReads = new Map( + discoverPolicyReadSites(repoRoot).map((site) => [site.relativePath, site.reads]), + ); + const auditedReads = [...MUTATION_READS, ...NON_MUTATION_POLICY_READS]; + for (const { relativePath, expectedReads } of auditedReads) { const sourcePath = path.join(repoRoot, relativePath); if (!existsSync(sourcePath)) { violations.push(`${relativePath}: audited policy read source is missing`); + discoveredReads.delete(relativePath); continue; } - const source = readFileSync(sourcePath, "utf8"); - if (!source.includes(baseCommand)) { - violations.push(`${relativePath}: expected the audited policy mutation read to use --base`); - } - if (unsafeBaseCommand && source.includes(unsafeBaseCommand)) { - violations.push(`${relativePath}: policy mutation reads must preserve command failures`); - } - if (!diagnosticFullRead && source.includes(fullCommand)) { - violations.push(`${relativePath}: audited policy mutation read must never use --full output`); - } - if (diagnosticFullRead) { - const diagnosticReads = source.split(diagnosticFullRead).length - 1; - if (!source.includes(fullCommand) || diagnosticReads === 0) { - violations.push(`${relativePath}: expected the audited diagnostic read to use --full`); - } - if (diagnosticReads !== 1) { - violations.push( - `${relativePath}: --full policy reads must remain isolated to the diagnostic path`, - ); - } - } - } - - const discoveredReads = new Map( - discoverPolicyReadSites(repoRoot).map((site) => [site.relativePath, site.readCalls]), - ); - const auditedReads = [...MUTATION_READS, ...NON_MUTATION_POLICY_READS]; - for (const { relativePath, expectedReadCalls } of auditedReads) { - const discoveredCount = discoveredReads.get(relativePath) ?? 0; - if (discoveredCount !== expectedReadCalls) { + const expected = policyReadDescriptions(expectedReads); + const discovered = policyReadDescriptions(discoveredReads.get(relativePath) ?? []); + if (expected.join("\n") !== discovered.join("\n")) { violations.push( - `${relativePath}: expected ${expectedReadCalls} audited policy read call(s), found ${discoveredCount}`, + `${relativePath}: expected audited policy reads [${expected.join("; ")}], found [${discovered.join("; ")}]`, ); } discoveredReads.delete(relativePath); } - for (const [relativePath, readCalls] of discoveredReads) { + for (const [relativePath, reads] of discoveredReads) { violations.push( - `${relativePath}: found ${readCalls} unaccounted policy read call(s); classify every read before merge`, + `${relativePath}: found ${reads.length} unaccounted policy read call(s); classify every read before merge`, ); } diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index f95952d7cec..b18a5d85eb8 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -263,8 +263,6 @@ describe("backupAll", () => { expect(mocks.withSandboxMutationLock.mock.calls.map(([name]) => name)).toEqual([ "alpha", "beta", - "beta", - "beta", ]); expect(mocks.backupSandboxState).toHaveBeenCalledOnce(); expect(mocks.backupSandboxState).toHaveBeenCalledWith("beta"); @@ -359,7 +357,7 @@ describe("backupAll", () => { logSpy.mockRestore(); }); - it("serializes each Shields window, backup, and relock in separate intervals (#7952)", async () => { + it("keeps each Shields window, backup, and relock in one lifecycle transaction (#7952)", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "alpha" }, { name: "beta" }], defaultSandbox: "alpha", @@ -417,32 +415,17 @@ describe("backupAll", () => { expect(events).toEqual([ "lock:start:alpha", "open:alpha", - "lock:end:alpha", - "lock:start:alpha", "backup:alpha", - "lock:end:alpha", - "lock:start:alpha", "relock:alpha", "lock:end:alpha", "lock:start:beta", "open:beta", - "lock:end:beta", - "lock:start:beta", "backup:beta", - "lock:end:beta", - "lock:start:beta", "relock:beta", "lock:end:beta", ]); - expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( - 3, - "alpha", - expect.any(Function), - { timeoutMs: 30_000 }, - ); - expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith(6, "beta", expect.any(Function), { - timeoutMs: 30_000, - }); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith(1, "alpha", expect.any(Function)); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith(2, "beta", expect.any(Function)); }); it("relocks shields after a credential permission failure and keeps the failure hard (#6455)", async () => { @@ -546,21 +529,16 @@ describe("backupAll", () => { mocks.backupSandboxState.mockImplementation(() => { throw backupError; }); - const relockLockError = new Error("mutation lock timed out"); - mocks.withSandboxMutationLock - .mockImplementationOnce(runSandboxMutationAction) - .mockImplementationOnce(runSandboxMutationAction) - .mockRejectedValueOnce(relockLockError); + const relockError = new Error("policy restore failed"); + mocks.relockBackupShieldsWindow.mockImplementation(() => { + throw relockError; + }); vi.spyOn(console, "log").mockImplementation(() => undefined); const failure = await backupAll().catch((error: unknown) => error); - expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( - 3, - "alpha", - expect.any(Function), - { timeoutMs: 30_000 }, - ); + expect(mocks.withSandboxMutationLock).toHaveBeenCalledOnce(); + expect(mocks.withSandboxMutationLock).toHaveBeenCalledWith("alpha", expect.any(Function)); expect(failure).toBeInstanceOf(AggregateError); expect((failure as AggregateError).message).toContain( "Backup for 'alpha' failed and Shields lockdown could not be restored", @@ -568,13 +546,13 @@ describe("backupAll", () => { expect((failure as AggregateError).errors).toEqual([ backupError, expect.objectContaining({ - cause: relockLockError, + cause: relockError, message: expect.stringContaining( "Shields lockdown could not be restored for 'alpha' after backup-all", ), }), ]); - expect(mocks.relockBackupShieldsWindow).not.toHaveBeenCalled(); + expect(mocks.relockBackupShieldsWindow).toHaveBeenCalledOnce(); }); it("preserves an orphan-manifest error when shields restoration also fails (#6455)", async () => { @@ -687,7 +665,7 @@ describe("backupAll", () => { expect(logOutput).not.toContain("Skipping 'sb-stopped'"); }); - it("keeps the stopped-container lifecycle inside the three backup lock intervals (#7952)", async () => { + it("keeps the stopped-container lifecycle inside one backup transaction (#7952)", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "sb-stopped" }], defaultSandbox: "sb-stopped", @@ -748,23 +726,14 @@ describe("backupAll", () => { "lock:start:sb-stopped", "start:sb-stopped", "open:sb-stopped", - "lock:end:sb-stopped", - "lock:start:sb-stopped", "backup:sb-stopped", - "lock:end:sb-stopped", - "lock:start:sb-stopped", "relock:sb-stopped", "stop:sb-stopped", "lock:end:sb-stopped", ]); expect(lockActive).toBe(false); - expect(mocks.withSandboxMutationLock).toHaveBeenCalledTimes(3); - expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( - 3, - "sb-stopped", - expect.any(Function), - { timeoutMs: 30_000 }, - ); + expect(mocks.withSandboxMutationLock).toHaveBeenCalledOnce(); + expect(mocks.withSandboxMutationLock).toHaveBeenCalledWith("sb-stopped", expect.any(Function)); }); it("returns the container to stopped and counts a failure when the started backup fails (#6500)", async () => { @@ -830,12 +799,23 @@ describe("backupAll", () => { ); }); - it("does not stop outside the lock when the cleanup interval cannot be acquired (#7952)", async () => { + it("keeps stopped-container cleanup in the transaction when Shields relock fails (#7952)", async () => { mocks.listSandboxes.mockReturnValue({ - sandboxes: [{ name: "sb-stopped" }, { name: "beta" }], + sandboxes: [{ name: "sb-stopped" }], defaultSandbox: "sb-stopped", }); - mocks.parseReadySandboxNames.mockReturnValue(new Set(["beta"])); + mocks.parseReadySandboxNames.mockReturnValue(new Set()); + let lockActive = false; + mocks.withSandboxMutationLock.mockImplementation( + async (_name: string, action: () => unknown) => { + lockActive = true; + try { + return await action(); + } finally { + lockActive = false; + } + }, + ); mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ containerName: "openshell-sb-stopped-abc", }); @@ -848,32 +828,30 @@ describe("backupAll", () => { failedFiles: [], manifest: { backupPath: "/backups/sb-stopped/timestamp" }, }); - const cleanupLockError = new Error("Timed out waiting for the cleanup mutation lock"); - mocks.withSandboxMutationLock - .mockImplementationOnce(runSandboxMutationAction) - .mockImplementationOnce(runSandboxMutationAction) - .mockRejectedValueOnce(cleanupLockError); + const relockError = new Error("policy restore failed"); + mocks.relockBackupShieldsWindow.mockImplementation(() => { + expect(lockActive).toBe(true); + throw relockError; + }); + mocks.returnSandboxContainerToStopped.mockImplementation(() => { + expect(lockActive).toBe(true); + return true; + }); vi.spyOn(console, "log").mockImplementation(() => undefined); - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const failure = await backupAll().catch((error: unknown) => error); - expect(failure).toBeInstanceOf(AggregateError); - expect((failure as AggregateError).errors).toEqual([ + expect(failure).toEqual( expect.objectContaining({ - cause: cleanupLockError, + cause: relockError, message: expect.stringContaining("Shields lockdown could not be restored"), }), - expect.objectContaining({ - cause: cleanupLockError, - message: expect.stringContaining("container was left running"), - }), - ]); - expect(mocks.withSandboxMutationLock).toHaveBeenCalledTimes(3); - expect(mocks.relockBackupShieldsWindow).not.toHaveBeenCalled(); - expect(mocks.returnSandboxContainerToStopped).not.toHaveBeenCalled(); + ); + expect(lockActive).toBe(false); + expect(mocks.withSandboxMutationLock).toHaveBeenCalledOnce(); + expect(mocks.relockBackupShieldsWindow).toHaveBeenCalledOnce(); + expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith("openshell-sb-stopped-abc"); expect(mocks.openBackupShieldsWindow).toHaveBeenCalledOnce(); - expect(errorSpy.mock.calls.flat().join("\n")).toContain("container was left running"); }); it("returns a started container to stopped when an orphan manifest skips backup (#6500)", async () => { diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 17a1eb2d19b..9019950ed1e 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -27,7 +27,6 @@ import * as registry from "../state/registry"; import * as sandboxState from "../state/sandbox"; import { nemoclawStateRoot, resolveHome } from "../state/state-root"; import { - type BackupShieldsWindow, type BackupShieldsWindowOptions, openBackupShieldsWindow, relockBackupShieldsWindow, @@ -79,26 +78,13 @@ interface BackupAllSandboxAttempt { mutationLockError?: unknown; } -interface BackupAllSandboxSetup { - window: BackupShieldsWindow | null; - startedForBackup: StartedForBackup | null; - stoppedContainerUnavailable: boolean; - stoppedContainerCleanupError: Error | null; -} - function returnStartedSandboxToStopped( sandboxName: string, startedForBackup: StartedForBackup, - cause?: unknown, ): Error | null { const failureDetail = "could not return its container to the stopped state; the container was left running"; const failureMessage = `Backup cleanup failed for '${sandboxName}': ${failureDetail}.`; - if (cause !== undefined) { - const error = new Error(failureMessage, { cause }); - console.error(` ${RD}✗${R} ${sandboxName}: backup cleanup failed (${failureDetail})`); - return error; - } try { if (returnSandboxContainerToStopped(startedForBackup.containerName)) { console.log(` ${D}Returned '${sandboxName}' to its stopped state.${R}`); @@ -127,36 +113,27 @@ async function backupSandboxWithinShieldsWindow( ) => sandboxState.BackupResult | Promise, ): Promise { const shieldsWindowOptions = backupAllShieldsWindowOptions(sandboxName); - let enteredOpenLock = false; - let setup: BackupAllSandboxSetup; + let enteredTransactionLock = false; try { - setup = await withSandboxMutationLock(sandboxName, () => { - enteredOpenLock = true; + return await withSandboxMutationLock(sandboxName, async () => { + enteredTransactionLock = true; const startedForBackup = shouldStartStoppedContainer ? startStoppedSandboxContainerForBackup(sandboxName) : null; if (shouldStartStoppedContainer && !startedForBackup) { return { - window: null, - startedForBackup: null, + result: null, + orphanManifestMessage: null, + shieldsWindowOpened: false, stoppedContainerUnavailable: true, - stoppedContainerCleanupError: null, }; } if (startedForBackup) { console.log(` Starting stopped sandbox '${sandboxName}' to back it up...`); } + let window; try { - const window = openBackupShieldsWindow(sandboxName, shieldsWindowOptions); - return { - window, - startedForBackup, - stoppedContainerUnavailable: false, - stoppedContainerCleanupError: - !window && startedForBackup - ? returnStartedSandboxToStopped(sandboxName, startedForBackup) - : null, - }; + window = openBackupShieldsWindow(sandboxName, shieldsWindowOptions); } catch (error) { if (!startedForBackup) throw error; const cleanupError = returnStartedSandboxToStopped(sandboxName, startedForBackup); @@ -168,9 +145,113 @@ async function backupSandboxWithinShieldsWindow( } throw error; } + if (!window) { + const cleanupError = startedForBackup + ? returnStartedSandboxToStopped(sandboxName, startedForBackup) + : null; + if (cleanupError) throw cleanupError; + return { + result: null, + orphanManifestMessage: null, + shieldsWindowOpened: false, + stoppedContainerUnavailable: false, + }; + } + + console.log(` Backing up '${sandboxName}'...`); + let result: sandboxState.BackupResult | null = null; + let orphanManifestMessage: string | null = null; + let backupError: unknown; + let hasBackupError = false; + let relockError: Error | null = null; + let stoppedContainerCleanupError: Error | null = null; + try { + result = await backup(startedForBackup); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + // Preserve the narrow pre-upgrade orphan exception, but classify it inside + // this window so a previously locked sandbox is always relocked before the + // caller counts the attempt as skipped. + if (/^Agent '[^']+' not found: .+\/manifest\.yaml$/.test(message)) { + orphanManifestMessage = message; + } else { + backupError = err; + hasBackupError = true; + } + } finally { + // One lifecycle transaction excludes concurrent NemoClaw destroy and + // recreate operations through the shields-down window, backup, and + // cleanup. If auto-restore expires, its deadline gate blocks new + // lifecycle mutations and waits for this owner. The relock path binds + // to the active timer token before the lifecycle lock is released. + try { + if (!relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions)) { + relockError = shieldsRelockError(sandboxName); + } + } catch (error) { + relockError = shieldsRelockError(sandboxName, error); + } finally { + if (startedForBackup) { + stoppedContainerCleanupError = returnStartedSandboxToStopped( + sandboxName, + startedForBackup, + ); + } + } + } + + if (relockError) { + if (hasBackupError) { + throw new AggregateError( + [ + backupError, + relockError, + ...(stoppedContainerCleanupError ? [stoppedContainerCleanupError] : []), + ], + `Backup for '${sandboxName}' failed and Shields lockdown could not be restored; aborting remaining backups.`, + ); + } + if (orphanManifestMessage) { + throw new AggregateError( + [ + new Error(orphanManifestMessage), + relockError, + ...(stoppedContainerCleanupError ? [stoppedContainerCleanupError] : []), + ], + `Backup for '${sandboxName}' encountered an orphan manifest and Shields lockdown could not be restored; aborting remaining backups.`, + ); + } + if (stoppedContainerCleanupError) { + throw new AggregateError( + [relockError, stoppedContainerCleanupError], + `Shields lockdown could not be restored for '${sandboxName}' and its started container could not be returned to the stopped state; aborting remaining backups.`, + ); + } + throw relockError; + } + if (stoppedContainerCleanupError && hasBackupError) { + throw new AggregateError( + [backupError, stoppedContainerCleanupError], + `Backup for '${sandboxName}' failed and its started container could not be returned to the stopped state; aborting remaining backups.`, + ); + } + if (stoppedContainerCleanupError && orphanManifestMessage) { + throw new AggregateError( + [new Error(orphanManifestMessage), stoppedContainerCleanupError], + `Backup for '${sandboxName}' encountered an orphan manifest and its started container could not be returned to the stopped state; aborting remaining backups.`, + ); + } + if (stoppedContainerCleanupError) throw stoppedContainerCleanupError; + if (hasBackupError) throw backupError; + return { + result, + orphanManifestMessage, + shieldsWindowOpened: true, + stoppedContainerUnavailable: false, + }; }); } catch (error) { - if (enteredOpenLock) throw error; + if (enteredTransactionLock) throw error; return { result: null, orphanManifestMessage: null, @@ -179,130 +260,6 @@ async function backupSandboxWithinShieldsWindow( mutationLockError: error, }; } - if (setup.stoppedContainerUnavailable) { - return { - result: null, - orphanManifestMessage: null, - shieldsWindowOpened: false, - stoppedContainerUnavailable: true, - }; - } - if (!setup.window) { - if (setup.stoppedContainerCleanupError) throw setup.stoppedContainerCleanupError; - return { - result: null, - orphanManifestMessage: null, - shieldsWindowOpened: false, - stoppedContainerUnavailable: false, - }; - } - const window = setup.window; - - console.log(` Backing up '${sandboxName}'...`); - let result: sandboxState.BackupResult | null = null; - let orphanManifestMessage: string | null = null; - let backupError: unknown; - let hasBackupError = false; - let relockError: Error | null = null; - let stoppedContainerCleanupError: Error | null = null; - try { - result = await withSandboxMutationLock(sandboxName, () => backup(setup.startedForBackup)); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - // Preserve the narrow pre-upgrade orphan exception, but classify it inside - // this window so a previously locked sandbox is always relocked before the - // caller counts the attempt as skipped. - if (/^Agent '[^']+' not found: .+\/manifest\.yaml$/.test(message)) { - orphanManifestMessage = message; - } else { - backupError = err; - hasBackupError = true; - } - } finally { - let enteredRelockLock = false; - try { - await withSandboxMutationLock( - sandboxName, - () => { - enteredRelockLock = true; - try { - if (!relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions)) { - relockError = shieldsRelockError(sandboxName); - } - } catch (error) { - relockError = shieldsRelockError(sandboxName, error); - } finally { - if (setup.startedForBackup) { - stoppedContainerCleanupError = returnStartedSandboxToStopped( - sandboxName, - setup.startedForBackup, - ); - } - } - }, - { timeoutMs: 30_000 }, - ); - } catch (error) { - relockError ??= shieldsRelockError(sandboxName, error); - if (!enteredRelockLock && setup.startedForBackup) { - stoppedContainerCleanupError = returnStartedSandboxToStopped( - sandboxName, - setup.startedForBackup, - error, - ); - } - } - } - - if (relockError) { - if (hasBackupError) { - throw new AggregateError( - [ - backupError, - relockError, - ...(stoppedContainerCleanupError ? [stoppedContainerCleanupError] : []), - ], - `Backup for '${sandboxName}' failed and Shields lockdown could not be restored; aborting remaining backups.`, - ); - } - if (orphanManifestMessage) { - throw new AggregateError( - [ - new Error(orphanManifestMessage), - relockError, - ...(stoppedContainerCleanupError ? [stoppedContainerCleanupError] : []), - ], - `Backup for '${sandboxName}' encountered an orphan manifest and Shields lockdown could not be restored; aborting remaining backups.`, - ); - } - if (stoppedContainerCleanupError) { - throw new AggregateError( - [relockError, stoppedContainerCleanupError], - `Shields lockdown could not be restored for '${sandboxName}' and its started container could not be returned to the stopped state; aborting remaining backups.`, - ); - } - throw relockError; - } - if (stoppedContainerCleanupError && hasBackupError) { - throw new AggregateError( - [backupError, stoppedContainerCleanupError], - `Backup for '${sandboxName}' failed and its started container could not be returned to the stopped state; aborting remaining backups.`, - ); - } - if (stoppedContainerCleanupError && orphanManifestMessage) { - throw new AggregateError( - [new Error(orphanManifestMessage), stoppedContainerCleanupError], - `Backup for '${sandboxName}' encountered an orphan manifest and its started container could not be returned to the stopped state; aborting remaining backups.`, - ); - } - if (stoppedContainerCleanupError) throw stoppedContainerCleanupError; - if (hasBackupError) throw backupError; - return { - result, - orphanManifestMessage, - shieldsWindowOpened: true, - stoppedContainerUnavailable: false, - }; } export async function backupAll(): Promise { diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index 5594dced93c..88a5e024655 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -1,14 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { isIP } from "node:net"; -import { isDeepStrictEqual } from "node:util"; -import YAML from "yaml"; - import type { AgentMcpAdapter } from "../../agent/defs"; -import { diagnosticPreview } from "../../name-validation"; import * as policies from "../../policy"; -import { isBlockedMcpUrlTargetHost } from "../../security/mcp-url-target"; import type { McpBridgeEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { @@ -22,7 +16,6 @@ import { buildMcpBridgePolicyYaml, } from "./mcp-bridge-policy-render"; -export { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; export { buildMcpBridgePolicyKey, buildMcpBridgePolicyName, @@ -31,463 +24,6 @@ export { MCP_BRIDGE_POLICY_MAX_BODY_BYTES, } from "./mcp-bridge-policy-render"; -export interface ExactManagedMcpPolicy { - key: string; - networkPolicy: unknown; - policyName: string; - server: string; -} - -export interface ManagedMcpPolicyOmission { - key?: string; - policyName?: string; - server?: string; - reason: string; -} - -export interface ProvableManagedMcpPolicies { - policies: ExactManagedMcpPolicy[]; - omissions: ManagedMcpPolicyOmission[]; -} - -type ManagedMcpPolicyInspectionDeps = { - getSandbox: typeof registry.getSandbox; -}; - -const managedMcpPolicyInspectionDeps: ManagedMcpPolicyInspectionDeps = { - getSandbox: registry.getSandbox, -}; - -function parseManagedPolicyDocument(source: string, label: string): Record { - let parsed: unknown; - try { - parsed = YAML.parse(source); - } catch { - throw new Error(`${label} is not valid YAML`); - } - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - throw new Error(`${label} must be a YAML mapping`); - } - return parsed as Record; -} - -function readManagedNetworkPolicies( - document: Record, - label: string, -): Record { - const networkPolicies = document.network_policies; - if (networkPolicies === undefined || networkPolicies === null) return {}; - if (typeof networkPolicies !== "object" || Array.isArray(networkPolicies)) { - throw new Error(`${label} network_policies must be a mapping`); - } - return networkPolicies as Record; -} - -function requireCanonicalAllowedIps(networkPolicy: unknown, policyName: string): readonly string[] { - if (!networkPolicy || typeof networkPolicy !== "object" || Array.isArray(networkPolicy)) { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); - } - const endpoints = (networkPolicy as Record).endpoints; - if (!Array.isArray(endpoints) || endpoints.length !== 1) { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); - } - const endpoint = endpoints[0]; - if (!endpoint || typeof endpoint !== "object" || Array.isArray(endpoint)) { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); - } - const allowedIps = (endpoint as Record).allowed_ips; - if (!Array.isArray(allowedIps) || allowedIps.length === 0) { - throw new Error(`Managed MCP policy '${policyName}' has no exact public address pins`); - } - if ( - allowedIps.some( - (address) => - typeof address !== "string" || - address !== address.toLowerCase() || - address.includes("%") || - isIP(address) === 0 || - isBlockedMcpUrlTargetHost(address), - ) - ) { - throw new Error(`Managed MCP policy '${policyName}' has invalid public address pins`); - } - const pins = allowedIps as string[]; - if (new Set(pins).size !== pins.length || !isDeepStrictEqual(pins, [...pins].sort())) { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical public address pins`); - } - return pins; -} - -function resolveCanonicalManagedMcpAdapter( - sandbox: registry.SandboxEntry, - bridge: McpBridgeEntry, -): AgentMcpAdapter { - if (isAgentMcpAdapter(bridge.adapter)) return bridge.adapter; - switch (sandbox.agent || "openclaw") { - case "openclaw": - return "mcporter"; - case "hermes": - return "hermes-config"; - case "langchain-deepagents-code": - return "deepagents-config"; - default: - throw new Error("Managed MCP bridge has no canonical adapter"); - } -} - -function requireCanonicalManagedPolicy( - sandbox: registry.SandboxEntry, - server: string, - livePolicies: Record, -): ExactManagedMcpPolicy { - const bridge = sandbox.mcp?.bridges[server]; - if (!bridge || bridge.addState || bridge.server !== server) { - throw new Error(`Managed MCP bridge '${server}' has an incomplete lifecycle transition`); - } - - const policyName = buildMcpBridgePolicyName(server); - const policyKey = buildMcpBridgePolicyKey(server); - if (bridge.policyName !== policyName) { - throw new Error(`Managed MCP bridge '${server}' has a non-canonical policy name`); - } - - const registrations = (sandbox.customPolicies ?? []).filter( - (policy) => policy.name === policyName, - ); - if (registrations.length !== 1) { - throw new Error( - `Managed MCP bridge '${server}' does not have one exact policy ownership record`, - ); - } - const [registration] = registrations; - if (registration?.sourcePath !== MCP_BRIDGE_POLICY_SOURCE) { - throw new Error(`Managed MCP bridge '${server}' has no NemoClaw-owned policy record`); - } - if (registration.pendingContent !== undefined) { - throw new Error(`Managed MCP bridge '${server}' has an incomplete policy transition`); - } - - const registeredDocument = parseManagedPolicyDocument( - registration.content, - `Managed MCP policy '${policyName}'`, - ); - const preset = registeredDocument.preset; - if ( - !preset || - typeof preset !== "object" || - Array.isArray(preset) || - (preset as Record).name !== policyName - ) { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical preset metadata`); - } - const registeredPolicies = readManagedNetworkPolicies( - registeredDocument, - `Managed MCP policy '${policyName}'`, - ); - const registeredKeys = Object.keys(registeredPolicies); - if (registeredKeys.length !== 1 || registeredKeys[0] !== policyKey) { - throw new Error(`Managed MCP policy '${policyName}' has a non-canonical network policy key`); - } - - const registeredNetworkPolicy = registeredPolicies[policyKey]; - const allowedIps = requireCanonicalAllowedIps(registeredNetworkPolicy, policyName); - let expectedDocument: Record; - try { - expectedDocument = parseManagedPolicyDocument( - buildMcpBridgePolicyYaml( - bridge.server, - bridge.url, - resolveCanonicalManagedMcpAdapter(sandbox, bridge), - allowedIps, - ), - `Canonical managed MCP policy '${policyName}'`, - ); - } catch { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); - } - if (!isDeepStrictEqual(registeredDocument, expectedDocument)) { - throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); - } - - if (!Object.hasOwn(livePolicies, policyKey)) { - throw new Error(`Managed MCP policy '${policyName}' is absent from the live gateway policy`); - } - if (!isDeepStrictEqual(livePolicies[policyKey], registeredNetworkPolicy)) { - throw new Error(`Managed MCP policy '${policyName}' has drifted from its ownership record`); - } - - return { - key: policyKey, - networkPolicy: registeredNetworkPolicy, - policyName, - server, - }; -} - -/** - * Resolve the exact generated MCP entries that NemoClaw currently owns. - * - * The registry is an ownership claim, not sufficient authority to overwrite - * the gateway. Every committed bridge must have one canonical, fully - * committed custom-policy record whose sole network entry exactly matches the - * live base policy. - */ -export function inspectExactManagedMcpPolicies( - sandboxName: string, - livePolicyYaml: string, - deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, -): ExactManagedMcpPolicy[] { - const liveDocument = parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"); - const livePolicies = readManagedNetworkPolicies(liveDocument, "Live gateway policy"); - const sandbox = deps.getSandbox(sandboxName); - if (!sandbox) { - const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); - if (unclassifiedKey) { - throw new Error( - `Reserved MCP policy key ${diagnosticPreview(unclassifiedKey)} has no committed managed bridge ownership`, - ); - } - return []; - } - const generatedRegistrations = (sandbox.customPolicies ?? []).filter( - (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, - ); - if (!sandbox.mcp) { - const orphaned = generatedRegistrations[0]; - if (orphaned) { - throw new Error( - `Generated MCP policy ${diagnosticPreview(orphaned.name)} has no committed managed bridge ownership`, - ); - } - const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); - if (unclassifiedKey) { - throw new Error( - `Reserved MCP policy key ${diagnosticPreview(unclassifiedKey)} has no committed managed bridge ownership`, - ); - } - return []; - } - if (sandbox.mcp.destroyPreparedAt || sandbox.mcp.destroyPendingAt) { - throw new Error("Managed MCP sandbox destruction is incomplete"); - } - - const bridgeEntries = Object.entries(sandbox.mcp.bridges); - if (bridgeEntries.some(([, bridge]) => bridge.addState !== undefined)) { - throw new Error("A managed MCP bridge lifecycle transition is incomplete"); - } - const exact = bridgeEntries.map(([server]) => - requireCanonicalManagedPolicy(sandbox, server, livePolicies), - ); - - const committedPolicyNames = new Set(exact.map((entry) => entry.policyName)); - const orphaned = generatedRegistrations.find( - (registration) => !committedPolicyNames.has(registration.name), - ); - if (orphaned) { - throw new Error( - `Generated MCP policy ${diagnosticPreview(orphaned.name)} has no committed managed bridge ownership`, - ); - } - - const keys = new Set(); - for (const entry of exact) { - if (keys.has(entry.key)) { - throw new Error(`Managed MCP policy key '${entry.key}' has ambiguous bridge ownership`); - } - keys.add(entry.key); - } - const unclassifiedKey = Object.keys(livePolicies).find( - (key) => key.startsWith("mcp_bridge_") && !keys.has(key), - ); - if (unclassifiedKey) { - throw new Error( - `Reserved MCP policy key ${diagnosticPreview(unclassifiedKey)} has no committed managed bridge ownership`, - ); - } - return exact.sort((left, right) => left.key.localeCompare(right.key)); -} - -/** - * Deadline-only inspection for automatic Shields restoration. - * - * Each entry is admitted independently through the same exact committed/live - * proof as the strict path. Incomplete, drifted, orphaned, or ambiguous claims - * are omitted instead of extending the mutable window; registry state is never - * reconciled or rewritten here. - */ -export function inspectProvableManagedMcpPoliciesForDeadline( - sandboxName: string, - livePolicyYaml: string, - deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, -): ProvableManagedMcpPolicies { - const derivedIdentity = (server: string): { key?: string; policyName?: string } => { - try { - return { - key: buildMcpBridgePolicyKey(server), - policyName: buildMcpBridgePolicyName(server), - }; - } catch { - return {}; - } - }; - const omit = (reason: string, server?: string, policyName?: string): ManagedMcpPolicyOmission => { - const identity = server ? derivedIdentity(server) : {}; - return { - ...(server ? { server } : {}), - ...identity, - ...(policyName ? { policyName } : {}), - reason, - }; - }; - const sandbox = deps.getSandbox(sandboxName); - const generatedRegistrations = (sandbox?.customPolicies ?? []).filter( - (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, - ); - const bridgeEntries = Object.entries(sandbox?.mcp?.bridges ?? {}); - - if (sandbox?.mcp?.destroyPreparedAt || sandbox?.mcp?.destroyPendingAt) { - const reason = "Managed MCP sandbox destruction is incomplete"; - const omissions = bridgeEntries.map(([server]) => omit(reason, server)); - for (const registration of generatedRegistrations) { - if (!omissions.some((entry) => entry.policyName === registration.name)) { - omissions.push(omit(reason, undefined, registration.name)); - } - } - if (omissions.length === 0) omissions.push({ reason }); - return { policies: [], omissions }; - } - - let livePolicies: Record; - try { - livePolicies = readManagedNetworkPolicies( - parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"), - "Live gateway policy", - ); - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - const omissions = bridgeEntries.map(([server]) => omit(reason, server)); - for (const registration of generatedRegistrations) { - if (!omissions.some((entry) => entry.policyName === registration.name)) { - omissions.push(omit(reason, undefined, registration.name)); - } - } - return { policies: [], omissions }; - } - - const policies: ExactManagedMcpPolicy[] = []; - const omissions: ManagedMcpPolicyOmission[] = []; - if (!sandbox) { - for (const key of Object.keys(livePolicies).filter((candidate) => - candidate.startsWith("mcp_bridge_"), - )) { - omissions.push({ - key, - reason: `Reserved MCP policy key '${key}' has no committed managed bridge ownership`, - }); - } - return { policies, omissions }; - } - const claimedServersByKey = new Map(); - const claimedServersByPolicyName = new Map(); - for (const [server] of bridgeEntries) { - const identity = derivedIdentity(server); - if (identity.key) { - const servers = claimedServersByKey.get(identity.key) ?? []; - servers.push(server); - claimedServersByKey.set(identity.key, servers); - } - if (identity.policyName) { - const servers = claimedServersByPolicyName.get(identity.policyName) ?? []; - servers.push(server); - claimedServersByPolicyName.set(identity.policyName, servers); - } - } - const ambiguousServers = new Set(); - for (const servers of [...claimedServersByKey.values(), ...claimedServersByPolicyName.values()]) { - if (servers.length <= 1) continue; - for (const server of servers) ambiguousServers.add(server); - } - for (const [server] of bridgeEntries) { - if (ambiguousServers.has(server)) { - omissions.push(omit("Managed MCP policy identity has ambiguous bridge ownership", server)); - continue; - } - try { - policies.push(requireCanonicalManagedPolicy(sandbox, server, livePolicies)); - } catch (error) { - omissions.push(omit(error instanceof Error ? error.message : String(error), server)); - } - } - - const bridgePolicyNames = new Set( - bridgeEntries - .map(([server]) => derivedIdentity(server).policyName) - .filter((name): name is string => name !== undefined), - ); - for (const registration of generatedRegistrations) { - if (!bridgePolicyNames.has(registration.name)) { - omissions.push( - omit( - `Generated MCP policy '${registration.name}' has no committed managed bridge ownership`, - undefined, - registration.name, - ), - ); - } - } - - const policiesByKey = new Map(); - for (const policy of policies) { - const entries = policiesByKey.get(policy.key) ?? []; - entries.push(policy); - policiesByKey.set(policy.key, entries); - } - const exact: ExactManagedMcpPolicy[] = []; - for (const entries of policiesByKey.values()) { - if (entries.length === 1) { - exact.push(entries[0]!); - continue; - } - for (const entry of entries) { - omissions.push( - omit(`Managed MCP policy key '${entry.key}' has ambiguous ownership`, entry.server), - ); - } - } - const exactKeys = new Set(exact.map((entry) => entry.key)); - for (const key of Object.keys(livePolicies).filter( - (candidate) => candidate.startsWith("mcp_bridge_") && !exactKeys.has(candidate), - )) { - if (omissions.some((entry) => entry.key === key)) continue; - omissions.push({ - key, - reason: `Reserved MCP policy key '${key}' has no exact committed managed bridge ownership`, - }); - } - return { - policies: exact.sort((left, right) => left.key.localeCompare(right.key)), - omissions, - }; -} - -export function hasManagedMcpPolicyClaims( - sandboxName: string, - deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, -): boolean { - const sandbox = deps.getSandbox(sandboxName); - if (!sandbox) return false; - return ( - Boolean( - sandbox.mcp && - (Object.keys(sandbox.mcp.bridges).length > 0 || - (sandbox.mcp.managedServerNames?.length ?? 0) > 0 || - sandbox.mcp.destroyPreparedAt || - sandbox.mcp.destroyPendingAt), - ) || - (sandbox.customPolicies ?? []).some((policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE) - ); -} - type GeneratedPolicyRegistrationState = { policy: registry.CustomPolicyEntry; state: "match" | "absent" | "drift" | null; diff --git a/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts b/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts index 16ca5aa73f2..c3017fcb8bd 100644 --- a/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts +++ b/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { runSandboxSnapshot } from "./snapshot"; const mocks = vi.hoisted(() => ({ backupSandboxState: vi.fn(), @@ -31,10 +30,6 @@ vi.mock("../../shields/timer-bound-lock", () => ({ ), })); -vi.mock("../../state/mcp-lifecycle-lock", () => ({ - withSandboxMutationLock: vi.fn((_sandboxName: string, operation: () => unknown) => operation()), -})); - vi.mock("../../state/registry", () => ({ getBaselineExclusions: mocks.getBaselineExclusions, getSandbox: vi.fn(() => ({ name: "alpha", agent: "hermes" })), @@ -45,10 +40,6 @@ vi.mock("../../state/sandbox", () => ({ findBackup: mocks.findBackup, })); -vi.mock("./snapshot/dependencies", () => ({ - backupSandboxStateWithManagedAuthority: mocks.backupSandboxState, -})); - vi.mock("./sandbox-gateway-routing", () => ({ probeGatewayRunning: vi.fn(() => true), selectSandboxGatewayIfRegistered: vi.fn(() => true), @@ -80,6 +71,7 @@ describe("snapshot baseline exclusion output", () => { it("reports active exclusions and support impact after a successful snapshot (#7178)", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); await runSandboxSnapshot("alpha", { kind: "create" }); diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 37c720ff039..1a2b99d81e1 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -8,20 +8,14 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; -import YAML from "yaml"; -import { buildMcpBridgePolicyYaml } from "../actions/sandbox/mcp-bridge-policy-render"; -import type { SandboxEntry } from "../state/registry"; const requireDist = createRequire(import.meta.url); const shieldsModulePath = "./index.js"; type ShieldsHarness = { - applyShieldsPolicySnapshot: typeof import("./index.js").applyShieldsPolicySnapshot; auditSpy: MockInstance; - cleanupTempDirSpy: MockInstance; errorSpy: MockInstance; logSpy: MockInstance; - policySetBodies: string[]; runSpy: MockInstance; shieldsDown: typeof import("./index.js").shieldsDown; shieldsStatus: typeof import("./index.js").shieldsStatus; @@ -40,7 +34,6 @@ type HarnessOptions = { directSandboxUnavailable?: boolean; dockerExecFileSync?: (argv: unknown) => string; failOpenClawGuardActions?: Array<"lock" | "unlock">; - failStateSave?: boolean; invokedAs?: "nemoclaw" | "nemohermes"; openClawGuardFailure?: { code: string; @@ -59,68 +52,19 @@ type HarnessOptions = { send: () => boolean; kill: () => boolean; }; - livePolicy?: string; livePolicyYaml?: string; run?: (cmd: unknown) => { status: number }; - sandboxEntry?: SandboxEntry; }; -function managedMcpPolicy(server: string, address = "8.8.8.8") { - const content = buildMcpBridgePolicyYaml( - server, - `https://${server}.example.com/mcp`, - "hermes-config", - [address], - ); - const entries = Object.entries(YAML.parse(content).network_policies as Record); - expect(entries, `rendered MCP policies for ${server}`).toHaveLength(1); - const [key, networkPolicy] = entries[0]!; - return { content, key, networkPolicy, server }; -} - -function managedMcpSandbox(policies: Array>): SandboxEntry { - return { - name: "openclaw", - openshellDriver: "docker", - customPolicies: policies.map(({ content, server }) => ({ - name: `mcp-bridge-${server}`, - content, - sourcePath: "generated:nemoclaw-mcp-bridge", - })), - mcp: { - bridges: Object.fromEntries( - policies.map(({ server }) => [ - server, - { - server, - agent: "hermes", - adapter: "hermes-config", - url: `https://${server}.example.com/mcp`, - env: ["MCP_SECRET"], - policyName: `mcp-bridge-${server}`, - addedAt: "2026-07-30T00:00:00.000Z", - }, - ]), - ), - }, - }; -} - function throwHarnessError(error: Error): never { throw error; } -function recordPolicySetBody(policySetBodies: string[], file: unknown): void { - policySetBodies.push(fs.readFileSync(String(file), "utf-8")); -} - function createHarness(options: HarnessOptions = {}): ShieldsHarness { vi.stubEnv("NEMOCLAW_INVOKED_AS", options.invokedAs ?? "nemoclaw"); delete require.cache[requireDist.resolve(shieldsModulePath)]; delete require.cache[requireDist.resolve("./timer-bound-lock.js")]; delete require.cache[requireDist.resolve("./transition-lock.js")]; - delete require.cache[requireDist.resolve("./permissive-runtime.js")]; - delete require.cache[requireDist.resolve("../actions/sandbox/mcp-bridge-policy.js")]; delete require.cache[requireDist.resolve("../sandbox/privileged-exec.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; const lifecycleLock = requireDist( @@ -142,24 +86,19 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { const privilegedExec = requireDist("../sandbox/privileged-exec.js"); const dockerExec = requireDist("../adapters/docker/exec.js"); const audit = requireDist("./audit.js"); - const tempFiles = requireDist("../onboard/temp-files.js"); const childProcess = requireDist("node:child_process"); - const policySetBodies: string[] = []; let openClawPosture: "locked" | "mutable" = "mutable"; vi.spyOn(runner, "validateName").mockImplementation((name: unknown) => String(name)); vi.spyOn(runner, "runCapture").mockReturnValue( - options.livePolicy ?? options.livePolicyYaml ?? "version: 1\nnetwork_policies:\n test: {}\n", + options.livePolicyYaml ?? "version: 1\nnetwork_policies:\n test: {}\n", ); const runSpy = vi.spyOn(runner, "run").mockImplementation((cmd: unknown) => { return options.run ? options.run(cmd) : { status: 0 }; }); options.fork && vi.spyOn(childProcess, "fork").mockImplementation(options.fork); vi.spyOn(policy, "buildPolicyGetCommand").mockReturnValue(["openshell", "policy", "get"]); - vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: unknown) => { - recordPolicySetBody(policySetBodies, file); - return ["openshell", "policy", "set"]; - }); + vi.spyOn(policy, "buildPolicySetCommand").mockReturnValue(["openshell", "policy", "set"]); vi.spyOn(policy, "parseCurrentPolicy").mockImplementation((raw: unknown) => String(raw)); vi.spyOn(policy, "resolvePermissivePolicyPath").mockReturnValue( path.join(tmpDir, "permissive.yaml"), @@ -172,13 +111,8 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { configPath: "/sandbox/.openclaw/openclaw.json", format: "json", }); - vi.spyOn(registry, "getSandbox").mockReturnValue( - options.sandboxEntry ?? { name: "openclaw", openshellDriver: "docker" }, - ); + vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "openclaw", openshellDriver: "docker" }); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [{ name: "openclaw" }] }); - const permissiveRuntime = requireDist( - "./permissive-runtime.js", - ) as typeof import("./permissive-runtime.js"); const directSandboxUnavailableError = new Error( "No running direct OpenShell sandbox container found for 'openclaw' (driver: docker). Expected a running container named openshell-openclaw or openshell-openclaw-*. Is the sandbox running?", ); @@ -274,33 +208,15 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { : ""; }); const auditSpy = vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined); - const cleanupTempDirSpy = vi.spyOn(tempFiles, "cleanupTempDir"); - const prepareStateSaveFailure = options.failStateSave - ? () => - fs.mkdirSync(path.join(tmpDir, ".nemoclaw", "state", "shields-openclaw.json"), { - recursive: true, - }) - : () => undefined; - const buildRuntimePermissivePolicy = permissiveRuntime.buildRuntimePermissivePolicy; - vi.spyOn(permissiveRuntime, "buildRuntimePermissivePolicy").mockImplementation( - (basePath, deps) => { - const runtimePolicy = buildRuntimePermissivePolicy(basePath, deps); - prepareStateSaveFailure(); - return runtimePolicy; - }, - ); const shields = requireDist(shieldsModulePath); logSpy.mockClear(); errorSpy.mockClear(); auditSpy.mockClear(); return { - applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot, auditSpy, - cleanupTempDirSpy, errorSpy, logSpy, - policySetBodies, runSpy, shieldsDown: shields.shieldsDown, shieldsStatus: shields.shieldsStatus, @@ -326,20 +242,6 @@ function expectStagedDriverNeutralRecovery( return output; } -function expectManagedPolicyCleanup(harness: ShieldsHarness): void { - expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( - expect.stringContaining("nemoclaw-permissive-runtime"), - "nemoclaw-permissive-runtime", - ); - expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); - const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); - expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); -} - -function writeOpenClawShieldsState(stateDir: string, state: Record): void { - fs.writeFileSync(path.join(stateDir, "shields-openclaw.json"), JSON.stringify(state)); -} - function writeExpiredShieldsFixture( processToken: string, reason: string, @@ -402,8 +304,6 @@ describe("shields command flow", () => { delete require.cache[requireDist.resolve(shieldsModulePath)]; delete require.cache[requireDist.resolve("./timer-bound-lock.js")]; delete require.cache[requireDist.resolve("./transition-lock.js")]; - delete require.cache[requireDist.resolve("./permissive-runtime.js")]; - delete require.cache[requireDist.resolve("../actions/sandbox/mcp-bridge-policy.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; }); @@ -434,247 +334,6 @@ describe("shields command flow", () => { ); }); - it("shieldsDown preserves an exact managed MCP policy and records its snapshot key (#7952)", { - timeout: 15_000, - }, () => { - const alpha = managedMcpPolicy("alpha"); - const harness = createHarness({ - livePolicy: YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - mcp_bridge_alpha: alpha.networkPolicy, - }, - }), - sandboxEntry: managedMcpSandbox([alpha]), - }); - - harness.shieldsDown("openclaw", { - timeout: "5m", - reason: "managed MCP transition coverage", - skipTimer: true, - throwOnError: true, - }); - - const state = JSON.parse( - fs.readFileSync(path.join(tmpDir, ".nemoclaw", "state", "shields-openclaw.json"), "utf-8"), - ); - expect(state.shieldsManagedMcpPolicyKeys).toEqual(["mcp_bridge_alpha"]); - const applied = YAML.parse(harness.policySetBodies.at(-1)!); - expect(applied.network_policies.mcp_bridge_alpha).toEqual(alpha.networkPolicy); - expect(applied.network_policies).not.toHaveProperty("restrictive_baseline"); - }); - - it("cleans the staged managed MCP policy when timer startup fails", () => { - const alpha = managedMcpPolicy("alpha"); - const harness = createHarness({ - fork: () => { - throw new Error("timer startup failed"); - }, - livePolicy: YAML.stringify({ - version: 1, - network_policies: { [alpha.key]: alpha.networkPolicy }, - }), - sandboxEntry: managedMcpSandbox([alpha]), - }); - - expect(() => - harness.shieldsDown("openclaw", { - timeout: "5m", - reason: "cleanup coverage", - throwOnError: true, - }), - ).toThrow("Cannot start auto-restore timer: timer startup failed"); - - expectManagedPolicyCleanup(harness); - }); - - it("cleans the staged managed MCP policy when state persistence fails", () => { - const alpha = managedMcpPolicy("alpha"); - const harness = createHarness({ - failStateSave: true, - livePolicy: YAML.stringify({ - version: 1, - network_policies: { [alpha.key]: alpha.networkPolicy }, - }), - sandboxEntry: managedMcpSandbox([alpha]), - }); - - expect(() => - harness.shieldsDown("openclaw", { - timeout: "5m", - reason: "cleanup coverage", - skipTimer: true, - throwOnError: true, - }), - ).toThrow(/EISDIR|directory/i); - - expectManagedPolicyCleanup(harness); - }); - - it("timer restore uses persisted MCP ownership after its transition marker clears (#7952)", () => { - const alpha = managedMcpPolicy("alpha", "8.8.8.8"); - const beta = managedMcpPolicy("beta", "1.1.1.1"); - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const snapshotPath = path.join(stateDir, "policy-snapshot-managed-restore.yaml"); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - snapshotPath, - YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - mcp_bridge_alpha: alpha.networkPolicy, - }, - }), - ); - writeOpenClawShieldsState(stateDir, { - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], - }); - const harness = createHarness({ - livePolicy: YAML.stringify({ - version: 1, - network_policies: { - permissive_baseline: { endpoints: [{ host: "*" }] }, - mcp_bridge_alpha: alpha.networkPolicy, - mcp_bridge_beta: beta.networkPolicy, - }, - }), - sandboxEntry: managedMcpSandbox([alpha, beta]), - }); - - const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { - transitionProcessToken: "6".repeat(32), - }); - - expect(result.status).toBe(0); - const restored = YAML.parse(harness.policySetBodies.at(-1)!); - expect(Object.keys(restored.network_policies).sort()).toEqual([ - "mcp_bridge_alpha", - "mcp_bridge_beta", - "restrictive_baseline", - ]); - expect(restored.network_policies.mcp_bridge_beta).toEqual(beta.networkPolicy); - }); - - it("refuses manual restoration when persisted MCP ownership is malformed (#7952)", () => { - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const snapshotPath = path.join(stateDir, "policy-snapshot-corrupt-state.yaml"); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - writeOpenClawShieldsState(stateDir, { - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: ["../not-a-managed-key"], - }); - const harness = createHarness(); - - expect(() => harness.applyShieldsPolicySnapshot("openclaw", snapshotPath)).toThrow( - /Saved Shields MCP policy ownership is invalid/, - ); - expect(harness.policySetBodies).toHaveLength(0); - }); - - it("refuses a legacy restore whose persisted state names a different snapshot (#7952)", () => { - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const expectedSnapshotPath = path.join(stateDir, "policy-snapshot-expected.yaml"); - const requestedSnapshotPath = path.join(stateDir, "policy-snapshot-requested.yaml"); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(expectedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync(requestedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); - writeOpenClawShieldsState(stateDir, { - shieldsDown: true, - shieldsPolicySnapshotPath: expectedSnapshotPath, - }); - const harness = createHarness(); - - expect(() => harness.applyShieldsPolicySnapshot("openclaw", requestedSnapshotPath)).toThrow( - /does not match the policy snapshot/, - ); - expect(harness.policySetBodies).toHaveLength(0); - }); - - it("uses token-bound transition ownership when the forward owner dies before state commit (#7952)", () => { - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const processToken = "8".repeat(32); - const snapshotPath = path.join(stateDir, "policy-snapshot-new-cycle.yaml"); - const oldSnapshotPath = path.join(stateDir, "policy-snapshot-old-cycle.yaml"); - const alpha = managedMcpPolicy("alpha", "8.8.8.8"); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - snapshotPath, - YAML.stringify({ - version: 1, - network_policies: { mcp_bridge_alpha: alpha.networkPolicy }, - }), - ); - fs.writeFileSync(oldSnapshotPath, "version: 1\nnetwork_policies: {}\n"); - writeOpenClawShieldsState(stateDir, { - shieldsDown: false, - shieldsPolicySnapshotPath: oldSnapshotPath, - shieldsManagedMcpPolicyKeys: [], - }); - fs.writeFileSync( - path.join(stateDir, `shields-transition-openclaw-${processToken}.json`), - JSON.stringify({ - version: 1, - phase: "preparing", - ownerPid: process.pid, - ownerStartIdentity: "forward-owner", - processToken, - sandboxName: "openclaw", - snapshotPath, - managedMcpPolicyKeys: ["mcp_bridge_alpha"], - }), - ); - const harness = createHarness({ - livePolicy: YAML.stringify({ - version: 1, - network_policies: { mcp_bridge_alpha: alpha.networkPolicy }, - }), - sandboxEntry: managedMcpSandbox([alpha]), - }); - - const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { - transitionProcessToken: processToken, - }); - - expect(result.status).toBe(0); - expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies.mcp_bridge_alpha).toEqual( - alpha.networkPolicy, - ); - }); - - it("loads 257 managed keys recorded by Shields down (#7952)", { timeout: 15_000 }, () => { - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const snapshotPath = path.join(stateDir, "policy-snapshot-many-managed-keys.yaml"); - const policies = Array.from({ length: 257 }, (_, index) => managedMcpPolicy(`server${index}`)); - const keys = policies.map(({ key }) => key); - const networkPolicies = Object.fromEntries( - policies.map(({ key, networkPolicy }) => [key, networkPolicy]), - ); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(snapshotPath, YAML.stringify({ network_policies: networkPolicies })); - writeOpenClawShieldsState(stateDir, { - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: keys, - }); - const harness = createHarness({ - livePolicy: YAML.stringify({ version: 1, network_policies: networkPolicies }), - sandboxEntry: managedMcpSandbox(policies), - }); - - expect(harness.applyShieldsPolicySnapshot("openclaw", snapshotPath).status).toBe(0); - const applied = YAML.parse(harness.policySetBodies.at(-1)!); - const appliedKeys = Object.keys(applied.network_policies); - expect([...appliedKeys].sort()).toEqual([...keys].sort()); - expect(appliedKeys).toHaveLength(257); - expect(appliedKeys).toContain("mcp_bridge_server256"); - }); - it("binds manual shields-up to the active auto-restore timer generation", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const sandboxName = "openclaw"; @@ -738,6 +397,80 @@ describe("shields command flow", () => { }); }); + it.skipIf(currentProcessStartIdentity === null)( + "lets the lifecycle owner raise Shields after a live timer's completion grace (#7952)", + { timeout: 15_000 }, + () => { + const sandboxName = "openclaw"; + const processToken = "7".repeat(32); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const timerControl = requireDist("./timer-control.js"); + const { stateDir, timerMarkerPath, transitionLockPath } = writeExpiredShieldsFixture( + processToken, + "long lifecycle operation", + "dead", + ); + fs.rmSync(transitionLockPath); + const marker = JSON.parse(fs.readFileSync(timerMarkerPath, "utf-8")); + marker.restoreAt = new Date(Date.now() + 60_000).toISOString(); + fs.writeFileSync(timerMarkerPath, JSON.stringify(marker)); + const transitionPath = path.join( + stateDir, + `shields-transition-${sandboxName}-${processToken}.json`, + ); + fs.writeFileSync( + transitionPath, + JSON.stringify({ + version: 1, + phase: "active", + ownerPid: process.pid, + ownerStartIdentity: currentProcessStartIdentity, + processToken, + sandboxName, + snapshotPath: marker.snapshotPath, + }), + ); + vi.spyOn(timerControl, "isProcessAlive").mockReturnValue(true); + vi.spyOn(timerControl, "verifyTimerMarkerIdentity").mockReturnValue({ verified: true }); + const waitSpy = vi.spyOn(Atomics, "wait"); + const harness = createHarness({ + dockerExecFileSync: (argv: unknown) => { + const args = Array.isArray(argv) ? argv.map(String) : []; + if (args.includes("sha256sum")) return `${"a".repeat(64)} ${String(args.at(-1))}\n`; + if (args.includes("lsattr")) return `----i---------e----- ${String(args.at(-1))}\n`; + if (!args.includes("stat")) return ""; + if (args.at(-1) === "/sandbox") return "1775 root:sandbox\n"; + if (args.at(-1) === "/sandbox/.openclaw") return "755 root:root\n"; + return "444 root:root\n"; + }, + }); + const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath(sandboxName, stateDir)}.containment`; + + lifecycleLock.withMcpLifecycleLockSync( + sandboxName, + () => { + marker.restoreAt = new Date(Date.now() - 60_000).toISOString(); + fs.writeFileSync(timerMarkerPath, JSON.stringify(marker)); + expect(lifecycleLock.isMcpLifecycleLockHeld(sandboxName, stateDir)).toBe(true); + harness.shieldsUp(sandboxName, { throwOnError: true }); + }, + { stateDir }, + ); + + expect( + JSON.parse(fs.readFileSync(path.join(stateDir, `shields-${sandboxName}.json`), "utf-8")) + .shieldsDown, + ).toBe(false); + expect(fs.existsSync(timerMarkerPath)).toBe(false); + expect(fs.existsSync(transitionPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(waitSpy.mock.calls.filter((call) => call[3] === 5_000)).toHaveLength(0); + expect(harness.auditSpy).not.toHaveBeenCalledWith( + expect.objectContaining({ action: "shields_up_failed" }), + ); + }, + ); + it("auto-restore waits for the forward shields-down commit before reclaiming policy", () => { const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -937,8 +670,8 @@ describe("shields command flow", () => { expect(fs.existsSync(containmentPath)).toBe(true); }); - it("lets an expired timer preempt its token-bound destroy owner and restore lockdown", { - timeout: 20_000, + it("waits for a token-bound destroy owner without signaling it, then restores lockdown", { + timeout: 10_000, }, async () => { const transitionLockPath = path.join(import.meta.dirname, "transition-lock.ts"); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -946,6 +679,7 @@ describe("shields command flow", () => { const processToken = "e".repeat(32); const snapshotPath = path.join(stateDir, "policy-snapshot-destroy.yaml"); const readyPath = path.join(stateDir, "destroy-owner.ready"); + const releasePath = path.join(stateDir, "destroy-owner.release"); const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); const statePath = path.join(stateDir, `shields-${sandboxName}.json`); @@ -985,12 +719,14 @@ describe("shields command flow", () => { [ `const {withShieldsTransitionLock}=require(${JSON.stringify(transitionLockPath)})`, "const fs=require('fs')", - "const [name,token,ready]=process.argv.slice(1)", - "withShieldsTransitionLock(name,'destroy sandbox',()=>{fs.writeFileSync(ready,'ready');Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,10000)},{takeoverToken:token})", + "const [name,token,ready,release]=process.argv.slice(1)", + "const waitBuffer=new Int32Array(new SharedArrayBuffer(4))", + "withShieldsTransitionLock(name,'destroy sandbox',()=>{fs.writeFileSync(ready,'ready');const deadline=Date.now()+5000;while(!fs.existsSync(release)){if(Date.now()>=deadline)throw new Error('release handshake timed out');Atomics.wait(waitBuffer,0,0,10)}},{takeoverToken:token})", ].join(";"), sandboxName, processToken, readyPath, + releasePath, ], { env: { ...process.env, HOME: tmpDir }, stdio: "ignore" }, ); @@ -1006,6 +742,33 @@ describe("shields command flow", () => { const timerControl = requireDist("./timer-control.js"); const ownerStartIdentity = timerControl.readProcessStartIdentity(owner.pid); expect(ownerStartIdentity).toBeTypeOf("string"); + const processKillSpy = vi.spyOn(process, "kill"); + const nativeAtomicsWait = Atomics.wait; + let releaseObserved = false; + vi.spyOn(Atomics, "wait").mockImplementation((( + _typedArray: Int32Array, + _index: number, + _value: number, + _timeout?: number, + ) => { + if (!releaseObserved) { + const ownerState = timerControl.readProcessState(owner.pid); + expect(ownerState).not.toBeNull(); + expect(ownerState?.startsWith("Z")).toBe(false); + expect(fs.existsSync(lockPath)).toBe(true); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); + fs.writeFileSync(releasePath, "release"); + const releaseDeadline = Date.now() + 5_000; + const waitBuffer = new Int32Array(new SharedArrayBuffer(4)); + while (fs.existsSync(lockPath) && Date.now() < releaseDeadline) { + nativeAtomicsWait(waitBuffer, 0, 0, 10); + } + expect(fs.existsSync(lockPath)).toBe(false); + releaseObserved = true; + } + return "timed-out"; + }) as typeof Atomics.wait); const harness = createHarness({ dockerExecFileSync: (argv: unknown) => { const args = Array.isArray(argv) ? argv.map(String) : []; @@ -1028,8 +791,16 @@ describe("shields command flow", () => { harness.shieldsStatus(sandboxName); - const ownerState = timerControl.readProcessState(owner.pid); - expect(ownerState === null || ownerState.startsWith("Z")).toBe(true); + expect(releaseObserved).toBe(true); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); + await vi.waitFor( + () => { + const ownerState = timerControl.readProcessState(owner.pid); + expect(ownerState === null || ownerState.startsWith("Z")).toBe(true); + }, + { timeout: 2_000, interval: 10 }, + ); expect(fs.existsSync(lockPath)).toBe(false); expect(JSON.parse(fs.readFileSync(statePath, "utf-8"))).toMatchObject({ shieldsDown: false, @@ -1042,7 +813,6 @@ describe("shields command flow", () => { ); expect(harness.logSpy).toHaveBeenCalledWith(" Shields: UP (lockdown active)"); } finally { - owner.kill("SIGCONT"); owner.kill("SIGKILL"); } }); @@ -1115,7 +885,6 @@ describe("shields command flow", () => { ownerPid: process.pid, sandboxName: "openclaw", snapshotPath: expect.stringContaining("policy-snapshot-"), - managedMcpPolicyKeys: [], }); expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(true); expect( @@ -1219,14 +988,17 @@ describe("shields command flow", () => { const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - writeOpenClawShieldsState(stateDir, { - shieldsDown: true, - shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), - shieldsDownTimeout: 300, - shieldsDownReason: "coverage", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: path.join(stateDir, "missing-snapshot.yaml"), - }); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "coverage", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: path.join(stateDir, "missing-snapshot.yaml"), + }), + ); expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( "Saved policy snapshot is missing", @@ -1256,14 +1028,17 @@ describe("shields command flow", () => { const snapshotPath = path.join(stateDir, "policy-snapshot-failed-restore.yaml"); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - writeOpenClawShieldsState(stateDir, { - shieldsDown: true, - shieldsDownAt: new Date().toISOString(), - shieldsDownTimeout: 300, - shieldsDownReason: "recovery-hint coverage", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, - }); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date().toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "recovery-hint coverage", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }), + ); expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( "policy restore exited with status 1", @@ -1319,57 +1094,73 @@ describe("shields command flow", () => { expect(output).not.toContain("CRITICAL: OpenClaw lock rollback"); }); - it.each<{ - scenario: string; - options: HarnessOptions; - expectedError: RegExp; - }>([ - { - scenario: "non-transient OpenClaw rollback failures", - options: { - failOpenClawGuardActions: ["lock"], - openClawGuardFailure: { - code: "unsafe-config-path", - path: "/sandbox/.openclaw/openclaw.json", - detail: "canonical config path is not a safe regular file", - }, + it("retains critical recovery for non-transient OpenClaw rollback failures (#6126)", () => { + const harness = createHarness({ + failOpenClawGuardActions: ["lock"], + openClawGuardFailure: { + code: "unsafe-config-path", + path: "/sandbox/.openclaw/openclaw.json", + detail: "canonical config path is not a safe regular file", }, - expectedError: /unsafe-config-path/, - }, - { - scenario: "structural startup-not-ready diagnostics", - options: { - failOpenClawGuardActions: ["lock"], - openClawGuardFailure: { + }); + + expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( + /unsafe-config-path/, + ); + + const output = harness.errorSpy.mock.calls.flat().map(String).join("\n"); + expect(output).toContain( + "CRITICAL: OpenClaw lock rollback could not restore the trusted posture. Restore from a trusted backup and recreate the sandbox.", + ); + expect(output).not.toContain( + "Warning: OpenClaw lock rollback could not restore the trusted posture", + ); + }); + + it("retains critical recovery for structural startup-not-ready diagnostics (#6126)", () => { + const harness = createHarness({ + failOpenClawGuardActions: ["lock"], + openClawGuardFailure: { + code: "startup-not-ready", + path: "/run/nemoclaw/openclaw-config-ready.json", + detail: "installed config guard requires NemoClaw PID 1", + }, + }); + + expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( + /requires NemoClaw PID 1/, + ); + + const output = harness.errorSpy.mock.calls.flat().map(String).join("\n"); + expect(output).toContain( + "CRITICAL: OpenClaw lock rollback could not restore the trusted posture. Restore from a trusted backup and recreate the sandbox.", + ); + expect(output).not.toContain( + "Warning: OpenClaw lock rollback could not restore the trusted posture", + ); + }); + + it("retains critical recovery when a transient diagnostic is followed by another issue (#6126)", () => { + const harness = createHarness({ + failOpenClawGuardActions: ["lock"], + openClawGuardFailures: [ + { code: "startup-not-ready", path: "/run/nemoclaw/openclaw-config-ready.json", - detail: "installed config guard requires NemoClaw PID 1", + detail: "OpenClaw startup is not ready for host config mutations", }, - }, - expectedError: /requires NemoClaw PID 1/, - }, - { - scenario: "a transient diagnostic followed by another issue", - options: { - failOpenClawGuardActions: ["lock"], - openClawGuardFailures: [ - { - code: "startup-not-ready", - path: "/run/nemoclaw/openclaw-config-ready.json", - detail: "OpenClaw startup is not ready for host config mutations", - }, - { - code: "unsafe-config-path", - path: "/sandbox/.openclaw/openclaw.json", - detail: "canonical config path is not a safe regular file", - }, - ], - }, - expectedError: /unsafe-config-path/, - }, - ])("retains critical recovery for $scenario (#6126)", ({ options, expectedError }) => { - const harness = createHarness(options); - expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow(expectedError); + { + code: "unsafe-config-path", + path: "/sandbox/.openclaw/openclaw.json", + detail: "canonical config path is not a safe regular file", + }, + ], + }); + + expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( + /unsafe-config-path/, + ); + const output = harness.errorSpy.mock.calls.flat().map(String).join("\n"); expect(output).toContain( "CRITICAL: OpenClaw lock rollback could not restore the trusted posture. Restore from a trusted backup and recreate the sandbox.", @@ -1383,14 +1174,17 @@ describe("shields command flow", () => { const harness = createHarness({ failOpenClawGuardActions: ["lock"] }); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - writeOpenClawShieldsState(stateDir, { - shieldsDown: false, - chattrApplied: false, - fileHashes: { - "/sandbox/.openclaw/openclaw.json": "a".repeat(64), - "/sandbox/.openclaw/.config-hash": "a".repeat(64), - }, - }); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: false, + chattrApplied: false, + fileHashes: { + "/sandbox/.openclaw/openclaw.json": "a".repeat(64), + "/sandbox/.openclaw/.config-hash": "a".repeat(64), + }, + }), + ); expect(() => harness.shieldsUp("openclaw", { throwOnError: true })).toThrow( /startup-not-ready/, @@ -1407,14 +1201,17 @@ describe("shields command flow", () => { const markerPath = path.join(stateDir, "shields-timer-openclaw.json"); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - writeOpenClawShieldsState(stateDir, { - shieldsDown: true, - shieldsDownAt: new Date().toISOString(), - shieldsDownTimeout: 1800, - shieldsDownReason: "rebuild", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, - }); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date().toISOString(), + shieldsDownTimeout: 1800, + shieldsDownReason: "rebuild", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }), + ); fs.writeFileSync( markerPath, JSON.stringify({ @@ -1500,13 +1297,7 @@ describe("shields command flow", () => { }); const harness = createHarness({ beginContainment: () => { - const error = new Error("state directory is read-only") as Error & { - code: string; - retainOwnedLifecycleGates: boolean; - }; - error.code = "NEMOCLAW_DURABLE_CONTAINMENT"; - error.retainOwnedLifecycleGates = true; - throw error; + throw new Error("state directory is read-only"); }, }); let containmentFailure: unknown; @@ -1522,7 +1313,6 @@ describe("shields command flow", () => { expect(result).toBe("handled"); expect(containmentFailure).toMatchObject({ code: "NEMOCLAW_DURABLE_CONTAINMENT", - retainOwnedLifecycleGates: true, }); expect(String(containmentFailure)).toContain("state directory is read-only"); expect(fs.existsSync(containmentPath)).toBe(false); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index 2c2cc8dfade..ad3bdbbd457 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -103,22 +103,6 @@ function withDefaultNodeExecFileSync( return defaultNodeExecFileSync(file, argv) || fallback(); } -function throwRegistryPermissionDenied(): never { - throw Object.assign(new Error("registry permission denied"), { code: "EACCES" }); -} - -function readFileWithUnreadableRegistry( - originalReadFileSync: typeof fs.readFileSync, - file: fs.PathOrFileDescriptor, - options?: unknown, -): unknown { - const readers = new Map unknown>([ - [true, throwRegistryPermissionDenied], - [false, () => originalReadFileSync(file, options as never)], - ]); - return readers.get(String(file).endsWith(`${path.sep}sandboxes.json`))!(); -} - function throwProcessNotRunning(): never { throw Object.assign(new Error("not running"), { code: "ESRCH" }); } @@ -132,23 +116,6 @@ function routeProcessKill(pid: number, signal?: string | number): true { return (processActions.get(`${pid}:${signal}`) ?? reportProcessRunning)(); } -function readRuntimePolicyBeforeCleanup( - cleanupDir: string, - readFile: typeof fs.readFileSync, -): string | null { - switch ( - path.basename(cleanupDir).startsWith("nemoclaw-permissive-runtime-") && - fs.existsSync(cleanupDir) - ) { - case false: - return null; - case true: { - const policyFile = fs.readdirSync(cleanupDir).find((name) => name.endsWith(".yaml")); - return policyFile ? readFile(path.join(cleanupDir, policyFile), "utf-8") : null; - } - } -} - beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-test-")); vi.stubEnv("HOME", tmpDir); @@ -529,65 +496,6 @@ describe("shields — unit logic", () => { expect(logSpy).toHaveBeenCalledWith(" Shields: DOWN (temporarily unlocked)"); }); - it("deadline composition removes an unproven MCP add from the restrictive policy", async () => { - const snapshot = - "version: 1\nnetwork_policies:\n restrictive_baseline: {}\n mcp_bridge_beta: {}\n"; - const { composeDeadlineManagedMcpPolicies } = await import("./mcp-policy-transition"); - const composition = composeDeadlineManagedMcpPolicies(snapshot, [], ["mcp_bridge_beta"]); - - expect(composition.yaml).toContain("restrictive_baseline"); - expect(composition.yaml).not.toContain("mcp_bridge_beta"); - }); - - it("deadline restore removes saved MCP keys when the registry cannot be read", async () => { - const sandboxName = "openclaw"; - const processToken = "b".repeat(32); - const snapshotPath = path.join(stateDir(), "policy-snapshot-unreadable-registry.yaml"); - fs.mkdirSync(stateDir(), { recursive: true }); - fs.writeFileSync( - snapshotPath, - "version: 1\nnetwork_policies:\n restrictive_baseline: {}\n mcp_bridge_alpha: {}\n", - ); - writeState(sandboxName, { - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], - }); - writeMarker(sandboxName, { - pid: 2_147_483_647, - sandboxName, - snapshotPath, - restoreAt: new Date(Date.now() - 1_000).toISOString(), - processToken, - }); - const originalReadFileSync = fs.readFileSync.bind(fs); - vi.spyOn(fs, "readFileSync").mockImplementation((file, options) => { - return readFileWithUnreadableRegistry(originalReadFileSync, file, options) as never; - }); - vi.spyOn(process, "kill").mockImplementation(routeProcessKill); - const originalRmSync = fs.rmSync.bind(fs); - let appliedPolicy = ""; - vi.spyOn(fs, "rmSync").mockImplementation((target, options) => { - const cleanupDir = String(target); - appliedPolicy = - readRuntimePolicyBeforeCleanup(cleanupDir, originalReadFileSync) ?? appliedPolicy; - originalRmSync(target, options); - }); - const { applyShieldsPolicySnapshot } = await loadShieldsModule(); - - const result = applyShieldsPolicySnapshot(sandboxName, snapshotPath, { - transitionProcessToken: processToken, - deadlineAuthoritative: true, - expiredTimerRecovery: true, - }); - - expect(result.managedMcpOmissions).toEqual([ - expect.objectContaining({ reason: expect.stringMatching(/Cannot read config file:/) }), - ]); - expect(appliedPolicy).toContain("restrictive_baseline"); - expect(appliedPolicy).not.toContain("mcp_bridge_alpha"); - }); - it("shieldsStatus warns and stays DOWN when inline recovery fails", async () => { const sandboxName = "openclaw"; const missingSnapshotPath = path.join(stateDir(), "missing-snapshot.yaml"); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 33c6b24f667..10eda298ec1 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -56,13 +56,7 @@ const { resolveNemoclawStateDir } = require("../state/paths"); const { appendAuditEntry } = require("./audit"); const { resolveAgentConfig } = require("../sandbox/agent-config"); const { - assertLegacyMcpPolicyRestoreSafe, - buildDeadlineRuntimeManagedMcpPolicy, - buildRuntimeManagedMcpPolicy, buildRuntimePermissivePolicy, - hasManagedMcpPolicyClaims, - inspectExactManagedMcpPolicies, - inspectProvableManagedMcpPoliciesForDeadline, }: typeof import("./permissive-runtime") = require("./permissive-runtime"); const { cleanupTempDir } = require("../onboard/temp-files"); const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock"); @@ -77,7 +71,6 @@ const { getMcpLifecycleLockPath, isMcpLifecycleLockHeld, durableMcpLifecycleContainmentFailure, - readMcpLockProcessIdentity, withMcpLifecycleDeadlineFenceSync, withMcpLifecycleLockSync, withTimerBoundAutoRestoreLock, @@ -112,7 +105,6 @@ const { }: typeof import("./mutable-config-repair") = require("./mutable-config-repair"); type MutableConfigPermsInspection = import("./mutable-config-perms").MutableConfigPermsInspection; type MutableConfigRepairResult = import("./mutable-config-perms").MutableConfigRepairResult; -type ManagedMcpPolicyOmission = import("./permissive-runtime").ManagedMcpPolicyOmission; type TimerMarker = import("./timer-control").TimerMarker; const STATE_DIR = resolveNemoclawStateDir(); const SHIELDS_TRANSITION_POLL_MS = 50; @@ -139,12 +131,9 @@ type ShieldsDownTransition = { phase: "preparing" | "active"; ownerPid: number; ownerStartIdentity: string; - ownerMcpProcessIdentity?: string; processToken: string; sandboxName: string; snapshotPath: string; - /** Exact generated MCP keys owned when snapshotPath was captured. */ - managedMcpPolicyKeys?: string[]; }; const transitionPollBuffer = new Int32Array(new SharedArrayBuffer(4)); @@ -190,25 +179,13 @@ function isShieldsDownTransition(value: unknown): value is ShieldsDownTransition value.ownerPid > 0 && typeof value.ownerStartIdentity === "string" && value.ownerStartIdentity.length > 0 && - (value.ownerMcpProcessIdentity === undefined || - (typeof value.ownerMcpProcessIdentity === "string" && - value.ownerMcpProcessIdentity.length > 0)) && typeof value.processToken === "string" && /^[0-9a-f]{32}$/.test(value.processToken) && typeof value.sandboxName === "string" && - typeof value.snapshotPath === "string" && - isOptionalManagedMcpPolicyKeys(value.managedMcpPolicyKeys) + typeof value.snapshotPath === "string" ); } -function sameManagedMcpPolicyKeys( - left: readonly string[] | undefined, - right: readonly string[] | undefined, -): boolean { - if (left === undefined || right === undefined) return left === right; - return left.length === right.length && left.every((key, index) => key === right[index]); -} - function readShieldsDownTransition( sandboxName: string, processToken: string, @@ -236,9 +213,7 @@ function writeShieldsDownTransition( !current || current.phase !== expectedPhase || current.ownerPid !== transition.ownerPid || - current.snapshotPath !== transition.snapshotPath || - current.ownerMcpProcessIdentity !== transition.ownerMcpProcessIdentity || - !sameManagedMcpPolicyKeys(current.managedMcpPolicyKeys, transition.managedMcpPolicyKeys) + current.snapshotPath !== transition.snapshotPath ) { throw new Error("Shields-down recovery ownership changed during the transition"); } @@ -351,10 +326,8 @@ function waitForShieldsDownForwardCommit( if ( next.ownerPid !== observed.ownerPid || next.ownerStartIdentity !== observed.ownerStartIdentity || - next.ownerMcpProcessIdentity !== observed.ownerMcpProcessIdentity || next.snapshotPath !== observed.snapshotPath || - next.processToken !== observed.processToken || - !sameManagedMcpPolicyKeys(next.managedMcpPolicyKeys, observed.managedMcpPolicyKeys) + next.processToken !== observed.processToken ) { throw new Error("Shields-down recovery ownership changed while waiting for forward commit"); } @@ -716,8 +689,6 @@ interface ShieldsState { shieldsDownReason?: string | null; shieldsDownPolicy?: string | null; shieldsPolicySnapshotPath?: string | null; - /** Exact generated MCP keys owned in the restrictive snapshot. */ - shieldsManagedMcpPolicyKeys?: string[]; chattrApplied?: boolean; // SHA-256 seal of each locked file, captured by `shields up` after the // lock verification passes. `shields status` re-hashes the same files @@ -1113,7 +1084,9 @@ function withExpiredAutoRestoreDeadlineFence( marker.snapshotPath, () => assertTimerMarkerGeneration(sandboxName, marker), ); - return recoverThenRun(); + // This lifecycle owner is what the live timer is waiting on. Run the + // nested operation without re-entering recovery against that timer. + return runWithHostLock(() => operation(false)); } if (!takeover) { return withMcpLifecycleLockSync(sandboxName, () => runWithHostLock(() => operation(true)), { @@ -1171,7 +1144,18 @@ function withExpiredAutoRestoreDeadlineFence( { stateDir: STATE_DIR, throwOnCommittedContainment: true, - onContainment: ({ ownerPid, reason }) => { + onContainment: ({ kind, ownerPid, reason }) => { + if (kind === "verified-live-wait") { + appendAuditEntryBestEffort({ + action: "shields_auto_restore_lock_warning", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + warning: reason, + }); + return; + } appendAuditEntryBestEffort({ action: "shields_up_failed", sandbox: sandboxName, @@ -1247,14 +1231,6 @@ function isOptionalHashMap(value: unknown): value is { [path: string]: string } return true; } -function isOptionalManagedMcpPolicyKeys(value: unknown): value is string[] | undefined { - if (value === undefined) return true; - // Preserve string entries exactly so deadline recovery can strip and audit - // malformed or duplicate ownership without delaying restrictive lockdown. - // Manual restoration validates the same entries strictly during composition. - return Array.isArray(value) && value.every((key) => typeof key === "string"); -} - function isShieldsState(value: unknown): value is ShieldsState { return ( isObjectRecord(value) && @@ -1264,7 +1240,6 @@ function isShieldsState(value: unknown): value is ShieldsState { isOptionalNullableString(value.shieldsDownReason) && isOptionalNullableString(value.shieldsDownPolicy) && isOptionalNullableString(value.shieldsPolicySnapshotPath) && - isOptionalManagedMcpPolicyKeys(value.shieldsManagedMcpPolicyKeys) && isOptionalBoolean(value.chattrApplied) && isOptionalHashMap(value.fileHashes) && isOptionalString(value.updatedAt) @@ -2352,16 +2327,19 @@ function repairMutableConfigPerms(sandboxName: string): MutableConfigRepairResul function restoreLockedStateDirStartupAccess(sandboxName: string): void { validateName(sandboxName, "sandbox name"); - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - withTimerBoundShieldsMutationLock(sandboxName, "restore locked startup access", () => { - const posture = getShieldsPostureWithoutHostLock(sandboxName, true); - if (!posture.locked) return; - const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); - const issues = restoreStateDirStartupAccess(stateDirLockExec(sandboxName), target.configDir); - if (issues.length > 0) { - throw new Error(`Locked startup access could not be restored: ${issues.join(", ")}`); - } - }); + withExpiredAutoRestoreDeadlineFence( + sandboxName, + "restore locked startup access", + (allowInlineRecovery) => { + const posture = getShieldsPostureWithoutHostLock(sandboxName, allowInlineRecovery); + if (!posture.locked) return; + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); + const issues = restoreStateDirStartupAccess(stateDirLockExec(sandboxName), target.configDir); + if (issues.length > 0) { + throw new Error(`Locked startup access could not be restored: ${issues.join(", ")}`); + } + }, + ); } // --------------------------------------------------------------------------- @@ -2685,7 +2663,6 @@ function synchronizeAutoRestoreTransition( processToken: string, snapshotPath: string, options: { - expiredTimerRecovery?: boolean; retainTransition?: boolean; assertTakeoverAuthority?: () => void; } = {}, @@ -2706,16 +2683,8 @@ function synchronizeAutoRestoreTransition( // above waits until the forward path has either committed its last weakening // mutation or its owner has died; restore the restrictive snapshot again at // that stable boundary before locking config. - const marker = readTimerMarker(sandboxName); - const timerOwnsRecovery = - marker?.pid === process.pid && - marker.processToken === processToken && - marker.snapshotPath === transition.snapshotPath; - const deadlineAuthoritative = timerOwnsRecovery || options.expiredTimerRecovery === true; - const restoreResult = applyShieldsPolicySnapshot(sandboxName, transition.snapshotPath, { - transitionProcessToken: processToken, - ...(deadlineAuthoritative ? { deadlineAuthoritative: true } : {}), - ...(options.expiredTimerRecovery ? { expiredTimerRecovery: true } : {}), + const restoreResult = run(buildPolicySetCommand(transition.snapshotPath, sandboxName), { + ignoreError: true, }); const status = typeof restoreResult.status === "number" ? restoreResult.status : 1; if (status !== 0) { @@ -2728,11 +2697,12 @@ function synchronizeAutoRestoreTransition( } } -function inspectAutoRestoreTransitionTakeoverOwner( +function prepareAutoRestoreTransitionTakeover( sandboxName: string, processToken: string, snapshotPath: string, -): { pid: number; processIdentity: string } | null { + assertTakeoverAuthority?: () => void, +): void { if (!/^[0-9a-f]{32}$/.test(processToken)) { throw new Error("Invalid auto-restore transition takeover token"); } @@ -2740,32 +2710,12 @@ function inspectAutoRestoreTransitionTakeoverOwner( if (transition && transition.snapshotPath !== snapshotPath) { throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); } - return transition?.ownerMcpProcessIdentity !== undefined - ? { pid: transition.ownerPid, processIdentity: transition.ownerMcpProcessIdentity } - : null; -} - -function prepareAutoRestoreTransitionTakeover( - sandboxName: string, - processToken: string, - snapshotPath: string, - assertTakeoverAuthority?: () => void, -): { pid: number; processIdentity: string } | null { - const initialTransitionOwner = inspectAutoRestoreTransitionTakeoverOwner( - sandboxName, - processToken, - snapshotPath, - ); - const transition = readShieldsDownTransition(sandboxName, processToken); - if (transition && transition.snapshotPath !== snapshotPath) { - throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); - } if (transition) { waitForShieldsDownForwardCommit(sandboxName, processToken, assertTakeoverAuthority); } const owner = inspectAnyShieldsTransitionLockOwner(sandboxName); - if (!owner) return initialTransitionOwner; + if (!owner) return; const ownerStatus = readExactProcessStatus( owner.pid, owner.processStartIdentity, @@ -2876,199 +2826,6 @@ function lockAgentConfig( }); } -function resolveExactManagedMcpPolicies( - sandboxName: string, - livePolicyYaml?: string, -): ReturnType { - let effectiveLivePolicy = livePolicyYaml; - if (!effectiveLivePolicy) { - let rawPolicy: string; - try { - rawPolicy = runCapture(buildPolicyGetCommand(sandboxName)); - } catch (error) { - throw new Error("Cannot read the live gateway policy for managed MCP reconciliation", { - cause: error, - }); - } - effectiveLivePolicy = parseCurrentPolicy(rawPolicy); - } - if (!effectiveLivePolicy) { - throw new Error("Cannot parse the live gateway policy for managed MCP reconciliation"); - } - return inspectExactManagedMcpPolicies(sandboxName, effectiveLivePolicy); -} - -function resolveProvableManagedMcpPoliciesForDeadline( - sandboxName: string, -): ReturnType { - try { - let effectiveLivePolicy = ""; - try { - effectiveLivePolicy = parseCurrentPolicy(runCapture(buildPolicyGetCommand(sandboxName))); - } catch { - // The tolerant deadline inspector records exact omissions for every claim - // when the live policy cannot be parsed or read. - } - return inspectProvableManagedMcpPoliciesForDeadline(sandboxName, effectiveLivePolicy); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { - policies: [], - omissions: [ - { - reason: `Managed MCP registry inspection failed at the auto-restore deadline: ${message}`, - }, - ], - }; - } -} - -/** - * Restore a saved complete policy while reconciling only exact generated MCP - * entries. Snapshot-time keys are removed before currently owned entries are - * overlaid, so changes made during the shields-down window survive both manual - * and timer restoration. - */ -interface ShieldsPolicySnapshotRestoreOptions { - transitionProcessToken?: string; - deadlineAuthoritative?: boolean; - expiredTimerRecovery?: boolean; -} - -type ShieldsPolicySnapshotRestoreResult = ReturnType & { - managedMcpOmissions?: ManagedMcpPolicyOmission[]; -}; - -function applyShieldsPolicySnapshot( - sandboxName: string, - snapshotPath: string, - options: ShieldsPolicySnapshotRestoreOptions = {}, -): ShieldsPolicySnapshotRestoreResult { - const state = loadShieldsState(sandboxName); - let transition: ShieldsDownTransition | null = null; - if (options.transitionProcessToken !== undefined) { - if (!/^[0-9a-f]{32}$/.test(options.transitionProcessToken)) { - throw new Error("Invalid Shields transition recovery token"); - } - transition = readShieldsDownTransition(sandboxName, options.transitionProcessToken); - if ( - !transition && - fs.existsSync(shieldsDownTransitionPath(sandboxName, options.transitionProcessToken)) - ) { - throw new Error("Shields transition recovery authority is invalid"); - } - if (transition && transition.snapshotPath !== snapshotPath) { - throw new Error("Shields transition does not authorize the policy snapshot being restored"); - } - } - if (options.deadlineAuthoritative) { - const marker = readTimerMarker(sandboxName); - const markerMatchesRecovery = - marker?.sandboxName === sandboxName && - marker.snapshotPath === snapshotPath && - marker.processToken === options.transitionProcessToken; - const restoreAtMs = marker ? new Date(marker.restoreAt).getTime() : Number.NaN; - const expiredTimerIsInactive = - options.expiredTimerRecovery === true && - markerMatchesRecovery && - Number.isFinite(restoreAtMs) && - restoreAtMs <= Date.now() && - (!isProcessAlive(marker!.pid) || !verifyTimerMarkerIdentity(marker!).verified); - if ( - options.transitionProcessToken === undefined || - !markerMatchesRecovery || - (marker!.pid !== process.pid && !expiredTimerIsInactive) - ) { - throw new Error("The active auto-restore timer does not authorize deadline restoration"); - } - } - - if (state._isCorrupt && !transition) { - throw new Error( - `Cannot restore a Shields policy while persisted state is corrupt: ${ - state._corruptError ?? "invalid state" - }`, - ); - } - // A preparing transition can outlive its owner before Shields state is - // committed; its token-bound marker is then the recovery authority. - // Every ordinary restore remains bound to the exact persisted snapshot. - if (!transition && state.shieldsPolicySnapshotPath !== snapshotPath) { - throw new Error("Shields state does not match the policy snapshot being restored"); - } - const persistedSnapshotMatches = state.shieldsPolicySnapshotPath === snapshotPath; - const ownershipOmissions: ManagedMcpPolicyOmission[] = []; - if ( - transition?.managedMcpPolicyKeys !== undefined && - persistedSnapshotMatches && - state.shieldsManagedMcpPolicyKeys !== undefined && - !sameManagedMcpPolicyKeys(transition.managedMcpPolicyKeys, state.shieldsManagedMcpPolicyKeys) - ) { - if (!options.deadlineAuthoritative) { - throw new Error("Shields transition ownership does not match persisted policy ownership"); - } - ownershipOmissions.push({ - reason: - "Shields transition ownership did not match persisted policy ownership at the auto-restore deadline", - }); - } - let snapshotManagedPolicyKeys = - transition?.managedMcpPolicyKeys ?? - (persistedSnapshotMatches ? state.shieldsManagedMcpPolicyKeys : undefined); - // Older Shields state has no exact snapshot-time ownership manifest. - // A manual restore preserves raw-snapshot behavior only when neither current - // state nor the snapshot can involve managed MCP. Deadline restoration - // instead strips every reserved key and overlays only independently proven - // current entries so legacy metadata cannot delay restrictive lockdown. - if (snapshotManagedPolicyKeys === undefined) { - if (options.deadlineAuthoritative) { - snapshotManagedPolicyKeys = []; - ownershipOmissions.push({ - reason: - "Legacy Shields state had no managed MCP ownership manifest at the auto-restore deadline", - }); - } else { - assertLegacyMcpPolicyRestoreSafe( - fs.readFileSync(snapshotPath, "utf-8"), - hasManagedMcpPolicyClaims(sandboxName), - ); - return run(buildPolicySetCommand(snapshotPath, sandboxName), { - ignoreError: true, - }); - } - } - let managedMcpOmissions: ManagedMcpPolicyOmission[] = []; - let runtimePolicyPath: string; - if (options.deadlineAuthoritative) { - const inspection = resolveProvableManagedMcpPoliciesForDeadline(sandboxName); - const runtime = buildDeadlineRuntimeManagedMcpPolicy(snapshotPath, { - managedMcpPolicies: inspection.policies, - snapshotManagedPolicyKeys, - readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), - }); - runtimePolicyPath = runtime.path; - managedMcpOmissions = [...ownershipOmissions, ...inspection.omissions, ...runtime.omissions]; - } else { - const managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName); - runtimePolicyPath = buildRuntimeManagedMcpPolicy(snapshotPath, { - managedMcpPolicies, - snapshotManagedPolicyKeys, - readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), - }); - } - const runtimePolicyIsTemp = runtimePolicyPath !== snapshotPath; - try { - const result = run(buildPolicySetCommand(runtimePolicyPath, sandboxName), { - ignoreError: true, - }); - return managedMcpOmissions.length > 0 ? { ...result, managedMcpOmissions } : result; - } finally { - if (runtimePolicyIsTemp) { - cleanupTempDir(runtimePolicyPath, "nemoclaw-permissive-runtime"); - } - } -} - function rollbackShieldsDown( sandboxName: string, target: AgentConfigTarget, @@ -3077,16 +2834,12 @@ function rollbackShieldsDown( cachedProtocol?: HermesShieldsProtocol, ): void { console.error(" Rolling back — restoring policy from snapshot..."); - let rollbackResult: ReturnType | null = null; - try { - rollbackResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Warning: Policy restore preparation failed during rollback: ${message}`); - } + const rollbackResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { + ignoreError: true, + }); let rollbackChattrApplied: boolean | null = null; let rollbackFileHashes: { [path: string]: string } | null = null; - if (rollbackResult?.status === 0) { + if (rollbackResult.status === 0) { // Re-confirm after the settle window so a reconciler revert cannot leave // the rolled-back config DRIFTED — same fail-closed treatment as the // auto-restore path. Leaves the hashes null (→ "manual intervention" @@ -3127,7 +2880,6 @@ interface LockdownActivationResult { error?: string; chattrApplied?: boolean; fileHashes?: { [path: string]: string }; - managedMcpOmissions?: ManagedMcpPolicyOmission[]; } function activateLockdownFromSnapshot( @@ -3136,23 +2888,14 @@ function activateLockdownFromSnapshot( allowLegacyHermesProtocol = false, cachedTarget?: AgentConfigTarget, cachedProtocol?: HermesShieldsProtocol, - restoreOptions: ShieldsPolicySnapshotRestoreOptions = {}, ): LockdownActivationResult { if (!snapshotPath || !fs.existsSync(snapshotPath)) { return { ok: false, error: "saved snapshot is missing" }; } - let restoreResult: ShieldsPolicySnapshotRestoreResult; - try { - restoreResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath, restoreOptions); - } catch (error) { - return { - ok: false, - error: `policy restore preparation failed: ${ - error instanceof Error ? error.message : String(error) - }`, - }; - } + const restoreResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { + ignoreError: true, + }); const restoreStatus = typeof restoreResult.status === "number" ? restoreResult.status : 1; if (restoreStatus !== 0) { return { @@ -3194,9 +2937,6 @@ function activateLockdownFromSnapshot( ok: true, chattrApplied: relock.lastResult.chattrApplied, fileHashes: relock.lastResult.fileHashes, - ...(restoreResult.managedMcpOmissions - ? { managedMcpOmissions: restoreResult.managedMcpOmissions } - : {}), }; } @@ -3236,7 +2976,6 @@ function recoverExpiredAutoRestoreInline( if (marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken)) { try { synchronizeAutoRestoreTransition(sandboxName, marker.processToken, marker.snapshotPath, { - expiredTimerRecovery: true, retainTransition: true, assertTakeoverAuthority: () => assertTimerMarkerGeneration(sandboxName, marker), }); @@ -3262,14 +3001,6 @@ function recoverExpiredAutoRestoreInline( marker.snapshotPath, marker.allowLegacyHermesProtocol === true, cachedTarget, - undefined, - marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken) - ? { - transitionProcessToken: marker.processToken, - deadlineAuthoritative: true, - expiredTimerRecovery: true, - } - : {}, ); const nowIso = new Date().toISOString(); if (!activation.ok) { @@ -3310,13 +3041,6 @@ function recoverExpiredAutoRestoreInline( restored_by: "auto_timer", policy_snapshot: marker.snapshotPath, restored_at: nowIso, - ...(activation.managedMcpOmissions?.length - ? { - warning: `Inline auto-restore omitted ${String( - activation.managedMcpOmissions.length, - )} unproven managed MCP policy entries`, - } - : {}), }); return { attempted: true, restored: true }; } @@ -3412,19 +3136,6 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = return failShieldsCommand("Cannot capture current policy", opts.throwOnError); } - let managedMcpPolicies: ReturnType; - try { - managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName, policyYaml); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Cannot preserve managed MCP policy state: ${message}`); - return failShieldsCommand( - `Cannot preserve managed MCP policy state: ${message}`, - opts.throwOnError, - ); - } - const snapshotManagedMcpPolicyKeys = managedMcpPolicies.map((policy) => policy.key); - const ts = Date.now(); const snapshotPath = path.join(STATE_DIR, `policy-snapshot-${ts}.yaml`); fs.mkdirSync(STATE_DIR, { recursive: true, mode: 0o700 }); @@ -3434,40 +3145,25 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = // 2. Determine and apply relaxed policy let policyFile: string; let policyFileIsTemp = false; - try { - if (policyName === "permissive") { - const basePath = resolvePermissivePolicyPath(sandboxName); - // Union the live sandbox's filesystem_policy.read_only/read_write into - // the static permissive baseline. OpenShell rejects removal of those - // paths on a live sandbox, and runtime-injected entries (/proc on - // GPU, /opt/hermes on Hermes, /home/linuxbrew on post-#3913 OpenClaw, - // etc.) are not present in the static YAML. See #3942, #3957, #3168. - // policyYaml is the pre-parsed body we already captured for the - // snapshot above — reuse it instead of re-fetching. Exact generated MCP - // entries are overlaid without copying any unrelated live egress. - policyFile = buildRuntimePermissivePolicy(basePath, { - livePolicyYaml: policyYaml, - managedMcpPolicies, - readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), - }); - policyFileIsTemp = policyFile !== basePath; - } else if (fs.existsSync(policyName)) { - const basePath = path.resolve(policyName); - policyFile = buildRuntimeManagedMcpPolicy(basePath, { - managedMcpPolicies, - readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), - }); - policyFileIsTemp = policyFile !== basePath; - } else { - console.error(` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`); - fs.rmSync(snapshotPath, { force: true }); - return failShieldsCommand(`Unknown policy "${policyName}"`, opts.throwOnError); - } - } catch (error) { - fs.rmSync(snapshotPath, { force: true }); - const message = error instanceof Error ? error.message : String(error); - console.error(` Cannot compose Shields-down policy: ${message}`); - return failShieldsCommand(`Cannot compose Shields-down policy: ${message}`, opts.throwOnError); + if (policyName === "permissive") { + const basePath = resolvePermissivePolicyPath(sandboxName); + // Union the live sandbox's filesystem_policy.read_only/read_write into + // the static permissive baseline. OpenShell rejects removal of those + // paths on a live sandbox, and runtime-injected entries (/proc on + // GPU, /opt/hermes on Hermes, /home/linuxbrew on post-#3913 OpenClaw, + // etc.) are not present in the static YAML. See #3942, #3957, #3168. + // policyYaml is the pre-parsed body we already captured for the + // snapshot above — reuse it instead of re-fetching. + policyFile = buildRuntimePermissivePolicy(basePath, { + livePolicyYaml: policyYaml, + readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), + }); + policyFileIsTemp = policyFile !== basePath; + } else if (fs.existsSync(policyName)) { + policyFile = path.resolve(policyName); + } else { + console.error(` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`); + return failShieldsCommand(`Unknown policy "${policyName}"`, opts.throwOnError); } // Every exit after the permissive merge builds a temp policy directory must @@ -3484,123 +3180,113 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = const now = new Date().toISOString(); let transition: ShieldsDownTransition | null = null; - try { - // Commit the host-side recovery authority before weakening policy or file - // permissions. If this process is killed later, the detached timer and its - // marker already exist and the persisted state honestly reports shields - // down. A crash can therefore never leave an untracked mutable window. - if (!opts.skipTimer) { - const restoreAt = new Date(Date.now() + timeoutSeconds * 1000); - const processToken = opts.processToken ?? randomBytes(16).toString("hex"); - if (!/^[0-9a-f]{32}$/.test(processToken)) { - throw new Error("Invalid shields-down recovery process token"); - } - const timerScript = path.join(__dirname, "timer.ts"); - const timerScriptJs = timerScript.replace(/\.ts$/, ".js"); - const actualScript = fs.existsSync(timerScriptJs) ? timerScriptJs : timerScript; - transition = { - version: 1, - phase: "preparing", - ownerPid: process.pid, - ownerStartIdentity: - readProcessStartIdentity(process.pid) ?? - (() => { - throw new Error("Cannot identify shields-down owner process"); - })(), - ownerMcpProcessIdentity: - readMcpLockProcessIdentity(process.pid, true) ?? - (() => { - throw new Error("Cannot identify shields-down lifecycle owner process"); - })(), - processToken, - sandboxName, - snapshotPath, - managedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, - }; - const leaseOwnerPid = opts.deferAutoRestoreWhileOwnerAlive ? transition.ownerPid : null; - const leaseOwnerStartIdentity = opts.deferAutoRestoreWhileOwnerAlive - ? transition.ownerStartIdentity - : null; - let timerChild: ReturnType | null = null; + // Commit the host-side recovery authority before weakening policy or file + // permissions. If this process is killed later, the detached timer and its + // marker already exist and the persisted state honestly reports shields + // down. A crash can therefore never leave an untracked mutable window. + if (!opts.skipTimer) { + const restoreAt = new Date(Date.now() + timeoutSeconds * 1000); + const processToken = opts.processToken ?? randomBytes(16).toString("hex"); + if (!/^[0-9a-f]{32}$/.test(processToken)) { + throw new Error("Invalid shields-down recovery process token"); + } + const timerScript = path.join(__dirname, "timer.ts"); + const timerScriptJs = timerScript.replace(/\.ts$/, ".js"); + const actualScript = fs.existsSync(timerScriptJs) ? timerScriptJs : timerScript; + transition = { + version: 1, + phase: "preparing", + ownerPid: process.pid, + ownerStartIdentity: + readProcessStartIdentity(process.pid) ?? + (() => { + throw new Error("Cannot identify shields-down owner process"); + })(), + processToken, + sandboxName, + snapshotPath, + }; + const leaseOwnerPid = opts.deferAutoRestoreWhileOwnerAlive ? transition.ownerPid : null; + const leaseOwnerStartIdentity = opts.deferAutoRestoreWhileOwnerAlive + ? transition.ownerStartIdentity + : null; + let timerChild: ReturnType | null = null; - try { - // Publish the forward-transition ownership marker before authorizing the - // timer. If the timeout expires while this command is still weakening - // policy/config, the timer waits for phase=active or owner death instead - // of racing the forward mutations. - writeShieldsDownTransition(transition, null); - timerChild = fork( - actualScript, - [ - sandboxName, - snapshotPath, - restoreAt.toISOString(), - target.configPath, - target.configDir, - processToken, - opts.allowLegacyHermesProtocol === true ? "1" : "0", - leaseOwnerPid === null ? "" : String(leaseOwnerPid), - leaseOwnerStartIdentity ?? "", - target.agentName ?? "", - ], - { - detached: true, - stdio: ["ignore", "ignore", "ignore", "ipc"], - }, - ); - if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); - writeTimerMarkerAtomic(sandboxName, { - pid: timerChild.pid, + try { + // Publish the forward-transition ownership marker before authorizing the + // timer. If the timeout expires while this command is still weakening + // policy/config, the timer waits for phase=active or owner death instead + // of racing the forward mutations. + writeShieldsDownTransition(transition, null); + timerChild = fork( + actualScript, + [ sandboxName, snapshotPath, - restoreAt: restoreAt.toISOString(), + restoreAt.toISOString(), + target.configPath, + target.configDir, processToken, - allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, - agentName: target.agentName, - configPath: target.configPath, - configDir: target.configDir, - ...(leaseOwnerPid !== null && leaseOwnerStartIdentity - ? { leaseOwnerPid, leaseOwnerStartIdentity } - : {}), - }); - if (!timerChild.send({ type: "authorize", processToken })) { - throw new Error("auto-restore timer authorization channel closed early"); - } - timerChild.disconnect(); - timerChild.unref(); - } catch (err) { - try { - timerChild?.kill("SIGTERM"); - } catch { - // Best effort; without a matching marker the child has no authority. - } - clearTimerMarker(sandboxName); - clearShieldsDownTransition(sandboxName, processToken); - const message = err instanceof Error ? err.message : String(err); - console.error(` Cannot start auto-restore timer: ${message}`); - return failShieldsCommand(`Cannot start auto-restore timer: ${message}`, opts.throwOnError); + opts.allowLegacyHermesProtocol === true ? "1" : "0", + leaseOwnerPid === null ? "" : String(leaseOwnerPid), + leaseOwnerStartIdentity ?? "", + target.agentName ?? "", + ], + { + detached: true, + stdio: ["ignore", "ignore", "ignore", "ipc"], + }, + ); + if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); + writeTimerMarkerAtomic(sandboxName, { + pid: timerChild.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAt.toISOString(), + processToken, + allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, + agentName: target.agentName, + configPath: target.configPath, + configDir: target.configDir, + ...(leaseOwnerPid !== null && leaseOwnerStartIdentity + ? { leaseOwnerPid, leaseOwnerStartIdentity } + : {}), + }); + if (!timerChild.send({ type: "authorize", processToken })) { + throw new Error("auto-restore timer authorization channel closed early"); } + timerChild.disconnect(); + timerChild.unref(); + } catch (err) { + clearTimerMarker(sandboxName); + clearShieldsDownTransition(sandboxName, processToken); + cleanupRuntimePolicyFile(); + const message = err instanceof Error ? err.message : String(err); + console.error(` Cannot start auto-restore timer: ${message}`); + return failShieldsCommand(`Cannot start auto-restore timer: ${message}`, opts.throwOnError); } + } - try { - saveShieldsState(sandboxName, { - shieldsDown: true, - shieldsDownAt: now, - shieldsDownTimeout: timeoutSeconds, - shieldsDownReason: reason, - shieldsDownPolicy: policyName, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, - }); - } catch (error) { - if (transition) { - clearShieldsDownTransition(sandboxName, transition.processToken); - killTimer(sandboxName); - } - throw error; + try { + saveShieldsState(sandboxName, { + shieldsDown: true, + shieldsDownAt: now, + shieldsDownTimeout: timeoutSeconds, + shieldsDownReason: reason, + shieldsDownPolicy: policyName, + shieldsPolicySnapshotPath: snapshotPath, + }); + } catch (error) { + if (transition) { + clearShieldsDownTransition(sandboxName, transition.processToken); + killTimer(sandboxName); } + cleanupRuntimePolicyFile(); + throw error; + } - console.log(` Applying ${policyName} policy...`); + console.log(` Applying ${policyName} policy...`); + try { run(buildPolicySetCommand(policyFile, sandboxName)); } finally { cleanupRuntimePolicyFile(); @@ -4215,13 +3901,11 @@ function clearShieldsState(sandboxName: string): void { // --------------------------------------------------------------------------- export { - applyShieldsPolicySnapshot, clearShieldsState, completeAutoRestoreTransition, DEFAULT_TIMEOUT_SECONDS, deriveShieldsMode, getShieldsPosture, - inspectAutoRestoreTransitionTakeoverOwner, inspectMutableConfigPerms, isShieldsDown, killTimer, diff --git a/src/lib/shields/mcp-policy-transition.test.ts b/src/lib/shields/mcp-policy-transition.test.ts deleted file mode 100644 index f754e7e2452..00000000000 --- a/src/lib/shields/mcp-policy-transition.test.ts +++ /dev/null @@ -1,685 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; -import YAML from "yaml"; - -import { - hasManagedMcpPolicyClaims, - inspectProvableManagedMcpPoliciesForDeadline, - inspectExactManagedMcpPolicies as inspectRegisteredManagedMcpPolicies, - MCP_BRIDGE_POLICY_SOURCE, -} from "../actions/sandbox/mcp-bridge-policy"; -import { - buildMcpBridgePolicyKey, - buildMcpBridgePolicyName, - buildMcpBridgePolicyYaml, -} from "../actions/sandbox/mcp-bridge-policy-render"; -import type { SandboxEntry } from "../state/registry"; -import { - assertLegacyMcpPolicyRestoreSafe, - composeDeadlineManagedMcpPolicies, - composeManagedMcpPolicies, -} from "./mcp-policy-transition"; - -const ADAPTER = "hermes-config"; - -function registeredPolicy( - server: string, - address: string, -): NonNullable[number] { - return { - name: buildMcpBridgePolicyName(server), - content: buildMcpBridgePolicyYaml(server, `https://${server}.example.com/mcp`, ADAPTER, [ - address, - ]), - sourcePath: MCP_BRIDGE_POLICY_SOURCE, - }; -} - -function bridge(server: string): NonNullable["bridges"]>[string] { - return { - server, - agent: "hermes", - adapter: ADAPTER, - url: `https://${server}.example.com/mcp`, - env: ["MCP_SECRET"], - providerName: `sandbox-mcp-${server}`, - providerId: `provider-${server}`, - policyName: buildMcpBridgePolicyName(server), - addedAt: "2026-07-30T00:00:00.000Z", - }; -} - -function sandboxWithPolicies( - policies: Array>, - bridgeServers = policies.map((policy) => policy.name.replace(/^mcp-bridge-/, "")), -): SandboxEntry { - return { - name: "alpha", - agent: "hermes", - customPolicies: policies, - mcp: { - bridges: Object.fromEntries(bridgeServers.map((server) => [server, bridge(server)])), - }, - }; -} - -function networkEntry(content: string, server: string): unknown { - return YAML.parse(content).network_policies[buildMcpBridgePolicyKey(server)]; -} - -function mutateRegisteredNetworkPolicy( - policy: ReturnType, - server: string, - mutate: (entry: Record) => void, -): void { - const document = YAML.parse(policy.content) as { - network_policies: Record>; - }; - mutate(document.network_policies[buildMcpBridgePolicyKey(server)]!); - policy.content = YAML.stringify(document); -} - -function livePolicy( - entries: Array<{ content: string; server: string }>, - extra: Record = {}, -): string { - return YAML.stringify({ - version: 1, - network_policies: { - ...extra, - ...Object.fromEntries( - entries.map(({ content, server }) => [ - buildMcpBridgePolicyKey(server), - networkEntry(content, server), - ]), - ), - }, - }); -} - -function inspectExactManagedMcpPolicies(sandbox: SandboxEntry, livePolicyYaml: string) { - return inspectRegisteredManagedMcpPolicies("alpha", livePolicyYaml, { - getSandbox: () => sandbox, - }); -} - -describe("managed MCP Shields policy transitions (#7952)", () => { - it("admits only canonical committed registrations that exactly match the live policy", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const sandbox = sandboxWithPolicies([alpha]); - - const exact = inspectExactManagedMcpPolicies( - sandbox, - livePolicy([{ content: alpha.content, server: "alpha" }], { - unrelated_live_entry: { endpoints: [{ host: "unrelated.example.com" }] }, - }), - ); - - expect(exact).toEqual([ - expect.objectContaining({ - key: "mcp_bridge_alpha", - policyName: "mcp-bridge-alpha", - server: "alpha", - }), - ]); - }); - - it.each([ - { - label: "pending policy content", - mutate: (sandbox: SandboxEntry) => { - sandbox.customPolicies![0]!.pendingContent = sandbox.customPolicies![0]!.content; - }, - expected: /incomplete policy transition/, - }, - { - label: "an orphaned generated registration", - mutate: (sandbox: SandboxEntry) => { - sandbox.customPolicies!.push(registeredPolicy("orphan", "1.1.1.1")); - }, - expected: /no committed managed bridge ownership/, - }, - { - label: "an incomplete bridge add", - mutate: (sandbox: SandboxEntry) => { - sandbox.mcp!.bridges.alpha!.addState = "prepared"; - }, - expected: /lifecycle transition is incomplete/, - }, - ])("fails closed on $label", ({ mutate, expected }) => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const sandbox = sandboxWithPolicies([alpha]); - mutate(sandbox); - - expect(() => - inspectExactManagedMcpPolicies( - sandbox, - livePolicy( - (sandbox.customPolicies ?? []).map((policy) => ({ - content: policy.content, - server: policy.name.replace(/^mcp-bridge-/, ""), - })), - ), - ), - ).toThrow(expected); - }); - - it("fails closed when the live policy differs from the ownership record", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const drifted = registeredPolicy("alpha", "1.1.1.1"); - - expect(() => - inspectExactManagedMcpPolicies( - sandboxWithPolicies([alpha]), - livePolicy([{ content: drifted.content, server: "alpha" }]), - ), - ).toThrow(/drifted from its ownership record/); - }); - - it("rejects matching registry and live documents with weakened generated semantics", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - mutateRegisteredNetworkPolicy(alpha, "alpha", (entry) => { - const endpoint = (entry.endpoints as Array>)[0]!; - endpoint.enforcement = "observe"; - }); - const sandbox = sandboxWithPolicies([alpha]); - const live = livePolicy([{ content: alpha.content, server: "alpha" }]); - - expect(() => inspectExactManagedMcpPolicies(sandbox, live)).toThrow( - /non-canonical generated content/, - ); - expect( - inspectProvableManagedMcpPoliciesForDeadline("alpha", live, { - getSandbox: () => sandbox, - }), - ).toEqual({ - policies: [], - omissions: [ - expect.objectContaining({ - server: "alpha", - reason: expect.stringMatching(/non-canonical generated content/), - }), - ], - }); - }); - - it.each([ - { - label: "a private literal", - pins: ["127.0.0.1"], - expected: /invalid public address pins/, - }, - { - label: "a scoped public IPv6 literal", - pins: ["2001:4860:4860::8888%lo0"], - expected: /invalid public address pins/, - }, - { - label: "duplicate literals", - pins: ["8.8.8.8", "8.8.8.8"], - expected: /non-canonical public address pins/, - }, - { - label: "unsorted literals", - pins: ["8.8.8.8", "1.1.1.1"], - expected: /non-canonical public address pins/, - }, - ])("rejects matching registry and live documents with $label", ({ pins, expected }) => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - mutateRegisteredNetworkPolicy(alpha, "alpha", (entry) => { - const endpoint = (entry.endpoints as Array>)[0]!; - endpoint.allowed_ips = pins; - }); - - expect(() => - inspectExactManagedMcpPolicies( - sandboxWithPolicies([alpha]), - livePolicy([{ content: alpha.content, server: "alpha" }]), - ), - ).toThrow(expected); - }); - - it("fails closed on a generated policy record without managed MCP state", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const sandbox: SandboxEntry = { - name: "alpha", - agent: "hermes", - customPolicies: [alpha], - }; - const deps = { getSandbox: () => sandbox }; - - expect(hasManagedMcpPolicyClaims("alpha", deps)).toBe(true); - expect(() => - inspectRegisteredManagedMcpPolicies( - "alpha", - livePolicy([{ content: alpha.content, server: "alpha" }]), - deps, - ), - ).toThrow(/no committed managed bridge ownership/); - }); - - it("treats residual managed server history as an ownership claim", () => { - const sandbox: SandboxEntry = { - name: "alpha", - agent: "hermes", - mcp: { bridges: {}, managedServerNames: ["retired"] }, - }; - const deps = { getSandbox: () => sandbox }; - - expect(hasManagedMcpPolicyClaims("alpha", deps)).toBe(true); - expect( - inspectRegisteredManagedMcpPolicies( - "alpha", - livePolicy([], { unrelated_live_entry: {} }), - deps, - ), - ).toEqual([]); - }); - - it.each([ - { - label: "no sandbox registry entry", - sandbox: undefined, - }, - { - label: "only residual ownership history", - sandbox: { - name: "alpha", - agent: "hermes", - mcp: { bridges: {}, managedServerNames: ["retired"] }, - } satisfies SandboxEntry, - }, - ])("rejects an unclassified reserved live key with $label", ({ sandbox }) => { - expect(() => - inspectRegisteredManagedMcpPolicies("alpha", livePolicy([], { mcp_bridge_retired: {} }), { - getSandbox: () => sandbox ?? null, - }), - ).toThrow( - /Reserved MCP policy key "mcp_bridge_retired".*no committed managed bridge ownership/, - ); - }); - - it("escapes unclassified live policy keys in operator diagnostics", () => { - const maliciousKey = "mcp_bridge_\u001b[31mforged\nline\u0085"; - - let failure: unknown; - try { - inspectRegisteredManagedMcpPolicies("alpha", livePolicy([], { [maliciousKey]: {} }), { - getSandbox: () => null, - }); - } catch (error) { - failure = error; - } - - expect(failure).toBeInstanceOf(Error); - expect((failure as Error).message).toContain( - String.raw`"mcp_bridge_\u001b[31mforged\u000aline\u0085"`, - ); - expect((failure as Error).message).not.toMatch(/[\u0000-\u001f\u007f-\u009f]/u); - }); - - it("retains additions while restoring the restrictive snapshot", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const beta = registeredPolicy("beta", "1.1.1.1"); - const current = inspectExactManagedMcpPolicies( - sandboxWithPolicies([alpha, beta]), - livePolicy([ - { content: alpha.content, server: "alpha" }, - { content: beta.content, server: "beta" }, - ]), - ); - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), - }, - }); - - const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"])); - - expect(Object.keys(restored.network_policies).sort()).toEqual([ - "mcp_bridge_alpha", - "mcp_bridge_beta", - "restrictive_baseline", - ]); - }); - - it("does not restore a managed MCP policy removed during the shields-down window", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), - }, - }); - - const restored = YAML.parse(composeManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"])); - - expect(restored.network_policies).toEqual({ - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - }); - }); - - it("replaces a stale snapshot entry with the current exact registration", () => { - const oldAlpha = registeredPolicy("alpha", "8.8.8.8"); - const currentAlpha = registeredPolicy("alpha", "1.1.1.1"); - const current = inspectExactManagedMcpPolicies( - sandboxWithPolicies([currentAlpha]), - livePolicy([{ content: currentAlpha.content, server: "alpha" }]), - ); - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - mcp_bridge_alpha: networkEntry(oldAlpha.content, "alpha"), - }, - }); - - const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"])); - - expect(restored.network_policies.mcp_bridge_alpha).toEqual( - networkEntry(currentAlpha.content, "alpha"), - ); - }); - - it("rejects an unclassified reserved key in the restrictive snapshot", () => { - const currentAlpha = registeredPolicy("alpha", "1.1.1.1"); - const current = inspectExactManagedMcpPolicies( - sandboxWithPolicies([currentAlpha]), - livePolicy([{ content: currentAlpha.content, server: "alpha" }]), - ); - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - mcp_bridge_alpha: { - name: "operator-owned-alpha", - endpoints: [{ host: "operator.example.com" }], - }, - }, - }); - - expect(() => composeManagedMcpPolicies(snapshot, current, [])).toThrow( - /Reserved MCP policy key 'mcp_bridge_alpha'.*absent from the saved ownership manifest/, - ); - }); - - it("accepts an empty ownership manifest when the snapshot has no reserved keys", () => { - const snapshot = YAML.stringify({ - version: 1, - network_policies: { restrictive_baseline: {} }, - }); - - expect(YAML.parse(composeManagedMcpPolicies(snapshot, [], [])).network_policies).toEqual({ - restrictive_baseline: {}, - }); - }); - - it("rejects a saved managed key that is absent from its policy snapshot", () => { - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - }, - }); - - expect(() => composeManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"])).toThrow( - /absent from its policy snapshot/, - ); - }); - - it.each([ - { - label: "current managed MCP ownership", - hasCurrentManagedClaims: true, - networkPolicies: { restrictive_baseline: {} }, - }, - { - label: "a managed-shaped key in the snapshot", - hasCurrentManagedClaims: false, - networkPolicies: { mcp_bridge_alpha: {} }, - }, - ])("refuses legacy restore with $label", ({ hasCurrentManagedClaims, networkPolicies }) => { - expect(() => - assertLegacyMcpPolicyRestoreSafe( - YAML.stringify({ version: 1, network_policies: networkPolicies }), - hasCurrentManagedClaims, - ), - ).toThrow(/no managed MCP ownership manifest/); - }); - - it("allows a legacy restore with no current or snapshot MCP ownership", () => { - expect(() => - assertLegacyMcpPolicyRestoreSafe( - YAML.stringify({ - version: 1, - network_policies: { restrictive_baseline: {} }, - }), - false, - ), - ).not.toThrow(); - }); - - it("proves committed bridges independently while omitting an incomplete add at the deadline", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const beta = registeredPolicy("beta", "1.1.1.1"); - const sandbox = sandboxWithPolicies([alpha, beta]); - sandbox.mcp!.bridges.beta!.addState = "prepared"; - - const result = inspectProvableManagedMcpPoliciesForDeadline( - "alpha", - livePolicy([ - { content: alpha.content, server: "alpha" }, - { content: beta.content, server: "beta" }, - ]), - { getSandbox: () => sandbox }, - ); - - expect(result.policies.map((policy) => policy.server)).toEqual(["alpha"]); - expect(result.omissions).toEqual([ - expect.objectContaining({ server: "beta", reason: expect.stringMatching(/incomplete/) }), - ]); - }); - - it("omits every deadline claimant whose canonical policy identity collides", () => { - const collidingPolicy = registeredPolicy("foo-bar", "8.8.8.8"); - const sandbox = sandboxWithPolicies([collidingPolicy], ["foo-bar", "foo_bar"]); - - const result = inspectProvableManagedMcpPoliciesForDeadline( - "alpha", - livePolicy([{ content: collidingPolicy.content, server: "foo-bar" }]), - { getSandbox: () => sandbox }, - ); - - expect(result.policies).toEqual([]); - expect(result.omissions).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - server: "foo-bar", - reason: expect.stringMatching(/ambiguous bridge ownership/), - }), - expect.objectContaining({ - server: "foo_bar", - reason: expect.stringMatching(/ambiguous bridge ownership/), - }), - ]), - ); - }); - - it.each([ - "destroyPreparedAt", - "destroyPendingAt", - ] as const)("omits every generated policy while %s is present", (marker) => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const sandbox = sandboxWithPolicies([alpha]); - sandbox.mcp![marker] = "2026-07-30T01:00:00.000Z"; - - const result = inspectProvableManagedMcpPoliciesForDeadline( - "alpha", - livePolicy([{ content: alpha.content, server: "alpha" }]), - { getSandbox: () => sandbox }, - ); - - expect(result.policies).toEqual([]); - expect(result.omissions).toEqual([ - expect.objectContaining({ server: "alpha", reason: expect.stringMatching(/destruction/) }), - ]); - }); - - it("omits drift and orphan claims without discarding another exact bridge", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const beta = registeredPolicy("beta", "1.1.1.1"); - const driftedBeta = registeredPolicy("beta", "9.9.9.9"); - const orphan = registeredPolicy("orphan", "4.4.4.4"); - const sandbox = sandboxWithPolicies([alpha, beta, orphan], ["alpha", "beta"]); - - const result = inspectProvableManagedMcpPoliciesForDeadline( - "alpha", - livePolicy([ - { content: alpha.content, server: "alpha" }, - { content: driftedBeta.content, server: "beta" }, - { content: orphan.content, server: "orphan" }, - ]), - { getSandbox: () => sandbox }, - ); - - expect(result.policies.map((policy) => policy.server)).toEqual(["alpha"]); - expect(result.omissions).toEqual( - expect.arrayContaining([ - expect.objectContaining({ server: "beta", reason: expect.stringMatching(/drifted/) }), - expect.objectContaining({ - policyName: "mcp-bridge-orphan", - reason: expect.stringMatching(/no committed managed bridge ownership/), - }), - ]), - ); - }); - - it("deadline inspection reports an unclassified reserved live key", () => { - const result = inspectProvableManagedMcpPoliciesForDeadline( - "alpha", - livePolicy([], { mcp_bridge_residual: {} }), - { getSandbox: () => null }, - ); - - expect(result).toEqual({ - policies: [], - omissions: [ - expect.objectContaining({ - key: "mcp_bridge_residual", - reason: expect.stringMatching(/no committed managed bridge ownership/), - }), - ], - }); - }); - - it("deadline composition strips unclassified reserved keys before overlaying proven entries", () => { - const alpha = registeredPolicy("alpha", "8.8.8.8"); - const beta = registeredPolicy("beta", "1.1.1.1"); - const current = inspectExactManagedMcpPolicies( - sandboxWithPolicies([alpha, beta]), - livePolicy([ - { content: alpha.content, server: "alpha" }, - { content: beta.content, server: "beta" }, - ]), - ); - const operatorEntry = { endpoints: [{ host: "operator.example.com" }] }; - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), - mcp_bridge_beta: operatorEntry, - restrictive_baseline: {}, - }, - }); - - const result = composeDeadlineManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"]); - const restored = YAML.parse(result.yaml); - - expect(restored.network_policies.mcp_bridge_alpha).toEqual( - networkEntry(alpha.content, "alpha"), - ); - expect(restored.network_policies.mcp_bridge_beta).toEqual(networkEntry(beta.content, "beta")); - expect(result.omissions).toEqual([ - expect.objectContaining({ - key: "mcp_bridge_beta", - reason: expect.stringMatching(/absent from the saved ownership manifest/), - }), - ]); - }); - - it("deadline composition strips every reserved shape with an empty manifest", () => { - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - mcp_bridge_: {}, - mcp_bridge_legacy_invalid_name: {}, - restrictive_baseline: {}, - }, - }); - - const result = composeDeadlineManagedMcpPolicies(snapshot, [], []); - - expect(YAML.parse(result.yaml).network_policies).toEqual({ restrictive_baseline: {} }); - expect(result.omissions.map((entry) => entry.key)).toEqual([ - "mcp_bridge_", - "mcp_bridge_legacy_invalid_name", - ]); - }); - - it("deadline composition omits malformed and duplicate manifest entries without delaying lockdown", () => { - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - mcp_bridge_: {}, - mcp_bridge_alpha: {}, - restrictive_baseline: {}, - }, - }); - - const result = composeDeadlineManagedMcpPolicies( - snapshot, - [], - ["mcp_bridge_", "restrictive_baseline", "mcp_bridge_alpha", "mcp_bridge_alpha"], - ); - - expect(YAML.parse(result.yaml).network_policies).toEqual({ restrictive_baseline: {} }); - expect(result.omissions).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - key: "mcp_bridge_", - reason: expect.stringMatching(/ownership key.*invalid/), - }), - expect.objectContaining({ - key: "restrictive_baseline", - reason: expect.stringMatching(/ownership key.*invalid/), - }), - expect.objectContaining({ - key: "mcp_bridge_alpha", - reason: expect.stringMatching(/more than once/), - }), - ]), - ); - }); - - it("deadline composition restores the restrictive baseline when a saved key is absent", () => { - const snapshot = YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - }, - }); - - const result = composeDeadlineManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"]); - const restored = YAML.parse(result.yaml); - - expect(restored.network_policies).toEqual({ - restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - }); - expect(result.omissions).toEqual([ - expect.objectContaining({ reason: expect.stringMatching(/already absent/) }), - ]); - }); -}); diff --git a/src/lib/shields/mcp-policy-transition.ts b/src/lib/shields/mcp-policy-transition.ts deleted file mode 100644 index 858536bb790..00000000000 --- a/src/lib/shields/mcp-policy-transition.ts +++ /dev/null @@ -1,182 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import YAML from "yaml"; - -import type { - ExactManagedMcpPolicy, - ManagedMcpPolicyOmission, -} from "../actions/sandbox/mcp-bridge-policy"; - -const CANONICAL_MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_[a-z][a-z0-9_]{0,63}$/; -const RESERVED_MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_/; - -function parsePolicyDocument(source: string, label: string): Record { - let parsed: unknown; - try { - parsed = YAML.parse(source); - } catch { - throw new Error(`${label} is not valid YAML`); - } - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - throw new Error(`${label} must be a YAML mapping`); - } - return parsed as Record; -} - -function readNetworkPolicies( - document: Record, - label: string, -): Record { - const policies = document.network_policies; - if (policies === undefined || policies === null) return {}; - if (typeof policies !== "object" || Array.isArray(policies)) { - throw new Error(`${label} network_policies must be a mapping`); - } - return policies as Record; -} - -/** - * Reconcile generated MCP entries into a complete target policy. - * - * Snapshot-time keys are removed first so an MCP server deleted during the - * shields-down window cannot be restored. The current exact entries are then overlaid, - * retaining additions and replacing stale pins. Every non-MCP target entry - * remains authoritative; unrelated live entries are never copied. - */ -export function composeManagedMcpPolicies( - targetPolicyYaml: string, - currentPolicies: readonly ExactManagedMcpPolicy[], - snapshotManagedPolicyKeys: readonly string[] = [], -): string { - const target = parsePolicyDocument(targetPolicyYaml, "Target Shields policy"); - const targetPolicies = readNetworkPolicies(target, "Target Shields policy"); - - const snapshotKeys = new Set(); - for (const key of snapshotManagedPolicyKeys) { - if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(key) || snapshotKeys.has(key)) { - throw new Error("Saved Shields MCP policy ownership is invalid"); - } - if (!Object.hasOwn(targetPolicies, key)) { - throw new Error(`Saved Shields MCP policy '${key}' is absent from its policy snapshot`); - } - snapshotKeys.add(key); - delete targetPolicies[key]; - } - const unclassifiedKey = Object.keys(targetPolicies).find((key) => - RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key), - ); - if (unclassifiedKey) { - throw new Error( - `Reserved MCP policy key '${unclassifiedKey}' is absent from the saved ownership manifest`, - ); - } - - const currentKeys = new Set(); - for (const policy of currentPolicies) { - if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { - throw new Error(`Managed MCP policy key '${policy.key}' has ambiguous ownership`); - } - currentKeys.add(policy.key); - targetPolicies[policy.key] = policy.networkPolicy; - } - - target.network_policies = targetPolicies; - return YAML.stringify(target); -} - -export interface DeadlineManagedMcpPolicyComposition { - yaml: string; - omissions: ManagedMcpPolicyOmission[]; -} - -/** - * Security-authoritative deadline composition. - * - * Every reserved key is removed from the snapshot, including keys missing from - * an incomplete manifest. Only independently proven current entries are then - * overlaid. - */ -export function composeDeadlineManagedMcpPolicies( - targetPolicyYaml: string, - currentPolicies: readonly ExactManagedMcpPolicy[], - snapshotManagedPolicyKeys: readonly string[], -): DeadlineManagedMcpPolicyComposition { - const target = parsePolicyDocument(targetPolicyYaml, "Target Shields policy"); - const targetPolicies = readNetworkPolicies(target, "Target Shields policy"); - - const snapshotKeys = new Set(); - const omissions: ManagedMcpPolicyOmission[] = []; - for (const key of snapshotManagedPolicyKeys) { - if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(key)) { - if (RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key)) { - delete targetPolicies[key]; - } - omissions.push({ - key, - reason: `Saved Shields MCP policy ownership key '${key}' is invalid`, - }); - continue; - } - if (snapshotKeys.has(key)) { - omissions.push({ - key, - reason: `Saved Shields MCP policy '${key}' appeared more than once in its ownership manifest`, - }); - continue; - } - if (!Object.hasOwn(targetPolicies, key)) { - omissions.push({ - reason: `Saved Shields MCP policy '${key}' was already absent from its policy snapshot`, - }); - } - snapshotKeys.add(key); - delete targetPolicies[key]; - } - for (const key of Object.keys(targetPolicies)) { - if (!RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key)) continue; - delete targetPolicies[key]; - omissions.push({ - key, - reason: `Reserved MCP policy key '${key}' was absent from the saved ownership manifest`, - }); - } - - const currentKeys = new Set(); - for (const policy of currentPolicies) { - if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { - throw new Error(`Managed MCP policy key '${policy.key}' has ambiguous ownership`); - } - currentKeys.add(policy.key); - targetPolicies[policy.key] = policy.networkPolicy; - } - - target.network_policies = targetPolicies; - return { yaml: YAML.stringify(target), omissions }; -} - -export function isManagedMcpPolicyKey(value: unknown): value is string { - return typeof value === "string" && RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(value); -} - -/** - * Refuse to guess managed ownership for a Shields snapshot captured before the - * ownership manifest existed. Current claims prove reconciliation is needed; - * a managed-shaped snapshot key may be a removed bridge or an operator entry. - * Either case requires explicit recovery instead of a destructive raw apply. - */ -export function assertLegacyMcpPolicyRestoreSafe( - snapshotPolicyYaml: string, - hasCurrentManagedClaims: boolean, -): void { - const snapshot = parsePolicyDocument(snapshotPolicyYaml, "Legacy Shields policy snapshot"); - const snapshotPolicies = readNetworkPolicies(snapshot, "Legacy Shields policy snapshot"); - if ( - hasCurrentManagedClaims || - Object.keys(snapshotPolicies).some((key) => isManagedMcpPolicyKey(key)) - ) { - throw new Error( - "Legacy Shields state has no managed MCP ownership manifest; refusing policy restore", - ); - } -} diff --git a/src/lib/shields/permissive-runtime.ts b/src/lib/shields/permissive-runtime.ts index 897ed4d52a9..46f523f60be 100644 --- a/src/lib/shields/permissive-runtime.ts +++ b/src/lib/shields/permissive-runtime.ts @@ -4,30 +4,8 @@ import fs from "node:fs"; import YAML from "yaml"; -export { - type ExactManagedMcpPolicy, - hasManagedMcpPolicyClaims, - inspectExactManagedMcpPolicies, - inspectProvableManagedMcpPoliciesForDeadline, - type ManagedMcpPolicyOmission, -} from "../actions/sandbox/mcp-bridge-policy"; - -import type { - ExactManagedMcpPolicy, - ManagedMcpPolicyOmission, -} from "../actions/sandbox/mcp-bridge-policy"; import { cleanupTempDir, secureTempFile } from "../onboard/temp-files"; -export { - assertLegacyMcpPolicyRestoreSafe, - isManagedMcpPolicyKey, -} from "./mcp-policy-transition"; - -import { - composeDeadlineManagedMcpPolicies, - composeManagedMcpPolicies, -} from "./mcp-policy-transition"; - const TEMP_FILE_PREFIX = "nemoclaw-permissive-runtime"; /** @@ -82,10 +60,6 @@ export interface PermissiveRuntimeDeps { // secureTempFile when omitted. Exposed so tests can drive the // write-failure fallback path without monkey-patching node:fs. writeTempPolicy?: (yaml: string) => string; - // Exact, live-matching generated MCP policies resolved by the Shields - // coordinator. These entries remain active while the static policy replaces - // the rest of the complete gateway policy. - managedMcpPolicies?: readonly ExactManagedMcpPolicy[]; } export function buildRuntimePermissivePolicy( @@ -95,31 +69,21 @@ export function buildRuntimePermissivePolicy( const live = deps.livePolicyYaml ? safeYamlObject(deps.livePolicyYaml) : null; const liveRw = readStringList(live, "read_write"); const liveRo = readStringList(live, "read_only"); - const managedMcpPolicies = deps.managedMcpPolicies ?? []; // No live filesystem section to merge — keep the static path so the - // caller's apply path is unchanged unless exact managed MCP entries must - // survive the complete-policy replacement. - if (liveRw.length === 0 && liveRo.length === 0 && managedMcpPolicies.length === 0) { + // caller's apply path is unchanged. + if (liveRw.length === 0 && liveRo.length === 0) { return basePermissivePath; } let baseYaml: string; try { baseYaml = deps.readBasePolicy(); - } catch (error) { - if (managedMcpPolicies.length > 0) { - throw new Error("Cannot read the Shields-down policy while managed MCP policies are active", { - cause: error, - }); - } + } catch { return basePermissivePath; } const base = safeYamlObject(baseYaml); if (!base) { - if (managedMcpPolicies.length > 0) { - throw new Error("Cannot parse the Shields-down policy while managed MCP policies are active"); - } return basePermissivePath; } const fsPolicy = @@ -144,17 +108,11 @@ export function buildRuntimePermissivePolicy( fsPolicy.read_write = [...baseRw]; fsPolicy.read_only = [...baseRo]; - const yaml = composeManagedMcpPolicies(YAML.stringify(base), managedMcpPolicies); + const yaml = YAML.stringify(base); if (deps.writeTempPolicy) { try { return deps.writeTempPolicy(yaml); - } catch (error) { - if (managedMcpPolicies.length > 0) { - throw new Error( - "Cannot stage the Shields-down policy while managed MCP policies are active", - { cause: error }, - ); - } + } catch { return basePermissivePath; } } @@ -163,111 +121,15 @@ export function buildRuntimePermissivePolicy( tmpPath = secureTempFile(TEMP_FILE_PREFIX, ".yaml"); fs.writeFileSync(tmpPath, yaml, { mode: 0o600 }); return tmpPath; - } catch (error) { + } catch { // secureTempFile may have created an mkdtemp directory before // writeFileSync failed. Clean it up so we do not leak a 0700 dir // on /tmp every time the write path errors. if (tmpPath) cleanupTempDir(tmpPath, TEMP_FILE_PREFIX); - if (managedMcpPolicies.length > 0) { - throw new Error( - "Cannot stage the Shields-down policy while managed MCP policies are active", - { cause: error }, - ); - } return basePermissivePath; } } -export interface ManagedMcpRuntimePolicyDeps { - managedMcpPolicies: readonly ExactManagedMcpPolicy[]; - readBasePolicy: () => string; - snapshotManagedPolicyKeys?: readonly string[]; - writeTempPolicy?: (yaml: string) => string; -} - -/** - * Reconcile current generated MCP policies into a custom Shields-down policy - * or a saved restrictive snapshot. Unlike the legacy filesystem-only fallback, - * this path must fail closed: returning the unmodified base could silently - * discard a managed entry or restore one that was removed during the - * shields-down window. - */ -export function buildRuntimeManagedMcpPolicy( - _basePolicyPath: string, - deps: ManagedMcpRuntimePolicyDeps, -): string { - const snapshotManagedPolicyKeys = deps.snapshotManagedPolicyKeys ?? []; - - let baseYaml: string; - try { - baseYaml = deps.readBasePolicy(); - } catch (error) { - throw new Error("Cannot read the Shields policy for managed MCP reconciliation", { - cause: error, - }); - } - const yaml = composeManagedMcpPolicies( - baseYaml, - deps.managedMcpPolicies, - snapshotManagedPolicyKeys, - ); - if (deps.writeTempPolicy) { - try { - return deps.writeTempPolicy(yaml); - } catch (error) { - throw new Error("Cannot stage the Shields policy for managed MCP reconciliation", { - cause: error, - }); - } - } - - let tmpPath: string | null = null; - try { - tmpPath = secureTempFile(TEMP_FILE_PREFIX, ".yaml"); - fs.writeFileSync(tmpPath, yaml, { mode: 0o600 }); - return tmpPath; - } catch (error) { - if (tmpPath) cleanupTempDir(tmpPath, TEMP_FILE_PREFIX); - throw new Error("Cannot stage the Shields policy for managed MCP reconciliation", { - cause: error, - }); - } -} - -export interface DeadlineManagedMcpRuntimePolicy { - path: string; - omissions: ManagedMcpPolicyOmission[]; -} - -export function buildDeadlineRuntimeManagedMcpPolicy( - _basePolicyPath: string, - deps: ManagedMcpRuntimePolicyDeps, -): DeadlineManagedMcpRuntimePolicy { - const baseYaml = deps.readBasePolicy(); - const composition = composeDeadlineManagedMcpPolicies( - baseYaml, - deps.managedMcpPolicies, - deps.snapshotManagedPolicyKeys ?? [], - ); - let runtimePath: string | null = null; - try { - runtimePath = deps.writeTempPolicy - ? deps.writeTempPolicy(composition.yaml) - : secureTempFile(TEMP_FILE_PREFIX, ".yaml"); - if (!deps.writeTempPolicy) { - fs.writeFileSync(runtimePath, composition.yaml, { mode: 0o600 }); - } - return { path: runtimePath, omissions: composition.omissions }; - } catch (error) { - if (runtimePath && !deps.writeTempPolicy) { - cleanupTempDir(runtimePath, TEMP_FILE_PREFIX); - } - throw new Error("Cannot stage the deadline Shields policy for managed MCP reconciliation", { - cause: error, - }); - } -} - function safeYamlObject(text: string): Record | null { try { const parsed = YAML.parse(text); diff --git a/src/lib/shields/policy-transition.test.ts b/src/lib/shields/policy-transition.test.ts index dcd61523852..4bc819394b2 100644 --- a/src/lib/shields/policy-transition.test.ts +++ b/src/lib/shields/policy-transition.test.ts @@ -7,7 +7,6 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; -import YAML from "yaml"; const requireSource = createRequire(import.meta.url); const SHIELDS_MODULE = "./index.js"; @@ -609,209 +608,3 @@ describe("shields config lock without a shipped config hash", () => { expect(entries.get(CONFIG_DIR)).toEqual({ mode: "2770", owner: "sandbox:sandbox" }); }); }); - -describe("managed MCP policy deadline restoration (#7952)", () => { - let homeDir: string; - - function createRestoreHarness() { - delete require.cache[requireSource.resolve(SHIELDS_MODULE)]; - delete require.cache[requireSource.resolve("./permissive-runtime.js")]; - delete require.cache[requireSource.resolve("../actions/sandbox/mcp-bridge-policy.js")]; - - const runner = requireSource("../runner.js") as typeof import("../runner.js"); - const policy = requireSource("../policy/index.js") as typeof import("../policy/index.js"); - const registry = requireSource("../state/registry.js") as typeof import("../state/registry.js"); - const policySetBodies: string[] = []; - - vi.spyOn(runner, "runCapture").mockReturnValue( - "version: 1\nnetwork_policies:\n live_baseline: {}\n", - ); - vi.spyOn(runner, "run").mockReturnValue({ status: 0 } as never); - vi.spyOn(policy, "buildPolicyGetCommand").mockReturnValue(["openshell", "policy", "get"]); - vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: unknown) => { - policySetBodies.push(fs.readFileSync(String(file), "utf-8")); - return ["openshell", "policy", "set"]; - }); - vi.spyOn(policy, "parseCurrentPolicy").mockImplementation((raw: unknown) => String(raw)); - vi.spyOn(registry, "getSandbox").mockReturnValue({ - name: "openclaw", - openshellDriver: "docker", - }); - - const shields = requireSource(SHIELDS_MODULE) as typeof import("./index.js"); - return { applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot, policySetBodies }; - } - - function writeCurrentProcessTimerMarker(snapshotPath: string, processToken: string): void { - fs.writeFileSync( - path.join(homeDir, ".nemoclaw", "state", "shields-timer-openclaw.json"), - JSON.stringify({ - pid: process.pid, - sandboxName: "openclaw", - snapshotPath, - restoreAt: new Date(Date.now() + 60_000).toISOString(), - processToken, - }), - { mode: 0o600 }, - ); - } - - beforeEach(() => { - homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-mcp-deadline-flow-")); - vi.stubEnv("HOME", homeDir); - }); - - afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllEnvs(); - fs.rmSync(homeDir, { recursive: true, force: true }); - delete require.cache[requireSource.resolve(SHIELDS_MODULE)]; - delete require.cache[requireSource.resolve("./permissive-runtime.js")]; - delete require.cache[requireSource.resolve("../actions/sandbox/mcp-bridge-policy.js")]; - }); - - it("restores lockdown with malformed and duplicate ownership", () => { - const stateDir = path.join(homeDir, ".nemoclaw", "state"); - const processToken = "a".repeat(32); - const snapshotPath = path.join(stateDir, "policy-snapshot-malformed-deadline.yaml"); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - snapshotPath, - YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: {}, - mcp_bridge_: {}, - mcp_bridge_alpha: {}, - }, - }), - ); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: ["mcp_bridge_", "mcp_bridge_alpha", "mcp_bridge_alpha"], - }), - ); - writeCurrentProcessTimerMarker(snapshotPath, processToken); - const harness = createRestoreHarness(); - - const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { - transitionProcessToken: processToken, - deadlineAuthoritative: true, - }); - - expect(result.status).toBe(0); - expect(result.managedMcpOmissions).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - key: "mcp_bridge_", - reason: expect.stringMatching(/ownership key.*invalid/), - }), - expect.objectContaining({ - key: "mcp_bridge_alpha", - reason: expect.stringMatching(/more than once/), - }), - ]), - ); - expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies).toEqual({ - restrictive_baseline: {}, - }); - }); - - it("restores lockdown when transition and persisted ownership differ", () => { - const stateDir = path.join(homeDir, ".nemoclaw", "state"); - const processToken = "b".repeat(32); - const snapshotPath = path.join(stateDir, "policy-snapshot-mismatched-deadline.yaml"); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - snapshotPath, - YAML.stringify({ - version: 1, - network_policies: { - restrictive_baseline: {}, - mcp_bridge_alpha: {}, - mcp_bridge_beta: {}, - }, - }), - ); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], - }), - ); - fs.writeFileSync( - path.join(stateDir, `shields-transition-openclaw-${processToken}.json`), - JSON.stringify({ - version: 1, - phase: "active", - ownerPid: process.pid, - ownerStartIdentity: "test-owner", - processToken, - sandboxName: "openclaw", - snapshotPath, - managedMcpPolicyKeys: ["mcp_bridge_beta"], - }), - { mode: 0o600 }, - ); - writeCurrentProcessTimerMarker(snapshotPath, processToken); - const harness = createRestoreHarness(); - - const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { - transitionProcessToken: processToken, - deadlineAuthoritative: true, - }); - - expect(result.status).toBe(0); - expect(result.managedMcpOmissions).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - reason: expect.stringMatching(/did not match persisted policy ownership/), - }), - ]), - ); - expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies).toEqual({ - restrictive_baseline: {}, - }); - }); - - it("restores lockdown from a legacy snapshot without ownership metadata", () => { - const stateDir = path.join(homeDir, ".nemoclaw", "state"); - const processToken = "c".repeat(32); - const snapshotPath = path.join(stateDir, "policy-snapshot-legacy-deadline.yaml"); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - snapshotPath, - YAML.stringify({ - version: 1, - network_policies: { restrictive_baseline: {}, mcp_bridge_alpha: {} }, - }), - ); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ shieldsDown: true, shieldsPolicySnapshotPath: snapshotPath }), - ); - writeCurrentProcessTimerMarker(snapshotPath, processToken); - const harness = createRestoreHarness(); - - const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { - transitionProcessToken: processToken, - deadlineAuthoritative: true, - }); - - expect(result.status).toBe(0); - expect(result.managedMcpOmissions).toEqual( - expect.arrayContaining([ - expect.objectContaining({ reason: expect.stringMatching(/no managed MCP ownership/) }), - expect.objectContaining({ key: "mcp_bridge_alpha" }), - ]), - ); - expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies).toEqual({ - restrictive_baseline: {}, - }); - }); -}); diff --git a/src/lib/shields/timer-process.test.ts b/src/lib/shields/timer-process.test.ts new file mode 100644 index 00000000000..84bd3a96560 --- /dev/null +++ b/src/lib/shields/timer-process.test.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { fork } from "node:child_process"; +import { once } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const PROCESS_TOKEN = "a".repeat(32); + +describe("detached Shields timer process", () => { + let tmpHome: string; + + beforeEach(() => { + tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-timer-process-")); + vi.stubEnv("HOME", tmpHome); + vi.resetModules(); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(tmpHome, { recursive: true, force: true }); + }); + + it.skipIf(process.platform === "win32")( + "exits after cooperative marker revocation", + { timeout: 20_000 }, + async () => { + const { killTimer } = await import("./timer-control"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + const sandboxName = "cooperative-cancellation"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + + const child = fork( + path.join(import.meta.dirname, "timer.ts"), + [ + sandboxName, + snapshotPath, + restoreAtIso, + "/sandbox/.openclaw/openclaw.json", + "/sandbox/.openclaw", + PROCESS_TOKEN, + "0", + "", + "", + "openclaw", + ], + { + env: { ...process.env, HOME: tmpHome }, + execArgv: ["--import", "tsx"], + stdio: ["ignore", "ignore", "pipe", "ipc"], + }, + ); + const childPid = child.pid; + if (childPid === undefined) throw new Error("Detached timer did not start"); + let childExited = false; + let childStderr = ""; + child.once("exit", () => { + childExited = true; + }); + child.stderr?.setEncoding("utf-8"); + child.stderr?.on("data", (chunk: string) => { + childStderr += chunk; + }); + + try { + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: childPid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + agentName: "openclaw", + configPath: "/sandbox/.openclaw/openclaw.json", + configDir: "/sandbox/.openclaw", + }), + { mode: 0o600 }, + ); + + const authorization = once(child, "message", { signal: AbortSignal.timeout(10_000) }); + child.send({ type: "authorize", processToken: PROCESS_TOKEN, acknowledge: true }); + const [message] = await authorization; + expect(message).toEqual({ type: "authorized", processToken: PROCESS_TOKEN }); + child.disconnect(); + + const exit = once(child, "exit", { signal: AbortSignal.timeout(10_000) }); + expect(killTimer(sandboxName)).toEqual({ + markerFound: true, + markerPid: childPid, + wasAlive: true, + terminated: false, + warnings: [], + }); + expect(fs.existsSync(markerPath)).toBe(false); + const [code, signal] = await exit; + expect({ code, signal }).toEqual({ code: 0, signal: null }); + } catch (error) { + throw new Error(`Detached timer process regression failed: ${childStderr}`, { + cause: error, + }); + } finally { + if (!childExited) { + const forcedExit = once(child, "exit", { signal: AbortSignal.timeout(1_000) }).catch( + () => undefined, + ); + child.kill("SIGKILL"); + await forcedExit; + } + } + }, + ); +}); diff --git a/src/lib/shields/timer-recovery-budget.test.ts b/src/lib/shields/timer-recovery-budget.test.ts index 1b2651d1983..030b602ac9a 100644 --- a/src/lib/shields/timer-recovery-budget.test.ts +++ b/src/lib/shields/timer-recovery-budget.test.ts @@ -13,18 +13,36 @@ import { } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ - applyShieldsPolicySnapshot: vi.fn(() => ({ status: 0 })), completeAutoRestoreTransition: vi.fn(() => true), lockAgentConfig: vi.fn(), prepareAutoRestoreTransitionTakeover: vi.fn(), resolvePersistedAutoRestoreTarget: vi.fn(), })); +const runMock = vi.fn(() => ({ status: 0 })); + +vi.mock("../runner", async (importOriginal) => ({ + ...(await importOriginal()), + run: runMock, +})); + +vi.mock("../policy", () => ({ + buildPolicySetCommand: vi.fn((file: string, name: string) => [ + "openshell", + "policy", + "set", + "--policy", + file, + "--wait", + name, + ]), +})); + vi.mock("./index", () => shieldsIndexMock); const PROCESS_TOKEN = "a".repeat(32); -describe("detached Shields recovery budget", () => { +describe("detached Shields recovery budget", { timeout: 15_000 }, () => { let tmpHome: string; let stateDir: string; @@ -32,6 +50,7 @@ describe("detached Shields recovery budget", () => { tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-recovery-budget-")); stateDir = path.join(tmpHome, ".nemoclaw", "state"); vi.stubEnv("HOME", tmpHome); + runMock.mockImplementation(() => ({ status: 0 })); vi.resetModules(); vi.clearAllMocks(); }); @@ -72,7 +91,7 @@ describe("detached Shields recovery budget", () => { return { args: args!, lockPath, markerPath, sandboxName, timer }; } - function readAuditEntries(): Array<{ error?: string }> { + function readAuditEntries(): Array<{ action: string; error?: string; warning?: string }> { return fs .readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8") .trim() @@ -90,7 +109,7 @@ describe("detached Shields recovery budget", () => { await timer.runRestoreTimer(args, { retryDelayMs: 0, maxRestoreAttempts: 7 }); await vi.waitFor(() => expect(exitSpy).toHaveBeenCalledWith(1), { interval: 1, - timeout: 2_000, + timeout: 10_000, }); const auditsAtExit = readAuditEntries(); @@ -100,7 +119,7 @@ describe("detached Shields recovery budget", () => { ).toHaveLength(1); expect(auditsAtExit.at(-1)?.error).toContain("recovery failed after 7 attempts"); expect(auditsAtExit.at(-1)?.error).toContain("Correct the state-directory write failure"); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); await expect( withMcpLifecycleLock(sandboxName, () => undefined, { stateDir }), ).rejects.toThrow(); @@ -113,19 +132,17 @@ describe("detached Shields recovery budget", () => { it("shares the budget across scheduled setup failures and restoration", async () => { const { args, lockPath, timer } = await createFixture("cross-phase-budget"); const containmentPath = `${lockPath}.containment`; - const lifecycleDirectory = path.dirname(lockPath); const originalMkdir = fs.promises.mkdir.bind(fs.promises); - let setupFailuresRemaining = 2; - vi.spyOn(fs.promises, "mkdir").mockImplementation(async (targetPath, options) => { - if (String(targetPath) === lifecycleDirectory && setupFailuresRemaining > 0) { - setupFailuresRemaining -= 1; - const error = new Error("simulated pre-fence setup failure") as NodeJS.ErrnoException; - error.code = "EIO"; - throw error; - } - return await originalMkdir(targetPath, options); - }); - shieldsIndexMock.applyShieldsPolicySnapshot.mockReturnValue({ status: 1 }); + const setupError = new Error("simulated pre-fence setup failure") as NodeJS.ErrnoException; + setupError.code = "EIO"; + const rejectSetup = async (): Promise => { + throw setupError; + }; + vi.spyOn(fs.promises, "mkdir") + .mockImplementationOnce(rejectSetup) + .mockImplementationOnce(rejectSetup) + .mockImplementation(originalMkdir); + runMock.mockReturnValue({ status: 1 }); const exitSpy = vi .spyOn(process, "exit") .mockImplementation((() => undefined) as typeof process.exit); @@ -133,11 +150,10 @@ describe("detached Shields recovery budget", () => { await timer.runRestoreTimer(args, { retryDelayMs: 0, maxRestoreAttempts: 7 }); await vi.waitFor(() => expect(exitSpy).toHaveBeenCalledWith(1), { interval: 1, - timeout: 2_000, + timeout: 10_000, }); - expect(setupFailuresRemaining).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(5); + expect(runMock).toHaveBeenCalledTimes(5); expect(fs.existsSync(containmentPath)).toBe(true); expect( readAuditEntries().filter((entry) => @@ -146,23 +162,92 @@ describe("detached Shields recovery budget", () => { ).toHaveLength(1); }); + it("waits beyond the recovery budget for a verified live lifecycle owner, then restores", async () => { + const sandboxName = "healthy-backup-owner"; + let markOwnerEntered!: () => void; + let releaseOwner!: () => void; + const ownerEntered = new Promise((resolve) => { + markOwnerEntered = resolve; + }); + const ownerReleased = new Promise((resolve) => { + releaseOwner = resolve; + }); + const owner = withMcpLifecycleLock( + sandboxName, + async () => { + markOwnerEntered(); + await ownerReleased; + }, + { stateDir, pollIntervalMs: 1, timeoutMs: 1_000 }, + ); + await ownerEntered; + + const { args, lockPath, timer } = await createFixture(sandboxName); + const containmentPath = `${lockPath}.containment`; + const deadlinePath = `${lockPath}.deadline`; + runMock.mockReturnValue({ status: 0 }); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + + try { + const restore = timer.runRestoreTimer(args, { + deadlineSetupTimeoutMs: 10, + maxRestoreAttempts: 1, + retryDelayMs: 0, + }); + await vi.waitFor(() => expect(fs.existsSync(deadlinePath)).toBe(true), { + interval: 1, + timeout: 10_000, + }); + await new Promise((resolve) => setTimeout(resolve, 120)); + + expect(exitSpy).not.toHaveBeenCalled(); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(runMock).not.toHaveBeenCalled(); + const waitingAudits = readAuditEntries(); + expect(waitingAudits).toEqual([ + expect.objectContaining({ + action: "shields_auto_restore_lock_warning", + warning: expect.stringContaining("verified live sandbox mutation owner"), + }), + ]); + expect(JSON.stringify(waitingAudits)).not.toContain("Contained owner PID"); + expect(waitingAudits).not.toContainEqual( + expect.objectContaining({ action: "shields_up_failed" }), + ); + + releaseOwner(); + await Promise.all([owner, restore]); + + expect(exitSpy).toHaveBeenCalledOnce(); + expect(exitSpy).toHaveBeenCalledWith(0); + expect(runMock).toHaveBeenCalledOnce(); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(readAuditEntries()).not.toContainEqual( + expect.objectContaining({ action: "shields_up_failed" }), + ); + } finally { + releaseOwner(); + await owner; + } + }); + it("charges deadline-main publication failures to the same bounded budget", async () => { const { args, lockPath, sandboxName, timer } = await createFixture("publication-budget"); const containmentPath = `${lockPath}.containment`; const deadlinePath = `${lockPath}.deadline`; const originalLink = fs.promises.link.bind(fs.promises); - let mainPublicationAttempts = 0; - vi.spyOn(fs.promises, "link").mockImplementation(async (existingPath, newPath) => { - if (String(newPath) === lockPath) { - mainPublicationAttempts += 1; - const error = new Error( - "simulated deadline-main publication failure", - ) as NodeJS.ErrnoException; - error.code = "EROFS"; - throw error; - } - return await originalLink(existingPath, newPath); - }); + const publicationError = new Error( + "simulated deadline-main publication failure", + ) as NodeJS.ErrnoException; + publicationError.code = "EROFS"; + const linkSpy = vi + .spyOn(fs.promises, "link") + .mockImplementationOnce(originalLink) + .mockRejectedValue(publicationError); const exitSpy = vi .spyOn(process, "exit") .mockImplementation((() => undefined) as typeof process.exit); @@ -171,8 +256,8 @@ describe("detached Shields recovery budget", () => { expect(exitSpy).toHaveBeenCalledTimes(1); expect(exitSpy).toHaveBeenCalledWith(1); - expect(mainPublicationAttempts).toBe(3); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(linkSpy).toHaveBeenCalledTimes(4); + expect(runMock).not.toHaveBeenCalled(); expect(fs.existsSync(deadlinePath)).toBe(false); expect(fs.existsSync(containmentPath)).toBe(true); expect(readAuditEntries()).toHaveLength(2); @@ -189,26 +274,18 @@ describe("detached Shields recovery budget", () => { const containmentPath = `${lockPath}.containment`; const deadlinePath = `${lockPath}.deadline`; const originalAsyncLink = fs.promises.link.bind(fs.promises); - vi.spyOn(fs.promises, "link").mockImplementation(async (existingPath, newPath) => { - if (String(newPath) === lockPath) { - const error = new Error( - "simulated deadline-main publication failure", - ) as NodeJS.ErrnoException; - error.code = "EROFS"; - throw error; - } - return await originalAsyncLink(existingPath, newPath); - }); - const originalSyncLink = fs.linkSync.bind(fs); - vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { - if (String(newPath) === containmentPath) { - const error = new Error( - "simulated containment publication failure", - ) as NodeJS.ErrnoException; - error.code = "EROFS"; - throw error; - } - return originalSyncLink(existingPath, newPath); + const publicationError = new Error( + "simulated deadline-main publication failure", + ) as NodeJS.ErrnoException; + publicationError.code = "EROFS"; + vi.spyOn(fs.promises, "link") + .mockImplementationOnce(originalAsyncLink) + .mockRejectedValue(publicationError); + vi.spyOn(fs, "linkSync").mockImplementation((_existingPath, newPath) => { + expect(String(newPath)).toBe(containmentPath); + const error = new Error("simulated containment publication failure") as NodeJS.ErrnoException; + error.code = "EROFS"; + throw error; }); const exitSpy = vi .spyOn(process, "exit") @@ -230,6 +307,7 @@ describe("detached Shields recovery budget", () => { expect(audits).toHaveLength(1); expect(audits[0]?.error).toContain("recovery failed after 1 attempt"); expect(audits[0]?.error).toContain("Correct the state-directory write failure"); + expect(audits[0]?.error).toContain("`nemoclaw publication-retained-gate shields status`"); expect(audits[0]?.error).not.toContain("setup is retrying"); await expect( withMcpLifecycleLock( @@ -260,7 +338,7 @@ describe("detached Shields recovery budget", () => { expect(exitSpy).toHaveBeenCalledTimes(1); expect(exitSpy).toHaveBeenCalledWith(1); expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); const audits = readAuditEntries(); expect(audits).toHaveLength(1); expect(audits[0]?.error).toContain("committed process-tree containment"); diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index a3e56a64d32..8c3812e0711 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -13,12 +13,6 @@ import { } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ - applyShieldsPolicySnapshot: vi.fn( - (): { - status: number; - managedMcpOmissions?: Array<{ server: string; reason: string }>; - } => ({ status: 0 }), - ), completeAutoRestoreTransition: vi.fn(() => true), lockAgentConfig: vi.fn() as unknown, prepareAutoRestoreTransitionTakeover: vi.fn(), @@ -27,13 +21,31 @@ const shieldsIndexMock = vi.hoisted(() => ({ const PROCESS_TOKEN = "a".repeat(32); +const runMock = vi.fn(() => ({ status: 0 })); + +vi.mock("../runner", async (importOriginal) => ({ + ...(await importOriginal()), + run: runMock, +})); + +vi.mock("../policy", () => ({ + buildPolicySetCommand: vi.fn((file: string, name: string) => [ + "openshell", + "policy", + "set", + "--policy", + file, + "--wait", + name, + ]), +})); + interface TimerTestOptions { retryDelayMs?: number; maxRestoreAttempts?: number; } vi.mock("./index", () => ({ - applyShieldsPolicySnapshot: shieldsIndexMock.applyShieldsPolicySnapshot, completeAutoRestoreTransition: shieldsIndexMock.completeAutoRestoreTransition, get lockAgentConfig() { return shieldsIndexMock.lockAgentConfig; @@ -50,7 +62,6 @@ describe("shields timer authorization", () => { beforeEach(() => { tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-timer-")); vi.stubEnv("HOME", tmpHome); - shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementation(() => ({ status: 0 })); shieldsIndexMock.lockAgentConfig = vi.fn(); shieldsIndexMock.resolvePersistedAutoRestoreTarget = vi.fn( ( @@ -71,6 +82,7 @@ describe("shields timer authorization", () => { } : undefined, ); + runMock.mockImplementation(() => ({ status: 0 })); vi.resetModules(); vi.clearAllMocks(); }); @@ -130,24 +142,18 @@ describe("shields timer authorization", () => { )}.deadline`; const auditPath = path.join(path.dirname(markerPath), "shields-audit.jsonl"); const pending = runRestoreTimer(args, { retryDelayMs: 50 }); - let policyApplicationsBeforeRevocation: number | undefined; try { await waitForRetryBoundary(deadlinePath, auditPath); expect(exitSpy).not.toHaveBeenCalled(); expect(fs.existsSync(deadlinePath)).toBe(true); - policyApplicationsBeforeRevocation = - shieldsIndexMock.applyShieldsPolicySnapshot.mock.calls.length; - } finally { + const policyApplicationsBeforeRevocation = runMock.mock.calls.length; fs.rmSync(markerPath, { force: true }); await pending; + expect(runMock).toHaveBeenCalledTimes(policyApplicationsBeforeRevocation); + } finally { fs.writeFileSync(markerPath, markerContents); exitSpy.mockRestore(); } - if (policyApplicationsBeforeRevocation !== undefined) { - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes( - policyApplicationsBeforeRevocation, - ); - } } function createFailedRestoreFixture( @@ -173,7 +179,7 @@ describe("shields timer authorization", () => { }), ); writeMarker(PROCESS_TOKEN); - shieldsIndexMock.applyShieldsPolicySnapshot.mockReturnValue({ status: 1 }); + runMock.mockReturnValue({ status: 1 }); const args = parseTimerArgs([sandboxName, snapshotPath, restoreAtIso, "", "", PROCESS_TOKEN]); expect(args).not.toBeNull(); return { @@ -208,7 +214,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); }); @@ -250,7 +256,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.existsSync(markerPath)).toBe(true); }); @@ -294,7 +300,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.lstatSync(markerPath).isSymbolicLink()).toBe(true); expect(fs.readFileSync(markerTargetPath, "utf-8")).toBe(markerTarget); @@ -430,7 +436,7 @@ describe("shields timer authorization", () => { await timer.runRestoreTimer(args!); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); expect(exitSpy).not.toHaveBeenCalled(); expect(vi.getTimerCount()).toBe(1); expect(fs.existsSync(markerPath)).toBe(true); @@ -440,7 +446,7 @@ describe("shields timer authorization", () => { } }); - it("audits a successful restore retry without stale MCP warnings or timestamps", async () => { + it("audits a successful restore retry while retaining deadline ownership", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); @@ -463,17 +469,10 @@ describe("shields timer authorization", () => { leaseOwnerStartIdentity: "proc:dead-owner", }), ); - shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { + runMock.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); expect(fs.existsSync(deadlinePath)).toBe(true); - return { - status: 17, - managedMcpOmissions: [{ server: "beta", reason: "incomplete add" }], - }; - }); - shieldsIndexMock.applyShieldsPolicySnapshot.mockReturnValueOnce({ - status: 0, - managedMcpOmissions: [], + return { status: 17 }; }); const args = timer.parseTimerArgs([ sandboxName, @@ -491,7 +490,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(2); + expect(runMock).toHaveBeenCalledTimes(2); expect(shieldsIndexMock.completeAutoRestoreTransition).toHaveBeenCalledWith( sandboxName, PROCESS_TOKEN, @@ -505,22 +504,14 @@ describe("shields timer authorization", () => { .split("\n") .filter(Boolean) .map((line) => JSON.parse(line)); - const successAudits = audits.filter((audit) => audit.action === "shields_auto_restore"); - expect(successAudits).toEqual([ + expect(audits).toContainEqual( expect.objectContaining({ - action: "shields_auto_restore", - sandbox: sandboxName, + action: "shields_up_failed", + error: "Policy restore exited with status 17", }), - ]); - expect(successAudits[0]).not.toHaveProperty("warning"); - const failedAudit = audits.find( - (audit) => - audit.action === "shields_up_failed" && - audit.error === "Policy restore exited with status 17", ); - expect(failedAudit).toEqual(expect.objectContaining({ timestamp: expect.any(String) })); - expect(Date.parse(successAudits[0].timestamp)).toBeGreaterThan( - Date.parse(failedAudit.timestamp), + expect(audits).toContainEqual( + expect.objectContaining({ action: "shields_auto_restore", sandbox: sandboxName }), ); }); @@ -562,7 +553,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.existsSync(markerPath)).toBe(true); }); @@ -659,7 +650,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(1); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); expect(shieldsIndexMock.completeAutoRestoreTransition).not.toHaveBeenCalled(); expect(fs.existsSync(mutationLockPath)).toBe(false); expect(fs.existsSync(deadlinePath)).toBe(false); @@ -701,7 +692,7 @@ describe("shields timer authorization", () => { const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); const deadlinePath = `${sandboxMutationLockPath}.deadline`; - shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { + runMock.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); expect(fs.existsSync(deadlinePath)).toBe(true); expect(JSON.parse(fs.readFileSync(lockPath, "utf-8"))).toMatchObject({ @@ -709,10 +700,7 @@ describe("shields timer authorization", () => { command: "shields auto-restore", takeoverToken: PROCESS_TOKEN, }); - return { - status: 0, - managedMcpOmissions: [{ server: "beta", reason: "incomplete add" }], - }; + return { status: 0 }; }); const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); @@ -720,7 +708,7 @@ describe("shields timer authorization", () => { const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); expect(exitCode).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); + expect(runMock).toHaveBeenCalledTimes(1); expect(updatedState.shieldsDown).toBe(false); expect(updatedState.shieldsDownAt).toBeNull(); expect(fs.existsSync(markerPath)).toBe(false); @@ -731,18 +719,6 @@ describe("shields timer authorization", () => { PROCESS_TOKEN, snapshotPath, ); - expect( - fs - .readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8") - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)), - ).toContainEqual( - expect.objectContaining({ - action: "shields_auto_restore", - warning: "Auto-restore omitted 1 unproven managed MCP policy entries", - }), - ); }); it("keeps the deadline gate closed while a failed restore retries", async () => { @@ -765,9 +741,7 @@ describe("shields timer authorization", () => { processToken: PROCESS_TOKEN, }), ); - shieldsIndexMock.applyShieldsPolicySnapshot - .mockReturnValueOnce({ status: 1 }) - .mockReturnValue({ status: 0 }); + runMock.mockReturnValueOnce({ status: 1 }).mockReturnValue({ status: 0 }); const args = timer.parseTimerArgs([ sandboxName, snapshotPath, @@ -784,13 +758,10 @@ describe("shields timer authorization", () => { try { const restore = timer.runRestoreTimer(args!, { retryDelayMs: 100 }); - await vi.waitFor( - () => expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1), - { - interval: 1, - timeout: 200, - }, - ); + await vi.waitFor(() => expect(runMock).toHaveBeenCalledTimes(1), { + interval: 1, + timeout: 2_000, + }); expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); const contender = withMcpLifecycleLock( @@ -805,7 +776,7 @@ describe("shields timer authorization", () => { expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); await Promise.all([restore, contender]); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(2); + expect(runMock).toHaveBeenCalledTimes(2); expect(contenderEntered).toBe(true); expect(shieldsIndexMock.completeAutoRestoreTransition).toHaveBeenCalledWith( sandboxName, @@ -836,7 +807,7 @@ describe("shields timer authorization", () => { }); expect(exitCode).toBe(1); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(7); + expect(runMock).toHaveBeenCalledTimes(7); expect(fs.existsSync(markerPath)).toBe(true); expect(fs.existsSync(mutationLockPath)).toBe(false); expect(fs.existsSync(deadlinePath)).toBe(false); @@ -871,10 +842,13 @@ describe("shields timer authorization", () => { fixture; const replacementToken = "f".repeat(32); const originalLink = fs.linkSync.bind(fs); + const publishReplacement = (existingPath: fs.PathLike, newPath: fs.PathLike): void => { + originalLink(existingPath, newPath); + writeMarker(replacementToken); + }; + const linkHandlers = new Map([[containmentPath, publishReplacement]]); const linkSpy = vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { - const result = originalLink(existingPath, newPath); - if (String(newPath) === containmentPath) writeMarker(replacementToken); - return result; + return (linkHandlers.get(String(newPath)) ?? originalLink)(existingPath, newPath); }); try { @@ -905,19 +879,19 @@ describe("shields timer authorization", () => { fixture; const replacementToken = "e".repeat(32); const originalLink = fs.linkSync.bind(fs); + const publishReplacement = (existingPath: fs.PathLike, newPath: fs.PathLike): void => { + originalLink(existingPath, newPath); + writeMarker(replacementToken); + }; + const linkHandlers = new Map([[containmentPath, publishReplacement]]); const linkSpy = vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { - const result = originalLink(existingPath, newPath); - if (String(newPath) === containmentPath) writeMarker(replacementToken); - return result; + return (linkHandlers.get(String(newPath)) ?? originalLink)(existingPath, newPath); }); - const originalRename = fs.renameSync.bind(fs); - const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((oldPath, newPath) => { - if (String(oldPath) === containmentPath) { - const error = new Error("simulated exact rollback failure") as NodeJS.ErrnoException; - error.code = "EACCES"; - throw error; - } - return originalRename(oldPath, newPath); + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((oldPath) => { + expect(String(oldPath)).toBe(containmentPath); + const error = new Error("simulated exact rollback failure") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; }); try { @@ -944,13 +918,16 @@ describe("shields timer authorization", () => { const { args, containmentPath, deadlinePath, markerPath, mutationLockPath, stateDir } = createFailedRestoreFixture("retry-containment-failure", timer.parseTimerArgs); const originalLink = fs.linkSync.bind(fs); - const linkSpy = vi.spyOn(fs, "linkSync").mockImplementation((existingPath, newPath) => { - if (String(newPath) === containmentPath) { - const error = new Error("simulated containment commit failure") as NodeJS.ErrnoException; - error.code = "EROFS"; - throw error; - } - return originalLink(existingPath, newPath); + const rejectContainment = (): never => { + const error = new Error("simulated containment commit failure") as NodeJS.ErrnoException; + error.code = "EROFS"; + throw error; + }; + const linkHandlers = new Map([ + [containmentPath, rejectContainment], + ]); + const linkSpy = vi.spyOn(fs, "linkSync").mockImplementation((_existingPath, newPath) => { + return (linkHandlers.get(String(newPath)) ?? originalLink)(_existingPath, newPath); }); try { @@ -963,7 +940,7 @@ describe("shields timer authorization", () => { linkSpy.mockRestore(); } - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); + expect(runMock).toHaveBeenCalledTimes(1); expect(fs.existsSync(markerPath)).toBe(true); expect(fs.existsSync(containmentPath)).toBe(false); expect(fs.existsSync(mutationLockPath)).toBe(true); @@ -994,7 +971,7 @@ describe("shields timer authorization", () => { expect(exitCode).toBe(1); expect(shieldsIndexMock.prepareAutoRestoreTransitionTakeover).toHaveBeenCalledTimes(1); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(runMock).not.toHaveBeenCalled(); expect(fs.existsSync(markerPath)).toBe(true); expect(fs.existsSync(mutationLockPath)).toBe(false); expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(false); @@ -1106,12 +1083,7 @@ describe("shields timer authorization", () => { const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); expect(exitCode).toBe(0); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledWith( - sandboxName, - snapshotPath, - { deadlineAuthoritative: true, transitionProcessToken: PROCESS_TOKEN }, - ); + expect(runMock).toHaveBeenCalledTimes(1); // #4663: relockAndReconfirm applies then re-confirms after the settle // window (0ms under test), so lockAgentConfig is invoked twice for a clean // lock. @@ -1240,7 +1212,7 @@ describe("shields timer authorization", () => { .split("\n") .map((line) => JSON.parse(line)); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); + expect(runMock).toHaveBeenCalledTimes(1); expect(updatedState.shieldsDown).toBe(true); expect(auditEntries).toContainEqual( expect.objectContaining({ diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index 920996533c6..51083167d9d 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -10,7 +10,10 @@ import fs from "node:fs"; import path from "node:path"; +import { CLI_NAME } from "../cli/branding"; import { isObjectRecord, type UnknownRecord } from "../core/json-types"; +import { buildPolicySetCommand } from "../policy"; +import { run } from "../runner"; import { beginCommittedMcpLifecycleContainmentSync, durableMcpLifecycleContainmentFailure, @@ -58,6 +61,7 @@ interface TimerArgs { interface TimerRuntimeOptions { retryDelayMs?: number; maxRestoreAttempts?: number; + deadlineSetupTimeoutMs?: number; } interface RecoveryAttemptBudget { @@ -70,6 +74,7 @@ type RestoreAttemptOutcome = "complete" | "retry" | "revoked"; const STATE_DIR = resolveNemoclawStateDir(); const AUTO_RESTORE_RETRY_MS = 5_000; const AUTO_RESTORE_MAX_ATTEMPTS = 7; +const AUTO_RESTORE_AUTHORITY_POLL_MS = 250; function parseTimerArgs(argv: string[]): TimerArgs | null { const [ @@ -266,10 +271,14 @@ async function runRestoreTimerWithBudget( (runtimeOptions.maxRestoreAttempts ?? 0) > 0 ? runtimeOptions.maxRestoreAttempts! : AUTO_RESTORE_MAX_ATTEMPTS; + const deadlineSetupTimeoutMs = + Number.isFinite(runtimeOptions.deadlineSetupTimeoutMs) && + (runtimeOptions.deadlineSetupTimeoutMs ?? 0) > 0 + ? Math.floor(runtimeOptions.deadlineSetupTimeoutMs!) + : 5_000; let exitCode = 0; let retryScheduled = false; let terminalContainment = false; - let managedMcpWarning: string | undefined; const scheduleRetry = (): boolean => { if (!markerMatchesCurrentTimer(args)) return false; retryScheduled = true; @@ -302,7 +311,7 @@ async function runRestoreTimerWithBudget( const message = error instanceof Error ? error.message : String(error); return durableMcpLifecycleContainmentFailure( new Error( - `${reason}; durable containment could not be committed: ${message}. Correct the state-directory write failure, then retry a NemoClaw command for this sandbox to obtain exact-generation recovery guidance`, + `${reason}; durable containment could not be committed: ${message}. Correct the state-directory write failure, then run \`${CLI_NAME} ${args.sandboxName} shields status\` to resume recovery or receive exact-generation recovery guidance`, ), lockPath, { retainOwnedLifecycleGates: true }, @@ -361,16 +370,10 @@ async function runRestoreTimerWithBudget( } // Restore policy (slow — openshell policy set --wait blocks) - const result = shields.applyShieldsPolicySnapshot(args.sandboxName, args.snapshotPath, { - transitionProcessToken: args.processToken, - deadlineAuthoritative: true, + const result = run(buildPolicySetCommand(args.snapshotPath, args.sandboxName), { + ignoreError: true, }); const status = typeof result.status === "number" ? result.status : 1; - managedMcpWarning = result.managedMcpOmissions?.length - ? `Auto-restore omitted ${String( - result.managedMcpOmissions.length, - )} unproven managed MCP policy entries` - : undefined; if (status !== 0) { appendAudit({ @@ -495,7 +498,6 @@ async function runRestoreTimerWithBudget( restored_by: "auto_timer", policy_snapshot: args.snapshotPath, scheduled_restore_at: args.restoreAtIso, - ...(managedMcpWarning ? { warning: managedMcpWarning } : {}), }); cleanupOwnedTimerMarker(args); exitCode = 0; @@ -568,7 +570,7 @@ async function runRestoreTimerWithBudget( { stateDir: STATE_DIR, pollIntervalMs: 50, - timeoutMs: 5_000, + timeoutMs: deadlineSetupTimeoutMs, throwOnCommittedContainment: true, onSetupFailure: async () => { const attempt = (recoveryBudget.attemptsUsed += 1); @@ -576,7 +578,18 @@ async function runRestoreTimerWithBudget( await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); assertTakeoverAuthority(); }, - onContainment: ({ ownerPid, reason }) => { + onContainment: ({ kind, ownerPid, reason }) => { + if (kind === "verified-live-wait") { + appendAudit({ + action: "shields_auto_restore_lock_warning", + sandbox: args.sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: args.snapshotPath, + warning: reason, + }); + return; + } appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, @@ -629,31 +642,59 @@ function main(): void { } let scheduled = false; - const authorize = (): void => { - if (scheduled || !markerMatchesCurrentTimer(args)) return; + const authorize = (): boolean => { + if (scheduled) return true; + if (!markerMatchesCurrentTimer(args)) return false; scheduled = true; - setTimeout( + const restoreTimeout = setTimeout( () => { + clearInterval(authorityPoll); + if (!markerMatchesCurrentTimer(args)) { + process.exit(0); + return; + } void runRestoreTimer(args); }, Math.max(0, args.restoreAtMs - Date.now()), ); + const authorityPoll = setInterval(() => { + if (markerMatchesCurrentTimer(args)) return; + clearTimeout(restoreTimeout); + clearInterval(authorityPoll); + process.exit(0); + }, AUTO_RESTORE_AUTHORITY_POLL_MS); + return true; }; // The parent publishes the PID/token marker before authorizing this child. // Without this barrier a short timeout can fire in the fork-to-marker gap, // causing the child to exit before the parent records a now-dead timer PID. process.on("message", (message: unknown) => { + const request = message as { + type?: unknown; + processToken?: unknown; + acknowledge?: unknown; + }; if ( typeof message === "object" && message !== null && - (message as { type?: unknown }).type === "authorize" && - (message as { processToken?: unknown }).processToken === args.processToken + request.type === "authorize" && + request.processToken === args.processToken ) { - authorize(); + const authorized = authorize(); + if ( + authorized && + request.acknowledge === true && + process.connected && + process.send !== undefined + ) { + process.send({ type: "authorized", processToken: args.processToken }, () => undefined); + } } }); - process.once("disconnect", authorize); + process.once("disconnect", () => { + authorize(); + }); } if (require.main === module) { diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index 688575b849c..4408b32849f 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -68,6 +68,7 @@ export interface McpLifecycleDeadlineFenceSyncOptions extends McpLifecycleLockOp } export interface McpLifecycleDeadlineContainment { + kind: "verified-live-wait" | "failure"; ownerPid: number | null; reason: string; } @@ -855,6 +856,7 @@ async function acquireDeadlineFence( performance.now() - blockedAt >= timeoutMs ) { await reportDeadlineContainment(options, { + kind: "failure", ownerPid: null, reason: "A committed process-tree containment requires operator resolution before auto-restore can continue.", @@ -907,6 +909,7 @@ async function acquireDeadlineFence( } if (generation !== notifiedGeneration) { await reportDeadlineContainment(options, { + kind: "failure", ownerPid: observation.owner?.pid ?? null, reason, }); @@ -957,6 +960,7 @@ function acquireDeadlineFenceSync( if (options.throwOnCommittedContainment) { const reason = `A committed process-tree containment requires operator resolution before auto-restore can continue. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${deadlinePath}', and '${containmentPath}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; reportDeadlineContainmentSync(options, { + kind: "failure", ownerPid: null, reason, }); @@ -967,6 +971,7 @@ function acquireDeadlineFenceSync( performance.now() - blockedAt >= timeoutMs ) { reportDeadlineContainmentSync(options, { + kind: "failure", ownerPid: null, reason: "A committed process-tree containment requires operator resolution before auto-restore can continue.", @@ -1012,6 +1017,7 @@ function acquireDeadlineFenceSync( }`; if (generation !== notifiedGeneration && performance.now() - blockedAt >= timeoutMs) { reportDeadlineContainmentSync(options, { + kind: "failure", ownerPid: observation.owner?.pid ?? null, reason: "Another auto-restore deadline owner is active or cannot be verified; the deadline gate remains closed pending operator or distributed-lease resolution.", @@ -1054,6 +1060,11 @@ async function clearDeadlineProtectedPath( Boolean(owner.processIdentity) && owner.hostIdentity === readMcpLockHostIdentity() && owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity(); + const exactCurrentOrdinaryOwner = + !targetPath.endsWith(".reaper") && + disposition === "active" && + exactLocalOwner && + readMcpLockProcessIdentity(owner.pid, true) === owner.processIdentity; if (disposition === "stale" && exactLocalOwner) { const confirmed = await readMcpLifecycleLockObservation(targetPath); if (!sameLockGeneration(observed, confirmed)) continue; @@ -1082,13 +1093,27 @@ async function clearDeadlineProtectedPath( owner?.token ?? "invalid" }`; if (performance.now() - blockedAt >= containmentTimeoutMs) { - const reason = `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`; + if (exactCurrentOrdinaryOwner) { + const reason = `The verified live ${targetLabel} owner PID ${String(owner.pid)} is still completing its lifecycle transaction. Shields remain DOWN and the deadline gate is blocking new mutations until that owner releases the lock.`; + if (generation !== notifiedGeneration) { + await reportDeadlineContainment(options, { + kind: "verified-live-wait", + ownerPid: owner.pid, + reason, + }); + notifiedGeneration = generation; + } + await sleep(pollIntervalMs); + continue; + } + const reason = `The active or unverifiable ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`; if (options.onSetupFailure) { blockedAt = performance.now(); await options.onSetupFailure(new Error(reason)); } if (generation !== notifiedGeneration) { await reportDeadlineContainment(options, { + kind: "failure", ownerPid: owner?.pid ?? null, reason, }); @@ -1185,6 +1210,7 @@ function clearDeadlineProtectedPathSync( performance.now() - blockedAt >= containmentTimeoutMs ) { reportDeadlineContainmentSync(options, { + kind: "failure", ownerPid: owner?.pid ?? null, reason: `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`, }); @@ -1263,6 +1289,7 @@ async function publishDeadlineMainOwner( } if (message !== notifiedError) { await reportDeadlineContainment(options, { + kind: "failure", ownerPid: null, reason: resolutionReason, }); @@ -1283,6 +1310,7 @@ async function publishDeadlineMainOwner( if (options.onSetupFailure) await options.onSetupFailure(error); if (message !== notifiedError) { await reportDeadlineContainment(options, { + kind: "failure", ownerPid: null, reason: `Auto-restore deadline setup is retrying while the gate remains closed: ${message}`, }); @@ -1360,6 +1388,7 @@ function publishDeadlineMainOwnerSync( const resolutionReason = `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; if (message !== notifiedError) { reportDeadlineContainmentSync(options, { + kind: "failure", ownerPid: null, reason: resolutionReason, }); @@ -1384,6 +1413,7 @@ function publishDeadlineMainOwnerSync( } if (message !== notifiedError) { reportDeadlineContainmentSync(options, { + kind: "failure", ownerPid: null, reason: `Auto-restore deadline setup is retrying while the gate remains closed: ${message}`, }); diff --git a/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.test.ts b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.test.ts new file mode 100644 index 00000000000..6dcdf8d9356 --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.test.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { readShieldsTimerMarker, shieldsTimerMarkerPath } from "./shields-timer-authority"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { force: true, recursive: true }); + } +}); + +function writeMarker(stateDir: string, requestedSandbox: string, markerSandbox: string): void { + fs.writeFileSync( + shieldsTimerMarkerPath(requestedSandbox, stateDir), + JSON.stringify({ + pid: process.pid, + restoreAt: "2026-08-03T12:00:00.000Z", + sandboxName: markerSandbox, + snapshotPath: path.join(stateDir, "snapshot.yaml"), + }), + ); +} + +describe("Shields timer marker authority", () => { + it("accepts a marker bound to the requested sandbox", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shields-marker-")); + tempDirs.push(stateDir); + writeMarker(stateDir, "alpha", "alpha"); + + expect(readShieldsTimerMarker("alpha", stateDir)).toMatchObject({ sandboxName: "alpha" }); + }); + + it("rejects a marker whose payload names another sandbox", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shields-marker-")); + tempDirs.push(stateDir); + writeMarker(stateDir, "alpha", "beta"); + + expect(readShieldsTimerMarker("alpha", stateDir)).toBeNull(); + }); +}); diff --git a/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts index 6da9abb0737..e2673beca34 100644 --- a/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts +++ b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts @@ -72,7 +72,8 @@ export function readShieldsTimerMarker( stateDir = resolveNemoclawStateDir(), ): ShieldsTimerMarker | null { try { - return readShieldsTimerMarkerFile(shieldsTimerMarkerPath(sandboxName, stateDir)); + const marker = readShieldsTimerMarkerFile(shieldsTimerMarkerPath(sandboxName, stateDir)); + return marker?.sandboxName === sandboxName ? marker : null; } catch { return null; } diff --git a/test/e2e/live/mcp-bridge-hermes-lifecycle.ts b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts index 5044cc5e31a..0f9b7a06881 100644 --- a/test/e2e/live/mcp-bridge-hermes-lifecycle.ts +++ b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts @@ -187,6 +187,7 @@ export async function assertHermesManagedAddSurvivesLockedGatewayRestartAndState }, ); expectExitZero(shieldsDown, "unlock Hermes config for remaining managed MCP lifecycle"); + await assertHermesReloadRollback(sandbox, sandboxName, mcpUrl); } /** diff --git a/test/e2e/live/mcp-bridge-sandbox.ts b/test/e2e/live/mcp-bridge-sandbox.ts index aa801ef4c28..5d9f5b8927e 100644 --- a/test/e2e/live/mcp-bridge-sandbox.ts +++ b/test/e2e/live/mcp-bridge-sandbox.ts @@ -1,91 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import assert from "node:assert/strict"; -import YAML from "yaml"; import { shellQuote } from "../../../src/lib/core/shell-quote"; -import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; -import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const MCP_CURL_HTTP_CODE_MARKER = "NEMOCLAW_MCP_CURL_HTTP_CODE="; export type McpDnsRebindingAdapter = "mcporter" | "hermes-config" | "deepagents-config"; -export type CapturedManagedMcpPolicy = { - networkPolicies: Record; - policy: McpNetworkPolicy; -}; - -type McpNetworkPolicy = { - endpoints?: Array<{ - host?: string; - allowed_ips?: string[]; - [key: string]: unknown; - }>; - [key: string]: unknown; -}; - -export async function captureManagedMcpPolicy( - sandbox: SandboxClient, - options: { - artifactName: string; - label: string; - policyKey: string; - sandboxName: string; - url: string; - }, -): Promise { - const result = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { - artifactName: options.artifactName, - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); - assertExitZero(result, options.label); - const document = YAML.parse(parseOpenShellPolicy(resultText(result)).yamlBody) as { - network_policies?: Record; - }; - const networkPolicies = document.network_policies ?? {}; - const policy = networkPolicies[options.policyKey]; - if (!policy) { - throw new Error(`${options.label}: managed MCP policy '${options.policyKey}' is absent`); - } - const endpoint = policy.endpoints?.[0]; - const expectedHost = new URL(options.url).hostname; - if (endpoint?.host !== expectedHost) { - throw new Error(`${options.label}: expected managed MCP host '${expectedHost}'`); - } - if ( - !Array.isArray(endpoint.allowed_ips) || - endpoint.allowed_ips.length === 0 || - endpoint.allowed_ips.some((address) => typeof address !== "string") - ) { - throw new Error(`${options.label}: expected at least one managed MCP address pin`); - } - return { networkPolicies, policy }; -} - -export function assertManagedMcpPolicySurvivedRemoval( - before: McpNetworkPolicy, - after: CapturedManagedMcpPolicy, - removedPolicyKey: string, -): void { - assert.deepStrictEqual(after.policy, before); - assert.equal(after.networkPolicies[removedPolicyKey], undefined); -} - -export function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { - assert.notEqual( - result.exitCode, - 0, - `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, - ); - assert.match(resultText(result), pattern); -} - export async function hostAddressForSandbox(_host: HostCliClient): Promise { return "host.openshell.internal"; } diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 84789a9161e..3cf1043cdea 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -4,12 +4,14 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import YAML from "yaml"; import { buildDeepAgentsMcpStatusCommand, buildHermesMcpStatusCommand, buildOpenClawMcporterInspectCommand, } from "../../../src/lib/actions/sandbox/mcp-bridge-adapter-status"; import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import type { McpBridgeEntry } from "../../../src/lib/state/registry"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; @@ -25,17 +27,13 @@ import { assertHermesConfig, assertHermesInspectionRejectsUnmanagedFields, assertHermesManagedAddSurvivesLockedGatewayRestartAndStateLayout, - assertHermesReloadRollback, assertHermesRemovalSurvivesGatewayRestart, } from "./mcp-bridge-hermes-lifecycle.ts"; import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv } from "./mcp-bridge-onboard-env.ts"; import { MCP_BRIDGE_PHASES } from "./mcp-bridge-phases.ts"; import { retryAfterHermesRestartTransportFailure } from "./mcp-bridge-reliability.ts"; import { - assertManagedMcpPolicySurvivedRemoval, buildMcpDnsRebindingProbeScript, - captureManagedMcpPolicy, - expectExitNonZero, hostAddressForSandbox, hostPrivateAddressForSandbox, isExpectedMcpCurlPolicyDenial, @@ -77,10 +75,12 @@ const COMPATIBLE_MODEL = "mock/mcp-bridge"; const TOOL_CHALLENGE = "nemoclaw-authenticated-mcp-proof"; const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); const selectedMcpBridgeShard = resolveMcpBridgeShard(); + function mcpBridgeShardTest(shard: McpBridgeShard) { return selectedMcpBridgeShard === shard ? e2eTest : e2eTest.skip; } const test = mcpBridgeShardTest("openclaw"); + type McpAgent = "openclaw" | "hermes" | "langchain-deepagents-code"; type McpAdapter = "mcporter" | "hermes-config" | "deepagents-config"; const MCP_MUTATION_TIMEOUT_MS: Record = { @@ -90,6 +90,19 @@ const MCP_MUTATION_TIMEOUT_MS: Record = { }; const MCP_BRIDGE_ALREADY_ABSENT = /No MCP servers are registered|No MCP server '.+' is registered|MCP server '.+' not found/iu; + +function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { + expect( + result.exitCode, + `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ).not.toBe(0); + expect(resultText(result)).toMatch(pattern); +} + +function parseCurrentPolicy(raw: string): string { + return parseOpenShellPolicy(raw).yamlBody; +} + async function cleanupMcpBridge( host: HostCliClient, sandboxName: string, @@ -107,6 +120,7 @@ async function cleanupMcpBridge( `cleanup MCP bridge ${server} on sandbox ${sandboxName}`, ); } + async function onboardAgent( host: HostCliClient, cleanup: CleanupRegistry, @@ -144,6 +158,7 @@ async function onboardAgent( ); expectExitZero(result, `onboard ${options.agent} sandbox for MCP bridge`); } + async function assertSecretAbsentFromSandbox( sandbox: SandboxClient, sandboxName: string, @@ -174,7 +189,6 @@ async function assertAdapterDnsRebindingDenied( artifactPrefix: string; sandboxName: string; secretPaths: string[]; - survivingMcpUrl: string; }, ): Promise { const rebindMcp = await startFakeMcpHttpsServer({ secret: REBIND_HOST_SECRET }); @@ -193,14 +207,6 @@ async function assertAdapterDnsRebindingDenied( cleanup.add(`restore ${options.artifactPrefix} DNS rebinding hosts fixture`, () => restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture), ); - const survivingPolicyBeforeAddResult = await captureManagedMcpPolicy(sandbox, { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-before-add`, - label: `${options.artifactPrefix} captures the surviving MCP policy before adding the rebinding route`, - policyKey: SERVER_POLICY_KEY, - sandboxName: options.sandboxName, - url: options.survivingMcpUrl, - }); - const survivingPolicyBeforeAdd = survivingPolicyBeforeAddResult.policy; await remapDnsRebindingHostname( host, options.sandboxName, @@ -255,14 +261,19 @@ async function assertAdapterDnsRebindingDenied( policy: { gatewayPresent: true }, adapter: { registered: true }, }); - const rebindingPolicy = await captureManagedMcpPolicy(sandbox, { + const policy = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-policy-pinned-public-ip`, - label: `${options.artifactPrefix} validates the add-time DNS pin`, - policyKey: REBIND_POLICY_KEY, - sandboxName: options.sandboxName, - url: rebindMcpUrl, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, }); - expect(rebindingPolicy.policy.endpoints?.[0]).toMatchObject({ + expectExitZero(policy, `${options.artifactPrefix} inspects add-time DNS pin`); + const policyJson = YAML.parse(parseCurrentPolicy(resultText(policy))) as { + network_policies?: Record< + string, + { endpoints?: Array<{ host?: string; allowed_ips?: string[] }> } + >; + }; + expect(policyJson.network_policies?.[REBIND_POLICY_KEY]?.endpoints?.[0]).toMatchObject({ host: REBIND_HOSTNAME, allowed_ips: [REBIND_PUBLIC_IP], }); @@ -311,18 +322,6 @@ async function assertAdapterDnsRebindingDenied( timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], }); expectExitZero(remove, `${options.artifactPrefix} removes DNS rebinding route after proof`); - const survivingPolicyAfterRemoveResult = await captureManagedMcpPolicy(sandbox, { - artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-after-remove`, - label: `${options.artifactPrefix} inspects MCP policy after removing the rebinding route`, - policyKey: SERVER_POLICY_KEY, - sandboxName: options.sandboxName, - url: options.survivingMcpUrl, - }); - assertManagedMcpPolicySurvivedRemoval( - survivingPolicyBeforeAdd, - survivingPolicyAfterRemoveResult, - REBIND_POLICY_KEY, - ); } async function addBridgeAndReadStatus( host: HostCliClient, @@ -355,6 +354,7 @@ async function addBridgeAndReadStatus( }, ); expectExitZero(add, `${options.artifactPrefix} mcp add fake server`); + const status = await host.nemoclaw( [options.sandboxName, "mcp", "status", SERVER_NAME, "--json"], { @@ -981,7 +981,6 @@ test("mcp-bridge", { artifactPrefix: "openclaw", sandboxName: OPENCLAW_SANDBOX_NAME, secretPaths: ["/sandbox/.openclaw", "/sandbox/.mcp.json"], - survivingMcpUrl: mcpUrl, }); const requestCountBeforeAllowedNodeProof = fakeMcp.requests.length; @@ -1172,13 +1171,6 @@ mcpBridgeShardTest("hermes")( challenge: TOOL_CHALLENGE, resultToken: hermesResult, }); - const assertHermesToolCall = (artifactName: string) => - assertRealAdapterToolCall(sandbox, fakeMcp, { - agent: "hermes", - sandboxName: HERMES_SANDBOX_NAME, - resultToken: hermesResult, - artifactName, - }); cleanup.add("stop fake Hermes MCP HTTPS server", () => fakeMcp.close()); const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ cleanup, @@ -1198,6 +1190,7 @@ mcpBridgeShardTest("hermes")( cleanup.add("remove Hermes MCP bridge", () => cleanupMcpBridge(host, HERMES_SANDBOX_NAME, SERVER_NAME, "hermes-config"), ); + progress.phase("configure and inspect the Hermes MCP bridge"); await assertConcurrentAddSerialized(host, cleanup, { sandboxName: HERMES_SANDBOX_NAME, @@ -1205,6 +1198,7 @@ mcpBridgeShardTest("hermes")( expectedAdapter: "hermes-config", artifactPrefix: "hermes", }); + const initialDiscoveryOffset = fakeMcp.requests.length; const providerName = await addBridgeAndReadStatus(host, { sandboxName: HERMES_SANDBOX_NAME, @@ -1239,8 +1233,6 @@ mcpBridgeShardTest("hermes")( HERMES_SANDBOX_NAME, mcpUrl, ); - await assertHermesToolCall("hermes-real-mcp-tool-call-immediately-after-shields-down"); - await assertHermesReloadRollback(sandbox, HERMES_SANDBOX_NAME, mcpUrl); await assertSecretAbsentFromSandbox( sandbox, HERMES_SANDBOX_NAME, @@ -1259,12 +1251,15 @@ mcpBridgeShardTest("hermes")( artifactPrefix: "hermes", sandboxName: HERMES_SANDBOX_NAME, secretPaths: ["/sandbox/.hermes"], - survivingMcpUrl: mcpUrl, }); - await assertHermesToolCall("hermes-real-mcp-tool-call-after-dns-rebinding-remove"); const survivingDiscoveryOffset = fakeMcp.requests.length; await restartBridgeWithoutHostSecret(host, HERMES_SANDBOX_NAME, "hermes"); - await assertHermesToolCall("hermes-real-mcp-tool-call-after-rediscovery-restart"); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + resultToken: hermesResult, + artifactName: "hermes-real-mcp-tool-call-after-rediscovery-restart", + }); await assertAuthenticatedMcpRediscovery(survivingMcp, survivingDiscoveryOffset); fakeMcp.setSecret(ROTATED_HOST_SECRET); await rotateBridgeCredential(host, HERMES_SANDBOX_NAME, "hermes"); @@ -1423,7 +1418,6 @@ mcpBridgeShardTest("deepagents")( artifactPrefix: "deepagents", sandboxName: DEEPAGENTS_SANDBOX_NAME, secretPaths: ["/sandbox/.deepagents"], - survivingMcpUrl: mcpUrl, }); progress.phase("exercise lifecycle and confirm Deep Agents bridge removal"); await assertRealAdapterToolCall(sandbox, fakeMcp, { diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts index c2e6017ee9f..4346ae4b62b 100644 --- a/test/e2e/support/mcp-bridge-sandbox.test.ts +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -12,7 +12,6 @@ import YAML from "yaml"; import { testTimeout } from "../../helpers/timeouts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { - assertManagedMcpPolicySurvivedRemoval, buildMcpDnsRebindingProbeScript, hostAddressForSandbox, hostPrivateAddressForSandbox, @@ -303,34 +302,45 @@ network_policies: expect(contractSource).not.toContain("assertAdapterDnsRebindingDenied"); }); - it("accepts an unchanged surviving policy only after the unrelated policy is absent", () => { - const survivingPolicy = { - endpoints: [{ host: "surviving.example.test", allowed_ips: ["203.0.113.10"] }], - }; + it("runs the zero-upstream rebinding proof for all three adapters", () => { + const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); - expect(() => - assertManagedMcpPolicySurvivedRemoval( - survivingPolicy, - { - networkPolicies: { mcp_bridge_surviving: survivingPolicy }, - policy: survivingPolicy, - }, - "mcp_bridge_rebinding", - ), - ).not.toThrow(); - expect(() => - assertManagedMcpPolicySurvivedRemoval( - survivingPolicy, - { - networkPolicies: { - mcp_bridge_rebinding: { endpoints: [] }, - mcp_bridge_surviving: survivingPolicy, - }, - policy: survivingPolicy, - }, - "mcp_bridge_rebinding", - ), - ).toThrow(); + expect(source.match(/await assertAdapterDnsRebindingDenied/g)).toHaveLength(3); + for (const adapter of [ + 'adapter: "mcporter"', + 'adapter: "hermes-config"', + 'adapter: "deepagents-config"', + ]) { + expect(source).toContain(adapter); + } + expect(source).toContain("rebound request must not reach the upstream MCP server"); + expect(source).toContain(").toHaveLength(0);"); + }); + + it("captures the Hermes rediscovery offset after route removal and before restart", () => { + const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); + const denialProof = source.indexOf("rebound request must not reach the upstream MCP server"); + const restore = source.indexOf("await restoreDnsRebindingHostsFixture", denialProof); + const remove = source.indexOf("const remove = await host.nemoclaw", denialProof); + const hermesTest = source.indexOf('mcpBridgeShardTest("hermes")'); + const rebinding = source.indexOf("await assertAdapterDnsRebindingDenied", hermesTest); + const offset = source.indexOf( + "const survivingDiscoveryOffset = fakeMcp.requests.length", + rebinding, + ); + const restart = source.indexOf("await restartBridgeWithoutHostSecret", offset); + const toolCall = source.indexOf("await assertRealAdapterToolCall", restart); + const rediscovery = source.indexOf("await assertAuthenticatedMcpRediscovery", toolCall); + + expect(denialProof).toBeGreaterThanOrEqual(0); + expect(restore).toBeGreaterThan(denialProof); + expect(remove).toBeGreaterThan(restore); + expect(rebinding).toBeGreaterThan(hermesTest); + expect(offset).toBeGreaterThan(rebinding); + expect(restart).toBeGreaterThan(offset); + expect(toolCall).toBeGreaterThan(restart); + expect(rediscovery).toBeGreaterThan(toolCall); + expect(source).toContain("Hermes MCP rediscovery after explicit restart"); }); it("restores host DNS strictly while treating the ephemeral sandbox as best effort", async () => { diff --git a/test/permissive-runtime.test.ts b/test/permissive-runtime.test.ts index 01b3f91212b..504d9a2898d 100644 --- a/test/permissive-runtime.test.ts +++ b/test/permissive-runtime.test.ts @@ -7,10 +7,7 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import YAML from "yaml"; -import { - buildRuntimePermissivePolicy, - type ExactManagedMcpPolicy, -} from "../src/lib/shields/permissive-runtime.js"; +import { buildRuntimePermissivePolicy } from "../src/lib/shields/permissive-runtime.js"; const BASE_PERMISSIVE = YAML.stringify({ filesystem_policy: { @@ -49,51 +46,6 @@ afterEach(() => { }); describe("buildRuntimePermissivePolicy (#3942)", () => { - it("preserves exact managed MCP entries without copying unrelated live egress (#7952)", () => { - const managedPolicy: ExactManagedMcpPolicy = { - key: "mcp_bridge_alpha", - networkPolicy: { - endpoints: [{ host: "alpha.example.com", port: 443, protocol: "mcp" }], - binaries: [{ path: "/opt/hermes/.venv/bin/python*" }], - }, - policyName: "mcp-bridge-alpha", - server: "alpha", - }; - const liveYaml = YAML.stringify({ - filesystem_policy: { read_write: ["/proc"] }, - network_policies: { - mcp_bridge_alpha: managedPolicy.networkPolicy, - unrelated_live_entry: { - endpoints: [{ host: "unrelated.example.com", port: 443 }], - }, - }, - }); - - const out = buildRuntimePermissivePolicy("/unused-base.yaml", { - livePolicyYaml: liveYaml, - managedMcpPolicies: [managedPolicy], - readBasePolicy: () => - YAML.stringify({ - ...YAML.parse(BASE_PERMISSIVE), - network_policies: { - permissive_baseline: { - endpoints: [{ host: "*", port: 443 }], - }, - }, - }), - }); - trackTempForCleanup(out, "/unused-base.yaml"); - - const result = YAML.parse(fs.readFileSync(out, "utf-8")); - expect(result.network_policies).toMatchObject({ - mcp_bridge_alpha: managedPolicy.networkPolicy, - permissive_baseline: { - endpoints: [{ host: "*", port: 443 }], - }, - }); - expect(result.network_policies).not.toHaveProperty("unrelated_live_entry"); - }); - it("preserves /proc when the live GPU sandbox has it in read_write", () => { const liveYaml = YAML.stringify({ filesystem_policy: { @@ -215,25 +167,6 @@ describe("buildRuntimePermissivePolicy (#3942)", () => { expect(out).toBe(basePath); }); - it("fails closed when the base cannot be read with managed MCP policies active (#7952)", () => { - expect(() => - buildRuntimePermissivePolicy("/path/to/static.yaml", { - livePolicyYaml: "version: 1\nnetwork_policies: {}\n", - managedMcpPolicies: [ - { - key: "mcp_bridge_alpha", - networkPolicy: {}, - policyName: "mcp-bridge-alpha", - server: "alpha", - }, - ], - readBasePolicy: () => { - throw new Error("ENOENT"); - }, - }), - ).toThrow(/Cannot read the Shields-down policy/); - }); - it("returns the static base path when base YAML is unparseable", () => { const basePath = "/path/to/static.yaml"; const liveYaml = YAML.stringify({ diff --git a/test/policy-mutation-read-discovery.test.ts b/test/policy-mutation-read-discovery.test.ts index c6e2ac102e5..94c580ecc4a 100644 --- a/test/policy-mutation-read-discovery.test.ts +++ b/test/policy-mutation-read-discovery.test.ts @@ -9,10 +9,45 @@ import { describe, expect, it } from "vitest"; import { auditOpenShellPolicyMutationReads, + classifyPolicyReadCalls, countPolicyReadCalls, discoverPolicyReadSites, } from "../scripts/checks/openshell-policy-mutation-read.mts"; +function createShieldsAuditFixture(source: string): string { + const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shields-read-audit-")); + const sourcePath = path.join(repoRoot, "src", "lib", "shields", "index.ts"); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync(sourcePath, source); + return repoRoot; +} + +function shieldsAuditSource( + downRead = "runCapture(buildPolicyGetCommand(sandboxName));", + extraRead = "", + aliasDeclaration = "", + ignoreErrors = true, +): string { + const downFunction = ignoreErrors + ? [ + "function shieldsDownWithoutHostLock(sandboxName: string) {", + " try {", + ` ${downRead}`, + " } catch {", + " return;", + " }", + "}", + ] + : ["function shieldsDownWithoutHostLock(sandboxName: string) {", ` ${downRead}`, "}"]; + return [ + 'const { buildPolicyGetCommand, buildPolicyGetFullCommand } = require("../policy");', + 'const { runCapture } = require("../runner");', + aliasDeclaration, + ...downFunction, + extraRead, + ].join("\n"); +} + describe("OpenShell policy mutation read discovery (#6921)", () => { it("counts canonical builder bindings and direct argv reads", () => { const source = [ @@ -36,6 +71,151 @@ describe("OpenShell policy mutation read discovery (#6921)", () => { expect(countPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toBe(7); }); + it("classifies each read by function, policy view, and failure handling", () => { + const source = [ + 'import { buildPolicyGetCommand as buildBase, buildPolicyGetFullCommand as buildFull } from "./policy/commands";', + "let readPolicy = buildBase;", + "function preserve(sandboxName: string) {", + " return runCapture(readPolicy(sandboxName));", + "}", + "const ignore = (sandboxName: string) =>", + " runCapture(buildFull(sandboxName), { ignoreError: true });", + "function unknown(sandboxName: string) {", + " return execute(buildBase(sandboxName));", + "}", + ].join("\n"); + + expect(classifyPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toEqual([ + { site: "preserve", view: "base", failureHandling: "error-preserving" }, + { site: "ignore", view: "full", failureHandling: "ignore-error" }, + { site: "unknown", view: "base", failureHandling: "unclassified" }, + ]); + }); + + it("discovers a full builder destructured from a canonical namespace import", () => { + const source = [ + 'import * as policy from "./policy/commands";', + "function inspectPolicy(sandboxName: string) {", + " const { buildPolicyGetFullCommand: readPolicy } = policy;", + " return runCapture(readPolicy(sandboxName));", + "}", + ].join("\n"); + + expect(classifyPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toEqual([ + { site: "inspectPolicy", view: "full", failureHandling: "error-preserving" }, + ]); + }); + + it("discovers a literal computed builder destructured from a policy namespace", () => { + const source = [ + 'import * as policy from "./policy/commands";', + "function inspectPolicy(sandboxName: string) {", + ' const { ["buildPolicyGetFullCommand"]: readPolicy } = policy;', + " return runCapture(readPolicy(sandboxName));", + "}", + ].join("\n"); + + expect(classifyPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toEqual([ + { site: "inspectPolicy", view: "full", failureHandling: "error-preserving" }, + ]); + }); + + it("follows an object-rest policy namespace alias", () => { + const source = [ + 'import * as policy from "./policy/commands";', + "function inspectPolicy(sandboxName: string) {", + " return runCapture(policyAlias.buildPolicyGetFullCommand(sandboxName));", + "}", + "const { ...firstPolicyAlias } = policy;", + "const policyAlias = firstPolicyAlias;", + ].join("\n"); + + expect(classifyPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toEqual([ + { site: "inspectPolicy", view: "full", failureHandling: "error-preserving" }, + ]); + }); + + it("follows chained namespace aliases before destructuring a builder", () => { + const source = [ + 'import * as policy from "./policy/commands";', + "function inspectPolicy(sandboxName: string) {", + " const { buildPolicyGetCommand: readPolicy } = policyAlias;", + " return runCapture(readPolicy(sandboxName));", + "}", + "const firstPolicyAlias = policy;", + "const policyAlias = firstPolicyAlias;", + ].join("\n"); + + expect(classifyPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toEqual([ + { site: "inspectPolicy", view: "base", failureHandling: "error-preserving" }, + ]); + }); + + it("classifies empty and fallback catches as ignored runner failures", () => { + const source = [ + 'import { buildPolicyGetCommand } from "./policy/commands";', + "function inspectPolicy(sandboxName: string) {", + " try {", + " return runCapture(buildPolicyGetCommand(sandboxName));", + " } catch {}", + "}", + "function inspectFallbackPolicy(sandboxName: string) {", + ' let rawPolicy = "";', + " try {", + " rawPolicy = runCapture(buildPolicyGetCommand(sandboxName));", + " } catch {", + ' rawPolicy = "";', + " }", + " return rawPolicy;", + "}", + ].join("\n"); + + expect(classifyPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toEqual([ + { site: "inspectPolicy", view: "base", failureHandling: "ignore-error" }, + { + site: "inspectFallbackPolicy", + view: "base", + failureHandling: "ignore-error", + }, + ]); + }); + + it("leaves conditional catch exits unclassified", () => { + const source = [ + 'import { buildPolicyGetCommand } from "./policy/commands";', + "function inspectPolicy(sandboxName: string) {", + " try {", + " return runCapture(buildPolicyGetCommand(sandboxName));", + " } catch (error) {", + " switch (String(error)) {", + ' case "retry":', + " throw error;", + " }", + " }", + "}", + "function inspectPolicyWithEarlyExit(sandboxName: string) {", + " try {", + " return runCapture(buildPolicyGetCommand(sandboxName));", + " } catch (error) {", + " switch (String(error)) {", + ' case "ignore":', + ' return "";', + " }", + " throw error;", + " }", + "}", + ].join("\n"); + + expect(classifyPolicyReadCalls(source, "/repo/src/lib/fixture.ts", "/repo")).toEqual([ + { site: "inspectPolicy", view: "base", failureHandling: "unclassified" }, + { + site: "inspectPolicyWithEarlyExit", + view: "base", + failureHandling: "unclassified", + }, + ]); + }); + it("ignores similarly named calls without canonical policy bindings", () => { const source = [ 'import { buildPolicyGetCommand as unrelated } from "./fixture-helpers";', @@ -155,8 +335,16 @@ describe("OpenShell policy mutation read discovery (#6921)", () => { try { expect(discoverPolicyReadSites(repoRoot)).toEqual([ - { relativePath: "nemoclaw/src/new-policy-diagnostic.ts", readCalls: 1 }, - { relativePath: "src/lib/new-policy-mutation.ts", readCalls: 1 }, + { + relativePath: "nemoclaw/src/new-policy-diagnostic.ts", + readCalls: 1, + reads: [{ site: "", view: "full", failureHandling: "error-preserving" }], + }, + { + relativePath: "src/lib/new-policy-mutation.ts", + readCalls: 1, + reads: [{ site: "", view: "base", failureHandling: "error-preserving" }], + }, ]); expect(auditOpenShellPolicyMutationReads(repoRoot)).toEqual( expect.arrayContaining([ @@ -168,4 +356,100 @@ describe("OpenShell policy mutation read discovery (#6921)", () => { fs.rmSync(repoRoot, { recursive: true, force: true }); } }); + + it("rejects an aliased full read substituted at an approved Shields site", () => { + const source = shieldsAuditSource( + "runCapture(readPolicy(sandboxName));", + "", + "const readPolicy = buildPolicyGetFullCommand;", + ); + const repoRoot = createShieldsAuditFixture(source); + + try { + expect( + countPolicyReadCalls(source, path.join(repoRoot, "src/lib/shields/index.ts"), repoRoot), + ).toBe(1); + expect(auditOpenShellPolicyMutationReads(repoRoot)).toEqual( + expect.arrayContaining([ + expect.stringContaining("shieldsDownWithoutHostLock (full, ignore-error)"), + ]), + ); + } finally { + fs.rmSync(repoRoot, { recursive: true, force: true }); + } + }); + + it("rejects a destructured full builder substituted at an approved Shields site", () => { + const source = shieldsAuditSource( + "runCapture(readPolicy(sandboxName));", + "", + [ + 'const policyNamespace = require("../policy");', + "const policyAlias = policyNamespace;", + "const { buildPolicyGetFullCommand: readPolicy } = policyAlias;", + ].join("\n"), + ); + const repoRoot = createShieldsAuditFixture(source); + + try { + expect( + countPolicyReadCalls(source, path.join(repoRoot, "src/lib/shields/index.ts"), repoRoot), + ).toBe(1); + expect(auditOpenShellPolicyMutationReads(repoRoot)).toEqual( + expect.arrayContaining([ + expect.stringContaining("shieldsDownWithoutHostLock (full, ignore-error)"), + ]), + ); + } finally { + fs.rmSync(repoRoot, { recursive: true, force: true }); + } + }); + + it("rejects error-preserving behavior substituted at an approved Shields site", () => { + const source = shieldsAuditSource( + "runCapture(buildPolicyGetCommand(sandboxName));", + "", + "", + false, + ); + const repoRoot = createShieldsAuditFixture(source); + + try { + expect( + countPolicyReadCalls(source, path.join(repoRoot, "src/lib/shields/index.ts"), repoRoot), + ).toBe(1); + expect(auditOpenShellPolicyMutationReads(repoRoot)).toEqual( + expect.arrayContaining([ + expect.stringContaining("shieldsDownWithoutHostLock (base, error-preserving)"), + ]), + ); + } finally { + fs.rmSync(repoRoot, { recursive: true, force: true }); + } + }); + + it("rejects moving a Shields read to an unapproved site without changing the count", () => { + const source = shieldsAuditSource( + "return;", + [ + "function inspectAnotherPolicy(sandboxName: string) {", + " runCapture(buildPolicyGetCommand(sandboxName));", + "}", + ].join("\n"), + ); + const repoRoot = createShieldsAuditFixture(source); + + try { + expect( + countPolicyReadCalls(source, path.join(repoRoot, "src/lib/shields/index.ts"), repoRoot), + ).toBe(1); + expect(auditOpenShellPolicyMutationReads(repoRoot)).toEqual( + expect.arrayContaining([ + expect.stringContaining("inspectAnotherPolicy (base, error-preserving)"), + ]), + ); + } finally { + fs.rmSync(repoRoot, { recursive: true, force: true }); + } + }); }); From 58bc8cbd53d0999bb8bc42b59aeab45781ca87ba Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 11:38:38 -0400 Subject: [PATCH 7/8] test(shields): stabilize recovery regressions Signed-off-by: Julie Yaunches --- src/lib/shields/flow.test.ts | 68 ++++++++++++++------------- src/lib/shields/timer-process.test.ts | 20 +++----- test/mcp-lifecycle-lock.test.ts | 2 +- 3 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 1a2b99d81e1..a0c76bd5705 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -436,12 +436,20 @@ describe("shields command flow", () => { const harness = createHarness({ dockerExecFileSync: (argv: unknown) => { const args = Array.isArray(argv) ? argv.map(String) : []; - if (args.includes("sha256sum")) return `${"a".repeat(64)} ${String(args.at(-1))}\n`; - if (args.includes("lsattr")) return `----i---------e----- ${String(args.at(-1))}\n`; - if (!args.includes("stat")) return ""; - if (args.at(-1) === "/sandbox") return "1775 root:sandbox\n"; - if (args.at(-1) === "/sandbox/.openclaw") return "755 root:root\n"; - return "444 root:root\n"; + switch (true) { + case args.includes("sha256sum"): + return `${"a".repeat(64)} ${String(args.at(-1))}\n`; + case args.includes("lsattr"): + return `----i---------e----- ${String(args.at(-1))}\n`; + case !args.includes("stat"): + return ""; + case args.at(-1) === "/sandbox": + return "1775 root:sandbox\n"; + case args.at(-1) === "/sandbox/.openclaw": + return "755 root:root\n"; + default: + return "444 root:root\n"; + } }, }); const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath(sandboxName, stateDir)}.containment`; @@ -744,31 +752,27 @@ describe("shields command flow", () => { expect(ownerStartIdentity).toBeTypeOf("string"); const processKillSpy = vi.spyOn(process, "kill"); const nativeAtomicsWait = Atomics.wait; - let releaseObserved = false; - vi.spyOn(Atomics, "wait").mockImplementation((( - _typedArray: Int32Array, - _index: number, - _value: number, - _timeout?: number, - ) => { - if (!releaseObserved) { - const ownerState = timerControl.readProcessState(owner.pid); - expect(ownerState).not.toBeNull(); - expect(ownerState?.startsWith("Z")).toBe(false); - expect(fs.existsSync(lockPath)).toBe(true); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); - fs.writeFileSync(releasePath, "release"); - const releaseDeadline = Date.now() + 5_000; - const waitBuffer = new Int32Array(new SharedArrayBuffer(4)); - while (fs.existsSync(lockPath) && Date.now() < releaseDeadline) { - nativeAtomicsWait(waitBuffer, 0, 0, 10); - } - expect(fs.existsSync(lockPath)).toBe(false); - releaseObserved = true; - } - return "timed-out"; - }) as typeof Atomics.wait); + const atomicsWaitSpy = vi + .spyOn(Atomics, "wait") + .mockImplementationOnce( + (_typedArray: Int32Array, _index: number, _value: number, _timeout?: number) => { + const ownerState = timerControl.readProcessState(owner.pid); + expect(ownerState).not.toBeNull(); + expect(ownerState?.startsWith("Z")).toBe(false); + expect(fs.existsSync(lockPath)).toBe(true); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); + fs.writeFileSync(releasePath, "release"); + const releaseDeadline = Date.now() + 5_000; + const waitBuffer = new Int32Array(new SharedArrayBuffer(4)); + while (fs.existsSync(lockPath) && Date.now() < releaseDeadline) { + nativeAtomicsWait(waitBuffer, 0, 0, 10); + } + expect(fs.existsSync(lockPath)).toBe(false); + return "timed-out"; + }, + ) + .mockReturnValue("timed-out"); const harness = createHarness({ dockerExecFileSync: (argv: unknown) => { const args = Array.isArray(argv) ? argv.map(String) : []; @@ -791,7 +795,7 @@ describe("shields command flow", () => { harness.shieldsStatus(sandboxName); - expect(releaseObserved).toBe(true); + expect(atomicsWaitSpy).toHaveBeenCalled(); expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); await vi.waitFor( diff --git a/src/lib/shields/timer-process.test.ts b/src/lib/shields/timer-process.test.ts index 84bd3a96560..a0977495730 100644 --- a/src/lib/shields/timer-process.test.ts +++ b/src/lib/shields/timer-process.test.ts @@ -58,13 +58,9 @@ describe("detached Shields timer process", () => { stdio: ["ignore", "ignore", "pipe", "ipc"], }, ); - const childPid = child.pid; - if (childPid === undefined) throw new Error("Detached timer did not start"); - let childExited = false; + expect(child.pid).toBeTypeOf("number"); + const childPid = child.pid as number; let childStderr = ""; - child.once("exit", () => { - childExited = true; - }); child.stderr?.setEncoding("utf-8"); child.stderr?.on("data", (chunk: string) => { childStderr += chunk; @@ -108,13 +104,11 @@ describe("detached Shields timer process", () => { cause: error, }); } finally { - if (!childExited) { - const forcedExit = once(child, "exit", { signal: AbortSignal.timeout(1_000) }).catch( - () => undefined, - ); - child.kill("SIGKILL"); - await forcedExit; - } + child.kill("SIGKILL"); + await vi.waitFor( + () => expect(child.exitCode !== null || child.signalCode !== null).toBe(true), + { timeout: 1_000, interval: 10 }, + ); } }, ); diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index 39036c8f6d4..2fddf6a91c7 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -609,7 +609,7 @@ const releasePath = process.argv[3]; lifecycleLock.withMcpLifecycleLock( "alpha", () => undefined, - options({ timeoutMs: 50, corruptLockGraceMs: 20 }), + options({ timeoutMs: 1_000, corruptLockGraceMs: 20 }), ), ).rejects.toThrow("Sandbox mutation containment is active"); expect(fs.existsSync(lockPath)).toBe(true); From eb0184e004f24c086afc4017ba2e7ec21752b234 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 11:42:55 -0400 Subject: [PATCH 8/8] test(shields): type overloaded wait mock Signed-off-by: Julie Yaunches --- src/lib/shields/flow.test.ts | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index a0c76bd5705..73dc30cb306 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -754,24 +754,22 @@ describe("shields command flow", () => { const nativeAtomicsWait = Atomics.wait; const atomicsWaitSpy = vi .spyOn(Atomics, "wait") - .mockImplementationOnce( - (_typedArray: Int32Array, _index: number, _value: number, _timeout?: number) => { - const ownerState = timerControl.readProcessState(owner.pid); - expect(ownerState).not.toBeNull(); - expect(ownerState?.startsWith("Z")).toBe(false); - expect(fs.existsSync(lockPath)).toBe(true); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); - fs.writeFileSync(releasePath, "release"); - const releaseDeadline = Date.now() + 5_000; - const waitBuffer = new Int32Array(new SharedArrayBuffer(4)); - while (fs.existsSync(lockPath) && Date.now() < releaseDeadline) { - nativeAtomicsWait(waitBuffer, 0, 0, 10); - } - expect(fs.existsSync(lockPath)).toBe(false); - return "timed-out"; - }, - ) + .mockImplementationOnce(() => { + const ownerState = timerControl.readProcessState(owner.pid); + expect(ownerState).not.toBeNull(); + expect(ownerState?.startsWith("Z")).toBe(false); + expect(fs.existsSync(lockPath)).toBe(true); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); + fs.writeFileSync(releasePath, "release"); + const releaseDeadline = Date.now() + 5_000; + const waitBuffer = new Int32Array(new SharedArrayBuffer(4)); + while (fs.existsSync(lockPath) && Date.now() < releaseDeadline) { + nativeAtomicsWait(waitBuffer, 0, 0, 10); + } + expect(fs.existsSync(lockPath)).toBe(false); + return "timed-out"; + }) .mockReturnValue("timed-out"); const harness = createHarness({ dockerExecFileSync: (argv: unknown) => {