diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 557999701dc..9af2f6aac85 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 - import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -29,9 +28,14 @@ import { shouldCleanupGatewayAfterDestroy, shouldStopHostServicesAfterDestroy, } from "../../domain/sandbox/destroy"; +import { resolveNemoclawStateDir } from "../../state/paths"; +import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; import { G, R, YW } from "../../cli/terminal-style"; -type DockerRmi = (tag: string, opts?: { ignoreError?: boolean }) => { status: number | null }; +type DockerRmi = ( + tag: string, + opts?: { ignoreError?: boolean }, +) => { status: number | null }; type RemoveSandboxImageDeps = { getSandbox?: typeof registry.getSandbox; @@ -56,20 +60,47 @@ export type CleanupSandboxServicesDeps = { rmSync?: typeof fs.rmSync; }; +type ShieldsTimerNeutralizeResult = { + warnings?: string[]; +}; + +type CleanupShieldsDestroyArtifactsDeps = { + killShieldsTimer?: ( + sandboxName: string, + ) => ShieldsTimerNeutralizeResult | void; + rmSync?: typeof fs.rmSync; + stateDir?: string; + warn?: (message: string) => void; +}; + +type RemoveShieldsStateDeps = { + rmSync?: typeof fs.rmSync; + warn?: (message: string) => void; +}; + const NEMOCLAW_GATEWAY_NAME = "nemoclaw"; const DASHBOARD_FORWARD_PORT = String(DASHBOARD_PORT); function dockerDriverGatewayPidFile(): string { const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; - const stateDir = configured && configured.trim() - ? path.resolve(configured.trim()) - : path.join(os.homedir(), ".local", "state", "nemoclaw", "openshell-docker-gateway"); + const stateDir = + configured && configured.trim() + ? path.resolve(configured.trim()) + : path.join( + os.homedir(), + ".local", + "state", + "nemoclaw", + "openshell-docker-gateway", + ); return path.join(stateDir, "openshell-gateway.pid"); } function isDockerDriverGatewayPid(pid: number): boolean { try { - const cmdline = fs.readFileSync(`/proc/${pid}/cmdline`, "utf-8").replace(/\0/g, " "); + const cmdline = fs + .readFileSync(`/proc/${pid}/cmdline`, "utf-8") + .replace(/\0/g, " "); return cmdline.includes("openshell-gateway") || cmdline.includes("openclaw-gateway"); } catch { return false; @@ -103,10 +134,16 @@ function stopDockerDriverGatewayProcess(): void { function cleanupGatewayAfterLastSandbox(): void { const { runOpenshell } = require("../../adapters/openshell/runtime") as { - runOpenshell: (args: string[], opts?: Record) => { status: number | null }; + runOpenshell: ( + args: string[], + opts?: Record, + ) => { status: number | null }; }; const { dockerRemoveVolumesByPrefix } = require("../../adapters/docker") as { - dockerRemoveVolumesByPrefix: (prefix: string, opts?: { ignoreError?: boolean }) => void; + dockerRemoveVolumesByPrefix: ( + prefix: string, + opts?: { ignoreError?: boolean }, + ) => void; }; runOpenshell(["forward", "stop", DASHBOARD_FORWARD_PORT], { @@ -120,10 +157,13 @@ function cleanupGatewayAfterLastSandbox(): void { stopStaleDashboardListeners(); if (process.platform === "linux") { stopDockerDriverGatewayProcess(); - const removeResult = runOpenshell(["gateway", "remove", NEMOCLAW_GATEWAY_NAME], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); + const removeResult = runOpenshell( + ["gateway", "remove", NEMOCLAW_GATEWAY_NAME], + { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }, + ); if (removeResult.status !== 0) { runOpenshell(["gateway", "destroy", "-g", NEMOCLAW_GATEWAY_NAME], { ignoreError: true, @@ -131,7 +171,9 @@ function cleanupGatewayAfterLastSandbox(): void { }); } } else { - runOpenshell(["gateway", "destroy", "-g", NEMOCLAW_GATEWAY_NAME], { ignoreError: true }); + runOpenshell(["gateway", "destroy", "-g", NEMOCLAW_GATEWAY_NAME], { + ignoreError: true, + }); } dockerRemoveVolumesByPrefix(`openshell-cluster-${NEMOCLAW_GATEWAY_NAME}`, { ignoreError: true, @@ -168,7 +210,9 @@ async function resolveCleanupGatewayDecision( console.log( " Also destroy the shared NemoClaw gateway (port forward, gateway pod, cluster volumes)?", ); - console.log(" Saying 'no' keeps the gateway so the next 'nemoclaw onboard' is faster."); + console.log( + " Saying 'no' keeps the gateway so the next 'nemoclaw onboard' is faster.", + ); const answer = await askPrompt( " Type 'yes' to destroy the gateway, or press Enter to keep it [y/N]: ", ); @@ -210,9 +254,10 @@ export function cleanupSandboxServices( const unloadOllamaModels = deps.unloadOllamaModels ?? (() => { - const { unloadOllamaModels: unload } = require("../../inference/ollama/proxy") as { - unloadOllamaModels: () => void; - }; + const { unloadOllamaModels: unload } = + require("../../inference/ollama/proxy") as { + unloadOllamaModels: () => void; + }; unload(); }); const runOpenshell = @@ -240,14 +285,22 @@ export function cleanupSandboxServices( } try { - rmSync(`/tmp/nemoclaw-services-${sandboxName}`, { recursive: true, force: true }); + rmSync(`/tmp/nemoclaw-services-${sandboxName}`, { + recursive: true, + force: true, + }); } catch { // PID directory may not exist — ignore. } // Delete messaging providers created during onboard. Suppress stderr so // "! Provider not found" noise doesn't appear when messaging was never configured. - for (const suffix of ["telegram-bridge", "discord-bridge", "slack-bridge", "slack-app"]) { + for (const suffix of [ + "telegram-bridge", + "discord-bridge", + "slack-bridge", + "slack-app", + ]) { runOpenshell(["provider", "delete", `${sandboxName}-${suffix}`], { ignoreError: true, stdio: ["ignore", "ignore", "ignore"], @@ -266,23 +319,35 @@ export function cleanupSandboxServices( */ export function removeShieldsState( sandboxName: string, - stateDir = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "state"), + stateDir = resolveNemoclawStateDir(), + deps: RemoveShieldsStateDeps = {}, ): void { + const rmSync = deps.rmSync ?? fs.rmSync; + const warn = + deps.warn ?? ((message: string) => console.warn(` ${YW}⚠${R} ${message}`)); const resolvedStateDir = path.resolve(stateDir); for (const prefix of ["shields-", "shields-timer-"]) { - const filePath = path.resolve(resolvedStateDir, `${prefix}${sandboxName}.json`); + const filePath = path.resolve( + resolvedStateDir, + `${prefix}${sandboxName}.json`, + ); if (!filePath.startsWith(`${resolvedStateDir}${path.sep}`)) { // Defense-in-depth: sandbox names are validated to [a-z0-9-] at // all entry points, but reject traversal attempts just in case. continue; } try { - fs.rmSync(filePath, { force: true }); + rmSync(filePath, { force: true }); } catch (error) { // force: true already suppresses ENOENT; warn on real failures // (e.g. EPERM) so stale state doesn't silently survive. - const message = error instanceof Error ? error.message : String(error); - console.warn(` ${YW}⚠${R} Failed to remove shields state '${filePath}': ${message}`); + const errno = error as NodeJS.ErrnoException; + if (errno.code !== "ENOENT") { + const message = error instanceof Error ? error.message : String(error); + warn( + `Failed to remove shields cleanup artifact '${filePath}': ${message}`, + ); + } } } } @@ -297,7 +362,8 @@ export function removeSandboxImage( ): void { const getSandbox = deps.getSandbox ?? registry.getSandbox; const removeImage = - deps.dockerRmi ?? (require("../../adapters/docker") as { dockerRmi: DockerRmi }).dockerRmi; + deps.dockerRmi ?? + (require("../../adapters/docker") as { dockerRmi: DockerRmi }).dockerRmi; const sb = getSandbox(sandboxName); if (!sb?.imageTag) return; const result = removeImage(sb.imageTag, { ignoreError: true }); @@ -320,6 +386,29 @@ export function removeSandboxRegistryEntry( return removeSandbox(sandboxName); } +function defaultDestroyWarn(message: string): void { + console.warn(` ${YW}⚠${R} ${message}`); +} + +export function cleanupShieldsDestroyArtifacts( + sandboxName: string, + deps: CleanupShieldsDestroyArtifactsDeps = {}, +): void { + const killShieldsTimer = deps.killShieldsTimer ?? defaultKillShieldsTimer; + const stateDir = deps.stateDir ?? resolveNemoclawStateDir(); + const warn = deps.warn ?? defaultDestroyWarn; + + const timerResult = killShieldsTimer(sandboxName); + for (const warning of timerResult?.warnings ?? []) { + warn(warning); + } + + removeShieldsState(sandboxName, stateDir, { + rmSync: deps.rmSync ?? fs.rmSync, + warn, + }); +} + export async function destroySandbox( sandboxName: string, options: string[] | DestroySandboxOptions = {}, @@ -332,7 +421,10 @@ export async function destroySandbox( const opsBin = resolveOpenshell(); if (opsBin) { try { - const sessionResult = getActiveSandboxSessions(sandboxName, createSessionDeps(opsBin)); + const sessionResult = getActiveSandboxSessions( + sandboxName, + createSessionDeps(opsBin), + ); if (sessionResult.detected) { activeSessionCount = sessionResult.sessions.length; } @@ -352,17 +444,27 @@ export async function destroySandbox( ` Destroying will terminate ${activeSessionCount === 1 ? "the" : "all"} active ${plural} with a Broken pipe error.`, ); } - console.log(" This will permanently delete the sandbox and all workspace files inside it."); + console.log( + " This will permanently delete the sandbox and all workspace files inside it.", + ); console.log(" This cannot be undone."); - const answer = await askPrompt(" Type 'yes' to confirm, or press Enter to cancel [y/N]: "); - if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { + const answer = await askPrompt( + " Type 'yes' to confirm, or press Enter to cancel [y/N]: ", + ); + if ( + answer.trim().toLowerCase() !== "y" && + answer.trim().toLowerCase() !== "yes" + ) { console.log(" Cancelled."); return; } } const nim = require("../../inference/nim") as { - stopNimContainer: (sandboxName: string, opts?: { silent?: boolean }) => void; + stopNimContainer: ( + sandboxName: string, + opts?: { silent?: boolean }, + ) => void; stopNimContainerByName: (name: string) => void; }; const sb = registry.getSandbox(sandboxName); @@ -397,7 +499,8 @@ export async function destroySandbox( ignoreError: true, stdio: ["ignore", "pipe", "pipe"], }); - const { output: deleteOutput, alreadyGone } = getSandboxDeleteOutcome(deleteResult); + const { output: deleteOutput, alreadyGone } = + getSandboxDeleteOutcome(deleteResult); if (deleteResult.status !== 0 && !alreadyGone) { if (deleteOutput) { @@ -414,8 +517,10 @@ export async function destroySandbox( sandboxStillRegistered: !!registry.getSandbox(sandboxName), }); - cleanupSandboxServices(sandboxName, { stopHostServices: shouldStopHostServices }); - removeShieldsState(sandboxName); + cleanupSandboxServices(sandboxName, { + stopHostServices: shouldStopHostServices, + }); + cleanupShieldsDestroyArtifacts(sandboxName); const removed = removeSandboxRegistryEntry(sandboxName); const session = onboardSession.loadSession(); if (session && session.sandboxName === sandboxName) { @@ -432,7 +537,8 @@ export async function destroySandbox( noLiveSandboxes: hasNoLiveSandboxes(), }) ) { - const shouldCleanupGateway = await resolveCleanupGatewayDecision(normalized); + const shouldCleanupGateway = + await resolveCleanupGatewayDecision(normalized); if (shouldCleanupGateway) { cleanupGatewayAfterLastSandbox(); } else { @@ -449,7 +555,9 @@ export async function destroySandbox( } } if (alreadyGone) { - console.log(` Sandbox '${sandboxName}' was already absent from the live gateway.`); + console.log( + ` Sandbox '${sandboxName}' was already absent from the live gateway.`, + ); } console.log(` ${G}✓${R} Sandbox '${sandboxName}' destroyed`); } diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index 6e0c90b8308..eece63299ef 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -620,12 +620,13 @@ export async function runSandboxDoctor(sandboxName: string, args: string[] = []) }); } + const shieldsDown = shields.isShieldsDown(sandboxName, true); checks.push({ group: "Sandbox", label: "Shields", - status: shields.isShieldsDown(sandboxName) ? "warn" : "ok", - detail: shields.isShieldsDown(sandboxName) ? "down" : "up", - hint: shields.isShieldsDown(sandboxName) + status: shieldsDown ? "warn" : "ok", + detail: shieldsDown ? "down" : "up", + hint: shieldsDown ? `run \`${CLI_NAME} ${sandboxName} shields status\` for details` : undefined, }); diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index f7d56ed6ea5..f760f2f7689 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -145,7 +145,7 @@ export async function showSandboxStatus(sandboxName: string): Promise { /* non-fatal */ } - if (shields.isShieldsDown(sandboxName)) { + if (shields.isShieldsDown(sandboxName, true)) { console.log(" Permissions: shields down (check `shields status` for details)"); } diff --git a/src/lib/shields/audit.ts b/src/lib/shields/audit.ts index 75a3484ad7f..8ac37d3c343 100644 --- a/src/lib/shields/audit.ts +++ b/src/lib/shields/audit.ts @@ -13,12 +13,18 @@ import { appendFileSync } from "node:fs"; import { join } from "node:path"; import { redactFull } from "../security/redact"; import { ensureConfigDir } from "../state/config-io"; +import { resolveNemoclawStateDir } from "../state/paths"; -const AUDIT_DIR = join(process.env.HOME ?? "/tmp", ".nemoclaw", "state"); +const AUDIT_DIR = resolveNemoclawStateDir(); const AUDIT_FILE = join(AUDIT_DIR, "shields-audit.jsonl"); export interface ShieldsAuditEntry { - action: "shields_down" | "shields_up" | "shields_auto_restore" | "shields_up_failed"; + action: + | "shields_down" + | "shields_up" + | "shields_auto_restore" + | "shields_up_failed" + | "shields_auto_restore_lock_warning"; sandbox: string; timestamp: string; timeout_seconds?: number; @@ -26,9 +32,12 @@ export interface ShieldsAuditEntry { policy_applied?: string; policy_snapshot?: string; restored_at?: string; + scheduled_restore_at?: string; restored_by?: "operator" | "auto_timer"; duration_seconds?: number; error?: string; + warning?: string; + lock_verified?: boolean; } /** diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index a7f72990a40..3ec7e5f02aa 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -19,7 +19,13 @@ vi.mock("../runner", () => ({ })); vi.mock("../policy", () => ({ - buildPolicyGetCommand: vi.fn((name) => ["openshell", "policy", "get", "--full", name]), + buildPolicyGetCommand: vi.fn((name) => [ + "openshell", + "policy", + "get", + "--full", + name, + ]), buildPolicySetCommand: vi.fn((file, name) => [ "openshell", "policy", @@ -43,6 +49,10 @@ vi.mock("../sandbox/config", () => ({ })), })); +vi.mock("../adapters/docker/exec", () => ({ + dockerExecFileSync: vi.fn((_argv: string[]) => ""), +})); + vi.mock("./audit", () => ({ appendAuditEntry: vi.fn(), })); @@ -50,6 +60,21 @@ vi.mock("./audit", () => ({ vi.mock("child_process", () => ({ fork: vi.fn(() => ({ pid: 12345, disconnect: vi.fn(), unref: vi.fn() })), execFileSync: vi.fn(), + spawnSync: vi.fn(() => ({ + status: 0, + stdout: Buffer.from(""), + stderr: Buffer.from(""), + })), +})); + +vi.mock("node:child_process", () => ({ + execFileSync: vi.fn(() => ""), + spawnSync: vi.fn(() => ({ + status: 0, + stdout: "", + stderr: "", + })), + spawn: vi.fn(), })); let tmpDir: string; @@ -57,10 +82,12 @@ let tmpDir: string; beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-test-")); vi.stubEnv("HOME", tmpDir); + vi.resetModules(); vi.clearAllMocks(); }); afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllEnvs(); fs.rmSync(tmpDir, { recursive: true, force: true }); }); @@ -109,8 +136,14 @@ describe("shields — unit logic", () => { fs.mkdirSync(stateDir, { recursive: true }); // Write state for two different sandboxes - const alphaState = { shieldsDown: true, updatedAt: new Date().toISOString() }; - const betaState = { shieldsDown: false, updatedAt: new Date().toISOString() }; + const alphaState = { + shieldsDown: true, + updatedAt: new Date().toISOString(), + }; + const betaState = { + shieldsDown: false, + updatedAt: new Date().toISOString(), + }; fs.writeFileSync( path.join(stateDir, "shields-alpha.json"), JSON.stringify(alphaState, null, 2), @@ -120,8 +153,12 @@ describe("shields — unit logic", () => { JSON.stringify(betaState, null, 2), ); - const alpha = JSON.parse(fs.readFileSync(path.join(stateDir, "shields-alpha.json"), "utf-8")); - const beta = JSON.parse(fs.readFileSync(path.join(stateDir, "shields-beta.json"), "utf-8")); + const alpha = JSON.parse( + fs.readFileSync(path.join(stateDir, "shields-alpha.json"), "utf-8"), + ); + const beta = JSON.parse( + fs.readFileSync(path.join(stateDir, "shields-beta.json"), "utf-8"), + ); expect(alpha.shieldsDown).toBe(true); expect(beta.shieldsDown).toBe(false); }); @@ -132,9 +169,13 @@ describe("shields — unit logic", () => { const ts = Date.now(); const snapshotPath = path.join(stateDir, `policy-snapshot-${ts}.yaml`); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}", { - mode: 0o600, - }); + fs.writeFileSync( + snapshotPath, + "version: 1\nnetwork_policies:\n test: {}", + { + mode: 0o600, + }, + ); const state = { shieldsDown: true, @@ -164,7 +205,10 @@ describe("shields — unit logic", () => { fs.mkdirSync(stateDir, { recursive: true }); const snapshotPath = path.join(stateDir, "policy-snapshot-test.yaml"); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}"); + fs.writeFileSync( + snapshotPath, + "version: 1\nnetwork_policies:\n test: {}", + ); const downState = { shieldsDown: true, @@ -266,14 +310,298 @@ describe("shields — unit logic", () => { // ------------------------------------------------------------------- describe("NC-2227-02: three-state shields model", () => { it("deriveShieldsMode encodes the fresh, locked, unlocked, and legacy-state cases", async () => { - const { deriveShieldsMode } = await import("../../../dist/lib/shields/index.js"); + const distModulePath = path.join( + process.cwd(), + "dist", + "lib", + "shields", + "index.js", + ); + const { deriveShieldsMode } = await import(distModulePath); expect(deriveShieldsMode({}, false)).toBe("mutable_default"); - expect(deriveShieldsMode({ shieldsDown: true }, true)).toBe("temporarily_unlocked"); + expect(deriveShieldsMode({ shieldsDown: true }, true)).toBe( + "temporarily_unlocked", + ); expect(deriveShieldsMode({ shieldsDown: false }, true)).toBe("locked"); expect(deriveShieldsMode({}, true)).toBe("mutable_default"); }); }); + + describe("NC-3112: status self-heals stale expired auto-restore markers", () => { + async function loadShieldsModule() { + const distModulePath = path.join( + process.cwd(), + "dist", + "lib", + "shields", + "index.js", + ); + return import(distModulePath); + } + + function stateDir(): string { + return path.join(tmpDir, ".nemoclaw", "state"); + } + + function writeState( + sandboxName: string, + state: Record, + ): void { + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync( + path.join(stateDir(), `shields-${sandboxName}.json`), + JSON.stringify(state, null, 2), + { mode: 0o600 }, + ); + } + + function writeMarker( + sandboxName: string, + marker: Record, + ): void { + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync( + path.join(stateDir(), `shields-timer-${sandboxName}.json`), + JSON.stringify(marker, null, 2), + { mode: 0o600 }, + ); + } + + it("shieldsStatus attempts inline recovery for expired marker when timer PID is dead", async () => { + const sandboxName = "openclaw"; + const snapshotPath = path.join(stateDir(), "policy-snapshot-test.yaml"); + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + writeState(sandboxName, { + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 5 * 60 * 1000).toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "testing", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + updatedAt: new Date().toISOString(), + }); + writeMarker(sandboxName, { + pid: 4242, + sandboxName, + snapshotPath, + restoreAt: new Date(Date.now() - 30_000).toISOString(), + processToken: "token-123", + }); + + const processKillSpy = vi + .spyOn(process, "kill") + .mockImplementation((pid: number, signal?: string | number) => { + if (signal === 0 && pid === 4242) { + const err = new Error("not running") as NodeJS.ErrnoException; + err.code = "ESRCH"; + throw err; + } + return true; + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const dockerExecFileSync = (await import("node:child_process")) + .execFileSync as ReturnType; + dockerExecFileSync.mockImplementation( + (_file: string, argv?: readonly string[]) => { + const cmd = Array.isArray(argv) ? argv.join(" ") : ""; + if ( + cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw/.config-hash") + ) { + return "444 root:root"; + } + if ( + cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw/openclaw.json") + ) { + return "444 root:root"; + } + if (cmd.includes(" lsattr -d /sandbox/.openclaw/.config-hash")) { + return "----i---------e----- /sandbox/.openclaw/.config-hash"; + } + if (cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw")) { + return "755 root:root"; + } + if (cmd.includes(" lsattr -d /sandbox/.openclaw/openclaw.json")) { + return "----i---------e----- /sandbox/.openclaw/openclaw.json"; + } + return ""; + }, + ); + + const { shieldsStatus } = await loadShieldsModule(); + + shieldsStatus(sandboxName); + + expect(processKillSpy).toHaveBeenCalledWith(4242, 0); + expect(errorSpy).toHaveBeenCalledWith( + " Warning: auto-restore timer marker is expired and the timer process is not the recorded shields timer; attempting inline restore.", + ); + expect(logSpy).toHaveBeenCalledWith( + " Shields: DOWN (temporarily unlocked)", + ); + }); + + it("shieldsStatus warns and stays DOWN when inline recovery fails", async () => { + const sandboxName = "openclaw"; + const missingSnapshotPath = path.join( + stateDir(), + "missing-snapshot.yaml", + ); + writeState(sandboxName, { + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 60_000).toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "testing", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: missingSnapshotPath, + updatedAt: new Date().toISOString(), + }); + writeMarker(sandboxName, { + pid: 4242, + sandboxName, + snapshotPath: missingSnapshotPath, + restoreAt: new Date(Date.now() - 30_000).toISOString(), + }); + + vi.spyOn(process, "kill").mockImplementation( + (pid: number, signal?: string | number) => { + if (signal === 0 && pid === 4242) { + const err = new Error("not running") as NodeJS.ErrnoException; + err.code = "ESRCH"; + throw err; + } + return true; + }, + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const { shieldsStatus } = await loadShieldsModule(); + + shieldsStatus(sandboxName); + + expect(logSpy).toHaveBeenCalledWith( + " Shields: DOWN (temporarily unlocked)", + ); + expect(errorSpy).toHaveBeenCalledWith( + " Recovery warning: inline auto-restore failed; shields remain DOWN.", + ); + expect(errorSpy).toHaveBeenCalledWith( + ` Recovery warning: run \`nemoclaw ${sandboxName} shields up\` manually.`, + ); + expect( + fs.existsSync( + path.join(stateDir(), `shields-timer-${sandboxName}.json`), + ), + ).toBe(true); + }); + + it("shieldsStatus attempts inline recovery when expired marker PID is alive but cmdline does not match recorded timer", async () => { + const sandboxName = "openclaw"; + const snapshotPath = path.join(stateDir(), "policy-snapshot-test.yaml"); + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + writeState(sandboxName, { + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 5 * 60 * 1000).toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "testing", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + updatedAt: new Date().toISOString(), + }); + writeMarker(sandboxName, { + pid: 4242, + sandboxName, + snapshotPath, + restoreAt: new Date(Date.now() - 30_000).toISOString(), + processToken: "token-123", + }); + + // PID is alive but belongs to an unrelated process (PID reuse after reboot). + vi.spyOn(process, "kill").mockImplementation( + (_pid: number, _signal?: string | number) => true, + ); + const originalExistsSync = fs.existsSync.bind(fs); + const originalReadFileSync = fs.readFileSync.bind(fs); + vi.spyOn(fs, "existsSync").mockImplementation((p: fs.PathLike) => { + if (String(p) === "/proc/4242/cmdline") return true; + return originalExistsSync(p); + }); + vi.spyOn(fs, "readFileSync").mockImplementation( + (p: fs.PathOrFileDescriptor, options?: unknown) => { + if (String(p) === "/proc/4242/cmdline") { + return "python\0unrelated-process\0"; + } + return originalReadFileSync(p, options as never) as never; + }, + ); + + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const dockerExecFileSync = (await import("node:child_process")) + .execFileSync as ReturnType; + dockerExecFileSync.mockImplementation( + (_file: string, argv?: readonly string[]) => { + const cmd = Array.isArray(argv) ? argv.join(" ") : ""; + if ( + cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw/.config-hash") + ) { + return "444 root:root"; + } + if ( + cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw/openclaw.json") + ) { + return "444 root:root"; + } + if (cmd.includes(" lsattr -d /sandbox/.openclaw/.config-hash")) { + return "----i---------e----- /sandbox/.openclaw/.config-hash"; + } + if (cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw")) { + return "755 root:root"; + } + if (cmd.includes(" lsattr -d /sandbox/.openclaw/openclaw.json")) { + return "----i---------e----- /sandbox/.openclaw/openclaw.json"; + } + return ""; + }, + ); + + const { shieldsStatus } = await loadShieldsModule(); + shieldsStatus(sandboxName); + + expect(errorSpy).toHaveBeenCalledWith( + " Warning: auto-restore timer marker is expired and the timer process is not the recorded shields timer; attempting inline restore.", + ); + expect(logSpy).toHaveBeenCalledWith( + " Shields: DOWN (temporarily unlocked)", + ); + }); + + it("status fails fast on corrupt shields state instead of reporting NOT CONFIGURED", async () => { + const sandboxName = "openclaw"; + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync( + path.join(stateDir(), `shields-${sandboxName}.json`), + "{not-json", + ); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((code?: string | number | null) => { + throw new Error(`exit ${String(code)}`); + }); + + const { shieldsStatus } = await loadShieldsModule(); + expect(() => shieldsStatus(sandboxName)).toThrow("exit 1"); + expect(errorSpy).toHaveBeenCalledWith( + " Shields: ERROR (state file is corrupt)", + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + }); }); // ------------------------------------------------------------------- @@ -281,7 +609,10 @@ describe("shields — unit logic", () => { // ------------------------------------------------------------------- describe("NC-2227-04: sandbox-state.ts tar commands do not follow symlinks", () => { function getSourceCode(): string { - return fs.readFileSync(path.join(import.meta.dirname, "..", "state", "sandbox.ts"), "utf-8"); + return fs.readFileSync( + path.join(import.meta.dirname, "..", "state", "sandbox.ts"), + "utf-8", + ); } it("backup tar command does not use -h flag (no symlink following)", () => { @@ -370,10 +701,14 @@ describe("NC-2227-04: sandbox-state.ts tar commands do not follow symlinks", () expect(chownCheck).toContain("2>/dev/null || true"); expect(chownCheck).toContain("chownResult.error"); expect(chownCheck).toContain("chownResult.signal"); - expect(chownCheck).toContain("WARNING: post-restore ownership repair did not complete"); + expect(chownCheck).toContain( + "WARNING: post-restore ownership repair did not complete", + ); expect(usabilityCheck).toContain("[ -r"); expect(usabilityCheck).toContain("[ -w"); - expect(usabilityCheck).toContain("FAILED: restored state usability check failed"); + expect(usabilityCheck).toContain( + "FAILED: restored state usability check failed", + ); expect(fnBody).toContain("failedDirs.push(...localDirs)"); }); }); @@ -414,7 +749,9 @@ describe("NC-2227-05: shields.ts locks state directories", () => { expect(fnBody).toContain("applyStateDirLockMode"); expect(fnBody).toContain('["chmod", "g-s", target.configDir]'); expect(src).toContain('["chmod", "g-s", dirPath]'); - expect(src).toContain("Best effort; do not skip recursive write stripping."); + expect(src).toContain( + "Best effort; do not skip recursive write stripping.", + ); expect(src).toContain('[ "$clear_setgid" = "1" ] && chmod g-s "$dir"'); expect(fnBody).toContain("chown"); expect(fnBody).toContain("g-s"); @@ -432,7 +769,10 @@ describe("NC-2227-05: shields.ts locks state directories", () => { ); expect(verificationBlock).toContain("for (const f of filesToLock)"); - expect(verificationBlock).toContain('["stat", "-c", "%a %U:%G", f]'); + expect(verificationBlock).toContain('"%a %U:%G"'); + expect(verificationBlock).toContain( + "privilegedSandboxExecCapture(sandboxName", + ); expect(verificationBlock).toContain("${f} mode="); expect(verificationBlock).toContain("${f} owner="); expect(verificationBlock).toContain("${f} immutable bit not set"); @@ -479,4 +819,245 @@ describe("NC-2227-05: shields.ts locks state directories", () => { expect(src).toContain("legacy symlink remains"); expect(fnBody).toContain("assertNoLegacyStateLayout"); }); + + it("readTimerMarker rejects invalid marker pid values", async () => { + const distModulePath = path.join( + process.cwd(), + "dist", + "lib", + "shields", + "timer-control.js", + ); + const { readTimerMarker } = await import(distModulePath); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const markerPath = path.join(stateDir, "shields-timer-openclaw.json"); + + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: 0, + sandboxName: "openclaw", + snapshotPath: "/tmp/snap.yaml", + restoreAt: new Date().toISOString(), + }), + ); + expect(readTimerMarker("openclaw")).toBeNull(); + + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: 12.5, + sandboxName: "openclaw", + snapshotPath: "/tmp/snap.yaml", + restoreAt: new Date().toISOString(), + }), + ); + expect(readTimerMarker("openclaw")).toBeNull(); + }); + + it("killTimer terminates verified live timer process and clears marker", async () => { + const distModulePath = path.join( + process.cwd(), + "dist", + "lib", + "shields", + "timer-control.js", + ); + const { killTimer } = await import(distModulePath); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "shields-timer-openclaw.json"), + JSON.stringify({ + pid: 7331, + sandboxName: "openclaw", + snapshotPath: "/tmp/snap.yaml", + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken: "proc-token-1", + }), + ); + const originalExistsSync = fs.existsSync.bind(fs); + const originalReadFileSync = fs.readFileSync.bind(fs); + const existsSyncSpy = vi.spyOn(fs, "existsSync"); + const readFileSyncSpy = vi.spyOn(fs, "readFileSync"); + existsSyncSpy.mockImplementation((p: fs.PathLike) => { + const asString = String(p); + if (asString === "/proc/7331/cmdline") return true; + return originalExistsSync(p); + }); + readFileSyncSpy.mockImplementation( + ( + p: fs.PathOrFileDescriptor, + options?: + | BufferEncoding + | { encoding?: null | BufferEncoding; flag?: string } + | null, + ) => { + const asString = String(p); + if (asString === "/proc/7331/cmdline") { + return "node\0dist/lib/shields/timer.js\0openclaw\0/tmp/snap.yaml\0proc-token-1\0"; + } + return originalReadFileSync(p, options as never) as never; + }, + ); + const processKillSpy = vi + .spyOn(process, "kill") + .mockImplementation((_pid: number, _signal?: string | number) => true); + + const result = killTimer("openclaw"); + + expect(result).toEqual({ + markerFound: true, + markerPid: 7331, + wasAlive: true, + terminated: true, + warnings: [], + }); + expect(processKillSpy).toHaveBeenCalledWith(7331, 0); + expect(processKillSpy).toHaveBeenCalledWith(7331, "SIGTERM"); + expect( + fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json")), + ).toBe(false); + }); + + it("killTimer does not signal a live PID when marker identity mismatches and still clears marker", async () => { + const distModulePath = path.join( + process.cwd(), + "dist", + "lib", + "shields", + "timer-control.js", + ); + const { killTimer } = await import(distModulePath); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "shields-timer-openclaw.json"), + JSON.stringify({ + pid: 7331, + sandboxName: "openclaw", + snapshotPath: "/tmp/snap.yaml", + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken: "expected-token", + }), + ); + const originalExistsSync = fs.existsSync.bind(fs); + const originalReadFileSync = fs.readFileSync.bind(fs); + const existsSyncSpy = vi.spyOn(fs, "existsSync"); + const readFileSyncSpy = vi.spyOn(fs, "readFileSync"); + existsSyncSpy.mockImplementation((p: fs.PathLike) => { + const asString = String(p); + if (asString === "/proc/7331/cmdline") return true; + return originalExistsSync(p); + }); + readFileSyncSpy.mockImplementation( + ( + p: fs.PathOrFileDescriptor, + options?: + | BufferEncoding + | { encoding?: null | BufferEncoding; flag?: string } + | null, + ) => { + const asString = String(p); + if (asString === "/proc/7331/cmdline") { + return "python\0some-other-process\0--token\0nope\0"; + } + return originalReadFileSync(p, options as never) as never; + }, + ); + const processKillSpy = vi + .spyOn(process, "kill") + .mockImplementation((_pid: number, _signal?: string | number) => true); + + const result = killTimer("openclaw"); + + expect(result.markerFound).toBe(true); + expect(result.wasAlive).toBe(true); + expect(result.terminated).toBe(false); + expect(result.warnings[0]).toContain( + "does not match shields timer identity", + ); + expect(processKillSpy).toHaveBeenCalledTimes(1); + expect(processKillSpy).toHaveBeenCalledWith(7331, 0); + expect( + fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json")), + ).toBe(false); + }); + + it("killTimer clears stale marker even when PID is not alive", async () => { + const distModulePath = path.join( + process.cwd(), + "dist", + "lib", + "shields", + "timer-control.js", + ); + const { killTimer } = await import(distModulePath); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const markerPath = path.join(stateDir, "shields-timer-openclaw.json"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: 7331, + sandboxName: "openclaw", + snapshotPath: "/tmp/snap.yaml", + restoreAt: new Date(Date.now() + 60_000).toISOString(), + }), + ); + + const processKillSpy = vi + .spyOn(process, "kill") + .mockImplementation((pid: number, signal?: string | number) => { + if (pid === 7331 && signal === 0) { + const err = new Error("gone") as NodeJS.ErrnoException; + err.code = "ESRCH"; + throw err; + } + return true; + }); + + const result = killTimer("openclaw"); + expect(result).toEqual({ + markerFound: true, + markerPid: 7331, + wasAlive: false, + terminated: false, + warnings: [], + }); + expect(processKillSpy).toHaveBeenCalledWith(7331, 0); + expect(fs.existsSync(markerPath)).toBe(false); + }); + + it("shieldsDown writes a process token into the timer marker and passes it to timer args", () => { + const src = getSourceCode(); + const downStart = src.indexOf("function shieldsDown"); + expect(downStart).not.toBe(-1); + const fnBody = src.slice(downStart, src.indexOf("function shieldsUp")); + + expect(fnBody).toContain( + 'const processToken = randomBytes(16).toString("hex")', + ); + expect(fnBody).toContain("processToken,"); + }); + + it("isShieldsDown fails closed when shields state is corrupt", async () => { + const distModulePath = path.join( + process.cwd(), + "dist", + "lib", + "shields", + "index.js", + ); + const { isShieldsDown } = await import(distModulePath); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + "{broken-json", + ); + + expect(isShieldsDown("openclaw")).toBe(false); + }); }); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index cb0454c6198..c21f2890165 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -13,6 +13,7 @@ const fs = require("fs"); const path = require("path"); const { fork } = require("child_process"); +const { randomBytes } = require("crypto"); const { run, runCapture, validateName, shellQuote } = require("../runner"); const { dockerExecFileSync } = require("../adapters/docker/exec"); const { dockerCapture } = require("../adapters/docker/run"); @@ -25,11 +26,24 @@ const { parseCurrentPolicy, PERMISSIVE_POLICY_PATH, } = require("../policy"); -const { parseDuration, MAX_SECONDS, DEFAULT_SECONDS } = require("../domain/duration"); +const { + parseDuration, + MAX_SECONDS, + DEFAULT_SECONDS, +} = require("../domain/duration"); +const { + timerMarkerPath, + readTimerMarker, + clearTimerMarker, + isProcessAlive, + verifyTimerMarkerIdentity, + killTimer, +} = require("./timer-control"); +const { resolveNemoclawStateDir } = require("../state/paths"); const { appendAuditEntry } = require("./audit"); const { resolveAgentConfig } = require("../sandbox/config"); -const STATE_DIR = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "state"); +const STATE_DIR = resolveNemoclawStateDir(); // --------------------------------------------------------------------------- // privileged sandbox exec — bypasses the sandbox's Landlock context @@ -44,7 +58,9 @@ const STATE_DIR = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "state"); const K3S_CONTAINER = "openshell-cluster-nemoclaw"; -function resolveDockerDriverSandboxContainer(sandboxName: string): string | null { +function resolveDockerDriverSandboxContainer( + sandboxName: string, +): string | null { try { if (registry.getSandbox?.(sandboxName)?.openshellDriver !== "docker") { return null; @@ -54,7 +70,9 @@ function resolveDockerDriverSandboxContainer(sandboxName: string): string | null } const prefix = `openshell-${sandboxName}-`; const exact = `openshell-${sandboxName}`; - const output = dockerCapture(["ps", "--format", "{{.Names}}"], { ignoreError: true }); + const output = dockerCapture(["ps", "--format", "{{.Names}}"], { + ignoreError: true, + }); return ( output .split("\n") @@ -79,8 +97,12 @@ function kubectlExecArgv(sandboxName: string, cmd: string[]): string[] { ]; } -function privilegedSandboxExecArgv(sandboxName: string, cmd: string[]): string[] { - const dockerDriverContainer = resolveDockerDriverSandboxContainer(sandboxName); +function privilegedSandboxExecArgv( + sandboxName: string, + cmd: string[], +): string[] { + const dockerDriverContainer = + resolveDockerDriverSandboxContainer(sandboxName); if (dockerDriverContainer) { return ["exec", "--user", "root", dockerDriverContainer, ...cmd]; } @@ -94,7 +116,10 @@ function privilegedSandboxExec(sandboxName: string, cmd: string[]): void { }); } -function privilegedSandboxExecCapture(sandboxName: string, cmd: string[]): string { +function privilegedSandboxExecCapture( + sandboxName: string, + cmd: string[], +): string { return dockerExecFileSync(privilegedSandboxExecArgv(sandboxName, cmd), { stdio: ["ignore", "pipe", "pipe"], timeout: 15000, @@ -137,7 +162,10 @@ interface ShieldsState { * shields up has actually been run (shieldsDown === false AND * the state file exists with an updatedAt timestamp). */ -function deriveShieldsMode(state: ShieldsState, hasStateFile: boolean): ShieldsMode { +function deriveShieldsMode( + state: ShieldsState, + hasStateFile: boolean, +): ShieldsMode { if (!hasStateFile) return "mutable_default"; if (state.shieldsDown === true) return "temporarily_unlocked"; if (state.shieldsDown === false) return "locked"; @@ -145,40 +173,60 @@ function deriveShieldsMode(state: ShieldsState, hasStateFile: boolean): ShieldsM return "mutable_default"; } -function loadShieldsState(sandboxName: string): ShieldsState & { _hasStateFile: boolean } { +function loadShieldsState(sandboxName: string): ShieldsState & { + _hasStateFile: boolean; + _isCorrupt?: boolean; + _corruptError?: string; +} { const filePath = stateFilePath(sandboxName); if (!fs.existsSync(filePath)) return { _hasStateFile: false }; try { const parsed = JSON.parse(fs.readFileSync(filePath, "utf-8")); - const state: ShieldsState = isShieldsState(parsed) ? parsed : {}; + if (!isShieldsState(parsed)) { + return { + _hasStateFile: true, + _isCorrupt: true, + _corruptError: "invalid shields state shape", + }; + } + const state: ShieldsState = parsed; return { ...state, _hasStateFile: true }; - } catch { - return { _hasStateFile: false }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + _hasStateFile: true, + _isCorrupt: true, + _corruptError: message, + }; } } -function saveShieldsState(sandboxName: string, patch: ShieldsState): ShieldsState { +function saveShieldsState( + sandboxName: string, + patch: ShieldsState, +): ShieldsState { const current = loadShieldsState(sandboxName); - // Strip the internal _hasStateFile flag before persisting — it is a - // runtime-only marker and must not leak into the JSON state file. - const { _hasStateFile: _, ...currentClean } = current; - const updated: ShieldsState = { ...currentClean, ...patch, updatedAt: new Date().toISOString() }; + // Strip runtime-only markers before persisting. + const { + _hasStateFile: _hasStateFile, + _isCorrupt: _isCorrupt, + _corruptError: _corruptError, + ...currentClean + } = current; + const updated: ShieldsState = { + ...currentClean, + ...patch, + updatedAt: new Date().toISOString(), + }; fs.mkdirSync(STATE_DIR, { recursive: true, mode: 0o700 }); - fs.writeFileSync(stateFilePath(sandboxName), JSON.stringify(updated, null, 2), { mode: 0o600 }); + fs.writeFileSync( + stateFilePath(sandboxName), + JSON.stringify(updated, null, 2), + { mode: 0o600 }, + ); return updated; } -// --------------------------------------------------------------------------- -// Timer marker — tracks the detached auto-restore process -// --------------------------------------------------------------------------- - -interface TimerMarker { - pid: number; - sandboxName: string; - snapshotPath: string; - restoreAt: string; -} - type UnknownRecord = { [key: string]: unknown }; function isObjectRecord(value: unknown): value is UnknownRecord { @@ -190,20 +238,28 @@ function isOptionalBoolean(value: unknown): value is boolean | undefined { } function isOptionalNumber(value: unknown): value is number | undefined { - return value === undefined || (typeof value === "number" && Number.isFinite(value)); + return ( + value === undefined || (typeof value === "number" && Number.isFinite(value)) + ); } function isOptionalString(value: unknown): value is string | undefined { return value === undefined || typeof value === "string"; } -function isOptionalNullableString(value: unknown): value is string | null | undefined { +function isOptionalNullableString( + value: unknown, +): value is string | null | undefined { return value === undefined || value === null || typeof value === "string"; } -function isOptionalNullableNumber(value: unknown): value is number | null | undefined { +function isOptionalNullableNumber( + value: unknown, +): value is number | null | undefined { return ( - value === undefined || value === null || (typeof value === "number" && Number.isFinite(value)) + value === undefined || + value === null || + (typeof value === "number" && Number.isFinite(value)) ); } @@ -220,46 +276,6 @@ function isShieldsState(value: unknown): value is ShieldsState { ); } -function isTimerMarker(value: unknown): value is TimerMarker { - return ( - isObjectRecord(value) && - typeof value.pid === "number" && - typeof value.sandboxName === "string" && - typeof value.snapshotPath === "string" && - typeof value.restoreAt === "string" - ); -} - -function timerMarkerPath(sandboxName: string): string { - return path.join(STATE_DIR, `shields-timer-${sandboxName}.json`); -} - -function readTimerMarker(sandboxName: string): TimerMarker | null { - const p = timerMarkerPath(sandboxName); - if (!fs.existsSync(p)) return null; - try { - const parsed = JSON.parse(fs.readFileSync(p, "utf-8")); - return isTimerMarker(parsed) ? parsed : null; - } catch { - return null; - } -} - -function killTimer(sandboxName: string): void { - const marker = readTimerMarker(sandboxName); - if (!marker) return; - try { - process.kill(marker.pid, "SIGTERM"); - } catch { - // Process already exited — fine - } - try { - fs.unlinkSync(timerMarkerPath(sandboxName)); - } catch { - // Best effort - } -} - // --------------------------------------------------------------------------- // NC-2227-05: State directories locked by shields-up. // @@ -289,7 +305,11 @@ const HIGH_RISK_STATE_DIRS = [ "telegram", ]; -function applyStateDirLockMode(sandboxName: string, configDir: string, owner: string): void { +function applyStateDirLockMode( + sandboxName: string, + configDir: string, + owner: string, +): void { // Locking (shields-up) strips group + world write. Unlocking (shields-down) // restores the same group-readable/writable + o-rwx mutable-default contract // as startup, plus setgid so the gateway UID — now in the sandbox group via @@ -322,7 +342,12 @@ function applyStateDirLockMode(sandboxName: string, configDir: string, owner: st } } try { - privilegedSandboxExec(sandboxName, ["chmod", "-R", recursiveMode, dirPath]); + privilegedSandboxExec(sandboxName, [ + "chmod", + "-R", + recursiveMode, + dirPath, + ]); } catch { // Silently skip } @@ -366,19 +391,34 @@ function legacyDataDirFor(configDir: string): string { return `${configDir}-data`; } -function assertNoLegacyStateLayout(sandboxName: string, configDir: string): void { +function assertNoLegacyStateLayout( + sandboxName: string, + configDir: string, +): void { const dataDir = legacyDataDirFor(configDir); const script = 'set -u; config_dir="$1"; data_dir="$2"; data_real="$(readlink -f "$data_dir" 2>/dev/null || printf "%s" "$data_dir")"; if [ -e "$data_dir" ] || [ -L "$data_dir" ]; then echo "legacy data dir exists: $data_dir"; exit 1; fi; for entry in "$config_dir"/*; do [ -L "$entry" ] || continue; target="$(readlink -f "$entry" 2>/dev/null || readlink "$entry" 2>/dev/null || true)"; case "$target" in "$data_real"/*|"$data_dir"/*) echo "legacy symlink remains: $entry -> $target"; exit 1;; esac; done'; try { - privilegedSandboxExecCapture(sandboxName, ["sh", "-c", script, "sh", configDir, dataDir]); + privilegedSandboxExecCapture(sandboxName, [ + "sh", + "-c", + script, + "sh", + configDir, + dataDir, + ]); } catch (err) { - const execErr = err as { stdout?: Buffer | string; stderr?: Buffer | string; message?: string }; + const execErr = err as { + stdout?: Buffer | string; + stderr?: Buffer | string; + message?: string; + }; const captured = [execErr.stdout, execErr.stderr] .map((value) => (value ? String(value).trim() : "")) .filter(Boolean) .join("\n"); - const message = captured || (err instanceof Error ? err.message : String(err)); + const message = + captured || (err instanceof Error ? err.message : String(err)); throw new Error(`legacy state layout still present: ${message}`); } } @@ -398,7 +438,12 @@ function assertNoLegacyStateLayout(sandboxName: string, configDir: string): void function unlockAgentConfig( sandboxName: string, - target: { agentName?: string; configPath: string; configDir: string; sensitiveFiles?: string[] }, + target: { + agentName?: string; + configPath: string; + configDir: string; + sensitiveFiles?: string[]; + }, ): void { const errors: string[] = []; const filesToUnlock = [target.configPath, ...(target.sensitiveFiles || [])]; @@ -430,7 +475,11 @@ function unlockAgentConfig( } } try { - privilegedSandboxExec(sandboxName, ["chown", "sandbox:sandbox", target.configDir]); + privilegedSandboxExec(sandboxName, [ + "chown", + "sandbox:sandbox", + target.configDir, + ]); } catch { errors.push("chown config dir"); } @@ -454,16 +503,27 @@ function unlockAgentConfig( const issues: string[] = []; for (const f of filesToUnlock) { try { - const perms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]); + const perms = privilegedSandboxExecCapture(sandboxName, [ + "stat", + "-c", + "%a %U:%G", + f, + ]); const [mode, owner] = perms.split(" "); - if (mode !== fileMode) issues.push(`${f} mode=${mode} (expected ${fileMode})`); - if (owner !== "sandbox:sandbox") issues.push(`${f} owner=${owner} (expected sandbox:sandbox)`); + if (mode !== fileMode) + issues.push(`${f} mode=${mode} (expected ${fileMode})`); + if (owner !== "sandbox:sandbox") + issues.push(`${f} owner=${owner} (expected sandbox:sandbox)`); } catch (err) { const msg = err instanceof Error ? err.message : String(err); issues.push(`${f} stat failed: ${msg}`); } try { - const attrs = privilegedSandboxExecCapture(sandboxName, ["lsattr", "-d", f]); + const attrs = privilegedSandboxExecCapture(sandboxName, [ + "lsattr", + "-d", + f, + ]); const [flags] = attrs.trim().split(/\s+/, 1); if (flags.includes("i")) issues.push(`${f} immutable bit still set`); } catch { @@ -472,9 +532,15 @@ function unlockAgentConfig( } try { - const dirPerms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", target.configDir]); + const dirPerms = privilegedSandboxExecCapture(sandboxName, [ + "stat", + "-c", + "%a %U:%G", + target.configDir, + ]); const [mode, owner] = dirPerms.split(" "); - if (mode !== dirMode) issues.push(`config dir mode=${mode} (expected ${dirMode})`); + if (mode !== dirMode) + issues.push(`config dir mode=${mode} (expected ${dirMode})`); if (owner !== "sandbox:sandbox") { issues.push(`config dir owner=${owner} (expected sandbox:sandbox)`); } @@ -508,7 +574,12 @@ function unlockAgentConfig( function lockAgentConfig( sandboxName: string, - target: { agentName?: string; configPath: string; configDir: string; sensitiveFiles?: string[] }, + target: { + agentName?: string; + configPath: string; + configDir: string; + sensitiveFiles?: string[]; + }, ): void { const errors: string[] = []; const filesToLock = [target.configPath, ...(target.sensitiveFiles || [])]; @@ -533,7 +604,11 @@ function lockAgentConfig( } try { - privilegedSandboxExec(sandboxName, ["chown", "root:root", target.configDir]); + privilegedSandboxExec(sandboxName, [ + "chown", + "root:root", + target.configDir, + ]); } catch { errors.push("chown root:root config dir"); } @@ -578,10 +653,17 @@ function lockAgentConfig( const issues: string[] = []; for (const f of filesToLock) { try { - const perms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]); + const perms = privilegedSandboxExecCapture(sandboxName, [ + "stat", + "-c", + "%a %U:%G", + f, + ]); const [mode, owner] = perms.split(" "); - if (!/^4[0-4][0-4]$/.test(mode)) issues.push(`${f} mode=${mode} (expected 444)`); - if (owner !== "root:root") issues.push(`${f} owner=${owner} (expected root:root)`); + if (!/^4[0-4][0-4]$/.test(mode)) + issues.push(`${f} mode=${mode} (expected 444)`); + if (owner !== "root:root") + issues.push(`${f} owner=${owner} (expected root:root)`); } catch (err) { const msg = err instanceof Error ? err.message : String(err); issues.push(`${f} stat failed: ${msg}`); @@ -589,10 +671,16 @@ function lockAgentConfig( } try { - const dirPerms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", target.configDir]); + const dirPerms = privilegedSandboxExecCapture(sandboxName, [ + "stat", + "-c", + "%a %U:%G", + target.configDir, + ]); const [dirMode, dirOwner] = dirPerms.split(" "); if (dirMode !== "755") issues.push(`dir mode=${dirMode} (expected 755)`); - if (dirOwner !== "root:root") issues.push(`dir owner=${dirOwner} (expected root:root)`); + if (dirOwner !== "root:root") + issues.push(`dir owner=${dirOwner} (expected root:root)`); } catch (err) { const msg = err instanceof Error ? err.message : String(err); issues.push(`dir stat failed: ${msg}`); @@ -601,7 +689,11 @@ function lockAgentConfig( if (chattrSucceeded) { for (const f of filesToLock) { try { - const attrs = privilegedSandboxExecCapture(sandboxName, ["lsattr", "-d", f]); + const attrs = privilegedSandboxExecCapture(sandboxName, [ + "lsattr", + "-d", + f, + ]); // lsattr format: "----i---------e----- /path/to/file" // First whitespace-delimited token is the flags field. const [flags] = attrs.trim().split(/\s+/, 1); @@ -624,6 +716,135 @@ function lockAgentConfig( } } +interface LockdownActivationResult { + ok: boolean; + error?: string; +} + +function activateLockdownFromSnapshot( + sandboxName: string, + snapshotPath: string, +): LockdownActivationResult { + if (!snapshotPath || !fs.existsSync(snapshotPath)) { + return { ok: false, error: "saved snapshot is missing" }; + } + + const restoreResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { + ignoreError: true, + }); + const restoreStatus = + typeof restoreResult.status === "number" ? restoreResult.status : 1; + if (restoreStatus !== 0) { + return { + ok: false, + error: `policy restore exited with status ${String(restoreStatus)}`, + }; + } + + const target = resolveAgentConfig(sandboxName); + try { + lockAgentConfig(sandboxName, target); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, error: message }; + } + + return { ok: true }; +} + +function recoverExpiredAutoRestoreInline( + sandboxName: string, + state: ShieldsState & { _isCorrupt?: boolean; _corruptError?: string }, +): { attempted: boolean; restored: boolean } { + if (state._isCorrupt) return { attempted: false, restored: false }; + if (state.shieldsDown !== true) return { attempted: false, restored: false }; + + const marker = readTimerMarker(sandboxName); + if (!marker) return { attempted: false, restored: false }; + + const restoreAtMs = new Date(marker.restoreAt).getTime(); + if (!Number.isFinite(restoreAtMs) || restoreAtMs > Date.now()) { + return { attempted: false, restored: false }; + } + + // PID liveness alone is unsafe: after a reboot/OOM the original timer's PID + // can be reassigned to an unrelated live process, which would otherwise block + // recovery forever and reproduce the #3112 fail-open. Treat a live PID as + // "our timer" only if cmdline + sandbox + processToken match. + if ( + isProcessAlive(marker.pid) && + verifyTimerMarkerIdentity(marker).verified + ) { + return { attempted: false, restored: false }; + } + + console.error( + " Warning: auto-restore timer marker is expired and the timer process is not the recorded shields timer; attempting inline restore.", + ); + + const activation = activateLockdownFromSnapshot( + sandboxName, + marker.snapshotPath, + ); + const nowIso = new Date().toISOString(); + if (!activation.ok) { + appendAuditEntry({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: nowIso, + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: `Inline auto-restore failed: ${activation.error ?? "unknown error"}`, + }); + console.error( + " Recovery warning: inline auto-restore failed; shields remain DOWN.", + ); + console.error( + ` Recovery warning: run \`nemoclaw ${sandboxName} shields up\` manually.`, + ); + return { attempted: true, restored: false }; + } + + saveShieldsState(sandboxName, { + shieldsDown: false, + shieldsDownAt: null, + shieldsDownTimeout: null, + shieldsDownReason: null, + shieldsDownPolicy: null, + }); + clearTimerMarker(sandboxName); + appendAuditEntry({ + action: "shields_auto_restore", + sandbox: sandboxName, + timestamp: nowIso, + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + restored_at: nowIso, + }); + return { attempted: true, restored: true }; +} + +function recoverExpiredAutoRestoreGate( + sandboxName: string, + allowInlineRecovery = true, +): ShieldsState & { + _hasStateFile: boolean; + _isCorrupt?: boolean; + _corruptError?: string; +} { + const state = loadShieldsState(sandboxName); + if (!allowInlineRecovery) return state; + if ( + deriveShieldsMode(state, state._hasStateFile) !== "temporarily_unlocked" + ) { + return state; + } + + const recovery = recoverExpiredAutoRestoreInline(sandboxName, state); + if (!recovery.restored) return state; + return loadShieldsState(sandboxName); +} + // --------------------------------------------------------------------------- // shields down — return to default (mutable) state // @@ -645,7 +866,9 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { console.error( ` Config is already unlocked for ${sandboxName} (since ${state.shieldsDownAt}).`, ); - console.error(" Run `nemoclaw shields up` first, or use --extend (not yet implemented)."); + console.error( + " Run `nemoclaw shields up` first, or use --extend (not yet implemented).", + ); process.exit(1); } @@ -654,7 +877,9 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { // active timer and leave the sandbox unlocked indefinitely. killTimer(sandboxName); - const timeoutSeconds = parseDuration(opts.timeout || `${DEFAULT_TIMEOUT_SECONDS}`); + const timeoutSeconds = parseDuration( + opts.timeout || `${DEFAULT_TIMEOUT_SECONDS}`, + ); const reason = opts.reason || null; const policyName = opts.policy || "permissive"; @@ -662,7 +887,9 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { console.log(" Capturing current policy snapshot..."); let rawPolicy: string; try { - rawPolicy = runCapture(buildPolicyGetCommand(sandboxName), { ignoreError: true }); + rawPolicy = runCapture(buildPolicyGetCommand(sandboxName), { + ignoreError: true, + }); } catch { rawPolicy = ""; } @@ -686,7 +913,9 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { } else if (fs.existsSync(policyName)) { policyFile = path.resolve(policyName); } else { - console.error(` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`); + console.error( + ` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`, + ); process.exit(1); } @@ -697,14 +926,20 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { // OpenClaw uses sandbox:sandbox 0660/2770 here so the gateway UID, which // is a member of the sandbox group, can mutate runtime config. const target = resolveAgentConfig(sandboxName); - console.log(` Unlocking ${target.agentName} config (${target.configPath})...`); + console.log( + ` Unlocking ${target.agentName} config (${target.configPath})...`, + ); try { unlockAgentConfig(sandboxName, target); } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error(` ERROR: ${message}`); - console.error(" Config did not reach the mutable-default state; refusing to save shields-down state."); - console.error(` Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`); + console.error( + " Config did not reach the mutable-default state; refusing to save shields-down state.", + ); + console.error( + ` Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`, + ); process.exit(1); } @@ -724,14 +959,24 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { // can take minutes (policy apply + kubectl chmod), so a relative timeout // passed at fork time would fire too early. const restoreAt = new Date(Date.now() + timeoutSeconds * 1000); + const processToken = randomBytes(16).toString("hex"); const timerScript = path.join(__dirname, "timer.ts"); const timerScriptJs = timerScript.replace(/\.ts$/, ".js"); - const actualScript = fs.existsSync(timerScriptJs) ? timerScriptJs : timerScript; + const actualScript = fs.existsSync(timerScriptJs) + ? timerScriptJs + : timerScript; try { const child = fork( actualScript, - [sandboxName, snapshotPath, restoreAt.toISOString(), target.configPath, target.configDir], + [ + sandboxName, + snapshotPath, + restoreAt.toISOString(), + target.configPath, + target.configDir, + processToken, + ], { detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"], @@ -749,6 +994,7 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { sandboxName, snapshotPath, restoreAt: restoreAt.toISOString(), + processToken, }), { mode: 0o600 }, ); @@ -756,16 +1002,21 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { const message = err instanceof Error ? err.message : String(err); console.error(` Cannot start auto-restore timer: ${message}`); console.error(" Rolling back — restoring policy from snapshot..."); - const rollbackResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { - ignoreError: true, - }); + const rollbackResult = run( + buildPolicySetCommand(snapshotPath, sandboxName), + { + ignoreError: true, + }, + ); let rollbackLocked = false; if (rollbackResult.status === 0) { try { lockAgentConfig(sandboxName, target); rollbackLocked = true; } catch { - console.error(" Warning: Rollback re-lock could not be verified. Check config manually."); + console.error( + " Warning: Rollback re-lock could not be verified. Check config manually.", + ); } } else { console.error(" Warning: Policy restore failed during rollback."); @@ -781,7 +1032,9 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { console.error(" Lockdown restored. Config was never left unguarded."); } else { // Leave state as shieldsDown: true — don't lie about protection level - console.error(" Config remains unlocked — manual intervention required."); + console.error( + " Config remains unlocked — manual intervention required.", + ); console.error( ` Re-lock manually via kubectl exec, then run: nemoclaw ${sandboxName} shields up`, ); @@ -808,7 +1061,9 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { ); console.log(""); console.log(" Sandbox is in default (mutable) state."); - console.log(` Run \`nemoclaw ${sandboxName} shields up\` to opt into lockdown.`); + console.log( + ` Run \`nemoclaw ${sandboxName} shields up\` to opt into lockdown.`, + ); } // --------------------------------------------------------------------------- @@ -825,6 +1080,7 @@ function shieldsUp(sandboxName: string): void { // shieldsDown === false means explicitly locked by a previous shields-up. // undefined (no state file) means fresh sandbox — mutable default, allow shields-up. if (state.shieldsDown === false) { + clearTimerMarker(sandboxName); console.log(" Lockdown is already active."); return; } @@ -835,37 +1091,59 @@ function shieldsUp(sandboxName: string): void { // 2. If coming from shields-down, restore the saved policy snapshot. // If first shields-up on a fresh sandbox (no prior shields-down), // the current policy is already the restrictive baseline — skip restore. - const snapshotPath = state.shieldsDown ? state.shieldsPolicySnapshotPath : undefined; + const snapshotPath = state.shieldsDown + ? state.shieldsPolicySnapshotPath + : undefined; if (state.shieldsDown && (!snapshotPath || !fs.existsSync(snapshotPath))) { - console.error(" Cannot restore restrictive policy: saved snapshot is missing."); - console.error(" Sandbox remains unlocked; recapture shields-down state before running shields up."); + console.error( + " Cannot restore restrictive policy: saved snapshot is missing.", + ); + console.error( + " Sandbox remains unlocked; recapture shields-down state before running shields up.", + ); process.exit(1); } if (snapshotPath) { console.log(" Restoring restrictive policy from snapshot..."); - run(buildPolicySetCommand(snapshotPath, sandboxName)); - } - - // 2b. Lock config file to read-only. - // Uses kubectl exec to bypass Landlock (same as shields down). - // Each operation runs independently and the result is verified. - // If verification fails, config remains unlocked — we do not lie about state. - const target = resolveAgentConfig(sandboxName); - console.log(` Locking ${target.agentName} config (${target.configPath})...`); - try { - lockAgentConfig(sandboxName, target); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.error(` ERROR: ${message}`); - console.error(" Config remains unlocked — manual intervention required."); - console.error( - ` Re-lock manually via kubectl exec, then run: nemoclaw ${sandboxName} shields up`, + const activation = activateLockdownFromSnapshot(sandboxName, snapshotPath); + if (!activation.ok) { + console.error(` ERROR: ${activation.error ?? "unknown restore error"}`); + console.error( + " Config remains unlocked — manual intervention required.", + ); + console.error( + ` Re-lock manually via kubectl exec, then run: nemoclaw ${sandboxName} shields up`, + ); + process.exit(1); + } + } else { + // 2b. Lock config file to read-only. + // Uses kubectl exec to bypass Landlock (same as shields down). + // Each operation runs independently and the result is verified. + // If verification fails, config remains unlocked — we do not lie about state. + const target = resolveAgentConfig(sandboxName); + console.log( + ` Locking ${target.agentName} config (${target.configPath})...`, ); - process.exit(1); + try { + lockAgentConfig(sandboxName, target); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(` ERROR: ${message}`); + console.error( + " Config remains unlocked — manual intervention required.", + ); + console.error( + ` Re-lock manually via kubectl exec, then run: nemoclaw ${sandboxName} shields up`, + ); + process.exit(1); + } } // 3. Calculate duration - const downAt = state.shieldsDownAt ? new Date(state.shieldsDownAt) : new Date(); + const downAt = state.shieldsDownAt + ? new Date(state.shieldsDownAt) + : new Date(); const now = new Date(); const durationSeconds = Math.floor((now.getTime() - downAt.getTime()) / 1000); @@ -878,6 +1156,7 @@ function shieldsUp(sandboxName: string): void { shieldsDownPolicy: null, // Keep snapshotPath for forensics — don't clear it }); + clearTimerMarker(sandboxName); // 5. Audit log appendAuditEntry({ @@ -903,17 +1182,29 @@ function shieldsUp(sandboxName: string): void { // shields status // --------------------------------------------------------------------------- -function shieldsStatus(sandboxName: string): void { +function shieldsStatus(sandboxName: string, allowInlineRecovery = true): void { validateName(sandboxName, "sandbox name"); - const state = loadShieldsState(sandboxName); + const state = recoverExpiredAutoRestoreGate(sandboxName, allowInlineRecovery); + if (state._isCorrupt) { + console.error(" Shields: ERROR (state file is corrupt)"); + console.error( + ` ${stateFilePath(sandboxName)} could not be parsed: ${state._corruptError ?? "unknown error"}`, + ); + console.error( + ` Recovery warning: run \`nemoclaw ${sandboxName} shields up\` to restore a known-good state.`, + ); + process.exit(1); + } const mode = deriveShieldsMode(state, state._hasStateFile); switch (mode) { case "mutable_default": // NC-2227-02: Fresh sandbox with no shields history — do NOT claim locked console.log(" Shields: NOT CONFIGURED (default mutable state)"); - console.log(" Config is mutable. Run `nemoclaw shields up` to opt into lockdown."); + console.log( + " Config is mutable. Run `nemoclaw shields up` to opt into lockdown.", + ); return; case "locked": @@ -927,10 +1218,16 @@ function shieldsStatus(sandboxName: string): void { return; case "temporarily_unlocked": { - const downSince = state.shieldsDownAt ? new Date(state.shieldsDownAt) : null; - const elapsed = downSince ? Math.floor((Date.now() - downSince.getTime()) / 1000) : 0; + const downSince = state.shieldsDownAt + ? new Date(state.shieldsDownAt) + : null; + const elapsed = downSince + ? Math.floor((Date.now() - downSince.getTime()) / 1000) + : 0; const remaining = - state.shieldsDownTimeout != null ? Math.max(0, state.shieldsDownTimeout - elapsed) : null; + state.shieldsDownTimeout != null + ? Math.max(0, state.shieldsDownTimeout - elapsed) + : null; console.log(" Shields: DOWN (temporarily unlocked)"); console.log(` Since: ${state.shieldsDownAt ?? "unknown"}`); @@ -956,8 +1253,9 @@ function shieldsStatus(sandboxName: string): void { * true since the config IS mutable. Only returns false when shields * have been explicitly locked via `shields up`. */ -function isShieldsDown(sandboxName: string): boolean { - const state = loadShieldsState(sandboxName); +function isShieldsDown(sandboxName: string, allowInlineRecovery = false): boolean { + const state = recoverExpiredAutoRestoreGate(sandboxName, allowInlineRecovery); + if (state._isCorrupt) return false; const mode = deriveShieldsMode(state, state._hasStateFile); return mode !== "locked"; } @@ -971,6 +1269,7 @@ export { shieldsUp, shieldsStatus, isShieldsDown, + killTimer, deriveShieldsMode, parseDuration, lockAgentConfig, diff --git a/src/lib/shields/timer-control.ts b/src/lib/shields/timer-control.ts new file mode 100644 index 00000000000..87ce081291d --- /dev/null +++ b/src/lib/shields/timer-control.ts @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { resolveNemoclawStateDir } from "../state/paths"; + +interface TimerMarker { + pid: number; + sandboxName: string; + snapshotPath: string; + restoreAt: string; + processToken?: string; +} + +type UnknownRecord = { [key: string]: unknown }; + +function isObjectRecord(value: unknown): value is UnknownRecord { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isTimerMarker(value: unknown): value is TimerMarker { + const pid = isObjectRecord(value) ? value.pid : undefined; + return ( + isObjectRecord(value) && + typeof pid === "number" && + Number.isInteger(pid) && + pid > 0 && + typeof value.sandboxName === "string" && + typeof value.snapshotPath === "string" && + typeof value.restoreAt === "string" && + (value.processToken === undefined || typeof value.processToken === "string") + ); +} + +function timerMarkerPath(sandboxName: string): string { + return path.join(resolveNemoclawStateDir(), `shields-timer-${sandboxName}.json`); +} + +function readTimerMarker(sandboxName: string): TimerMarker | null { + const p = timerMarkerPath(sandboxName); + if (!fs.existsSync(p)) return null; + try { + const parsed = JSON.parse(fs.readFileSync(p, "utf-8")); + return isTimerMarker(parsed) ? parsed : null; + } catch { + return null; + } +} + +interface ClearTimerMarkerResult { + cleared: boolean; + warning?: string; +} + +function clearTimerMarker(sandboxName: string): ClearTimerMarkerResult { + const markerPath = timerMarkerPath(sandboxName); + try { + fs.unlinkSync(markerPath); + return { cleared: true }; + } catch (error) { + const errno = error as NodeJS.ErrnoException; + if (errno.code === "ENOENT") { + return { cleared: false }; + } + return { + cleared: false, + warning: `Failed to remove shields timer marker '${markerPath}': ${errno.message}`, + }; + } +} + +function isProcessAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + return code === "EPERM"; + } +} + +function readProcessCommandLine(pid: number): string | null { + const procCmdline = `/proc/${String(pid)}/cmdline`; + try { + if (fs.existsSync(procCmdline)) { + const cmdline = fs.readFileSync(procCmdline, "utf-8").replaceAll("\0", " ").trim(); + return cmdline || null; + } + } catch { + // Fall through to ps-based lookup. + } + + try { + const psCommand = execFileSync("ps", ["-o", "command=", "-p", String(pid)], { + stdio: ["ignore", "pipe", "ignore"], + }) + .toString() + .trim(); + return psCommand || null; + } catch { + return null; + } +} + +function verifyTimerMarkerIdentity( + marker: TimerMarker, +): { verified: boolean; warning?: string } { + const commandLine = readProcessCommandLine(marker.pid); + if (!commandLine) { + return { + verified: false, + warning: `Unable to verify shields timer PID ${String(marker.pid)} for sandbox '${marker.sandboxName}'; clearing marker without signaling.`, + }; + } + + const looksLikeTimerProcess = + commandLine.includes("shields/timer.js") || commandLine.includes("shields/timer.ts"); + const hasSandboxArg = commandLine.includes(marker.sandboxName); + + if (!looksLikeTimerProcess || !hasSandboxArg) { + return { + verified: false, + warning: `PID ${String(marker.pid)} does not match shields timer identity for sandbox '${marker.sandboxName}'; clearing marker without signaling.`, + }; + } + + if (marker.processToken && !commandLine.includes(marker.processToken)) { + return { + verified: false, + warning: `PID ${String(marker.pid)} token mismatch for sandbox '${marker.sandboxName}'; clearing marker without signaling.`, + }; + } + + return { verified: true }; +} + +interface KillTimerResult { + markerFound: boolean; + markerPid: number | null; + wasAlive: boolean; + terminated: boolean; + warnings: string[]; +} + +function killTimer(sandboxName: string): KillTimerResult { + const marker = readTimerMarker(sandboxName); + let wasAlive = false; + let terminated = false; + const warnings: string[] = []; + + if (marker) { + wasAlive = isProcessAlive(marker.pid); + if (wasAlive) { + const verification = verifyTimerMarkerIdentity(marker); + if (!verification.verified) { + if (verification.warning) { + warnings.push(verification.warning); + } + } else { + try { + process.kill(marker.pid, "SIGTERM"); + terminated = true; + } catch (error) { + const errno = error as NodeJS.ErrnoException; + if (errno.code !== "ESRCH") { + warnings.push( + `Failed to terminate shields timer PID ${String(marker.pid)} for sandbox '${sandboxName}': ${errno.message}`, + ); + } + } + } + } + } + + const markerClear = clearTimerMarker(sandboxName); + if (markerClear.warning) { + warnings.push(markerClear.warning); + } + + return { + markerFound: marker !== null, + markerPid: marker?.pid ?? null, + wasAlive, + terminated, + warnings, + }; +} + +export { + timerMarkerPath, + readTimerMarker, + clearTimerMarker, + isProcessAlive, + verifyTimerMarkerIdentity, + killTimer, +}; + +export type { + TimerMarker, + ClearTimerMarkerResult, + KillTimerResult, +}; diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts new file mode 100644 index 00000000000..4650c1bbc8b --- /dev/null +++ b/src/lib/shields/timer.test.ts @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const runMock = vi.fn(() => ({ status: 0 })); + +vi.mock("../runner", () => ({ + run: runMock, +})); + +vi.mock("../policy", () => ({ + buildPolicySetCommand: vi.fn((file: string, name: string) => [ + "openshell", + "policy", + "set", + "--policy", + file, + "--wait", + name, + ]), +})); + +vi.mock("../sandbox/config", () => ({ + DEFAULT_AGENT_CONFIG: Symbol("DEFAULT_AGENT_CONFIG"), + resolveAgentConfig: vi.fn(() => ({ + configPath: "/sandbox/.openclaw/openclaw.json", + configDir: "/sandbox/.openclaw", + })), +})); + +vi.mock("./index", () => ({ + lockAgentConfig: vi.fn(), +})); + +describe("shields timer authorization", () => { + let tmpHome: string; + + beforeEach(() => { + tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-timer-")); + vi.stubEnv("HOME", tmpHome); + vi.resetModules(); + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(tmpHome, { recursive: true, force: true }); + }); + + function invokeTimerAndCaptureExit( + runRestoreTimer: (args: any) => void, + args: unknown, + ): number { + const exitSpy = vi.spyOn(process, "exit").mockImplementation((code?: any) => { + throw new Error(`process.exit:${String(code ?? 0)}`); + }); + + try { + runRestoreTimer(args); + throw new Error("Expected runRestoreTimer to exit"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + expect(message.startsWith("process.exit:")).toBe(true); + const code = Number.parseInt(message.slice("process.exit:".length), 10); + return Number.isNaN(code) ? 0 : code; + } finally { + exitSpy.mockRestore(); + } + } + + it("does not restore or rewrite state when marker is missing", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + + const sandboxName = "alpha"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); + const stateFile = path.join(stateDir, `shields-${sandboxName}.json`); + const initialState = { shieldsDown: true, updatedAt: "2026-01-01T00:00:00.000Z" }; + + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync(stateFile, JSON.stringify(initialState, null, 2)); + + const args = timer.parseTimerArgs([sandboxName, snapshotPath, restoreAtIso, "", "", "tok"]); + expect(args).not.toBeNull(); + + const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + + expect(exitCode).toBe(0); + expect(runMock).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); + }); + + it("does not restore or rewrite state when marker processToken mismatches", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + + const sandboxName = "alpha"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); + const stateFile = path.join(stateDir, `shields-${sandboxName}.json`); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const initialState = { shieldsDown: true, updatedAt: "2026-01-01T00:00:00.000Z" }; + + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync(stateFile, JSON.stringify(initialState, null, 2)); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: "wrong-token", + }), + ); + + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + "right-token", + ]); + expect(args).not.toBeNull(); + + const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + + expect(exitCode).toBe(0); + expect(runMock).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); + expect(fs.existsSync(markerPath)).toBe(true); + }); + + it("does not restore or rewrite state when marker pid mismatches", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + + const sandboxName = "alpha"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); + const stateFile = path.join(stateDir, `shields-${sandboxName}.json`); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const initialState = { shieldsDown: true, updatedAt: "2026-01-01T00:00:00.000Z" }; + + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync(stateFile, JSON.stringify(initialState, null, 2)); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid + 1, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: "tok", + }), + ); + + const args = timer.parseTimerArgs([sandboxName, snapshotPath, restoreAtIso, "", "", "tok"]); + expect(args).not.toBeNull(); + + const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + + expect(exitCode).toBe(0); + expect(runMock).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); + expect(fs.existsSync(markerPath)).toBe(true); + }); + + it("restores and updates state when marker matches current timer invocation", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + + const sandboxName = "alpha"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: "tok", + }), + ); + + const args = timer.parseTimerArgs([sandboxName, snapshotPath, restoreAtIso, "", "", "tok"]); + expect(args).not.toBeNull(); + + const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + const stateFile = path.join(stateDir, `shields-${sandboxName}.json`); + const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); + + expect(exitCode).toBe(0); + expect(runMock).toHaveBeenCalledTimes(1); + expect(updatedState.shieldsDown).toBe(false); + expect(updatedState.shieldsDownAt).toBeNull(); + expect(fs.existsSync(markerPath)).toBe(false); + }); +}); diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index c7c79f095ed..6bc45727613 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -6,7 +6,7 @@ // restores the captured policy snapshot. // // Usage (internal — called by shields.ts via fork()): -// node shields-timer.js +// node shields-timer.js import fs from "node:fs"; import path from "node:path"; @@ -14,6 +14,8 @@ import path from "node:path"; import { buildPolicySetCommand } from "../policy"; import { run } from "../runner"; import { DEFAULT_AGENT_CONFIG, resolveAgentConfig } from "../sandbox/config"; +import { resolveNemoclawStateDir } from "../state/paths"; +import { appendAuditEntry, type ShieldsAuditEntry } from "./audit"; import { lockAgentConfig } from "./index"; type UnknownRecord = { [key: string]: unknown }; @@ -36,17 +38,17 @@ interface TimerArgs { markerPath: string; configPath?: string; configDir?: string; + processToken?: string; } -const STATE_DIR = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "state"); -const AUDIT_FILE = path.join(STATE_DIR, "shields-audit.jsonl"); +const STATE_DIR = resolveNemoclawStateDir(); function isRecord(value: unknown): value is UnknownRecord { return typeof value === "object" && value !== null && !Array.isArray(value); } function parseTimerArgs(argv: string[]): TimerArgs | null { - const [sandboxName, snapshotPath, restoreAtIso, configPath, configDir] = argv; + const [sandboxName, snapshotPath, restoreAtIso, configPath, configDir, processToken] = argv; const restoreAtMs = restoreAtIso ? new Date(restoreAtIso).getTime() : Number.NaN; if (!sandboxName || !snapshotPath || !restoreAtIso || Number.isNaN(restoreAtMs)) { @@ -63,12 +65,13 @@ function parseTimerArgs(argv: string[]): TimerArgs | null { markerPath: path.join(STATE_DIR, `shields-timer-${sandboxName}.json`), configPath, configDir, + processToken, }; } -function appendAudit(entry: UnknownRecord): void { +function appendAudit(entry: ShieldsAuditEntry): void { try { - fs.appendFileSync(AUDIT_FILE, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); + appendAuditEntry(entry); } catch { // Best effort — don't crash the timer } @@ -106,11 +109,51 @@ function cleanupMarker(markerPath: string): void { } } +function readTimerMarker(markerPath: string): UnknownRecord | null { + try { + if (!fs.existsSync(markerPath)) { + return null; + } + const parsed = JSON.parse(fs.readFileSync(markerPath, "utf-8")); + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } +} + +function markerMatchesCurrentTimer(args: TimerArgs): boolean { + const marker = readTimerMarker(args.markerPath); + if (!marker) return false; + + const markerPid = marker.pid; + const markerSandboxName = marker.sandboxName; + const markerSnapshotPath = marker.snapshotPath; + const markerRestoreAt = marker.restoreAt; + const markerProcessToken = marker.processToken; + + return ( + markerPid === process.pid && + markerSandboxName === args.sandboxName && + markerSnapshotPath === args.snapshotPath && + markerRestoreAt === args.restoreAtIso && + markerProcessToken === args.processToken + ); +} + function runRestoreTimer(args: TimerArgs): void { const now = new Date().toISOString(); let exitCode = 0; + let ownedMarker = false; try { + // Timer markers are the source of authority. If the marker was removed or + // replaced (e.g., destroy-time neutralization), this process must not + // restore policy or rewrite shields state. + if (!markerMatchesCurrentTimer(args)) { + return; + } + ownedMarker = true; + if (!fs.existsSync(args.snapshotPath)) { appendAudit({ action: "shields_up_failed", @@ -214,7 +257,7 @@ function runRestoreTimer(args: TimerArgs): void { timestamp: now, restored_by: "auto_timer", policy_snapshot: args.snapshotPath, - restore_at: args.restoreAtIso, + scheduled_restore_at: args.restoreAtIso, }); return; } @@ -241,7 +284,9 @@ function runRestoreTimer(args: TimerArgs): void { }); exitCode = 1; } finally { - cleanupMarker(args.markerPath); + if (ownedMarker && markerMatchesCurrentTimer(args)) { + cleanupMarker(args.markerPath); + } process.exit(exitCode); } } @@ -260,3 +305,10 @@ function main(): void { if (require.main === module) { main(); } + +export { + markerMatchesCurrentTimer, + parseTimerArgs, + readTimerMarker, + runRestoreTimer, +}; diff --git a/src/lib/state/paths.ts b/src/lib/state/paths.ts index a1d442ffb4e..e5c0553141b 100644 --- a/src/lib/state/paths.ts +++ b/src/lib/state/paths.ts @@ -1,7 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import os from "node:os"; import path from "node:path"; export const ROOT = path.resolve(__dirname, "..", "..", ".."); export const SCRIPTS = path.join(ROOT, "scripts"); + +export function resolveNemoclawHomeDir( + homeDir: string = process.env.HOME ?? os.homedir(), +): string { + return path.join(homeDir, ".nemoclaw"); +} + +export function resolveNemoclawStateDir( + homeDir: string = process.env.HOME ?? os.homedir(), +): string { + return path.join(resolveNemoclawHomeDir(homeDir), "state"); +} diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index 22daea98b92..10c8eafe54e 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -4,18 +4,20 @@ // Verify that sandbox lifecycle operations clean up host-side Docker images. // See: https://github.com/NVIDIA/NemoClaw/issues/2086 -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { + cleanupShieldsDestroyArtifacts, removeSandboxImage, removeSandboxRegistryEntry, removeShieldsState, } from "../src/lib/actions/sandbox/destroy"; import { getSandboxDeleteOutcome } from "../src/lib/domain/sandbox/destroy"; import { normalizeGarbageCollectImagesOptions } from "../src/lib/domain/lifecycle/options"; +import { resolveNemoclawStateDir } from "../src/lib/state/paths"; import { help as renderRootHelp } from "../src/lib/actions/root-help"; import { COMMANDS, globalCommandTokens } from "../src/lib/cli/command-registry"; import { getRegisteredOclifCommandMetadata } from "../src/lib/cli/oclif-metadata"; @@ -74,6 +76,73 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { alreadyGone: true, }); }); + + it("destroy neutralizes active shields timer and only deletes target sandbox files", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "destroy-shields-")); + const alphaState = path.join(stateDir, "shields-alpha.json"); + const alphaTimer = path.join(stateDir, "shields-timer-alpha.json"); + const betaState = path.join(stateDir, "shields-beta.json"); + const betaTimer = path.join(stateDir, "shields-timer-beta.json"); + + fs.writeFileSync(alphaState, '{"shieldsDown":true}'); + fs.writeFileSync(alphaTimer, '{"pid":9999}'); + fs.writeFileSync(betaState, '{"shieldsDown":true}'); + fs.writeFileSync(betaTimer, '{"pid":9999}'); + + const killCalls: string[] = []; + cleanupShieldsDestroyArtifacts("alpha", { + stateDir, + killShieldsTimer: (sandboxName) => { + killCalls.push(sandboxName); + return { + warnings: [], + }; + }, + }); + + expect(killCalls).toEqual(["alpha"]); + expect(fs.existsSync(alphaState)).toBe(false); + expect(fs.existsSync(alphaTimer)).toBe(false); + expect(fs.existsSync(betaState)).toBe(true); + expect(fs.existsSync(betaTimer)).toBe(true); + + fs.rmSync(stateDir, { recursive: true, force: true }); + }); + + it("destroy shields cleanup warns on timer/cleanup failures but keeps best-effort flow", () => { + const warnings: string[] = []; + const rmSync = vi.fn((artifactPath: string) => { + if (artifactPath.endsWith("shields-alpha.json")) { + const error = new Error("permission denied") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + } + }); + + cleanupShieldsDestroyArtifacts("alpha", { + stateDir: "/tmp/nonexistent-state-dir", + rmSync: rmSync as unknown as typeof fs.rmSync, + killShieldsTimer: () => ({ + warnings: ["Failed to terminate shields timer PID 4242"], + }), + warn: (message) => warnings.push(message), + }); + + expect(warnings).toContain("Failed to terminate shields timer PID 4242"); + expect( + warnings.some((message) => + message.includes("Failed to remove shields cleanup artifact"), + ), + ).toBe(true); + expect(rmSync).toHaveBeenCalledTimes(2); + expect(rmSync.mock.calls[0][0]).toContain("shields-alpha.json"); + expect(rmSync.mock.calls[1][0]).toContain("shields-timer-alpha.json"); + }); + + it("state-dir helper resolves ~/.nemoclaw/state from a single shared helper", () => { + const resolved = resolveNemoclawStateDir("/tmp/example-home"); + expect(resolved).toBe(path.join("/tmp/example-home", ".nemoclaw", "state")); + }); }); describe("image cleanup: onboard records imageTag in registry (#2086)", () => {