diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index b6454371128..a1d93822dfe 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -12,15 +12,21 @@ const requireDist = createRequire(import.meta.url); const shieldsModulePath = "../../../dist/lib/shields/index.js"; type ShieldsHarness = { + auditSpy: MockInstance; logSpy: MockInstance; shieldsDown: typeof import("../../../dist/lib/shields/index.js").shieldsDown; + shieldsStatus: typeof import("../../../dist/lib/shields/index.js").shieldsStatus; shieldsUp: typeof import("../../../dist/lib/shields/index.js").shieldsUp; isShieldsDown: typeof import("../../../dist/lib/shields/index.js").isShieldsDown; }; let tmpDir: string; -function createHarness(): ShieldsHarness { +type HarnessOptions = { + dockerExecFileSync?: (argv: unknown) => string; +}; + +function createHarness(options: HarnessOptions = {}): ShieldsHarness { delete require.cache[requireDist.resolve(shieldsModulePath)]; delete require.cache[requireDist.resolve("../../../dist/lib/sandbox/privileged-exec.js")]; const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); @@ -64,22 +70,26 @@ function createHarness(): ShieldsHarness { ], ); vi.spyOn(dockerExec, "dockerExecFileSync").mockImplementation((argv: unknown) => { + if (options.dockerExecFileSync) return options.dockerExecFileSync(argv); const args = Array.isArray(argv) ? argv.map(String) : []; - if (args.includes("sha256sum")) return "a".repeat(64) + " /sandbox/.openclaw/openclaw.json\n"; - if (args.includes("stat")) { - return args.at(-1) === "/sandbox/.openclaw" - ? "2770 sandbox:sandbox\n" - : "660 sandbox:sandbox\n"; - } - return ""; + return args.includes("sha256sum") + ? "a".repeat(64) + " /sandbox/.openclaw/openclaw.json\n" + : args.includes("stat") + ? args.at(-1) === "/sandbox/.openclaw" + ? "2770 sandbox:sandbox\n" + : "660 sandbox:sandbox\n" + : ""; }); - vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined); + const auditSpy = vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined); const shields = requireDist(shieldsModulePath); logSpy.mockClear(); + auditSpy.mockClear(); return { + auditSpy, logSpy, shieldsDown: shields.shieldsDown, + shieldsStatus: shields.shieldsStatus, shieldsUp: shields.shieldsUp, isShieldsDown: shields.isShieldsDown, }; @@ -143,4 +153,88 @@ describe("shields command flow", () => { "Saved policy snapshot is missing", ); }); + + it("shieldsStatus restores an expired dead timer through the same lock path as shields up", () => { + const configPath = "/sandbox/.openclaw/openclaw.json"; + const configDir = "/sandbox/.openclaw"; + const hashPath = `${configDir}/.config-hash`; + const configHash = "a".repeat(64); + const hashHash = "b".repeat(64); + const execCalls: string[] = []; + const execResponses = new Map([ + [` stat -c %a %U:%G ${hashPath}`, "444 root:root\n"], + [` stat -c %a %U:%G ${configPath}`, "444 root:root\n"], + [` stat -c %a %U:%G ${configDir}`, "755 root:root\n"], + [` lsattr -d ${hashPath}`, `----i---------e----- ${hashPath}\n`], + [` lsattr -d ${configPath}`, `----i---------e----- ${configPath}\n`], + [` sha256sum ${hashPath}`, `${hashHash} ${hashPath}\n`], + [` sha256sum ${configPath}`, `${configHash} ${configPath}\n`], + ]); + const harness = createHarness({ + dockerExecFileSync: (argv: unknown) => { + const args = Array.isArray(argv) ? argv.map(String) : []; + const cmd = args.join(" "); + execCalls.push(cmd); + return [...execResponses].find(([needle]) => cmd.includes(needle))?.[1] ?? ""; + }, + }); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const snapshotPath = path.join(stateDir, "policy-snapshot-expired.yaml"); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), + shieldsDownTimeout: 60, + shieldsDownReason: "coverage", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }), + ); + fs.writeFileSync( + path.join(stateDir, "shields-timer-openclaw.json"), + JSON.stringify({ + pid: 4242, + sandboxName: "openclaw", + snapshotPath, + restoreAt: new Date(Date.now() - 30_000).toISOString(), + processToken: "timer-token", + }), + ); + vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { + const failDeadTimerProbe = () => { + const error = new Error("timer is gone") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + }; + const deadTimerProbe = `${pid}:${signal}` === "4242:0" ? failDeadTimerProbe : undefined; + deadTimerProbe?.(); + return true; + }); + + harness.shieldsStatus("openclaw"); + + const state = JSON.parse( + fs.readFileSync(path.join(stateDir, "shields-openclaw.json"), "utf-8"), + ); + expect(harness.logSpy).toHaveBeenCalledWith(" Shields: UP (lockdown active)"); + expect(state.shieldsDown).toBe(false); + expect(state.fileHashes).toMatchObject({ + [configPath]: configHash, + [hashPath]: hashHash, + }); + expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(false); + expect(harness.auditSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: "shields_auto_restore", + policy_snapshot: snapshotPath, + restored_by: "auto_timer", + sandbox: "openclaw", + }), + ); + expect(execCalls.some((cmd) => cmd.includes(` chmod 444 ${hashPath}`))).toBe(true); + expect(execCalls.some((cmd) => cmd.includes(` chown root:root ${hashPath}`))).toBe(true); + }); }); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index 0339724c698..bc96d79b490 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -380,6 +380,8 @@ describe("shields — unit logic", () => { it("shieldsStatus attempts inline recovery for expired marker when timer PID is dead", async () => { const sandboxName = "openclaw"; + const configPath = "/sandbox/.openclaw/openclaw.json"; + const hashPath = "/sandbox/.openclaw/.config-hash"; const snapshotPath = path.join(stateDir(), "policy-snapshot-test.yaml"); fs.mkdirSync(stateDir(), { recursive: true }); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); @@ -417,20 +419,20 @@ describe("shields — unit logic", () => { >; dockerExecFileSync.mockImplementation((_file: string, argv?: readonly string[]) => { const cmd = Array.isArray(argv) ? argv.join(" ") : ""; - if (cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw/.config-hash")) { + if (cmd.includes(` stat -c %a %U:%G ${hashPath}`)) { return "444 root:root"; } - if (cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw/openclaw.json")) { + if (cmd.includes(` stat -c %a %U:%G ${configPath}`)) { return "444 root:root"; } - if (cmd.includes(" lsattr -d /sandbox/.openclaw/.config-hash")) { - return "----i---------e----- /sandbox/.openclaw/.config-hash"; + if (cmd.includes(` lsattr -d ${hashPath}`)) { + return `----i---------e----- ${hashPath}`; } if (cmd.includes(" stat -c %a %U:%G /sandbox/.openclaw")) { return "755 root:root"; } - if (cmd.includes(" lsattr -d /sandbox/.openclaw/openclaw.json")) { - return "----i---------e----- /sandbox/.openclaw/openclaw.json"; + if (cmd.includes(` lsattr -d ${configPath}`)) { + return `----i---------e----- ${configPath}`; } return ""; }); @@ -825,6 +827,39 @@ describe("shields — unit logic", () => { expect(exitSpy).toHaveBeenCalledWith(2); }); + it("prints baseline-acceptance recovery when the verifier only reports missing seals", async () => { + const sandboxName = "openclaw"; + writeSealedLockedState(sandboxName); + const driftIssues = [ + "/sandbox/.openclaw/.config-hash content drifted (no seal recorded; expected SHA-256)", + ]; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((code?: string | number | null) => { + throw new Error(`exit ${String(code)}`); + }); + + const { shieldsStatus } = await loadShieldsModule(); + expect(() => + shieldsStatus(sandboxName, true, { + verifyLockState: () => ({ ok: false, issues: driftIssues }), + resolveConfig: () => ({ + agentName: "openclaw", + configPath: "/sandbox/.openclaw/openclaw.json", + configDir: "/sandbox/.openclaw", + }), + }), + ).toThrow("exit 2"); + + const allErrors = errorSpy.mock.calls.map((args) => args[0]).join("\n"); + expect(allErrors).toContain("no seal recorded"); + expect(allErrors).toContain("Recovery: rebuild the sandbox for a known-good baseline"); + expect(allErrors).toContain("NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE=1"); + expect(allErrors).not.toContain("restore the original file content from a trusted source"); + expect(exitSpy).toHaveBeenCalledWith(2); + }); + it("treats a resolveConfig throw as drift so the locked status cannot mask a setup gap", async () => { const sandboxName = "openclaw"; writeLockedState(sandboxName); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index cea3a165676..ca30933db4c 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -148,6 +148,17 @@ type AgentConfigTarget = { sensitiveFiles?: string[]; }; +function configHashPath(configDir: string): string { + return `${configDir.replace(/\/+$/, "")}/.config-hash`; +} + +function ensureConfigHashSensitiveFile(target: T): T { + const hashPath = configHashPath(target.configDir); + const sensitiveFiles = target.sensitiveFiles || []; + if (sensitiveFiles.includes(hashPath)) return target; + return { ...target, sensitiveFiles: [...sensitiveFiles, hashPath] } as T; +} + function failShieldsCommand(message: string, shouldThrow?: boolean): never { if (shouldThrow) throw new Error(message); process.exit(1); @@ -528,7 +539,8 @@ function assertNoLegacyStateLayout(sandboxName: string, configDir: string): void // read_only) + chown/chmod below. // --------------------------------------------------------------------------- -function unlockAgentConfig(sandboxName: string, target: AgentConfigTarget): void { +function unlockAgentConfig(sandboxName: string, rawTarget: AgentConfigTarget): void { + const target = ensureConfigHashSensitiveFile(rawTarget); const errors: string[] = []; const filesToUnlock = [target.configPath, ...(target.sensitiveFiles || [])]; // Mutable-default mode for OpenClaw: group-writable + setgid on the @@ -622,7 +634,7 @@ function unlockAgentConfig(sandboxName: string, target: AgentConfigTarget): void function inspectMutableConfigPerms(sandboxName: string): MutableConfigPermsInspection { validateName(sandboxName, "sandbox name"); - const target = resolveAgentConfig(sandboxName); + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); return inspectMutableConfigPermsCore(target, getShieldsPosture(sandboxName, true).mode, (p) => privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", p]), ); @@ -630,7 +642,7 @@ function inspectMutableConfigPerms(sandboxName: string): MutableConfigPermsInspe function repairMutableConfigPerms(sandboxName: string): MutableConfigRepairResult { validateName(sandboxName, "sandbox name"); - const target = resolveAgentConfig(sandboxName); + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); return repairMutableConfigPermsCore(target, getShieldsPosture(sandboxName, true).mode, () => unlockAgentConfig(sandboxName, target), ); @@ -675,8 +687,9 @@ function captureSealHashes(sandboxName: string, filesToHash: string[]): { [path: function lockAgentConfig( sandboxName: string, - target: AgentConfigTarget, + rawTarget: AgentConfigTarget, ): { chattrApplied: boolean; fileHashes: { [path: string]: string } } { + const target = ensureConfigHashSensitiveFile(rawTarget); const errors: string[] = []; const filesToLock = [target.configPath, ...(target.sensitiveFiles || [])]; @@ -856,7 +869,7 @@ function activateLockdownFromSnapshot( }; } - const target = resolveAgentConfig(sandboxName); + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); // Re-confirm the lock after a settle window. This restore feeds the // auto-restore inline recovery and the `shields up` snapshot path, both of // which mark shields UP on this result — so a reconciler revert here would @@ -1054,7 +1067,7 @@ function shieldsDown(sandboxName: string, opts: ShieldsDownOpts = {}): void { // 2b. Return config to default mutable state. // OpenClaw uses sandbox:sandbox 0660/2770 here so the gateway UID, which // is a member of the sandbox group, can mutate runtime config. - const target = resolveAgentConfig(sandboxName); + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); console.log(` Unlocking ${target.agentName} config (${target.configPath})...`); try { unlockAgentConfig(sandboxName, target); @@ -1179,7 +1192,7 @@ function shieldsUp(sandboxName: string, opts: { throwOnError?: boolean } = {}): // host-root tamper has reverted protected perms or rewritten file // content (even when the mode/owner is restored), re-apply the lock // so the recovery hint surfaced by `shields status` actually works. - const target = resolveAgentConfig(sandboxName); + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); const { issues } = verifyShieldsLockState(sandboxName, target, { verifyChattr: state.chattrApplied === true, exec: (cmd: string[]) => privilegedSandboxExecCapture(sandboxName, cmd), @@ -1340,7 +1353,7 @@ function shieldsUp(sandboxName: string, opts: { throwOnError?: boolean } = {}): // Uses kubectl exec to bypass Landlock (same as shields down). // Each operation runs independently and the result is verified. // If verification fails, config remains unlocked — we do not lie about state. - const target = resolveAgentConfig(sandboxName); + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); console.log(` Locking ${target.agentName} config (${target.configPath})...`); let lockResult: { chattrApplied: boolean; fileHashes: { [path: string]: string } }; try { @@ -1449,7 +1462,7 @@ function shieldsStatus( // instead of reported as a clean lockdown. let driftIssues: string[] = []; try { - const target = resolveConfig(sandboxName); + const target = ensureConfigHashSensitiveFile(resolveConfig(sandboxName)); driftIssues = verify(sandboxName, target, { verifyChattr: state.chattrApplied === true, exec: (cmd: string[]) => privilegedSandboxExecCapture(sandboxName, cmd), @@ -1475,13 +1488,22 @@ function shieldsStatus( // would just seal the tampered or unverifiable content. Perm // drift (mode/owner/chattr/legacy-layout) is launderable by // re-up. Surface the right recovery for the failure mode. - const hasHashTrouble = driftIssues.some(isHashVerificationIssue); - if (hasHashTrouble) { - console.error( - ` Recovery: restore the original file content from a trusted source, or rebuild the sandbox, then run \`nemoclaw ${sandboxName} shields up\` to re-seal.`, - ); - } else { - console.error(` Recovery: nemoclaw ${sandboxName} shields up # re-lock and re-verify`); + const hashIssues = driftIssues.filter(isHashVerificationIssue); + const realHashDrift = hashIssues.filter((entry) => !entry.includes("no seal recorded")); + const hasMissingSeals = hashIssues.length > realHashDrift.length; + const recoveryLines = + realHashDrift.length > 0 + ? [ + ` Recovery: restore the original file content from a trusted source, or rebuild the sandbox, then run \`nemoclaw ${sandboxName} shields up\` to re-seal.`, + ] + : hasMissingSeals + ? [ + " Recovery: rebuild the sandbox for a known-good baseline,", + ` or set NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE=1 and re-run \`nemoclaw ${sandboxName} shields up\` to seal the current bytes.`, + ] + : [` Recovery: nemoclaw ${sandboxName} shields up # re-lock and re-verify`]; + for (const line of recoveryLines) { + console.error(line); } process.exit(2); } diff --git a/test/e2e-scenario/live/shields-config.test.ts b/test/e2e-scenario/live/shields-config.test.ts index 567e66c3b4d..153107bfa70 100644 --- a/test/e2e-scenario/live/shields-config.test.ts +++ b/test/e2e-scenario/live/shields-config.test.ts @@ -24,11 +24,13 @@ import { } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CONFIG_PATH = "/sandbox/.openclaw/openclaw.json"; const CONFIG_DIR = path.dirname(CONFIG_PATH); +const CONFIG_HASH_PATH = `${CONFIG_DIR}/.config-hash`; const AUDIT_FILE = path.join(os.homedir(), ".nemoclaw", "state", "shields-audit.jsonl"); const STATE_FILE = (sandboxName: string) => path.join(os.homedir(), ".nemoclaw", "state", `shields-${sandboxName}.json`); @@ -50,6 +52,10 @@ function resultText(result: Pick): string return [result.stdout, result.stderr].filter(Boolean).join("\n"); } +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { ...buildAvailabilityProbeEnv(), @@ -212,6 +218,14 @@ function readAuditEntries(): unknown[] { .map((line) => JSON.parse(line)); } +function readTimerMarker(sandboxName: string): { + pid: number; + restoreAt: string; + snapshotPath: string; +} { + return JSON.parse(fs.readFileSync(TIMER_FILE(sandboxName), "utf8")); +} + RUN_SHIELDS_TEST( "shields-config: live shields up/down locks config and detects drift", { timeout: TEST_TIMEOUT_MS }, @@ -227,7 +241,7 @@ RUN_SHIELDS_TEST( "shields up locks config/workspace and config get redacts secrets", "host-root chmod-write-chmod tamper is detected as content drift", "shields down restores mutable modes and records audit JSONL", - "auto-restore timer re-locks shields", + "dead auto-restore timer inline recovery re-locks config and .config-hash", "double shields-up/down operations are rejected", ], }); @@ -245,10 +259,8 @@ RUN_SHIELDS_TEST( skip("Docker is required for shields-config live E2E"); } - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - expect(apiKey.startsWith("nvapi-"), "NVIDIA_INFERENCE_API_KEY must start with nvapi-").toBe( - true, - ); + const hosted = requireHostedInferenceConfig(secrets); + const apiKey = hosted.apiKey; await cleanupSandbox(host, sandbox, "pre-cleanup"); cleanup.add(`destroy shields-config sandbox ${SANDBOX_NAME}`, async () => { @@ -261,7 +273,7 @@ RUN_SHIELDS_TEST( { artifactName: "phase-1-install-shields-config", env: commandEnv({ - NVIDIA_INFERENCE_API_KEY: apiKey, + ...hosted.env, NEMOCLAW_RECREATE_SANDBOX: "1", }), redactionValues: [apiKey], @@ -498,6 +510,8 @@ RUN_SHIELDS_TEST( { artifactName: "phase-9-shields-down-timer" }, ); expect(timerDown.exitCode, resultText(timerDown)).toBe(0); + const timerMarker = readTimerMarker(SANDBOX_NAME); + process.kill(timerMarker.pid, "SIGKILL"); const statusTimer = await runNemoclaw(host, [SANDBOX_NAME, "shields", "status"], { artifactName: "phase-9-status-down-before-auto-restore", }); @@ -507,9 +521,10 @@ RUN_SHIELDS_TEST( let restored = false; let lastTimerStatus = ""; for (let attempt = 1; Date.now() < deadline; attempt += 1) { - await new Promise((resolve) => setTimeout(resolve, TIMER_POLL_INTERVAL_MS)); + const waitForRestoreAt = Math.max(0, new Date(timerMarker.restoreAt).getTime() - Date.now()); + await delay(Math.max(TIMER_POLL_INTERVAL_MS, waitForRestoreAt + 1_000)); const poll = await runNemoclaw(host, [SANDBOX_NAME, "shields", "status"], { - artifactName: `phase-9-status-auto-restore-poll-${attempt}`, + artifactName: `phase-9-status-dead-timer-inline-restore-poll-${attempt}`, }); lastTimerStatus = resultText(poll); if (lastTimerStatus.includes("Shields: UP")) { @@ -518,10 +533,37 @@ RUN_SHIELDS_TEST( } } expect(restored, lastTimerStatus).toBe(true); - const configTimer = await sandboxShell(sandbox, `stat -c '%a' ${CONFIG_PATH}`, { - artifactName: "phase-9-config-perms-after-auto-restore", + const dirTimer = await statPath( + sandbox, + CONFIG_DIR, + "phase-9-config-dir-perms-after-dead-timer-inline-restore", + ); + expect(dirTimer).toMatchObject({ mode: "755", owner: "root:root" }); + const configTimer = await statPath( + sandbox, + CONFIG_PATH, + "phase-9-config-perms-after-dead-timer-inline-restore", + ); + expect(configTimer).toMatchObject({ mode: "444", owner: "root:root" }); + const hashTimer = await statPath( + sandbox, + CONFIG_HASH_PATH, + "phase-9-config-hash-perms-after-dead-timer-inline-restore", + ); + expect(hashTimer).toMatchObject({ mode: "444", owner: "root:root" }); + const stateAfterTimer = JSON.parse(fs.readFileSync(STATE_FILE(SANDBOX_NAME), "utf8")); + expect(stateAfterTimer.fileHashes).toMatchObject({ + [CONFIG_PATH]: expect.any(String), + [CONFIG_HASH_PATH]: expect.any(String), }); - expect(configTimer.stdout.trim()).toMatch(/^4[0-4][0-4]$/); + expect(readAuditEntries()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + action: "shields_auto_restore", + policy_snapshot: timerMarker.snapshotPath, + }), + ]), + ); const doubleUp = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { artifactName: "phase-10-double-shields-up", @@ -555,7 +597,7 @@ RUN_SHIELDS_TEST( contentDriftDetection: true, shieldsDownMutableRestore: true, auditTrail: true, - autoRestore: true, + deadTimerInlineAutoRestore: true, doubleOperationRejection: true, }, shellDeletion: "deferred to #5098 Phase 11 cleanup",