From 8209b7ba098ac5fef9edf1c3d9453977813e0a67 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 17 Jun 2026 05:08:50 +0000 Subject: [PATCH 1/4] fix(sandbox/recover): enforce Hermes env-file secret boundary on probe path Signed-off-by: Tinson Lai --- docs/reference/commands-nemohermes.mdx | 4 + docs/reference/commands.mdx | 8 + src/lib/actions/sandbox/connect.ts | 12 +- src/lib/actions/sandbox/process-recovery.ts | 81 ++++++- src/lib/agent/hermes-recovery-boundary.ts | 51 ++++ ...hermes-secret-boundary-behavioural.test.ts | 40 +++ test/process-recovery.test.ts | 229 ++++++++++++++++++ 7 files changed, 423 insertions(+), 2 deletions(-) diff --git a/docs/reference/commands-nemohermes.mdx b/docs/reference/commands-nemohermes.mdx index 0f60d3464f9..8b121dc8894 100644 --- a/docs/reference/commands-nemohermes.mdx +++ b/docs/reference/commands-nemohermes.mdx @@ -500,6 +500,10 @@ If the gateway is already running, the command exits zero with a probe message a nemohermes my-assistant recover ``` +`recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. +If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. +Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. + ### `nemohermes status` Show sandbox status, health, and inference configuration. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 268a2d55714..2a38721b0db 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -618,6 +618,14 @@ If the gateway is already running, the command exits zero with a probe message a $$nemoclaw my-assistant recover ``` + + +`recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. +If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. +Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. + + + ### `$$nemoclaw status` Show sandbox status, health, and inference configuration. diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 58a4f1e8af4..3abd1278d85 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -27,7 +27,6 @@ import { findReachableOllamaHost, probeLocalProviderHealth } from "../../inferen import { ensureOllamaAuthProxy, probeOllamaAuthProxyHealth } from "../../inference/ollama/proxy"; import { LOCAL_INFERENCE_TIMEOUT_SECS } from "../../onboard/env"; import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; -import { getSandboxTargetGatewayName } from "./gateway-target"; import { isWsl } from "../../platform"; import { ROOT } from "../../runner"; import * as sandboxVersion from "../../sandbox/version"; @@ -53,6 +52,7 @@ import { import { preflightVllmModelEnvOrExit } from "./connect-vllm-preflight"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state"; +import { getSandboxTargetGatewayName } from "./gateway-target"; import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand } from "./process-recovery"; import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; @@ -193,6 +193,16 @@ function runSandboxConnectProbe(sandboxName: string): void { ); process.exit(1); } + if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { + console.error(""); + console.error( + ` Probe failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, + ); + console.error( + " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + process.exit(1); + } if (processCheck.wasRunning) { ensureSandboxInferenceRoute(sandboxName, { quiet: true }); // Defense-in-depth scope-upgrade approval on the probe-only / `recover` diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 871daee87a0..3310942542e 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -11,6 +11,12 @@ import { runOpenshell, } from "../../adapters/openshell/runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; +import { + buildHermesEnvFileBoundaryStandaloneCheck, + SECRET_BOUNDARY_OK_MARKER, + SECRET_BOUNDARY_REFUSED_MARKER, + SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER, +} from "../../agent/hermes-recovery-boundary"; import * as agentRuntime from "../../agent/runtime"; import { G, R } from "../../cli/terminal-style"; import { DASHBOARD_PORT } from "../../core/ports"; @@ -482,12 +488,71 @@ function recoverHermesDashboardProcessIfEnabled(sandboxName: string): boolean | return recoverHermesDashboardProcess(sandboxName, { executeCommand: executeSandboxCommand }); } +function isHermesAgent(agent: ReturnType): boolean { + return !!agent && agent.name === "hermes"; +} + +/** + * Re-run the Hermes env-file secret-boundary validator against a running + * gateway, before the probe path returns control to the caller. The + * relaunch path already runs the same validator inline as part of + * `buildRecoveryScript`, but the probe path returns early as soon as the + * gateway is reported healthy, so a poisoned `.env` injected after cold + * start would otherwise never be re-evaluated. The check is invoked via + * `openshell sandbox exec` (root) so the validator's kill snippet can + * actually signal the gateway-user process when refusing — a sandbox-user + * SSH shell cannot (test/e2e-gateway-isolation.sh test 13). Returns + * `null` when the helper has nothing to evaluate (unknown agent, exec + * unreachable, validator missing on this sandbox image); returns + * `refused: true` when the validator refused the env file and the + * snippet brought the gateway down. + */ +function enforceHermesSecretBoundaryOnRunningGateway( + sandboxName: string, + agent: ReturnType, + { quiet }: { quiet: boolean }, +): { refused: boolean } | null { + if (!isHermesAgent(agent)) return null; + const script = buildHermesEnvFileBoundaryStandaloneCheck(); + const result = executeSandboxExecCommand(sandboxName, script, 30000); + if (!result) return null; + const stdoutMarker = result.stdout + .split(/\r?\n/) + .reverse() + .find((line) => line.trim().startsWith("SECRET_BOUNDARY_")); + if (stdoutMarker === SECRET_BOUNDARY_REFUSED_MARKER || result.status !== 0) { + if (!quiet) { + console.error(""); + console.error( + ` ${R}Secret-boundary check refused recovery of Hermes gateway in '${sandboxName}'.${R}`, + ); + if (result.stderr.trim()) { + for (const line of result.stderr.split(/\r?\n/)) { + if (line.trim()) console.error(` ${line}`); + } + } + console.error(" /sandbox/.hermes/.env contains raw secret-shaped values. Replace them with"); + console.error( + " openshell:resolve:env: placeholders and re-run `nemoclaw recover`.", + ); + } + return { refused: true }; + } + if ( + stdoutMarker === SECRET_BOUNDARY_OK_MARKER || + stdoutMarker === SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER + ) { + return { refused: false }; + } + return null; +} + /** * Detect and recover from a sandbox that survived a gateway restart but * whose OpenClaw processes are not running. Also re-establishes the * host-side dashboard port-forward when it has gone dead independently * of the gateway. Returns an object describing the outcome: - * `{ checked, wasRunning, recovered, forwardRecovered }`. + * `{ checked, wasRunning, recovered, forwardRecovered, secretBoundaryRefused? }`. */ export function checkAndRecoverSandboxProcesses( sandboxName: string, @@ -499,6 +564,20 @@ export function checkAndRecoverSandboxProcesses( } const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); const recoveryPort = resolveSandboxDashboardPort(sandboxName); + if (running && isHermesAgent(recoveryAgent)) { + const enforcement = enforceHermesSecretBoundaryOnRunningGateway(sandboxName, recoveryAgent, { + quiet, + }); + if (enforcement?.refused) { + return { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + }; + } + } if (running) { // Gateway is alive but the host-side forward can still be dead or // owned by another sandbox. Probe and re-establish only when diff --git a/src/lib/agent/hermes-recovery-boundary.ts b/src/lib/agent/hermes-recovery-boundary.ts index dbaa6797cb9..a80008f44f0 100644 --- a/src/lib/agent/hermes-recovery-boundary.ts +++ b/src/lib/agent/hermes-recovery-boundary.ts @@ -22,6 +22,10 @@ import { shellQuote } from "../runner"; export const HERMES_SECRET_BOUNDARY_VALIDATOR_PATH = "/usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py"; +export const SECRET_BOUNDARY_REFUSED_MARKER = "SECRET_BOUNDARY_REFUSED"; +export const SECRET_BOUNDARY_OK_MARKER = "SECRET_BOUNDARY_OK"; +export const SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER = "SECRET_BOUNDARY_VALIDATOR_MISSING"; + const HERMES_GATEWAY_PROC_PATTERN = "[h]ermes[[:space:]]+gateway([[:space:]]|$)"; const HERMES_DASHBOARD_PROC_PATTERN = "[h]ermes[[:space:]]+dashboard([[:space:]]|$)"; const HERMES_BOUNDARY_RECOVERY_LOG = "/tmp/gateway-recovery.log"; @@ -118,9 +122,56 @@ export function buildHermesRuntimeEnvBoundaryGuard(): string { return `if [ ! -f ${shellQuote(validator)} ]; then ${missingLog} elif ! ${invocation}; then ${kill} echo SECRET_BOUNDARY_REFUSED; exit 1; fi;`; } +/** + * Build a standalone shell snippet that evaluates the Hermes env-file + * secret-boundary contract without relaunching anything. Intended for the + * `sandbox recover` / `connect --probe-only` probe path, where the gateway + * is already running and the relaunch script is not reached: the host can + * exec this snippet inside the sandbox, parse the marker on stdout, and + * decide whether to refuse the probe. + * + * Marker contract on stdout (one of): + * - `SECRET_BOUNDARY_OK` — validator ran and accepted the env file. + * - `SECRET_BOUNDARY_REFUSED` — validator ran and refused; the snippet + * killed any running gateway/dashboard process before exiting non-zero. + * - `SECRET_BOUNDARY_VALIDATOR_MISSING` — validator script absent on this + * sandbox image (older image, fail-open by design). + * + * Validator stderr (`[SECURITY] …` lines) is left on the exec command's + * stderr; the caller surfaces it directly. This keeps the snippet + * independent of any `/tmp/gateway-recovery.log` setup, which matters when + * the snippet runs via `openshell sandbox exec` (root) rather than the + * sandbox-user SSH recovery shell that the relaunch path uses. + * + * The kill snippet is intentionally invoked from a context the caller + * arranges to have authority over: a sandbox-user SSH shell cannot signal + * gateway-user processes (test/e2e-gateway-isolation.sh test 13), so a + * refusal that did not also bring the listener down would log a refusal + * while `/health` kept serving. Run this via the root sandbox-exec path so + * the kill has authority. + */ +export function buildHermesEnvFileBoundaryStandaloneCheck(): string { + const validator = HERMES_SECRET_BOUNDARY_VALIDATOR_PATH; + const kill = buildHermesBoundaryKillSnippet(); + const invocation = `python3 ${shellQuote(validator)} env-file /sandbox/.hermes/.env`; + return [ + `if [ ! -f ${shellQuote(validator)} ]; then`, + ` echo ${SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER}; exit 0;`, + `fi;`, + `if ${invocation}; then`, + ` echo ${SECRET_BOUNDARY_OK_MARKER}; exit 0;`, + `else`, + ` ${kill}`, + ` echo ${SECRET_BOUNDARY_REFUSED_MARKER};`, + ` exit 1;`, + `fi;`, + ].join("\n"); +} + export const __testing = { buildHermesEnvFileBoundaryGuard, buildHermesRuntimeEnvBoundaryGuard, + buildHermesEnvFileBoundaryStandaloneCheck, buildHermesBoundaryKillSnippet, HERMES_GATEWAY_PROC_PATTERN, HERMES_DASHBOARD_PROC_PATTERN, diff --git a/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts b/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts index df6e053391a..55a1aec3adb 100644 --- a/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts +++ b/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts @@ -196,6 +196,46 @@ describe("Hermes secret-boundary guard — guard snippet behaviour", () => { expect(result.pkillCalls.length).toBeGreaterThanOrEqual(2); expect(result.recoveryLog).toContain("[SECURITY]"); }); + + it("standalone env-file check exits 1, emits SECRET_BOUNDARY_REFUSED, kills processes when validator refuses", { + timeout: 15_000, + }, () => { + const result = runGuard({ + guard: __testing.buildHermesEnvFileBoundaryStandaloneCheck(), + pythonExit: 1, + validatorExists: true, + }); + expect(result.status).toBe(1); + expect(result.stdout).toContain("SECRET_BOUNDARY_REFUSED"); + expect(result.stdout).not.toContain("SECRET_BOUNDARY_OK"); + expect(result.stdout).not.toContain("REACHED_LAUNCH"); + expect(result.pkillCalls.length).toBeGreaterThanOrEqual(2); + expect(result.stderr).toContain("[SECURITY]"); + }); + + it("standalone env-file check exits 0 and emits SECRET_BOUNDARY_OK when validator accepts", () => { + const result = runGuard({ + guard: __testing.buildHermesEnvFileBoundaryStandaloneCheck(), + pythonExit: 0, + validatorExists: true, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("SECRET_BOUNDARY_OK"); + expect(result.stdout).not.toContain("SECRET_BOUNDARY_REFUSED"); + expect(result.pkillCalls.length).toBe(0); + }); + + it("standalone env-file check emits SECRET_BOUNDARY_VALIDATOR_MISSING and exits 0 when validator script is absent", () => { + const result = runGuard({ + guard: __testing.buildHermesEnvFileBoundaryStandaloneCheck(), + pythonExit: 0, + validatorExists: false, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("SECRET_BOUNDARY_VALIDATOR_MISSING"); + expect(result.stdout).not.toContain("SECRET_BOUNDARY_REFUSED"); + expect(result.pkillCalls.length).toBe(0); + }); }); describe("Hermes secret-boundary guard — full recovery script behaviour", () => { diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index ce784cc33ce..311d1955217 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -535,4 +535,233 @@ hermes-box 127.0.0.1 8642 12346 running`; expect(result.wasRunning).toBe(true); expect(result.forwardRecovered).toBe(false); }); + + it("refuses recovery of a running Hermes gateway when /sandbox/.hermes/.env contains raw secret-shaped values", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + let forwardListCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { + status: 1, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_REFUSED\n", + stderr: + "[SECURITY] Refusing Hermes startup because /sandbox/.hermes/.env contains raw secret-shaped values\n[SECURITY] TELEGRAM_BOT_TOKEN (line 3)", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockImplementation(() => { + forwardListCalls += 1; + return { + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }; + }); + const runOpenshell = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + }); + expect(secretBoundaryCalls).toBe(1); + expect(forwardListCalls).toBe(0); + expect(captureOpenshell).not.toHaveBeenCalled(); + expect( + runOpenshell.mock.calls.some(([rawArgs]) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + return args[0] === "forward"; + }), + ).toBe(false); + }); + + it("falls through to the forward-refresh path when the Hermes secret-boundary check passes", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + }); + expect(secretBoundaryCalls).toBe(1); + }); + + it("falls through when the Hermes secret-boundary validator is absent on an older sandbox image", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); + const childProcess = requireDist("node:child_process"); + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_VALIDATOR_MISSING\n", + stderr: "", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + }); + }); + + it("does not invoke the Hermes secret-boundary check for an OpenClaw sandbox", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "beta", + agent: "openclaw", + dashboardPort: 18789, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nbeta 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + withFakeOpenshellBinary(() => checkAndRecoverSandboxProcesses("beta", { quiet: true })); + expect(secretBoundaryCalls).toBe(0); + }); }); From 2e03367d32758628c1b0581525de5074cf484cc0 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 17 Jun 2026 08:51:14 +0000 Subject: [PATCH 2/4] fix(sandbox/recover): surface validator stderr and fail safe on inconclusive secret-boundary check Signed-off-by: Tinson Lai --- src/lib/actions/sandbox/connect.ts | 23 +- src/lib/actions/sandbox/process-recovery.ts | 66 ++++-- test/process-recovery.test.ts | 238 ++++++++++++++++++-- 3 files changed, 280 insertions(+), 47 deletions(-) diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 3abd1278d85..2e6af863b16 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -195,12 +195,23 @@ function runSandboxConnectProbe(sandboxName: string): void { } if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { console.error(""); - console.error( - ` Probe failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, - ); - console.error( - " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", - ); + const reason = + "secretBoundaryReason" in processCheck ? processCheck.secretBoundaryReason : undefined; + if (reason === "raw-secret") { + console.error( + ` Probe failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, + ); + console.error( + " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + } else { + console.error( + ` Probe failed: secret-boundary check did not complete for ${agentName} gateway in '${sandboxName}'.`, + ); + console.error( + " Inspect the validator output above and re-run `nemoclaw recover`.", + ); + } process.exit(1); } if (processCheck.wasRunning) { diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 3310942542e..59f23924dc1 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -492,6 +492,19 @@ function isHermesAgent(agent: ReturnType): return !!agent && agent.name === "hermes"; } +type SecretBoundaryRefusalReason = "raw-secret" | "inconclusive"; + +type HermesSecretBoundaryEnforcement = + | { refused: false } + | { refused: true; reason: SecretBoundaryRefusalReason; stderr: string }; + +function printValidatorStderr(stderr: string): void { + if (!stderr.trim()) return; + for (const line of stderr.split(/\r?\n/)) { + if (line.trim()) console.error(` ${line}`); + } +} + /** * Re-run the Hermes env-file secret-boundary validator against a running * gateway, before the probe path returns control to the caller. The @@ -501,42 +514,52 @@ function isHermesAgent(agent: ReturnType): * start would otherwise never be re-evaluated. The check is invoked via * `openshell sandbox exec` (root) so the validator's kill snippet can * actually signal the gateway-user process when refusing — a sandbox-user - * SSH shell cannot (test/e2e-gateway-isolation.sh test 13). Returns - * `null` when the helper has nothing to evaluate (unknown agent, exec - * unreachable, validator missing on this sandbox image); returns - * `refused: true` when the validator refused the env file and the - * snippet brought the gateway down. + * SSH shell cannot (test/e2e-gateway-isolation.sh test 13). Validator + * `[SECURITY]` stderr is surfaced on every refusal path, regardless of + * `quiet`, so the offending key (e.g. `TELEGRAM_BOT_TOKEN (line N)`) + * always reaches the operator. Returns `null` only when the agent is not + * Hermes (no boundary to enforce); a running Hermes gateway whose root + * exec channel is unreachable is treated as a fail-safe inconclusive + * refusal rather than a healthy path. Non-zero validator status without + * a `SECRET_BOUNDARY_REFUSED` marker is also reported as inconclusive, + * not as a raw-secret refusal, so a shell or validator crash does not + * masquerade as a poisoned env file. */ function enforceHermesSecretBoundaryOnRunningGateway( sandboxName: string, agent: ReturnType, { quiet }: { quiet: boolean }, -): { refused: boolean } | null { +): HermesSecretBoundaryEnforcement | null { if (!isHermesAgent(agent)) return null; const script = buildHermesEnvFileBoundaryStandaloneCheck(); const result = executeSandboxExecCommand(sandboxName, script, 30000); - if (!result) return null; + if (!result) { + if (!quiet) { + console.error(""); + console.error( + ` ${R}Secret-boundary check could not run against the Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error(" Refusing recovery to keep the validator-enforced boundary intact."); + } + return { refused: true, reason: "inconclusive", stderr: "" }; + } const stdoutMarker = result.stdout .split(/\r?\n/) .reverse() .find((line) => line.trim().startsWith("SECRET_BOUNDARY_")); - if (stdoutMarker === SECRET_BOUNDARY_REFUSED_MARKER || result.status !== 0) { + if (stdoutMarker === SECRET_BOUNDARY_REFUSED_MARKER) { + printValidatorStderr(result.stderr); if (!quiet) { console.error(""); console.error( ` ${R}Secret-boundary check refused recovery of Hermes gateway in '${sandboxName}'.${R}`, ); - if (result.stderr.trim()) { - for (const line of result.stderr.split(/\r?\n/)) { - if (line.trim()) console.error(` ${line}`); - } - } console.error(" /sandbox/.hermes/.env contains raw secret-shaped values. Replace them with"); console.error( " openshell:resolve:env: placeholders and re-run `nemoclaw recover`.", ); } - return { refused: true }; + return { refused: true, reason: "raw-secret", stderr: result.stderr }; } if ( stdoutMarker === SECRET_BOUNDARY_OK_MARKER || @@ -544,7 +567,17 @@ function enforceHermesSecretBoundaryOnRunningGateway( ) { return { refused: false }; } - return null; + printValidatorStderr(result.stderr); + if (!quiet) { + console.error(""); + console.error( + ` ${R}Secret-boundary check did not complete cleanly for Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error( + " Refusing recovery; inspect the validator output above before re-running `nemoclaw recover`.", + ); + } + return { refused: true, reason: "inconclusive", stderr: result.stderr }; } /** @@ -552,7 +585,7 @@ function enforceHermesSecretBoundaryOnRunningGateway( * whose OpenClaw processes are not running. Also re-establishes the * host-side dashboard port-forward when it has gone dead independently * of the gateway. Returns an object describing the outcome: - * `{ checked, wasRunning, recovered, forwardRecovered, secretBoundaryRefused? }`. + * `{ checked, wasRunning, recovered, forwardRecovered, secretBoundaryRefused?, secretBoundaryReason? }`. */ export function checkAndRecoverSandboxProcesses( sandboxName: string, @@ -575,6 +608,7 @@ export function checkAndRecoverSandboxProcesses( recovered: false, forwardRecovered: false, secretBoundaryRefused: true, + secretBoundaryReason: enforcement.reason, }; } } diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index 311d1955217..cf6840ce2d6 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -186,11 +186,24 @@ beta 127.0.0.1 18789 12345 dead`; beta 127.0.0.1 18789 12345 running`; let forwardListCalls = 0; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "beta", @@ -318,11 +331,24 @@ hermes-box 127.0.0.1 18789 12345 running hermes-box 127.0.0.1 8642 12346 running`; let secondaryStarted = false; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -396,11 +422,24 @@ hermes-box 127.0.0.1 8642 12346 running`; hermes-box 127.0.0.1 18789 12345 running sibling-box 127.0.0.1 8642 99999 running`; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -445,11 +484,24 @@ sibling-box 127.0.0.1 8642 99999 running`; const primaryOnlyForward = `SANDBOX BIND PORT PID STATUS hermes-box 127.0.0.1 18789 12345 running`; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -490,11 +542,24 @@ hermes-box 127.0.0.1 18789 12345 running`; hermes-box 127.0.0.1 18789 12345 running hermes-box 127.0.0.1 8642 12346 running`; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -599,6 +664,7 @@ hermes-box 127.0.0.1 8642 12346 running`; recovered: false, forwardRecovered: false, secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", }); expect(secretBoundaryCalls).toBe(1); expect(forwardListCalls).toBe(0); @@ -764,4 +830,126 @@ hermes-box 127.0.0.1 8642 12346 running`; withFakeOpenshellBinary(() => checkAndRecoverSandboxProcesses("beta", { quiet: true })); expect(secretBoundaryCalls).toBe(0); }); + + it("fails safe on a running Hermes gateway when the root exec channel is unreachable", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + let forwardListCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { status: 1, stdout: "", stderr: "openshell: connection refused" } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockImplementation(() => { + forwardListCalls += 1; + return { + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }; + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }); + expect(secretBoundaryCalls).toBe(1); + expect(forwardListCalls).toBe(0); + expect(captureOpenshell).not.toHaveBeenCalled(); + }); + + it("treats a non-zero boundary check without the REFUSED marker as inconclusive, not raw-secret", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { + status: 2, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\n", + stderr: "python3: validator crashed: ImportError: no module named foo", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }); + expect(secretBoundaryCalls).toBe(1); + }); }); From bff177c823f77575505d9ad1a26eb256dff1631b Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 17 Jun 2026 09:22:31 +0000 Subject: [PATCH 3/4] fix(sandbox/recover): gate Hermes boundary on registry agent and cover probe refusal at the command boundary Signed-off-by: Tinson Lai --- docs/reference/commands-nemohermes.mdx | 1 + docs/reference/commands.mdx | 1 + src/lib/actions/sandbox/connect-flow.test.ts | 73 ++++++++++++++++++++ src/lib/actions/sandbox/process-recovery.ts | 29 ++++++-- test/process-recovery.test.ts | 64 +++++++++++++++++ 5 files changed, 161 insertions(+), 7 deletions(-) diff --git a/docs/reference/commands-nemohermes.mdx b/docs/reference/commands-nemohermes.mdx index 8b121dc8894..200387544b8 100644 --- a/docs/reference/commands-nemohermes.mdx +++ b/docs/reference/commands-nemohermes.mdx @@ -503,6 +503,7 @@ nemohermes my-assistant recover `recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. +Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` records the missing validator and proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. ### `nemohermes status` diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 2a38721b0db..f4113d4e7bd 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -623,6 +623,7 @@ $$nemoclaw my-assistant recover `recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. +Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` records the missing validator and proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index 32e874eae08..1a442f5144f 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -29,6 +29,8 @@ type ConnectHarnessOptions = { wasRunning?: boolean; recovered?: boolean; forwardRecovered?: boolean; + secretBoundaryRefused?: boolean; + secretBoundaryReason?: "raw-secret" | "inconclusive"; }; spawnStatus?: number | null; }; @@ -222,4 +224,75 @@ describe("connectSandbox flow", () => { ); expect(exitSpy).toHaveBeenCalledWith(1); }); + + it("probe-only mode exits with raw-secret remediation when the Hermes boundary refuses recovery", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", + }, + }); + const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + const errorSpy = vi.spyOn(console, "error"); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( + "openshell", + ["sandbox", "connect", "alpha"], + expect.any(Object), + ); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Probe failed: refused to confirm Hermes gateway in 'alpha' — /sandbox/.hermes/.env contains raw secret-shaped values.", + ); + expect(errorOutput).toContain( + "Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + const logOutput = harness.logSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(logOutput).not.toContain("Probe complete"); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + + it("probe-only mode exits with inconclusive guidance when the Hermes boundary check could not run", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }, + }); + const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + const errorSpy = vi.spyOn(console, "error"); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Probe failed: secret-boundary check did not complete for Hermes gateway in 'alpha'.", + ); + expect(errorOutput).toContain( + "Inspect the validator output above and re-run `nemoclaw recover`.", + ); + const logOutput = harness.logSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(logOutput).not.toContain("Probe complete"); + expect(exitSpy).toHaveBeenCalledWith(1); + }); }); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 59f23924dc1..f34adfc3100 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -517,12 +517,16 @@ function printValidatorStderr(stderr: string): void { * SSH shell cannot (test/e2e-gateway-isolation.sh test 13). Validator * `[SECURITY]` stderr is surfaced on every refusal path, regardless of * `quiet`, so the offending key (e.g. `TELEGRAM_BOT_TOKEN (line N)`) - * always reaches the operator. Returns `null` only when the agent is not - * Hermes (no boundary to enforce); a running Hermes gateway whose root - * exec channel is unreachable is treated as a fail-safe inconclusive + * always reaches the operator. Returns `null` only when the persisted + * sandbox registry entry is not Hermes (no boundary to enforce). When + * the registry says Hermes but the in-memory agent definition failed to + * load (`getSessionAgent()` returned `null` from its catch path), the + * helper fails safe with an inconclusive refusal rather than silently + * skipping the boundary. A running Hermes gateway whose root exec + * channel is unreachable is also treated as a fail-safe inconclusive * refusal rather than a healthy path. Non-zero validator status without - * a `SECRET_BOUNDARY_REFUSED` marker is also reported as inconclusive, - * not as a raw-secret refusal, so a shell or validator crash does not + * a `SECRET_BOUNDARY_REFUSED` marker is reported as inconclusive, not as + * a raw-secret refusal, so a shell or validator crash does not * masquerade as a poisoned env file. */ function enforceHermesSecretBoundaryOnRunningGateway( @@ -530,7 +534,18 @@ function enforceHermesSecretBoundaryOnRunningGateway( agent: ReturnType, { quiet }: { quiet: boolean }, ): HermesSecretBoundaryEnforcement | null { - if (!isHermesAgent(agent)) return null; + const persistedAgent = registry.getSandbox(sandboxName)?.agent; + if (persistedAgent !== "hermes") return null; + if (!isHermesAgent(agent)) { + if (!quiet) { + console.error(""); + console.error( + ` ${R}Hermes agent definition could not be loaded for sandbox '${sandboxName}'.${R}`, + ); + console.error(" Refusing recovery to keep the validator-enforced boundary intact."); + } + return { refused: true, reason: "inconclusive", stderr: "" }; + } const script = buildHermesEnvFileBoundaryStandaloneCheck(); const result = executeSandboxExecCommand(sandboxName, script, 30000); if (!result) { @@ -597,7 +612,7 @@ export function checkAndRecoverSandboxProcesses( } const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); const recoveryPort = resolveSandboxDashboardPort(sandboxName); - if (running && isHermesAgent(recoveryAgent)) { + if (running) { const enforcement = enforceHermesSecretBoundaryOnRunningGateway(sandboxName, recoveryAgent, { quiet, }); diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index cf6840ce2d6..f550e720c86 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -606,6 +606,7 @@ hermes-box 127.0.0.1 8642 12346 running`; const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); const registry = requireDist("../dist/lib/state/registry.js"); const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); let secretBoundaryCalls = 0; let forwardListCalls = 0; @@ -675,6 +676,69 @@ hermes-box 127.0.0.1 8642 12346 running`; return args[0] === "forward"; }), ).toBe(false); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "[SECURITY] Refusing Hermes startup because /sandbox/.hermes/.env contains raw secret-shaped values", + ); + expect(errorOutput).toContain("[SECURITY] TELEGRAM_BOT_TOKEN (line 3)"); + }); + + it("fails safe on a running Hermes sandbox when the agent definition cannot be loaded", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + let forwardListCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockImplementation(() => { + forwardListCalls += 1; + return { + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }; + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }); + expect(secretBoundaryCalls).toBe(0); + expect(forwardListCalls).toBe(0); + expect(captureOpenshell).not.toHaveBeenCalled(); }); it("falls through to the forward-refresh path when the Hermes secret-boundary check passes", () => { From fd13c69793f1414d48eb09d454ddfc59c20868a8 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 17 Jun 2026 09:59:06 +0000 Subject: [PATCH 4/4] fix(sandbox/recover): surface every secret-boundary refusal diagnostic and stop the non-probe connect path on refusal Signed-off-by: Tinson Lai --- docs/reference/commands-nemohermes.mdx | 2 +- docs/reference/commands.mdx | 2 +- src/lib/actions/sandbox/connect-flow.test.ts | 35 +++++++++ src/lib/actions/sandbox/connect.ts | 53 ++++++++----- src/lib/actions/sandbox/process-recovery.ts | 80 +++++++++----------- test/process-recovery.test.ts | 22 ++++++ 6 files changed, 129 insertions(+), 65 deletions(-) diff --git a/docs/reference/commands-nemohermes.mdx b/docs/reference/commands-nemohermes.mdx index 200387544b8..89c31edb2d2 100644 --- a/docs/reference/commands-nemohermes.mdx +++ b/docs/reference/commands-nemohermes.mdx @@ -503,7 +503,7 @@ nemohermes my-assistant recover `recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. -Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` records the missing validator and proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. +Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` prints a `[boundary]` warning naming the sandbox and noting that `/sandbox/.hermes/.env` was not re-evaluated, then proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. ### `nemohermes status` diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f4113d4e7bd..5fbed3b396b 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -623,7 +623,7 @@ $$nemoclaw my-assistant recover `recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. -Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` records the missing validator and proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. +Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` prints a `[boundary]` warning naming the sandbox and noting that `/sandbox/.hermes/.env` was not re-evaluated, then proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index 1a442f5144f..cc8ae9db02a 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -263,6 +263,41 @@ describe("connectSandbox flow", () => { expect(exitSpy).toHaveBeenCalledWith(1); }); + it("non-probe connect exits before Ollama/inference-route/auto-pair when the Hermes boundary refuses", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", + }, + }); + const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + const errorSpy = vi.spyOn(console, "error"); + + await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(1)"); + + expect(harness.ensureOllamaAuthProxySpy).not.toHaveBeenCalled(); + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( + "openshell", + ["sandbox", "connect", "alpha"], + expect.any(Object), + ); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Connect failed: refused to confirm Hermes gateway in 'alpha' — /sandbox/.hermes/.env contains raw secret-shaped values.", + ); + expect(errorOutput).toContain( + "Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + it("probe-only mode exits with inconclusive guidance when the Hermes boundary check could not run", async () => { const harness = createConnectHarness({ processCheck: { diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 2e6af863b16..81cc6f5f93b 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -183,6 +183,33 @@ export function parseSandboxConnectArgs( return options; } +function exitOnSecretBoundaryRefusal( + sandboxName: string, + agentName: string, + processCheck: Record, + contextLabel: "Probe" | "Connect", +): never { + console.error(""); + const reason = + "secretBoundaryReason" in processCheck + ? (processCheck.secretBoundaryReason as "raw-secret" | "inconclusive" | undefined) + : undefined; + if (reason === "raw-secret") { + console.error( + ` ${contextLabel} failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, + ); + console.error( + " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + } else { + console.error( + ` ${contextLabel} failed: secret-boundary check did not complete for ${agentName} gateway in '${sandboxName}'.`, + ); + console.error(" Inspect the validator output above and re-run `nemoclaw recover`."); + } + process.exit(1); +} + function runSandboxConnectProbe(sandboxName: string): void { const processCheck = checkAndRecoverSandboxProcesses(sandboxName, { quiet: true }); const agent = agentRuntime.getSessionAgent(sandboxName); @@ -194,25 +221,7 @@ function runSandboxConnectProbe(sandboxName: string): void { process.exit(1); } if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { - console.error(""); - const reason = - "secretBoundaryReason" in processCheck ? processCheck.secretBoundaryReason : undefined; - if (reason === "raw-secret") { - console.error( - ` Probe failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, - ); - console.error( - " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", - ); - } else { - console.error( - ` Probe failed: secret-boundary check did not complete for ${agentName} gateway in '${sandboxName}'.`, - ); - console.error( - " Inspect the validator output above and re-run `nemoclaw recover`.", - ); - } - process.exit(1); + exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Probe"); } if (processCheck.wasRunning) { ensureSandboxInferenceRoute(sandboxName, { quiet: true }); @@ -882,7 +891,11 @@ export async function connectSandbox( /* non-fatal — don't block connect on session detection failure */ } - checkAndRecoverSandboxProcesses(sandboxName); + const processCheck = checkAndRecoverSandboxProcesses(sandboxName); + if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { + const agentName = agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName)); + exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Connect"); + } // Ensure Ollama auth proxy is running (recovers from host reboots) ensureOllamaAuthProxy(); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index f34adfc3100..2693bd0408b 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -514,10 +514,12 @@ function printValidatorStderr(stderr: string): void { * start would otherwise never be re-evaluated. The check is invoked via * `openshell sandbox exec` (root) so the validator's kill snippet can * actually signal the gateway-user process when refusing — a sandbox-user - * SSH shell cannot (test/e2e-gateway-isolation.sh test 13). Validator - * `[SECURITY]` stderr is surfaced on every refusal path, regardless of - * `quiet`, so the offending key (e.g. `TELEGRAM_BOT_TOKEN (line N)`) - * always reaches the operator. Returns `null` only when the persisted + * SSH shell cannot (test/e2e-gateway-isolation.sh test 13). Every + * refusal diagnostic — validator `[SECURITY]` stderr, the helper's own + * context line, and the remediation hint — is written to `console.error` + * unconditionally, so the offending key (e.g. `TELEGRAM_BOT_TOKEN (line + * N)`) and the reason for refusal always reach the operator, including + * on the quiet probe/recover path. Returns `null` only when the persisted * sandbox registry entry is not Hermes (no boundary to enforce). When * the registry says Hermes but the in-memory agent definition failed to * load (`getSessionAgent()` returned `null` from its catch path), the @@ -532,30 +534,25 @@ function printValidatorStderr(stderr: string): void { function enforceHermesSecretBoundaryOnRunningGateway( sandboxName: string, agent: ReturnType, - { quiet }: { quiet: boolean }, ): HermesSecretBoundaryEnforcement | null { const persistedAgent = registry.getSandbox(sandboxName)?.agent; if (persistedAgent !== "hermes") return null; if (!isHermesAgent(agent)) { - if (!quiet) { - console.error(""); - console.error( - ` ${R}Hermes agent definition could not be loaded for sandbox '${sandboxName}'.${R}`, - ); - console.error(" Refusing recovery to keep the validator-enforced boundary intact."); - } + console.error(""); + console.error( + ` ${R}Hermes agent definition could not be loaded for sandbox '${sandboxName}'.${R}`, + ); + console.error(" Refusing recovery to keep the validator-enforced boundary intact."); return { refused: true, reason: "inconclusive", stderr: "" }; } const script = buildHermesEnvFileBoundaryStandaloneCheck(); const result = executeSandboxExecCommand(sandboxName, script, 30000); if (!result) { - if (!quiet) { - console.error(""); - console.error( - ` ${R}Secret-boundary check could not run against the Hermes gateway in '${sandboxName}'.${R}`, - ); - console.error(" Refusing recovery to keep the validator-enforced boundary intact."); - } + console.error(""); + console.error( + ` ${R}Secret-boundary check could not run against the Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error(" Refusing recovery to keep the validator-enforced boundary intact."); return { refused: true, reason: "inconclusive", stderr: "" }; } const stdoutMarker = result.stdout @@ -564,34 +561,33 @@ function enforceHermesSecretBoundaryOnRunningGateway( .find((line) => line.trim().startsWith("SECRET_BOUNDARY_")); if (stdoutMarker === SECRET_BOUNDARY_REFUSED_MARKER) { printValidatorStderr(result.stderr); - if (!quiet) { - console.error(""); - console.error( - ` ${R}Secret-boundary check refused recovery of Hermes gateway in '${sandboxName}'.${R}`, - ); - console.error(" /sandbox/.hermes/.env contains raw secret-shaped values. Replace them with"); - console.error( - " openshell:resolve:env: placeholders and re-run `nemoclaw recover`.", - ); - } + console.error(""); + console.error( + ` ${R}Secret-boundary check refused recovery of Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error(" /sandbox/.hermes/.env contains raw secret-shaped values. Replace them with"); + console.error( + " openshell:resolve:env: placeholders and re-run `nemoclaw recover`.", + ); return { refused: true, reason: "raw-secret", stderr: result.stderr }; } - if ( - stdoutMarker === SECRET_BOUNDARY_OK_MARKER || - stdoutMarker === SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER - ) { + if (stdoutMarker === SECRET_BOUNDARY_OK_MARKER) { return { refused: false }; } - printValidatorStderr(result.stderr); - if (!quiet) { - console.error(""); - console.error( - ` ${R}Secret-boundary check did not complete cleanly for Hermes gateway in '${sandboxName}'.${R}`, - ); + if (stdoutMarker === SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER) { console.error( - " Refusing recovery; inspect the validator output above before re-running `nemoclaw recover`.", + ` [boundary] Hermes secret-boundary validator missing in sandbox '${sandboxName}'; recover proceeded without re-evaluating /sandbox/.hermes/.env. Re-image the sandbox to enable per-run enforcement.`, ); + return { refused: false }; } + printValidatorStderr(result.stderr); + console.error(""); + console.error( + ` ${R}Secret-boundary check did not complete cleanly for Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error( + " Refusing recovery; inspect the validator output above before re-running `nemoclaw recover`.", + ); return { refused: true, reason: "inconclusive", stderr: result.stderr }; } @@ -613,9 +609,7 @@ export function checkAndRecoverSandboxProcesses( const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); const recoveryPort = resolveSandboxDashboardPort(sandboxName); if (running) { - const enforcement = enforceHermesSecretBoundaryOnRunningGateway(sandboxName, recoveryAgent, { - quiet, - }); + const enforcement = enforceHermesSecretBoundaryOnRunningGateway(sandboxName, recoveryAgent); if (enforcement?.refused) { return { checked: true, diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index f550e720c86..b79d4b90e4b 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -688,6 +688,7 @@ hermes-box 127.0.0.1 8642 12346 running`; const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); const registry = requireDist("../dist/lib/state/registry.js"); const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); let secretBoundaryCalls = 0; let forwardListCalls = 0; @@ -739,6 +740,10 @@ hermes-box 127.0.0.1 8642 12346 running`; expect(secretBoundaryCalls).toBe(0); expect(forwardListCalls).toBe(0); expect(captureOpenshell).not.toHaveBeenCalled(); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Hermes agent definition could not be loaded for sandbox 'hermes-box'", + ); }); it("falls through to the forward-refresh path when the Hermes secret-boundary check passes", () => { @@ -806,6 +811,7 @@ hermes-box 127.0.0.1 8642 12346 running`; const registry = requireDist("../dist/lib/state/registry.js"); const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(childProcess, "spawnSync").mockImplementation( (_command: unknown, rawArgs: unknown) => { @@ -854,6 +860,11 @@ hermes-box 127.0.0.1 8642 12346 running`; recovered: false, forwardRecovered: false, }); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "[boundary] Hermes secret-boundary validator missing in sandbox 'hermes-box'", + ); + expect(errorOutput).toContain("Re-image the sandbox to enable per-run enforcement."); }); it("does not invoke the Hermes secret-boundary check for an OpenClaw sandbox", () => { @@ -900,6 +911,7 @@ hermes-box 127.0.0.1 8642 12346 running`; const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); const registry = requireDist("../dist/lib/state/registry.js"); const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); let secretBoundaryCalls = 0; let forwardListCalls = 0; @@ -956,6 +968,10 @@ hermes-box 127.0.0.1 8642 12346 running`; expect(secretBoundaryCalls).toBe(1); expect(forwardListCalls).toBe(0); expect(captureOpenshell).not.toHaveBeenCalled(); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Secret-boundary check could not run against the Hermes gateway in 'hermes-box'", + ); }); it("treats a non-zero boundary check without the REFUSED marker as inconclusive, not raw-secret", () => { @@ -963,6 +979,7 @@ hermes-box 127.0.0.1 8642 12346 running`; const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); const registry = requireDist("../dist/lib/state/registry.js"); const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); let secretBoundaryCalls = 0; vi.spyOn(childProcess, "spawnSync").mockImplementation( @@ -1015,5 +1032,10 @@ hermes-box 127.0.0.1 8642 12346 running`; secretBoundaryReason: "inconclusive", }); expect(secretBoundaryCalls).toBe(1); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("python3: validator crashed: ImportError: no module named foo"); + expect(errorOutput).toContain( + "Secret-boundary check did not complete cleanly for Hermes gateway in 'hermes-box'", + ); }); });