diff --git a/docs/reference/commands-nemohermes.mdx b/docs/reference/commands-nemohermes.mdx index 0f60d3464f9..89c31edb2d2 100644 --- a/docs/reference/commands-nemohermes.mdx +++ b/docs/reference/commands-nemohermes.mdx @@ -500,6 +500,11 @@ If the gateway is already running, the command exits zero with a probe message a nemohermes my-assistant recover ``` +`recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. +If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. +Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. +Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` prints a `[boundary]` warning naming the sandbox and noting that `/sandbox/.hermes/.env` was not re-evaluated, then proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. + ### `nemohermes status` Show sandbox status, health, and inference configuration. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 268a2d55714..5fbed3b396b 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -618,6 +618,15 @@ If the gateway is already running, the command exits zero with a probe message a $$nemoclaw my-assistant recover ``` + + +`recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` on every run, including when the gateway is already healthy. +If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command stops the running gateway, exits non-zero, and prints the offending key. +Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. +Older Hermes sandbox images that predate the standalone validator are detected and left untouched: `recover` prints a `[boundary]` warning naming the sandbox and noting that `/sandbox/.hermes/.env` was not re-evaluated, then proceeds with the rest of the recovery path rather than blocking the gateway. Re-image the sandbox to a current Hermes build to enable the per-run boundary re-evaluation described above. + + + ### `$$nemoclaw status` Show sandbox status, health, and inference configuration. diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index 32e874eae08..cc8ae9db02a 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -29,6 +29,8 @@ type ConnectHarnessOptions = { wasRunning?: boolean; recovered?: boolean; forwardRecovered?: boolean; + secretBoundaryRefused?: boolean; + secretBoundaryReason?: "raw-secret" | "inconclusive"; }; spawnStatus?: number | null; }; @@ -222,4 +224,110 @@ describe("connectSandbox flow", () => { ); expect(exitSpy).toHaveBeenCalledWith(1); }); + + it("probe-only mode exits with raw-secret remediation when the Hermes boundary refuses recovery", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", + }, + }); + const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + const errorSpy = vi.spyOn(console, "error"); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( + "openshell", + ["sandbox", "connect", "alpha"], + expect.any(Object), + ); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Probe failed: refused to confirm Hermes gateway in 'alpha' — /sandbox/.hermes/.env contains raw secret-shaped values.", + ); + expect(errorOutput).toContain( + "Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + const logOutput = harness.logSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(logOutput).not.toContain("Probe complete"); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + + it("non-probe connect exits before Ollama/inference-route/auto-pair when the Hermes boundary refuses", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", + }, + }); + const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + const errorSpy = vi.spyOn(console, "error"); + + await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(1)"); + + expect(harness.ensureOllamaAuthProxySpy).not.toHaveBeenCalled(); + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( + "openshell", + ["sandbox", "connect", "alpha"], + expect.any(Object), + ); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Connect failed: refused to confirm Hermes gateway in 'alpha' — /sandbox/.hermes/.env contains raw secret-shaped values.", + ); + expect(errorOutput).toContain( + "Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + + it("probe-only mode exits with inconclusive guidance when the Hermes boundary check could not run", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }, + }); + const agentRuntime = requireDist("../../../../dist/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + const errorSpy = vi.spyOn(console, "error"); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Probe failed: secret-boundary check did not complete for Hermes gateway in 'alpha'.", + ); + expect(errorOutput).toContain( + "Inspect the validator output above and re-run `nemoclaw recover`.", + ); + const logOutput = harness.logSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(logOutput).not.toContain("Probe complete"); + expect(exitSpy).toHaveBeenCalledWith(1); + }); }); diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 58a4f1e8af4..81cc6f5f93b 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -27,7 +27,6 @@ import { findReachableOllamaHost, probeLocalProviderHealth } from "../../inferen import { ensureOllamaAuthProxy, probeOllamaAuthProxyHealth } from "../../inference/ollama/proxy"; import { LOCAL_INFERENCE_TIMEOUT_SECS } from "../../onboard/env"; import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; -import { getSandboxTargetGatewayName } from "./gateway-target"; import { isWsl } from "../../platform"; import { ROOT } from "../../runner"; import * as sandboxVersion from "../../sandbox/version"; @@ -53,6 +52,7 @@ import { import { preflightVllmModelEnvOrExit } from "./connect-vllm-preflight"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state"; +import { getSandboxTargetGatewayName } from "./gateway-target"; import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand } from "./process-recovery"; import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; @@ -183,6 +183,33 @@ export function parseSandboxConnectArgs( return options; } +function exitOnSecretBoundaryRefusal( + sandboxName: string, + agentName: string, + processCheck: Record, + contextLabel: "Probe" | "Connect", +): never { + console.error(""); + const reason = + "secretBoundaryReason" in processCheck + ? (processCheck.secretBoundaryReason as "raw-secret" | "inconclusive" | undefined) + : undefined; + if (reason === "raw-secret") { + console.error( + ` ${contextLabel} failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, + ); + console.error( + " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + } else { + console.error( + ` ${contextLabel} failed: secret-boundary check did not complete for ${agentName} gateway in '${sandboxName}'.`, + ); + console.error(" Inspect the validator output above and re-run `nemoclaw recover`."); + } + process.exit(1); +} + function runSandboxConnectProbe(sandboxName: string): void { const processCheck = checkAndRecoverSandboxProcesses(sandboxName, { quiet: true }); const agent = agentRuntime.getSessionAgent(sandboxName); @@ -193,6 +220,9 @@ function runSandboxConnectProbe(sandboxName: string): void { ); process.exit(1); } + if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { + exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Probe"); + } if (processCheck.wasRunning) { ensureSandboxInferenceRoute(sandboxName, { quiet: true }); // Defense-in-depth scope-upgrade approval on the probe-only / `recover` @@ -861,7 +891,11 @@ export async function connectSandbox( /* non-fatal — don't block connect on session detection failure */ } - checkAndRecoverSandboxProcesses(sandboxName); + const processCheck = checkAndRecoverSandboxProcesses(sandboxName); + if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { + const agentName = agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName)); + exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Connect"); + } // Ensure Ollama auth proxy is running (recovers from host reboots) ensureOllamaAuthProxy(); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 871daee87a0..2693bd0408b 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -11,6 +11,12 @@ import { runOpenshell, } from "../../adapters/openshell/runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; +import { + buildHermesEnvFileBoundaryStandaloneCheck, + SECRET_BOUNDARY_OK_MARKER, + SECRET_BOUNDARY_REFUSED_MARKER, + SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER, +} from "../../agent/hermes-recovery-boundary"; import * as agentRuntime from "../../agent/runtime"; import { G, R } from "../../cli/terminal-style"; import { DASHBOARD_PORT } from "../../core/ports"; @@ -482,12 +488,115 @@ function recoverHermesDashboardProcessIfEnabled(sandboxName: string): boolean | return recoverHermesDashboardProcess(sandboxName, { executeCommand: executeSandboxCommand }); } +function isHermesAgent(agent: ReturnType): boolean { + return !!agent && agent.name === "hermes"; +} + +type SecretBoundaryRefusalReason = "raw-secret" | "inconclusive"; + +type HermesSecretBoundaryEnforcement = + | { refused: false } + | { refused: true; reason: SecretBoundaryRefusalReason; stderr: string }; + +function printValidatorStderr(stderr: string): void { + if (!stderr.trim()) return; + for (const line of stderr.split(/\r?\n/)) { + if (line.trim()) console.error(` ${line}`); + } +} + +/** + * Re-run the Hermes env-file secret-boundary validator against a running + * gateway, before the probe path returns control to the caller. The + * relaunch path already runs the same validator inline as part of + * `buildRecoveryScript`, but the probe path returns early as soon as the + * gateway is reported healthy, so a poisoned `.env` injected after cold + * start would otherwise never be re-evaluated. The check is invoked via + * `openshell sandbox exec` (root) so the validator's kill snippet can + * actually signal the gateway-user process when refusing — a sandbox-user + * SSH shell cannot (test/e2e-gateway-isolation.sh test 13). Every + * refusal diagnostic — validator `[SECURITY]` stderr, the helper's own + * context line, and the remediation hint — is written to `console.error` + * unconditionally, so the offending key (e.g. `TELEGRAM_BOT_TOKEN (line + * N)`) and the reason for refusal always reach the operator, including + * on the quiet probe/recover path. Returns `null` only when the persisted + * sandbox registry entry is not Hermes (no boundary to enforce). When + * the registry says Hermes but the in-memory agent definition failed to + * load (`getSessionAgent()` returned `null` from its catch path), the + * helper fails safe with an inconclusive refusal rather than silently + * skipping the boundary. A running Hermes gateway whose root exec + * channel is unreachable is also treated as a fail-safe inconclusive + * refusal rather than a healthy path. Non-zero validator status without + * a `SECRET_BOUNDARY_REFUSED` marker is reported as inconclusive, not as + * a raw-secret refusal, so a shell or validator crash does not + * masquerade as a poisoned env file. + */ +function enforceHermesSecretBoundaryOnRunningGateway( + sandboxName: string, + agent: ReturnType, +): HermesSecretBoundaryEnforcement | null { + const persistedAgent = registry.getSandbox(sandboxName)?.agent; + if (persistedAgent !== "hermes") return null; + if (!isHermesAgent(agent)) { + console.error(""); + console.error( + ` ${R}Hermes agent definition could not be loaded for sandbox '${sandboxName}'.${R}`, + ); + console.error(" Refusing recovery to keep the validator-enforced boundary intact."); + return { refused: true, reason: "inconclusive", stderr: "" }; + } + const script = buildHermesEnvFileBoundaryStandaloneCheck(); + const result = executeSandboxExecCommand(sandboxName, script, 30000); + if (!result) { + console.error(""); + console.error( + ` ${R}Secret-boundary check could not run against the Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error(" Refusing recovery to keep the validator-enforced boundary intact."); + return { refused: true, reason: "inconclusive", stderr: "" }; + } + const stdoutMarker = result.stdout + .split(/\r?\n/) + .reverse() + .find((line) => line.trim().startsWith("SECRET_BOUNDARY_")); + if (stdoutMarker === SECRET_BOUNDARY_REFUSED_MARKER) { + printValidatorStderr(result.stderr); + console.error(""); + console.error( + ` ${R}Secret-boundary check refused recovery of Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error(" /sandbox/.hermes/.env contains raw secret-shaped values. Replace them with"); + console.error( + " openshell:resolve:env: placeholders and re-run `nemoclaw recover`.", + ); + return { refused: true, reason: "raw-secret", stderr: result.stderr }; + } + if (stdoutMarker === SECRET_BOUNDARY_OK_MARKER) { + return { refused: false }; + } + if (stdoutMarker === SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER) { + console.error( + ` [boundary] Hermes secret-boundary validator missing in sandbox '${sandboxName}'; recover proceeded without re-evaluating /sandbox/.hermes/.env. Re-image the sandbox to enable per-run enforcement.`, + ); + return { refused: false }; + } + printValidatorStderr(result.stderr); + console.error(""); + console.error( + ` ${R}Secret-boundary check did not complete cleanly for Hermes gateway in '${sandboxName}'.${R}`, + ); + console.error( + " Refusing recovery; inspect the validator output above before re-running `nemoclaw recover`.", + ); + return { refused: true, reason: "inconclusive", stderr: result.stderr }; +} + /** * Detect and recover from a sandbox that survived a gateway restart but * whose OpenClaw processes are not running. Also re-establishes the * host-side dashboard port-forward when it has gone dead independently * of the gateway. Returns an object describing the outcome: - * `{ checked, wasRunning, recovered, forwardRecovered }`. + * `{ checked, wasRunning, recovered, forwardRecovered, secretBoundaryRefused?, secretBoundaryReason? }`. */ export function checkAndRecoverSandboxProcesses( sandboxName: string, @@ -499,6 +608,19 @@ export function checkAndRecoverSandboxProcesses( } const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); const recoveryPort = resolveSandboxDashboardPort(sandboxName); + if (running) { + const enforcement = enforceHermesSecretBoundaryOnRunningGateway(sandboxName, recoveryAgent); + if (enforcement?.refused) { + return { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: enforcement.reason, + }; + } + } if (running) { // Gateway is alive but the host-side forward can still be dead or // owned by another sandbox. Probe and re-establish only when diff --git a/src/lib/agent/hermes-recovery-boundary.ts b/src/lib/agent/hermes-recovery-boundary.ts index dbaa6797cb9..a80008f44f0 100644 --- a/src/lib/agent/hermes-recovery-boundary.ts +++ b/src/lib/agent/hermes-recovery-boundary.ts @@ -22,6 +22,10 @@ import { shellQuote } from "../runner"; export const HERMES_SECRET_BOUNDARY_VALIDATOR_PATH = "/usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py"; +export const SECRET_BOUNDARY_REFUSED_MARKER = "SECRET_BOUNDARY_REFUSED"; +export const SECRET_BOUNDARY_OK_MARKER = "SECRET_BOUNDARY_OK"; +export const SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER = "SECRET_BOUNDARY_VALIDATOR_MISSING"; + const HERMES_GATEWAY_PROC_PATTERN = "[h]ermes[[:space:]]+gateway([[:space:]]|$)"; const HERMES_DASHBOARD_PROC_PATTERN = "[h]ermes[[:space:]]+dashboard([[:space:]]|$)"; const HERMES_BOUNDARY_RECOVERY_LOG = "/tmp/gateway-recovery.log"; @@ -118,9 +122,56 @@ export function buildHermesRuntimeEnvBoundaryGuard(): string { return `if [ ! -f ${shellQuote(validator)} ]; then ${missingLog} elif ! ${invocation}; then ${kill} echo SECRET_BOUNDARY_REFUSED; exit 1; fi;`; } +/** + * Build a standalone shell snippet that evaluates the Hermes env-file + * secret-boundary contract without relaunching anything. Intended for the + * `sandbox recover` / `connect --probe-only` probe path, where the gateway + * is already running and the relaunch script is not reached: the host can + * exec this snippet inside the sandbox, parse the marker on stdout, and + * decide whether to refuse the probe. + * + * Marker contract on stdout (one of): + * - `SECRET_BOUNDARY_OK` — validator ran and accepted the env file. + * - `SECRET_BOUNDARY_REFUSED` — validator ran and refused; the snippet + * killed any running gateway/dashboard process before exiting non-zero. + * - `SECRET_BOUNDARY_VALIDATOR_MISSING` — validator script absent on this + * sandbox image (older image, fail-open by design). + * + * Validator stderr (`[SECURITY] …` lines) is left on the exec command's + * stderr; the caller surfaces it directly. This keeps the snippet + * independent of any `/tmp/gateway-recovery.log` setup, which matters when + * the snippet runs via `openshell sandbox exec` (root) rather than the + * sandbox-user SSH recovery shell that the relaunch path uses. + * + * The kill snippet is intentionally invoked from a context the caller + * arranges to have authority over: a sandbox-user SSH shell cannot signal + * gateway-user processes (test/e2e-gateway-isolation.sh test 13), so a + * refusal that did not also bring the listener down would log a refusal + * while `/health` kept serving. Run this via the root sandbox-exec path so + * the kill has authority. + */ +export function buildHermesEnvFileBoundaryStandaloneCheck(): string { + const validator = HERMES_SECRET_BOUNDARY_VALIDATOR_PATH; + const kill = buildHermesBoundaryKillSnippet(); + const invocation = `python3 ${shellQuote(validator)} env-file /sandbox/.hermes/.env`; + return [ + `if [ ! -f ${shellQuote(validator)} ]; then`, + ` echo ${SECRET_BOUNDARY_VALIDATOR_MISSING_MARKER}; exit 0;`, + `fi;`, + `if ${invocation}; then`, + ` echo ${SECRET_BOUNDARY_OK_MARKER}; exit 0;`, + `else`, + ` ${kill}`, + ` echo ${SECRET_BOUNDARY_REFUSED_MARKER};`, + ` exit 1;`, + `fi;`, + ].join("\n"); +} + export const __testing = { buildHermesEnvFileBoundaryGuard, buildHermesRuntimeEnvBoundaryGuard, + buildHermesEnvFileBoundaryStandaloneCheck, buildHermesBoundaryKillSnippet, HERMES_GATEWAY_PROC_PATTERN, HERMES_DASHBOARD_PROC_PATTERN, diff --git a/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts b/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts index df6e053391a..55a1aec3adb 100644 --- a/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts +++ b/src/lib/agent/runtime-hermes-secret-boundary-behavioural.test.ts @@ -196,6 +196,46 @@ describe("Hermes secret-boundary guard — guard snippet behaviour", () => { expect(result.pkillCalls.length).toBeGreaterThanOrEqual(2); expect(result.recoveryLog).toContain("[SECURITY]"); }); + + it("standalone env-file check exits 1, emits SECRET_BOUNDARY_REFUSED, kills processes when validator refuses", { + timeout: 15_000, + }, () => { + const result = runGuard({ + guard: __testing.buildHermesEnvFileBoundaryStandaloneCheck(), + pythonExit: 1, + validatorExists: true, + }); + expect(result.status).toBe(1); + expect(result.stdout).toContain("SECRET_BOUNDARY_REFUSED"); + expect(result.stdout).not.toContain("SECRET_BOUNDARY_OK"); + expect(result.stdout).not.toContain("REACHED_LAUNCH"); + expect(result.pkillCalls.length).toBeGreaterThanOrEqual(2); + expect(result.stderr).toContain("[SECURITY]"); + }); + + it("standalone env-file check exits 0 and emits SECRET_BOUNDARY_OK when validator accepts", () => { + const result = runGuard({ + guard: __testing.buildHermesEnvFileBoundaryStandaloneCheck(), + pythonExit: 0, + validatorExists: true, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("SECRET_BOUNDARY_OK"); + expect(result.stdout).not.toContain("SECRET_BOUNDARY_REFUSED"); + expect(result.pkillCalls.length).toBe(0); + }); + + it("standalone env-file check emits SECRET_BOUNDARY_VALIDATOR_MISSING and exits 0 when validator script is absent", () => { + const result = runGuard({ + guard: __testing.buildHermesEnvFileBoundaryStandaloneCheck(), + pythonExit: 0, + validatorExists: false, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("SECRET_BOUNDARY_VALIDATOR_MISSING"); + expect(result.stdout).not.toContain("SECRET_BOUNDARY_REFUSED"); + expect(result.pkillCalls.length).toBe(0); + }); }); describe("Hermes secret-boundary guard — full recovery script behaviour", () => { diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index ce784cc33ce..b79d4b90e4b 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -186,11 +186,24 @@ beta 127.0.0.1 18789 12345 dead`; beta 127.0.0.1 18789 12345 running`; let forwardListCalls = 0; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "beta", @@ -318,11 +331,24 @@ hermes-box 127.0.0.1 18789 12345 running hermes-box 127.0.0.1 8642 12346 running`; let secondaryStarted = false; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -396,11 +422,24 @@ hermes-box 127.0.0.1 8642 12346 running`; hermes-box 127.0.0.1 18789 12345 running sibling-box 127.0.0.1 8642 99999 running`; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -445,11 +484,24 @@ sibling-box 127.0.0.1 8642 99999 running`; const primaryOnlyForward = `SANDBOX BIND PORT PID STATUS hermes-box 127.0.0.1 18789 12345 running`; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -490,11 +542,24 @@ hermes-box 127.0.0.1 18789 12345 running`; hermes-box 127.0.0.1 18789 12345 running hermes-box 127.0.0.1 8642 12346 running`; - vi.spyOn(childProcess, "spawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", - stderr: "", - } as never); + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes", forwardPort: 18789, @@ -535,4 +600,442 @@ hermes-box 127.0.0.1 8642 12346 running`; expect(result.wasRunning).toBe(true); expect(result.forwardRecovered).toBe(false); }); + + it("refuses recovery of a running Hermes gateway when /sandbox/.hermes/.env contains raw secret-shaped values", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + let secretBoundaryCalls = 0; + let forwardListCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { + status: 1, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_REFUSED\n", + stderr: + "[SECURITY] Refusing Hermes startup because /sandbox/.hermes/.env contains raw secret-shaped values\n[SECURITY] TELEGRAM_BOT_TOKEN (line 3)", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockImplementation(() => { + forwardListCalls += 1; + return { + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }; + }); + const runOpenshell = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", + }); + expect(secretBoundaryCalls).toBe(1); + expect(forwardListCalls).toBe(0); + expect(captureOpenshell).not.toHaveBeenCalled(); + expect( + runOpenshell.mock.calls.some(([rawArgs]) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + return args[0] === "forward"; + }), + ).toBe(false); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "[SECURITY] Refusing Hermes startup because /sandbox/.hermes/.env contains raw secret-shaped values", + ); + expect(errorOutput).toContain("[SECURITY] TELEGRAM_BOT_TOKEN (line 3)"); + }); + + it("fails safe on a running Hermes sandbox when the agent definition cannot be loaded", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + let secretBoundaryCalls = 0; + let forwardListCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockImplementation(() => { + forwardListCalls += 1; + return { + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }; + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }); + expect(secretBoundaryCalls).toBe(0); + expect(forwardListCalls).toBe(0); + expect(captureOpenshell).not.toHaveBeenCalled(); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Hermes agent definition could not be loaded for sandbox 'hermes-box'", + ); + }); + + it("falls through to the forward-refresh path when the Hermes secret-boundary check passes", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", + stderr: "", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + }); + expect(secretBoundaryCalls).toBe(1); + }); + + it("falls through when the Hermes secret-boundary validator is absent on an older sandbox image", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); + const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_VALIDATOR_MISSING\n", + stderr: "", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + }); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "[boundary] Hermes secret-boundary validator missing in sandbox 'hermes-box'", + ); + expect(errorOutput).toContain("Re-image the sandbox to enable per-run enforcement."); + }); + + it("does not invoke the Hermes secret-boundary check for an OpenClaw sandbox", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); + const childProcess = requireDist("node:child_process"); + let secretBoundaryCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + } + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "beta", + agent: "openclaw", + dashboardPort: 18789, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nbeta 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + withFakeOpenshellBinary(() => checkAndRecoverSandboxProcesses("beta", { quiet: true })); + expect(secretBoundaryCalls).toBe(0); + }); + + it("fails safe on a running Hermes gateway when the root exec channel is unreachable", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + let secretBoundaryCalls = 0; + let forwardListCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { status: 1, stdout: "", stderr: "openshell: connection refused" } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockImplementation(() => { + forwardListCalls += 1; + return { + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }; + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }); + expect(secretBoundaryCalls).toBe(1); + expect(forwardListCalls).toBe(0); + expect(captureOpenshell).not.toHaveBeenCalled(); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain( + "Secret-boundary check could not run against the Hermes gateway in 'hermes-box'", + ); + }); + + it("treats a non-zero boundary check without the REFUSED marker as inconclusive, not raw-secret", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const childProcess = requireDist("node:child_process"); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + let secretBoundaryCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockImplementation( + (_command: unknown, rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + const shellCommand = String(args.at(-1) ?? ""); + if (shellCommand.includes("HTTP_CODE=$(curl")) { + return { + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never; + } + if (shellCommand.includes("validate-hermes-env-secret-boundary.py")) { + secretBoundaryCalls += 1; + return { + status: 2, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\n", + stderr: "python3: validator crashed: ImportError: no module named foo", + } as never; + } + return { status: 0, stdout: "", stderr: "" } as never; + }, + ); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "hermes", + forwardPort: 8642, + displayName: "Hermes Agent", + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "hermes-box", + agent: "hermes", + dashboardPort: 18789, + }); + vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ + status: 0, + output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, + }); + vi.spyOn(openshellRuntime, "runOpenshell").mockReturnValue({ status: 0 } as never); + + const result = withFakeOpenshellBinary(() => + checkAndRecoverSandboxProcesses("hermes-box", { quiet: true }), + ); + expect(result).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "inconclusive", + }); + expect(secretBoundaryCalls).toBe(1); + const errorOutput = errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("python3: validator crashed: ImportError: no module named foo"); + expect(errorOutput).toContain( + "Secret-boundary check did not complete cleanly for Hermes gateway in 'hermes-box'", + ); + }); });