diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index ad7499adc1d..884bc0d99fd 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -89,6 +89,10 @@ NONCE_RE = re.compile(r"[0-9a-f]{64}\Z") ENV_KEY_RE = re.compile(rb"[A-Za-z_][A-Za-z0-9_]*\Z") SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +HERMES_MCP_STATE_RE = re.compile( + r"# nemoclaw-hermes-mcp-state-v1 " + r"intended=[0-9a-f]{64} applied=[0-9a-f]{64}\Z" +) CONTROL_STAGES = frozenset( { "detect-agent", @@ -1340,6 +1344,29 @@ def _read_regular(path: str, limit: int) -> tuple[bytes, os.stat_result]: os.close(fd) +def _parse_locked_hermes_hash(strict: bytes) -> dict[str, str]: + records: dict[str, str] = {} + mcp_state_seen = False + try: + lines = strict.decode("ascii").splitlines() + for line in lines: + if line.startswith("#"): + if mcp_state_seen or HERMES_MCP_STATE_RE.fullmatch(line) is None: + raise ValueError("invalid Hermes MCP state metadata") + mcp_state_seen = True + continue + if mcp_state_seen: + raise ValueError("Hermes MCP state metadata must be terminal") + digest, pathname = line.split(maxsplit=1) + canonical_path = pathname.strip() + if canonical_path in records: + raise ValueError("duplicate Hermes config hash path") + records[canonical_path] = digest.lower() + except (UnicodeDecodeError, ValueError) as exc: + raise ControlError("GATEWAY_CONFIG_HASH_MISMATCH") from exc + return records + + def _verify_locked_hermes_hash() -> None: config_path = _system_path("/sandbox/.hermes/config.yaml") env_path = _system_path("/sandbox/.hermes/.env") @@ -1361,13 +1388,7 @@ def _verify_locked_hermes_hash() -> None: strict, strict_stat = _read_regular(hash_path, MAX_HASH_BYTES) if strict_stat.st_uid != 0 or stat.S_IMODE(strict_stat.st_mode) & 0o022: raise ControlError("GATEWAY_UNSAFE_CONFIG_PATH") - try: - records = {} - for line in strict.decode("ascii").splitlines(): - digest, pathname = line.split(maxsplit=1) - records[pathname.strip()] = digest.lower() - except (UnicodeDecodeError, ValueError) as exc: - raise ControlError("GATEWAY_CONFIG_HASH_MISMATCH") from exc + records = _parse_locked_hermes_hash(strict) expected_paths = { "/sandbox/.hermes/config.yaml": hashlib.sha256(config).hexdigest(), "/sandbox/.hermes/.env": hashlib.sha256(environment).hexdigest(), diff --git a/test/e2e/live/mcp-bridge-hermes-lifecycle.ts b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts index f48dc3f9c49..bf5c3329dbb 100644 --- a/test/e2e/live/mcp-bridge-hermes-lifecycle.ts +++ b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts @@ -44,6 +44,251 @@ export async function assertHermesConfig( expectExitZero(result, "Hermes MCP config contains placeholder and no raw host secret"); } +/** + * Focused #7499 live regression after the supported managed add has executed + * the real unprivileged Hermes transaction: explicitly restore shields, + * restart the real gateway, and prove the locked files and transaction state + * remain current. + */ +export async function assertHermesManagedAddSurvivesLockedGatewayRestart( + host: HostCliClient, + sandbox: SandboxClient, + sandboxName: string, + mcpUrl: string, +): Promise { + const shieldsUp = await host.nemoclaw([sandboxName, "shields", "up"], { + artifactName: "hermes-mcp-shields-up-after-add", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 3 * 60_000, + }); + expectExitZero(shieldsUp, "restore Hermes shields after managed MCP add"); + + const shieldsStatus = await host.nemoclaw([sandboxName, "shields", "status"], { + artifactName: "hermes-mcp-shields-status-after-add", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 60_000, + }); + expectExitZero(shieldsStatus, "read Hermes shields status after managed MCP add"); + expect(resultText(shieldsStatus)).toContain("Shields: UP"); + + const restart = await host.nemoclaw([sandboxName, "gateway", "restart"], { + artifactName: "hermes-mcp-add-gateway-restart", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 12 * 60_000, + }); + expectExitZero(restart, "Hermes gateway restart with managed MCP present"); + expect(resultText(restart)).toContain("Gateway restarted"); + expect(resultText(restart)).toContain("health passed"); + + const lockedIntegrity = await sandbox.execShell( + sandboxName, + trustedSandboxShellScript( + [ + "set -eu", + "test \"$(stat -c '%a %U:%G' /sandbox)\" = '1775 root:sandbox'", + "test \"$(stat -c '%a %U:%G' /sandbox/.hermes)\" = '755 root:root'", + "for path in /sandbox/.hermes/config.yaml /sandbox/.hermes/.env /etc/nemoclaw/hermes.config-hash /sandbox/.hermes/.config-hash; do", + " test \"$(stat -c '%a %U:%G' \"$path\")\" = '444 root:root'", + "done", + "cmp -s /etc/nemoclaw/hermes.config-hash /sandbox/.hermes/.config-hash", + "sha256sum -c /etc/nemoclaw/hermes.config-hash --status", + "sha256sum -c /sandbox/.hermes/.config-hash --status", + "echo HERMES_MCP_LOCKED_INTEGRITY_CURRENT", + ].join("\n"), + ), + { + artifactName: "hermes-mcp-locked-integrity-after-add-gateway-restart", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 60_000, + }, + ); + expectExitZero(lockedIntegrity, "Hermes MCP integrity anchors after gateway restart"); + expect(lockedIntegrity.stdout).toContain("HERMES_MCP_LOCKED_INTEGRITY_CURRENT"); + + const list = await host.nemoclaw([sandboxName, "mcp", "list", "--json"], { + artifactName: "hermes-mcp-list-after-add-gateway-restart", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 60_000, + }); + expectExitZero(list, "Hermes MCP list after add gateway restart"); + const listJson = JSON.parse(list.stdout) as { + bridges: Array<{ server: string; url: string; adapter: { registered: boolean | null } }>; + }; + expect(listJson.bridges).toEqual([ + expect.objectContaining({ + server: SERVER_NAME, + url: mcpUrl, + adapter: expect.objectContaining({ registered: true }), + }), + ]); + + const expectedPayload = Buffer.from( + JSON.stringify({ + present: { + [SERVER_NAME]: { + url: mcpUrl, + headers: { + Authorization: "Bearer openshell:resolve:env:FAKE_MCP_SECRET", + }, + timeout: 120, + connect_timeout: 60, + tools: { prompts: true, resources: true }, + enabled: true, + }, + }, + absent: [], + }), + "utf8", + ).toString("base64"); + const effectiveConfig = await sandbox.execShell( + sandboxName, + trustedSandboxShellScript( + [ + "set -eu", + `payload="$(printf '%s' '${expectedPayload}' | base64 -d)"`, + '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py inspect --payload "$payload"', + ].join("\n"), + ), + { + artifactName: "hermes-mcp-effective-config-after-add-gateway-restart", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET, expectedPayload], + timeoutMs: 60_000, + }, + ); + expectExitZero(effectiveConfig, "Hermes effective MCP config after add gateway restart"); + const effectiveConfigJson = JSON.parse(effectiveConfig.stdout) as { state: string }; + expect(effectiveConfigJson.state).toBe("matched"); + + const shieldsDown = await host.nemoclaw( + [ + sandboxName, + "shields", + "down", + "--timeout", + "15m", + "--reason", + "Continue managed MCP lifecycle E2E", + ], + { + artifactName: "hermes-mcp-shields-down-after-restart-proof", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 3 * 60_000, + }, + ); + expectExitZero(shieldsDown, "unlock Hermes config for remaining managed MCP lifecycle"); + await assertHermesReloadRollback(sandbox, sandboxName, mcpUrl); +} + +/** + * Inject a first-reload failure around the packaged transaction helper, then + * require its real rollback reload to restore the prior config, both integrity + * anchors, and a healthy managed gateway in the live sandbox. + */ +async function assertHermesReloadRollback( + sandbox: SandboxClient, + sandboxName: string, + mcpUrl: string, +): Promise { + const payload = JSON.stringify({ + server: "rollback_probe", + url: mcpUrl, + headers: { + Authorization: "Bearer openshell:resolve:env:FAKE_MCP_SECRET", + }, + replace_existing: false, + }); + const inspectionPayload = Buffer.from( + JSON.stringify({ + present: { + [SERVER_NAME]: { + url: mcpUrl, + headers: { + Authorization: "Bearer openshell:resolve:env:FAKE_MCP_SECRET", + }, + timeout: 120, + connect_timeout: 60, + tools: { prompts: true, resources: true }, + enabled: true, + }, + }, + absent: ["rollback_probe"], + }), + "utf8", + ).toString("base64"); + const script = [ + "set -eu", + "/opt/hermes/.venv/bin/python - <<'PY'", + "import importlib.util, json, os, pathlib, sys", + "module_path = '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py'", + "spec = importlib.util.spec_from_file_location('nemoclaw_mcp_tx_rollback_e2e', module_path)", + "assert spec is not None and spec.loader is not None", + "module = importlib.util.module_from_spec(spec)", + "sys.modules[spec.name] = module", + "spec.loader.exec_module(module)", + `payload = json.loads(${JSON.stringify(payload)})`, + "paths = (", + " pathlib.Path('/sandbox/.hermes/config.yaml'),", + " pathlib.Path('/etc/nemoclaw/hermes.config-hash'),", + " pathlib.Path('/sandbox/.hermes/.config-hash'),", + ")", + "before = {path: path.read_bytes() for path in paths}", + "if os.geteuid() == 0:", + " raise RuntimeError('rollback regression must run as the sandbox identity')", + "real_reload = module.reload_gateway", + "reload_calls = 0", + "rollback_reloaded = None", + "def fail_then_reload():", + " global reload_calls, rollback_reloaded", + " reload_calls += 1", + " if reload_calls == 1:", + " raise RuntimeError('injected first managed reload failure')", + " rollback_reloaded = real_reload()", + " return rollback_reloaded", + "module.reload_gateway = fail_then_reload", + "try:", + " module.execute('add', payload)", + "except RuntimeError as error:", + " failure = str(error)", + "else:", + " raise RuntimeError('injected managed reload failure unexpectedly succeeded')", + "if reload_calls != 2:", + " raise RuntimeError(f'rollback performed {reload_calls} reload attempts instead of 2')", + "if 'injected first managed reload failure' not in failure:", + " raise RuntimeError('transaction did not report the injected reload failure')", + "if rollback_reloaded is not True:", + " raise RuntimeError('rollback did not restore a healthy managed gateway')", + "after = {path: path.read_bytes() for path in paths}", + "if after != before:", + " raise RuntimeError('rollback did not restore config and both hash anchors exactly')", + "PY", + `inspection_payload="$(printf '%s' '${inspectionPayload}' | base64 -d)"`, + '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py inspect --payload "$inspection_payload"', + ].join("\n"); + const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { + artifactName: "hermes-mcp-failed-reload-rollback", + env: buildAvailabilityProbeEnv(), + redactionValues: [ + HOST_SECRET, + ROTATED_HOST_SECRET, + inspectionPayload, + Buffer.from(script, "utf8").toString("base64"), + ], + timeoutMs: 7 * 60_000, + }); + expectExitZero(result, "Hermes failed MCP reload restores config, hashes, and gateway"); + expect(JSON.parse(result.stdout)).toEqual({ + ok: true, + state: "current", + }); +} + // No host `nemoclaw mcp inspect` command exists; exercise the packaged CLI // through the same OpenShell sandbox boundary used by live MCP reconciliation. export async function assertHermesInspectionRejectsUnmanagedFields( diff --git a/test/e2e/live/mcp-bridge-phases.ts b/test/e2e/live/mcp-bridge-phases.ts index 90e05fd9a48..fc18e31a985 100644 --- a/test/e2e/live/mcp-bridge-phases.ts +++ b/test/e2e/live/mcp-bridge-phases.ts @@ -12,6 +12,7 @@ export const MCP_BRIDGE_PHASES = { "start Hermes inference and MCP endpoints", "onboard the Hermes MCP sandbox", "configure and inspect the Hermes MCP bridge", + "restore Hermes shields, restart, and prove rollback", "exercise lifecycle and confirm Hermes bridge removal", ], deepagents: [ diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 8268e1029bb..2b76c5ac8d8 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -26,6 +26,7 @@ import { type McpBridgeShard, resolveMcpBridgeShard } from "./mcp-bridge-agent-s import { assertHermesConfig, assertHermesInspectionRejectsUnmanagedFields, + assertHermesManagedAddSurvivesLockedGatewayRestart, assertHermesRemovalSurvivesGatewayRestart, } from "./mcp-bridge-hermes-lifecycle.ts"; import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv } from "./mcp-bridge-onboard-env.ts"; @@ -1236,13 +1237,27 @@ mcpBridgeShardTest("hermes")( await assertHermesConfig(sandbox, HERMES_SANDBOX_NAME, mcpUrl); await assertHermesInspectionRejectsUnmanagedFields(sandbox, HERMES_SANDBOX_NAME); await assertSecretAbsentFromSandbox(sandbox, HERMES_SANDBOX_NAME, ["/sandbox/.hermes"]); + progress.phase("restore Hermes shields, restart, and prove rollback"); + await assertHermesManagedAddSurvivesLockedGatewayRestart( + host, + sandbox, + HERMES_SANDBOX_NAME, + mcpUrl, + ); + await assertSecretAbsentFromSandbox( + sandbox, + HERMES_SANDBOX_NAME, + ["/sandbox/.hermes", "/tmp/nemoclaw-start.log"], + [HOST_SECRET], + "hermes-assert-secret-absent-after-add-gateway-restart", + ); + progress.phase("exercise lifecycle and confirm Hermes bridge removal"); await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { adapter: "hermes-config", artifactPrefix: "hermes", sandboxName: HERMES_SANDBOX_NAME, secretPaths: ["/sandbox/.hermes"], }); - progress.phase("exercise lifecycle and confirm Hermes bridge removal"); await assertRealAdapterToolCall(sandbox, fakeMcp, { agent: "hermes", sandboxName: HERMES_SANDBOX_NAME, diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index 69360e6cb9e..d63c8a6cbc9 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -904,7 +904,7 @@ it("carries the generated planner matrix through the workflow output and PR repo } finally { fs.rmSync(directory, { force: true, recursive: true }); } -}, 30_000); +}, 40_000); it("builds controller target matrices only from trusted runner mappings (#7031)", () => { const target = "ubuntu-repo-cloud-langchain-deepagents-code"; diff --git a/test/managed-gateway-control.test.ts b/test/managed-gateway-control.test.ts index 1e40dccc266..535eb398e77 100644 --- a/test/managed-gateway-control.test.ts +++ b/test/managed-gateway-control.test.ts @@ -15,6 +15,44 @@ const BOUNDARY_VALIDATOR = path.join( ); const NONCE = "a".repeat(64); +const HERMES_HASH_HARNESS = String.raw` +import importlib.util +import json +import sys + +spec = importlib.util.spec_from_file_location("managed_control_hash", sys.argv[1]) +control = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = control +spec.loader.exec_module(control) + +digest = "a" * 64 +config = f"{digest} /sandbox/.hermes/config.yaml" +environment = f"{digest} /sandbox/.hermes/.env" +state = ( + "# nemoclaw-hermes-mcp-state-v1 " + f"intended={digest} applied={digest}" +) + +def parse(*lines): + try: + return control._parse_locked_hermes_hash( + ("\n".join(lines) + "\n").encode("ascii") + ) + except control.ControlError as error: + return error.code + +print(json.dumps({ + "legacy": parse(config, environment), + "current": parse(config, environment, state), + "state_first": parse(state, config, environment), + "state_between": parse(config, state, environment), + "malformed_state": parse(config, environment, state + " trailing"), + "duplicate_state": parse(config, environment, state, state), + "unknown_comment": parse(config, environment, "# untrusted metadata"), + "duplicate_path": parse(config, config, environment), +}, sort_keys=True)) +`; + const PROCESS_HARNESS = String.raw` import importlib.util import contextlib @@ -1222,6 +1260,30 @@ with tempfile.TemporaryDirectory() as root: `; describe("managed gateway root control", () => { + it("accepts the authenticated Hermes MCP state record and rejects ambiguous hash files (#7499)", () => { + const result = spawnSync("python3", ["-c", HERMES_HASH_HARNESS, HELPER], { + encoding: "utf-8", + timeout: 5000, + }); + + expect(result.status, result.stderr).toBe(0); + const digest = "a".repeat(64); + const expectedRecords = { + "/sandbox/.hermes/config.yaml": digest, + "/sandbox/.hermes/.env": digest, + }; + expect(JSON.parse(result.stdout)).toEqual({ + legacy: expectedRecords, + current: expectedRecords, + state_first: "GATEWAY_CONFIG_HASH_MISMATCH", + state_between: "GATEWAY_CONFIG_HASH_MISMATCH", + malformed_state: "GATEWAY_CONFIG_HASH_MISMATCH", + duplicate_state: "GATEWAY_CONFIG_HASH_MISMATCH", + unknown_comment: "GATEWAY_CONFIG_HASH_MISMATCH", + duplicate_path: "GATEWAY_CONFIG_HASH_MISMATCH", + }); + }); + it("pins the OpenShell process tree, rejects ambiguity/reuse, and proves restart/recover", () => { const result = spawnSync("python3", ["-c", PROCESS_HARNESS, HELPER, BOUNDARY_VALIDATOR], { encoding: "utf-8",