diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index f804e4dd1d1..bbe4b726792 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -128,6 +128,7 @@ COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway- COPY agents/hermes/validate-env-secret-boundary.py /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py COPY agents/hermes/seed-dashboard-config.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py COPY agents/hermes/runtime-config-guard.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py +COPY agents/hermes/build-mcp-digest.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py COPY agents/hermes/mcp-config-transaction.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py @@ -138,10 +139,10 @@ COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/ # minimum supported Hermes sandbox base tag guarantees those artifacts and # test/sandbox-rlimit-hooks.test.ts covers that base. RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py \ - && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json \ + && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json \ && chmod 700 /usr/local/bin/nemoclaw-gateway-control \ && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py \ - && chmod 444 /usr/local/lib/nemoclaw/gateway-supervisor.sh \ + && chmod 444 /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py \ && chmod 444 /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json \ && if [ -d /usr/local/lib/nemoclaw/preloads ]; then \ chown -R 0:0 /usr/local/lib/nemoclaw/preloads \ @@ -483,16 +484,21 @@ RUN set -eu; \ && chown sandbox:sandbox /sandbox/.hermes/.hermes_history \ && chmod 660 /sandbox/.hermes/.hermes_history -# Pin config hash at build time for integrity verification at startup. +# Pin config hash at build time for integrity verification at startup. Invoke +# the installed runtime guard's `_canonical_mcp_servers_digest` directly so +# image sealing and runtime verification cannot drift onto different JSON +# canonicalization contracts. RUN mkdir -p /etc/nemoclaw \ && sha256sum /sandbox/.hermes/config.yaml /sandbox/.hermes/.env \ > /etc/nemoclaw/hermes.config-hash \ + && mcp_digest="$(/opt/hermes/.venv/bin/python -I /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py --guard /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py --config /sandbox/.hermes/config.yaml)" \ + && printf '# nemoclaw-hermes-mcp-state-v1 intended=%s applied=%s\n' "$mcp_digest" "$mcp_digest" \ + >> /etc/nemoclaw/hermes.config-hash \ && chown root:root /etc/nemoclaw/hermes.config-hash \ && chmod 444 /etc/nemoclaw/hermes.config-hash # Backward-compatible marker for host-side shields logic on older sandboxes. -RUN sha256sum /sandbox/.hermes/config.yaml /sandbox/.hermes/.env \ - > /sandbox/.hermes/.config-hash \ +RUN cp /etc/nemoclaw/hermes.config-hash /sandbox/.hermes/.config-hash \ && chmod 640 /sandbox/.hermes/.config-hash \ && chown sandbox:sandbox /sandbox/.hermes/.config-hash diff --git a/agents/hermes/build-mcp-digest.py b/agents/hermes/build-mcp-digest.py new file mode 100644 index 00000000000..90ef917c5ed --- /dev/null +++ b/agents/hermes/build-mcp-digest.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the image seal with the runtime guard's canonical MCP function.""" + +from __future__ import annotations + +import argparse +import importlib.util +import sys +from pathlib import Path +from types import ModuleType +from typing import Callable, cast + + +def _load_guard(path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location("hermes_runtime_config_guard", path) + if spec is None or spec.loader is None: + raise RuntimeError("Hermes runtime config guard cannot be loaded") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--guard", required=True, type=Path) + parser.add_argument("--config", required=True, type=Path) + args = parser.parse_args() + + guard = _load_guard(args.guard) + canonicalizer = cast(Callable[[str], str], guard._canonical_mcp_servers_digest) + print(canonicalizer(args.config.read_text(encoding="utf-8"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agents/hermes/mcp-config-transaction.py b/agents/hermes/mcp-config-transaction.py index b877567bbc0..d848a26c6ac 100755 --- a/agents/hermes/mcp-config-transaction.py +++ b/agents/hermes/mcp-config-transaction.py @@ -116,10 +116,18 @@ def _load_credential_boundary_manifest() -> dict[str, object]: # sourceBoundary: NemoClaw owns one reviewed manifest installed beside this # helper in images; the second path is the deterministic source-checkout layout. # whyNotSourceFix: OpenShell v0.0.72 has no machine-readable child-env contract. + # It also deliberately hides the supervisor identity mount from workload + # children and the Hermes image contains no OpenShell CLI. Executing + # ``openshell --version`` here would therefore either fail every real + # transaction or verify an unrelated in-image stub. The host MCP lifecycle + # gate verifies the selected OpenShell CLI before provider mutation; the + # exact loaded MCP policy remains the running-supervisor capability proof. # regressionTest: hermes-mcp-config-transaction and image packaging tests cover # both layouts, strict parsing, version alignment, and reserved-name parity. - # removalCondition: use an upstream capability manifest once the minimum - # supported OpenShell release provides one. + # removalCondition: replace this manifest boundary only when the live + # supervisor exposes an authenticated, machine-readable attestation binding + # both its running version and child-env contract to workload startup; #6256 + # tracks that upstream capability boundary. candidates = ( Path(__file__).with_name(BOUNDARY_MANIFEST_NAME), Path(__file__).resolve().parents[2] @@ -388,6 +396,85 @@ def _managed_candidate(payload: dict[str, object]) -> dict[str, object]: return candidate +_MANAGED_CANDIDATE_FIELDS = frozenset( + {"url", "enabled", "timeout", "connect_timeout", "tools", "headers"} +) + + +def _validate_inspection_payload(payload: dict[str, object]) -> None: + if set(payload) != {"present", "absent"}: + raise ValueError("Hermes MCP inspection payload has invalid fields") + present = payload.get("present") + absent = payload.get("absent") + if not isinstance(present, dict) or not isinstance(absent, list): + raise ValueError("Hermes MCP inspection payload has invalid shape") + if not all( + isinstance(name, str) and SERVER_NAME_RE.fullmatch(name) for name in present + ): + raise ValueError("Hermes MCP inspection payload has an invalid server name") + if not all( + isinstance(name, str) and SERVER_NAME_RE.fullmatch(name) for name in absent + ): + raise ValueError("Hermes MCP inspection payload has an invalid absent server") + if len(absent) != len(set(absent)) or set(present).intersection(absent): + raise ValueError("Hermes MCP inspection payload has overlapping server state") + for server, expected in present.items(): + if not isinstance(expected, dict): + raise ValueError("Hermes MCP inspection expected config must be an object") + if not set(expected).issubset(_MANAGED_CANDIDATE_FIELDS): + raise ValueError("Hermes MCP inspection expected config has invalid fields") + synthetic = { + "server": server, + "url": expected.get("url"), + "headers": expected.get("headers"), + "replace_existing": True, + } + _validate_payload("add", synthetic) + if expected != _managed_candidate(synthetic): + raise ValueError("Hermes MCP inspection expected config is not canonical") + + +def inspect_managed_config(payload: dict[str, object]) -> dict[str, object]: + _validate_inspection_payload(payload) + privileged = os.geteuid() == 0 + guard = _load_guard() + hash_path = ( + STRICT_HASH_PATH if privileged else os.path.join(HERMES_DIR, ".config-hash") + ) + compatibility_hash_path = ( + os.path.join(HERMES_DIR, ".config-hash") if privileged else None + ) + # TOCTOU contract: this call reads config, env, and every hash anchor into + # one authenticated snapshot. After comparing the returned config bytes to + # host intent, `assert_mcp_integrity_snapshot_current` reopens every path and + # requires the same inode/content metadata before any match is reported. + integrity = guard.inspect_mcp_integrity_snapshot( + HERMES_DIR, hash_path, compatibility_hash_path + ) + if integrity.state != "current": + raise RuntimeError("Hermes MCP config does not match applied gateway state") + parsed = yaml.safe_load(integrity.config_text) + if parsed is None: + parsed = {} + if not isinstance(parsed, dict): + raise RuntimeError("Hermes MCP config does not match persisted managed intent") + servers = parsed.get("mcp_servers", {}) + if servers is None: + servers = {} + if not isinstance(servers, dict): + raise RuntimeError("Hermes MCP config does not match persisted managed intent") + present = payload["present"] + absent = payload["absent"] + if not isinstance(present, dict) or not isinstance(absent, list): + raise RuntimeError("Hermes MCP config does not match persisted managed intent") + matches = all(servers.get(name) == expected for name, expected in present.items()) + matches = matches and all(name not in servers for name in absent) + if not matches: + raise RuntimeError("Hermes MCP config does not match persisted managed intent") + guard.assert_mcp_integrity_snapshot_current(integrity) + return {"ok": True, "state": "matched"} + + def _mutate(data: object, action: str, payload: dict[str, object]) -> tuple[dict, bool]: if not isinstance(data, dict): raise ValueError("Invalid Hermes config: expected a YAML object") @@ -435,14 +522,32 @@ def _managed_hash_paths(privileged: bool) -> tuple[str, ...]: return (STRICT_HASH_PATH, compatibility) if privileged else (compatibility,) -def _refresh_and_verify_hashes(guard: ModuleType, privileged: bool) -> None: +def _refresh_and_verify_hashes( + guard: ModuleType, privileged: bool, mcp_transition: str = "preserve" +) -> None: if privileged: - guard.refresh_hashes(HERMES_DIR, STRICT_HASH_PATH, "strict") - guard.refresh_hashes(HERMES_DIR, STRICT_HASH_PATH, "compat") + guard.refresh_hashes( + HERMES_DIR, + STRICT_HASH_PATH, + "strict", + mcp_transition=mcp_transition, + ) + guard.refresh_hashes( + HERMES_DIR, + STRICT_HASH_PATH, + "compat", + mcp_transition=mcp_transition, + ) compat_text, _ = guard._read_text(os.path.join(HERMES_DIR, ".config-hash")) + _config_digest, _env_digest, mcp_state = guard._parse_config_hash( + compat_text, + os.path.join(HERMES_DIR, "config.yaml"), + os.path.join(HERMES_DIR, ".env"), + ) expected_text, _, _ = guard._hash_text( os.path.join(HERMES_DIR, "config.yaml"), os.path.join(HERMES_DIR, ".env"), + mcp_state, ) if compat_text != expected_text: raise RuntimeError("Hermes compatibility config hash is stale") @@ -452,6 +557,16 @@ def _refresh_and_verify_hashes(guard: ModuleType, privileged: bool) -> None: strict_text = compat_text if strict_text != compat_text: raise RuntimeError("Hermes strict and compatibility config hashes differ") + state = guard.inspect_mcp_integrity( + HERMES_DIR, + STRICT_HASH_PATH if privileged else os.path.join(HERMES_DIR, ".config-hash"), + ) + expected_state = { + "apply": "current", + "rollback": "pending", + }.get(mcp_transition) + if expected_state is not None and state != expected_state: + raise RuntimeError("Hermes MCP applied hash state is stale") def _restore_hash_snapshots( @@ -479,13 +594,17 @@ def apply_transaction(action: str, payload: dict[str, object]) -> bool: hash_originals = { path: guard._read_text(path) for path in _managed_hash_paths(privileged) } + integrity_path = ( + STRICT_HASH_PATH if privileged else os.path.join(HERMES_DIR, ".config-hash") + ) + guard.inspect_mcp_integrity(HERMES_DIR, integrity_path) parsed = yaml.safe_load(original_text) if parsed is None: parsed = {} updated, changed = _mutate(parsed, action, payload) if not changed: try: - _refresh_and_verify_hashes(guard, privileged) + _refresh_and_verify_hashes(guard, privileged, "intend") except Exception as hash_error: try: _restore_hash_snapshots(guard, hash_originals) @@ -507,7 +626,7 @@ def apply_transaction(action: str, payload: dict[str, object]) -> bool: mode=original_snapshot.mode, ) _, replacement_snapshot = guard._read_text(CONFIG_PATH) - _refresh_and_verify_hashes(guard, privileged) + _refresh_and_verify_hashes(guard, privileged, "intend") except Exception as mutation_error: if replacement_snapshot is None: raise @@ -518,7 +637,7 @@ def apply_transaction(action: str, payload: dict[str, object]) -> bool: replacement_snapshot, mode=original_snapshot.mode, ) - _refresh_and_verify_hashes(guard, privileged) + _restore_hash_snapshots(guard, hash_originals) except Exception as rollback_error: raise RuntimeError( f"Hermes MCP config update failed ({mutation_error}); rollback also failed ({rollback_error})" @@ -535,9 +654,6 @@ def apply_transaction_and_reload( privileged = os.geteuid() == 0 guard = _load_guard() original_text, original_snapshot = guard._read_text(CONFIG_PATH) - hash_originals = { - path: guard._read_text(path) for path in _managed_hash_paths(privileged) - } parsed = yaml.safe_load(original_text) if parsed is None: parsed = {} @@ -551,6 +667,14 @@ def apply_transaction_and_reload( changed = apply_transaction(action, payload) try: reloaded = reload_gateway() + if not reloaded: + raise RuntimeError("Hermes gateway stopped before managed MCP reload") + current_text, _current_snapshot = guard._read_text(CONFIG_PATH) + if current_text != expected_text: + raise RuntimeError( + "Hermes config changed concurrently after MCP reload; refusing applied-state commit" + ) + _refresh_and_verify_hashes(guard, privileged, "apply") except Exception as reload_error: if not changed: raise RuntimeError( @@ -569,11 +693,11 @@ def apply_transaction_and_reload( current_snapshot, mode=int(getattr(original_snapshot, "mode")), ) - try: - _refresh_and_verify_hashes(guard, privileged) - except Exception: - _restore_hash_snapshots(guard, hash_originals) - raise + # Do not restore the original current/current anchors before the + # old config is proven live. Record a pending rollback anchor so + # startup and host reconciliation remain fail-closed if this + # second reload also fails. + _refresh_and_verify_hashes(guard, privileged, "rollback") except Exception as rollback_error: rollback_errors.append(f"config/hash rollback failed: {rollback_error}") else: @@ -583,6 +707,8 @@ def apply_transaction_and_reload( rollback_errors.append( "old-config runtime reload was not verified because the gateway stopped" ) + else: + _refresh_and_verify_hashes(guard, privileged, "apply") except Exception as rollback_reload_error: rollback_errors.append( f"old-config runtime reload failed: {rollback_reload_error}" @@ -768,6 +894,10 @@ def _gateway_identity() -> tuple[int, object] | None: raise PermissionError( "Hermes gateway PID does not identify the trusted launcher" ) + if not _gateway_has_managed_parent(numeric_pid): + raise PermissionError( + "Hermes gateway is not running under the managed service lifecycle" + ) start_time = get_process_start_time(numeric_pid) if start_time is None: raise PermissionError("Hermes gateway process start identity is unavailable") @@ -845,13 +975,22 @@ def reload_gateway() -> bool: if now >= deadline: break current = _gateway_identity() - if current is not None and current != previous: + observed_phase = "waiting-for-replacement-identity" + if ( + current is not None + and current != previous + and _gateway_has_managed_parent(current[0]) + ): healthy, observed_phase = _gateway_health_phase(deadline) if phase_order[observed_phase] > phase_order[last_safe_phase]: last_safe_phase = observed_phase if healthy: confirmed = _gateway_identity() - if confirmed == current and time.monotonic() < deadline: + if ( + confirmed == current + and _gateway_has_managed_parent(current[0]) + and time.monotonic() < deadline + ): return True # A pinned Hermes gateway can remain alive without converging after the @@ -864,6 +1003,10 @@ def reload_gateway() -> bool: not re_kick_attempted and now >= re_kick_not_before and now < deadline + # The managed supervisor owns the public socat relay. Once the + # replacement gateway is internally healthy, another gateway + # signal cannot repair that relay and only creates crash churn. + and observed_phase != "waiting-for-public-relay-health-on-8642" and current is not None and _gateway_has_managed_parent(current[0]) and _gateway_identity() == current @@ -944,7 +1087,7 @@ def execute(action: str, payload: dict[str, object]) -> dict[str, object]: def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("action", choices=("add", "remove", "probe")) + parser.add_argument("action", choices=("add", "remove", "inspect", "probe")) parser.add_argument("--payload") args = parser.parse_args() payload: dict[str, object] | None = None @@ -953,6 +1096,11 @@ def main() -> int: if args.payload is not None: raise ValueError("Hermes MCP lifecycle probe does not accept --payload") result = probe() + elif args.action == "inspect": + if args.payload is None: + raise ValueError("Hermes MCP inspection requires --payload") + payload = _parse_payload(args.payload) + result = inspect_managed_config(payload) elif args.payload is None: raise ValueError("Hermes MCP mutation requires --payload") else: diff --git a/agents/hermes/runtime-config-guard.py b/agents/hermes/runtime-config-guard.py index 34550600afe..a6cc4d1370c 100755 --- a/agents/hermes/runtime-config-guard.py +++ b/agents/hermes/runtime-config-guard.py @@ -25,7 +25,9 @@ import sys import tempfile import time -from dataclasses import dataclass +from dataclasses import dataclass, field + +import yaml API_SERVER_KEY_RE = re.compile(r"^[0-9a-f]{64}$") @@ -45,6 +47,11 @@ MAX_PROC_BYTES = 1024 * 1024 PROC_ROOT = "/proc" MAX_PROC_ENTRIES = 32768 +MCP_HASH_STATE_PREFIX = "# nemoclaw-hermes-mcp-state-v1" +MCP_INTEGRITY_PENDING_EXIT_CODE = 10 +MCP_HASH_STATE_RE = re.compile( + rf"{re.escape(MCP_HASH_STATE_PREFIX)} intended=([0-9a-f]{{64}}) applied=([0-9a-f]{{64}})" +) NEMOCLAW_START_ARGV = (b"nemoclaw-start", b"/usr/local/bin/nemoclaw-start") OPENSHELL_SUPERVISOR_ARGV0 = b"/opt/openshell/bin/openshell-sandbox" SEALED_FILE_NAMES = ("config.yaml", ".env", ".config-hash") @@ -130,6 +137,50 @@ def from_stat(cls, st: os.stat_result) -> "FileSnapshot": ) +# invalidState: persisted intent is treated as gateway-applied before a healthy +# replacement gateway has consumed it, or mutable config bytes race an anchor +# transition and are accidentally blessed. +# sourceBoundary: this guard owns parsing and atomically advancing the durable +# intended/applied marker; the transaction helper owns candidate writes and +# rollback, while startup advances `applied` only after replacement health. +# whyNotSourceFix: config bytes prove desired state but cannot prove which bytes +# a long-lived Hermes gateway consumed; Hermes/OpenShell exposes no authenticated +# applied-config digest in the pinned runtime. +# regressionTest: hermes-mcp-integrity-state covers pending/current transitions, +# metadata-only apply, stale snapshots, rollback, and startup commit ordering. +# removalCondition: replace this marker only when the runtime exposes an +# authenticated applied-config digest with equivalent transactional rollback. +@dataclass(frozen=True) +class McpHashState: + intended: str + applied: str + + +# invalidState: managed-state inspection authenticates one config snapshot, then +# reopens mutable config and accidentally compares different bytes to host intent. +# sourceBoundary: this guard owns the authenticated config/env/hash snapshots; +# the transaction helper may parse the returned config text, but must revalidate +# this opaque snapshot immediately before reporting a managed-state match. +# whyNotSourceFix: the Hermes config has no runtime-provided authenticated read +# API, so host reconciliation must bind its comparison to the local trust anchor. +# regressionTest: hermes-mcp-integrity-state mutates config after authentication +# and proves the final snapshot validation refuses the raced managed-state match. +# removalCondition: remove this snapshot token only when Hermes exposes an +# authenticated applied-config digest and exact config bytes through one API; +# #6257 tracks that upstream attestation boundary. +@dataclass(frozen=True) +class McpIntegritySnapshot: + state: str + # Authenticated config bytes can include credentials; never expose them + # through the generated dataclass representation. + config_text: str = field(repr=False) + config_path: str + config_snapshot: FileSnapshot + env_path: str + env_snapshot: FileSnapshot + hash_snapshots: tuple[tuple[str, FileSnapshot], ...] + + class OpenFile: def __init__(self, path: str, fd: int, snapshot: FileSnapshot): self.path = path @@ -524,6 +575,11 @@ def _pinned_process_matches_supervised_nonroot_start( supervisor_identity: tuple[str, int | None], expected_effective_uid: int, ) -> bool: + # OpenShell 0.0.72 keeps its supervisor at PID 1 and launches the non-root + # NemoClaw entrypoint as a child, so startup authority must be proved from + # pinned procfs identity rather than a PID-1 equality check. Remove this + # compatibility proof when #6256 provides authenticated supervisor/runtime + # attestation with a unified workload topology. proc_pid_fd = -1 try: numeric_pid = int(pid, 10) @@ -679,7 +735,9 @@ def _validate_action_readiness(action: str, startup_owner: bool) -> None: except KeyError: sandbox_uid = -1 startup_actions = { + "commit-mcp-applied", "ensure-api-key", + "inspect-mcp-integrity", "refresh-hashes", "provider-placeholders", "publish-startup-ready", @@ -1092,14 +1150,77 @@ def _write_hash(path: str, text: str) -> None: _atomic_replace_preserving_flags(path, text.encode("utf-8"), snapshot) +def _canonical_mcp_servers_digest(config_text: str) -> str: + """Hash the effective MCP map without persisting or logging its contents.""" + try: + parsed = yaml.safe_load(config_text) + except yaml.YAMLError as exc: + raise UnsafePathError("refusing invalid Hermes MCP configuration") from exc + if parsed is None: + parsed = {} + if not isinstance(parsed, dict): + raise UnsafePathError("refusing non-object Hermes configuration") + servers = parsed.get("mcp_servers", {}) + if servers is None: + servers = {} + if not isinstance(servers, dict): + raise UnsafePathError("refusing non-object Hermes mcp_servers configuration") + try: + canonical = json.dumps( + servers, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise UnsafePathError( + "refusing non-canonical Hermes mcp_servers configuration" + ) from exc + return hashlib.sha256(canonical).hexdigest() + + +def _current_mcp_servers_digest(config_path: str) -> tuple[str, FileSnapshot]: + config_text, snapshot = _read_text(config_path, MAX_CONFIG_INPUT_BYTES) + return _canonical_mcp_servers_digest(config_text), snapshot + + +def _hash_text_and_mcp_digest( + config_path: str, + env_path: str, + mcp_state: McpHashState | None = None, +) -> tuple[str, FileSnapshot, FileSnapshot, str, str]: + config_text, config_snapshot = _read_text(config_path, MAX_CONFIG_INPUT_BYTES) + config_digest = hashlib.sha256(config_text.encode("utf-8")).hexdigest() + config_entry = f"{config_digest} {config_path}\n" + env_entry, env_snapshot = _sha256_entry(env_path, MAX_ENV_BYTES) + current_mcp = _canonical_mcp_servers_digest(config_text) + state = mcp_state or McpHashState(current_mcp, current_mcp) + state_entry = ( + f"{MCP_HASH_STATE_PREFIX} intended={state.intended} applied={state.applied}\n" + ) + return ( + config_entry + env_entry + state_entry, + config_snapshot, + env_snapshot, + current_mcp, + config_text, + ) + + def _hash_text( - config_path: str, env_path: str + config_path: str, + env_path: str, + mcp_state: McpHashState | None = None, ) -> tuple[str, FileSnapshot, FileSnapshot]: - config_entry, config_snapshot = _sha256_entry( - config_path, MAX_CONFIG_INPUT_BYTES - ) - env_entry, env_snapshot = _sha256_entry(env_path, MAX_ENV_BYTES) - return config_entry + env_entry, config_snapshot, env_snapshot + ( + text, + config_snapshot, + env_snapshot, + _current_mcp, + _config_text, + ) = _hash_text_and_mcp_digest(config_path, env_path, mcp_state) + return text, config_snapshot, env_snapshot def _sealed_file_limit(name: str) -> int: @@ -1127,11 +1248,112 @@ def _decode_bounded_base64(value: str, max_bytes: int, label: str) -> bytes: return decoded -def refresh_hashes(hermes_dir: str, hash_file: str, mode: str) -> None: +def _hash_state_from_file(path: str, config_path: str, env_path: str) -> McpHashState: + text = _read_hash_file(path) + _config_digest, _env_digest, state = _parse_config_hash(text, config_path, env_path) + return state + + +def refresh_hashes( + hermes_dir: str, + hash_file: str, + mode: str, + mcp_transition: str = "preserve", +) -> None: + """Advance the durable MCP intended/applied state without blessing drift. + + ``preserve`` requires current config to equal intended. ``intend`` records + current config as the next intent while retaining the last applied digest. + ``rollback`` requires restored config to equal the prior applied digest, + then conservatively records restored/failed-candidate until reload health is + proven. ``apply`` is a metadata-only intended/intended commit and requires + the complete pending config/env anchor to remain byte-identical. Thus a new + image begins current/current, add/remove moves to new/old, rollback moves to + old/new, and only a healthy replacement advances either pending state to + current/current; concurrent config or env changes fail closed. + """ config_path = os.path.join(hermes_dir, "config.yaml") env_path = os.path.join(hermes_dir, ".env") compat_hash = os.path.join(hermes_dir, ".config-hash") - hash_text, config_snapshot, env_snapshot = _hash_text(config_path, env_path) + if mcp_transition not in {"preserve", "intend", "rollback", "apply"}: + raise UnsafePathError("refusing unsupported Hermes MCP hash transition") + + # Snapshot-stability/TOCTOU contract: derive the config hash and canonical + # MCP digest from one `_read_text` result, retain both config/env inode + # snapshots, and reopen/compare them before each anchor write and once after + # the final write. For `apply`, the complete pending anchor must also remain + # byte-identical, so advancing only the metadata line cannot bless unrelated + # config/env drift between gateway health and commit. + state_path = hash_file if mode in ("strict", "both") else compat_hash + # Runtime refresh is allowed to advance an existing trust anchor, never to + # create one from the mutable config it is supposed to authenticate. Image + # construction emits the initial intended/applied marker; missing or + # malformed metadata must therefore fail closed. + source_hash_text = _read_hash_file(state_path) + _config_digest, _env_digest, state = _parse_config_hash( + source_hash_text, config_path, env_path + ) + current_mcp, _ = _current_mcp_servers_digest(config_path) + if mcp_transition == "preserve": + if not secrets.compare_digest(current_mcp, state.intended): + raise UnsafePathError( + "Hermes MCP config differs from persisted intended state" + ) + elif mcp_transition == "intend": + if state.intended != state.applied and not secrets.compare_digest( + current_mcp, state.intended + ): + raise UnsafePathError( + "Hermes MCP configuration has an incomplete prior transaction" + ) + state = McpHashState(current_mcp, state.applied) + elif mcp_transition == "rollback": + # A failed desired-config reload leaves the runtime identity uncertain. + # Re-anchor the restored config as intended, but retain the failed + # candidate digest as the conservative applied value until a healthy + # old-config replacement is observed. This keeps startup/recovery + # fail-closed if the rollback reload also fails. + if secrets.compare_digest(state.intended, state.applied): + raise UnsafePathError( + "Hermes MCP rollback requires a pending desired configuration" + ) + if not secrets.compare_digest(current_mcp, state.applied): + raise UnsafePathError( + "Hermes MCP rollback config does not match the previously applied state" + ) + state = McpHashState(current_mcp, state.intended) + else: + if not secrets.compare_digest(current_mcp, state.intended): + raise UnsafePathError( + "Hermes MCP config changed before applied-state commit" + ) + # Applying intent is a metadata-only commit. Require the complete + # config/env snapshot to still match the pending trust anchor rather + # than re-hashing and blessing unrelated concurrent changes. + pending_hash_text, config_snapshot, env_snapshot = _hash_text( + config_path, env_path, state + ) + if not secrets.compare_digest(pending_hash_text, source_hash_text): + raise UnsafePathError( + "Hermes config or env changed before applied-state commit" + ) + if mode == "both" and not secrets.compare_digest( + _read_hash_file(compat_hash), source_hash_text + ): + raise UnsafePathError( + "Hermes strict and compatibility MCP state differ before applied-state commit" + ) + state = McpHashState(state.intended, state.intended) + lines = pending_hash_text.splitlines(keepends=True) + lines[2] = ( + f"{MCP_HASH_STATE_PREFIX} intended={state.intended} applied={state.applied}\n" + ) + hash_text = "".join(lines) + + if mcp_transition != "apply": + hash_text, config_snapshot, env_snapshot = _hash_text( + config_path, env_path, state + ) def assert_inputs_stable() -> None: config = _open_regular(config_path) @@ -1152,7 +1374,13 @@ def assert_inputs_stable() -> None: # Hash refresh is an atomic rename, so directory write authority is what # matters; a correctly shields-locked compatibility file is itself 0444. compat_writable = os.access(hermes_dir, os.W_OK) - if mode == "both" or (mode == "compat" and compat_writable): + # Applying a healthy gateway's intent must use the real atomic write as + # the authority check. `os.access` is only a best-effort legacy probe and + # can disagree with the effective credentials used by the write itself. + compat_commit_required = mcp_transition == "apply" and mode == "compat" + if mode == "both" or ( + mode == "compat" and (compat_writable or compat_commit_required) + ): assert_inputs_stable() _write_hash(compat_hash, hash_text) @@ -1169,6 +1397,72 @@ def assert_inputs_stable() -> None: assert_inputs_stable() +def inspect_mcp_integrity_snapshot( + hermes_dir: str, + hash_file: str, + compatibility_hash_file: str | None = None, +) -> McpIntegritySnapshot: + config_path = os.path.join(hermes_dir, "config.yaml") + env_path = os.path.join(hermes_dir, ".env") + text, hash_snapshot = _read_text(hash_file, MAX_HASH_BYTES) + hash_snapshots = [(hash_file, hash_snapshot)] + if compatibility_hash_file is not None: + compatibility_text, compatibility_snapshot = _read_text( + compatibility_hash_file, MAX_HASH_BYTES + ) + if not secrets.compare_digest(compatibility_text, text): + raise UnsafePathError( + "Hermes strict and compatibility MCP integrity anchors differ" + ) + hash_snapshots.append((compatibility_hash_file, compatibility_snapshot)) + _config_digest, _env_digest, state = _parse_config_hash(text, config_path, env_path) + ( + actual, + config_snapshot, + env_snapshot, + current_mcp, + config_text, + ) = _hash_text_and_mcp_digest(config_path, env_path, state) + if not secrets.compare_digest(actual, text): + raise UnsafePathError("Hermes config hash does not match persisted inputs") + if not secrets.compare_digest(current_mcp, state.intended): + raise UnsafePathError("Hermes MCP config differs from persisted intended state") + return McpIntegritySnapshot( + state="pending" if state.intended != state.applied else "current", + config_text=config_text, + config_path=config_path, + config_snapshot=config_snapshot, + env_path=env_path, + env_snapshot=env_snapshot, + hash_snapshots=tuple(hash_snapshots), + ) + + +def assert_mcp_integrity_snapshot_current(snapshot: McpIntegritySnapshot) -> None: + for path, expected in ( + (snapshot.config_path, snapshot.config_snapshot), + (snapshot.env_path, snapshot.env_snapshot), + *snapshot.hash_snapshots, + ): + try: + opened = _open_regular(path) + except OSError as exc: + raise UnsafePathError( + "refusing raced Hermes MCP integrity snapshot" + ) from exc + try: + if opened.snapshot != expected: + raise UnsafePathError("refusing raced Hermes MCP integrity snapshot") + finally: + opened.close() + + +def inspect_mcp_integrity(hermes_dir: str, hash_file: str) -> str: + snapshot = inspect_mcp_integrity_snapshot(hermes_dir, hash_file) + assert_mcp_integrity_snapshot_current(snapshot) + return snapshot.state + + def _inode_metadata(st: os.stat_result) -> dict[str, int]: return { "dev": st.st_dev, @@ -1237,9 +1531,11 @@ def _read_hash_file(path: str) -> str: def _verify_strict_hash(hermes_dir: str, hash_file: str) -> None: config_path = os.path.join(hermes_dir, "config.yaml") env_path = os.path.join(hermes_dir, ".env") - actual, _config_snapshot, _env_snapshot = _hash_text(config_path, env_path) strict = _read_hash_file(hash_file) - _parse_two_file_hash(strict, config_path, env_path) + _config_digest, _env_digest, state = _parse_config_hash( + strict, config_path, env_path + ) + actual, _config_snapshot, _env_snapshot = _hash_text(config_path, env_path, state) if actual != strict: raise StrictHashMismatchError( "strict hash verification failed for Hermes restart seal" @@ -1251,13 +1547,13 @@ def _verify_compat_hash(hash_file: str, compat_hash_file: str) -> None: raise UnsafePathError("compat hash verification failed for Hermes restart seal") -def _parse_two_file_hash( +def _parse_config_hash( text: str, config_path: str, env_path: str -) -> tuple[str, str]: +) -> tuple[str, str, McpHashState]: parts = text.split("\n") - if len(parts) != 3 or parts[-1] != "": + if len(parts) != 4 or parts[-1] != "": raise UnsafePathError("refusing malformed Hermes config hash") - lines = parts[:-1] + lines = parts[:2] expected_paths = (config_path, env_path) if len(lines) != len(expected_paths): raise UnsafePathError("refusing malformed Hermes config hash") @@ -1267,7 +1563,14 @@ def _parse_two_file_hash( if match is None or match.group(2) != expected_path: raise UnsafePathError("refusing malformed Hermes config hash") digests.append(match.group(1)) - return digests[0], digests[1] + state_match = MCP_HASH_STATE_RE.fullmatch(parts[2]) + if state_match is None: + raise UnsafePathError("refusing malformed Hermes MCP hash state") + return ( + digests[0], + digests[1], + McpHashState(state_match.group(1), state_match.group(2)), + ) def _without_single_generated_api_server_key(text: str) -> str: @@ -1366,11 +1669,13 @@ def _reconcile_nonroot_startup_api_key_hash( env_path = os.path.join(hermes_dir, ".env") compat_hash_path = os.path.join(hermes_dir, ".config-hash") strict_text = _read_hash_file(hash_file) - strict_config_sha256, strict_env_sha256 = _parse_two_file_hash( + strict_config_sha256, strict_env_sha256, strict_mcp_state = _parse_config_hash( strict_text, config_path, env_path ) - actual_text, config_snapshot, env_snapshot = _hash_text(config_path, env_path) - actual_config_sha256, _actual_env_sha256 = _parse_two_file_hash( + actual_text, config_snapshot, env_snapshot = _hash_text( + config_path, env_path, strict_mcp_state + ) + actual_config_sha256, _actual_env_sha256, _actual_mcp_state = _parse_config_hash( actual_text, config_path, env_path ) @@ -2786,11 +3091,20 @@ def _seal_shields_locked( ) unavailable = bool(unavailable_reasons) file_mode = 0o400 if unavailable else 0o444 + try: + mcp_digest = _canonical_mcp_servers_digest( + inputs["config.yaml"].decode("utf-8") + ) + except (UnicodeDecodeError, UnsafePathError): + # Containment cannot let a malformed mutable input veto shields-up. + # Semantic MCP inspection still rejects those frozen config bytes. + mcp_digest = hashlib.sha256(b"{}").hexdigest() hash_text = ( f"{hashlib.sha256(inputs['config.yaml']).hexdigest()} " f"{os.path.join(hermes_dir, 'config.yaml')}\n" f"{hashlib.sha256(inputs['.env']).hexdigest()} " f"{os.path.join(hermes_dir, '.env')}\n" + f"{MCP_HASH_STATE_PREFIX} intended={mcp_digest} applied={mcp_digest}\n" ) if len(hash_text.encode("utf-8")) > MAX_HASH_BYTES: raise UnsafePathError("refusing oversized synthesized Hermes hash") @@ -3922,6 +4236,18 @@ def write_config_transaction( raise UnsafePathError( "Hermes config changed after the host read it; retry the command" ) + try: + replacement_text = config_bytes.decode("utf-8") + except UnicodeDecodeError as exc: + raise UnsafePathError("refusing non-UTF-8 Hermes config input") from exc + current_state = _hash_state_from_file( + hash_file, config_path, os.path.join(hermes_dir, ".env") + ) + replacement_mcp = _canonical_mcp_servers_digest(replacement_text) + if not secrets.compare_digest(replacement_mcp, current_state.intended): + raise UnsafePathError( + "non-MCP config transaction cannot change Hermes mcp_servers" + ) state_data["phase"] = "config-write-prepared" state_data["config_write"] = { @@ -4328,6 +4654,8 @@ def main() -> int: choices=( "ensure-api-key", "refresh-hashes", + "inspect-mcp-integrity", + "commit-mcp-applied", "provider-placeholders", "publish-startup-ready", "seal-restart", @@ -4359,11 +4687,16 @@ def main() -> int: "--rollback-shields-mode", choices=("locked", "mutable"), default="" ) parser.add_argument("--startup-owner", action="store_true") + parser.add_argument("--mcp-state-exit-code", action="store_true") args = parser.parse_args() previous_alarm_handler = signal.signal(signal.SIGALRM, _deadline_expired) signal.alarm(GUARD_DEADLINE_SECONDS) try: + if args.mcp_state_exit_code and args.action != "inspect-mcp-integrity": + raise UnsafePathError( + "--mcp-state-exit-code requires inspect-mcp-integrity" + ) _validate_action_readiness(args.action, args.startup_owner) if args.action == "ensure-api-key": if not args.hash_file: @@ -4373,6 +4706,30 @@ def main() -> int: if not args.hash_file: raise UnsafePathError("refresh-hashes requires --hash-file") refresh_hashes(args.hermes_dir, args.hash_file, args.mode) + elif args.action == "inspect-mcp-integrity": + if not args.hash_file: + raise UnsafePathError("inspect-mcp-integrity requires --hash-file") + state = inspect_mcp_integrity(args.hermes_dir, args.hash_file) + if args.mcp_state_exit_code: + # Startup uses an exit-only protocol so no same-UID process can + # forge a named result file and no shell parser can truncate an + # embedded NUL or accept a non-canonical response. + if state == "current": + return 0 + if state == "pending": + return MCP_INTEGRITY_PENDING_EXIT_CODE + raise UnsafePathError("refusing unknown Hermes MCP integrity state") + print(f"mcp_state={state}") + elif args.action == "commit-mcp-applied": + if not args.hash_file: + raise UnsafePathError("commit-mcp-applied requires --hash-file") + refresh_hashes( + args.hermes_dir, + args.hash_file, + args.mode, + mcp_transition="apply", + ) + print("mcp_applied=1") elif args.action == "provider-placeholders": if not args.hash_file: raise UnsafePathError("provider-placeholders requires --hash-file") diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 8955d266ff2..b49c9a9ace4 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -242,6 +242,8 @@ HERMES_RESTART_SEALED=0 HERMES_RESTART_ORIGINAL_LOCKED=0 HERMES_RESTART_UNSEALING=0 HERMES_RESTART_SIGNAL_PENDING=0 +HERMES_MCP_RECONCILE_PENDING=0 +HERMES_MCP_INTEGRITY_FAILED=0 # A same-container PID 1 restart can retain /run. Revoke the prior readiness # lease before any startup migration or mutable config read; host mutations are @@ -340,10 +342,18 @@ verify_hermes_config_integrity() { # that owns the mutable Hermes home. export -f verify_config_integrity "${STEP_DOWN_PREFIX_SANDBOX[@]}" bash -c "verify_config_integrity \"\$1\" \"\$2\"" bash \ - "${HERMES_DIR}" "${HERMES_HASH_FILE}" - return $? + "${HERMES_DIR}" "${HERMES_HASH_FILE}" || return 1 + if ! inspect_hermes_mcp_integrity "${HERMES_HASH_FILE}"; then + HERMES_RESTART_FAILURE_CODE=mcp-integrity + return 1 + fi + return 0 + fi + verify_config_integrity "${HERMES_DIR}" "${HERMES_HASH_FILE}" || return 1 + if ! inspect_hermes_mcp_integrity "${HERMES_HASH_FILE}"; then + HERMES_RESTART_FAILURE_CODE=mcp-integrity + return 1 fi - verify_config_integrity "${HERMES_DIR}" "${HERMES_HASH_FILE}" } # configure_messaging_channels is provided by sandbox-init.sh (shared). @@ -476,6 +486,16 @@ cleanup_orphan_socat_forwarders() { cmdline="$(tr '\0' ' ' <"$cmdline_file" 2>/dev/null || true)" case "$cmdline" in *socat*"TCP-LISTEN:${PUBLIC_PORT}"*"TCP:127.0.0.1:${INTERNAL_PORT}"*) + if [ "$pid" = "${SOCAT_PID:-}" ] \ + && hermes_tracked_role_is_current \ + api-socat "$pid" current "$PUBLIC_PORT"; then + # A managed gateway reload temporarily leaves no gateway process, + # but its exact tracked relay may still be safe to reuse. Preserve + # only the fully identity-proven parent; listener ownership and + # public readiness are re-proven before convergence, while every + # other matching socat is still removed below. + continue + fi echo "[gateway] Removing orphaned socat forwarder for ${PUBLIC_PORT}->${INTERNAL_PORT} (pid ${pid})" >&2 kill "$pid" 2>/dev/null || true ;; @@ -483,6 +503,11 @@ cleanup_orphan_socat_forwarders() { if [ -z "$dashboard_public_port" ] || [ -z "$dashboard_internal_port" ]; then continue fi + if [ "$pid" = "${DASHBOARD_SOCAT_PID:-}" ] \ + && hermes_tracked_role_is_current \ + dashboard-socat "$pid" current "$dashboard_public_port"; then + continue + fi echo "[gateway] Removing orphaned dashboard socat forwarder for ${dashboard_public_port}->${dashboard_internal_port} (pid ${pid})" >&2 kill "$pid" 2>/dev/null || true ;; @@ -1664,6 +1689,54 @@ refresh_hermes_runtime_config_hashes() { "${cmd[@]}" } +inspect_hermes_mcp_integrity() { + local hash_file="${1:-}" + local guard_status + [ -n "$hash_file" ] || { + if [ "$(id -u)" -eq 0 ]; then + hash_file="$HERMES_HASH_FILE" + else + hash_file="${HERMES_DIR}/.config-hash" + fi + } + # Keep the guard as the startup owner's direct child. A command + # substitution here would interpose a shell process and invalidate the + # exact-parent proof used by --startup-owner. State is returned only through + # the kernel-owned exit status: 0=current, 10=pending, anything else=failure. + # This avoids a same-UID writable result file or ambiguous shell byte parsing. + if "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" inspect-mcp-integrity \ + --hermes-dir "$HERMES_DIR" \ + --hash-file "$hash_file" \ + --startup-owner \ + --mcp-state-exit-code >/dev/null; then + guard_status=0 + else + guard_status=$? + fi + case "$guard_status" in + 0) HERMES_MCP_RECONCILE_PENDING=0 ;; + 10) HERMES_MCP_RECONCILE_PENDING=1 ;; + *) + HERMES_MCP_INTEGRITY_FAILED=1 + echo "[SECURITY] HERMES_MCP_CONFIG_DRIFT: MCP intent cannot be matched to the persisted gateway state; rebuild the sandbox from its NemoClaw registry state" >&2 + return 1 + ;; + esac + HERMES_MCP_INTEGRITY_FAILED=0 +} + +commit_hermes_mcp_applied_if_pending() { + local mode=compat + [ "$HERMES_MCP_RECONCILE_PENDING" -eq 1 ] || return 0 + [ "$(id -u)" -eq 0 ] && mode=both + "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" commit-mcp-applied \ + --hermes-dir "$HERMES_DIR" \ + --hash-file "$HERMES_HASH_FILE" \ + --mode "$mode" \ + --startup-owner >/dev/null || return 1 + HERMES_MCP_RECONCILE_PENDING=0 +} + ensure_hermes_runtime_api_server_key() { local mode="${1:-strict}" local env_file="${HERMES_DIR}/.env" @@ -1751,6 +1824,13 @@ hermes_gateway_healthy() { HERMES_RESTART_FAILURE_CODE=internal +hermes_restart_failure_revokes_gateway() { + case "${1:-}" in + secret-boundary-refusal | mcp-integrity) return 0 ;; + *) return 1 ;; + esac +} + validate_running_hermes_boundary() { HERMES_RESTART_FAILURE_CODE=validator-missing [ -f "$_HERMES_BOUNDARY_VALIDATOR" ] || return 1 @@ -2154,13 +2234,18 @@ ensure_hermes_supervised_auxiliaries() { dashboard_user=sandbox fi - if ! hermes_api_socat_bridge_healthy "${SOCAT_PID:-}" "$PUBLIC_PORT"; then + # Structural identity/listener loss requires exact relay replacement. A + # transient public HTTP miss does not: forked socat accepts each request on + # a fresh backend connection, so churning its proven listener can prolong + # the outage while the replacement gateway is still settling. Preserve the + # exact parent and let the supervised recovery loop retry readiness instead. + if ! hermes_socat_bridge_healthy api-socat "${SOCAT_PID:-}" "$PUBLIC_PORT"; then hermes_stop_tracked_role api-socat "${SOCAT_PID:-0}" current "$PUBLIC_PORT" || return 1 SOCAT_PID="" start_socat_forwarder \ "$PUBLIC_PORT" "$INTERNAL_PORT" "API" SOCAT_PID "$GATEWAY_PID" "$gateway_user" || return 1 - hermes_api_socat_bridge_healthy "$SOCAT_PID" "$PUBLIC_PORT" || return 1 fi + hermes_api_socat_bridge_healthy "$SOCAT_PID" "$PUBLIC_PORT" || return 1 if ! hermes_dashboard_healthy "${DASHBOARD_PID:-}"; then # A live PID is not sufficient: it may be reused, alive without the exact # dashboard listener, or serving a wedged HTTP process. Stop both tracked @@ -2285,6 +2370,10 @@ handle_hermes_gateway_control_request() { gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" return 1 fi + if [ "$HERMES_MCP_RECONCILE_PENDING" -eq 1 ]; then + gateway_control_fail mcp-reconcile-required "$old_pid" + return 1 + fi if ! gateway_control_pid_is_live "$old_pid" \ || ! hermes_gateway_healthy "$old_pid" \ || hermes_auxiliaries_need_recovery; then @@ -2302,60 +2391,62 @@ handle_hermes_gateway_control_request() { # Verify the root-owned trust anchor before any auxiliary consumes it; a # healthy gateway is not authority to bless direct sandbox config drift. if ! prepare_hermes_gateway_restart; then - if [ "$HERMES_RESTART_FAILURE_CODE" = "secret-boundary-refusal" ]; then + if hermes_restart_failure_revokes_gateway "$HERMES_RESTART_FAILURE_CODE"; then stop_hermes_gateway_fail_closed fi gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" return 1 fi - if hermes_auxiliaries_need_recovery; then - if ! seal_hermes_restart_inputs; then - if [ "$HERMES_RESTART_SEALED" -eq 1 ]; then - stop_hermes_gateway_fail_closed + if [ "$HERMES_MCP_RECONCILE_PENDING" -eq 0 ]; then + if hermes_auxiliaries_need_recovery; then + if ! seal_hermes_restart_inputs; then + if [ "$HERMES_RESTART_SEALED" -eq 1 ]; then + stop_hermes_gateway_fail_closed + fi + gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" + return 1 fi - gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" - return 1 - fi - # Re-run boundary + hash validation against the fresh sealed inodes. A - # pre-open attacker fd cannot change these pathnames after this point. - if ! prepare_hermes_gateway_restart; then - failure_code="$HERMES_RESTART_FAILURE_CODE" - if [ "$failure_code" = "secret-boundary-refusal" ]; then - # A post-seal boundary refusal means the currently running service no - # longer has a boundary we can prove safe. Stop it even if metadata - # restoration subsequently fails. - stop_hermes_gateway_fail_closed + # Re-run boundary + hash validation against the fresh sealed inodes. A + # pre-open attacker fd cannot change these pathnames after this point. + if ! prepare_hermes_gateway_restart; then + failure_code="$HERMES_RESTART_FAILURE_CODE" + if hermes_restart_failure_revokes_gateway "$failure_code"; then + # A post-seal boundary refusal means the currently running service no + # longer has a boundary we can prove safe. Stop it even if metadata + # restoration subsequently fails. + stop_hermes_gateway_fail_closed + fi + if ! unseal_hermes_restart_inputs; then + gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" + return 1 + fi + gateway_control_fail "$failure_code" "$old_pid" + return 1 fi - if ! unseal_hermes_restart_inputs; then - gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" + if ! ensure_hermes_supervised_auxiliaries; then + if ! unseal_hermes_restart_inputs; then + stop_hermes_gateway_fail_closed + gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" + else + gateway_control_fail launch-failed "$old_pid" + fi + refresh_hermes_supervised_child_pids return 1 fi - gateway_control_fail "$failure_code" "$old_pid" - return 1 - fi - if ! ensure_hermes_supervised_auxiliaries; then if ! unseal_hermes_restart_inputs; then stop_hermes_gateway_fail_closed gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" - else - gateway_control_fail launch-failed "$old_pid" + return 1 fi - refresh_hermes_supervised_child_pids - return 1 - fi - if ! unseal_hermes_restart_inputs; then - stop_hermes_gateway_fail_closed - gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" - return 1 fi + refresh_hermes_supervised_child_pids + gateway_control_complete already-running "$old_pid" "$old_pid" + return 0 fi - refresh_hermes_supervised_child_pids - gateway_control_complete already-running "$old_pid" "$old_pid" - return 0 fi if ! prepare_hermes_gateway_restart; then - if [ "$HERMES_RESTART_FAILURE_CODE" = "secret-boundary-refusal" ]; then + if hermes_restart_failure_revokes_gateway "$HERMES_RESTART_FAILURE_CODE"; then stop_hermes_gateway_fail_closed fi gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" @@ -2373,7 +2464,7 @@ handle_hermes_gateway_control_request() { fi if ! prepare_hermes_gateway_restart; then failure_code="$HERMES_RESTART_FAILURE_CODE" - if [ "$failure_code" = "secret-boundary-refusal" ]; then + if hermes_restart_failure_revokes_gateway "$failure_code"; then # Do not leave the old gateway alive after a boundary refusal merely # because restoring the restart seal also fails. stop_hermes_gateway_fail_closed @@ -2429,6 +2520,11 @@ handle_hermes_gateway_control_request() { gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" return 1 fi + if ! commit_hermes_mcp_applied_if_pending; then + stop_hermes_gateway_fail_closed + gateway_control_fail mcp-integrity "$old_pid" + return 1 + fi refresh_hermes_supervised_child_pids gateway_control_complete ok "$old_pid" "$GATEWAY_PID" } @@ -2438,12 +2534,20 @@ prepare_hermes_nonroot_runtime() { echo "[SECURITY] Config integrity check failed — refusing to start (non-root mode)" >&2 return 1 fi + # Classify raw .env material at its dedicated boundary before the MCP + # integrity guard authenticates the full config/env snapshot. Otherwise a + # mutable default with a raw secret fails as generic MCP drift and bypasses + # the actionable, redacted secret-boundary refusal. Repeat after the trusted + # startup mutations below so their outputs remain covered as well. + validate_hermes_env_secret_boundary || return 1 + inspect_hermes_mcp_integrity "${HERMES_DIR}/.config-hash" || return 1 ensure_hermes_runtime_api_server_key compat || return 1 apply_shields_up_runtime_env || return 1 validate_hermes_env_secret_boundary || return 1 validate_hermes_runtime_env_secret_boundary || return 1 refresh_hermes_provider_placeholders compat || return 1 refresh_hermes_runtime_config_hashes compat || return 1 + inspect_hermes_mcp_integrity "${HERMES_DIR}/.config-hash" || return 1 configure_messaging_channels || return 1 retry_tirith_marker_if_needed || return 1 } @@ -2598,6 +2702,11 @@ record_hermes_managed_gateway_exit() { recover_hermes_gateway_current_user() { while :; do until prepare_hermes_nonroot_runtime; do + if [ "$HERMES_MCP_INTEGRITY_FAILED" -eq 1 ]; then + echo "[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox" >&2 + quarantine_hermes_managed_gateway_relaunch + return 1 + fi echo "[gateway] Hermes runtime preparation refused automatic respawn; retrying in 5s" >&2 sleep 5 || true done @@ -2606,10 +2715,33 @@ recover_hermes_gateway_current_user() { sleep 5 || true continue fi - if wait_for_hermes_gateway_internal "$GATEWAY_PID" \ - && ensure_hermes_supervised_auxiliaries; then - refresh_hermes_supervised_child_pids - return 0 + if wait_for_hermes_gateway_internal "$GATEWAY_PID"; then + # The gateway and its socat relay are separate supervised children. A + # transient relay repair failure must not churn an internally healthy, + # identity-pinned replacement or charge that churn against the gateway + # crash budget. Retry only while the exact gateway remains healthy, and + # re-prove it after auxiliary repair before committing applied MCP state. + while hermes_tracked_role_is_current \ + gateway "$GATEWAY_PID" current "$INTERNAL_PORT" \ + && hermes_gateway_healthy "$GATEWAY_PID"; do + if ensure_hermes_supervised_auxiliaries; then + if ! hermes_tracked_role_is_current \ + gateway "$GATEWAY_PID" current "$INTERNAL_PORT" \ + || ! hermes_gateway_healthy "$GATEWAY_PID"; then + break + fi + if ! commit_hermes_mcp_applied_if_pending; then + echo "[SECURITY] HERMES_MCP_APPLIED_COMMIT_FAILED: stopping the uncommitted Hermes gateway" >&2 + hermes_stop_tracked_role gateway "$GATEWAY_PID" current "$INTERNAL_PORT" || return 1 + mark_hermes_gateway_stopped + return 1 + fi + refresh_hermes_supervised_child_pids + return 0 + fi + echo "[gateway] Hermes auxiliary repair failed; retrying while the exact gateway remains healthy" >&2 + sleep 1 || true + done fi echo "[gateway] Hermes replacement failed health or auxiliary validation; stopping the exact child" >&2 @@ -2684,6 +2816,12 @@ bootstrap_hermes_gateway_current_user() { if wait_for_hermes_gateway_internal "$GATEWAY_PID" \ && ensure_hermes_supervised_auxiliaries; then + if ! commit_hermes_mcp_applied_if_pending; then + echo "[SECURITY] HERMES_MCP_APPLIED_COMMIT_FAILED: stopping the uncommitted Hermes gateway" >&2 + hermes_stop_tracked_role gateway "$GATEWAY_PID" current "$INTERNAL_PORT" || return 1 + mark_hermes_gateway_stopped + return 1 + fi refresh_hermes_supervised_child_pids return 0 fi @@ -2813,6 +2951,11 @@ launch_hermes_gateway start_gateway_log_stream wait_for_hermes_gateway_internal "$GATEWAY_PID" ensure_hermes_supervised_auxiliaries +if ! commit_hermes_mcp_applied_if_pending; then + echo "[SECURITY] HERMES_MCP_APPLIED_COMMIT_FAILED: stopping the uncommitted Hermes gateway" >&2 + stop_hermes_gateway_fail_closed + exit 1 +fi restore_hermes_config_permissions_after_dashboard_start # NOTE: PIDs are collected after launch; a signal arriving between trap # registration and the final append is a small race window (same as before diff --git a/scripts/gateway-control.sh b/scripts/gateway-control.sh index 7fe81e12ac8..bcfca804bfb 100755 --- a/scripts/gateway-control.sh +++ b/scripts/gateway-control.sh @@ -117,6 +117,7 @@ for _ in $(seq 1 900); do secret-boundary-refusal) fail "SECRET_BOUNDARY_REFUSED" ;; unsafe-config) fail "GATEWAY_UNSAFE_CONFIG_PATH" ;; hash-mismatch) fail "GATEWAY_CONFIG_HASH_MISMATCH" ;; + mcp-integrity | mcp-reconcile-required) fail "HERMES_MCP_CONFIG_DRIFT" ;; preload-missing) fail "GATEWAY_GUARDS_MISSING" ;; health-timeout) fail "GATEWAY_HEALTH_TIMEOUT" ;; *) fail "GATEWAY_FAILED" ;; diff --git a/scripts/lib/gateway-supervisor.sh b/scripts/lib/gateway-supervisor.sh index 47de1f44a9c..3a07a5e6c7d 100755 --- a/scripts/lib/gateway-supervisor.sh +++ b/scripts/lib/gateway-supervisor.sh @@ -88,7 +88,7 @@ gateway_control_fail() { local code="$1" local old_pid="${2:-0}" case "$code" in - validator-missing | secret-boundary-refusal | unsafe-config | hash-mismatch | preload-missing | launch-failed | health-timeout | internal) ;; + validator-missing | secret-boundary-refusal | unsafe-config | hash-mismatch | preload-missing | launch-failed | health-timeout | mcp-integrity | mcp-reconcile-required | internal) ;; *) code=internal ;; esac gateway_control_atomic_status "$GATEWAY_CONTROL_NONCE" "failed ${code} ${old_pid} 0" diff --git a/scripts/update-hermes-agent.sh b/scripts/update-hermes-agent.sh index 60f6c90a6c3..8cb5f09b294 100755 --- a/scripts/update-hermes-agent.sh +++ b/scripts/update-hermes-agent.sh @@ -198,6 +198,8 @@ installed_copy_schema_error() { for item in \ "validate-hermes-env-secret-boundary.py" \ "seed-hermes-dashboard-config.py" \ + "COPY agents/hermes/build-mcp-digest.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py" \ + "/opt/hermes/.venv/bin/python -I /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py --guard /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py" \ "hermes-mcp-config-transaction.py" \ "openshell-child-visible-credentials.v0.0.72.json" \ "HERMES_HOME=/sandbox/.hermes /usr/local/bin/hermes doctor --fix" \ diff --git a/src/lib/actions/sandbox/connect-boundary-refusal.ts b/src/lib/actions/sandbox/connect-boundary-refusal.ts new file mode 100644 index 00000000000..dc9e3a62f91 --- /dev/null +++ b/src/lib/actions/sandbox/connect-boundary-refusal.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SecretBoundaryRefusalReason } from "./hermes-secret-boundary-recovery"; +import { + hermesMcpReconciliationRemediationLines, + sanitizeHermesMcpReconciliationDetail, +} from "./mcp-bridge-hermes-reconciliation"; + +type ConnectBoundaryContext = "Probe" | "Connect"; + +export function exitOnSecretBoundaryRefusal( + sandboxName: string, + agentName: string, + processCheck: Record, + contextLabel: ConnectBoundaryContext, +): never { + console.error(""); + const reason = + "secretBoundaryReason" in processCheck + ? (processCheck.secretBoundaryReason as SecretBoundaryRefusalReason | undefined) + : undefined; + if (reason === "raw-secret") { + console.error( + ` ${contextLabel} failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, + ); + console.error( + " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", + ); + } else if (reason === "exec-failed") { + console.error( + ` ${contextLabel} failed: could not execute the secret-boundary check for ${agentName} gateway in '${sandboxName}'.`, + ); + console.error( + " Check sandbox connectivity, then re-run `nemoclaw recover` before connecting.", + ); + } else if (reason === "validator-missing") { + console.error( + ` ${contextLabel} failed: the secret-boundary validator is missing from Hermes gateway in '${sandboxName}'.`, + ); + console.error(" Re-image the sandbox with a current Hermes build before connecting."); + } else if (reason === "agent-missing") { + console.error( + ` ${contextLabel} failed: the Hermes agent definition is unavailable for sandbox '${sandboxName}'.`, + ); + console.error(" Repair the NemoClaw installation, then re-run recovery before connecting."); + } else { + console.error( + ` ${contextLabel} failed: secret-boundary check did not complete for ${agentName} gateway in '${sandboxName}'.`, + ); + console.error(" Inspect the validator output above and re-run `nemoclaw recover`."); + } + process.exit(1); +} + +export function exitOnMcpReconciliationRefusal( + sandboxName: string, + agentName: string, + processCheck: Record, + contextLabel: ConnectBoundaryContext, +): never { + const detail = + "mcpReconciliationReason" in processCheck + ? String(processCheck.mcpReconciliationReason) + : "the effective Hermes MCP configuration does not match persisted managed intent"; + const sanitizedDetail = sanitizeHermesMcpReconciliationDetail(detail); + console.error(""); + console.error( + ` ${contextLabel} failed: refused to confirm ${agentName} gateway in '${sandboxName}' — ${sanitizedDetail}.`, + ); + for (const line of hermesMcpReconciliationRemediationLines(sandboxName)) { + console.error(` ${line}`); + } + process.exit(1); +} diff --git a/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts b/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts index eb135f65a1c..0cf1f382f68 100644 --- a/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts +++ b/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts @@ -78,6 +78,72 @@ describe("connectSandbox Hermes secret-boundary refusals", () => { expect(exitSpy).toHaveBeenCalledWith(1); }); + it("fails closed on Hermes MCP drift with restart and rebuild guidance", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + mcpReconciliationReason: "Hermes MCP config does not match persisted managed intent", + }, + }); + const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("Probe failed: refused to confirm Hermes gateway in 'alpha'"); + expect(errorOutput).toContain("nemoclaw alpha mcp restart"); + expect(errorOutput).toContain("nemoclaw alpha rebuild --yes"); + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + + it("refuses an HTTP-healthy Hermes gateway while MCP integrity is pending", async () => { + const harness = createConnectHarness({ + processCheck: { + checked: true, + // Process recovery normalizes both HTTP 200 and authenticated HTTP 401 + // health probes to this positive running state before reconciliation. + wasRunning: true, + recovered: false, + forwardRecovered: true, + mcpReconciliationRefused: true, + mcpReconciliationReason: + "\x1b[31mHermes MCP integrity is pending\x1b[0m\nFORGED SUCCESS ghp_0123456789abcdefghij", + }, + }); + const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); + + await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(1)"); + + const failureLine = harness.errorSpy.mock.calls + .map((call) => String(call[0] ?? "")) + .find((line) => line.includes("Connect failed:")); + expect(failureLine).toContain("Hermes MCP integrity is pending FORGED SUCCESS "); + expect(failureLine).not.toMatch(/[\r\n\x1b]/); + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain("nemoclaw alpha mcp restart"); + expect(errorOutput).toContain("nemoclaw alpha rebuild --yes"); + expect(errorOutput).not.toContain("ghp_0123456789abcdefghij"); + expect(harness.ensureOllamaAuthProxySpy).not.toHaveBeenCalled(); + expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); + expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( + "openshell", + ["sandbox", "connect", "alpha"], + expect.any(Object), + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + it.each([ [ "raw-secret", diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index dc96ab23ca7..3cd7b0d8335 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -49,6 +49,10 @@ import { CONNECT_AUTO_PAIR_MAX_APPROVALS, CONNECT_AUTO_PAIR_TIMEOUT_MS, } from "./connect-autopair-budget"; +import { + exitOnMcpReconciliationRefusal, + exitOnSecretBoundaryRefusal, +} from "./connect-boundary-refusal"; import { buildSandboxInferenceRouteProbeArgs, type InferenceRouteProbeAgent, @@ -58,7 +62,6 @@ import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-f import { ensureLiveSandboxOrExit, printGatewayLifecycleHint } from "./gateway-state"; import { getSandboxTargetGatewayName } from "./gateway-target"; import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; -import type { SecretBoundaryRefusalReason } from "./hermes-secret-boundary-recovery"; import { checkAndRecoverSandboxProcesses, executeSandboxExecCommand, @@ -193,50 +196,6 @@ export function parseSandboxConnectArgs( return options; } -function exitOnSecretBoundaryRefusal( - sandboxName: string, - agentName: string, - processCheck: Record, - contextLabel: "Probe" | "Connect", -): never { - console.error(""); - const reason = - "secretBoundaryReason" in processCheck - ? (processCheck.secretBoundaryReason as SecretBoundaryRefusalReason | undefined) - : undefined; - if (reason === "raw-secret") { - console.error( - ` ${contextLabel} failed: refused to confirm ${agentName} gateway in '${sandboxName}' — /sandbox/.hermes/.env contains raw secret-shaped values.`, - ); - console.error( - " Replace raw secret values with openshell:resolve:env: placeholders and re-run.", - ); - } else if (reason === "exec-failed") { - console.error( - ` ${contextLabel} failed: could not execute the secret-boundary check for ${agentName} gateway in '${sandboxName}'.`, - ); - console.error( - " Check sandbox connectivity, then re-run `nemoclaw recover` before connecting.", - ); - } else if (reason === "validator-missing") { - console.error( - ` ${contextLabel} failed: the secret-boundary validator is missing from Hermes gateway in '${sandboxName}'.`, - ); - console.error(" Re-image the sandbox with a current Hermes build before connecting."); - } else if (reason === "agent-missing") { - console.error( - ` ${contextLabel} failed: the Hermes agent definition is unavailable for sandbox '${sandboxName}'.`, - ); - console.error(" Repair the NemoClaw installation, then re-run recovery before connecting."); - } else { - console.error( - ` ${contextLabel} failed: secret-boundary check did not complete for ${agentName} gateway in '${sandboxName}'.`, - ); - console.error(" Inspect the validator output above and re-run `nemoclaw recover`."); - } - process.exit(1); -} - function exitOnForwardRecoveryFailure( sandboxName: string, agentName: string, @@ -277,6 +236,9 @@ function runSandboxConnectProbe(sandboxName: string): void { if ("secretBoundaryRefused" in processCheck && processCheck.secretBoundaryRefused) { exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Probe"); } + if ("mcpReconciliationRefused" in processCheck && processCheck.mcpReconciliationRefused) { + exitOnMcpReconciliationRefusal(sandboxName, agentName, processCheck, "Probe"); + } if ("forwardRecoveryFailed" in processCheck && processCheck.forwardRecoveryFailed) { const detail = "forwardRecoveryFailureDetail" in processCheck @@ -962,6 +924,10 @@ export async function connectSandbox( const agentName = agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName)); exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Connect"); } + if ("mcpReconciliationRefused" in processCheck && processCheck.mcpReconciliationRefused) { + const agentName = agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName)); + exitOnMcpReconciliationRefusal(sandboxName, agentName, processCheck, "Connect"); + } // Ensure Ollama auth proxy is running (recovers from host reboots) ensureOllamaAuthProxy(); diff --git a/src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts b/src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts new file mode 100644 index 00000000000..93aa2638562 --- /dev/null +++ b/src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; + +import { hermesAgent } from "../../agent/hermes-recovery-boundary-fixtures"; +import { type GatewayRestartDeps, restartSandboxGatewayWithDeps } from "./gateway-restart"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../../.."); +const HERMES_GUARD = path.join(REPO_ROOT, "agents/hermes/runtime-config-guard.py"); +const HERMES_TRANSACTION = path.join(REPO_ROOT, "agents/hermes/mcp-config-transaction.py"); + +function fixtureSnapshot(paths: readonly string[]): Record { + return Object.fromEntries( + paths.map((filePath) => [path.basename(filePath), fs.readFileSync(filePath, "utf8")]), + ); +} + +it("detects real Hermes config/hash drift without mutating the inspected fixture", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-gateway-drift-")); + const hermesDir = path.join(root, ".hermes"); + const configPath = path.join(hermesDir, "config.yaml"); + const envPath = path.join(hermesDir, ".env"); + const strictHashPath = path.join(root, "hermes.config-hash"); + const compatHashPath = path.join(hermesDir, ".config-hash"); + const fixturePaths = [configPath, envPath, strictHashPath, compatHashPath] as const; + const setup = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, yaml + +def load(name, file_path): + spec = importlib.util.spec_from_file_location(name, file_path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + +transaction = load("gateway_drift_transaction", sys.argv[1]) +guard = load("gateway_drift_guard", sys.argv[2]) +root = sys.argv[3] +hermes = os.path.join(root, ".hermes") +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +strict = os.path.join(root, "hermes.config-hash") +compat = os.path.join(hermes, ".config-hash") +os.mkdir(hermes) +candidate = transaction._managed_candidate({ + "url": "https://api.githubcopilot.com/mcp/", + "headers": {"Authorization": "Bearer openshell:resolve:env:GITHUB_TOKEN"}, +}) +payload = {"present": {"github": candidate}, "absent": []} +open(config, "w", encoding="utf-8").write( + yaml.safe_dump({"model": "test", "mcp_servers": {"github": candidate}}, sort_keys=False) +) +open(env, "w", encoding="utf-8").write("SAFE=1\n") +hash_text, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(strict, hash_text) +guard._write_hash(compat, hash_text) +print(json.dumps(payload, sort_keys=True)) +`, + HERMES_TRANSACTION, + HERMES_GUARD, + root, + ], + { encoding: "utf8", timeout: 10_000 }, + ); + + try { + expect(setup.status, setup.stderr).toBe(0); + const payload = setup.stdout.trim(); + fs.writeFileSync( + configPath, + fs + .readFileSync(configPath, "utf8") + .replace("https://api.githubcopilot.com/mcp/", "https://drift.example.test/mcp"), + ); + const driftedFixture = fixtureSnapshot(fixturePaths); + const inspection = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, os, sys + +spec = importlib.util.spec_from_file_location("gateway_drift_inspection", sys.argv[1]) +transaction = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = transaction +spec.loader.exec_module(transaction) +root = sys.argv[3] +transaction.HERMES_DIR = os.path.join(root, ".hermes") +transaction.CONFIG_PATH = os.path.join(transaction.HERMES_DIR, "config.yaml") +transaction.STRICT_HASH_PATH = os.path.join(root, "hermes.config-hash") +transaction.GUARD_PATH = sys.argv[2] +transaction.os.geteuid = lambda: 0 +sys.argv = [transaction.__file__, "inspect", "--payload", sys.argv[4]] +raise SystemExit(transaction.main()) +`, + HERMES_TRANSACTION, + HERMES_GUARD, + root, + payload, + ], + { encoding: "utf8", timeout: 10_000 }, + ); + expect(inspection.status).toBe(2); + expect(inspection.stderr).toContain("Hermes config hash does not match persisted inputs"); + expect(fixtureSnapshot(fixturePaths)).toEqual(driftedFixture); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +it("sanitizes an injected Hermes reconciliation refusal before post-restart mutations", () => { + try { + const postReconciliationMutations = [ + vi.fn(() => true), + vi.fn(() => null), + vi.fn(() => null), + vi.fn(() => null), + ] as const; + const deps: GatewayRestartDeps = { + getSessionAgent: () => hermesAgent, + getSandbox: () => ({ agent: "hermes" }), + resolveSandboxDashboardPort: () => 18789, + requestGatewaySupervisorAction: vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=123", + stderr: "", + })), + executeSandboxExecCommand: vi.fn(() => null), + waitForRecoveredSandboxGateway: vi.fn(() => true), + ensureSandboxPortForward: postReconciliationMutations[0], + ensureHermesDashboardPortForwardIfEnabled: postReconciliationMutations[1], + recoverMessagingHostForward: postReconciliationMutations[2], + recoverDeclaredAgentForwardPorts: postReconciliationMutations[3], + printGatewayWedgeDiagnostics: vi.fn(() => false), + inspectHermesMcpReconciliationRefusal: vi.fn(() => ({ + detail: "Hermes config hash does not match persisted inputs FORGED SUCCESS ", + })), + }; + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + expect(restartSandboxGatewayWithDeps("alpha", { quiet: true, deps })).toEqual({ + ok: false, + failureLayer: "MCP reconciliation refusal", + detail: "Hermes config hash does not match persisted inputs FORGED SUCCESS ", + }); + expect(postReconciliationMutations[0]).not.toHaveBeenCalled(); + expect(postReconciliationMutations[1]).not.toHaveBeenCalled(); + expect(postReconciliationMutations[2]).not.toHaveBeenCalled(); + expect(postReconciliationMutations[3]).not.toHaveBeenCalled(); + expect(error.mock.calls.flat().join("\n")).not.toMatch(/\x1b|ghp_0123456789abcdefghij/u); + } finally { + vi.restoreAllMocks(); + } +}); diff --git a/src/lib/actions/sandbox/gateway-restart-mcp.test.ts b/src/lib/actions/sandbox/gateway-restart-mcp.test.ts new file mode 100644 index 00000000000..c665a9fa336 --- /dev/null +++ b/src/lib/actions/sandbox/gateway-restart-mcp.test.ts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { hermesAgent } from "../../agent/hermes-recovery-boundary-fixtures"; +import type { GatewayRestartDeps } from "./gateway-restart"; +import { restartSandboxGateway } from "./process-recovery"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function silenceConsole() { + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + return () => { + log.mockRestore(); + error.mockRestore(); + }; +} + +function baseDeps(overrides: Partial = {}): GatewayRestartDeps { + return { + getSessionAgent: () => hermesAgent, + getSandbox: () => ({ name: "alpha", agent: "hermes" }), + resolveSandboxDashboardPort: () => 18789, + requestGatewaySupervisorAction: vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=123", + stderr: "", + })), + executeSandboxExecCommand: vi.fn(() => null), + waitForRecoveredSandboxGateway: vi.fn(() => true), + ensureSandboxPortForward: vi.fn(() => true), + ensureHermesDashboardPortForwardIfEnabled: vi.fn(() => null), + recoverMessagingHostForward: vi.fn(() => null), + recoverDeclaredAgentForwardPorts: vi.fn(() => null), + printGatewayWedgeDiagnostics: vi.fn(() => false), + inspectHermesMcpReconciliationRefusal: vi.fn(() => null), + ...overrides, + }; +} + +describe("Hermes MCP gateway restart", () => { + it("refuses to report a restarted gateway with stale MCP intent", () => { + const restore = silenceConsole(); + try { + const deps = baseDeps({ + inspectHermesMcpReconciliationRefusal: vi.fn(() => ({ + detail: "Hermes MCP config does not match persisted managed intent", + })), + }); + + expect(restartSandboxGateway("alpha", { quiet: true, deps })).toEqual({ + ok: false, + failureLayer: "MCP reconciliation refusal", + detail: "Hermes MCP config does not match persisted managed intent", + }); + expect(deps.ensureSandboxPortForward).not.toHaveBeenCalled(); + } finally { + restore(); + } + }); + + it("returns only sanitized MCP reconciliation detail", () => { + const restore = silenceConsole(); + try { + const deps = baseDeps({ + inspectHermesMcpReconciliationRefusal: vi.fn(() => ({ + detail: "integrity pending FORGED SUCCESS ", + })), + }); + + expect(restartSandboxGateway("alpha", { quiet: true, deps })).toEqual({ + ok: false, + failureLayer: "MCP reconciliation refusal", + detail: "integrity pending FORGED SUCCESS ", + }); + expect(deps.ensureSandboxPortForward).not.toHaveBeenCalled(); + } finally { + restore(); + } + }); + + it("prints MCP recovery guidance for a supervisor-side integrity refusal", () => { + const restore = silenceConsole(); + try { + const deps = baseDeps({ + requestGatewaySupervisorAction: vi.fn(() => ({ + status: 1, + stdout: "", + stderr: + "v1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa failed mcp-integrity 4242 0\nHERMES_MCP_CONFIG_DRIFT", + })), + }); + + expect(restartSandboxGateway("alpha", { quiet: true, deps })).toMatchObject({ + ok: false, + failureLayer: "MCP reconciliation refusal", + }); + expect(deps.waitForRecoveredSandboxGateway).not.toHaveBeenCalled(); + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain("nemoclaw alpha mcp restart"); + expect(output).toContain("nemoclaw alpha rebuild --yes"); + } finally { + restore(); + } + }); +}); diff --git a/src/lib/actions/sandbox/gateway-restart.test.ts b/src/lib/actions/sandbox/gateway-restart.test.ts index 56ef2b02f3e..e41540c18d7 100644 --- a/src/lib/actions/sandbox/gateway-restart.test.ts +++ b/src/lib/actions/sandbox/gateway-restart.test.ts @@ -23,6 +23,9 @@ describe("gateway restart failure markers", () => { [MARKERS.SECRET_BOUNDARY_REFUSED, "secret-boundary refusal"], [MARKERS.SECRET_BOUNDARY_VALIDATOR_MISSING, "unsafe config path"], [MARKERS.GATEWAY_UNSAFE_CONFIG_PATH, "unsafe config path"], + ["mcp-integrity", "MCP reconciliation refusal"], + ["mcp-reconcile-required", "MCP reconciliation refusal"], + ["HERMES_MCP_CONFIG_DRIFT", "MCP reconciliation refusal"], [MARKERS.GATEWAY_CONFIG_HASH_MISMATCH, "config hash mismatch"], ["HERMES_UNSAFE_CONFIG_PATH", "unsafe config path"], ["HERMES_LOCKED_HASH_MISMATCH", "config hash mismatch"], @@ -70,6 +73,7 @@ describe("restartSandboxGateway — host-mediated gateway restart", () => { recoverMessagingHostForward: vi.fn(() => null), recoverDeclaredAgentForwardPorts: vi.fn(() => null), printGatewayWedgeDiagnostics: vi.fn(() => false), + inspectHermesMcpReconciliationRefusal: vi.fn(() => null), ...overrides, }; } diff --git a/src/lib/actions/sandbox/gateway-restart.ts b/src/lib/actions/sandbox/gateway-restart.ts index 42ade503ff5..529b5922978 100644 --- a/src/lib/actions/sandbox/gateway-restart.ts +++ b/src/lib/actions/sandbox/gateway-restart.ts @@ -5,6 +5,8 @@ import { GATEWAY_RESTART_MARKERS as MARKERS } from "../../agent/gateway-restart- import * as agentRuntime from "../../agent/runtime"; import { G, R } from "../../cli/terminal-style"; import { redactFull } from "../../security/redact"; +import { hermesMcpReconciliationRemediationLines } from "./mcp-bridge-hermes-reconciliation"; +import { inspectHermesMcpReconciliationRefusal } from "./mcp-bridge-recovery"; export type GatewayRestartCommandResult = { status: number; @@ -18,6 +20,7 @@ export type GatewayRestartFailureLayer = | "secret-boundary refusal" | "unsafe config path" | "config hash mismatch" + | "MCP reconciliation refusal" | "launch failure" | "health timeout" | "forward recovery failure"; @@ -77,6 +80,7 @@ export type GatewayRestartDeps = { sandboxName: string, exec: (sandboxName: string, command: string) => GatewayRestartCommandResult | null, ) => boolean; + inspectHermesMcpReconciliationRefusal: typeof inspectHermesMcpReconciliationRefusal; }; export type RestartSandboxGatewayOptions = { @@ -151,6 +155,16 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul ) { return { layer: "unsafe config path", detail: detail || "unsafe config path" }; } + if ( + output.includes("mcp-integrity") || + output.includes("mcp-reconcile-required") || + output.includes("HERMES_MCP_CONFIG_DRIFT") + ) { + return { + layer: "MCP reconciliation refusal", + detail: detail || "Hermes MCP reconciliation refused", + }; + } if ( output.includes(MARKERS.GATEWAY_CONFIG_HASH_MISMATCH) || output.includes("HERMES_LOCKED_HASH_MISMATCH") || @@ -182,6 +196,11 @@ export function printGatewayRestartFailure( for (const line of lines) { console.error(` ${line}`); } + if (layer === "MCP reconciliation refusal") { + for (const line of hermesMcpReconciliationRemediationLines(sandboxName)) { + console.error(` ${line}`); + } + } } function unsupportedGatewayRestartAgentDetail(agentName: string, reason: string): string { @@ -291,6 +310,19 @@ export function restartSandboxGatewayWithDeps( return { ok: false, failureLayer: "health timeout", detail }; } + if (agentName === "hermes") { + const refusal = deps.inspectHermesMcpReconciliationRefusal(sandboxName); + if (refusal) { + const { detail } = refusal; + printGatewayRestartFailure(sandboxName, "MCP reconciliation refusal", detail); + return { + ok: false, + failureLayer: "MCP reconciliation refusal", + detail, + }; + } + } + const forwardRecovered = deps.ensureSandboxPortForward(sandboxName); const dashboardForwardRecovered = deps.ensureHermesDashboardPortForwardIfEnabled(sandboxName); const messagingForwardRecovered = deps.recoverMessagingHostForward(sandboxName, { quiet }); diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-status.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-status.ts index ee51f0bc943..07942dd9c2c 100644 --- a/src/lib/actions/sandbox/mcp-bridge-adapter-status.ts +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-status.ts @@ -68,7 +68,7 @@ export function mcporterHeaderMatcherSource(): string { return `const mcporterHeadersMatchExpected = ${mcporterHeadersMatchExpected.toString()};`; } -function hermesManagedServerConfig(entry: McpBridgeEntry): Record { +export function hermesManagedServerConfig(entry: McpBridgeEntry): Record { const headers = entryHeaders(entry); return { url: entry.url, @@ -80,6 +80,25 @@ function hermesManagedServerConfig(entry: McpBridgeEntry): Record>; + absent: string[]; +} + +/** Render the host registry into the credential-safe shape persisted by Hermes. */ +export function buildHermesMcpIntentPayload( + entries: readonly McpBridgeEntry[], + managedServerNames: readonly string[], +): HermesMcpIntentPayload { + const sortedEntries = [...entries].sort((left, right) => left.server.localeCompare(right.server)); + const present = Object.fromEntries( + sortedEntries.map((entry) => [entry.server, hermesManagedServerConfig(entry)]), + ); + const presentNames = new Set(Object.keys(present)); + const absent = [...new Set(managedServerNames)].filter((name) => !presentNames.has(name)).sort(); + return { present, absent }; +} + export function deepAgentsManagedServerConfig(entry: McpBridgeEntry): Record { const headers = entryHeaders(entry); return { diff --git a/src/lib/actions/sandbox/mcp-bridge-add-restart.ts b/src/lib/actions/sandbox/mcp-bridge-add-restart.ts index 810f078ae67..6f86b044486 100644 --- a/src/lib/actions/sandbox/mcp-bridge-add-restart.ts +++ b/src/lib/actions/sandbox/mcp-bridge-add-restart.ts @@ -16,6 +16,7 @@ import { unregisterAgentAdapter, } from "./mcp-bridge-adapters"; import { type McpBridgeAddOptions, McpBridgeError } from "./mcp-bridge-contracts"; +import { assertHermesMcpRuntimeIntent } from "./mcp-bridge-hermes-reconciliation"; import { applyGeneratedPolicy, buildMcpBridgePolicyKey, @@ -52,6 +53,7 @@ import { } from "./mcp-bridge-state"; import { assertAuthenticatedCredentialReference, + assertMcpCredentialBoundaryRuntimeVersion, buildMcpBridgeProviderName, normalizeMcpServerUrl, resolveCredentialEnv, @@ -214,6 +216,9 @@ async function addMcpBridgeUnlocked( // prepared manifest is written. The in-sandbox helper repeats the check at // the actual config write so a concurrent posture change still fails closed. assertAgentMcpConfigMutationAllowed(sandboxName, adapter); + // Bind the static credential-name deny-list to the OpenShell binary before + // persisting ownership or mutating a provider, policy, or adapter. + assertMcpCredentialBoundaryRuntimeVersion(); // This is the durable ownership manifest for every resource created below. // It intentionally precedes gateway selection and all OpenShell mutations, // so process death can never leave an unowned provider/policy/adapter entry. @@ -344,6 +349,7 @@ async function addMcpBridgeUnlocked( // Replacing it is idempotent and, for Hermes, re-verifies runtime reload. replaceExisting: resumingPreflightedAdd && adapterInspection.state === "registered", }); + if (adapter === "hermes-config") assertHermesMcpRuntimeIntent(sandboxName); const { addState: _completedAddState, ...committedEntry } = entry; writeBridgeEntry(sandboxName, committedEntry); } catch (error) { diff --git a/src/lib/actions/sandbox/mcp-bridge-destroy.ts b/src/lib/actions/sandbox/mcp-bridge-destroy.ts index 3607ee0c847..5d70c2ca281 100644 --- a/src/lib/actions/sandbox/mcp-bridge-destroy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-destroy.ts @@ -147,6 +147,9 @@ export async function prepareMcpBridgesForDestroy( bridges: Object.fromEntries( entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), ), + ...(sandbox.mcp?.managedServerNames + ? { managedServerNames: sandbox.mcp.managedServerNames } + : {}), destroyPreparedAt: nowIso(), }, }); @@ -179,6 +182,9 @@ export async function prepareMcpBridgesForDestroy( bridges: Object.fromEntries( entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), ), + ...(current.mcp.managedServerNames + ? { managedServerNames: current.mcp.managedServerNames } + : {}), }, }); } catch (rollbackError) { @@ -218,6 +224,9 @@ export async function restoreMcpBridgesAfterDestroyAbort( bridges: Object.fromEntries( preparation.entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), ), + ...(preparedSandbox.mcp?.managedServerNames + ? { managedServerNames: preparedSandbox.mcp.managedServerNames } + : {}), }, }); if (!cleared) { @@ -241,6 +250,9 @@ export async function restoreMcpBridgesAfterDestroyAbort( bridges: Object.fromEntries( preparation.entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), ), + ...(preparedSandbox.mcp?.managedServerNames + ? { managedServerNames: preparedSandbox.mcp.managedServerNames } + : {}), destroyPreparedAt, }, }); @@ -280,6 +292,9 @@ export async function finalizeMcpBridgesAfterSandboxDelete( bridges: Object.fromEntries( entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), ), + ...(sandbox.mcp?.managedServerNames + ? { managedServerNames: sandbox.mcp.managedServerNames } + : {}), destroyPendingAt: nowIso(), }, }); diff --git a/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.test.ts b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.test.ts new file mode 100644 index 00000000000..cb2d0f88488 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.test.ts @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { McpBridgeEntry, SandboxEntry } from "../../state/registry"; + +const mocks = vi.hoisted(() => ({ + getSandbox: vi.fn(), + runOpenshellProviderCommand: vi.fn(), +})); + +vi.mock("../../state/registry", () => ({ + getSandbox: mocks.getSandbox, +})); + +vi.mock("../../actions/global", () => ({ + runOpenshellProviderCommand: mocks.runOpenshellProviderCommand, +})); + +import { + assertHermesMcpRuntimeIntent, + inspectHermesMcpRuntimeIntent, +} from "./mcp-bridge-hermes-reconciliation"; + +const entry: McpBridgeEntry = { + server: "github", + agent: "hermes", + adapter: "hermes-config", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), +}; + +function sandbox(overrides: Partial = {}): SandboxEntry { + return { + name: "alpha", + agent: "hermes", + mcp: { + bridges: { github: entry }, + managedServerNames: ["github", "retired"], + }, + ...overrides, + }; +} + +describe("Hermes MCP host reconciliation", () => { + beforeEach(() => { + mocks.getSandbox.mockReset().mockReturnValue(sandbox()); + mocks.runOpenshellProviderCommand.mockReset().mockReturnValue({ + status: 0, + stdout: '{"ok":true,"state":"matched"}\n', + stderr: "", + }); + }); + + afterEach(() => { + delete process.env.GITHUB_TOKEN; + }); + + it("sends the complete credential-safe present and absent projection", () => { + expect(inspectHermesMcpRuntimeIntent("alpha")).toEqual({ ok: true, state: "matched" }); + + const [args, options] = mocks.runOpenshellProviderCommand.mock.calls[0]; + expect(args.slice(0, 8)).toEqual([ + "sandbox", + "exec", + "--name", + "alpha", + "--timeout", + "45", + "--no-tty", + "--", + ]); + expect(args.slice(8, 11)).toEqual([ + "/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", + "inspect", + "--payload", + ]); + expect(JSON.parse(args[11])).toEqual({ + present: { + github: { + url: "https://api.githubcopilot.com/mcp/", + enabled: true, + timeout: 120, + connect_timeout: 60, + tools: { resources: true, prompts: true }, + headers: { Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN" }, + }, + }, + absent: ["retired"], + }); + expect(JSON.stringify(args)).not.toContain("host-only-secret"); + expect(options).toMatchObject({ ignoreError: true, timeout: 60_000 }); + }); + + it("can inspect a removal intent while retaining the removed name as a tombstone", () => { + expect( + inspectHermesMcpRuntimeIntent("alpha", { + entries: [], + managedServerNames: ["github", "retired"], + }), + ).toEqual({ ok: true, state: "matched" }); + + expect(JSON.parse(mocks.runOpenshellProviderCommand.mock.calls[0][0][11])).toEqual({ + present: {}, + absent: ["github", "retired"], + }); + }); + + it("fails closed and sanitizes helper stdout and stderr", () => { + process.env.GITHUB_TOKEN = "host-only-secret"; + mocks.runOpenshellProviderCommand.mockReturnValue({ + status: 2, + stdout: "\x1b[32mFORGED SUCCESS\x1b[0m\ngeneric ghp_0123456789abcdefghij", + stderr: "Hermes MCP config drifted: host-only-secret\r\n\x1b]0;spoof\x07SECOND", + }); + + expect(inspectHermesMcpRuntimeIntent("alpha")).toEqual({ + ok: false, + state: "mismatch", + detail: "Hermes MCP config drifted: ***REDACTED*** SECOND FORGED SUCCESS generic ", + }); + expect(() => assertHermesMcpRuntimeIntent("alpha")).toThrow( + /does not match the persisted managed intent/, + ); + }); + + it("sanitizes thrown helper failures before returning or throwing them", () => { + process.env.GITHUB_TOKEN = "host-only-secret"; + mocks.runOpenshellProviderCommand.mockImplementation(() => { + throw new Error( + "\x1b[31mhelper failed\x1b[0m\nFORGED READY host-only-secret sk-proj-0123456789abcdef", + ); + }); + + expect(inspectHermesMcpRuntimeIntent("alpha")).toEqual({ + ok: false, + state: "error", + detail: "helper failed FORGED READY ***REDACTED*** ", + }); + + let thrown: unknown; + try { + assertHermesMcpRuntimeIntent("alpha"); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(Error); + expect((thrown as Error).message).not.toMatch(/[\r\n\x1b]/); + expect((thrown as Error).message).not.toContain("host-only-secret"); + expect((thrown as Error).message).not.toContain("sk-proj-0123456789abcdef"); + expect((thrown as Error).message).toContain( + "helper failed FORGED READY ***REDACTED*** ", + ); + }); + + it("does not execute the Hermes helper for an untracked non-Hermes sandbox", () => { + mocks.getSandbox.mockReturnValue({ name: "alpha", agent: "openclaw" }); + + expect(inspectHermesMcpRuntimeIntent("alpha")).toEqual({ + ok: true, + state: "not-applicable", + }); + expect(mocks.runOpenshellProviderCommand).not.toHaveBeenCalled(); + }); + + it("fails closed when a Hermes bridge is attached to another explicit agent", () => { + mocks.getSandbox.mockReturnValue(sandbox({ agent: "openclaw" })); + + expect(inspectHermesMcpRuntimeIntent("alpha")).toEqual({ + ok: false, + state: "error", + detail: "Registry entry agent mismatch for Hermes MCP sandbox 'alpha'.", + }); + expect(mocks.runOpenshellProviderCommand).not.toHaveBeenCalled(); + }); + + it("retains the Hermes adapter fallback for legacy entries without an agent", () => { + mocks.getSandbox.mockReturnValue(sandbox({ agent: null })); + + expect(inspectHermesMcpRuntimeIntent("alpha")).toEqual({ ok: true, state: "matched" }); + expect(mocks.runOpenshellProviderCommand).toHaveBeenCalledOnce(); + }); + + it("fails closed when a corrupted registry key returns another sandbox name", () => { + mocks.getSandbox.mockReturnValue(sandbox({ name: "other" })); + + expect(inspectHermesMcpRuntimeIntent("alpha")).toEqual({ + ok: false, + state: "error", + detail: "Registry entry name mismatch for sandbox 'alpha'.", + }); + expect(mocks.runOpenshellProviderCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.ts b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.ts new file mode 100644 index 00000000000..83b55fca7f6 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-hermes-reconciliation.ts @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { runOpenshellProviderCommand } from "../../actions/global"; +import { redactFull } from "../../security/redact"; +import type { McpBridgeEntry, SandboxEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { buildHermesMcpIntentPayload } from "./mcp-bridge-adapter-status"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; + +const HERMES_MCP_TRANSACTION_HELPER = "/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py"; +const HERMES_MCP_INSPECT_TIMEOUT_SECONDS = 45; +const HERMES_MCP_INSPECT_TIMEOUT_MS = 60_000; +const HERMES_MCP_RECONCILIATION_FAILURE = + "Hermes MCP runtime does not match the persisted managed intent"; +const ANSI_OR_UNSAFE_CONTROL_RE = + /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])|[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/g; +const DISPLAY_LINE_BREAK_RE = /[\r\n\u2028\u2029]+/g; + +export type HermesMcpReconciliationResult = + | { ok: true; state: "matched" | "not-applicable" } + | { ok: false; state: "mismatch" | "error"; detail: string }; + +export interface HermesMcpReconciliationOptions { + entries?: readonly McpBridgeEntry[]; + managedServerNames?: readonly string[]; +} + +export function hermesMcpReconciliationRemediationLines(sandboxName: string): readonly string[] { + return [ + `Run \`nemoclaw ${sandboxName} mcp restart\` to restore the managed MCP configuration, then retry.`, + `If the sandbox has an old helper or missing runtime metadata, run \`nemoclaw ${sandboxName} rebuild --yes\` instead.`, + ]; +} + +function bridgeEntries(sandbox: SandboxEntry): McpBridgeEntry[] { + return Object.values(sandbox.mcp?.bridges ?? {}); +} + +function appliesToHermes(sandbox: SandboxEntry, entries: readonly McpBridgeEntry[]): boolean { + return sandbox.agent === "hermes" || entries.some((entry) => entry.adapter === "hermes-config"); +} + +function buildInspectArgs(sandboxName: string, payload: string): string[] { + return [ + "sandbox", + "exec", + "--name", + sandboxName, + "--timeout", + String(HERMES_MCP_INSPECT_TIMEOUT_SECONDS), + "--no-tty", + "--", + HERMES_MCP_TRANSACTION_HELPER, + "inspect", + "--payload", + payload, + ]; +} + +function parseLastJsonObject(output: string): Record | null { + for (const line of output.trim().split(/\r?\n/).reverse()) { + try { + const parsed = JSON.parse(line) as unknown; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // OpenShell can frame diagnostics around the helper's single JSON line. + } + } + return null; +} + +export function sanitizeHermesMcpReconciliationDetail( + detail: string, + entries: readonly McpBridgeEntry[] = [], +): string { + // Reconciliation detail crosses from an untrusted sandbox helper into host + // exceptions and terminal output. Remove terminal controls before matching + // secrets so escape bytes cannot split a token and evade redaction. + let sanitized = String(detail || "").replace(ANSI_OR_UNSAFE_CONTROL_RE, ""); + for (const entry of entries) { + const envValues = Object.fromEntries( + entry.env.flatMap((name) => (process.env[name] ? [[name, process.env[name]]] : [])), + ); + sanitized = redactBridgeSecretsForDisplay(sanitized, entry, envValues); + } + return ( + redactFull(sanitized).replace(DISPLAY_LINE_BREAK_RE, " ").replace(/\s+/g, " ").trim() || + HERMES_MCP_RECONCILIATION_FAILURE + ); +} + +function commandStream(value: string | Buffer | null | undefined): string { + return typeof value === "string" ? value : (value?.toString() ?? ""); +} + +function sanitizedCommandDetail( + result: ReturnType, + entries: readonly McpBridgeEntry[], +): string { + return sanitizeHermesMcpReconciliationDetail( + [commandStream(result.stderr), commandStream(result.stdout), result.error?.message] + .filter(Boolean) + .join("\n"), + entries, + ); +} + +export function inspectHermesMcpRuntimeIntent( + sandboxName: string, + options: HermesMcpReconciliationOptions = {}, +): HermesMcpReconciliationResult { + const sandbox = registry.getSandbox(sandboxName); + if (!sandbox) { + return { + ok: false, + state: "error", + detail: sanitizeHermesMcpReconciliationDetail(`Sandbox '${sandboxName}' not found.`), + }; + } + if (sandbox.name !== sandboxName) { + return { + ok: false, + state: "error", + detail: sanitizeHermesMcpReconciliationDetail( + `Registry entry name mismatch for sandbox '${sandboxName}'.`, + ), + }; + } + const entries = options.entries ? [...options.entries] : bridgeEntries(sandbox); + const managedServerNames = options.managedServerNames + ? [...options.managedServerNames] + : [...(sandbox.mcp?.managedServerNames ?? entries.map((entry) => entry.server))]; + if (!appliesToHermes(sandbox, entries) || (!sandbox.mcp && options.entries === undefined)) { + return { ok: true, state: "not-applicable" }; + } + if (sandbox.agent != null && sandbox.agent !== "hermes") { + return { + ok: false, + state: "error", + detail: sanitizeHermesMcpReconciliationDetail( + `Registry entry agent mismatch for Hermes MCP sandbox '${sandboxName}'.`, + entries, + ), + }; + } + + const payload = buildHermesMcpIntentPayload(entries, managedServerNames); + let result: ReturnType; + try { + result = runOpenshellProviderCommand(buildInspectArgs(sandboxName, JSON.stringify(payload)), { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + timeout: HERMES_MCP_INSPECT_TIMEOUT_MS, + }); + } catch (error) { + return { + ok: false, + state: "error", + detail: sanitizeHermesMcpReconciliationDetail( + error instanceof Error ? error.message : String(error), + entries, + ), + }; + } + const response = parseLastJsonObject(result.stdout || ""); + if ( + result.status === 0 && + !result.error && + response?.ok === true && + response.state === "matched" + ) { + return { ok: true, state: "matched" }; + } + return { + ok: false, + state: result.status === 2 ? "mismatch" : "error", + detail: sanitizedCommandDetail(result, entries), + }; +} + +export function assertHermesMcpRuntimeIntent( + sandboxName: string, + options: HermesMcpReconciliationOptions = {}, +): void { + const inspection = inspectHermesMcpRuntimeIntent(sandboxName, options); + if (inspection.ok) return; + throw new McpBridgeError( + `${sanitizeHermesMcpReconciliationDetail( + `${HERMES_MCP_RECONCILIATION_FAILURE} for sandbox '${sandboxName}': ${inspection.detail}`, + options.entries, + )}.`, + ); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts index 20c654be597..69c749a5382 100644 --- a/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts @@ -14,9 +14,85 @@ import { parseMcpAddArgs, resolveCredentialEnv, } from "./mcp-bridge"; +import { assertMcpCredentialBoundaryRuntimeVersion } from "./mcp-bridge-validation"; import childVisibleCredentialManifest from "./openshell-child-visible-credentials.v0.0.72.json"; +function matchingOpenshellRuntime() { + return { + resolveOpenshell: () => "/test/openshell", + runVersionCommand: () => ({ + status: 0, + stdout: "openshell 0.0.72\n", + stderr: "", + }), + }; +} + describe("MCP CLI input validation", () => { + it("requires the runtime OpenShell version to match the credential boundary manifest", () => { + expect(() => + assertMcpCredentialBoundaryRuntimeVersion(matchingOpenshellRuntime()), + ).not.toThrow(); + + expect(() => + assertMcpCredentialBoundaryRuntimeVersion({ + ...matchingOpenshellRuntime(), + runVersionCommand: () => ({ + status: 0, + stdout: "openshell 0.0.73\n", + stderr: "", + }), + }), + ).toThrow( + /expected 0\.0\.72, actual 0\.0\.73 \(version mismatch\)\. Install OpenShell 0\.0\.72, or point NEMOCLAW_OPENSHELL_BIN to that version, then retry\./, + ); + }); + + it("fails closed when the runtime OpenShell binary is missing", () => { + expect(() => + assertMcpCredentialBoundaryRuntimeVersion({ resolveOpenshell: () => null }), + ).toThrow(/expected 0\.0\.72, actual \(openshell binary not found\)/); + }); + + it("fails closed when openshell --version exits unsuccessfully", () => { + const deps = { + ...matchingOpenshellRuntime(), + runVersionCommand: () => ({ + status: 23, + stdout: "", + stderr: "credential-shaped-output-must-not-be-repeated", + }), + }; + expect(() => assertMcpCredentialBoundaryRuntimeVersion(deps)).toThrow( + /expected 0\.0\.72, actual \(openshell --version exited with status 23\)/, + ); + try { + assertMcpCredentialBoundaryRuntimeVersion(deps); + } catch (error) { + expect(String(error)).not.toContain("credential-shaped-output-must-not-be-repeated"); + } + }); + + it("fails closed without reflecting unparseable version output", () => { + const deps = { + ...matchingOpenshellRuntime(), + runVersionCommand: () => ({ + status: 0, + stdout: "not-a-version credential-shaped-output-must-not-be-repeated\n", + stderr: "", + }), + }; + try { + assertMcpCredentialBoundaryRuntimeVersion(deps); + throw new Error("expected runtime version validation to fail"); + } catch (error) { + expect(String(error)).toMatch( + /expected 0\.0\.72, actual \(invalid openshell --version output\)/, + ); + expect(String(error)).not.toContain("credential-shaped-output-must-not-be-repeated"); + } + }); + it("parses server, URL, and env references", () => { const parsed = parseMcpAddArgs([ "github", diff --git a/src/lib/actions/sandbox/mcp-bridge-recovery.test.ts b/src/lib/actions/sandbox/mcp-bridge-recovery.test.ts new file mode 100644 index 00000000000..623b45a04c5 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-recovery.test.ts @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + inspectHermesMcpReconciliationRefusal, + processRecoveryMcpReconciliationRefusal, +} from "./mcp-bridge-recovery"; + +describe("Hermes MCP recovery boundary (#6257)", () => { + it("continues when runtime intent matches", () => { + expect( + inspectHermesMcpReconciliationRefusal("alpha", () => ({ + ok: true, + state: "matched", + })), + ).toBeNull(); + }); + + it("sanitizes a reconciliation refusal once at the shared boundary", () => { + expect( + inspectHermesMcpReconciliationRefusal("alpha", () => ({ + ok: false, + state: "mismatch", + detail: "\u001b[31mdrifted\u001b[0m\nFORGED", + })), + ).toEqual({ detail: "drifted FORGED" }); + }); + + it.each([true, false])("maps refusal into the process recovery contract (%s)", (wasRunning) => { + expect( + processRecoveryMcpReconciliationRefusal("alpha", wasRunning, () => ({ + ok: false, + state: "error", + detail: "runtime mismatch", + })), + ).toEqual({ + checked: true, + wasRunning, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + mcpReconciliationReason: "runtime mismatch", + }); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-recovery.ts b/src/lib/actions/sandbox/mcp-bridge-recovery.ts new file mode 100644 index 00000000000..a9e85582dc0 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-recovery.ts @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type HermesMcpReconciliationResult, + inspectHermesMcpRuntimeIntent, + sanitizeHermesMcpReconciliationDetail, +} from "./mcp-bridge-hermes-reconciliation"; + +export type McpReconciliationRefusalRecoveryResult = { + checked: true; + wasRunning: boolean; + recovered: false; + forwardRecovered: false; + forwardRecoveryFailed?: undefined; + forwardRecoveryFailureDetail?: undefined; + mcpReconciliationRefused: true; + mcpReconciliationReason: string; +}; + +type InspectHermesMcpRuntimeIntent = (sandboxName: string) => HermesMcpReconciliationResult; + +export function inspectHermesMcpReconciliationRefusal( + sandboxName: string, + inspect: InspectHermesMcpRuntimeIntent = inspectHermesMcpRuntimeIntent, +): { detail: string } | null { + const reconciliation = inspect(sandboxName); + if (reconciliation.ok) return null; + return { detail: sanitizeHermesMcpReconciliationDetail(reconciliation.detail) }; +} + +export function processRecoveryMcpReconciliationRefusal( + sandboxName: string, + wasRunning: boolean, + inspect: InspectHermesMcpRuntimeIntent = inspectHermesMcpRuntimeIntent, +): McpReconciliationRefusalRecoveryResult | null { + const refusal = inspectHermesMcpReconciliationRefusal(sandboxName, inspect); + if (!refusal) return null; + return { + checked: true, + wasRunning, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + mcpReconciliationReason: refusal.detail, + }; +} diff --git a/src/lib/actions/sandbox/mcp-bridge-remove.ts b/src/lib/actions/sandbox/mcp-bridge-remove.ts index 8a31a1586f8..afe9722b5c1 100644 --- a/src/lib/actions/sandbox/mcp-bridge-remove.ts +++ b/src/lib/actions/sandbox/mcp-bridge-remove.ts @@ -10,6 +10,7 @@ import { unregisterAgentAdapter, } from "./mcp-bridge-adapters"; import { isAgentMcpAdapter, McpBridgeError } from "./mcp-bridge-contracts"; +import { assertHermesMcpRuntimeIntent } from "./mcp-bridge-hermes-reconciliation"; import { assertGeneratedPolicyMutationSafe, removeGeneratedPolicy } from "./mcp-bridge-policy"; import { deleteProvider, @@ -244,6 +245,14 @@ async function removeMcpBridgeUnlocked( `Could not prove removal of the exact managed adapter entry for MCP server '${entry.server}'. Preserved provider, policy, and registry ownership state.`, ); } + if (adapter === "hermes-config") { + assertHermesMcpRuntimeIntent(sandboxName, { + entries: Object.values(bridgeState(sandbox)).filter( + (candidate) => candidate.server !== server, + ), + managedServerNames: sandbox.mcp?.managedServerNames, + }); + } } catch (error) { const detail = error instanceof Error ? error.message : String(error); if (!options.force) throw new McpBridgeError(detail); diff --git a/src/lib/actions/sandbox/mcp-bridge-restart.ts b/src/lib/actions/sandbox/mcp-bridge-restart.ts index bbdeac5c0e0..680760c60fe 100644 --- a/src/lib/actions/sandbox/mcp-bridge-restart.ts +++ b/src/lib/actions/sandbox/mcp-bridge-restart.ts @@ -6,6 +6,7 @@ import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import type { McpBridgeEntry } from "../../state/registry"; import { registerAgentAdapter } from "./mcp-bridge-adapters"; import { McpBridgeError } from "./mcp-bridge-contracts"; +import { assertHermesMcpRuntimeIntent } from "./mcp-bridge-hermes-reconciliation"; import { applyGeneratedPolicy, assertGeneratedPolicyMutationSafe } from "./mcp-bridge-policy"; import { assertMcpProviderRecoverable, @@ -37,6 +38,7 @@ import { } from "./mcp-bridge-state"; import { assertAuthenticatedBridgeEntry, + assertMcpCredentialBoundaryRuntimeVersion, resolveCredentialEnv, validateSandboxName, } from "./mcp-bridge-validation"; @@ -67,6 +69,7 @@ async function restartMcpBridgeUnlocked(sandboxName: string, server?: string): P const bridges = bridgeState(sandbox); const targets = server ? [[server, bridges[server]] as const] : Object.entries(bridges); if (targets.length === 0) { + if (adapter === "hermes-config") assertHermesMcpRuntimeIntent(sandboxName); console.log(` No MCP servers for sandbox '${sandboxName}'.`); return; } @@ -88,6 +91,7 @@ async function restartMcpBridgeUnlocked(sandboxName: string, server?: string): P // recovery/selection, provider inspection, or any lifecycle mutation. assertMcpAdapterConfigMutationsAllowed(sandboxName, sandbox, targetEntries); const resolvedByServer = await preflightMcpEntryTargets(targetEntries); + assertMcpCredentialBoundaryRuntimeVersion(); await ensureSandboxGatewaySelected(sandboxName); // Prove every policy key is absent or still matches its recorded ownership // before inspecting or updating any provider. `applyGeneratedPolicy` repeats @@ -173,6 +177,7 @@ async function restartMcpBridgeUnlocked(sandboxName: string, server?: string): P }); console.log(` Refreshed MCP server '${name}'.`); } + if (adapter === "hermes-config") assertHermesMcpRuntimeIntent(sandboxName); } export async function restoreExistingMcpBridgeRuntime( @@ -183,6 +188,9 @@ export async function restoreExistingMcpBridgeRuntime( if (entries.length === 0) return; for (const entry of entries) assertAuthenticatedBridgeEntry(entry); const resolvedByServer = await preflightMcpEntryTargets(entries); + if (options.lifecyclePhase !== "teardown-rollback") { + assertMcpCredentialBoundaryRuntimeVersion(); + } await ensureSandboxGatewaySelected(sandboxName); const sandbox = getSandboxOrThrow(sandboxName); assertMcpDestroyNotPending(sandbox); @@ -195,6 +203,7 @@ export async function restoreExistingMcpBridgeRuntime( } else { assertMcpAdapterMutationRuntimeCapabilities(sandboxName, sandbox, entries); } + const defaultAdapter = getBridgeAdapter(getSandboxAgent(sandbox)); for (const entry of entries) { assertGeneratedPolicyMutationSafe(sandboxName, entry); const provider = assertMcpProviderRecoverable(entry); @@ -207,8 +216,7 @@ export async function restoreExistingMcpBridgeRuntime( applyGeneratedPolicy(sandboxName, entry, resolvedTargetPins(resolvedByServer, entry)); attachProvider(sandboxName, entry); waitForAttachedMcpCredential(sandboxName, entry); - const adapter = - (entry.adapter as AgentMcpAdapter | undefined) ?? getBridgeAdapter(getSandboxAgent(sandbox)); + const adapter = (entry.adapter as AgentMcpAdapter | undefined) ?? defaultAdapter; registerAgentAdapter( sandboxName, adapter, @@ -221,4 +229,10 @@ export async function restoreExistingMcpBridgeRuntime( ); writeBridgeEntry(sandboxName, { ...entry, adapter, updatedAt: nowIso() }); } + if ( + defaultAdapter === "hermes-config" || + entries.some((entry) => entry.adapter === "hermes-config") + ) { + assertHermesMcpRuntimeIntent(sandboxName, { entries }); + } } diff --git a/src/lib/actions/sandbox/mcp-bridge-state.ts b/src/lib/actions/sandbox/mcp-bridge-state.ts index e1c0963ae98..8a128683376 100644 --- a/src/lib/actions/sandbox/mcp-bridge-state.ts +++ b/src/lib/actions/sandbox/mcp-bridge-state.ts @@ -74,12 +74,19 @@ export function setBridgeState(sandboxName: string, bridges: Record !entry.addState) + .map((entry) => entry.server); + const managedServerNames = [ + ...new Set([...(mcpState?.managedServerNames ?? []), ...committedServerNames]), + ].sort(); const hasDestroyState = !!destroyPreparedAt || !!destroyPendingAt; const updated = registry.updateSandbox(sandboxName, { mcp: - Object.keys(bridges).length > 0 || hasDestroyState + Object.keys(bridges).length > 0 || managedServerNames.length > 0 || hasDestroyState ? { bridges, + ...(managedServerNames.length > 0 ? { managedServerNames } : {}), ...(destroyPreparedAt ? { destroyPreparedAt } : {}), ...(destroyPendingAt ? { destroyPendingAt } : {}), } diff --git a/src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts index 463457f59cb..74ab96abeb7 100644 --- a/src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts @@ -97,9 +97,12 @@ bridge.removeMcpBridge("legacy-sandbox", "github").then( expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const jsonStart = result.stdout.indexOf("{"); const sandbox = JSON.parse(result.stdout.slice(jsonStart)) as { - mcp?: unknown; + mcp?: { bridges?: Record; managedServerNames?: string[] }; }; - expect(sandbox.mcp).toBeUndefined(); + expect(sandbox.mcp).toEqual({ + bridges: {}, + managedServerNames: ["github"], + }); }); it("preserves the registry entry when force cleanup leaves residual policy state", () => { diff --git a/src/lib/actions/sandbox/mcp-bridge-status-state.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-state.test.ts index 46376419e01..59d095da66a 100644 --- a/src/lib/actions/sandbox/mcp-bridge-status-state.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-status-state.test.ts @@ -113,14 +113,75 @@ process.stdout.write(JSON.stringify(markers.map((_, index) => registry.getSandbo }>; expect(sandboxes[0]?.mcp).toEqual({ bridges: {}, + managedServerNames: ["github"], destroyPreparedAt: "2026-06-27T01:00:00.000Z", }); expect(sandboxes[1]?.mcp).toEqual({ bridges: {}, + managedServerNames: ["github"], destroyPendingAt: "2026-06-27T01:00:00.000Z", }); }); + it("reconciles Hermes removal tombstones when no active bridges remain", () => { + const home = createTempHome("nemoclaw-hermes-mcp-tombstone-status-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const payloads = []; +let mismatch = false; +globalActions.runOpenshellProviderCommand = (args) => { + if (args[0] !== "sandbox" || args[1] !== "exec") { + throw new Error("Unexpected OpenShell call: " + args.join(" ")); + } + payloads.push(JSON.parse(args[args.length - 1])); + return mismatch + ? { status: 2, stdout: "", stderr: "managed Hermes MCP entry is still present" } + : { status: 0, stdout: '{"ok":true,"state":"matched"}\\n', stderr: "" }; +}; +registry.registerSandbox({ + name: "hermes-sandbox", + agent: "hermes", + mcp: { bridges: {}, managedServerNames: ["retired"] }, +}); +const status = require("./src/lib/actions/sandbox/mcp-bridge-status.js"); +(async () => { + const matched = await status.statusMcpBridge("hermes-sandbox"); + mismatch = true; + let refusal = ""; + try { + await status.statusMcpBridge("hermes-sandbox"); + } catch (error) { + refusal = error instanceof Error ? error.message : String(error); + } + process.stdout.write(JSON.stringify({ matched, payloads, refusal })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + matched: unknown[]; + payloads: Array<{ present: Record; absent: string[] }>; + refusal: string; + }; + expect(payload.matched).toEqual([]); + expect(payload.payloads).toEqual([ + { present: {}, absent: ["retired"] }, + { present: {}, absent: ["retired"] }, + ]); + expect(payload.refusal).toContain("does not match the persisted managed intent"); + expect(payload.refusal).toContain("managed Hermes MCP entry is still present"); + }); + it("validates requested server names and does not read inherited bridge keys", () => { const home = createTempHome("nemoclaw-mcp-status-key-"); const script = ` diff --git a/src/lib/actions/sandbox/mcp-bridge-status.ts b/src/lib/actions/sandbox/mcp-bridge-status.ts index 57474f1baf1..f16018bef76 100644 --- a/src/lib/actions/sandbox/mcp-bridge-status.ts +++ b/src/lib/actions/sandbox/mcp-bridge-status.ts @@ -8,7 +8,11 @@ import { buildHermesMcpStatusCommand, buildOpenClawMcporterInspectCommand, } from "./mcp-bridge-adapters"; -import { isAgentMcpAdapter, type McpBridgeStatus } from "./mcp-bridge-contracts"; +import { isAgentMcpAdapter, McpBridgeError, type McpBridgeStatus } from "./mcp-bridge-contracts"; +import { + type HermesMcpReconciliationResult, + inspectHermesMcpRuntimeIntent, +} from "./mcp-bridge-hermes-reconciliation"; import { redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; import { getPolicyPresence, getRegisteredGeneratedPolicy } from "./mcp-bridge-policy"; import { @@ -80,9 +84,15 @@ function getAdapterRegistration( sandboxName: string, adapter: AgentMcpAdapter | undefined, entry: McpBridgeEntry | undefined, + hermesReconciliation?: HermesMcpReconciliationResult, ): McpBridgeStatus["adapter"] { if (!entry) return { registered: null }; if (!adapter) return { registered: null, detail: "MCP adapter is not declared" }; + if (adapter === "hermes-config" && hermesReconciliation) { + return hermesReconciliation.ok + ? { registered: true } + : { registered: false, detail: hermesReconciliation.detail }; + } const command = adapter === "mcporter" ? buildOpenClawMcporterInspectCommand(entry, false) @@ -148,6 +158,18 @@ export async function statusMcpBridge( ]; } + const hermesReconciliation = + agent.name === "hermes" && + (entries.length > 0 || (sandbox.mcp?.managedServerNames?.length ?? 0) > 0) && + entries.every(([, entry]) => !entry || storedCredentialWarning(entry) === undefined) + ? inspectHermesMcpRuntimeIntent(sandboxName) + : undefined; + if (entries.length === 0 && hermesReconciliation && !hermesReconciliation.ok) { + throw new McpBridgeError( + `Hermes MCP runtime does not match the persisted managed intent for sandbox '${sandboxName}': ${hermesReconciliation.detail}.`, + ); + } + return entries.map(([name, entry]) => { const support = entry ? getPersistedBridgeSupport(entry) : getSupportSummary(agent); const registeredPolicy = getRegisteredGeneratedPolicy(sandboxName, entry); @@ -220,7 +242,7 @@ export async function statusMcpBridge( detail: "Adapter inspection was skipped because the unsupported legacy credential may still be attached to fresh sandbox children.", } - : getAdapterRegistration(sandboxName, support.adapter, entry), + : getAdapterRegistration(sandboxName, support.adapter, entry, hermesReconciliation), ...(entry?.addedAt ? { addedAt: entry.addedAt } : {}), ...(entry?.updatedAt ? { updatedAt: entry.updatedAt } : {}), }; diff --git a/src/lib/actions/sandbox/mcp-bridge-validation.ts b/src/lib/actions/sandbox/mcp-bridge-validation.ts index 6fff7bceef0..121bd1f9934 100644 --- a/src/lib/actions/sandbox/mcp-bridge-validation.ts +++ b/src/lib/actions/sandbox/mcp-bridge-validation.ts @@ -1,10 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { type SpawnSyncReturns, spawnSync } from "node:child_process"; import crypto from "node:crypto"; +import { resolveOpenshell } from "../../adapters/openshell/resolve"; import type { McpBridgeEntry } from "../../state/registry"; -import { isSubprocessEnvNameAllowed } from "../../subprocess-env"; +import { buildSubprocessEnv, isSubprocessEnvNameAllowed } from "../../subprocess-env"; import { McpBridgeError, type ParsedEnvReference, @@ -27,6 +29,78 @@ export { const VALID_SERVER_RE = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; const VALID_ENV_RE = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; const VALID_SANDBOX_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; +const OPENSHELL_VERSION_OUTPUT_RE = + /^openshell\s+([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/; +const OPENSHELL_VERSION_PROBE_TIMEOUT_MS = 5_000; +const OPENSHELL_VERSION_PROBE_MAX_BUFFER_BYTES = 16 * 1_024; +const EXPECTED_OPENSHELL_VERSION = childVisibleCredentialManifest.openshellVersion; + +type OpenshellVersionCommandResult = Pick< + SpawnSyncReturns, + "error" | "status" | "stderr" | "stdout" +>; + +export interface McpCredentialBoundaryRuntimeDeps { + resolveOpenshell?: () => string | null; + runVersionCommand?: (binary: string) => OpenshellVersionCommandResult; +} + +function runOpenshellVersionCommand(binary: string): OpenshellVersionCommandResult { + return spawnSync(binary, ["--version"], { + encoding: "utf8", + env: buildSubprocessEnv(), + maxBuffer: OPENSHELL_VERSION_PROBE_MAX_BUFFER_BYTES, + stdio: ["ignore", "pipe", "pipe"], + timeout: OPENSHELL_VERSION_PROBE_TIMEOUT_MS, + }); +} + +function credentialBoundaryVersionError(actual: string, detail: string): McpBridgeError { + return new McpBridgeError( + `OpenShell credential boundary runtime version check failed: expected ${EXPECTED_OPENSHELL_VERSION}, actual ${actual} (${detail}). Install OpenShell ${EXPECTED_OPENSHELL_VERSION}, or point NEMOCLAW_OPENSHELL_BIN to that version, then retry.`, + ); +} + +/** + * Bind the static child-visible credential manifest to the host OpenShell CLI + * that will establish a provider credential. Credential-establishing lifecycle + * boundaries call this once immediately before their first side effect; + * deliberately avoiding a cache ensures a long-running CLI process cannot + * retain stale approval after the binary changes. Teardown skips this check so + * a version mismatch cannot strand detach/delete cleanup that only revokes + * credential access. + */ +export function assertMcpCredentialBoundaryRuntimeVersion( + deps: McpCredentialBoundaryRuntimeDeps = {}, +): void { + const binary = (deps.resolveOpenshell ?? resolveOpenshell)(); + if (!binary) { + throw credentialBoundaryVersionError("", "openshell binary not found"); + } + + const result = (deps.runVersionCommand ?? runOpenshellVersionCommand)(binary); + if (result.error) { + const code = (result.error as NodeJS.ErrnoException).code; + const detail = code === "ENOENT" ? "openshell binary not found" : "openshell --version failed"; + throw credentialBoundaryVersionError("", detail); + } + if (result.status !== 0) { + throw credentialBoundaryVersionError( + "", + `openshell --version exited with status ${String(result.status)}`, + ); + } + + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`.trim(); + const actualVersion = output.match(OPENSHELL_VERSION_OUTPUT_RE)?.[1]; + if (!actualVersion) { + throw credentialBoundaryVersionError("", "invalid openshell --version output"); + } + if (actualVersion !== EXPECTED_OPENSHELL_VERSION) { + throw credentialBoundaryVersionError(actualVersion, "version mismatch"); + } +} + // invalidState: an MCP bearer name aliases a child-visible or process-control // key and exposes or executes the provider value outside the intended request. // sourceBoundary: the versioned JSON manifest pins OpenShell-owned keys to the diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index e3876574430..607e071462b 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -42,6 +42,10 @@ import { } from "./gateway-restart"; import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; import { enforceHermesSecretBoundaryOnRunningGateway } from "./hermes-secret-boundary-recovery"; +import { + inspectHermesMcpReconciliationRefusal, + processRecoveryMcpReconciliationRefusal, +} from "./mcp-bridge-recovery"; import { buildSandboxExecMarkedCommand, extractSandboxExecCommandStdout, @@ -547,6 +551,7 @@ export function restartSandboxGateway( recoverMessagingHostForward, recoverDeclaredAgentForwardPorts, printGatewayWedgeDiagnostics, + inspectHermesMcpReconciliationRefusal, ...deps, }, }), @@ -760,6 +765,8 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( secretBoundaryReason: enforcement.reason, }; } + const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, true); + if (mcpRefusal) return mcpRefusal; } if (running) { // Gateway is alive but the host-side forward can still be dead or @@ -911,6 +918,8 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } + const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, false); + if (mcpRefusal) return mcpRefusal; const forwardRecovered = ensureSandboxPortForward(sandboxName); const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet }); diff --git a/src/lib/state/registry-mcp.ts b/src/lib/state/registry-mcp.ts index 2fdf7fc2d02..c0d970ec1f2 100644 --- a/src/lib/state/registry-mcp.ts +++ b/src/lib/state/registry-mcp.ts @@ -27,6 +27,12 @@ export interface McpBridgeEntry { export interface SandboxMcpState { bridges: Record; + /** + * Durable ownership history for adapter reconciliation. Names remain after a + * bridge is removed so a later startup can prove that the retired managed + * entry is absent without claiming unrelated user-managed MCP definitions. + */ + managedServerNames?: string[]; /** Set after in-sandbox adapter scrub/provider detach and before delete. */ destroyPreparedAt?: string; /** @@ -63,6 +69,17 @@ export function normalizeSandboxMcpState(value: unknown): SandboxMcpState | unde const entry = normalizeMcpBridgeEntry(name, rawEntry); if (entry) bridges[entry.server] = entry; } + const persistedManagedServerNames = Array.isArray(value.managedServerNames) + ? value.managedServerNames.filter( + (name): name is string => typeof name === "string" && MCP_SERVER_RE.test(name), + ) + : []; + const committedServerNames = Object.values(bridges) + .filter((entry) => !entry.addState) + .map((entry) => entry.server); + const managedServerNames = [ + ...new Set([...persistedManagedServerNames, ...committedServerNames]), + ].sort(); const destroyPendingAt = typeof value.destroyPendingAt === "string" && value.destroyPendingAt ? value.destroyPendingAt @@ -71,11 +88,17 @@ export function normalizeSandboxMcpState(value: unknown): SandboxMcpState | unde typeof value.destroyPreparedAt === "string" && value.destroyPreparedAt ? value.destroyPreparedAt : undefined; - if (Object.keys(bridges).length === 0 && !destroyPreparedAt && !destroyPendingAt) { + if ( + Object.keys(bridges).length === 0 && + managedServerNames.length === 0 && + !destroyPreparedAt && + !destroyPendingAt + ) { return undefined; } return { bridges, + ...(managedServerNames.length > 0 ? { managedServerNames } : {}), ...(destroyPreparedAt ? { destroyPreparedAt } : {}), ...(destroyPendingAt ? { destroyPendingAt } : {}), }; diff --git a/test/deepagents-mcp-legacy-lifecycle.test.ts b/test/deepagents-mcp-legacy-lifecycle.test.ts index e49c8f7d49e..3a7a8347e76 100644 --- a/test/deepagents-mcp-legacy-lifecycle.test.ts +++ b/test/deepagents-mcp-legacy-lifecycle.test.ts @@ -8,6 +8,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.72"); + function runLegacyLifecycle(body: string) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-deepagents-mcp-legacy-")); const script = String.raw` @@ -158,7 +160,7 @@ ${body} const result = spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, }); fs.rmSync(home, { recursive: true, force: true }); return result; diff --git a/test/e2e/live/mcp-bridge-hermes-lifecycle.ts b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts new file mode 100644 index 00000000000..704d4f26168 --- /dev/null +++ b/test/e2e/live/mcp-bridge-hermes-lifecycle.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { MCP_BRIDGE_TEST_CREDENTIALS } from "../fixtures/mcp-bridge-credentials.ts"; + +const SERVER_NAME = "fake"; +const HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.host; +const ROTATED_HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.rotatedHost; +const INSPECTION_CONTROL_MARKER = "MCP_INSPECT_FORGED_CONTROL_LINE"; +const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); + +function resultText(result: { stdout: string; stderr: string }): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function expectExitZero( + result: { exitCode: number | null; stdout: string; stderr: string }, + label: string, +): void { + expect(result.exitCode, `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); +} + +export async function assertHermesConfig( + sandbox: SandboxClient, + sandboxName: string, + mcpUrl: string, +): Promise { + const script = [ + "set -eu", + "/opt/hermes/.venv/bin/python - <<'PY'", + "import pathlib, yaml", + "path = pathlib.Path('/sandbox/.hermes/config.yaml')", + "text = path.read_text(encoding='utf-8')", + "data = yaml.safe_load(text) or {}", + `entry = data['mcp_servers'][${JSON.stringify(SERVER_NAME)}]`, + `assert entry['url'] == ${JSON.stringify(mcpUrl)}`, + "assert entry['headers']['Authorization'] == 'Bearer openshell:resolve:env:FAKE_MCP_SECRET'", + `assert ${JSON.stringify(HOST_SECRET)} not in text`, + "PY", + ].join("\n"); + const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { + artifactName: "hermes-mcp-config-assertions", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, Buffer.from(script, "utf8").toString("base64")], + timeoutMs: 60_000, + }); + expectExitZero(result, "Hermes MCP config contains placeholder and no raw host secret"); +} + +// No host `nemoclaw mcp inspect` command exists; exercise the packaged CLI +// through the same OpenShell sandbox boundary used by live MCP reconciliation. +export async function assertHermesInspectionRejectsUnmanagedFields( + sandbox: SandboxClient, + sandboxName: string, +): Promise { + const payload = Buffer.from( + JSON.stringify({ + present: { + [SERVER_NAME]: { + command: [HOST_SECRET, `\u001b[31m${INSPECTION_CONTROL_MARKER}\u001b[0m`], + transport: `stdio\r\n${ROTATED_HOST_SECRET}`, + }, + }, + absent: [], + }), + "utf8", + ).toString("base64"); + const script = [ + "set -eu", + `payload="$(printf '%s' '${payload}' | base64 -d)"`, + '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py inspect --payload "$payload"', + ].join("\n"); + const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { + artifactName: "hermes-mcp-inspect-rejects-unmanaged-fields", + env: buildAvailabilityProbeEnv(), + redactionValues: [ + HOST_SECRET, + ROTATED_HOST_SECRET, + payload, + Buffer.from(script, "utf8").toString("base64"), + ], + timeoutMs: 60_000, + }); + const output = resultText(result); + expect(result.exitCode, `malformed Hermes MCP inspection must fail\n${output}`).not.toBe(0); + expect(output).toContain("Hermes MCP inspection expected config has invalid fields"); + expect(output).not.toContain(HOST_SECRET); + expect(output).not.toContain(ROTATED_HOST_SECRET); + expect(output).not.toContain(INSPECTION_CONTROL_MARKER); + expect(output).not.toContain("\u001b"); + expect(output).not.toContain("\r"); +} + +/** + * Prove the removal tombstone survives an actual supervisor-mediated Hermes + * gateway restart. A successful post-restart `mcp list` runs the in-sandbox + * integrity inspector against the empty registry projection, so it covers the + * current intended/applied digest and the absence of the retired server in the + * config used by the newly healthy gateway. + */ +export async function assertHermesRemovalSurvivesGatewayRestart( + host: HostCliClient, + sandbox: SandboxClient, + sandboxName: string, +): Promise { + expect(fs.existsSync(REGISTRY_FILE), `registry file not found: ${REGISTRY_FILE}`).toBe(true); + const registryRaw = fs.readFileSync(REGISTRY_FILE, "utf8"); + expect(registryRaw).not.toContain(HOST_SECRET); + expect(registryRaw).not.toContain(ROTATED_HOST_SECRET); + const registry = JSON.parse(registryRaw) as { + sandboxes?: Record< + string, + { mcp?: { bridges?: Record; managedServerNames?: string[] } } + >; + }; + const mcpState = registry.sandboxes?.[sandboxName]?.mcp; + expect(mcpState?.bridges, "removed Hermes bridge must leave no active registry intent").toEqual( + {}, + ); + expect( + mcpState?.managedServerNames, + "removed Hermes bridge must retain its managed-name tombstone", + ).toContain(SERVER_NAME); + + const restart = await host.nemoclaw([sandboxName, "gateway", "restart"], { + artifactName: "hermes-mcp-removal-gateway-restart", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 12 * 60_000, + }); + expectExitZero(restart, "Hermes gateway restart after managed MCP removal"); + expect(resultText(restart)).toContain("Gateway restarted"); + expect(resultText(restart)).toContain("health passed"); + expect(resultText(restart)).not.toContain(HOST_SECRET); + expect(resultText(restart)).not.toContain(ROTATED_HOST_SECRET); + + const list = await host.nemoclaw([sandboxName, "mcp", "list", "--json"], { + artifactName: "hermes-mcp-list-after-removal-gateway-restart", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 60_000, + }); + expectExitZero(list, "Hermes MCP list after removal gateway restart"); + expect(JSON.parse(list.stdout).bridges).toEqual([]); + expect(resultText(list)).not.toContain(HOST_SECRET); + expect(resultText(list)).not.toContain(ROTATED_HOST_SECRET); + + const config = await sandbox.execShell( + sandboxName, + trustedSandboxShellScript( + [ + "set -eu", + '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py inspect --payload \'{"present":{},"absent":["fake"]}\'', + ].join("\n"), + ), + { + artifactName: "hermes-mcp-effective-config-after-removal-gateway-restart", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 60_000, + }, + ); + expectExitZero(config, "Hermes effective MCP config after removal gateway restart"); + expect(config.stdout).toContain('"state": "matched"'); + expect(resultText(config)).not.toContain(HOST_SECRET); + expect(resultText(config)).not.toContain(ROTATED_HOST_SECRET); +} diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 1d626a1dd60..7c2c32ae497 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -22,6 +22,11 @@ import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clien import { expect, test } from "../fixtures/e2e-test.ts"; import { MCP_BRIDGE_TEST_CREDENTIALS } from "../fixtures/mcp-bridge-credentials.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + assertHermesConfig, + assertHermesInspectionRejectsUnmanagedFields, + assertHermesRemovalSurvivesGatewayRestart, +} from "./mcp-bridge-hermes-lifecycle.ts"; import { buildMcpDnsRebindingProbeScript, hostAddressForSandbox, @@ -667,32 +672,6 @@ async function assertAdapterRequestDeniedAfterRemove( ).toBe(true); expect(fakeMcp.requests).toHaveLength(requestCount); } -async function assertHermesConfig( - sandbox: SandboxClient, - sandboxName: string, - mcpUrl: string, -): Promise { - const script = [ - "set -eu", - "/opt/hermes/.venv/bin/python - <<'PY'", - "import pathlib, yaml", - "path = pathlib.Path('/sandbox/.hermes/config.yaml')", - "text = path.read_text(encoding='utf-8')", - "data = yaml.safe_load(text) or {}", - `entry = data['mcp_servers'][${JSON.stringify(SERVER_NAME)}]`, - `assert entry['url'] == ${JSON.stringify(mcpUrl)}`, - "assert entry['headers']['Authorization'] == 'Bearer openshell:resolve:env:FAKE_MCP_SECRET'", - `assert ${JSON.stringify(HOST_SECRET)} not in text`, - "PY", - ].join("\n"); - const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { - artifactName: "hermes-mcp-config-assertions", - env: buildAvailabilityProbeEnv(), - redactionValues: [HOST_SECRET, Buffer.from(script, "utf8").toString("base64")], - timeoutMs: 60_000, - }); - expectExitZero(result, "Hermes MCP config contains placeholder and no raw host secret"); -} async function assertDeepAgentsConfig( sandbox: SandboxClient, sandboxName: string, @@ -1290,6 +1269,7 @@ liveAgentMatrixTest( mcpUrl, }); await assertHermesConfig(sandbox, HERMES_SANDBOX_NAME, mcpUrl); + await assertHermesInspectionRejectsUnmanagedFields(sandbox, HERMES_SANDBOX_NAME); await assertSecretAbsentFromSandbox(sandbox, HERMES_SANDBOX_NAME, ["/sandbox/.hermes"]); await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { adapter: "hermes-config", @@ -1363,6 +1343,20 @@ liveAgentMatrixTest( mcpUrl, artifactPrefix: "hermes", }); + await assertHermesRemovalSurvivesGatewayRestart(host, sandbox, HERMES_SANDBOX_NAME); + await assertAdapterRequestDeniedAfterRemove(sandbox, fakeMcp, { + adapter: "hermes-config", + sandboxName: HERMES_SANDBOX_NAME, + mcpUrl, + artifactPrefix: "hermes-after-removal-gateway-restart", + }); + await assertSecretAbsentFromSandbox( + sandbox, + HERMES_SANDBOX_NAME, + ["/sandbox/.hermes", "/tmp/nemoclaw-start.log"], + [HOST_SECRET, ROTATED_HOST_SECRET], + "hermes-assert-secrets-absent-after-removal-gateway-restart", + ); }, ); diff --git a/test/fixtures/openshell-v0.0.72 b/test/fixtures/openshell-v0.0.72 new file mode 100755 index 00000000000..1e7f79b48e6 --- /dev/null +++ b/test/fixtures/openshell-v0.0.72 @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +if [ "$#" -ne 1 ] || [ "$1" != "--version" ]; then + exit 64 +fi +printf '%s\n' 'openshell 0.0.72' diff --git a/test/gateway-supervisor-mcp-failure-contract.test.ts b/test/gateway-supervisor-mcp-failure-contract.test.ts new file mode 100644 index 00000000000..d09f0759127 --- /dev/null +++ b/test/gateway-supervisor-mcp-failure-contract.test.ts @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const CONTROL_HELPER = path.join(REPO_ROOT, "scripts", "gateway-control.sh"); +const NONCE = "a".repeat(64); + +function withTmpDir(run: (tmpDir: string) => void): void { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-supervisor-contract-")); + try { + run(tmpDir); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} + +describe("gateway supervisor MCP failure contract (#6257)", () => { + it.each([ + "mcp-integrity", + "mcp-reconcile-required", + ])("preserves the %s failure code in supervisor status", (failureCode) => { + withTmpDir((tmpDir) => { + const controlDir = path.join(tmpDir, "control"); + fs.mkdirSync(controlDir, { mode: 0o700 }); + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -eu", + 'export NEMOCLAW_GATEWAY_CONTROL_DIR="$1"', + ". scripts/lib/gateway-supervisor.sh", + 'GATEWAY_CONTROL_NONCE="$2"', + 'gateway_control_fail "$3" 4242', + 'cat "$NEMOCLAW_GATEWAY_CONTROL_STATUS"', + ].join("\n"), + "gateway-supervisor-mcp-failure-contract", + controlDir, + NONCE, + failureCode, + ], + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim()).toBe(`v1 ${NONCE} failed ${failureCode} 4242 0`); + }); + }); + + it.each([ + "mcp-integrity", + "mcp-reconcile-required", + ])("maps %s to the stable host-visible MCP drift marker", (failureCode) => { + withTmpDir((tmpDir) => { + const controlDir = path.join(tmpDir, "control"); + const procRoot = path.join(tmpDir, "proc"); + fs.mkdirSync(controlDir, { mode: 0o700 }); + fs.mkdirSync(path.join(procRoot, "1"), { recursive: true }); + fs.writeFileSync(path.join(procRoot, "1", "cmdline"), "bash\0nemoclaw-start\0"); + const wrapper = path.join(tmpDir, "run-control.sh"); + fs.writeFileSync( + wrapper, + [ + "#!/usr/bin/env bash", + "set -eu", + 'stat() { printf "%s\\n" "root:root 700"; }', + 'FAILURE_CODE="${NEMOCLAW_TEST_FAILURE_CODE:?}"', + 'TEST_NONCE="${NEMOCLAW_TEST_NONCE:?}"', + 'kill() { printf "v1 %s failed %s 4242 0\\n" "$TEST_NONCE" "$FAILURE_CODE" >"$NEMOCLAW_GATEWAY_CONTROL_DIR/status"; }', + 'set -- restart "$TEST_NONCE"', + '. "${NEMOCLAW_TEST_CONTROL_HELPER:?}"', + ].join("\n"), + { mode: 0o700 }, + ); + + const result = spawnSync("bash", [wrapper], { + encoding: "utf8", + env: { + ...process.env, + NEMOCLAW_GATEWAY_CONTROL_DIR: controlDir, + NEMOCLAW_TEST_CONTROL_HELPER: CONTROL_HELPER, + NEMOCLAW_TEST_FAILURE_CODE: failureCode, + NEMOCLAW_TEST_GATEWAY_CONTROL_CALLER_UID: "0", + NEMOCLAW_TEST_GATEWAY_CONTROL_PROC_ROOT: procRoot, + NEMOCLAW_TEST_NONCE: NONCE, + }, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(`failed ${failureCode} 4242 0`); + expect(result.stderr).toContain("HERMES_MCP_CONFIG_DRIFT"); + expect(result.stderr).not.toContain("GATEWAY_FAILED"); + }); + }); +}); diff --git a/test/hermes-doctor-config-hash.test.ts b/test/hermes-doctor-config-hash.test.ts index f766fb032b6..b244bf624e7 100644 --- a/test/hermes-doctor-config-hash.test.ts +++ b/test/hermes-doctor-config-hash.test.ts @@ -10,6 +10,8 @@ import { dockerRunCommandBetween, runDockerShell } from "./helpers/hermes-docker const ROOT = path.resolve(import.meta.dirname, ".."); const HERMES_DOCKERFILE = path.join(ROOT, "agents", "hermes", "Dockerfile"); +const HERMES_BUILD_MCP_DIGEST = path.join(ROOT, "agents", "hermes", "build-mcp-digest.py"); +const HERMES_RUNTIME_CONFIG_GUARD = path.join(ROOT, "agents", "hermes", "runtime-config-guard.py"); describe("Hermes doctor and config hash boundary", () => { it("locks trusted gateway recovery preloads as image-owned read-only files", () => { @@ -18,6 +20,7 @@ describe("Hermes doctor and config hash boundary", () => { const binDir = path.join(tmp, "usr-local-bin"); const libDir = path.join(tmp, "usr-local-lib-nemoclaw"); const preloadsDir = path.join(libDir, "preloads"); + const buildMcpDigestPath = path.join(libDir, "build-hermes-mcp-digest.py"); const mcpConfigTransactionPath = path.join(libDir, "hermes-mcp-config-transaction.py"); const mcpCredentialBoundaryPath = path.join( libDir, @@ -41,6 +44,7 @@ describe("Hermes doctor and config hash boundary", () => { path.join(libDir, "validate-hermes-env-secret-boundary.py"), path.join(libDir, "seed-hermes-dashboard-config.py"), path.join(libDir, "hermes-runtime-config-guard.py"), + buildMcpDigestPath, mcpConfigTransactionPath, mcpCredentialBoundaryPath, path.join(libDir, "state-dir-guard.py"), @@ -76,7 +80,7 @@ describe("Hermes doctor and config hash boundary", () => { expect(result.stderr).toBe(""); expect(fs.readFileSync(chownLogPath, "utf-8")).toBe( [ - `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${path.join(libDir, "managed-gateway-control.py")} ${mcpCredentialBoundaryPath}`, + `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${path.join(libDir, "managed-gateway-control.py")} ${buildMcpDigestPath} ${mcpCredentialBoundaryPath}`, `-R 0:0 ${preloadsDir}`, "", ].join("\n"), @@ -84,6 +88,7 @@ describe("Hermes doctor and config hash boundary", () => { expect(mode(path.join(binDir, "nemoclaw-gateway-control"))).toBe("700"); expect(mode(mcpConfigTransactionPath)).toBe("755"); expect(mode(mcpCredentialBoundaryPath)).toBe("444"); + expect(mode(buildMcpDigestPath)).toBe("444"); expect(mode(path.join(libDir, "gateway-supervisor.sh"))).toBe("444"); expect(mode(path.join(libDir, "state-dir-guard.py"))).toBe("500"); expect(mode(path.join(libDir, "managed-gateway-control.py"))).toBe("500"); @@ -149,12 +154,22 @@ describe("Hermes doctor and config hash boundary", () => { dockerfile, "# Pin config hash at build time", "# Backward-compatible marker", - ).replaceAll("/etc/nemoclaw", etcDir); + ) + .replaceAll("/etc/nemoclaw", etcDir) + .replaceAll("/opt/hermes/.venv/bin/python", "python3") + .replaceAll( + "/usr/local/lib/nemoclaw/build-hermes-mcp-digest.py", + JSON.stringify(HERMES_BUILD_MCP_DIGEST), + ) + .replaceAll( + "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", + JSON.stringify(HERMES_RUNTIME_CONFIG_GUARD), + ); const compatHashCommand = dockerRunCommandBetween( dockerfile, "# Backward-compatible marker", "# OpenShell's macOS VM backend", - ); + ).replaceAll("/etc/nemoclaw", etcDir); try { const doctorAndGenerate = spawnSync("bash", ["-c", doctorAndGenerateCommand], { diff --git a/test/hermes-gateway-auxiliary-retry.test.ts b/test/hermes-gateway-auxiliary-retry.test.ts new file mode 100644 index 00000000000..22132b835c9 --- /dev/null +++ b/test/hermes-gateway-auxiliary-retry.test.ts @@ -0,0 +1,262 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + extractShellFunction, + runHermesBashHarness as runBashHarness, +} from "./support/hermes-shell-harness"; + +const START_SCRIPT = path.join(import.meta.dirname, "..", "agents", "hermes", "start.sh"); + +function writeFakeProcCmdline(procRoot: string, pid: number, args: string[]): void { + const processDir = path.join(procRoot, String(pid)); + fs.mkdirSync(processDir, { recursive: true }); + fs.writeFileSync(path.join(processDir, "cmdline"), Buffer.from(`${args.join("\0")}\0`)); +} + +describe("Hermes gateway auxiliary retry", () => { + it("retries transient auxiliary failures without churning the healthy gateway", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + "prepare_hermes_nonroot_runtime() { return 0; }", + 'launch_hermes_gateway_current_user() { launch_calls=$((launch_calls + 1)); GATEWAY_PID=6001; trace "launch:$GATEWAY_PID"; }', + 'wait_for_hermes_gateway_internal() { trace "internal:$1"; return 0; }', + 'hermes_tracked_role_is_current() { trace "identity:$2"; return 0; }', + 'hermes_gateway_healthy() { trace "health:$1"; return 0; }', + 'ensure_hermes_supervised_auxiliaries() { auxiliary_calls=$((auxiliary_calls + 1)); trace "auxiliary:$auxiliary_calls"; [ "$auxiliary_calls" -ge 3 ]; }', + "commit_hermes_mcp_applied_if_pending() { trace commit-applied; return 0; }", + "refresh_hermes_supervised_child_pids() { trace refresh; }", + 'hermes_stop_tracked_role() { trace "unexpected-stop:$2"; return 1; }', + "mark_hermes_gateway_stopped() { trace unexpected-mark; }", + "record_hermes_managed_gateway_exit() { trace unexpected-exit-record; }", + 'sleep() { trace "sleep:$1"; }', + extractShellFunction(source, "recover_hermes_gateway_current_user"), + "INTERNAL_PORT=18642", + "launch_calls=0", + "auxiliary_calls=0", + "recover_hermes_gateway_current_user", + 'trace "launch-count:$launch_calls"', + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "launch:6001", + "internal:6001", + "identity:6001", + "health:6001", + "auxiliary:1", + "sleep:1", + "identity:6001", + "health:6001", + "auxiliary:2", + "sleep:1", + "identity:6001", + "health:6001", + "auxiliary:3", + "identity:6001", + "health:6001", + "commit-applied", + "refresh", + "launch-count:1", + ]); + expect(result.stderr.match(/auxiliary repair failed/g)).toHaveLength(2); + expect(result.stdout).not.toContain("unexpected-"); + }); + + it("stops and charges a replacement that loses health during auxiliary retry", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + "prepare_hermes_nonroot_runtime() { return 0; }", + 'launch_hermes_gateway_current_user() { GATEWAY_PID=6001; trace "launch:$GATEWAY_PID"; }', + 'wait_for_hermes_gateway_internal() { trace "internal:$1"; return 0; }', + 'hermes_tracked_role_is_current() { trace "identity:$2"; return 0; }', + 'hermes_gateway_healthy() { health_calls=$((health_calls + 1)); trace "health:$health_calls"; [ "$health_calls" -eq 1 ]; }', + "ensure_hermes_supervised_auxiliaries() { trace auxiliary-failed; return 1; }", + 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', + "mark_hermes_gateway_stopped() { trace mark-stopped; GATEWAY_PID=0; }", + "record_hermes_managed_gateway_exit() { trace exit-record; return 1; }", + 'sleep() { trace "sleep:$1"; }', + extractShellFunction(source, "recover_hermes_gateway_current_user"), + "INTERNAL_PORT=18642", + "health_calls=0", + 'if recover_hermes_gateway_current_user; then trace unexpected-success; else trace "failure:$?"; fi', + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "launch:6001", + "internal:6001", + "identity:6001", + "health:1", + "auxiliary-failed", + "sleep:1", + "identity:6001", + "health:2", + "stop:6001", + "mark-stopped", + "exit-record", + "failure:1", + ]); + expect(result.stdout).not.toContain("unexpected-success"); + }); +}); + +describe("Hermes gateway relay convergence", () => { + it("preserves exact tracked relays while removing matching orphan processes", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness( + [ + 'trace() { printf "%s\\n" "$*"; }', + 'kill() { trace "kill:$1"; }', + 'hermes_tracked_role_is_current() { case "$1:$2" in api-socat:101|dashboard-socat:303) trace "preserve:$1:$2"; return 0 ;; *) return 1 ;; esac; }', + extractShellFunction(source, "cleanup_orphan_socat_forwarders"), + 'NEMOCLAW_PROC_ROOT="$TEST_PROC_ROOT"', + "PUBLIC_PORT=8642", + "INTERNAL_PORT=18642", + "DASHBOARD_PUBLIC_PORT=18789", + "DASHBOARD_INTERNAL_PORT=19119", + "SOCAT_PID=101", + "DASHBOARD_SOCAT_PID=303", + "cleanup_orphan_socat_forwarders", + ], + (tmpDir) => { + const procRoot = path.join(tmpDir, "proc"); + const apiArgs = [ + "socat", + "TCP-LISTEN:8642,bind=0.0.0.0,fork,reuseaddr", + "TCP:127.0.0.1:18642", + ]; + const dashboardArgs = [ + "socat", + "TCP-LISTEN:18789,bind=0.0.0.0,fork,reuseaddr", + "TCP:127.0.0.1:19119", + ]; + writeFakeProcCmdline(procRoot, 101, apiArgs); + writeFakeProcCmdline(procRoot, 202, apiArgs); + writeFakeProcCmdline(procRoot, 303, dashboardArgs); + writeFakeProcCmdline(procRoot, 404, dashboardArgs); + return { TEST_PROC_ROOT: procRoot }; + }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("preserve:api-socat:101"); + expect(result.stdout).toContain("preserve:dashboard-socat:303"); + expect(result.stdout).toContain("kill:202"); + expect(result.stdout).toContain("kill:404"); + expect(result.stdout).not.toContain("kill:101"); + expect(result.stdout).not.toContain("kill:303"); + }); + + it("removes a recorded relay when its exact tracked identity is not proven", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness( + [ + 'trace() { printf "%s\\n" "$*"; }', + 'kill() { trace "kill:$1"; }', + "hermes_tracked_role_is_current() { return 1; }", + extractShellFunction(source, "cleanup_orphan_socat_forwarders"), + 'NEMOCLAW_PROC_ROOT="$TEST_PROC_ROOT"', + "PUBLIC_PORT=8642", + "INTERNAL_PORT=18642", + "DASHBOARD_PUBLIC_PORT=18789", + "DASHBOARD_INTERNAL_PORT=19119", + "SOCAT_PID=101", + 'DASHBOARD_SOCAT_PID=""', + "cleanup_orphan_socat_forwarders", + ], + (tmpDir) => { + const procRoot = path.join(tmpDir, "proc"); + writeFakeProcCmdline(procRoot, 101, [ + "socat", + "TCP-LISTEN:8642,bind=0.0.0.0,fork,reuseaddr", + "TCP:127.0.0.1:18642", + ]); + return { TEST_PROC_ROOT: procRoot }; + }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("kill:101\n"); + }); + + it("retries transient public health without churning an exact listener", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness( + [ + 'trace() { printf "%s\\n" "$*"; }', + "exec 3>&1", + 'id() { [ "${1:-}" = "-u" ] && printf "1000\\n"; }', + "hermes_socat_bridge_healthy() { return 0; }", + 'curl() { count="$(cat "$TEST_PROBE_FILE")"; count=$((count + 1)); printf "%s\\n" "$count" >"$TEST_PROBE_FILE"; printf "public-probe:%s\\n" "$count" >&3; if [ "$count" -lt 3 ]; then printf "503"; else printf "200"; fi; }', + 'hermes_stop_tracked_role() { trace "unexpected-stop:$2"; return 1; }', + 'start_socat_forwarder() { trace "unexpected-start:$*"; return 1; }', + "hermes_dashboard_healthy() { return 0; }", + "ensure_gateway_log_stream() { trace gateway-log; }", + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), + extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), + "PUBLIC_PORT=8642", + "INTERNAL_PORT=18642", + "DASHBOARD_PUBLIC_PORT=18789", + "DASHBOARD_INTERNAL_PORT=19119", + "SOCAT_PID=101", + "DASHBOARD_PID=202", + "DASHBOARD_SOCAT_PID=303", + "GATEWAY_PID=4242", + 'for attempt in 1 2 3; do if ensure_hermes_supervised_auxiliaries; then trace "result:$attempt:ready"; else trace "result:$attempt:waiting"; fi; done', + ], + (tmpDir) => { + const probeFile = path.join(tmpDir, "probe-count"); + fs.writeFileSync(probeFile, "0\n"); + return { TEST_PROBE_FILE: probeFile }; + }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("result:1:waiting"); + expect(result.stdout).toContain("result:2:waiting"); + expect(result.stdout).toContain("result:3:ready"); + expect(result.stdout).not.toContain("unexpected-"); + expect(result.stdout.match(/public-probe:/g)).toHaveLength(3); + }); + + it("replaces structural listener loss once and preserves a public-red replacement", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + 'id() { [ "${1:-}" = "-u" ] && printf "1000\\n"; }', + 'hermes_socat_bridge_healthy() { [ "$1:$2" != "api-socat:101" ]; }', + 'curl() { printf "503"; }', + 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', + 'start_socat_forwarder() { trace "start:$*"; printf -v "$4" 111; return 0; }', + "hermes_dashboard_healthy() { trace unexpected-dashboard; return 0; }", + "ensure_gateway_log_stream() { trace unexpected-log; }", + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), + extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), + "PUBLIC_PORT=8642", + "INTERNAL_PORT=18642", + "DASHBOARD_PUBLIC_PORT=18789", + "DASHBOARD_INTERNAL_PORT=19119", + "SOCAT_PID=101", + "DASHBOARD_PID=202", + "DASHBOARD_SOCAT_PID=303", + "GATEWAY_PID=4242", + 'for attempt in 1 2; do if ensure_hermes_supervised_auxiliaries; then trace "unexpected-ready:$attempt"; else trace "waiting:$attempt"; fi; done', + 'trace "final-api-bridge:$SOCAT_PID"', + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.match(/^stop:/gm)).toHaveLength(1); + expect(result.stdout.match(/^start:/gm)).toHaveLength(1); + expect(result.stdout).toContain("waiting:1"); + expect(result.stdout).toContain("waiting:2"); + expect(result.stdout).toContain("final-api-bridge:111"); + expect(result.stdout).not.toContain("unexpected-"); + }); +}); diff --git a/test/hermes-gateway-supervisor-recovery.test.ts b/test/hermes-gateway-supervisor-recovery.test.ts index 4ab0be5fec5..05da8ca890a 100644 --- a/test/hermes-gateway-supervisor-recovery.test.ts +++ b/test/hermes-gateway-supervisor-recovery.test.ts @@ -1,12 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { + extractShellFunction, + runHermesBashHarness as runBashHarness, +} from "./support/hermes-shell-harness"; + const START_SCRIPT = path.join(import.meta.dirname, "..", "agents", "hermes", "start.sh"); const SUPERVISOR_LIB = path.join( import.meta.dirname, @@ -16,38 +19,6 @@ const SUPERVISOR_LIB = path.join( "gateway-supervisor.sh", ); -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -function extractShellFunction(source: string, name: string): string { - const match = source.match(new RegExp(`${escapeRegExp(name)}\\(\\) \\{([\\s\\S]*?)^\\}`, "m")); - const resolved = - match ?? - (() => { - throw new Error(`Expected ${name} in agents/hermes/start.sh`); - })(); - return `${name}() {${resolved[1]}\n}`; -} - -function runBashHarness(lines: string[], configure?: (tmpDir: string) => Record) { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-supervisor-test-")); - const script = path.join(tmpDir, "run.sh"); - fs.writeFileSync(script, ["#!/usr/bin/env bash", "set -uo pipefail", ...lines].join("\n"), { - mode: 0o700, - }); - - try { - return spawnSync("bash", [script], { - encoding: "utf-8", - timeout: 5000, - env: { ...process.env, ...configure?.(tmpDir) }, - }); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } -} - function runHermesHealthyGatewayRecovery(integrityStatus: 0 | 1) { const source = fs.readFileSync(START_SCRIPT, "utf-8"); return runBashHarness([ @@ -266,6 +237,55 @@ describe("Hermes PID 1 supervisor recovery", () => { expect(result.stdout).not.toContain("unexpected-"); }); + it("stops a healthy replacement gateway when the pending MCP applied-state commit fails", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + "gateway_control_take_request() { GATEWAY_CONTROL_ACTION=restart; trace take-request; }", + 'prepare_hermes_gateway_restart() { prepare_calls=$((prepare_calls + 1)); trace "prepare:$prepare_calls"; return 0; }', + "seal_hermes_restart_inputs() { trace seal-inputs; return 0; }", + 'hermes_stop_tracked_role() { trace "stop-old:$2"; return 0; }', + "mark_hermes_gateway_stopped() { trace mark-stopped; GATEWAY_PID=0; }", + "cleanup_sealed_hermes_gateway_runtime() { trace cleanup-runtime; return 0; }", + 'launch_hermes_gateway() { GATEWAY_PID=5252; trace "launch:$GATEWAY_PID"; return 0; }', + 'wait_for_hermes_gateway_internal() { trace "health:$1"; return 0; }', + "ensure_hermes_supervised_auxiliaries() { trace auxiliaries; return 0; }", + "unseal_hermes_restart_inputs() { trace unseal-inputs; return 0; }", + "commit_hermes_mcp_applied_if_pending() { trace commit-applied; return 1; }", + 'stop_hermes_gateway_fail_closed() { trace "stop-fail-closed:$GATEWAY_PID"; GATEWAY_PID=0; }', + 'gateway_control_fail() { trace "fail:$1:$2"; }', + 'gateway_control_complete() { trace "unexpected-complete:$1:$2:$3"; }', + "refresh_hermes_supervised_child_pids() { trace unexpected-refresh; }", + extractShellFunction(source, "handle_hermes_gateway_control_request"), + "INTERNAL_PORT=18642", + "GATEWAY_PID=4242", + "HERMES_RESTART_FAILURE_CODE=internal", + "prepare_calls=0", + 'if handle_hermes_gateway_control_request; then trace "handler-rc:0"; else trace "handler-rc:$?"; fi', + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "take-request", + "prepare:1", + "seal-inputs", + "prepare:2", + "stop-old:4242", + "mark-stopped", + "cleanup-runtime", + "launch:5252", + "health:5252", + "auxiliaries", + "unseal-inputs", + "commit-applied", + "stop-fail-closed:5252", + "fail:mcp-integrity:4242", + "handler-rc:1", + ]); + expect(result.stdout).not.toContain("unexpected-complete"); + expect(result.stdout).not.toContain("unexpected-refresh"); + }); + it("routes a secret-boundary refusal through whole-container gateway revocation", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ @@ -275,6 +295,7 @@ describe("Hermes PID 1 supervisor recovery", () => { "stop_hermes_gateway_fail_closed() { trace fail-closed-stop; }", 'gateway_control_fail() { trace "fail:$1:$2"; }', "mark_hermes_gateway_stopped() { trace unexpected-direct-mark; }", + extractShellFunction(source, "hermes_restart_failure_revokes_gateway"), extractShellFunction(source, "handle_hermes_gateway_control_request"), "GATEWAY_PID=4242", "HERMES_RESTART_FAILURE_CODE=internal", @@ -550,7 +571,7 @@ describe("Hermes supervised auxiliary recovery", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ 'trace() { printf "%s\\n" "$*"; }', - 'hermes_tracked_role_is_current() { [ "$2" = "6262" ] && { trace "supervised:$2"; exit 0; }; return 1; }', + 'hermes_tracked_role_is_current() { case "$2" in 5252) tracked_5252=$((tracked_5252 + 1)); [ "$tracked_5252" -le 2 ] ;; 6262) tracked_6262=$((tracked_6262 + 1)); [ "$tracked_6262" -le 2 ] || { trace "supervised:$2"; exit 0; } ;; *) return 1 ;; esac; }', 'wait() { trace "wait:$1"; return 143; }', "mark_hermes_gateway_stopped() { trace mark-stopped; GATEWAY_PID=0; }", "hermes_managed_gateway_exit_was_host_authorized() { return 1; }", @@ -562,6 +583,7 @@ describe("Hermes supervised auxiliary recovery", () => { 'launch_hermes_gateway_current_user() { launch_calls=$((launch_calls + 1)); [ "$launch_calls" -eq 1 ] && GATEWAY_PID=5252 || GATEWAY_PID=6262; trace "launch:$GATEWAY_PID"; }', 'wait_for_hermes_gateway_internal() { trace "health:$1"; }', "ensure_hermes_supervised_auxiliaries() { trace auxiliaries; }", + "commit_hermes_mcp_applied_if_pending() { return 0; }", 'refresh_hermes_supervised_child_pids() { trace "refresh:$GATEWAY_PID"; }', "hermes_gateway_healthy() { return 0; }", 'hermes_stop_tracked_role() { trace "unexpected-stop:$2"; return 1; }', @@ -572,6 +594,8 @@ describe("Hermes supervised auxiliary recovery", () => { "INTERNAL_PORT=18642", "HERMES_MANAGED_GATEWAY_EXIT_TIMES=()", "HERMES_MANAGED_GATEWAY_EXIT_COUNT=0", + "tracked_5252=0", + "tracked_6262=0", "GATEWAY_PID=4242", "supervise_hermes_gateway_current_user", ]); @@ -628,17 +652,14 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.stderr).toContain("relaunch is quarantined until sandbox recreation"); }); - it.each([ - ["health validation", 1, 0], - ["auxiliary validation", 0, 1], - ])("counts repeated %s failures and never launches a sixth candidate", (_label, health, auxiliaries) => { + it("counts repeated gateway health failures and never launches a sixth candidate", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ 'trace() { printf "%s\\n" "$*"; }', "prepare_hermes_nonroot_runtime() { return 0; }", 'launch_hermes_gateway_current_user() { launch_calls=$((launch_calls + 1)); GATEWAY_PID=$((6000 + launch_calls)); trace "launch:$GATEWAY_PID"; }', - `wait_for_hermes_gateway_internal() { return ${health}; }`, - `ensure_hermes_supervised_auxiliaries() { return ${auxiliaries}; }`, + "wait_for_hermes_gateway_internal() { return 1; }", + "ensure_hermes_supervised_auxiliaries() { trace unexpected-auxiliary; return 0; }", 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', "mark_hermes_gateway_stopped() { GATEWAY_PID=0; }", "refresh_hermes_supervised_child_pids() { :; }", @@ -661,6 +682,7 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.stdout.match(/^stop:/gm)).toHaveLength(5); expect(result.stdout).toContain("quarantine"); expect(result.stderr).toContain("5 exits in 60s window"); + expect(result.stdout).not.toContain("unexpected-auxiliary"); }); it("does not count preparation refusals or launch before preparation succeeds", () => { @@ -670,7 +692,10 @@ describe("Hermes supervised auxiliary recovery", () => { 'prepare_hermes_nonroot_runtime() { prepare_calls=$((prepare_calls + 1)); trace "prepare:$prepare_calls"; [ "$prepare_calls" -ge 3 ]; }', 'launch_hermes_gateway_current_user() { launch_calls=$((launch_calls + 1)); GATEWAY_PID=7001; trace "launch:$GATEWAY_PID"; }', "wait_for_hermes_gateway_internal() { return 0; }", + "hermes_tracked_role_is_current() { return 0; }", + "hermes_gateway_healthy() { return 0; }", "ensure_hermes_supervised_auxiliaries() { return 0; }", + "commit_hermes_mcp_applied_if_pending() { return 0; }", "refresh_hermes_supervised_child_pids() { trace refresh; }", 'date() { trace unexpected-exit-record; printf "100\\n"; }', 'sleep() { trace "sleep:$1"; }', @@ -1131,6 +1156,8 @@ describe("Hermes supervised auxiliary recovery", () => { "listener:101:8642", "live:101", "listener:101:8642", + "live:101", + "listener:101:8642", "live:202", "service-listener:202:19119:sandbox", "stop:303", @@ -1192,95 +1219,6 @@ describe("Hermes supervised auxiliary recovery", () => { ]); }); - it("replaces a listener-owning API bridge that fails public HTTP health", () => { - const source = fs.readFileSync(START_SCRIPT, "utf-8"); - const result = runBashHarness([ - 'trace() { printf "%s\\n" "$*"; }', - 'id() { [ "${1:-}" = "-u" ] && printf "1000\\n"; }', - 'gateway_control_pid_is_live() { trace "live:$1"; return 0; }', - 'hermes_tracked_role_is_current() { gateway_control_pid_is_live "$2"; }', - 'gateway_control_pid_owns_tcp_listener() { trace "listener:$1:$2"; return 0; }', - 'curl() { if [ "$PUBLIC_HEALTH" = "stale" ]; then printf "503"; else printf "200"; fi; }', - 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', - 'start_socat_forwarder() { trace "start-forward:$*"; printf -v "$4" 111; PUBLIC_HEALTH=ready; return 0; }', - "hermes_dashboard_healthy() { trace dashboard-healthy; return 0; }", - "ensure_gateway_log_stream() { trace gateway-log; }", - extractShellFunction(source, "hermes_socat_bridge_healthy"), - extractShellFunction(source, "hermes_api_socat_bridge_healthy"), - extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), - "PUBLIC_PORT=8642", - "INTERNAL_PORT=18642", - "DASHBOARD_PUBLIC_PORT=18789", - "DASHBOARD_INTERNAL_PORT=19119", - "PUBLIC_HEALTH=stale", - "SOCAT_PID=101", - "DASHBOARD_PID=202", - "DASHBOARD_SOCAT_PID=303", - "GATEWAY_PID=4242", - 'if ensure_hermes_supervised_auxiliaries; then trace success; else trace "failure:$?"; fi', - 'trace "final-api-bridge:$SOCAT_PID"', - ]); - - expect(result.status, result.stderr).toBe(0); - expect(result.stdout.trim().split("\n")).toEqual([ - "live:101", - "listener:101:8642", - "stop:101", - "start-forward:8642 18642 API SOCAT_PID 4242 current", - "live:111", - "listener:111:8642", - "live:111", - "listener:111:8642", - "dashboard-healthy", - "live:303", - "listener:303:18789", - "gateway-log", - "success", - "final-api-bridge:111", - ]); - }); - - it("fails closed when a replacement API bridge still cannot serve public health", () => { - const source = fs.readFileSync(START_SCRIPT, "utf-8"); - const result = runBashHarness([ - 'trace() { printf "%s\\n" "$*"; }', - 'id() { [ "${1:-}" = "-u" ] && printf "1000\\n"; }', - 'gateway_control_pid_is_live() { trace "live:$1"; return 0; }', - 'hermes_tracked_role_is_current() { gateway_control_pid_is_live "$2"; }', - 'gateway_control_pid_owns_tcp_listener() { trace "listener:$1:$2"; return 0; }', - 'curl() { printf "503"; }', - 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', - 'start_socat_forwarder() { trace "start-forward:$*"; printf -v "$4" 111; return 0; }', - "hermes_dashboard_healthy() { trace unexpected-dashboard-health; return 0; }", - "ensure_gateway_log_stream() { trace unexpected-gateway-log; }", - extractShellFunction(source, "hermes_socat_bridge_healthy"), - extractShellFunction(source, "hermes_api_socat_bridge_healthy"), - extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), - "PUBLIC_PORT=8642", - "INTERNAL_PORT=18642", - "DASHBOARD_PUBLIC_PORT=18789", - "DASHBOARD_INTERNAL_PORT=19119", - "SOCAT_PID=101", - "DASHBOARD_PID=202", - "DASHBOARD_SOCAT_PID=303", - "GATEWAY_PID=4242", - 'if ensure_hermes_supervised_auxiliaries; then trace success; else trace "failure:$?"; fi', - 'trace "final-api-bridge:$SOCAT_PID"', - ]); - - expect(result.status, result.stderr).toBe(0); - expect(result.stdout.trim().split("\n")).toEqual([ - "live:101", - "listener:101:8642", - "stop:101", - "start-forward:8642 18642 API SOCAT_PID 4242 current", - "live:111", - "listener:111:8642", - "failure:1", - "final-api-bridge:111", - ]); - }); - it("restarts a dashboard that owns its listener but fails HTTP health", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ @@ -1316,6 +1254,8 @@ describe("Hermes supervised auxiliary recovery", () => { "listener:101:8642", "live:101", "listener:101:8642", + "live:101", + "listener:101:8642", "live:202", "service-listener:202:19119:sandbox", "stop:303", diff --git a/test/hermes-mcp-config-transaction.test.ts b/test/hermes-mcp-config-transaction.test.ts index 07ff0e08b01..f4972996543 100644 --- a/test/hermes-mcp-config-transaction.test.ts +++ b/test/hermes-mcp-config-transaction.test.ts @@ -382,7 +382,7 @@ snapshot = types.SimpleNamespace(mode=0o600) module.os.geteuid = lambda: 1000 module._assert_mutable_snapshot = lambda received: None module._managed_hash_paths = lambda privileged: [] -module._refresh_and_verify_hashes = lambda guard, privileged: None +module._refresh_and_verify_hashes = lambda guard, privileged, transition="preserve": None module.reload_gateway = lambda: True def run(method_name, original): @@ -395,6 +395,7 @@ def run(method_name, original): module._load_guard = lambda: types.SimpleNamespace( _read_text=read_text, _write_existing=write_existing, + inspect_mcp_integrity=lambda *_args: "current", ) error = "" try: @@ -561,9 +562,12 @@ def fixture(name): with open(path, "w", encoding="utf-8") as handle: handle.write(text) os.chmod(path, 0o600) + empty_mcp = hashlib.sha256(b"{}").hexdigest() hash_text = ( hashlib.sha256(CONFIG_TEXT.encode()).hexdigest() + " " + config_path + "\\n" + hashlib.sha256(ENV_TEXT.encode()).hexdigest() + " " + env_path + "\\n" + + "# nemoclaw-hermes-mcp-state-v1 intended=" + empty_mcp + + " applied=" + empty_mcp + "\\n" ) with open(hash_path, "w", encoding="utf-8") as handle: handle.write(hash_text) @@ -874,6 +878,7 @@ sys.modules["gateway.status"] = status module.os.geteuid = lambda: 0 module.pwd.getpwnam = lambda name: types.SimpleNamespace(pw_uid=2000) module._is_trusted_gateway_process = lambda pid: True +module._gateway_has_managed_parent = lambda pid: True module.os.stat = lambda path: types.SimpleNamespace(st_uid=1000) try: module._gateway_identity() @@ -925,6 +930,7 @@ sys.modules["gateway.status"] = status module.GATEWAY_PID_PATH = sys.argv[3] module.os.stat = lambda path: types.SimpleNamespace(st_uid=expected_uid) module._is_trusted_gateway_process = lambda pid: pid == 4242 +module._gateway_has_managed_parent = lambda pid: True recognized = module._gateway_identity() runtime["lock_active"] = False @@ -1045,6 +1051,7 @@ statuses[module.GATEWAY_INTERNAL_PORT] = 200 statuses[module.GATEWAY_PUBLIC_PORT] = [503, 401, 401] identities = iter(((1, 10), (2, 20), (2, 20), (3, 30), (3, 30), (3, 30))) module._gateway_identity = lambda: next(identities) +module._gateway_has_managed_parent = lambda pid: True signals = [] module.os.kill = lambda pid, sent_signal: signals.append((pid, signal.Signals(sent_signal).name)) module.time.monotonic = lambda: 0 @@ -1126,15 +1133,20 @@ module.pwd.getpwnam = lambda name: (_ for _ in ()).throw( ) snapshot = types.SimpleNamespace(mode=0o600, uid=sandbox_uid, gid=sandbox_uid) +config_state = {"text": "model: test\\n"} guard = types.SimpleNamespace( - _read_text=lambda path: ("model: test\\n", snapshot), + _read_text=lambda path: (config_state["text"], snapshot), ) module._load_guard = lambda: guard def apply_transaction(action, payload): observed["helper_uid"] = module.os.geteuid() observed["action"] = action + parsed = module.yaml.safe_load(config_state["text"]) + updated, _changed = module._mutate(parsed, action, payload) + config_state["text"] = module.yaml.safe_dump(updated, sort_keys=False) return True module.apply_transaction = apply_transaction +module._refresh_and_verify_hashes = lambda guard, privileged, transition="preserve": None gateway = types.ModuleType("gateway") status = types.ModuleType("gateway.status") @@ -1198,7 +1210,7 @@ print(json.dumps(observed, sort_keys=True)) }); }); - it("repairs and verifies strict and compatibility hashes on an unchanged retry", () => { + it("verifies strict and compatibility MCP hash state on an unchanged retry", () => { const temp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-tx-")); const hermesDir = path.join(temp, ".hermes"); const configPath = path.join(hermesDir, "config.yaml"); @@ -1239,6 +1251,12 @@ module.STRICT_HASH_PATH = sys.argv[4] module.os.geteuid = lambda: 0 module._require_lifecycle_identity = lambda: None module._assert_mutable_snapshot = lambda snapshot: None +guard = module._load_guard() +hash_text, _config_snapshot, _env_snapshot = guard._hash_text( + module.CONFIG_PATH, os.path.join(module.HERMES_DIR, ".env") +) +guard._write_hash(sys.argv[4], hash_text) +guard._write_hash(os.path.join(module.HERMES_DIR, ".config-hash"), hash_text) changed = module.apply_transaction("add", { "server": "fake", "url": "https://mcp.example.test/mcp", @@ -1410,7 +1428,8 @@ print(json.dumps(module.probe(), sort_keys=True)) const strictHash = path.join(temp, "strict-hash"); const config = "model: test\n"; const env = "HERMES_TEST=1\n"; - const originalHash = `${crypto.createHash("sha256").update(config).digest("hex")} ${configPath}\n${crypto.createHash("sha256").update(env).digest("hex")} ${envPath}\n`; + const emptyMcp = crypto.createHash("sha256").update("{}").digest("hex"); + const originalHash = `${crypto.createHash("sha256").update(config).digest("hex")} ${configPath}\n${crypto.createHash("sha256").update(env).digest("hex")} ${envPath}\n# nemoclaw-hermes-mcp-state-v1 intended=${emptyMcp} applied=${emptyMcp}\n`; fs.mkdirSync(hermesDir); fs.writeFileSync(configPath, config, { mode: 0o600 }); fs.writeFileSync(envPath, env, { mode: 0o600 }); diff --git a/test/hermes-mcp-credential-boundary-manifest.test.ts b/test/hermes-mcp-credential-boundary-manifest.test.ts new file mode 100644 index 00000000000..312ca934b0c --- /dev/null +++ b/test/hermes-mcp-credential-boundary-manifest.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const TRANSACTION = path.join( + import.meta.dirname, + "..", + "agents", + "hermes", + "mcp-config-transaction.py", +); +const MANIFEST_NAME = "openshell-child-visible-credentials.v0.0.72.json"; +const validManifest: Record = { + openshellVersion: "0.0.72", + rawChildValueKeys: ["RAW_CHILD_VALUE"], + rewrittenChildValueKeys: ["REWRITTEN_CHILD_VALUE"], + runtimeControlKeys: ["RUNTIME_CONTROL"], + runtimeControlPrefixes: ["RUNTIME_CONTROL_"], +}; + +function runEmbeddedTransactionImport(setup: (helperDir: string) => void = () => {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-manifest-")); + const helperDir = path.join(root, "isolated", "helper"); + const helper = path.join(helperDir, "mcp-config-transaction.py"); + fs.mkdirSync(helperDir, { recursive: true }); + fs.copyFileSync(TRANSACTION, helper); + setup(helperDir); + + try { + const result = spawnSync( + "python3", + [ + "-I", + "-c", + String.raw` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("isolated_mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +try: + spec.loader.exec_module(module) +except Exception as error: + outcome = {"loaded": False, "type": type(error).__name__, "message": str(error)} +else: + outcome = {"loaded": True, "type": "", "message": ""} +print(json.dumps(outcome)) +`, + helper, + ], + { encoding: "utf-8", timeout: 5000 }, + ); + expect(result.status, result.stderr).toBe(0); + return JSON.parse(result.stdout) as { loaded: boolean; type: string; message: string }; + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +} + +function runEmbeddedTransactionImportWithManifest(manifest: Record) { + return runEmbeddedTransactionImport((helperDir) => { + fs.writeFileSync(path.join(helperDir, MANIFEST_NAME), JSON.stringify(manifest)); + }); +} + +describe("Hermes MCP credential boundary manifest (#6256)", () => { + it("fails closed when the manifest is missing", () => { + expect(runEmbeddedTransactionImport()).toEqual({ + loaded: false, + type: "RuntimeError", + message: "Hermes MCP credential boundary manifest is missing", + }); + }); + + it("fails closed on a manifest for another OpenShell version", () => { + expect( + runEmbeddedTransactionImportWithManifest({ + ...validManifest, + openshellVersion: "0.0.73", + }), + ).toEqual({ + loaded: false, + type: "RuntimeError", + message: "Hermes MCP credential boundary manifest is invalid", + }); + }); + + it.each([ + "rawChildValueKeys", + "rewrittenChildValueKeys", + "runtimeControlKeys", + "runtimeControlPrefixes", + ])("fails closed when %s is missing", (key) => { + const incomplete = { ...validManifest }; + delete incomplete[key]; + expect(runEmbeddedTransactionImportWithManifest(incomplete)).toEqual({ + loaded: false, + type: "RuntimeError", + message: `Hermes MCP credential boundary manifest has invalid ${key}`, + }); + }); +}); diff --git a/test/hermes-mcp-integrity-state.test.ts b/test/hermes-mcp-integrity-state.test.ts new file mode 100644 index 00000000000..914e074a026 --- /dev/null +++ b/test/hermes-mcp-integrity-state.test.ts @@ -0,0 +1,844 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { bashPrintfQ, extractShellFunction } from "./support/hermes-shell-harness"; + +const GUARD = path.join(import.meta.dirname, "..", "agents", "hermes", "runtime-config-guard.py"); +const BUILD_DIGEST = path.join( + import.meta.dirname, + "..", + "agents", + "hermes", + "build-mcp-digest.py", +); +const TRANSACTION = path.join( + import.meta.dirname, + "..", + "agents", + "hermes", + "mcp-config-transaction.py", +); +const START = path.join(import.meta.dirname, "..", "agents", "hermes", "start.sh"); + +function runHermesRootMcpStartup(commitStatus: 0 | 1) { + const source = fs.readFileSync(START, "utf-8"); + const startupBlock = source.match( + /^launch_hermes_gateway\nstart_gateway_log_stream\nwait_for_hermes_gateway_internal "\$GATEWAY_PID"\nensure_hermes_supervised_auxiliaries\nif ! commit_hermes_mcp_applied_if_pending; then\n[\s\S]*?^restore_hermes_config_permissions_after_dashboard_start$/m, + )?.[0]; + expect(startupBlock).toBeDefined(); + const startupScript = startupBlock as string; + + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-root-start-")); + const scriptPath = path.join(tempDir, "run.sh"); + fs.writeFileSync( + scriptPath, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'trace() { printf "%s\\n" "$*"; }', + 'launch_hermes_gateway() { GATEWAY_PID=4242; trace "launch:$GATEWAY_PID"; }', + "start_gateway_log_stream() { trace log-stream; }", + 'wait_for_hermes_gateway_internal() { trace "health:$1"; }', + "ensure_hermes_supervised_auxiliaries() { trace auxiliaries; }", + `commit_hermes_mcp_applied_if_pending() { trace commit-applied; return ${commitStatus}; }`, + "stop_hermes_gateway_fail_closed() { trace stop-fail-closed; }", + "restore_hermes_config_permissions_after_dashboard_start() { trace restore-permissions; }", + startupScript, + "trace startup-complete", + ].join("\n"), + { mode: 0o700 }, + ); + + try { + return spawnSync("bash", [scriptPath], { + encoding: "utf-8", + timeout: 5000, + env: process.env, + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +describe("Hermes MCP intended/applied integrity state", () => { + it("uses the runtime canonicalizer for the build-time MCP seal", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-build-seal-")); + const config = path.join(tempDir, "config.yaml"); + fs.writeFileSync( + config, + "mcp_servers:\n zed:\n url: https://zed.example/mcp\n alpha:\n url: https://alpha.example/mcp\n", + ); + + try { + const buildDigest = spawnSync( + "python3", + ["-I", BUILD_DIGEST, "--guard", GUARD, "--config", config], + { encoding: "utf-8", timeout: 5000 }, + ); + const runtimeDigest = spawnSync( + "python3", + [ + "-I", + "-c", + String.raw` +import importlib.util, sys +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +print(guard._canonical_mcp_servers_digest(open(sys.argv[2], encoding="utf-8").read())) +`, + GUARD, + config, + ], + { encoding: "utf-8", timeout: 5000 }, + ); + + expect(buildDigest.status, buildDigest.stderr).toBe(0); + expect(runtimeDigest.status, runtimeDigest.stderr).toBe(0); + expect(buildDigest.stdout).toMatch(/^[0-9a-f]{64}\n$/u); + expect(buildDigest.stdout).toBe(runtimeDigest.stdout); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("omits authenticated config bytes from integrity snapshot representations", () => { + const result = spawnSync( + "python3", + [ + "-I", + "-c", + String.raw` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +metadata = guard.FileSnapshot( + dev=1, + ino=2, + mode=0o600, + uid=1000, + gid=1000, + nlink=1, + size=64, + mtime_ns=3, + ctime_ns=4, +) +secret = "API_SERVER_KEY=must-not-appear" +snapshot = guard.McpIntegritySnapshot( + state="current", + config_text=secret, + config_path="/sandbox/.hermes/config.yaml", + config_snapshot=metadata, + env_path="/sandbox/.hermes/.env", + env_snapshot=metadata, + hash_snapshots=(), +) +rendered = repr(snapshot) +print(json.dumps({ + "contains_config_field": "config_text=" in rendered, + "contains_secret": secret in rendered, + "is_snapshot_repr": rendered.startswith("McpIntegritySnapshot("), +})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 5000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + contains_config_field: false, + contains_secret: false, + is_snapshot_repr: true, + }); + }); + + it("returns current and pending through the guarded CLI status protocol", () => { + const result = spawnSync( + "python3", + [ + "-I", + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +root = tempfile.mkdtemp(prefix="hermes-mcp-cli-status-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +anchor = os.path.join(root, "hermes.config-hash") +open(config, "w", encoding="utf-8").write("model: test\n") +open(env, "w", encoding="utf-8").write("SAFE=1\n") +hash_text, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(anchor, hash_text) + +def inspect_status(): + sys.argv = [ + "runtime-config-guard.py", + "inspect-mcp-integrity", + "--hermes-dir", hermes, + "--hash-file", anchor, + "--startup-owner", + "--mcp-state-exit-code", + ] + return guard.main() + +current = inspect_status() +open(config, "w", encoding="utf-8").write( + "model: test\nmcp_servers:\n alpha:\n url: https://alpha.example/mcp\n" +) +guard.refresh_hashes(hermes, anchor, "strict", mcp_transition="intend") +pending = inspect_status() +sys.argv = [ + "runtime-config-guard.py", + "ensure-api-key", + "--hermes-dir", hermes, + "--mcp-state-exit-code", +] +try: + guard.main() +except SystemExit as error: + misuse = error.code +else: + misuse = 0 +print(json.dumps({"current": current, "pending": pending, "misuse": misuse})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ current: 0, pending: 10, misuse: 1 }); + }); + + it("uses the atomic write outcome for compat applied-state commits", () => { + const result = spawnSync( + "python3", + [ + "-I", + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +root = tempfile.mkdtemp(prefix="hermes-mcp-compat-apply-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +anchor = os.path.join(hermes, ".config-hash") +open(config, "w", encoding="utf-8").write("model: test\n") +open(env, "w", encoding="utf-8").write("SAFE=1\n") +hash_text, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(anchor, hash_text) +open(config, "w", encoding="utf-8").write( + "model: test\nmcp_servers:\n alpha:\n url: https://alpha.example/mcp\n" +) +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="intend") +original_access = guard.os.access +guard.os.access = lambda *_args: False +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="apply") +false_negative_state = guard.inspect_mcp_integrity(hermes, anchor) +guard.os.access = original_access +open(config, "w", encoding="utf-8").write( + "model: test\nmcp_servers:\n beta:\n url: https://beta.example/mcp\n" +) +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="intend") +pending_text = open(anchor, encoding="utf-8").read() +guard._write_hash = lambda *_args: (_ for _ in ()).throw( + PermissionError(13, "permission denied") +) +try: + guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="apply") +except PermissionError: + write_denied = True +else: + write_denied = False +print(json.dumps({ + "false_negative_state": false_negative_state, + "write_denied": write_denied, + "unchanged": open(anchor, encoding="utf-8").read() == pending_text, +})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + false_negative_state: "current", + write_denied: true, + unchanged: true, + }); + }); + + it("runs startup-owned MCP inspection as a direct child", () => { + const source = fs.readFileSync(START, "utf-8"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-parent-")); + const helper = path.join(tempDir, "guard-helper.sh"); + const parentFile = path.join(tempDir, "guard-parent"); + fs.writeFileSync( + helper, + [ + "#!/bin/bash", + "set -euo pipefail", + 'printf "%s\\n" "$PPID" >"$NEMOCLAW_TEST_GUARD_PARENT_FILE"', + 'printf "%s\\n" "mcp_state=current"', + ].join("\n"), + { mode: 0o700 }, + ); + + try { + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + extractShellFunction(source, "inspect_hermes_mcp_integrity"), + `_HERMES_PYTHON=${bashPrintfQ(helper)}`, + "_HERMES_RUNTIME_CONFIG_GUARD=/test/runtime-config-guard.py", + "HERMES_DIR=/test/.hermes", + "HERMES_HASH_FILE=/test/hermes.config-hash", + `NEMOCLAW_TEST_GUARD_PARENT_FILE=${bashPrintfQ(parentFile)}`, + "export NEMOCLAW_TEST_GUARD_PARENT_FILE", + "HERMES_MCP_RECONCILE_PENDING=9", + "caller_pid=$BASHPID", + "inspect_hermes_mcp_integrity", + 'IFS= read -r guard_parent <"$NEMOCLAW_TEST_GUARD_PARENT_FILE"', + '[ "$guard_parent" = "$caller_pid" ]', + 'printf "pending=%s\\n" "$HERMES_MCP_RECONCILE_PENDING"', + ].join("\n"), + ], + { encoding: "utf-8", timeout: 5000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("pending=0\n"); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it.each([ + { status: 0, expected: "rc=0 pending=0 failed=0\n" }, + { status: 10, expected: "rc=0 pending=1 failed=0\n" }, + { status: 1, expected: "rc=1 pending=9 failed=1\n" }, + ])("uses only the authenticated guard exit status ($status)", ({ status, expected }) => { + const source = fs.readFileSync(START, "utf-8"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-status-")); + const helper = path.join(tempDir, "guard-helper.sh"); + fs.writeFileSync( + helper, + [ + "#!/bin/bash", + "set -euo pipefail", + "printf 'mcp_state=current\\0attacker\\nmcp_state=pending'", + `exit ${status}`, + ].join("\n"), + { mode: 0o700 }, + ); + + try { + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -uo pipefail", + extractShellFunction(source, "inspect_hermes_mcp_integrity"), + `_HERMES_PYTHON=${bashPrintfQ(helper)}`, + "_HERMES_RUNTIME_CONFIG_GUARD=/test/runtime-config-guard.py", + "HERMES_DIR=/test/.hermes", + "HERMES_HASH_FILE=/test/hermes.config-hash", + "HERMES_MCP_RECONCILE_PENDING=9", + "HERMES_MCP_INTEGRITY_FAILED=0", + "if inspect_hermes_mcp_integrity; then rc=0; else rc=$?; fi", + 'printf "rc=%s pending=%s failed=%s\\n" "$rc" "$HERMES_MCP_RECONCILE_PENDING" "$HERMES_MCP_INTEGRITY_FAILED"', + ].join("\n"), + ], + { encoding: "utf-8", timeout: 5000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe(expected); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("rejects unmanaged fields in the host inspection projection", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +candidate = module._managed_candidate({ + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, +}) +rejected = [] +for field, value in ( + ("command", "touch /tmp/pwned"), + ("transport", "stdio"), + ("extra", True), +): + payload = {"present": {"safe": {**candidate, field: value}}, "absent": []} + try: + module._validate_inspection_payload(payload) + except ValueError as error: + rejected.append(str(error)) +print(json.dumps(rejected)) +`, + TRANSACTION, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual([ + "Hermes MCP inspection expected config has invalid fields", + "Hermes MCP inspection expected config has invalid fields", + "Hermes MCP inspection expected config has invalid fields", + ]); + }); + + it("reports a managed config match only after the gateway-applied state is current", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, sys, types, yaml +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.HERMES_DIR = "/tmp/.hermes" +module.CONFIG_PATH = "/tmp/.hermes/config.yaml" +module.os.geteuid = lambda: 1000 +candidate = module._managed_candidate({ + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, +}) +payload = {"present": {"safe": candidate}, "absent": []} +outcomes = {} +for integrity_state in ("current", "pending"): + module._load_guard = lambda state=integrity_state: types.SimpleNamespace( + inspect_mcp_integrity_snapshot=lambda *_args: types.SimpleNamespace( + state=state, + config_text=yaml.safe_dump( + {"mcp_servers": {"safe": candidate}}, sort_keys=False + ), + ), + assert_mcp_integrity_snapshot_current=lambda *_args: None, + ) + try: + outcomes[integrity_state] = module.inspect_managed_config(payload) + except RuntimeError as error: + outcomes[integrity_state] = str(error) +print(json.dumps(outcomes)) +`, + TRANSACTION, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + current: { ok: true, state: "matched" }, + pending: "Hermes MCP config does not match applied gateway state", + }); + }); + + it("refuses diverged root anchors and config races after integrity verification", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile, yaml + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + +transaction = load("mcp_tx", sys.argv[1]) +guard = load("hermes_guard", sys.argv[2]) +root = tempfile.mkdtemp(prefix="hermes-mcp-inspect-race-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +strict = os.path.join(root, "strict-hash") +compat = os.path.join(hermes, ".config-hash") +candidate = transaction._managed_candidate({ + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, +}) +with open(config, "w", encoding="utf-8") as handle: + handle.write(yaml.safe_dump({"mcp_servers": {"safe": candidate}}, sort_keys=False)) +with open(env, "w", encoding="utf-8") as handle: + handle.write("SAFE=1\n") +hash_text, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(strict, hash_text) +guard._write_hash(compat, "diverged\n") + +transaction.HERMES_DIR = hermes +transaction.CONFIG_PATH = config +transaction.STRICT_HASH_PATH = strict +transaction.os.geteuid = lambda: 0 +transaction._load_guard = lambda: guard +try: + transaction.inspect_managed_config({"present": {"safe": candidate}, "absent": []}) +except Exception as error: + diverged = str(error) +guard._write_hash(compat, hash_text) +original_inspect = guard.inspect_mcp_integrity_snapshot +def race_after_authentication(*args): + inspection = original_inspect(*args) + changed = {**candidate, "url": "https://attacker.example.test/mcp"} + with open(config, "w", encoding="utf-8") as handle: + handle.write(yaml.safe_dump({"mcp_servers": {"safe": changed}}, sort_keys=False)) + return inspection +guard.inspect_mcp_integrity_snapshot = race_after_authentication + +try: + raced = transaction.inspect_managed_config( + {"present": {"safe": candidate}, "absent": []} + ) +except Exception as error: + raced = str(error) +print(json.dumps({"diverged": diverged, "raced": raced})) +`, + TRANSACTION, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + diverged: "Hermes strict and compatibility MCP integrity anchors differ", + raced: "refusing raced Hermes MCP integrity snapshot", + }); + }); + + it("derives the full config hash and MCP digest from one config snapshot", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +root = tempfile.mkdtemp(prefix="hermes-mcp-single-read-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +strict = os.path.join(root, "hash") +open(config, "w", encoding="utf-8").write("model: test\nmcp_servers: {}\n") +open(env, "w", encoding="utf-8").write("SAFE=1\n") +initial, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(strict, initial) +original_read_text = guard._read_text +config_reads = 0 +def counted_read_text(path, *args, **kwargs): + global config_reads + if path == config: + config_reads += 1 + return original_read_text(path, *args, **kwargs) +guard._read_text = counted_read_text +state = guard.inspect_mcp_integrity(hermes, strict) +print(json.dumps({"state": state, "config_reads": config_reads})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ state: "current", config_reads: 1 }); + }); + + it("commits pending state after root gateway health before continuing startup", () => { + const success = runHermesRootMcpStartup(0); + expect(success.status, success.stderr).toBe(0); + expect(success.stdout.trim().split("\n")).toEqual([ + "launch:4242", + "log-stream", + "health:4242", + "auxiliaries", + "commit-applied", + "restore-permissions", + "startup-complete", + ]); + }); + + it("fails root startup closed when the applied-state commit fails after gateway health", () => { + const failure = runHermesRootMcpStartup(1); + expect(failure.status).toBe(1); + expect(failure.stdout.trim().split("\n")).toEqual([ + "launch:4242", + "log-stream", + "health:4242", + "auxiliaries", + "commit-applied", + "stop-fail-closed", + ]); + expect(failure.stderr).toContain("HERMES_MCP_APPLIED_COMMIT_FAILED"); + expect(failure.stdout).not.toContain("restore-permissions"); + expect(failure.stdout).not.toContain("startup-complete"); + }); + + it("tracks add and removal as pending until the gateway-applied commit", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile + +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +root = tempfile.mkdtemp(prefix="hermes-mcp-integrity-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +strict = os.path.join(root, "hermes.config-hash") +compat = os.path.join(hermes, ".config-hash") +open(config, "w", encoding="utf-8").write("model: test\n") +open(env, "w", encoding="utf-8").write("SAFE=1\n") +initial_hash, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(strict, initial_hash) +guard._write_hash(compat, initial_hash) +states = [guard.inspect_mcp_integrity(hermes, strict)] + +managed = """model: test +mcp_servers: + fake: + url: https://mcp.example.test/mcp + enabled: true + timeout: 120 + connect_timeout: 60 + tools: {resources: true, prompts: true} + headers: + Authorization: Bearer openshell:resolve:env:FAKE_TOKEN +""" +open(config, "w", encoding="utf-8").write(managed) +guard.refresh_hashes(hermes, strict, "strict", mcp_transition="intend") +guard.refresh_hashes(hermes, strict, "compat", mcp_transition="intend") +states.append(guard.inspect_mcp_integrity(hermes, strict)) +guard.refresh_hashes(hermes, strict, "strict", mcp_transition="apply") +guard.refresh_hashes(hermes, strict, "compat", mcp_transition="apply") +states.append(guard.inspect_mcp_integrity(hermes, strict)) + +open(config, "w", encoding="utf-8").write("model: test\n") +guard.refresh_hashes(hermes, strict, "strict", mcp_transition="intend") +guard.refresh_hashes(hermes, strict, "compat", mcp_transition="intend") +states.append(guard.inspect_mcp_integrity(hermes, strict)) +guard.refresh_hashes(hermes, strict, "strict", mcp_transition="apply") +guard.refresh_hashes(hermes, strict, "compat", mcp_transition="apply") +states.append(guard.inspect_mcp_integrity(hermes, strict)) +hash_text = open(strict, encoding="utf-8").read() +print(json.dumps({"states": states, "hash": hash_text})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + const proof = JSON.parse(result.stdout) as { states: string[]; hash: string }; + expect(proof.states).toEqual(["current", "pending", "current", "pending", "current"]); + expect(proof.hash).toMatch( + /# nemoclaw-hermes-mcp-state-v1 intended=[0-9a-f]{64} applied=[0-9a-f]{64}/u, + ); + expect(proof.hash).not.toContain("FAKE_TOKEN"); + }); + + it("refuses a second intent while a prior MCP transaction is incomplete", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +root = tempfile.mkdtemp(prefix="hermes-mcp-incomplete-intent-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +strict = os.path.join(root, "hash") +open(config, "w", encoding="utf-8").write("model: test\n") +open(env, "w", encoding="utf-8").write("SAFE=1\n") +initial, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(strict, initial) +open(config, "w", encoding="utf-8").write( + "model: test\nmcp_servers: {fake: {url: https://first.example.test/mcp}}\n" +) +guard.refresh_hashes(hermes, strict, "strict", mcp_transition="intend") +pending_hash = open(strict, encoding="utf-8").read() +open(config, "w", encoding="utf-8").write( + "model: test\nmcp_servers: {fake: {url: https://second.example.test/mcp}}\n" +) +try: + guard.refresh_hashes(hermes, strict, "strict", mcp_transition="intend") +except Exception as error: + refusal = str(error) +else: + refusal = "" +print(json.dumps({ + "refusal": refusal, + "hash_unchanged": open(strict, encoding="utf-8").read() == pending_hash, +})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + refusal: "Hermes MCP configuration has an incomplete prior transaction", + hash_unchanged: true, + }); + }); + + it("does not bless unrelated config or env drift while committing applied state", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +root = tempfile.mkdtemp(prefix="hermes-mcp-apply-race-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +strict = os.path.join(root, "hash") +compat = os.path.join(hermes, ".config-hash") +open(config, "w", encoding="utf-8").write("model: test\n") +open(env, "w", encoding="utf-8").write("SAFE=1\n") +initial, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(strict, initial) +guard._write_hash(compat, initial) +pending_config = "model: test\nmcp_servers: {fake: {url: https://mcp.example.test/mcp}}\n" +open(config, "w", encoding="utf-8").write(pending_config) +guard.refresh_hashes(hermes, strict, "strict", mcp_transition="intend") +guard.refresh_hashes(hermes, strict, "compat", mcp_transition="intend") +pending_hash = open(strict, encoding="utf-8").read() +errors = [] +open(env, "w", encoding="utf-8").write("SAFE=changed-canary\n") +try: + guard.refresh_hashes(hermes, strict, "strict", mcp_transition="apply") +except Exception as error: + errors.append(str(error)) +open(env, "w", encoding="utf-8").write("SAFE=1\n") +open(config, "w", encoding="utf-8").write(pending_config.replace("model: test", "model: drift-canary")) +try: + guard.refresh_hashes(hermes, strict, "strict", mcp_transition="apply") +except Exception as error: + errors.append(str(error)) +print(json.dumps({"errors": errors, "hash_unchanged": open(strict, encoding="utf-8").read() == pending_hash})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + const proof = JSON.parse(result.stdout) as { errors: string[]; hash_unchanged: boolean }; + expect(proof.errors).toHaveLength(2); + expect(proof.hash_unchanged).toBe(true); + expect(proof.errors.join("\n")).not.toMatch(/changed-canary|drift-canary/u); + }); + + it("fails closed on drift and malformed or missing MCP metadata", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile +spec = importlib.util.spec_from_file_location("hermes_guard", sys.argv[1]) +guard = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = guard +spec.loader.exec_module(guard) +root = tempfile.mkdtemp(prefix="hermes-mcp-refusal-") +hermes = os.path.join(root, ".hermes") +os.mkdir(hermes) +config = os.path.join(hermes, "config.yaml") +env = os.path.join(hermes, ".env") +strict = os.path.join(root, "hash") +open(config, "w", encoding="utf-8").write("model: test\n") +open(env, "w", encoding="utf-8").write("SAFE=1\n") +initial_hash, _config_snapshot, _env_snapshot = guard._hash_text(config, env) +guard._write_hash(strict, initial_hash) +errors = [] +open(config, "w", encoding="utf-8").write("mcp_servers: {fake: {token: raw-canary}}\n") +for operation in ( + lambda: guard.inspect_mcp_integrity(hermes, strict), + lambda: (open(strict, "w", encoding="utf-8").write("malformed\n"), guard.inspect_mcp_integrity(hermes, strict))[1], + lambda: (os.unlink(strict), guard.inspect_mcp_integrity(hermes, strict))[1], + lambda: guard.refresh_hashes(hermes, strict, "strict"), +): + try: + operation() + except Exception as error: + errors.append(str(error)) +print(json.dumps({"errors": errors, "hash_exists": os.path.exists(strict)})) +`, + GUARD, + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + const proof = JSON.parse(result.stdout) as { errors: string[]; hash_exists: boolean }; + expect(proof.errors).toHaveLength(4); + expect(proof.hash_exists).toBe(false); + expect(proof.errors.join("\n")).not.toContain("raw-canary"); + }); +}); diff --git a/test/hermes-mcp-reload-convergence.test.ts b/test/hermes-mcp-reload-convergence.test.ts index c1c3c6a7191..ad6d56b3d86 100644 --- a/test/hermes-mcp-reload-convergence.test.ts +++ b/test/hermes-mcp-reload-convergence.test.ts @@ -110,6 +110,96 @@ else: }); }); + it("does not probe or accept an unmanaged replacement gateway", () => { + const result = runPython(` +import importlib.util, json, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +module.RELOAD_TIMEOUT_SECONDS = 3 +clock = {"now": 0} +gateway = {"identity": (4242, 99)} +signals = [] +health_calls = [] +module._gateway_identity = lambda: gateway["identity"] +module._gateway_has_managed_parent = lambda pid: False +module._gateway_health_phase = lambda deadline=None: ( + health_calls.append(deadline) or (True, "waiting-for-stable-replacement-identity") +) +module.time.monotonic = lambda: clock["now"] +module.time.sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) +def signal_gateway(pid, sent_signal): + signals.append((pid, signal.Signals(sent_signal).name)) + gateway["identity"] = (4243, 100) +module.os.kill = signal_gateway +try: + module.reload_gateway() +except TimeoutError as error: + print(json.dumps({"error": str(error), "health_calls": health_calls, "signals": signals})) +else: + raise SystemExit(9) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-replacement-identity; re-kick attempted: no; re-kick sent: no)", + health_calls: [], + signals: [[4242, "SIGUSR1"]], + }); + }); + + it("revalidates the managed parent after replacement health", () => { + const result = runPython(` +import importlib.util, json, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +module.RELOAD_TIMEOUT_SECONDS = 3 +clock = {"now": 0} +gateway = {"identity": (4242, 99)} +signals = [] +parent_checks = [] +health_calls = [] +module._gateway_identity = lambda: gateway["identity"] +def managed_parent(pid): + parent_checks.append(pid) + return len(parent_checks) == 1 +module._gateway_has_managed_parent = managed_parent +module._gateway_health_phase = lambda deadline=None: ( + health_calls.append(deadline) or (True, "waiting-for-stable-replacement-identity") +) +module.time.monotonic = lambda: clock["now"] +module.time.sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) +def signal_gateway(pid, sent_signal): + signals.append((pid, signal.Signals(sent_signal).name)) + gateway["identity"] = (4243, 100) +module.os.kill = signal_gateway +try: + module.reload_gateway() +except TimeoutError as error: + print(json.dumps({ + "error": str(error), + "health_calls": len(health_calls), + "signals": signals, + })) +else: + raise SystemExit(9) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-stable-replacement-identity; re-kick attempted: no; re-kick sent: no)", + health_calls: 1, + signals: [[4242, "SIGUSR1"]], + }); + }); + it("attempts a vanished re-kick target only once", () => { const result = runPython(` import importlib.util, json, signal, sys @@ -209,9 +299,7 @@ def health_phase(deadline=None): return False, "waiting-for-internal-health-on-18642" module._gateway_identity = identity module._gateway_health_phase = health_phase -module._gateway_has_managed_parent = lambda pid: (_ for _ in ()).throw( - AssertionError("deadline exhaustion must precede re-kick authority checks") -) +module._gateway_has_managed_parent = lambda pid: True module.time.monotonic = lambda: clock["now"] module.time.sleep = lambda seconds: (_ for _ in ()).throw( AssertionError("deadline exhaustion must not sleep") @@ -304,11 +392,8 @@ print(json.dumps({name: run_case(name) for name in ( }, public: { error: - "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-public-relay-health-on-8642; re-kick attempted: yes; re-kick sent: yes)", - signals: [ - [4242, "SIGUSR1"], - [4243, "SIGUSR1"], - ], + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-public-relay-health-on-8642; re-kick attempted: no; re-kick sent: no)", + signals: [[4242, "SIGUSR1"]], }, stable: { error: diff --git a/test/hermes-mcp-rollback-pending.test.ts b/test/hermes-mcp-rollback-pending.test.ts new file mode 100644 index 00000000000..ce2e8b6c9f7 --- /dev/null +++ b/test/hermes-mcp-rollback-pending.test.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const TRANSACTION = path.join( + import.meta.dirname, + "..", + "agents", + "hermes", + "mcp-config-transaction.py", +); +const GUARD = path.join(import.meta.dirname, "..", "agents", "hermes", "runtime-config-guard.py"); + +describe("Hermes MCP rollback integrity", () => { + it("keeps a failed runtime rollback pending until a healthy old-config reload", () => { + const result = spawnSync( + "python3", + [ + "-c", + String.raw` +import importlib.util, json, os, sys, tempfile + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + +transaction = load("rollback_pending_transaction", sys.argv[1]) +guard = load("rollback_pending_guard", sys.argv[2]) +with tempfile.TemporaryDirectory(prefix="hermes-mcp-rollback-pending-") as root: + hermes = os.path.join(root, ".hermes") + os.mkdir(hermes) + config = os.path.join(hermes, "config.yaml") + env = os.path.join(hermes, ".env") + strict = os.path.join(root, "hermes.config-hash") + compat = os.path.join(hermes, ".config-hash") + original_config = "model: test\n" + open(config, "w", encoding="utf-8").write(original_config) + open(env, "w", encoding="utf-8").write("SAFE=1\n") + initial, _config_snapshot, _env_snapshot = guard._hash_text(config, env) + guard._write_hash(strict, initial) + guard._write_hash(compat, initial) + + transaction.GUARD_PATH = sys.argv[2] + transaction.HERMES_DIR = hermes + transaction.CONFIG_PATH = config + transaction.STRICT_HASH_PATH = strict + transaction.os.geteuid = lambda: 0 + transaction._assert_mutable_snapshot = lambda _snapshot: None + reload_calls = {"count": 0} + def fail_reload(): + reload_calls["count"] += 1 + raise RuntimeError(f"reload-{reload_calls['count']}-failed") + transaction.reload_gateway = fail_reload + + error = "" + try: + transaction.apply_transaction_and_reload("add", { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, + }) + except RuntimeError as caught: + error = str(caught) + + strict_pending = open(strict, encoding="utf-8").read() + compat_pending = open(compat, encoding="utf-8").read() + _config_digest, _env_digest, pending_marker = guard._parse_config_hash( + strict_pending, config, env + ) + pending_state = guard.inspect_mcp_integrity(hermes, strict) + restored_config = open(config, encoding="utf-8").read() + + guard.refresh_hashes(hermes, strict, "strict", mcp_transition="apply") + guard.refresh_hashes(hermes, strict, "compat", mcp_transition="apply") + repaired_state = guard.inspect_mcp_integrity(hermes, strict) + rejected = "" + try: + guard.refresh_hashes(hermes, strict, "strict", mcp_transition="rollback") + except Exception as caught: + rejected = str(caught) + + print(json.dumps({ + "compat_matches": compat_pending == strict_pending, + "error": error, + "marker_differs": pending_marker.intended != pending_marker.applied, + "pending_state": pending_state, + "rejected": rejected, + "reload_calls": reload_calls["count"], + "repaired_state": repaired_state, + "restored_config": restored_config, + })) +`, + TRANSACTION, + GUARD, + ], + { encoding: "utf-8", timeout: 15_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + const proof = JSON.parse(result.stdout) as { + compat_matches: boolean; + error: string; + marker_differs: boolean; + pending_state: string; + rejected: string; + reload_calls: number; + repaired_state: string; + restored_config: string; + }; + expect(proof).toMatchObject({ + compat_matches: true, + marker_differs: true, + pending_state: "pending", + reload_calls: 2, + repaired_state: "current", + restored_config: "model: test\n", + }); + expect(proof.error).toContain("reload-1-failed"); + expect(proof.error).toContain("old-config runtime reload failed: reload-2-failed"); + expect(proof.rejected).toContain("rollback requires a pending desired configuration"); + }); +}); diff --git a/test/hermes-nonroot-strict-hash-reconciliation.test.ts b/test/hermes-nonroot-strict-hash-reconciliation.test.ts index 8286a186723..0cde8450c08 100644 --- a/test/hermes-nonroot-strict-hash-reconciliation.test.ts +++ b/test/hermes-nonroot-strict-hash-reconciliation.test.ts @@ -35,7 +35,8 @@ function hashInputs(fixture: ReconciliationFixture): string { timeout: 5000, }); expect(result.status, result.stderr).toBe(0); - return result.stdout; + const mcpDigest = createHash("sha256").update("{}").digest("hex"); + return `${result.stdout}# nemoclaw-hermes-mcp-state-v1 intended=${mcpDigest} applied=${mcpDigest}\n`; } function createFixture(hermesMode = 0o3770): ReconciliationFixture { @@ -399,6 +400,32 @@ print(json.dumps([private_live, canonical_mutable, foreign_private, unexpected_m } }); + it.each([ + ["config", "configPath"], + ["environment", "envPath"], + ] as const)("binds non-root strict hash parsing to the live %s path (#2426)", (_label, pathKey) => { + const fixture = createFixture(); + fs.appendFileSync(fixture.envPath, `API_SERVER_KEY=${"4".repeat(64)}\n`); + refreshCompatOnly(fixture); + const livePath = fixture[pathKey]; + const malformed = fs + .readFileSync(fixture.hashPath, "utf-8") + .replace(` ${livePath}\n`, ` ${livePath}.stale\n`); + fs.writeFileSync(fixture.hashPath, malformed); + try { + const result = runManagedNonrootWrite( + fixture, + expectedConfigDigest(fixture), + "model:\n default: must-not-apply\n", + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("malformed Hermes config hash"); + assertCleanRefusal(fixture, malformed); + } finally { + fs.rmSync(fixture.root, { recursive: true, force: true }); + } + }); + it("refuses any env drift beyond the generated API key", () => { const fixture = createFixture(); fs.appendFileSync( diff --git a/test/hermes-restart-config-seal.test.ts b/test/hermes-restart-config-seal.test.ts index a36cd41a87a..7286ed0b499 100644 --- a/test/hermes-restart-config-seal.test.ts +++ b/test/hermes-restart-config-seal.test.ts @@ -33,6 +33,16 @@ function mode(pathname: string): number { return fs.statSync(pathname).mode & 0o7777; } +function hashInputs(configPath: string, envPath: string): string { + const result = spawnSync("sha256sum", [configPath, envPath], { + encoding: "utf-8", + timeout: 5000, + }); + expect(result.status, result.stderr).toBe(0); + const mcpDigest = createHash("sha256").update("{}").digest("hex"); + return `${result.stdout}# nemoclaw-hermes-mcp-state-v1 intended=${mcpDigest} applied=${mcpDigest}\n`; +} + function createRestartFixture(): RestartFixture { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-restart-seal-")); const sandboxDir = path.join(root, "sandbox"); @@ -53,13 +63,9 @@ function createRestartFixture(): RestartFixture { fs.writeFileSync(envPath, trustedEnv, { mode: 0o600 }); fs.chmodSync(envPath, 0o600); - const hash = spawnSync("sha256sum", [configPath, envPath], { - encoding: "utf-8", - timeout: 5000, - }); - expect(hash.status, hash.stderr).toBe(0); - fs.writeFileSync(hashPath, hash.stdout, { mode: 0o600 }); - fs.writeFileSync(compatHashPath, hash.stdout, { mode: 0o600 }); + const hash = hashInputs(configPath, envPath); + fs.writeFileSync(hashPath, hash, { mode: 0o600 }); + fs.writeFileSync(compatHashPath, hash, { mode: 0o600 }); return { root, @@ -251,16 +257,13 @@ describe.skipIf(process.platform === "win32")("Hermes mutable restart input seal }, () => { const fixture = createRestartFixture(); const boundarySize = 16 * 1024 * 1024; - const originalConfig = `${"a".repeat(boundarySize - 1)}\n`; - const updatedConfig = `${"b".repeat(boundarySize - 1)}\n`; + const payloadSize = boundarySize - "payload: \n".length; + const originalConfig = `payload: ${"a".repeat(payloadSize)}\n`; + const updatedConfig = `payload: ${"b".repeat(payloadSize)}\n`; fs.writeFileSync(fixture.configPath, originalConfig); - const hash = spawnSync("sha256sum", [fixture.configPath, fixture.envPath], { - encoding: "utf-8", - timeout: 10_000, - }); - expect(hash.status, hash.stderr).toBe(0); - fs.writeFileSync(fixture.hashPath, hash.stdout); - fs.writeFileSync(fixture.compatHashPath, hash.stdout); + const hash = hashInputs(fixture.configPath, fixture.envPath); + fs.writeFileSync(fixture.hashPath, hash); + fs.writeFileSync(fixture.compatHashPath, hash); const expectedDigest = createHash("sha256").update(originalConfig).digest("hex"); try { diff --git a/test/hermes-runtime-api-key.test.ts b/test/hermes-runtime-api-key.test.ts index 05440f2c1df..2712a22aa01 100644 --- a/test/hermes-runtime-api-key.test.ts +++ b/test/hermes-runtime-api-key.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -44,7 +45,9 @@ function writeHermesHash(hashPath: string, configPath: string, envPath: string): timeout: 5000, }); expect(result.status, result.stderr).toBe(0); - fs.writeFileSync(hashPath, result.stdout, { mode: 0o644 }); + const mcpDigest = createHash("sha256").update("{}").digest("hex"); + const hash = `${result.stdout}# nemoclaw-hermes-mcp-state-v1 intended=${mcpDigest} applied=${mcpDigest}\n`; + fs.writeFileSync(hashPath, hash, { mode: 0o644 }); } function parseApiServerKey(envFileContent: string): string | null { diff --git a/test/hermes-runtime-config-guard.test.ts b/test/hermes-runtime-config-guard.test.ts index 8a42410c8e2..25ae138f8f3 100644 --- a/test/hermes-runtime-config-guard.test.ts +++ b/test/hermes-runtime-config-guard.test.ts @@ -306,6 +306,8 @@ with tempfile.TemporaryDirectory() as tmp: with open(env_path, "wb") as handle: handle.write(b"API_SERVER_PORT=18642\\n") + initial_hash, _config_snapshot, _env_snapshot = guard._hash_text(config_path, env_path) + guard._write_hash(hash_path, initial_hash) before = os.stat(config_path) original_write_hash = guard._write_hash @@ -371,15 +373,17 @@ with tempfile.TemporaryDirectory() as tmp: with open(env_path, "w", encoding="utf-8") as handle: handle.write("API_SERVER_PORT=18642\\n") + initial_hash, _config_snapshot, _env_snapshot = guard._hash_text(config_path, env_path) + guard._write_hash(strict_hash_path, initial_hash) original_hash_text = guard._hash_text original_write_hash = guard._write_hash hash_text_calls = 0 writes = [] - def counted_hash_text(config, env): + def counted_hash_text(config, env, *args): global hash_text_calls hash_text_calls += 1 - return original_hash_text(config, env) + return original_hash_text(config, env, *args) def captured_write_hash(path, text): writes.append({"path": path, "text": text}) @@ -435,7 +439,9 @@ with tempfile.TemporaryDirectory() as tmp: with open(env_path, "w", encoding="utf-8") as handle: handle.write("API_SERVER_PORT=18642\\n") - guard.refresh_hashes(hermes_dir, strict_hash_path, "both") + initial_hash, _config_snapshot, _env_snapshot = guard._hash_text(config_path, env_path) + guard._write_hash(strict_hash_path, initial_hash) + guard._write_hash(compat_hash_path, initial_hash) with open(strict_hash_path, encoding="utf-8") as handle: old_strict = handle.read() with open(config_path, "w", encoding="utf-8") as handle: @@ -986,6 +992,34 @@ except guard.UnsafePathError: else: startup_without_owner_allowed = True +try: + guard._validate_action_readiness("inspect-mcp-integrity", True) +except guard.UnsafePathError: + inspect_allowed = False +else: + inspect_allowed = True + +try: + guard._validate_action_readiness("inspect-mcp-integrity", False) +except guard.UnsafePathError: + inspect_without_owner_allowed = False +else: + inspect_without_owner_allowed = True + +try: + guard._validate_action_readiness("commit-mcp-applied", True) +except guard.UnsafePathError: + commit_allowed = False +else: + commit_allowed = True + +try: + guard._validate_action_readiness("commit-mcp-applied", False) +except guard.UnsafePathError: + commit_without_owner_allowed = False +else: + commit_without_owner_allowed = True + guard._startup_ready_marker_absent = lambda: False try: guard._validate_action_readiness("seal-restart", False) @@ -995,7 +1029,11 @@ else: stale_marker_error = "" print(json.dumps({ + "commit_allowed": commit_allowed, + "commit_without_owner_allowed": commit_without_owner_allowed, "host_allowed": host_allowed, + "inspect_allowed": inspect_allowed, + "inspect_without_owner_allowed": inspect_without_owner_allowed, "startup_allowed": startup_allowed, "startup_without_owner_allowed": startup_without_owner_allowed, "stale_marker_error": stale_marker_error, @@ -1004,7 +1042,11 @@ print(json.dumps({ expect(result.status, result.stderr).toBe(0); expect(JSON.parse(result.stdout)).toEqual({ + commit_allowed: true, + commit_without_owner_allowed: false, host_allowed: true, + inspect_allowed: true, + inspect_without_owner_allowed: false, startup_allowed: true, startup_without_owner_allowed: false, stale_marker_error: "Hermes runtime config guard refuses mutation under a foreign PID 1", diff --git a/test/hermes-start-config-integrity.test.ts b/test/hermes-start-config-integrity.test.ts index 6aa6d6d5c93..5f9e2e246d1 100644 --- a/test/hermes-start-config-integrity.test.ts +++ b/test/hermes-start-config-integrity.test.ts @@ -22,7 +22,7 @@ function extractShellFunctionFromSource(src: string, name: string): string { return `${name}() {${match?.[1] ?? ""}\n}`; } -function runHermesConfigIntegrityVerifierAsRoot() { +function runHermesConfigIntegrityVerifierAsRoot(inspectStatus: 0 | 1) { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-integrity-")); const scriptPath = path.join(tmpDir, "run.sh"); const src = fs.readFileSync(START_SCRIPT, "utf-8"); @@ -37,11 +37,13 @@ function runHermesConfigIntegrityVerifierAsRoot() { "set -euo pipefail", 'id() { if [ "${1:-}" = "-u" ]; then printf "0\\n"; else command id "$@"; fi; }', 'verify_config_integrity() { printf "verify:%s:%s:stepped=%s\\n" "$1" "$2" "${NEMOCLAW_TEST_STEPPED_DOWN:-0}"; }', + `inspect_hermes_mcp_integrity() { return ${inspectStatus}; }`, extractShellFunctionFromSource(src, "verify_hermes_config_integrity"), `HERMES_DIR=${shellQuote(hermesHome)}`, `HERMES_HASH_FILE=${shellQuote(hashFile)}`, "STEP_DOWN_PREFIX_SANDBOX=(env NEMOCLAW_TEST_STEPPED_DOWN=1)", - "verify_hermes_config_integrity", + "HERMES_RESTART_FAILURE_CODE=internal", + 'if verify_hermes_config_integrity; then printf "result=success failure-code=%s\\n" "$HERMES_RESTART_FAILURE_CODE"; else printf "result=failure failure-code=%s\\n" "$HERMES_RESTART_FAILURE_CODE"; fi', ].join("\n"), { mode: 0o700 }, ); @@ -173,10 +175,19 @@ function runLockedParentStartupPreflight(parentMetadata: string) { describe("agents/hermes/start.sh config integrity", () => { it("verifies the strict Hermes hash through the sandbox identity in root mode", () => { - const result = runHermesConfigIntegrityVerifierAsRoot(); + const result = runHermesConfigIntegrityVerifierAsRoot(0); expect(result.status).toBe(0); expect(result.stderr).toBe(""); - expect(result.stdout.trim()).toMatch(/:stepped=1$/); + expect(result.stdout).toMatch(/:stepped=1$/m); + expect(result.stdout).toContain("result=success failure-code=internal"); + }); + + it("classifies failed MCP integrity inspection as an MCP restart failure", () => { + const result = runHermesConfigIntegrityVerifierAsRoot(1); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toMatch(/:stepped=1$/m); + expect(result.stdout).toContain("result=failure failure-code=mcp-integrity"); }); it("prepares root dashboard home and seeds config through the sandbox identity", { diff --git a/test/hermes-start.test.ts b/test/hermes-start.test.ts index 05efb79c316..e8686ae666b 100644 --- a/test/hermes-start.test.ts +++ b/test/hermes-start.test.ts @@ -8,6 +8,10 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { shellQuote } from "../src/lib/core/shell-quote"; +import { + bashPrintfQ, + extractShellFunction as extractShellFunctionFromSource, +} from "./support/hermes-shell-harness"; const START_SCRIPT = path.join(import.meta.dirname, "..", "agents", "hermes", "start.sh"); const SECRET_BOUNDARY_VALIDATOR_SCRIPT = path.join( @@ -21,31 +25,6 @@ const GENERATED_API_SERVER_KEY = Array.from({ length: 64 }, (_value, index) => (index % 16).toString(16), ).join(""); -function bashPrintfQ(value: string): string { - const result = spawnSync("bash", ["-c", "printf '%q' \"$1\"", "bash-printf-q", value], { - encoding: "utf-8", - timeout: 5000, - env: process.env, - }); - if (result.status !== 0) { - throw new Error(`bash printf %q failed: ${result.stderr}`); - } - return result.stdout; -} - -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -function extractShellFunctionFromSource(src: string, name: string): string { - const escapedName = escapeRegExp(name); - const match = src.match(new RegExp(`${escapedName}\\(\\) \\{([\\s\\S]*?)^\\}`, "m")); - if (!match) { - throw new Error(`Expected ${name} in agents/hermes/start.sh`); - } - return `${name}() {${match[1]}\n}`; -} - function extractRuntimeShellEnvBlock(src: string): string { const start = src.indexOf("write_runtime_shell_env() {"); const end = src.indexOf("\nwrite_runtime_shell_env\n", start); @@ -1032,6 +1011,49 @@ describe("agents/hermes/start.sh env secret boundary", () => { expect(result.stderr).not.toContain(rawToken); }); + it("checks the .env secret boundary before MCP integrity", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + 'trace() { printf "%s\\n" "$1"; }', + "verify_config_integrity_if_locked() { trace integrity; }", + "validate_hermes_env_secret_boundary() { trace env-boundary; }", + "inspect_hermes_mcp_integrity() { trace mcp-integrity; }", + "ensure_hermes_runtime_api_server_key() { trace api-key; }", + "apply_shields_up_runtime_env() { trace shields-env; }", + "validate_hermes_runtime_env_secret_boundary() { trace runtime-boundary; }", + "refresh_hermes_provider_placeholders() { trace placeholders; }", + "refresh_hermes_runtime_config_hashes() { trace hashes; }", + "configure_messaging_channels() { trace channels; }", + "retry_tirith_marker_if_needed() { trace tirith; }", + extractShellFunctionFromSource(source, "prepare_hermes_nonroot_runtime"), + "HERMES_DIR=/sandbox/.hermes; prepare_hermes_nonroot_runtime", + ].join("\n"), + ], + { encoding: "utf-8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "integrity", + "env-boundary", + "mcp-integrity", + "api-key", + "shields-env", + "env-boundary", + "runtime-boundary", + "placeholders", + "hashes", + "mcp-integrity", + "channels", + "tirith", + ]); + }); + it("rejects bare API-named raw values without printing the value", () => { const rawToken = "SENTINEL_RAW_SECRET_VALUE"; const result = runHermesEnvSecretBoundary({ diff --git a/test/mcp-add-crash-consistency.test.ts b/test/mcp-add-crash-consistency.test.ts index a3b09baa5ca..e3fdb2f31e3 100644 --- a/test/mcp-add-crash-consistency.test.ts +++ b/test/mcp-add-crash-consistency.test.ts @@ -8,6 +8,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.72"); + type CrashBoundary = | "provider" | "policy" @@ -203,7 +205,7 @@ bridge.addMcpBridge("crash-test", { return spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, timeout: 30_000, }); } @@ -305,7 +307,7 @@ bridge.removeMcpBridge("crash-test", "fake").then( return spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, timeout: 30_000, }); } @@ -361,7 +363,7 @@ bridge.statusMcpBridge("crash-test", "fake").then( return spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, timeout: 30_000, }); } @@ -721,7 +723,7 @@ bridge.removeMcpBridge("crash-test", "fake", { force: true }).then( const cancelled = spawnSync(process.execPath, ["-e", cancelScript], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, timeout: 30_000, }); expect(cancelled.status, `${cancelled.stdout}\n${cancelled.stderr}`).toBe(0); @@ -758,8 +760,17 @@ describe("MCP remove crash consistency", () => { expect(resumed.status, `${resumed.stdout}\n${resumed.stderr}`).toBe(0); const registry = JSON.parse( fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), - ) as { sandboxes: { "crash-test": { mcp?: unknown } } }; - expect(registry.sandboxes["crash-test"].mcp).toBeUndefined(); + ) as { + sandboxes: { + "crash-test": { + mcp?: { bridges: Record; managedServerNames: string[] }; + }; + }; + }; + expect(registry.sandboxes["crash-test"].mcp).toEqual({ + bridges: {}, + managedServerNames: ["fake"], + }); } finally { fs.rmSync(home, { recursive: true, force: true }); } diff --git a/test/mcp-destroy-lifecycle.test.ts b/test/mcp-destroy-lifecycle.test.ts index 9ac200ec20e..afa69d1a9aa 100644 --- a/test/mcp-destroy-lifecycle.test.ts +++ b/test/mcp-destroy-lifecycle.test.ts @@ -8,6 +8,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.72"); + function runDestroyLifecycleScenario(body: string) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-destroy-")); const script = ` @@ -181,7 +183,7 @@ ${body} const result = spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, }); fs.rmSync(home, { recursive: true, force: true }); return result; @@ -390,7 +392,10 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); registry.registerSandbox({ name: "alpha", agent: "openclaw", - mcp: { bridges: { github: bridgeEntries.github } }, + mcp: { + bridges: { github: bridgeEntries.github }, + managedServerNames: ["github", "retired"], + }, }); registry.addCustomPolicy("alpha", ownedPolicy("github")); const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); @@ -429,7 +434,10 @@ process.env.GITHUB_TOKEN = "ambient-value-that-must-not-rotate"; registry.registerSandbox({ name: "alpha", agent: "openclaw", - mcp: { bridges: { github: bridgeEntries.github } }, + mcp: { + bridges: { github: bridgeEntries.github }, + managedServerNames: ["github", "retired"], + }, }); registry.addCustomPolicy("alpha", ownedPolicy("github")); const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); @@ -452,6 +460,7 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); sandbox: { mcp: { bridges: Record; + managedServerNames?: string[]; destroyPreparedAt?: string; destroyPendingAt?: string; }; @@ -474,6 +483,7 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); payload.adapterCalls.some((call) => call.includes("openshell:resolve:env:GITHUB_TOKEN")), ).toBe(true); expect(payload.sandbox.mcp.bridges).toHaveProperty("github"); + expect(payload.sandbox.mcp.managedServerNames).toEqual(["github", "retired"]); expect(payload.sandbox.mcp.destroyPreparedAt).toBeUndefined(); expect(payload.sandbox.mcp.destroyPendingAt).toBeUndefined(); }); @@ -483,7 +493,10 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); registry.registerSandbox({ name: "alpha", agent: "openclaw", - mcp: { bridges: { github: bridgeEntries.github } }, + mcp: { + bridges: { github: bridgeEntries.github }, + managedServerNames: ["github", "retired"], + }, }); registry.addCustomPolicy("alpha", ownedPolicy("github")); const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); @@ -509,13 +522,18 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); const payload = JSON.parse(result.stdout) as { error: string; sandbox: { - mcp: { bridges: Record; destroyPreparedAt?: string }; + mcp: { + bridges: Record; + managedServerNames?: string[]; + destroyPreparedAt?: string; + }; }; attached: string[]; adapterRegistered: boolean; }; expect(payload.error).toMatch(/failed to activate generated MCP policy/i); expect(payload.sandbox.mcp.bridges).toHaveProperty("github"); + expect(payload.sandbox.mcp.managedServerNames).toEqual(["github", "retired"]); expect(payload.sandbox.mcp.destroyPreparedAt).toBeTruthy(); expect(payload.attached).not.toContain("alpha-mcp-github"); expect(payload.adapterRegistered).toBe(false); @@ -722,7 +740,10 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); registry.registerSandbox({ name: "alpha", agent: "openclaw", - mcp: { bridges: bridgeEntries }, + mcp: { + bridges: bridgeEntries, + managedServerNames: ["github", "retired", "slack"], + }, }); registry.addCustomPolicy("alpha", ownedPolicy("github")); registry.addCustomPolicy("alpha", ownedPolicy("slack")); @@ -757,6 +778,7 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); afterFailure: { mcp: { bridges: Record; + managedServerNames?: string[]; destroyPreparedAt?: string; destroyPendingAt?: string; }; @@ -770,6 +792,7 @@ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); expect(payload.firstError).toContain("provider delete failed"); expect(payload.afterFailure.mcp.destroyPendingAt).toBeTruthy(); expect(payload.afterFailure.mcp.destroyPreparedAt).toBeUndefined(); + expect(payload.afterFailure.mcp.managedServerNames).toEqual(["github", "retired", "slack"]); expect(Object.keys(payload.afterFailure.mcp.bridges)).toEqual(["github", "slack"]); expect(payload.afterFailure.customPolicies).toHaveLength(2); expect(payload.retry.destroyAlreadyPending).toBe(true); diff --git a/test/mcp-policy-key-ownership.test.ts b/test/mcp-policy-key-ownership.test.ts index f82cfa95c20..aef859677fe 100644 --- a/test/mcp-policy-key-ownership.test.ts +++ b/test/mcp-policy-key-ownership.test.ts @@ -8,6 +8,9 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.72"); +const MATCHING_OPENSHELL_VERSION_CLAUSE = `if [ "$1" = "--version" ]; then printf '%s\\n' 'openshell 0.0.72'; exit 0; fi`; + const PRESET = `network_policies: example: name: generated-policy @@ -25,6 +28,7 @@ function runApply( fs.writeFileSync( path.join(binDir, "openshell"), `#!/bin/sh +${MATCHING_OPENSHELL_VERSION_CLAUSE} printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} if [ "$1 $2" = "policy get" ]; then printf 'Version: 1\nHash: test\n---\nversion: 1\n${ @@ -74,6 +78,7 @@ function runContentMatch(liveName: string) { fs.writeFileSync( path.join(binDir, "openshell"), `#!/bin/sh +${MATCHING_OPENSHELL_VERSION_CLAUSE} printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n example:\n name: ${liveName}\n endpoints: []\n' `, { mode: 0o755 }, @@ -103,6 +108,7 @@ function runFailedPolicyMutation(operation: "apply" | "remove") { fs.writeFileSync( path.join(binDir, "openshell"), `#!/bin/sh +${MATCHING_OPENSHELL_VERSION_CLAUSE} if [ "$1 $2" = "policy get" ]; then printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n example:\n name: generated-policy\n endpoints: []\n' exit 0 @@ -167,6 +173,7 @@ function runSuccessfulPolicyRemoval(skipRegistryUpdate: boolean) { fs.writeFileSync( path.join(binDir, "openshell"), `#!/bin/sh +${MATCHING_OPENSHELL_VERSION_CLAUSE} if [ "$1 $2" = "policy get" ]; then printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n example:\n name: generated-policy\n endpoints: []\n' fi @@ -282,6 +289,7 @@ describe("MCP-generated network policy ownership", () => { fs.writeFileSync( path.join(binDir, "openshell"), `#!/bin/sh +${MATCHING_OPENSHELL_VERSION_CLAUSE} printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} if [ "$1 $2 $3" = "status --output json" ]; then printf '%s\n' 'ready' @@ -368,6 +376,7 @@ bridge.addMcpBridge("alpha", { fs.writeFileSync( path.join(binDir, "openshell"), `#!/bin/sh +${MATCHING_OPENSHELL_VERSION_CLAUSE} printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} if [ "$1 $2 $3" = "status --output json" ]; then printf '%s\n' 'ready' @@ -543,7 +552,7 @@ bridge.restartMcpBridge("alpha", "example").then( const result = spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, timeout: 30_000, }); fs.rmSync(home, { recursive: true, force: true }); diff --git a/test/mcp-restart-policy-order.test.ts b/test/mcp-restart-policy-order.test.ts index bc602143099..78dea8030e5 100644 --- a/test/mcp-restart-policy-order.test.ts +++ b/test/mcp-restart-policy-order.test.ts @@ -8,6 +8,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.72"); + describe("MCP restart policy ordering", () => { it("rejects a foreign attached credential key before policy or provider mutation", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-restart-order-")); @@ -110,7 +112,7 @@ bridge.restartMcpBridge("alpha", "example").then( const result = spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, timeout: 30_000, }); fs.rmSync(home, { recursive: true, force: true }); @@ -231,7 +233,7 @@ bridge.restartMcpBridge("alpha", "example").then( const result = spawnSync(process.execPath, ["-e", script], { cwd: process.cwd(), encoding: "utf8", - env: { ...process.env, HOME: home }, + env: { ...process.env, HOME: home, NEMOCLAW_OPENSHELL_BIN: MATCHING_OPENSHELL }, timeout: 30_000, }); fs.rmSync(home, { recursive: true, force: true }); diff --git a/test/registry.test.ts b/test/registry.test.ts index 5eb31803237..c9141bf8603 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -201,6 +201,25 @@ describe("registry", () => { expect(entry.token).toBeUndefined(); expect(entry.command).toBeUndefined(); expect(entry.port).toBeUndefined(); + expect(raw.sandboxes.alpha.mcp.managedServerNames).toEqual(["github"]); + }); + + it("retains sanitized managed MCP names after the active bridge map is emptied", () => { + registry.registerSandbox({ + name: "alpha", + agent: "hermes", + mcp: { + bridges: {}, + managedServerNames: ["retired", "../invalid", "retired", "still_active"], + }, + }); + + const stored = registry.getSandbox("alpha").mcp; + expect(stored).toEqual({ + bridges: {}, + managedServerNames: ["retired", "still_active"], + }); + expect(JSON.parse(fs.readFileSync(regFile, "utf-8")).sandboxes.alpha.mcp).toEqual(stored); }); it("normalizes MCP bridge maps by the recovered server name", () => { diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index b629e05dda2..faebe8884cd 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -1202,11 +1202,9 @@ describe("Hermes sandbox provisioning", () => { const bashrcPath = path.join(etcDir, "bash.bashrc"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); + const buildMcpDigestPath = path.join(localLib, "build-hermes-mcp-digest.py"); const mcpConfigTransactionPath = path.join(localLib, "hermes-mcp-config-transaction.py"); - const mcpCredentialBoundaryPath = path.join( - localLib, - "openshell-child-visible-credentials.v0.0.72.json", - ); + const mcpManifest = path.join(localLib, "openshell-child-visible-credentials.v0.0.72.json"); const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); const managedGatewayControlPath = path.join(localLib, "managed-gateway-control.py"); const files = [ @@ -1216,8 +1214,9 @@ describe("Hermes sandbox provisioning", () => { path.join(localLib, "validate-hermes-env-secret-boundary.py"), path.join(localLib, "seed-hermes-dashboard-config.py"), path.join(localLib, "hermes-runtime-config-guard.py"), + buildMcpDigestPath, mcpConfigTransactionPath, - mcpCredentialBoundaryPath, + mcpManifest, gatewaySupervisorPath, stateDirGuardPath, managedGatewayControlPath, @@ -1246,11 +1245,12 @@ describe("Hermes sandbox provisioning", () => { expect(result.status, result.stderr).toBe(0); expect(calls).toContain( - `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${managedGatewayControlPath} ${mcpCredentialBoundaryPath}`, + `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${managedGatewayControlPath} ${buildMcpDigestPath} ${mcpManifest}`, ); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(mcpConfigTransactionPath).mode & 0o777).toString(8)).toBe("755"); - expect((fs.statSync(mcpCredentialBoundaryPath).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(mcpManifest).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(buildMcpDigestPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(managedGatewayControlPath).mode & 0o777).toString(8)).toBe("500"); diff --git a/test/sandbox-rlimit-hooks.test.ts b/test/sandbox-rlimit-hooks.test.ts index 83f2073ca15..dbf861969ea 100644 --- a/test/sandbox-rlimit-hooks.test.ts +++ b/test/sandbox-rlimit-hooks.test.ts @@ -408,6 +408,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { const validator = path.join(localLib, "validate-hermes-env-secret-boundary.py"); const dashboardSeeder = path.join(localLib, "seed-hermes-dashboard-config.py"); const runtimeGuard = path.join(localLib, "hermes-runtime-config-guard.py"); + const buildMcpDigest = path.join(localLib, "build-hermes-mcp-digest.py"); const mcpTransaction = path.join(localLib, "hermes-mcp-config-transaction.py"); const mcpCredentialBoundary = path.join( localLib, @@ -432,6 +433,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { fs.writeFileSync(validator, "# validator fixture\n"); fs.writeFileSync(dashboardSeeder, "# dashboard seeder fixture\n"); fs.writeFileSync(runtimeGuard, "# runtime guard fixture\n"); + fs.writeFileSync(buildMcpDigest, "# build MCP digest fixture\n"); fs.writeFileSync(mcpTransaction, "# MCP transaction fixture\n"); fs.writeFileSync(mcpCredentialBoundary, "{}\n"); fs.mkdirSync(preloadDir, { mode: 0o777 }); @@ -459,6 +461,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { .replaceAll("/usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py", validator) .replaceAll("/usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py", dashboardSeeder) .replaceAll("/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", runtimeGuard) + .replaceAll("/usr/local/lib/nemoclaw/build-hermes-mcp-digest.py", buildMcpDigest) .replaceAll("/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", mcpTransaction) .replaceAll( "/usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json", @@ -491,6 +494,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { expect(hardenedSafetyNet.mode & 0o777).toBe(0o444); expect(hardenedCiaoGuard.mode & 0o777).toBe(0o444); expect(fs.statSync(mcpCredentialBoundary).mode & 0o777).toBe(0o444); + expect(fs.statSync(buildMcpDigest).mode & 0o777).toBe(0o444); expect(hardenedDir.uid).toBe(fixtureOwner.uid); expect(hardenedDir.gid).toBe(fixtureOwner.gid); expect(hardenedSafetyNet.uid).toBe(fixtureOwner.uid); diff --git a/test/support/connect-flow-test-harness.ts b/test/support/connect-flow-test-harness.ts index bd16436537d..5738d2dd0cf 100644 --- a/test/support/connect-flow-test-harness.ts +++ b/test/support/connect-flow-test-harness.ts @@ -42,6 +42,8 @@ export type ConnectHarnessOptions = { forwardRecoveryFailureDetail?: string; secretBoundaryRefused?: boolean; secretBoundaryReason?: SecretBoundaryRefusalReason; + mcpReconciliationRefused?: boolean; + mcpReconciliationReason?: string; }; spawnSignal?: NodeJS.Signals | null; spawnStatus?: number | null; diff --git a/test/support/hermes-shell-harness.ts b/test/support/hermes-shell-harness.ts new file mode 100644 index 00000000000..1768934a6d5 --- /dev/null +++ b/test/support/hermes-shell-harness.ts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +export function bashPrintfQ(value: string): string { + const result = spawnSync("bash", ["-c", "printf '%q' \"$1\"", "bash-printf-q", value], { + encoding: "utf-8", + timeout: 5000, + env: process.env, + }); + if (result.status !== 0) throw new Error(`bash printf %q failed: ${result.stderr}`); + return result.stdout; +} + +export function extractShellFunction(source: string, name: string): string { + const match = source.match(new RegExp(`${escapeRegExp(name)}\\(\\) \\{([\\s\\S]*?)^\\}`, "m")); + if (!match) throw new Error(`Expected shell function ${name}`); + return `${name}() {${match[1]}\n}`; +} + +export function runHermesBashHarness( + lines: string[], + configure?: (tmpDir: string) => Record, +) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-supervisor-test-")); + const script = path.join(tmpDir, "run.sh"); + fs.writeFileSync( + script, + [ + "#!/usr/bin/env bash", + "set -uo pipefail", + "HERMES_MCP_RECONCILE_PENDING=0", + "HERMES_MCP_INTEGRITY_FAILED=0", + ...lines, + ].join("\n"), + { mode: 0o700 }, + ); + + try { + return spawnSync("bash", [script], { + encoding: "utf-8", + timeout: 5000, + env: { ...process.env, ...configure?.(tmpDir) }, + }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} diff --git a/test/update-hermes-agent-script.test.ts b/test/update-hermes-agent-script.test.ts index 8adc842dd76..d173dee170d 100644 --- a/test/update-hermes-agent-script.test.ts +++ b/test/update-hermes-agent-script.test.ts @@ -30,6 +30,8 @@ const CURRENT_INSTALLED_BASE = [ const CURRENT_INSTALLED_DOCKERFILE = [ "COPY agents/hermes/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py", "COPY agents/hermes/seed-dashboard-config.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py", + "COPY agents/hermes/build-mcp-digest.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py", + 'RUN mcp_digest="$(/opt/hermes/.venv/bin/python -I /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py --guard /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py --config /sandbox/.hermes/config.yaml)"', "COPY agents/hermes/mcp-config-transaction.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", "COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json", "RUN HERMES_HOME=/sandbox/.hermes /usr/local/bin/hermes doctor --fix \\", @@ -273,7 +275,7 @@ fi ); const installedAgentDockerfile = path.join(path.dirname(installedDockerfile), "Dockerfile"); const preMcpDockerfile = CURRENT_INSTALLED_DOCKERFILE.replace( - /^COPY (?:agents\/hermes\/mcp-config-transaction\.py|src\/lib\/actions\/sandbox\/openshell-child-visible-credentials\.v0\.0\.72\.json) .*\n/gm, + /^(?:COPY (?:agents\/hermes\/(?:build-mcp-digest|mcp-config-transaction)\.py|src\/lib\/actions\/sandbox\/openshell-child-visible-credentials\.v0\.0\.72\.json) .*|RUN mcp_digest=.*build-hermes-mcp-digest\.py.*)\n/gm, "", ); fs.mkdirSync(path.dirname(installedDockerfile), { recursive: true }); @@ -299,6 +301,8 @@ fi expect(run.stdout).toContain("INVALID: installed copy"); expect(run.stdout).toContain("marker hermes-mcp-config-transaction.py"); expect(run.stdout).toContain("marker openshell-child-visible-credentials.v0.0.72.json"); + expect(run.stdout).toContain("marker COPY agents/hermes/build-mcp-digest.py"); + expect(run.stdout).toContain("marker /opt/hermes/.venv/bin/python -I"); expect(fs.readFileSync(installedDockerfile, "utf-8")).toBe(CURRENT_INSTALLED_BASE); expect(fs.readFileSync(installedAgentDockerfile, "utf-8")).toBe(preMcpDockerfile); } finally {