diff --git a/agents/hermes/runtime-config-guard.py b/agents/hermes/runtime-config-guard.py index 3f1e8f2fc5a..f6d2c5d2795 100755 --- a/agents/hermes/runtime-config-guard.py +++ b/agents/hermes/runtime-config-guard.py @@ -1326,10 +1326,12 @@ def refresh_hashes( mode: str, mcp_transition: str = "preserve", ) -> None: - """Advance the durable MCP intended/applied state without blessing drift. + """Advance the durable MCP intended/applied state around one config snapshot. - ``preserve`` requires current config to equal intended. ``intend`` records - current config as the next intent while retaining the last applied digest. + ``preserve`` requires current MCP config to equal intended. ``adopt`` records + Hermes-owned MCP config as the next intent and supersedes stale host intent. + ``intend`` records current config as the next managed intent while retaining + the last applied digest. ``rollback`` requires restored config to equal the prior applied digest, then conservatively records restored/failed-candidate until reload health is proven. ``apply`` is a metadata-only intended/intended commit and requires @@ -1341,7 +1343,7 @@ def refresh_hashes( config_path = os.path.join(hermes_dir, "config.yaml") env_path = os.path.join(hermes_dir, ".env") compat_hash = os.path.join(hermes_dir, ".config-hash") - if mcp_transition not in {"preserve", "intend", "rollback", "apply"}: + if mcp_transition not in {"preserve", "adopt", "intend", "rollback", "apply"}: raise UnsafePathError("refusing unsupported Hermes MCP hash transition") # Snapshot-stability/TOCTOU contract: derive the config hash and canonical @@ -1387,6 +1389,9 @@ def refresh_hashes( raise UnsafePathError( "Hermes MCP config differs from persisted intended state" ) + elif mcp_transition == "adopt": + if not secrets.compare_digest(current_mcp, state.intended): + state = McpHashState(current_mcp, state.applied) elif mcp_transition == "intend": if state.intended != state.applied and not secrets.compare_digest( current_mcp, state.intended @@ -1433,6 +1438,28 @@ def assert_inputs_stable() -> None: config.close() env.close() + # Restart validates once before sealing and once after. Keep a current + # anchor on the same inode so the second pass cannot invalidate seal state. + if mcp_transition == "adopt" and secrets.compare_digest( + hash_text, source_hash_text + ): + compatibility_matches = True + if mode == "both": + try: + compatibility_matches = secrets.compare_digest( + _read_hash_file(compat_hash), source_hash_text + ) + except FileNotFoundError: + compatibility_matches = False + if compatibility_matches: + integrity = inspect_mcp_integrity_snapshot( + hermes_dir, + state_path, + compat_hash if mode == "both" else None, + ) + assert_mcp_integrity_snapshot_current(integrity) + return + # `both` is the transaction contract: both trust anchors must advance or # the caller rolls the config write back. `compat` remains best-effort for # legacy startup paths where an old image can expose a read-only in-tree @@ -3361,6 +3388,9 @@ def main() -> int: parser.add_argument( "--mode", choices=("strict", "compat", "both"), default="strict" ) + parser.add_argument( + "--mcp-transition", choices=("preserve", "adopt"), default="preserve" + ) parser.add_argument("--state-file", default="") parser.add_argument("--expected-config-sha256", default="") parser.add_argument("--lock-token", default="") @@ -3375,6 +3405,8 @@ def main() -> int: raise UnsafePathError( "--mcp-state-exit-code requires inspect-mcp-integrity" ) + if args.mcp_transition != "preserve" and args.action != "refresh-hashes": + raise UnsafePathError("--mcp-transition requires refresh-hashes") _validate_action_readiness(args.action, args.startup_owner) if args.action == "ensure-api-key": if not args.hash_file: @@ -3383,7 +3415,12 @@ def main() -> int: elif args.action == "refresh-hashes": if not args.hash_file: raise UnsafePathError("refresh-hashes requires --hash-file") - refresh_hashes(args.hermes_dir, args.hash_file, args.mode) + refresh_hashes( + args.hermes_dir, + args.hash_file, + args.mode, + mcp_transition=args.mcp_transition, + ) elif args.action == "inspect-mcp-integrity": if not args.hash_file: raise UnsafePathError("inspect-mcp-integrity requires --hash-file") diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index c15388ee541..3b47b1e4a3b 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -11,9 +11,10 @@ # - Gateway listens on internal port 18642, socat forwards the API to 8642 # - Dashboard listens on a private loopback port, socat forwards it to 18789 # -# SECURITY: The gateway runs as a separate user so the sandboxed agent cannot -# kill it or restart it with a tampered config. Config hash is verified at -# startup to detect tampering. +# SECURITY: The direct-root gateway runs as a separate user so the sandboxed +# agent cannot control its process lifecycle. Hermes config remains mutable; +# restart transactions validate its paths, secret boundary, and managed MCP +# state before a replacement consumes it. set -euo pipefail @@ -336,7 +337,6 @@ HERMES_RESTART_SEALED=0 HERMES_RESTART_UNSEALING=0 HERMES_RESTART_SIGNAL_PENDING=0 HERMES_MCP_RECONCILE_PENDING=0 -HERMES_MCP_INTEGRITY_FAILED=0 # A same-container PID 1 restart can retain /run. Revoke the prior readiness # lease before any startup migration or mutable config read; host mutations are @@ -554,12 +554,13 @@ hermes_fatal_unproven_child() { # In managed OpenShell, exiting this non-root supervisor would leave PID 1 # and the unproven child alive. Bash's job table can still wait for the exact # `$!` child without treating a reused numeric PID as authority to signal it. - # Quarantine the supervisor after that child exits; only sandbox destruction - # may tear down a process tree whose identities could not be established. + # Quarantine this supervisor after that child exits. A sandbox stop/start + # replaces the supervisor through the sandbox lifecycle without asking this + # process to signal a child whose identity it could not establish. echo "[CRITICAL] Newly launched Hermes ${role} pid ${pid} failed exact role identity capture; quarantining the managed startup supervisor without signaling the unproven child" >&2 trap ':' TERM INT wait "$pid" 2>/dev/null || true - echo "[CRITICAL] Unproven Hermes ${role} child exited; managed supervisor remains quarantined until sandbox recreation" >&2 + echo "[CRITICAL] Unproven Hermes ${role} child exited; relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox" >&2 while :; do sleep 60 || true done @@ -2018,11 +2019,13 @@ refresh_hermes_provider_placeholders() { refresh_hermes_runtime_config_hashes() { local mode="${1:-strict}" + local mcp_transition="${2:-preserve}" local cmd=( "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" refresh-hashes --hermes-dir "$HERMES_DIR" --hash-file "$HERMES_HASH_FILE" --mode "$mode" + --mcp-transition "$mcp_transition" --startup-owner ) if [ "$mode" = "compat" ] && [ "$(id -u)" -eq 0 ]; then @@ -2070,12 +2073,10 @@ inspect_hermes_mcp_integrity() { 0) HERMES_MCP_RECONCILE_PENDING=0 ;; 10) HERMES_MCP_RECONCILE_PENDING=1 ;; *) - HERMES_MCP_INTEGRITY_FAILED=1 echo "[SECURITY] HERMES_MCP_CONFIG_DRIFT: MCP intent cannot be matched to the persisted gateway state; rebuild the sandbox from its NemoClaw registry state" >&2 return 1 ;; esac - HERMES_MCP_INTEGRITY_FAILED=0 } commit_hermes_mcp_applied_if_pending() { @@ -2204,13 +2205,14 @@ prepare_hermes_gateway_restart() { return 1 fi - # A restart is a lifecycle action, not authority to bless arbitrary bytes - # written by the sandbox user. Supported host config commands refresh the - # root-owned strict hash when they make a change; direct in-sandbox edits do - # not. Require that trusted anchor instead of chowning attacker-controlled - # paths or adopting a new hash here. + # Hermes owns its mutable config. Adopt one stable snapshot before sealing + # restart inputs. A direct MCP change becomes pending and is committed only + # after replacement health. Host reconciliation reports any registry mismatch + # without making that host state a precondition for Hermes to run. HERMES_RESTART_FAILURE_CODE=hash-mismatch - verify_hermes_config_integrity || return 1 + refresh_hermes_runtime_config_hashes both adopt || return 1 + HERMES_RESTART_FAILURE_CODE=mcp-integrity + inspect_hermes_mcp_integrity "$HERMES_HASH_FILE" || return 1 prepare_hermes_lazy_dependencies } @@ -2638,14 +2640,12 @@ handle_hermes_gateway_control_request() { local failure_code if [ "$GATEWAY_CONTROL_ACTION" = "probe" ]; then - if ! prepare_hermes_gateway_restart; then + # Probe verifies the running process and credential boundary. It does not + # adopt mutable config or change MCP transaction state. + if ! validate_running_hermes_boundary; then gateway_control_fail "$HERMES_RESTART_FAILURE_CODE" "$old_pid" return 1 fi - if [ "$HERMES_MCP_RECONCILE_PENDING" -eq 1 ]; then - gateway_control_fail mcp-reconcile-required "$old_pid" - return 1 - fi if ! gateway_control_pid_is_live "$old_pid" \ || ! hermes_gateway_healthy "$old_pid" \ || hermes_auxiliaries_need_recovery; then @@ -2660,8 +2660,8 @@ handle_hermes_gateway_control_request() { && gateway_control_pid_is_live "$old_pid" \ && hermes_gateway_healthy "$old_pid"; then # Recovery may also recreate the dashboard from the shared Hermes config. - # Verify the root-owned trust anchor before any auxiliary consumes it; a - # healthy gateway is not authority to bless direct sandbox config drift. + # Adopt one stable snapshot before any auxiliary consumes current config; + # the old gateway's health does not prove that snapshot stayed unchanged. if ! prepare_hermes_gateway_restart; then if hermes_restart_failure_revokes_gateway "$HERMES_RESTART_FAILURE_CODE"; then stop_hermes_gateway_fail_closed @@ -2809,10 +2809,9 @@ prepare_hermes_nonroot_runtime() { # startup mutations below so their outputs remain covered as well. validate_hermes_env_secret_boundary || return 1 # The non-root Hermes runtime can persist safe config/env changes while it is - # running. Reconcile that mutable compatibility anchor only after the secret - # boundary is valid; refresh-hashes still requires the recorded MCP intent to - # match exactly before it advances the anchor. - refresh_hermes_runtime_config_hashes compat || return 1 + # running. Adopt one stable snapshot only after the secret boundary is valid. + # Direct MCP drift becomes pending until the replacement gateway is healthy. + refresh_hermes_runtime_config_hashes compat adopt || return 1 inspect_hermes_mcp_integrity "${HERMES_DIR}/.config-hash" || return 1 prepare_hermes_lazy_dependencies || return 1 ensure_hermes_runtime_api_server_key compat || return 1 @@ -2920,7 +2919,10 @@ publish_hermes_root_runtime_marker() { } prepare_hermes_root_runtime() { - verify_hermes_config_integrity || return 1 + validate_hermes_env_secret_boundary || return 1 + validate_hermes_runtime_env_secret_boundary || return 1 + refresh_hermes_runtime_config_hashes both adopt || return 1 + inspect_hermes_mcp_integrity "$HERMES_HASH_FILE" || return 1 prepare_hermes_lazy_dependencies || return 1 ensure_hermes_config_root_mode || return 1 ensure_hermes_runtime_api_server_key both || return 1 @@ -3075,26 +3077,27 @@ record_hermes_managed_gateway_exit() { HERMES_MANAGED_GATEWAY_EXIT_TIMES=("${retained[@]+"${retained[@]}"}") HERMES_MANAGED_GATEWAY_EXIT_COUNT=${#HERMES_MANAGED_GATEWAY_EXIT_TIMES[@]} if [ "$HERMES_MANAGED_GATEWAY_EXIT_COUNT" -ge 5 ]; then - echo "[gateway] CRITICAL: $HERMES_MANAGED_GATEWAY_EXIT_COUNT exits in 60s window — Hermes relaunch is quarantined until sandbox recreation; check /tmp/gateway.log" >&2 + echo "[gateway] CRITICAL: $HERMES_MANAGED_GATEWAY_EXIT_COUNT exits in 60s window — Hermes relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox; check /tmp/gateway.log" >&2 quarantine_hermes_managed_gateway_relaunch return 1 fi } recover_hermes_gateway_current_user() { - local replacement_reached_internal_health + local replacement_reached_internal_health preparation_failures=0 preparation_failure_limit=5 while :; do replacement_reached_internal_health=0 until prepare_hermes_nonroot_runtime; do - if [ "$HERMES_MCP_INTEGRITY_FAILED" -eq 1 ]; then - echo "[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox" >&2 - quarantine_hermes_managed_gateway_relaunch + preparation_failures=$((preparation_failures + 1)) + if [ "$preparation_failures" -ge "$preparation_failure_limit" ]; then + echo "[gateway] Hermes runtime preparation failed after ${preparation_failures} consecutive attempts; supervisor exiting without launching a gateway; correct the reported failure, then stop and start the sandbox" >&2 return 1 fi echo "[gateway] Hermes runtime preparation refused automatic respawn; retrying in 5s" >&2 sleep 5 || true done + preparation_failures=0 if ! launch_hermes_gateway_current_user; then echo "[gateway] Hermes gateway launch failed; retrying under the same supervisor" >&2 sleep 5 || true diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 8a4b056e257..01bfd40a8b2 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -44,7 +44,7 @@ "src/lib/actions/sandbox/gateway-state.ts": 21, "src/lib/actions/sandbox/status-snapshot.ts": 19, "src/lib/actions/sandbox/policy-channel.ts": 30, - "src/lib/actions/sandbox/process-recovery.ts": 20, + "src/lib/actions/sandbox/process-recovery.ts": 19, "src/lib/actions/sandbox/rebuild-pipeline.ts": 30, "src/lib/actions/sandbox/snapshot.ts": 37, "src/lib/actions/uninstall/run-plan.ts": 25, @@ -62,7 +62,7 @@ "maxRootFiles": { "src/lib/onboard": 301, "src/lib/actions": 18, - "src/lib/actions/sandbox": 178, + "src/lib/actions/sandbox": 177, "src/lib/state": 38, "src/lib/inference": 63, "scripts": 42 diff --git a/docs/manage-sandboxes/gateway-lifecycle-control.mdx b/docs/manage-sandboxes/gateway-lifecycle-control.mdx index 6731f9184fd..44cb842bf50 100644 --- a/docs/manage-sandboxes/gateway-lifecycle-control.mdx +++ b/docs/manage-sandboxes/gateway-lifecycle-control.mdx @@ -25,12 +25,16 @@ For `recover` and `gateway restart`, the managed controller acquires the expecte Lock acquisition, gateway termination, and replacement health share one recovery deadline. If lock acquisition reaches that deadline, the controller returns `SUPERVISOR_BUSY` without publishing an expected-exit marker. -Mutable managed config retains the trust and time-of-check/time-of-use limits of managed cold start. -In the direct root-entrypoint topology, PID 1 validates the Hermes secret boundary and runtime environment and verifies the strict root-owned hash without recomputing it. -For Hermes, the managed controller preflights the secret boundary before it signals the observed child. - -Mutable config in the managed topology has no durable root-owned hash anchor, so a restart cannot promise a `config hash mismatch` for direct drift. +Hermes configuration remains mutable in both topologies. +Before a restart, the supervisor validates the secret boundary and records one stable config snapshot in its transaction metadata. +A direct MCP change becomes pending and supersedes stale host-managed intent. +The supervisor marks that change as applied only after the replacement gateway passes its health checks. +The host operation that owns a managed MCP transaction can report a registry mismatch after Hermes is healthy. + +The direct root-entrypoint topology keeps the integrity metadata under root ownership while it seals a restart transaction. +That metadata does not make the complete Hermes config a relaunch allowlist. +The managed topology has no durable root-owned config anchor because the supervisor, gateway, and agent share one UID. This compatibility path remains necessary while the OpenShell-managed topology owns a nonroot supervisor and shared gateway-agent UID. @@ -50,9 +54,12 @@ Custom agents that recover through an SSH script do not use this controller prob The nonroot Hermes supervisor continuously repairs the gateway, API relay, dashboard, dashboard relay, and gateway log stream. Four consecutive gateway health failures trigger recovery of the observed gateway child. -Five unexpected gateway exits or failed replacement candidates within 60 seconds quarantine relaunch until the sandbox is recreated. +Five unexpected gateway exits or failed replacement candidates within 60 seconds stop relaunch for the current supervisor instance. An authenticated host action authorizes one exit bound to the gateway process ID and kernel start identity while the root controller process remains live, so deliberate `gateway restart` and controller-driven replacement do not consume that crash budget. +Correct the reported process or health failure, then stop and start the sandbox to reset the supervisor. +Rebuild only if the sandbox still cannot start. + The authorization records host intent for that exit; it does not claim that the host signal was the only possible cause of process termination in the shared-UID topology. After the in-sandbox processes are healthy, the host repairs only the host-side OpenShell forwards. diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 83709a66430..fe42b368748 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -138,7 +138,11 @@ $$nemoclaw gateway restart The restart command asks the topology-specific controller to stop the tracked gateway child, wait for the entrypoint to launch a replacement, and prove listener and HTTP health. The host then checks or recovers host-side dashboard, messaging, and agent forwards. -For Hermes, the entrypoint supervisor owns the gateway, dashboard process, internal API relay, dashboard relay, and gateway log stream. The nonroot managed supervisor repairs those processes continuously, stops an alive but deaf gateway after four consecutive failed health checks, and quarantines relaunch after five exits within 60 seconds until sandbox recreation. +For Hermes, the entrypoint supervisor owns the gateway, dashboard process, internal API relay, dashboard relay, and gateway log stream. +The nonroot managed supervisor repairs those processes continuously and stops an alive but deaf gateway after four consecutive failed health checks. +Five unexpected gateway exits or failed replacement candidates within 60 seconds stop relaunch for the current supervisor instance. Host-authorized exits do not consume this crash budget. +Correct the reported failure, then run `$$nemoclaw stop` and `$$nemoclaw start` to reset the supervisor. +Rebuild only if the sandbox still cannot start. The host does not start the in-sandbox processes independently. Refer to [`$$nemoclaw recover`](../../reference/commands#$$nemoclaw-name-recover) and [`$$nemoclaw gateway restart`](../../reference/commands#$$nemoclaw-name-gateway-restart) for details. diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 9dffdaeb427..7be18b0cfcc 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -67,16 +67,17 @@ If preflight detects an unsafe path, invalid config, invalid ownership posture, | Channel tokens | Rebuild required because the channel configuration and credential attachment are created during onboarding or rebuild | `$$nemoclaw channels add `, then accept the rebuild prompt | | Channel enable or disable | Rebuild required because `/sandbox/.hermes/.env` and Hermes config are baked at image build time | `$$nemoclaw channels stop `, then rebuild | | API or dashboard forward port | Runtime; the host-side forward is re-resolved on the next `connect` | `$$nemoclaw connect` or `$$nemoclaw recover` | -| Hermes plugin code, Langfuse settings, or other startup-only runtime config | Runtime after a supported host-side update and gateway restart | Bake plugin code into the image or use a supported host config command, then run `$$nemoclaw gateway restart` | +| Hermes plugin code, Langfuse settings, or other startup-only runtime config | Runtime after Hermes or the sandbox user changes valid mutable config and the gateway restarts | Change Hermes-owned config in the sandbox; use a supported host config command for settings owned by NemoClaw, then run `$$nemoclaw gateway restart` | | Web search provider | Rebuild required because onboarding bakes `web.backend`, the environment placeholder, and the credential attachment into the image | Set `NEMOCLAW_WEB_SEARCH_PROVIDER=tavily` or `none`, then rerun onboarding and recreate the sandbox | | Filesystem layout | Locked at creation | Re-onboard with `$$nemoclaw onboard --recreate-sandbox` | | Sandbox name | Locked at creation | Re-onboard with a different `--name` | | GPU passthrough or device selector | Locked at creation | Re-onboard with `--gpu` or `--sandbox-gpu-device` | -| Hermes `config.yaml` keys | Mixed; inference and supported config keys can be patched by host commands, while image, policy, and channel changes still require rebuild | Use `$$nemoclaw inference set` or `$$nemoclaw config set` so the config and root-owned trust anchor change together | +| Hermes `config.yaml` keys | Mixed; valid Hermes-owned changes are adopted on restart, while image, policy, and channel changes still require rebuild | Change Hermes-owned settings in the sandbox; use `$$nemoclaw inference set` or `$$nemoclaw config set` for settings owned by NemoClaw so its registry stays aligned | The runtime source of truth is `/sandbox/.hermes/config.yaml` plus `/sandbox/.hermes/.env`. The host registry caches metadata, but the image and Hermes runtime read from the in-sandbox files. +Valid changes made by Hermes or the sandbox user are authoritative. Startup and restart validate the secret boundary, safe paths, and a stable transaction snapshot, then adopt the current mutable configuration. A direct config change alone does not block lifecycle operations. -Do not edit those files or their hash files directly and then expect `gateway restart` to establish the bytes as trusted. Use supported host config and inference commands so NemoClaw updates the managed config metadata together. +Use supported host config and inference commands for settings that NemoClaw owns so its registry stays aligned with the runtime. Those commands can report a registry mismatch for the operation they manage, but generic startup, restart, recovery, health, probe, and connect paths do not enforce registry equality. Hermes host-side config writes run as a sealed transaction. NemoClaw binds the write to the SHA-256 digest of the matching read, temporarily seals the mutable config paths, atomically installs fresh config inodes, refreshes the strict and compatibility hashes, and then restores the mutable paths. @@ -94,8 +95,8 @@ NemoClaw does not provide post-provisioning immutability for agent configuration OpenShell remains authoritative for sandbox filesystem and network policy enforcement. An agent process can change files that its sandbox identity can write. -Use supported host commands for intended configuration changes so NemoClaw updates validation hashes and managed metadata with the config. -Direct in-sandbox edits can cause a later gateway restart to reject the changed config when its integrity metadata no longer matches. +For Hermes, direct changes to valid mutable configuration do not block restart. Lifecycle validation can still refuse raw secrets, unsafe paths, raced snapshots, or missing or malformed transaction metadata. +Use supported host commands for settings owned by NemoClaw so its registry and the running Hermes projection remain aligned for those host-managed operations. NemoClaw serializes host-side gateway recovery, config and inference writes, snapshots, policy updates, channel updates, and sandbox destruction for each sandbox. This mutation lock prevents concurrent host operations from racing on the same registered sandbox. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index d35f0582d09..862797c89a6 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1304,7 +1304,17 @@ If a container or OpenShell identity check, reconnect check, or health check lat It then requires Podman to report the container as `exited` and OpenShell to report `Error` or `Stopped`. If NemoClaw cannot prove that rollback, the command reports both the recovery failure and the rollback failure. -`recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` and the supervisor runtime environment on every run, including when the gateway is already healthy. If the file contains raw secret-shaped values (for example a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder), the command exits non-zero and prints the offending key. The direct root-entrypoint supervisor stops a running gateway after this refusal, while the managed controller refuses before signaling the observed child. Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. The direct root-entrypoint supervisor verifies the strict root-owned config hash used by the managed transaction. Mutable config under the managed topology has no durable root-owned hash anchor and retains the same trust and time-of-check/time-of-use limits as managed cold start. If the boundary validator or supervisor helper is missing, recovery fails closed, names the sandbox, explains that `/sandbox/.hermes/.env` could not be re-evaluated, and leaves an otherwise healthy gateway untouched. Rebuild an older sandbox image with `$$nemoclaw rebuild --yes` before retrying. +`recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` and the supervisor runtime environment on every run, including when the gateway is already healthy. +If the file contains raw secret-shaped values, the command exits non-zero and prints the offending key. +Raw values include a pasted Telegram, Discord, or Slack bot token in place of the expected `openshell:resolve:env:` placeholder. +The direct root-entrypoint supervisor stops a running gateway after this refusal, while the managed controller refuses before signaling the observed child. +Replace each flagged value with the `openshell:resolve:env:` placeholder and re-run. +Both supervisors adopt one stable mutable config snapshot after this boundary passes. +A direct MCP change becomes pending, supersedes stale host-managed intent, and becomes applied only after gateway health passes. +A host-managed MCP operation can report a registry mismatch after Hermes is healthy. +If the boundary validator or supervisor helper is missing, recovery fails closed and names the sandbox. +It explains that `/sandbox/.hermes/.env` could not be re-evaluated and leaves an otherwise healthy gateway untouched. +Rebuild an older sandbox image with `$$nemoclaw rebuild --yes` before retrying. While a NemoClaw cron restore gate exists, Hermes `recover` keeps the same lifecycle lock through restore validation and gate release. That controller call has a 130-second host timeout; the earlier 30-second limit applies only to lifecycle-lock acquisition. @@ -1324,18 +1334,30 @@ Force-restart the supported in-sandbox gateway process through the controller fo $$nemoclaw my-assistant gateway restart [--quiet|-q] ``` -On success, the command reports that the gateway was restarted, health passed, and forwards were checked or recovered. It also checks the dashboard forward, messaging forward, and manifest-declared agent forwards. `--quiet` suppresses progress lines but still prints refusal diagnostics. In the direct root-entrypoint topology, PID 1 stops only the gateway child whose process ID and process start identity match the tracked child, applies the restart seal, and launches the replacement under the separate `gateway` UID. In the OpenShell-managed topology, the installed root controller verifies a stable OpenShell to `nemoclaw-start` to gateway process shape, holds a root-only lifecycle lock, publishes one root-owned exit authorization bound to the exact gateway process ID, kernel start identity, and live controller identity, pidfd-targets the observed child, waits for the nonroot entrypoint supervisor to respawn it under the sandbox UID, and proves the replacement listener and HTTP health. That managed process proof prevents PID reuse from redirecting the signal but cannot establish provenance against a malicious same-UID process or create gateway and agent UID isolation. For Hermes, the entrypoint supervisor also owns the dashboard process, internal API relay, dashboard relay, and gateway log stream. The managed nonroot supervisor continuously repairs those processes, stops an alive but deaf gateway after four consecutive failed health checks, and quarantines relaunch after five unexpected exits or failed replacement candidates within 60 seconds until sandbox recreation. That authorization keeps an authenticated host-requested exit out of the crash budget while its exact root controller remains live; it records host intent for the exit but does not claim that the host signal was the only possible cause in the shared-UID topology. The host repairs only the host-side OpenShell forwards after the supervisor reports a healthy gateway. +On success, the command reports that the gateway was restarted, health passed, and forwards were checked or recovered. +It also checks the dashboard forward, messaging forward, and manifest-declared agent forwards. +`--quiet` suppresses progress lines but still prints refusal diagnostics. +In the direct root-entrypoint topology, PID 1 stops only the gateway child whose process ID and process start identity match the tracked child, applies the restart seal, and launches the replacement under the separate `gateway` UID. +In the OpenShell-managed topology, the installed root controller verifies a stable OpenShell to `nemoclaw-start` to gateway process shape, holds a root-only lifecycle lock, publishes one root-owned exit authorization bound to the exact gateway process ID, kernel start identity, and live controller identity, pidfd-targets the observed child, waits for the nonroot entrypoint supervisor to respawn it under the sandbox UID, and proves the replacement listener and HTTP health. +That managed process proof prevents PID reuse from redirecting the signal but cannot establish provenance against a malicious same-UID process or create gateway and agent UID isolation. +For Hermes, the entrypoint supervisor also owns the dashboard process, internal API relay, dashboard relay, and gateway log stream. +The managed nonroot supervisor continuously repairs those processes, stops an alive but deaf gateway after four consecutive failed health checks, and stops relaunch for the current supervisor instance after five unexpected exits or failed replacement candidates within 60 seconds. +That authorization keeps an authenticated host-requested exit out of the crash budget while its exact root controller remains live. +It records host intent for the exit but does not claim that the host signal was the only possible cause in the shared-UID topology. +The host repairs only the host-side OpenShell forwards after the supervisor reports a healthy gateway. For Hermes, both controllers validate `/sandbox/.hermes/.env` against the secret-boundary guard and validate the supervisor runtime environment before restart. -The direct root-entrypoint supervisor verifies `/sandbox/.hermes/config.yaml` and `.env` against the strict hash and relaunches the process as the `gateway` user. -Mutable managed config retains cold-start-equivalent trust and time-of-check/time-of-use limits. -Neither controller recomputes a trusted strict hash to adopt direct in-sandbox edits. -Use supported host commands such as `$$nemoclaw config set` and `$$nemoclaw inference set` for intended runtime configuration changes because those commands update the managed config metadata together. -When a strict hash is available and does not match, the command reports the `config hash mismatch` failure layer. +Both supervisors adopt one stable snapshot of mutable `config.yaml` and `.env` after the secret boundary passes. +A direct MCP change becomes pending and supersedes stale host-managed intent. +The supervisor marks it as applied only after the replacement gateway passes its health checks. +The host operation that owns a managed MCP transaction can report a registry mismatch after Hermes is healthy. +The direct root-entrypoint supervisor relaunches the process as the `gateway` user. +Use supported host commands such as `$$nemoclaw config set` and `$$nemoclaw inference set` for settings that must match host registry state. +The command reports `unsafe config path` when a config or metadata path is unsafe. It reports `config hash mismatch` when hash or restart transaction metadata is missing, malformed, mismatched, or changed during validation. Hermes host config writes and lifecycle seals share one root-only mutation lock. Config writes bind to the digest of the matching read and atomically refresh the strict and compatibility hashes before restoring mutable access. If a concurrent lifecycle request reports `SUPERVISOR_BUSY` or `Hermes config mutation is already in progress`, wait for the active operation to finish and retry. @@ -1344,7 +1366,13 @@ If a concurrent lifecycle request reports `SUPERVISOR_BUSY` or `Hermes config mu -The command can fail at these layers: unsupported agent, privileged control unavailable, supervisor not running, secret-boundary refusal, unsafe config path, config hash mismatch when a strict hash is available, MCP reconciliation refusal, relaunch quarantined, launch failure, health timeout, or forward recovery failure. `relaunch quarantined` means the in-sandbox supervisor stopped attempting relaunch after a startup refusal or repeated gateway exits, so restart and recovery report the supported repair, `$$nemoclaw rebuild --yes`, instead of a retry. An older direct-container image without the matching supervisor or managed controller helper reports `privileged control unavailable` and requires `$$nemoclaw rebuild --yes`. Ordinary OpenShell exec and manual in-sandbox relaunch are not fallback paths. Terminal agents do not have a gateway runtime and fail as unsupported. +The command can fail at these layers: unsupported agent, privileged control unavailable, supervisor not running, secret-boundary refusal, unsafe config path, config hash mismatch when transaction metadata cannot be validated, MCP reconciliation refusal, relaunch quarantined, launch failure, health timeout, or forward recovery failure. +`relaunch quarantined` means the in-sandbox supervisor stopped attempting relaunch after repeated process or health failures. +Correct the reported cause, then run `$$nemoclaw stop` and `$$nemoclaw start` to reset the supervisor. +Rebuild only if the sandbox still cannot start. +An older direct-container image without the matching supervisor or managed controller helper reports `privileged control unavailable` and requires `$$nemoclaw rebuild --yes`. +Ordinary OpenShell exec and manual in-sandbox relaunch are not fallback paths. +Terminal agents do not have a gateway runtime and fail as unsupported. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 54a37901c2f..12f90dcf081 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -4344,35 +4344,62 @@ nemohermes start Then run `nemohermes inference get` and verify Dashboard Chat uses the selected model. If the command succeeds because the dashboard profile is missing, the dashboard is disabled and no dashboard recovery is required. -### Hermes restart reports `config hash mismatch` +### Hermes restart reports a config integrity failure -A Hermes restart reports `config hash mismatch` when an available managed hash does not match `/sandbox/.hermes/config.yaml` or `/sandbox/.hermes/.env`. -The direct root-entrypoint supervisor uses the strict hash at `/etc/nemoclaw/hermes.config-hash`. -Mutable config in the OpenShell-managed topology has no durable tamper-proof hash anchor, so restart retains the same trust and time-of-check/time-of-use limits as managed cold start. -Both controllers validate the secret boundary and supervisor runtime environment before they stop the tracked gateway. -They do not recompute a trusted strict hash to adopt direct edits made inside the sandbox. +Hermes configuration is mutable. +A current Hermes supervisor does not refuse startup, restart, recovery, or automatic respawn only because `config.yaml` or `.env` changed. +It validates the secret boundary and records one stable snapshot before a replacement gateway consumes the config. +A direct MCP change becomes pending and is marked as applied only after gateway health passes. -For intended changes, use supported host commands such as `nemohermes config set` and `nemohermes inference set` so NemoClaw updates the config and its strict and compatibility hashes together. Do not edit either hash file manually. +An unsafe config or metadata path reports `unsafe config path`. +A missing, malformed, mismatched, or raced hash or restart metadata file reports `config hash mismatch`. +Do not edit either hash file manually. -If direct edits have already caused a mismatch, restore the original config and environment files or rebuild from the registered configuration: +If the error reports an MCP reconciliation refusal, restore the managed MCP projection and retry recovery: + +```bash +nemohermes mcp restart +nemohermes recover +``` + +If the error reports `secret-boundary refusal`, inspect `/sandbox/.hermes/.env` for raw secret-shaped values. +Replace them through the supported credential flow so the file contains `openshell:resolve:env:` placeholders, then run `nemohermes recover`. + +If the error reports `unsafe config path`, do not follow or edit the unsafe path in place. Rebuild from registered configuration to restore the managed layout. +If it reports `config hash mismatch`, do not repair the hash or restart metadata by hand. Rebuild from registered configuration: ```bash nemohermes rebuild --yes ``` -If the command instead reports `secret-boundary refusal`, inspect `/sandbox/.hermes/.env` for raw secret-shaped values. Replace them through the supported credential flow so the file contains `openshell:resolve:env:` placeholders, then run `nemohermes recover`. The Hermes entrypoint supervisor remains responsible for the gateway, dashboard, internal API relay, dashboard relay, and gateway log stream throughout recovery. In the OpenShell-managed topology, that nonroot supervisor repairs failed auxiliaries continuously, recovers a gateway after four consecutive failed listener or HTTP health checks, and quarantines relaunch after five exits within 60 seconds until the sandbox is recreated. The host only repairs the host-side OpenShell forwards after the supervised processes pass health checks. +The Hermes entrypoint supervisor remains responsible for the gateway, dashboard, internal API relay, dashboard relay, and gateway log stream throughout recovery. +In the OpenShell-managed topology, that nonroot supervisor repairs failed auxiliaries continuously and recovers a gateway after four consecutive failed listener or HTTP health checks. +Five exits within 60 seconds stop relaunch for the current supervisor instance. +The host repairs only the host-side OpenShell forwards after the supervised processes pass health checks. ### Restart or recovery reports `relaunch quarantined` -In the OpenShell-managed topology the strict root-owned hash is not a trust anchor for mutable config, so a direct edit of `/sandbox/.hermes/config.yaml` or `/sandbox/.hermes/.env` is not refused by the host controller. The in-sandbox supervisor still refuses to start a gateway on configuration it cannot match to the persisted managed state, and it stops attempting relaunch once that refusal or repeated gateway exits exhaust its crash budget. `$$nemoclaw gateway restart`, `$$nemoclaw recover`, and `$$nemoclaw connect` then report the `relaunch quarantined` failure layer. +Valid Hermes config changes do not trigger relaunch quarantine. +The supervisor quarantines relaunch after five unexpected gateway exits or failed replacement candidates within 60 seconds. +Invalid Hermes config can make each replacement exit and consume that crash budget. + +The quarantine is deterministic, so retrying restart or recovery cannot clear it. +Inspect the Hermes log and correct the process or config failure. +Then restart the sandbox to reset the supervisor's crash budget: + +```bash +nemohermes logs --tail 50 +nemohermes stop +nemohermes start +``` -The refusal is deterministic, so retrying any of those commands cannot clear it. Restore the registered configuration and refresh its integrity metadata in one transaction: +If the sandbox still cannot start, rebuild it from registered configuration: ```bash nemohermes rebuild --yes ``` -After the rebuild, make the intended change through a supported command such as `$$nemoclaw config set` or `$$nemoclaw inference set`, which update the configuration and its hashes together. +After recovery, verify that Hermes starts and the gateway health check passes. ### Port 8642 in a browser shows a blank page or `Cannot GET /` diff --git a/docs/security/filesystem-controls.mdx b/docs/security/filesystem-controls.mdx index 1c6a9911d75..084295be1a9 100644 --- a/docs/security/filesystem-controls.mdx +++ b/docs/security/filesystem-controls.mdx @@ -70,13 +70,16 @@ Messaging sessions such as WhatsApp pairing can remain mutable by design so they The Hermes config and state tree remains mutable after provisioning. NemoClaw does not prevent the sandbox identity from changing paths that its Unix permissions allow. +Hermes startup and restart adopt a stable config snapshot after validating its paths and secret boundary. +A direct MCP change becomes applied only after the replacement gateway passes its health checks. +A host-managed MCP operation can report a registry mismatch without preventing Hermes from running. | Aspect | Detail | |---|---| | Default | The Hermes config tree contains NemoClaw-generated config plus mutable runtime state. | -| What you can change | Use host-side NemoClaw commands for durable model, provider, messaging, and policy changes; inspect files directly only for debugging. | -| Risk of direct edits | Direct edits to generated config can drift from the host registry and may be lost on rebuild. | -| Recommendation | For sensitive workloads, keep generated config under NemoClaw control and back up Hermes state before destructive operations. | +| What you can change | Hermes and the sandbox user can change mutable runtime config. Use host-side NemoClaw commands for settings that must match the host registry. | +| Risk of direct edits | Invalid config can prevent Hermes from starting. Changes to host-managed settings can drift from the registry and may be lost on rebuild. | +| Recommendation | Keep credentials in OpenShell providers. Back up Hermes state before destructive operations. | diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index e62f554a049..7d108478eca 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -143,19 +143,19 @@ ), re.compile(r"\[gateway\] Hermes gateway respawned \(pid [1-9][0-9]*\)"), re.compile( - r"\[gateway\] CRITICAL: [1-9][0-9]* exits in 60s window — Hermes relaunch is quarantined until sandbox recreation; check /tmp/gateway\.log" + r"\[gateway\] Hermes runtime preparation failed after 5 consecutive attempts; supervisor exiting without launching a gateway; correct the reported failure, then stop and start the sandbox" ), re.compile( - r"\[gateway\] CRITICAL: (?:exact Hermes replacement|unhealthy Hermes gateway|initial Hermes gateway) could not be stopped; managed supervisor is quarantined without another launch" + r"\[gateway\] CRITICAL: [1-9][0-9]* exits in 60s window — Hermes relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox; check /tmp/gateway\.log" ), re.compile( - r"\[SECURITY\] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox" + r"\[gateway\] CRITICAL: (?:exact Hermes replacement|unhealthy Hermes gateway|initial Hermes gateway) could not be stopped; managed supervisor is quarantined without another launch" ), re.compile( r"\[CRITICAL\] Newly launched Hermes (?:gateway|gateway-log|dashboard|dashboard-log|api-socat|dashboard-socat) pid [1-9][0-9]* failed exact role identity capture; quarantining the managed startup supervisor without signaling the unproven child" ), re.compile( - r"\[CRITICAL\] Unproven Hermes (?:gateway|gateway-log|dashboard|dashboard-log|api-socat|dashboard-socat) child exited; managed supervisor remains quarantined until sandbox recreation" + r"\[CRITICAL\] Unproven Hermes (?:gateway|gateway-log|dashboard|dashboard-log|api-socat|dashboard-socat) child exited; relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox" ), ) ANSI_ESCAPE_RE = re.compile( diff --git a/src/lib/actions/sandbox/connect-boundary-refusal.ts b/src/lib/actions/sandbox/connect-boundary-refusal.ts index 6beb5a48683..a2af0434ec3 100644 --- a/src/lib/actions/sandbox/connect-boundary-refusal.ts +++ b/src/lib/actions/sandbox/connect-boundary-refusal.ts @@ -3,31 +3,25 @@ import { type GatewayRestartFailureLayer, - gatewayIntegrityRepairLines, - isGatewayIntegrityRepairLayer, + gatewayTerminalRepairLines, + isGatewayTerminalRepairLayer, } from "./gateway-restart"; import type { SecretBoundaryRefusalReason } from "./hermes-secret-boundary-recovery"; -import { - hermesMcpReconciliationRemediationLines, - sanitizeHermesMcpReconciliationDetail, -} from "./mcp-bridge-hermes-reconciliation"; type ConnectBoundaryContext = "Probe" | "Connect"; /** - * A managed recovery that failed on a deterministic integrity refusal cannot be - * retried: every relaunch re-reads the same drifted protected configuration. - * The probe path recovers quietly, so without this the operator only sees the - * generic "check the gateway log" and never learns the supported repair (#7801). + * The probe path recovers quietly. Report the repair for a terminal transaction + * or process state before generic gateway-log guidance hides it (#7801). * Returns false when the layer is a retryable failure, leaving the caller's * existing wedge diagnostics in charge. */ -export function printGatewayIntegrityRepairGuidance( +export function printGatewayTerminalRepairGuidance( sandboxName: string, layer: GatewayRestartFailureLayer | null | undefined, ): boolean { - if (!isGatewayIntegrityRepairLayer(layer)) return false; - for (const line of gatewayIntegrityRepairLines(sandboxName, layer)) { + if (!isGatewayTerminalRepairLayer(layer)) return false; + for (const line of gatewayTerminalRepairLines(sandboxName, layer)) { console.error(` ${line}`); } return true; @@ -76,24 +70,3 @@ export function exitOnSecretBoundaryRefusal( } process.exit(1); } - -export function exitOnMcpReconciliationRefusal( - sandboxName: string, - agentName: string, - processCheck: Record, - contextLabel: ConnectBoundaryContext, -): never { - const detail = - "mcpReconciliationReason" in processCheck - ? String(processCheck.mcpReconciliationReason) - : "the effective Hermes MCP configuration does not match persisted managed intent"; - const sanitizedDetail = sanitizeHermesMcpReconciliationDetail(detail); - console.error(""); - console.error( - ` ${contextLabel} failed: refused to confirm ${agentName} gateway in '${sandboxName}' — ${sanitizedDetail}.`, - ); - for (const line of hermesMcpReconciliationRemediationLines(sandboxName)) { - console.error(` ${line}`); - } - process.exit(1); -} diff --git a/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts b/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts index 0cf1f382f68..eb135f65a1c 100644 --- a/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts +++ b/src/lib/actions/sandbox/connect-flow-hermes-boundary.test.ts @@ -78,72 +78,6 @@ describe("connectSandbox Hermes secret-boundary refusals", () => { expect(exitSpy).toHaveBeenCalledWith(1); }); - it("fails closed on Hermes MCP drift with restart and rebuild guidance", async () => { - const harness = createConnectHarness({ - processCheck: { - checked: true, - wasRunning: true, - recovered: false, - forwardRecovered: false, - mcpReconciliationRefused: true, - mcpReconciliationReason: "Hermes MCP config does not match persisted managed intent", - }, - }); - const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); - vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); - vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); - - await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( - "process.exit(1)", - ); - - const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); - expect(errorOutput).toContain("Probe failed: refused to confirm Hermes gateway in 'alpha'"); - expect(errorOutput).toContain("nemoclaw alpha mcp restart"); - expect(errorOutput).toContain("nemoclaw alpha rebuild --yes"); - expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); - expect(exitSpy).toHaveBeenCalledWith(1); - }); - - it("refuses an HTTP-healthy Hermes gateway while MCP integrity is pending", async () => { - const harness = createConnectHarness({ - processCheck: { - checked: true, - // Process recovery normalizes both HTTP 200 and authenticated HTTP 401 - // health probes to this positive running state before reconciliation. - wasRunning: true, - recovered: false, - forwardRecovered: true, - mcpReconciliationRefused: true, - mcpReconciliationReason: - "\x1b[31mHermes MCP integrity is pending\x1b[0m\nFORGED SUCCESS ghp_0123456789abcdefghij", - }, - }); - const agentRuntime = requireDist("../../src/lib/agent/runtime.js"); - vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "hermes" }); - vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("Hermes"); - - await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(1)"); - - const failureLine = harness.errorSpy.mock.calls - .map((call) => String(call[0] ?? "")) - .find((line) => line.includes("Connect failed:")); - expect(failureLine).toContain("Hermes MCP integrity is pending FORGED SUCCESS "); - expect(failureLine).not.toMatch(/[\r\n\x1b]/); - const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); - expect(errorOutput).toContain("nemoclaw alpha mcp restart"); - expect(errorOutput).toContain("nemoclaw alpha rebuild --yes"); - expect(errorOutput).not.toContain("ghp_0123456789abcdefghij"); - expect(harness.ensureOllamaAuthProxySpy).not.toHaveBeenCalled(); - expect(harness.runAutoPairSpy).not.toHaveBeenCalled(); - expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( - "openshell", - ["sandbox", "connect", "alpha"], - expect.any(Object), - ); - expect(exitSpy).toHaveBeenCalledWith(1); - }); - it.each([ [ "raw-secret", diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index 29324fd9587..586a9079a69 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -986,8 +986,11 @@ describe("connectSandbox flow", () => { ); const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); - expect(errorOutput).toContain("quarantined gateway relaunch"); + expect(errorOutput).toContain("repeated process or health failures"); + expect(errorOutput).toContain("nemoclaw alpha stop"); + expect(errorOutput).toContain("nemoclaw alpha start"); expect(errorOutput).toContain("nemoclaw alpha rebuild --yes"); + expect(errorOutput).not.toContain("config set"); expect(errorOutput).not.toContain("Check /tmp/gateway.log inside the sandbox for details."); expect(exitSpy).toHaveBeenCalledWith(1); }); diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 209a88dbad4..8205f7d4fea 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -64,9 +64,8 @@ import { runSetupDnsProxy } from "../dns"; import { runSandboxExecChild } from "./exec"; import { runConnectAutoPairApprovalPass } from "./auto-pair-approval"; import { - exitOnMcpReconciliationRefusal, exitOnSecretBoundaryRefusal, - printGatewayIntegrityRepairGuidance, + printGatewayTerminalRepairGuidance, } from "./connect-boundary-refusal"; import { prepareHermesLightTerminalSkin } from "./connect-hermes-light-skin"; import { @@ -432,10 +431,6 @@ async function runSandboxConnectProbe( probeTiming?.markFailureStage("processes"); exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Probe"); } - if ("mcpReconciliationRefused" in processCheck && processCheck.mcpReconciliationRefused) { - probeTiming?.markFailureStage("processes"); - exitOnMcpReconciliationRefusal(sandboxName, agentName, processCheck, "Probe"); - } if ("forwardRecoveryFailed" in processCheck && processCheck.forwardRecoveryFailed) { probeTiming?.markFailureStage("forward"); const detail = @@ -498,7 +493,7 @@ async function runSandboxConnectProbe( ` Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`, ); probeTiming?.markFailureStage("processes"); - if (printGatewayIntegrityRepairGuidance(sandboxName, recoveryFailureLayer)) { + if (printGatewayTerminalRepairGuidance(sandboxName, recoveryFailureLayer)) { process.exit(1); } // Surface the #4710 wedge signature: recovery ran with quiet=true, so this @@ -2180,12 +2175,6 @@ export async function prepareInteractiveSession(sandboxName: string): Promise<{ ); exitOnSecretBoundaryRefusal(sandboxName, agentName, processCheck, "Connect"); } - if ("mcpReconciliationRefused" in processCheck && processCheck.mcpReconciliationRefused) { - const agentName = agentRuntime.getAgentDisplayName( - agentRuntime.getSessionAgent(sandboxName), - ); - exitOnMcpReconciliationRefusal(sandboxName, agentName, processCheck, "Connect"); - } const recoveryFailureDetail = "recoveryFailureDetail" in processCheck && processCheck.recoveryFailureDetail ? String(processCheck.recoveryFailureDetail) diff --git a/src/lib/actions/sandbox/exec-googlechat-pairing-restart.test.ts b/src/lib/actions/sandbox/exec-googlechat-pairing-restart.test.ts index 231b8c32cff..c0112a24f76 100644 --- a/src/lib/actions/sandbox/exec-googlechat-pairing-restart.test.ts +++ b/src/lib/actions/sandbox/exec-googlechat-pairing-restart.test.ts @@ -254,7 +254,6 @@ describe("Google Chat pairing approval gateway activation (#8553)", () => { recoverMessagingHostForward: () => null, recoverDeclaredAgentForwardPorts: () => null, printGatewayWedgeDiagnostics: () => false, - inspectHermesMcpReconciliationRefusal: () => null, }, }), policyHint: { diff --git a/src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts b/src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts index f2fe82632cd..90caae69bce 100644 --- a/src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts +++ b/src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts @@ -5,10 +5,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { expect, it, vi } from "vitest"; - -import { hermesAgent } from "../../agent/hermes-recovery-boundary-fixtures"; -import { type GatewayRestartDeps, restartSandboxGatewayWithDeps } from "./gateway-restart"; +import { expect, it } from "vitest"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../../.."); const HERMES_GUARD = path.join(REPO_ROOT, "agents/hermes/runtime-config-guard.py"); @@ -129,50 +126,3 @@ raise SystemExit(transaction.main()) fs.rmSync(root, { recursive: true, force: true }); } }); - -it("sanitizes an injected Hermes reconciliation refusal before post-restart mutations", () => { - try { - const postReconciliationMutations = [ - vi.fn(() => true), - vi.fn(() => null), - vi.fn(() => null), - vi.fn(() => null), - ] as const; - const deps: GatewayRestartDeps = { - getSessionAgent: () => hermesAgent, - getSandbox: () => ({ agent: "hermes" }), - resolveSandboxDashboardPort: () => 18789, - requestGatewaySupervisorAction: vi.fn(() => ({ - status: 0, - stdout: "GATEWAY_PID=123", - stderr: "", - })), - executeSandboxExecCommand: vi.fn(() => null), - waitForRecoveredSandboxGateway: vi.fn(() => true), - ensureSandboxPortForward: postReconciliationMutations[0], - ensureHermesDashboardPortForwardIfEnabled: postReconciliationMutations[1], - recoverMessagingHostForward: postReconciliationMutations[2], - recoverDeclaredAgentForwardPorts: postReconciliationMutations[3], - printGatewayWedgeDiagnostics: vi.fn(() => false), - inspectHermesMcpReconciliationRefusal: vi.fn(() => ({ - detail: "Hermes config hash does not match persisted inputs FORGED SUCCESS ", - })), - }; - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - - expect(restartSandboxGatewayWithDeps("alpha", { quiet: true, deps })).toEqual({ - ok: false, - failureLayer: "MCP reconciliation refusal", - detail: "Hermes config hash does not match persisted inputs FORGED SUCCESS ", - restarted: true, - healthPassed: true, - }); - expect(postReconciliationMutations[0]).not.toHaveBeenCalled(); - expect(postReconciliationMutations[1]).not.toHaveBeenCalled(); - expect(postReconciliationMutations[2]).not.toHaveBeenCalled(); - expect(postReconciliationMutations[3]).not.toHaveBeenCalled(); - expect(error.mock.calls.flat().join("\n")).not.toMatch(/\x1b|ghp_0123456789abcdefghij/u); - } finally { - vi.restoreAllMocks(); - } -}); diff --git a/src/lib/actions/sandbox/gateway-restart-mcp.test.ts b/src/lib/actions/sandbox/gateway-restart-mcp.test.ts index fa4cf1a6f03..47c45f98038 100644 --- a/src/lib/actions/sandbox/gateway-restart-mcp.test.ts +++ b/src/lib/actions/sandbox/gateway-restart-mcp.test.ts @@ -36,51 +36,23 @@ function baseDeps(overrides: Partial = {}): GatewayRestartDe recoverMessagingHostForward: vi.fn(() => null), recoverDeclaredAgentForwardPorts: vi.fn(() => null), printGatewayWedgeDiagnostics: vi.fn(() => false), - inspectHermesMcpReconciliationRefusal: vi.fn(() => null), ...overrides, }; } describe("Hermes MCP gateway restart", () => { - it("refuses to report a restarted gateway with stale MCP intent", () => { + it("completes generic restart without host MCP reconciliation (#11108)", () => { const restore = silenceConsole(); try { - const deps = baseDeps({ - inspectHermesMcpReconciliationRefusal: vi.fn(() => ({ - detail: "Hermes MCP config does not match persisted managed intent", - })), - }); - - expect(restartSandboxGateway("alpha", { quiet: true, deps })).toEqual({ - ok: false, - failureLayer: "MCP reconciliation refusal", - detail: "Hermes MCP config does not match persisted managed intent", - restarted: true, - healthPassed: true, - }); - expect(deps.ensureSandboxPortForward).not.toHaveBeenCalled(); - } finally { - restore(); - } - }); - - it("returns only sanitized MCP reconciliation detail", () => { - const restore = silenceConsole(); - try { - const deps = baseDeps({ - inspectHermesMcpReconciliationRefusal: vi.fn(() => ({ - detail: "integrity pending FORGED SUCCESS ", - })), - }); + const deps = baseDeps(); expect(restartSandboxGateway("alpha", { quiet: true, deps })).toEqual({ - ok: false, - failureLayer: "MCP reconciliation refusal", - detail: "integrity pending FORGED SUCCESS ", restarted: true, + ok: true, healthPassed: true, + forwardRecovered: true, }); - expect(deps.ensureSandboxPortForward).not.toHaveBeenCalled(); + expect(deps.ensureSandboxPortForward).toHaveBeenCalledWith("alpha"); } finally { restore(); } diff --git a/src/lib/actions/sandbox/gateway-restart-quarantine-repair.test.ts b/src/lib/actions/sandbox/gateway-restart-quarantine-repair.test.ts index a6600f053b1..c3a102b521c 100644 --- a/src/lib/actions/sandbox/gateway-restart-quarantine-repair.test.ts +++ b/src/lib/actions/sandbox/gateway-restart-quarantine-repair.test.ts @@ -4,8 +4,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { classifyGatewayRestartFailure, - gatewayIntegrityRepairLines, - isGatewayIntegrityRepairLayer, + gatewayTerminalRepairLines, + isGatewayTerminalRepairLayer, printGatewayRestartFailure, } from "./gateway-restart"; @@ -13,22 +13,18 @@ import { // attempting relaunch. `scripts/managed-gateway-control.py` allowlists these // before forwarding them to the host as `NEMOCLAW_START_LOG=` lines. const QUARANTINE_LINES = [ - "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is quarantined until sandbox recreation; check /tmp/gateway.log", - "[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox", + "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox; check /tmp/gateway.log", "[gateway] CRITICAL: exact Hermes replacement could not be stopped; managed supervisor is quarantined without another launch", - "[CRITICAL] Unproven Hermes gateway child exited; managed supervisor remains quarantined until sandbox recreation", + "[CRITICAL] Unproven Hermes gateway child exited; relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox", "[CRITICAL] Newly launched Hermes gateway pid 4242 failed exact role identity capture; quarantining the managed startup supervisor without signaling the unproven child", ] as const; -// Verbatim controller output captured on a Hermes sandbox whose protected -// `config.yaml` was edited outside a supported command, then restarted. -const REPORTED_RESTART_OUTPUT = [ +const CRASH_LOOP_RESTART_OUTPUT = [ "GATEWAY_HEALTH_TIMEOUT", "NEMOCLAW_CONTROL_STAGE=await-replacement", "NEMOCLAW_SUPERVISOR_PID=42", "NEMOCLAW_GATEWAY_PID=0", - "NEMOCLAW_START_LOG=[gateway] Hermes gateway respawned (pid 18424)", - "NEMOCLAW_START_LOG=[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox", + "NEMOCLAW_START_LOG=[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox; check /tmp/gateway.log", ].join("\n"); function classify(stdout: string) { @@ -54,16 +50,10 @@ describe("supervisor relaunch quarantine classification (#7801)", () => { expect(classify(line)).toMatchObject({ layer: "relaunch quarantined" }); }); - it("classifies the reported restart output as a quarantine, not a health timeout", () => { - expect(classify(REPORTED_RESTART_OUTPUT)).toMatchObject({ layer: "relaunch quarantined" }); - }); - - it("prefers the quarantine over the MCP drift it is reported through", () => { - const output = [ - "HERMES_MCP_CONFIG_DRIFT", - "[SECURITY] Hermes automatic respawn is quarantined until MCP integrity is restored by rebuilding the sandbox", - ].join("\n"); - expect(classify(output)).toMatchObject({ layer: "relaunch quarantined" }); + it("classifies a crash-loop quarantine ahead of its health timeout", () => { + expect(classify(CRASH_LOOP_RESTART_OUTPUT)).toMatchObject({ + layer: "relaunch quarantined", + }); }); it("keeps the pre-existing layers for output without a quarantine line", () => { @@ -84,39 +74,43 @@ describe("supervisor relaunch quarantine classification (#7801)", () => { }); }); -describe("integrity repair guidance (#7801)", () => { - it("treats both deterministic integrity refusals as repairable layers", () => { - expect(isGatewayIntegrityRepairLayer("relaunch quarantined")).toBe(true); - expect(isGatewayIntegrityRepairLayer("config hash mismatch")).toBe(true); - expect(isGatewayIntegrityRepairLayer("health timeout")).toBe(false); - expect(isGatewayIntegrityRepairLayer("launch failure")).toBe(false); - expect(isGatewayIntegrityRepairLayer(null)).toBe(false); - expect(isGatewayIntegrityRepairLayer(undefined)).toBe(false); +describe("terminal restart repair guidance (#7801)", () => { + it("recognizes the terminal repair layers", () => { + expect(isGatewayTerminalRepairLayer("relaunch quarantined")).toBe(true); + expect(isGatewayTerminalRepairLayer("config hash mismatch")).toBe(true); + expect(isGatewayTerminalRepairLayer("health timeout")).toBe(false); + expect(isGatewayTerminalRepairLayer("launch failure")).toBe(false); + expect(isGatewayTerminalRepairLayer(null)).toBe(false); + expect(isGatewayTerminalRepairLayer(undefined)).toBe(false); }); it.each([ "relaunch quarantined", "config hash mismatch", ] as const)("names the supported repair command for %s", (layer) => { - const lines = gatewayIntegrityRepairLines("repro-7801", layer).join("\n"); + const lines = gatewayTerminalRepairLines("repro-7801", layer).join("\n"); expect(lines).toContain("nemoclaw repro-7801 rebuild --yes"); - expect(lines).toContain("Retrying the restart cannot clear it."); - expect(lines).toContain("nemoclaw repro-7801 config set"); }); - it("describes the two refusals differently", () => { - const quarantined = gatewayIntegrityRepairLines("alpha", "relaunch quarantined")[0]; - const drifted = gatewayIntegrityRepairLines("alpha", "config hash mismatch")[0]; - expect(quarantined).not.toEqual(drifted); - expect(drifted).toContain("integrity hash"); - expect(quarantined).toContain("quarantined"); + it("resets process quarantine without blaming mutable config (#11108)", () => { + const lines = gatewayTerminalRepairLines("alpha", "relaunch quarantined").join("\n"); + expect(lines).toContain("repeated process or health failures"); + expect(lines).toContain("nemoclaw alpha stop"); + expect(lines).toContain("nemoclaw alpha start"); + expect(lines).not.toContain("config set"); + }); + + it("keeps metadata repair separate from process quarantine", () => { + const lines = gatewayTerminalRepairLines("alpha", "config hash mismatch").join("\n"); + expect(lines).toContain("integrity metadata"); + expect(lines).not.toContain("nemoclaw alpha stop"); }); }); describe("printGatewayRestartFailure repair guidance (#7801)", () => { it("appends the repair to a quarantined restart failure", () => { const lines = captureStderr(() => - printGatewayRestartFailure("repro-7801", "relaunch quarantined", REPORTED_RESTART_OUTPUT), + printGatewayRestartFailure("repro-7801", "relaunch quarantined", CRASH_LOOP_RESTART_OUTPUT), ).join("\n"); expect(lines).toContain("Failure layer: relaunch quarantined"); expect(lines).toContain("nemoclaw repro-7801 rebuild --yes"); diff --git a/src/lib/actions/sandbox/gateway-restart.test.ts b/src/lib/actions/sandbox/gateway-restart.test.ts index f9fe5f0defe..9b11b732f00 100644 --- a/src/lib/actions/sandbox/gateway-restart.test.ts +++ b/src/lib/actions/sandbox/gateway-restart.test.ts @@ -141,7 +141,6 @@ describe("restartSandboxGateway — host-mediated gateway restart", () => { recoverMessagingHostForward: vi.fn(() => null), recoverDeclaredAgentForwardPorts: vi.fn(() => null), printGatewayWedgeDiagnostics: vi.fn(() => false), - inspectHermesMcpReconciliationRefusal: vi.fn(() => null), ...overrides, }; } diff --git a/src/lib/actions/sandbox/gateway-restart.ts b/src/lib/actions/sandbox/gateway-restart.ts index 24c5fa2a5e8..58be95be36b 100644 --- a/src/lib/actions/sandbox/gateway-restart.ts +++ b/src/lib/actions/sandbox/gateway-restart.ts @@ -6,7 +6,6 @@ import * as agentRuntime from "../../agent/runtime"; import { G, R } from "../../cli/terminal-style"; import { redactFullWithUrls } from "../../security/redact"; import { hermesMcpReconciliationRemediationLines } from "./mcp-bridge-hermes-reconciliation"; -import { inspectHermesMcpReconciliationRefusal } from "./mcp-bridge-recovery"; import { assertHermesPortableCommandUnavailable } from "../../onboard/experimental/portable-agent-lifecycle"; import { withMcpLifecycleLockSync } from "../../state/mcp-lifecycle-lock-acquisition"; @@ -85,13 +84,6 @@ export type GatewayRestartResult = detail: string; restarted?: never; healthPassed?: never; - } - | { - ok: false; - failureLayer: "MCP reconciliation refusal"; - detail: string; - restarted: true; - healthPassed: true; }; type SandboxAgentLookup = (sandboxName: string) => { agent?: string | null } | null | undefined; @@ -112,12 +104,11 @@ const GATEWAY_RESTART_SUPPORTED_AGENTS = ["openclaw", "hermes"] as const; // Substrings of the in-sandbox supervisor's quarantine lines. The supervisor // only forwards allowlisted lines to the host, so matching them is what tells -// the host that no further relaunch will be attempted until the sandbox is -// rebuilt. Keep in sync with the quarantine messages in agents/hermes/start.sh +// the host that no further relaunch will be attempted by this supervisor +// instance. Keep in sync with the messages in agents/hermes/start.sh // and their allowlist in scripts/managed-gateway-control.py. const GATEWAY_RELAUNCH_QUARANTINE_MARKERS = [ - "quarantined until sandbox recreation", - "quarantined until MCP integrity is restored", + "relaunch is stopped for this supervisor instance", "quarantined without another launch", "quarantining the managed startup supervisor", ] as const; @@ -148,7 +139,6 @@ export type GatewayRestartDeps = { sandboxName: string, exec: (sandboxName: string, command: string) => GatewayRestartCommandResult | null, ) => boolean; - inspectHermesMcpReconciliationRefusal: typeof inspectHermesMcpReconciliationRefusal; }; export type RestartSandboxGatewayOptions = { @@ -257,10 +247,9 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul } // A quarantined supervisor is the strictly more specific and terminal fact: // it stops attempting relaunch entirely, so the controller then reports the - // generic health timeout it would report for any unresponsive gateway, and a - // config refusal that tripped the crash budget is reported as MCP drift by the - // non-root startup guard. Classify the quarantine ahead of both so the host - // names the state that actually blocks recovery instead of its side effect. + // generic health timeout it would report for any unresponsive gateway. + // Classify the quarantine first so the host names the state that blocks + // recovery instead of its health-check side effect. if (GATEWAY_RELAUNCH_QUARANTINE_MARKERS.some((marker) => output.includes(marker))) { return { layer: "relaunch quarantined", @@ -293,32 +282,28 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul return { layer: "launch failure", detail: detail || `restart exited ${result.status}` }; } -export function isGatewayIntegrityRepairLayer( +export function isGatewayTerminalRepairLayer( layer: GatewayRestartFailureLayer | null | undefined, ): layer is "config hash mismatch" | "relaunch quarantined" { return layer === "config hash mismatch" || layer === "relaunch quarantined"; } -/** - * The supported repair for a sandbox whose protected configuration drifted away - * from its recorded integrity metadata. Both layers are deterministic refusals: - * every relaunch re-reads the same drifted file, so retrying a restart or a - * recover only burns the supervisor's crash budget. `rebuild` is the documented - * command that restores the registered configuration, refreshes the integrity - * hashes, and brings the gateway back in one transaction (#7801). - */ -export function gatewayIntegrityRepairLines( +/** Report terminal restart repair without treating process quarantine as config drift. */ +export function gatewayTerminalRepairLines( sandboxName: string, layer: "config hash mismatch" | "relaunch quarantined", ): readonly string[] { - const cause = - layer === "config hash mismatch" - ? "A protected configuration file no longer matches its recorded integrity hash." - : "The in-sandbox supervisor quarantined gateway relaunch after a startup refusal."; + if (layer === "relaunch quarantined") { + return [ + "The in-sandbox supervisor stopped relaunch after repeated process or health failures.", + `Inspect the Hermes failure with \`nemoclaw ${sandboxName} logs --tail 50\`.`, + `After correcting the cause, reset the supervisor with \`nemoclaw ${sandboxName} stop\`, then \`nemoclaw ${sandboxName} start\`.`, + `If the sandbox still cannot start, rebuild it with \`nemoclaw ${sandboxName} rebuild --yes\`.`, + ]; + } return [ - `${cause} Retrying the restart cannot clear it.`, + "The restart transaction could not validate its integrity metadata.", `Restore the registered configuration and refresh its integrity metadata with \`nemoclaw ${sandboxName} rebuild --yes\`.`, - `Then make intended changes through supported commands such as \`nemoclaw ${sandboxName} config set\` or \`nemoclaw inference set --sandbox ${sandboxName}\`, which update the configuration and its hashes together.`, ]; } @@ -365,8 +350,8 @@ export function printGatewayRestartFailure( console.error(" Hermes gateway log tail (sanitized):"); for (const line of gatewayLogTail) console.error(` ${line}`); } - if (isGatewayIntegrityRepairLayer(layer)) { - for (const line of gatewayIntegrityRepairLines(sandboxName, layer)) { + if (isGatewayTerminalRepairLayer(layer)) { + for (const line of gatewayTerminalRepairLines(sandboxName, layer)) { console.error(` ${line}`); } } @@ -483,21 +468,6 @@ export function restartSandboxGatewayWithDeps( return { ok: false, failureLayer: "health timeout", detail }; } - if (agentName === "hermes") { - const refusal = deps.inspectHermesMcpReconciliationRefusal(sandboxName); - if (refusal) { - const { detail } = refusal; - printGatewayRestartFailure(sandboxName, "MCP reconciliation refusal", detail); - return { - ok: false, - failureLayer: "MCP reconciliation refusal", - detail, - restarted: true, - healthPassed: true, - }; - } - } - const forwardRecovered = deps.ensureSandboxPortForward(sandboxName); const dashboardForwardRecovered = deps.ensureHermesDashboardPortForwardIfEnabled(sandboxName); const messagingForwardRecovered = deps.recoverMessagingHostForward(sandboxName, { quiet }); diff --git a/src/lib/actions/sandbox/mcp-bridge-recovery.test.ts b/src/lib/actions/sandbox/mcp-bridge-recovery.test.ts deleted file mode 100644 index 623b45a04c5..00000000000 --- a/src/lib/actions/sandbox/mcp-bridge-recovery.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - inspectHermesMcpReconciliationRefusal, - processRecoveryMcpReconciliationRefusal, -} from "./mcp-bridge-recovery"; - -describe("Hermes MCP recovery boundary (#6257)", () => { - it("continues when runtime intent matches", () => { - expect( - inspectHermesMcpReconciliationRefusal("alpha", () => ({ - ok: true, - state: "matched", - })), - ).toBeNull(); - }); - - it("sanitizes a reconciliation refusal once at the shared boundary", () => { - expect( - inspectHermesMcpReconciliationRefusal("alpha", () => ({ - ok: false, - state: "mismatch", - detail: "\u001b[31mdrifted\u001b[0m\nFORGED", - })), - ).toEqual({ detail: "drifted FORGED" }); - }); - - it.each([true, false])("maps refusal into the process recovery contract (%s)", (wasRunning) => { - expect( - processRecoveryMcpReconciliationRefusal("alpha", wasRunning, () => ({ - ok: false, - state: "error", - detail: "runtime mismatch", - })), - ).toEqual({ - checked: true, - wasRunning, - recovered: false, - forwardRecovered: false, - mcpReconciliationRefused: true, - mcpReconciliationReason: "runtime mismatch", - }); - }); -}); diff --git a/src/lib/actions/sandbox/mcp-bridge-recovery.ts b/src/lib/actions/sandbox/mcp-bridge-recovery.ts deleted file mode 100644 index 927a35e51c6..00000000000 --- a/src/lib/actions/sandbox/mcp-bridge-recovery.ts +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { - type HermesMcpReconciliationResult, - inspectHermesMcpRuntimeIntent, - sanitizeHermesMcpReconciliationDetail, -} from "./mcp-bridge-hermes-reconciliation"; -import type { McpProviderInspectionRuntimeSelection } from "./mcp-bridge-provider-inspection"; - -export type McpReconciliationRefusalRecoveryResult = { - checked: true; - wasRunning: boolean; - recovered: false; - forwardRecovered: false; - forwardRecoveryFailed?: undefined; - forwardRecoveryFailureDetail?: undefined; - mcpReconciliationRefused: true; - mcpReconciliationReason: string; -}; - -type InspectHermesMcpRuntimeIntent = (sandboxName: string) => HermesMcpReconciliationResult; - -export function inspectHermesMcpReconciliationRefusal( - sandboxName: string, - inspect: InspectHermesMcpRuntimeIntent = inspectHermesMcpRuntimeIntent, - runtimeSelection?: McpProviderInspectionRuntimeSelection, -): { detail: string } | null { - const reconciliation = - inspect === inspectHermesMcpRuntimeIntent - ? inspectHermesMcpRuntimeIntent(sandboxName, { runtimeSelection }) - : inspect(sandboxName); - if (reconciliation.ok) return null; - return { detail: sanitizeHermesMcpReconciliationDetail(reconciliation.detail) }; -} - -export function processRecoveryMcpReconciliationRefusal( - sandboxName: string, - wasRunning: boolean, - inspect: InspectHermesMcpRuntimeIntent = inspectHermesMcpRuntimeIntent, - runtimeSelection?: McpProviderInspectionRuntimeSelection, -): McpReconciliationRefusalRecoveryResult | null { - const refusal = inspectHermesMcpReconciliationRefusal(sandboxName, inspect, runtimeSelection); - if (!refusal) return null; - return { - checked: true, - wasRunning, - recovered: false, - forwardRecovered: false, - mcpReconciliationRefused: true, - mcpReconciliationReason: refusal.detail, - }; -} diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 868cf33ca03..92e1b0cc5d7 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -62,8 +62,8 @@ import { type GatewayRestartDeps, type GatewayRestartFailureLayer, type GatewayRestartResult, - gatewayIntegrityRepairLines, - isGatewayIntegrityRepairLayer, + gatewayTerminalRepairLines, + isGatewayTerminalRepairLayer, MANAGED_CONTROL_IDENTITY_CHANGED_MARKER, type ManagedGatewayControlCompletion, parseManagedGatewayControlCompletion, @@ -75,10 +75,6 @@ import { } from "./gateway-restart"; import { printGatewayWedgeDiagnostics } from "./gateway-wedge-diagnostics"; import { enforceHermesSecretBoundaryOnRunningGateway } from "./hermes-secret-boundary-recovery"; -import { - inspectHermesMcpReconciliationRefusal, - processRecoveryMcpReconciliationRefusal, -} from "./mcp-bridge-recovery"; import { buildSandboxExecMarkedCommand, extractSandboxExecCommandStdout, @@ -1011,12 +1007,6 @@ export function restartSandboxGateway( runtimeSelection, }), printGatewayWedgeDiagnostics, - inspectHermesMcpReconciliationRefusal: (name) => - inspectHermesMcpReconciliationRefusal( - name, - undefined, - runtimeSelection, - ), ...deps, }, }), @@ -1333,11 +1323,10 @@ function printHostManagedGatewayRecoveryHints( console.error(" If rebuild is blocked, destroy and re-onboard the sandbox to restore it."); return; } - // A drifted protected config and a quarantined supervisor both refuse every - // relaunch deterministically, so the generic "retry the managed restart" hint - // below would send the operator into a loop that cannot succeed (#7801). - if (isGatewayIntegrityRepairLayer(failureLayer)) { - for (const line of gatewayIntegrityRepairLines(quotedSandboxName, failureLayer)) { + // These terminal states need their specific repair before another managed + // restart. The generic hint below would otherwise repeat the same failure. + if (isGatewayTerminalRepairLayer(failureLayer)) { + for (const line of gatewayTerminalRepairLines(quotedSandboxName, failureLayer)) { console.error(` ${line}`); } return; @@ -1595,13 +1584,6 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( secretBoundaryReason: enforcement.reason, }; } - const mcpRefusal = processRecoveryMcpReconciliationRefusal( - sandboxName, - true, - undefined, - runtimeSelection, - ); - if (mcpRefusal) return mcpRefusal; } if (running) { // Gateway is alive but the host-side forward can still be dead or @@ -1929,13 +1911,6 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( ); if (finalizationFailure) return finalizationFailure; } - const mcpRefusal = processRecoveryMcpReconciliationRefusal( - sandboxName, - false, - undefined, - runtimeSelection, - ); - if (mcpRefusal) return mcpRefusal; const forwardRecovered = measure("forward", () => ensureSandboxPortForward(sandboxName, { afterSuccess: confirmRelaunchedManagedHealthForForward ?? undefined, diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts index 7fe32822b58..9bb966f1250 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -23,58 +23,16 @@ const RESTART_FAILED = { failureLayer: "health timeout", detail: "gateway did not become healthy", } as const; -const RESTARTED_WITH_MCP_MISMATCH = { - ok: false, - failureLayer: "MCP reconciliation refusal", - detail: "Hermes MCP config does not match persisted managed intent", - restarted: true, - healthPassed: true, -} as const; -const MCP_REFUSED_BEFORE_RESTART = { +const RESTART_REFUSED = { ok: false, failureLayer: "MCP reconciliation refusal", detail: "supervisor refused the restart before replacing the gateway", } as const; describe("binding the Hermes gateway to restored state", () => { - - it("keeps restart evidence while rebuild restores the managed MCP projection (#8671)", () => { - const restartState = restartHermesGatewayAfterStateRestore("alpha", "hermes", { - restartSandboxGateway: () => RESTARTED_WITH_MCP_MISMATCH, - }); - - expect(restartState).toBe("restarted"); - expect( - verifyHermesGatewayAfterStateRestore("alpha", "hermes", restartState, { - checkAndRecoverSandboxProcesses: () => ({ - checked: true, - wasRunning: true, - recovered: false, - }), - }), - ).toBe("healthy"); - }); - - it("rejects managed MCP drift that remains after rebuild restoration (#8671)", () => { - const restartState = restartHermesGatewayAfterStateRestore("alpha", "hermes", { - restartSandboxGateway: () => RESTARTED_WITH_MCP_MISMATCH, - }); - - expect( - verifyHermesGatewayAfterStateRestore("alpha", "hermes", restartState, { - checkAndRecoverSandboxProcesses: () => ({ - checked: true, - wasRunning: true, - recovered: false, - mcpReconciliationRefused: true, - }), - }), - ).toBe("unverified"); - }); - it("preserves an MCP refusal before gateway replacement (#8671)", () => { const restartState = restartHermesGatewayAfterStateRestore("alpha", "hermes", { - restartSandboxGateway: () => MCP_REFUSED_BEFORE_RESTART, + restartSandboxGateway: () => RESTART_REFUSED, }); expect(restartState).toBe("restart-failed"); @@ -139,22 +97,6 @@ describe("binding the Hermes gateway to restored state", () => { expect(restartSandboxGateway).not.toHaveBeenCalled(); }); - it("preserves MCP reconciliation refusal during restart-free final verification (#7084)", () => { - const restartSandboxGateway = vi.fn(() => RESTART_SUCCEEDED); - - expect( - verifyHermesGatewayAfterStateRestore("alpha", "hermes", "restarted", { - restartSandboxGateway, - checkAndRecoverSandboxProcesses: () => ({ - checked: true, - wasRunning: true, - recovered: false, - mcpReconciliationRefused: true, - }), - }), - ).toBe("unverified"); - expect(restartSandboxGateway).not.toHaveBeenCalled(); - }); }); describe("Hermes rebuild post-restore verification", () => { @@ -254,38 +196,6 @@ describe("Hermes rebuild post-restore verification", () => { expect(output).not.toContain("rebuilt successfully"); }); - it("fails when the final gateway check refuses MCP reconciliation (#7084)", async () => { - const mcpEntry = { - server: "blender", - providerName: "nemoclaw-mcp-alpha-blender", - }; - const harness = createRebuildFlowHarness({ - agentName: "hermes", - checkAndRecoverSandboxProcesses: () => ({ - checked: true, - wasRunning: true, - recovered: false, - forwardRecovered: false, - mcpReconciliationRefused: true, - }), - mcpPreparation: { - entries: [mcpEntry], - detachedProviderEntries: [mcpEntry], - scrubbedAdapterEntries: [mcpEntry], - }, - sandboxEntry: { agent: "hermes" }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Hermes post-restore verification failed"); - - expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); - expect(harness.logSpy).not.toHaveBeenCalledWith( - expect.stringContaining("rebuilt successfully"), - ); - }); - it.each(["forwardRecoveryFailed", "secretBoundaryRefused"] as const)( "fails when the final gateway check reports %s (#7084)", async (failureFlag) => { diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts index 8d26fb4ea60..24686b3f569 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts @@ -97,7 +97,6 @@ type GatewayRecoveryObservation = { recovered: boolean; forwardRecoveryFailed?: boolean; secretBoundaryRefused?: boolean; - mcpReconciliationRefused?: boolean; }; interface HermesPostRestoreGatewayDeps { @@ -152,12 +151,7 @@ export function restartHermesGatewayAfterStateRestore( ...(deps.runtimeSelection ? { runtimeSelection: deps.runtimeSelection } : {}), }); if (result.ok) return "restarted"; - const mcpRestoreCanSupersede = - result.failureLayer === "MCP reconciliation refusal" && - result.restarted === true && - result.healthPassed === true; - // Final verification still requires MCP reconciliation after restoration. - return mcpRestoreCanSupersede ? "restarted" : "restart-failed"; + return "restart-failed"; } export function verifyHermesGatewayAfterStateRestore( @@ -222,11 +216,7 @@ function verifyHermesGatewayAfterStateRestoreImpl( quiet: true, ...(deps.runtimeSelection ? { runtimeSelection: deps.runtimeSelection } : {}), }); - if ( - observation.forwardRecoveryFailed === true || - observation.secretBoundaryRefused === true || - observation.mcpReconciliationRefused === true - ) { + if (observation.forwardRecoveryFailed === true || observation.secretBoundaryRefused === true) { return { state: "unverified" }; } if (!observation.checked) continue; diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 3a35164be90..f70fa1bc97b 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -100,9 +100,6 @@ function startupRecoveryFailure(check: SandboxStartupRecoveryResult): string | n if ("secretBoundaryRefused" in check && check.secretBoundaryRefused) { return `secret-boundary refusal: ${String(check.secretBoundaryReason)}`; } - if ("mcpReconciliationRefused" in check && check.mcpReconciliationRefused) { - return `MCP reconciliation refusal: ${String(check.mcpReconciliationReason)}`; - } if ("forwardRecoveryFailed" in check && check.forwardRecoveryFailed) { return String(check.forwardRecoveryFailureDetail); } diff --git a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts index 77dc97a3e45..0d1fa24b80b 100644 --- a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts @@ -197,17 +197,6 @@ describe("collectSandboxStatusSnapshot Docker recovery", () => { secretBoundaryReason: "persisted secret boundary refused recovery", }, ], - [ - "mcp-reconciliation", - { - checked: true, - wasRunning: true, - recovered: false, - forwardRecovered: false, - mcpReconciliationRefused: true, - mcpReconciliationReason: "MCP intent mismatch", - }, - ], [ "gateway-recovery", { diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index b47becf14c9..65d0c44a853 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -268,7 +268,6 @@ type SandboxProcessRecoveryFailure = { layer: | "inspection" | "secret-boundary" - | "mcp-reconciliation" | "gateway-recovery" | "forward-recovery" | "recovery-error"; @@ -328,16 +327,6 @@ function processRecoveryFailure( ), }; } - if ("mcpReconciliationRefused" in result && result.mcpReconciliationRefused) { - return { - layer: "mcp-reconciliation", - detail: sanitizedStatusDetail( - "mcpReconciliationReason" in result - ? result.mcpReconciliationReason - : "MCP reconciliation refused recovery", - ), - }; - } if ("forwardRecoveryFailed" in result && result.forwardRecoveryFailed) { return { layer: "forward-recovery", diff --git a/test/agents/hermes/hermes-gateway-supervisor-recovery.test.ts b/test/agents/hermes/hermes-gateway-supervisor-recovery.test.ts index 4da797955a9..7c68b639001 100644 --- a/test/agents/hermes/hermes-gateway-supervisor-recovery.test.ts +++ b/test/agents/hermes/hermes-gateway-supervisor-recovery.test.ts @@ -19,7 +19,7 @@ const SUPERVISOR_LIB = path.join( "gateway-supervisor.sh", ); -function runHermesHealthyGatewayRecovery(integrityStatus: 0 | 1) { +function runHermesHealthyGatewayRecovery(adoptionStatus: 0 | 1) { const source = fs.readFileSync(START_SCRIPT, "utf-8"); return runBashHarness([ 'trace() { printf "%s\\n" "$*"; }', @@ -27,7 +27,9 @@ function runHermesHealthyGatewayRecovery(integrityStatus: 0 | 1) { 'gateway_control_pid_is_live() { trace "pid-live:$1"; return 0; }', "hermes_gateway_healthy() { trace gateway-healthy; return 0; }", "validate_running_hermes_boundary() { trace boundary-validation; return 0; }", - `verify_hermes_config_integrity() { trace strict-integrity; return ${integrityStatus}; }\nprepare_hermes_lazy_dependencies() { return 0; }`, + `refresh_hermes_runtime_config_hashes() { trace "adopt-config:$*"; return ${adoptionStatus}; }`, + "inspect_hermes_mcp_integrity() { trace mcp-integrity; return 0; }", + "prepare_hermes_lazy_dependencies() { return 0; }", "hermes_auxiliaries_need_recovery() { trace auxiliaries-needed; return 0; }", "seal_hermes_restart_inputs() { trace seal-inputs; return 0; }", "unseal_hermes_restart_inputs() { trace unseal-inputs; return 0; }", @@ -40,6 +42,7 @@ function runHermesHealthyGatewayRecovery(integrityStatus: 0 | 1) { extractShellFunction(source, "prepare_hermes_gateway_restart"), extractShellFunction(source, "handle_hermes_gateway_control_request"), "GATEWAY_PID=4242", + "HERMES_HASH_FILE=/etc/nemoclaw/hermes.config-hash", "HERMES_RESTART_FAILURE_CODE=internal", 'if handle_hermes_gateway_control_request; then trace "handler-rc:0"; else trace "handler-rc:$?"; fi', ]); @@ -54,7 +57,9 @@ function runHermesGatewayProbe(opts: { return runBashHarness([ 'trace() { printf "%s\\n" "$*"; }', "gateway_control_take_request() { GATEWAY_CONTROL_ACTION=probe; trace take-request; }", - `prepare_hermes_gateway_restart() { HERMES_RESTART_FAILURE_CODE=hash-mismatch; trace preflight; return ${opts.prepareStatus}; }`, + `validate_running_hermes_boundary() { HERMES_RESTART_FAILURE_CODE=secret-boundary-refusal; trace preflight; return ${opts.prepareStatus}; }`, + "refresh_hermes_runtime_config_hashes() { trace unexpected-adopt; }", + "inspect_hermes_mcp_integrity() { trace unexpected-mcp-inspection; }", 'gateway_control_pid_is_live() { trace "pid-live:$1"; return 0; }', `hermes_gateway_healthy() { trace "gateway-healthy:$1"; return ${opts.healthStatus}; }`, `hermes_auxiliaries_need_recovery() { trace auxiliaries-check; return ${opts.auxiliariesStatus}; }`, @@ -68,6 +73,7 @@ function runHermesGatewayProbe(opts: { "kill() { trace unexpected-signal; }", extractShellFunction(source, "handle_hermes_gateway_control_request"), "GATEWAY_PID=4242", + "HERMES_MCP_RECONCILE_PENDING=1", "HERMES_RESTART_FAILURE_CODE=internal", 'if handle_hermes_gateway_control_request; then trace "handler-rc:0"; else trace "handler-rc:$?"; fi', ]); @@ -134,7 +140,7 @@ describe("Hermes PID 1 supervisor recovery", () => { expect(result.stderr).toContain("privileged gateway control unavailable"); }); - it("validates the strict trust anchor before healthy-recover auxiliaries", () => { + it("adopts mutable config before healthy-recover auxiliaries (#11108)", () => { const result = runHermesHealthyGatewayRecovery(0); expect(result.status, result.stderr).toBe(0); @@ -143,11 +149,13 @@ describe("Hermes PID 1 supervisor recovery", () => { "pid-live:4242", "gateway-healthy", "boundary-validation", - "strict-integrity", + "adopt-config:both adopt", + "mcp-integrity", "auxiliaries-needed", "seal-inputs", "boundary-validation", - "strict-integrity", + "adopt-config:both adopt", + "mcp-integrity", "auxiliaries", "unseal-inputs", "refresh-child-pids", @@ -156,7 +164,7 @@ describe("Hermes PID 1 supervisor recovery", () => { ]); }); - it("does not start healthy-recover auxiliaries when strict validation fails", () => { + it("does not start healthy-recover auxiliaries when config adoption fails (#11108)", () => { const result = runHermesHealthyGatewayRecovery(1); expect(result.status, result.stderr).toBe(0); @@ -165,7 +173,7 @@ describe("Hermes PID 1 supervisor recovery", () => { "pid-live:4242", "gateway-healthy", "boundary-validation", - "strict-integrity", + "adopt-config:both adopt", "fail:hash-mismatch:4242", "handler-rc:1", ]); @@ -193,7 +201,12 @@ describe("Hermes PID 1 supervisor recovery", () => { prepareStatus: 1 as const, healthStatus: 0 as const, auxiliariesStatus: 1 as const, - expected: ["take-request", "preflight", "fail:hash-mismatch:4242", "handler-rc:1"], + expected: [ + "take-request", + "preflight", + "fail:secret-boundary-refusal:4242", + "handler-rc:1", + ], }, { label: "reports an unhealthy gateway", @@ -570,7 +583,8 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.stdout).toContain("recover:5004"); expect(result.stdout).not.toContain("recover:5005"); expect(result.stdout).toContain("quarantine"); - expect(result.stderr).toContain("relaunch is quarantined until sandbox recreation"); + expect(result.stderr).toContain("relaunch is stopped for this supervisor instance"); + expect(result.stderr).toContain("stop and start the sandbox"); }); it("starts a recovered gateway with a fresh consecutive health-failure budget", () => { @@ -656,7 +670,7 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.stdout).not.toContain("unexpected-auxiliary"); }); - it("does not count preparation refusals or launch before preparation succeeds", () => { + it("retries MCP preparation refusals without entering quarantine (#11108)", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ 'trace() { printf "%s\\n" "$*"; }', @@ -697,6 +711,39 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.stdout).not.toContain("unexpected-exit-record"); }); + it("bounds persistent preparation failure without config quarantine (#11108)", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + 'prepare_hermes_nonroot_runtime() { prepare_calls=$((prepare_calls + 1)); trace "prepare:$prepare_calls"; return 1; }', + "launch_hermes_gateway_current_user() { trace unexpected-launch; }", + 'sleep() { trace "sleep:$1"; }', + extractShellFunction(source, "recover_hermes_gateway_current_user"), + "prepare_calls=0", + "if recover_hermes_gateway_current_user; then trace unexpected-success; else trace failed; fi", + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "prepare:1", + "sleep:5", + "prepare:2", + "sleep:5", + "prepare:3", + "sleep:5", + "prepare:4", + "sleep:5", + "prepare:5", + "failed", + ]); + expect(result.stderr).toContain( + "runtime preparation failed after 5 consecutive attempts; supervisor exiting", + ); + expect(result.stderr).toContain("correct the reported failure, then stop and start the sandbox"); + expect(result.stderr).not.toContain("quarantin"); + expect(result.stdout).not.toContain("unexpected-"); + }); + it("keeps the initial non-root supervisor alive and recovers a failed first child", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ @@ -972,7 +1019,8 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.status, result.stderr).toBe(0); expect(result.stdout.trim().split("\n")).toEqual(["wait:4242", "quarantine-sleep:60"]); expect(result.stderr).toContain("quarantining the managed startup supervisor"); - expect(result.stderr).toContain("remains quarantined until sandbox recreation"); + expect(result.stderr).toContain("relaunch is stopped for this supervisor instance"); + expect(result.stderr).toContain("stop and start the sandbox"); expect(result.stdout).not.toContain("unexpected-signal"); expect(result.stdout).not.toContain("unexpected-return"); }); diff --git a/test/agents/hermes/hermes-mcp-integrity-state.test.ts b/test/agents/hermes/hermes-mcp-integrity-state.test.ts index 423e944cb4d..47567b0b370 100644 --- a/test/agents/hermes/hermes-mcp-integrity-state.test.ts +++ b/test/agents/hermes/hermes-mcp-integrity-state.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -255,7 +256,7 @@ print(json.dumps({"current": current, "pending": pending, "misuse": misuse})) expect(JSON.parse(result.stdout)).toEqual({ current: 0, pending: 10, misuse: 1 }); }); - it("refreshes safe mutable compatibility drift without changing MCP intent (#9203)", () => { + it("adopts valid runtime config regardless of stale host MCP intent (#11108)", () => { const result = spawnSync( "python3", [ @@ -313,13 +314,65 @@ except guard.UnsafePathError as error: else: mcp_drift_error = "" +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="adopt") +adopted_state = guard.inspect_mcp_integrity(hermes, anchor) +adopted_text = open(anchor, encoding="utf-8").read() +_config_digest, _env_digest, adopted_mcp = guard._parse_config_hash( + adopted_text, config, env +) +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="apply") +applied_state = guard.inspect_mcp_integrity(hermes, anchor) + +write_inputs("after", "two", "https://pending.example/mcp") +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="intend") +pending_text = open(anchor, encoding="utf-8").read() +write_inputs("after", "two", "https://conflict.example/mcp") +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="adopt") +superseded_state = guard.inspect_mcp_integrity(hermes, anchor) +superseded_text = open(anchor, encoding="utf-8").read() +_config_digest, _env_digest, superseded_mcp = guard._parse_config_hash( + superseded_text, config, env +) +guard.refresh_hashes(hermes, anchor, "compat", mcp_transition="apply") +superseded_applied_state = guard.inspect_mcp_integrity(hermes, anchor) + +strict = os.path.join(root, "hermes.config-hash") +current_text = open(anchor, encoding="utf-8").read() +guard._write_hash(strict, current_text) +_config_digest, _env_digest, root_before = guard._parse_config_hash( + current_text, config, env +) +write_inputs("after", "two", "https://root.example/mcp") +guard.refresh_hashes(hermes, strict, "both", mcp_transition="adopt") +root_pending_text = open(strict, encoding="utf-8").read() +_config_digest, _env_digest, root_pending = guard._parse_config_hash( + root_pending_text, config, env +) +root_pending_state = guard.inspect_mcp_integrity(hermes, strict) +root_anchors_equal = root_pending_text == open(anchor, encoding="utf-8").read() +guard.refresh_hashes(hermes, strict, "both", mcp_transition="apply") +root_applied_state = guard.inspect_mcp_integrity(hermes, strict) + proof = { "stale_rejected": stale_rejected, "refreshed_state": refreshed_state, "intended_preserved": refreshed_mcp.intended == initial_state.intended, "applied_preserved": refreshed_mcp.applied == initial_state.applied, "mcp_drift_error": mcp_drift_error, - "anchor_unchanged_after_mcp_drift": open(anchor, encoding="utf-8").read() == refreshed_text, + "adopted_state": adopted_state, + "adopted_intended_changed": adopted_mcp.intended != initial_state.intended, + "adopted_applied_preserved": adopted_mcp.applied == initial_state.applied, + "applied_state": applied_state, + "superseded_state": superseded_state, + "superseded_intended_changed": superseded_mcp.intended != adopted_mcp.intended, + "superseded_applied_preserved": superseded_mcp.applied == adopted_mcp.intended, + "pending_anchor_replaced": superseded_text != pending_text, + "superseded_applied_state": superseded_applied_state, + "root_pending_state": root_pending_state, + "root_anchors_equal": root_anchors_equal, + "root_intended_changed": root_pending.intended != root_before.intended, + "root_applied_preserved": root_pending.applied == root_before.applied, + "root_applied_state": root_applied_state, } shutil.rmtree(root) print(json.dumps(proof)) @@ -336,10 +389,82 @@ print(json.dumps(proof)) intended_preserved: true, applied_preserved: true, mcp_drift_error: "Hermes MCP config differs from persisted intended state", - anchor_unchanged_after_mcp_drift: true, + adopted_state: "pending", + adopted_intended_changed: true, + adopted_applied_preserved: true, + applied_state: "current", + superseded_state: "pending", + superseded_intended_changed: true, + superseded_applied_preserved: true, + pending_anchor_replaced: true, + superseded_applied_state: "current", + root_pending_state: "pending", + root_anchors_equal: true, + root_intended_changed: true, + root_applied_preserved: true, + root_applied_state: "current", }); }); + it("passes adopt from the shell wrapper to the guard CLI (#11108)", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-adopt-cli-")); + const hermesDir = path.join(root, ".hermes"); + const configPath = path.join(hermesDir, "config.yaml"); + const envPath = path.join(hermesDir, ".env"); + const anchor = path.join(hermesDir, ".config-hash"); + const strict = path.join(root, "hermes.config-hash"); + const beforeConfig = "model: test\nmcp_servers: {}\n"; + const afterConfig = + "model: test\nmcp_servers:\n alpha:\n url: https://alpha.example/mcp\n"; + const env = "SAFE=1\n"; + const digest = (value: string) => createHash("sha256").update(value).digest("hex"); + const beforeMcp = digest("{}"); + const afterMcp = digest('{"alpha":{"url":"https://alpha.example/mcp"}}'); + const initialHash = + `${digest(beforeConfig)} ${configPath}\n` + + `${digest(env)} ${envPath}\n` + + `# nemoclaw-hermes-mcp-state-v1 intended=${beforeMcp} applied=${beforeMcp}\n`; + const source = fs.readFileSync(START, "utf-8"); + + fs.mkdirSync(hermesDir); + fs.writeFileSync(configPath, afterConfig); + fs.writeFileSync(envPath, env); + fs.writeFileSync(anchor, initialHash); + + try { + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -uo pipefail", + extractShellFunction(source, "refresh_hermes_runtime_config_hashes"), + `_HERMES_PYTHON=${bashPrintfQ(process.env.PYTHON || "python3")}`, + `_HERMES_RUNTIME_CONFIG_GUARD=${bashPrintfQ(GUARD)}`, + `HERMES_DIR=${bashPrintfQ(hermesDir)}`, + `HERMES_HASH_FILE=${bashPrintfQ(strict)}`, + "STEP_DOWN_PREFIX_SANDBOX=(env)", + "if refresh_hermes_runtime_config_hashes compat; then preserve=0; else preserve=$?; fi", + "refresh_hermes_runtime_config_hashes compat adopt", + 'printf "preserve=%s\\n" "$preserve"', + ].join("\n"), + ], + { encoding: "utf-8", timeout: 10_000 }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("preserve=1\n"); + expect(result.stderr).toContain("Hermes MCP config differs from persisted intended state"); + expect(fs.readFileSync(anchor, "utf-8")).toBe( + `${digest(afterConfig)} ${configPath}\n` + + `${digest(env)} ${envPath}\n` + + `# nemoclaw-hermes-mcp-state-v1 intended=${afterMcp} applied=${beforeMcp}\n`, + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("uses the atomic write outcome for compat applied-state commits", () => { const result = spawnSync( "python3", @@ -404,7 +529,7 @@ print(json.dumps({ }); }); - it("validates but does not replace already-current applied-state anchors", () => { + it("validates without replacing current apply or adopt anchors (#11108)", () => { const result = spawnSync( "python3", [ @@ -439,6 +564,8 @@ def captured_write_hash(path, text): guard._write_hash = captured_write_hash guard.refresh_hashes(hermes, strict, "both", mcp_transition="apply") guard.refresh_hashes(hermes, strict, "compat", mcp_transition="apply") +guard.refresh_hashes(hermes, strict, "both", mcp_transition="adopt") +guard.refresh_hashes(hermes, strict, "compat", mcp_transition="adopt") after = {path: os.stat(path).st_ino for path in (strict, compat)} print(json.dumps({ "state": guard.inspect_mcp_integrity(hermes, strict), @@ -510,9 +637,9 @@ print(json.dumps({ }); it.each([ - { status: 0, expected: "rc=0 pending=0 failed=0\n" }, - { status: 10, expected: "rc=0 pending=1 failed=0\n" }, - { status: 1, expected: "rc=1 pending=9 failed=1\n" }, + { status: 0, expected: "rc=0 pending=0\n" }, + { status: 10, expected: "rc=0 pending=1\n" }, + { status: 1, expected: "rc=1 pending=9\n" }, ])("uses only the authenticated guard exit status ($status)", ({ status, expected }) => { const source = fs.readFileSync(START, "utf-8"); const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-status-")); @@ -541,9 +668,8 @@ print(json.dumps({ "HERMES_DIR=/test/.hermes", "HERMES_HASH_FILE=/test/hermes.config-hash", "HERMES_MCP_RECONCILE_PENDING=9", - "HERMES_MCP_INTEGRITY_FAILED=0", "if inspect_hermes_mcp_integrity; then rc=0; else rc=$?; fi", - 'printf "rc=%s pending=%s failed=%s\\n" "$rc" "$HERMES_MCP_RECONCILE_PENDING" "$HERMES_MCP_INTEGRITY_FAILED"', + 'printf "rc=%s pending=%s\\n" "$rc" "$HERMES_MCP_RECONCILE_PENDING"', ].join("\n"), ], { encoding: "utf-8", timeout: 5000 }, diff --git a/test/agents/hermes/hermes-start.test.ts b/test/agents/hermes/hermes-start.test.ts index 352421a17e2..696fe3102af 100644 --- a/test/agents/hermes/hermes-start.test.ts +++ b/test/agents/hermes/hermes-start.test.ts @@ -177,7 +177,11 @@ function runHermesPortValidation(opts: { } } -function runHermesEnvSecretBoundary(opts: { envFile?: string; symlinkEnvFile?: boolean }) { +function runHermesEnvSecretBoundary(opts: { + envFile?: string; + symlinkEnvFile?: boolean; + prepareMode?: "nonroot" | "root"; +}) { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-env-boundary-")); const hermesHome = path.join(tmpDir, ".hermes"); const envFile = path.join(hermesHome, ".env"); @@ -193,6 +197,13 @@ function runHermesEnvSecretBoundary(opts: { envFile?: string; symlinkEnvFile?: b } const src = fs.readFileSync(START_SCRIPT, "utf-8"); + const boundaryInvocation = opts.prepareMode + ? [ + 'refresh_hermes_runtime_config_hashes() { printf "unexpected-adopt:%s\\n" "$*"; }', + extractShellFunctionFromSource(src, `prepare_hermes_${opts.prepareMode}_runtime`), + `prepare_hermes_${opts.prepareMode}_runtime`, + ] + : ["validate_hermes_env_secret_boundary"]; fs.writeFileSync( scriptPath, [ @@ -207,7 +218,7 @@ function runHermesEnvSecretBoundary(opts: { envFile?: string; symlinkEnvFile?: b extractShellFunctionFromSource(src, "validate_hermes_env_secret_boundary"), `HERMES_DIR=${shellQuote(hermesHome)}`, `_HERMES_BOUNDARY_VALIDATOR=${shellQuote(SECRET_BOUNDARY_VALIDATOR_SCRIPT)}`, - "validate_hermes_env_secret_boundary", + ...boundaryInvocation, ].join("\n"), { mode: 0o700 }, ); @@ -356,13 +367,13 @@ function runHermesRootStartupMutableRootPreflight() { "HERMES_DIR_MODE=750", 'chmod() { if [ "${1:-}" = "3770" ] && [ "${2:-}" = "$HERMES_DIR" ]; then printf "%s\\n" "$1" > "$CHMOD_LOG"; HERMES_DIR_MODE=770; command chmod 770 "$2"; return 0; fi; command chmod "$@"; }', 'dir_mode() { printf "%s\\n" "$HERMES_DIR_MODE"; }', - 'verify_hermes_config_integrity() { printf "verify mode=%s\\n" "$(dir_mode)"; }', + 'refresh_hermes_runtime_config_hashes() { printf "adopt mode=%s args=%s\\n" "$(dir_mode)" "$*"; }', + 'inspect_hermes_mcp_integrity() { printf "mcp-integrity mode=%s\\n" "$(dir_mode)"; }', 'prepare_hermes_lazy_dependencies() { printf "lazy mode=%s\\n" "$(dir_mode)"; }', 'ensure_hermes_runtime_api_server_key() { printf "api-key mode=%s\\n" "$(dir_mode)"; }', "validate_hermes_env_secret_boundary() { :; }", "validate_hermes_runtime_env_secret_boundary() { :; }", "refresh_hermes_provider_placeholders() { :; }", - "refresh_hermes_runtime_config_hashes() { :; }", "configure_messaging_channels() { :; }", 'retry_tirith_marker_if_needed() { printf "tirith-state=%s\\n" "$TIRITH_RETRY_MARKER_CLEARED"; }', "prepare_tirith_marker_retry() { TIRITH_RETRY_MARKER_CLEARED=0; retry_tirith_marker_if_needed; }", @@ -1001,7 +1012,24 @@ describe("agents/hermes/start.sh env secret boundary", () => { expect(result.stderr).not.toContain(rawToken); }); - it("reconciles mutable hashes after the env boundary and before MCP integrity (#9203)", () => { + it.each(["nonroot", "root"] as const)( + "stops %s preparation before config adoption when the secret boundary refuses", + (prepareMode) => { + const rawToken = `SENTINEL_${prepareMode.toUpperCase()}_RAW_SECRET_VALUE`; + const result = runHermesEnvSecretBoundary({ + envFile: `DEVTEST_API_TOKEN=${rawToken}\n`, + prepareMode, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("raw secret-shaped values"); + expect(result.stderr).toContain("DEVTEST_API_TOKEN (line 1)"); + expect(result.stderr).not.toContain(rawToken); + expect(result.stdout).not.toContain("unexpected-adopt"); + }, + ); + + it("adopts mutable config after the env boundary and before MCP integrity (#11108)", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = spawnSync( "bash", @@ -1017,7 +1045,7 @@ describe("agents/hermes/start.sh env secret boundary", () => { "ensure_hermes_runtime_api_server_key() { trace api-key; }", "validate_hermes_runtime_env_secret_boundary() { trace runtime-boundary; }", "refresh_hermes_provider_placeholders() { trace placeholders; }", - "refresh_hermes_runtime_config_hashes() { trace hashes; hash_state=current; }", + 'refresh_hermes_runtime_config_hashes() { trace "hashes:$1:${2:-preserve}"; hash_state=current; }', "configure_messaging_channels() { trace channels; }", "retry_tirith_marker_if_needed() { trace tirith; }", extractShellFunctionFromSource(source, "prepare_tirith_marker_retry"), @@ -1031,14 +1059,14 @@ describe("agents/hermes/start.sh env secret boundary", () => { expect(result.status, result.stderr).toBe(0); expect(result.stdout.trim().split("\n")).toEqual([ "env-boundary", - "hashes", + "hashes:compat:adopt", "mcp-integrity", "lazy-dependencies", "api-key", "env-boundary", "runtime-boundary", "placeholders", - "hashes", + "hashes:compat:preserve", "mcp-integrity", "channels", "tirith", @@ -1329,7 +1357,8 @@ describe("agents/hermes/start.sh Tirith marker bootstrap", () => { const run = runHermesRootStartupMutableRootPreflight(); expect(run.result.status).toBe(0); - expect(run.result.stdout).toContain("verify mode=750"); + expect(run.result.stdout).toContain("adopt mode=750 args=both adopt"); + expect(run.result.stdout).toContain("mcp-integrity mode=750"); expect(run.result.stdout).toContain("lazy mode=750"); expect(run.result.stdout).toContain("api-key mode=770"); expect(run.result.stdout).toContain("tirith-state=0"); diff --git a/test/agents/hermes/hermes-tirith-retry-finalization.test.ts b/test/agents/hermes/hermes-tirith-retry-finalization.test.ts index 61dd90d5295..2b057532eee 100644 --- a/test/agents/hermes/hermes-tirith-retry-finalization.test.ts +++ b/test/agents/hermes/hermes-tirith-retry-finalization.test.ts @@ -169,7 +169,9 @@ describe("agents/hermes/start.sh Tirith retry finalization", () => { it("runs reset-aware retry preparation in the root startup path", () => { const run = runTirithFinalizer([ - "verify_hermes_config_integrity() { :; }", + "refresh_hermes_runtime_config_hashes() { :; }", + "inspect_hermes_mcp_integrity() { :; }", + "HERMES_HASH_FILE=/etc/nemoclaw/hermes.config-hash", "prepare_hermes_lazy_dependencies() { :; }", "ensure_hermes_config_root_mode() { :; }", "ensure_hermes_runtime_api_server_key() { :; }", diff --git a/test/helpers/rebuild-flow-test-support.ts b/test/helpers/rebuild-flow-test-support.ts index 378ac425e3a..88986235e92 100644 --- a/test/helpers/rebuild-flow-test-support.ts +++ b/test/helpers/rebuild-flow-test-support.ts @@ -54,7 +54,6 @@ export type RebuildFlowOverrides = { forwardRecovered: boolean; forwardRecoveryFailed?: boolean; secretBoundaryRefused?: boolean; - mcpReconciliationRefused?: boolean; }; restartSandboxGateway?: () => GatewayRestartResult; onboard?: ( diff --git a/test/inference/managed/managed-gateway-control-trust-contract.test.ts b/test/inference/managed/managed-gateway-control-trust-contract.test.ts index cb1a635b7ff..4638fa381dd 100644 --- a/test/inference/managed/managed-gateway-control-trust-contract.test.ts +++ b/test/inference/managed/managed-gateway-control-trust-contract.test.ts @@ -28,7 +28,8 @@ describe("managed gateway control trust contract", () => { ); expect(docs).toContain("malicious process running under the same sandbox UID"); - expect(docs).toContain("time-of-check/time-of-use limits of managed cold start"); + expect(docs).toContain("does not make the complete Hermes config a relaunch allowlist"); + expect(docs).toContain("no durable root-owned config anchor"); expect(docs).toContain("does not create gateway and agent UID isolation"); expect(docs).toContain( "minimum supported OpenShell provides a root-owned lifecycle supervisor", diff --git a/test/inference/managed/managed-gateway-control.test.ts b/test/inference/managed/managed-gateway-control.test.ts index 638bb3b4307..a54bbe8b6ee 100644 --- a/test/inference/managed/managed-gateway-control.test.ts +++ b/test/inference/managed/managed-gateway-control.test.ts @@ -1003,7 +1003,7 @@ with tempfile.TemporaryDirectory() as root: "[gateway] Hermes replacement gateway failed listener or health validation; stopping the exact child", "[gateway] Hermes replacement gateway lost its listener or health endpoint during auxiliary validation; stopping the exact child", "[gateway] CRITICAL: Hermes gateway lost its listener or health endpoint; stopping the exact child for recovery", - "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is quarantined until sandbox recreation; check /tmp/gateway.log", + "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox; check /tmp/gateway.log", "[CRITICAL] Newly launched Hermes gateway pid 5252 failed exact role identity capture; quarantining the managed startup supervisor without signaling the unproven child", ] supervisor_log_uid = 1000 if os.geteuid() == 0 else os.geteuid() @@ -1419,7 +1419,7 @@ describe("managed gateway root control", () => { "[gateway] Hermes replacement gateway failed listener or health validation; stopping the exact child", "[gateway] Hermes replacement gateway lost its listener or health endpoint during auxiliary validation; stopping the exact child", "[gateway] CRITICAL: Hermes gateway lost its listener or health endpoint; stopping the exact child for recovery", - "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is quarantined until sandbox recreation; check /tmp/gateway.log", + "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox; check /tmp/gateway.log", "[CRITICAL] Newly launched Hermes gateway pid 5252 failed exact role identity capture; quarantining the managed startup supervisor without signaling the unproven child", ], excerpt: [ @@ -1427,7 +1427,7 @@ describe("managed gateway root control", () => { "[gateway] Hermes replacement gateway failed listener or health validation; stopping the exact child", "[gateway] Hermes replacement gateway lost its listener or health endpoint during auxiliary validation; stopping the exact child", "[gateway] CRITICAL: Hermes gateway lost its listener or health endpoint; stopping the exact child for recovery", - "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is quarantined until sandbox recreation; check /tmp/gateway.log", + "[gateway] CRITICAL: 5 exits in 60s window — Hermes relaunch is stopped for this supervisor instance; correct the reported failure, then stop and start the sandbox; check /tmp/gateway.log", "[CRITICAL] Newly launched Hermes gateway pid 5252 failed exact role identity capture; quarantining the managed startup supervisor without signaling the unproven child", ], rejected_lines: [null, null, null, null, null], diff --git a/test/support/connect-flow-test-harness.ts b/test/support/connect-flow-test-harness.ts index 3f997544f21..37b99a91cbd 100644 --- a/test/support/connect-flow-test-harness.ts +++ b/test/support/connect-flow-test-harness.ts @@ -117,8 +117,6 @@ export type ConnectHarnessOptions = { recoveryFailureDetail?: string; secretBoundaryRefused?: boolean; secretBoundaryReason?: SecretBoundaryRefusalReason; - mcpReconciliationRefused?: boolean; - mcpReconciliationReason?: string; }; portableRecoveryResult?: { kind: "not-installed" | "already-running" | "recovered" }; portableReceiptDisposition?: diff --git a/test/support/hermes-shell-harness.ts b/test/support/hermes-shell-harness.ts index 9730e8a514d..fea6a788889 100644 --- a/test/support/hermes-shell-harness.ts +++ b/test/support/hermes-shell-harness.ts @@ -40,7 +40,6 @@ export function runHermesBashHarness( "#!/usr/bin/env bash", "set -uo pipefail", "HERMES_MCP_RECONCILE_PENDING=0", - "HERMES_MCP_INTEGRITY_FAILED=0", ...lines, ].join("\n"), { mode: 0o700 },