test(hermes): cover MCP integrity through restart

scripts/managed-gateway-control.py

@@ -89,6 +89,10 @@
 NONCE_RE = re.compile(r"[0-9a-f]{64}\Z")
 ENV_KEY_RE = re.compile(rb"[A-Za-z_][A-Za-z0-9_]*\Z")
 SHA256_RE = re.compile(r"[0-9a-f]{64}\Z")
+HERMES_MCP_STATE_RE = re.compile(
+    r"# nemoclaw-hermes-mcp-state-v1 "
+    r"intended=[0-9a-f]{64} applied=[0-9a-f]{64}\Z"
+)
 CONTROL_STAGES = frozenset(
     {
         "detect-agent",
@@ -1340,6 +1344,29 @@ def _read_regular(path: str, limit: int) -> tuple[bytes, os.stat_result]:
         os.close(fd)
 
 
+def _parse_locked_hermes_hash(strict: bytes) -> dict[str, str]:
+    records: dict[str, str] = {}
+    mcp_state_seen = False
+    try:
+        lines = strict.decode("ascii").splitlines()
+        for line in lines:
+            if line.startswith("#"):
+                if mcp_state_seen or HERMES_MCP_STATE_RE.fullmatch(line) is None:
+                    raise ValueError("invalid Hermes MCP state metadata")
+                mcp_state_seen = True
+                continue
+            if mcp_state_seen:
+                raise ValueError("Hermes MCP state metadata must be terminal")
+            digest, pathname = line.split(maxsplit=1)
+            canonical_path = pathname.strip()
+            if canonical_path in records:
+                raise ValueError("duplicate Hermes config hash path")
+            records[canonical_path] = digest.lower()
+    except (UnicodeDecodeError, ValueError) as exc:
+        raise ControlError("GATEWAY_CONFIG_HASH_MISMATCH") from exc
+    return records
+
+
 def _verify_locked_hermes_hash() -> None:
     config_path = _system_path("/sandbox/.hermes/config.yaml")
     env_path = _system_path("/sandbox/.hermes/.env")
@@ -1361,13 +1388,7 @@ def _verify_locked_hermes_hash() -> None:
     strict, strict_stat = _read_regular(hash_path, MAX_HASH_BYTES)
     if strict_stat.st_uid != 0 or stat.S_IMODE(strict_stat.st_mode) & 0o022:
         raise ControlError("GATEWAY_UNSAFE_CONFIG_PATH")
-    try:
-        records = {}
-        for line in strict.decode("ascii").splitlines():
-            digest, pathname = line.split(maxsplit=1)
-            records[pathname.strip()] = digest.lower()
-    except (UnicodeDecodeError, ValueError) as exc:
-        raise ControlError("GATEWAY_CONFIG_HASH_MISMATCH") from exc
+    records = _parse_locked_hermes_hash(strict)
     expected_paths = {
         "/sandbox/.hermes/config.yaml": hashlib.sha256(config).hexdigest(),
         "/sandbox/.hermes/.env": hashlib.sha256(environment).hexdigest(),

test/e2e/live/mcp-bridge-hermes-lifecycle.ts

@@ -44,6 +44,251 @@ export async function assertHermesConfig(
   expectExitZero(result, "Hermes MCP config contains placeholder and no raw host secret");
 }
 
+/**
+ * Focused #7499 live regression after the supported managed add has executed
+ * the real unprivileged Hermes transaction: explicitly restore shields,
+ * restart the real gateway, and prove the locked files and transaction state
+ * remain current.
+ */
+export async function assertHermesManagedAddSurvivesLockedGatewayRestart(
+  host: HostCliClient,
+  sandbox: SandboxClient,
+  sandboxName: string,
+  mcpUrl: string,
+): Promise<void> {
+  const shieldsUp = await host.nemoclaw([sandboxName, "shields", "up"], {
+    artifactName: "hermes-mcp-shields-up-after-add",
+    env: buildAvailabilityProbeEnv(),
+    redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET],
+    timeoutMs: 3 * 60_000,
+  });
+  expectExitZero(shieldsUp, "restore Hermes shields after managed MCP add");
+
+  const shieldsStatus = await host.nemoclaw([sandboxName, "shields", "status"], {
+    artifactName: "hermes-mcp-shields-status-after-add",
+    env: buildAvailabilityProbeEnv(),
+    redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET],
+    timeoutMs: 60_000,
+  });
+  expectExitZero(shieldsStatus, "read Hermes shields status after managed MCP add");
+  expect(resultText(shieldsStatus)).toContain("Shields: UP");
+
+  const restart = await host.nemoclaw([sandboxName, "gateway", "restart"], {
+    artifactName: "hermes-mcp-add-gateway-restart",
+    env: buildAvailabilityProbeEnv(),
+    redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET],
+    timeoutMs: 12 * 60_000,
+  });
+  expectExitZero(restart, "Hermes gateway restart with managed MCP present");
+  expect(resultText(restart)).toContain("Gateway restarted");
+  expect(resultText(restart)).toContain("health passed");
+
+  const lockedIntegrity = await sandbox.execShell(
+    sandboxName,
+    trustedSandboxShellScript(
+      [
+        "set -eu",
+        "test \"$(stat -c '%a %U:%G' /sandbox)\" = '1775 root:sandbox'",
+        "test \"$(stat -c '%a %U:%G' /sandbox/.hermes)\" = '755 root:root'",
+        "for path in /sandbox/.hermes/config.yaml /sandbox/.hermes/.env /etc/nemoclaw/hermes.config-hash /sandbox/.hermes/.config-hash; do",
+        "  test \"$(stat -c '%a %U:%G' \"$path\")\" = '444 root:root'",
+        "done",
+        "cmp -s /etc/nemoclaw/hermes.config-hash /sandbox/.hermes/.config-hash",
+        "sha256sum -c /etc/nemoclaw/hermes.config-hash --status",
+        "sha256sum -c /sandbox/.hermes/.config-hash --status",
+        "echo HERMES_MCP_LOCKED_INTEGRITY_CURRENT",
+      ].join("\n"),
+    ),
+    {
+      artifactName: "hermes-mcp-locked-integrity-after-add-gateway-restart",
+      env: buildAvailabilityProbeEnv(),
+      redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET],
+      timeoutMs: 60_000,
+    },
+  );
+  expectExitZero(lockedIntegrity, "Hermes MCP integrity anchors after gateway restart");
+  expect(lockedIntegrity.stdout).toContain("HERMES_MCP_LOCKED_INTEGRITY_CURRENT");
+
+  const list = await host.nemoclaw([sandboxName, "mcp", "list", "--json"], {
+    artifactName: "hermes-mcp-list-after-add-gateway-restart",
+    env: buildAvailabilityProbeEnv(),
+    redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET],
+    timeoutMs: 60_000,
+  });
+  expectExitZero(list, "Hermes MCP list after add gateway restart");
+  const listJson = JSON.parse(list.stdout) as {
+    bridges: Array<{ server: string; url: string; adapter: { registered: boolean | null } }>;
+  };
+  expect(listJson.bridges).toEqual([
+    expect.objectContaining({
+      server: SERVER_NAME,
+      url: mcpUrl,
+      adapter: expect.objectContaining({ registered: true }),
+    }),
+  ]);
+
+  const expectedPayload = Buffer.from(
+    JSON.stringify({
+      present: {
+        [SERVER_NAME]: {
+          url: mcpUrl,
+          headers: {
+            Authorization: "Bearer openshell:resolve:env:FAKE_MCP_SECRET",
+          },
+          timeout: 120,
+          connect_timeout: 60,
+          tools: { prompts: true, resources: true },
+          enabled: true,
+        },
+      },
+      absent: [],
+    }),
+    "utf8",
+  ).toString("base64");
+  const effectiveConfig = await sandbox.execShell(
+    sandboxName,
+    trustedSandboxShellScript(
+      [
+        "set -eu",
+        `payload="$(printf '%s' '${expectedPayload}' | base64 -d)"`,
+        '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py inspect --payload "$payload"',
+      ].join("\n"),
+    ),
+    {
+      artifactName: "hermes-mcp-effective-config-after-add-gateway-restart",
+      env: buildAvailabilityProbeEnv(),
+      redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET, expectedPayload],
+      timeoutMs: 60_000,
+    },
+  );
+  expectExitZero(effectiveConfig, "Hermes effective MCP config after add gateway restart");
+  const effectiveConfigJson = JSON.parse(effectiveConfig.stdout) as { state: string };
+  expect(effectiveConfigJson.state).toBe("matched");
+
+  const shieldsDown = await host.nemoclaw(
+    [
+      sandboxName,
+      "shields",
+      "down",
+      "--timeout",
+      "15m",
+      "--reason",
+      "Continue managed MCP lifecycle E2E",
+    ],
+    {
+      artifactName: "hermes-mcp-shields-down-after-restart-proof",
+      env: buildAvailabilityProbeEnv(),
+      redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET],
+      timeoutMs: 3 * 60_000,
+    },
+  );
+  expectExitZero(shieldsDown, "unlock Hermes config for remaining managed MCP lifecycle");
+  await assertHermesReloadRollback(sandbox, sandboxName, mcpUrl);
+}
+
+/**
+ * Inject a first-reload failure around the packaged transaction helper, then
+ * require its real rollback reload to restore the prior config, both integrity
+ * anchors, and a healthy managed gateway in the live sandbox.
+ */
+async function assertHermesReloadRollback(
+  sandbox: SandboxClient,
+  sandboxName: string,
+  mcpUrl: string,
+): Promise<void> {
+  const payload = JSON.stringify({
+    server: "rollback_probe",
+    url: mcpUrl,
+    headers: {
+      Authorization: "Bearer openshell:resolve:env:FAKE_MCP_SECRET",
+    },
+    replace_existing: false,
+  });
+  const inspectionPayload = Buffer.from(
+    JSON.stringify({
+      present: {
+        [SERVER_NAME]: {
+          url: mcpUrl,
+          headers: {
+            Authorization: "Bearer openshell:resolve:env:FAKE_MCP_SECRET",
+          },
+          timeout: 120,
+          connect_timeout: 60,
+          tools: { prompts: true, resources: true },
+          enabled: true,
+        },
+      },
+      absent: ["rollback_probe"],
+    }),
+    "utf8",
+  ).toString("base64");
+  const script = [
+    "set -eu",
+    "/opt/hermes/.venv/bin/python - <<'PY'",
+    "import importlib.util, json, os, pathlib, sys",
+    "module_path = '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py'",
+    "spec = importlib.util.spec_from_file_location('nemoclaw_mcp_tx_rollback_e2e', module_path)",
+    "assert spec is not None and spec.loader is not None",
+    "module = importlib.util.module_from_spec(spec)",
+    "sys.modules[spec.name] = module",
+    "spec.loader.exec_module(module)",
+    `payload = json.loads(${JSON.stringify(payload)})`,
+    "paths = (",
+    "    pathlib.Path('/sandbox/.hermes/config.yaml'),",
+    "    pathlib.Path('/etc/nemoclaw/hermes.config-hash'),",
+    "    pathlib.Path('/sandbox/.hermes/.config-hash'),",
+    ")",
+    "before = {path: path.read_bytes() for path in paths}",
+    "if os.geteuid() == 0:",
+    "    raise RuntimeError('rollback regression must run as the sandbox identity')",
+    "real_reload = module.reload_gateway",
+    "reload_calls = 0",
+    "rollback_reloaded = None",
+    "def fail_then_reload():",
+    "    global reload_calls, rollback_reloaded",
+    "    reload_calls += 1",
+    "    if reload_calls == 1:",
+    "        raise RuntimeError('injected first managed reload failure')",
+    "    rollback_reloaded = real_reload()",
+    "    return rollback_reloaded",
+    "module.reload_gateway = fail_then_reload",
+    "try:",
+    "    module.execute('add', payload)",
+    "except RuntimeError as error:",
+    "    failure = str(error)",
+    "else:",
+    "    raise RuntimeError('injected managed reload failure unexpectedly succeeded')",
+    "if reload_calls != 2:",
+    "    raise RuntimeError(f'rollback performed {reload_calls} reload attempts instead of 2')",
+    "if 'injected first managed reload failure' not in failure:",
+    "    raise RuntimeError('transaction did not report the injected reload failure')",
+    "if rollback_reloaded is not True:",
+    "    raise RuntimeError('rollback did not restore a healthy managed gateway')",
+    "after = {path: path.read_bytes() for path in paths}",
+    "if after != before:",
+    "    raise RuntimeError('rollback did not restore config and both hash anchors exactly')",
+    "PY",
+    `inspection_payload="$(printf '%s' '${inspectionPayload}' | base64 -d)"`,
+    '/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py inspect --payload "$inspection_payload"',
+  ].join("\n");
+  const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), {
+    artifactName: "hermes-mcp-failed-reload-rollback",
+    env: buildAvailabilityProbeEnv(),
+    redactionValues: [
+      HOST_SECRET,
+      ROTATED_HOST_SECRET,
+      inspectionPayload,
+      Buffer.from(script, "utf8").toString("base64"),
+    ],
+    timeoutMs: 7 * 60_000,
+  });
+  expectExitZero(result, "Hermes failed MCP reload restores config, hashes, and gateway");
+  expect(JSON.parse(result.stdout)).toEqual({
+    ok: true,
+    state: "current",
+  });
+}
+
 // No host `nemoclaw mcp inspect` command exists; exercise the packaged CLI
 // through the same OpenShell sandbox boundary used by live MCP reconciliation.
 export async function assertHermesInspectionRejectsUnmanagedFields(

test/e2e/live/mcp-bridge-phases.ts

@@ -12,6 +12,7 @@ export const MCP_BRIDGE_PHASES = {
     "start Hermes inference and MCP endpoints",
     "onboard the Hermes MCP sandbox",
     "configure and inspect the Hermes MCP bridge",
+    "restore Hermes shields, restart, and prove rollback",
     "exercise lifecycle and confirm Hermes bridge removal",
   ],
   deepagents: [

test/e2e/live/mcp-bridge.test.ts

@@ -26,6 +26,7 @@ import { type McpBridgeShard, resolveMcpBridgeShard } from "./mcp-bridge-agent-s
 import {
   assertHermesConfig,
   assertHermesInspectionRejectsUnmanagedFields,
+  assertHermesManagedAddSurvivesLockedGatewayRestart,
   assertHermesRemovalSurvivesGatewayRestart,
 } from "./mcp-bridge-hermes-lifecycle.ts";
 import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv } from "./mcp-bridge-onboard-env.ts";
@@ -1236,13 +1237,27 @@ mcpBridgeShardTest("hermes")(
     await assertHermesConfig(sandbox, HERMES_SANDBOX_NAME, mcpUrl);
     await assertHermesInspectionRejectsUnmanagedFields(sandbox, HERMES_SANDBOX_NAME);
     await assertSecretAbsentFromSandbox(sandbox, HERMES_SANDBOX_NAME, ["/sandbox/.hermes"]);
+    progress.phase("restore Hermes shields, restart, and prove rollback");
+    await assertHermesManagedAddSurvivesLockedGatewayRestart(
+      host,
+      sandbox,
+      HERMES_SANDBOX_NAME,
+      mcpUrl,
+    );
+    await assertSecretAbsentFromSandbox(
+      sandbox,
+      HERMES_SANDBOX_NAME,
+      ["/sandbox/.hermes", "/tmp/nemoclaw-start.log"],
+      [HOST_SECRET],
+      "hermes-assert-secret-absent-after-add-gateway-restart",
+    );
+    progress.phase("exercise lifecycle and confirm Hermes bridge removal");
     await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, {
       adapter: "hermes-config",
       artifactPrefix: "hermes",
       sandboxName: HERMES_SANDBOX_NAME,
       secretPaths: ["/sandbox/.hermes"],
     });
-    progress.phase("exercise lifecycle and confirm Hermes bridge removal");
     await assertRealAdapterToolCall(sandbox, fakeMcp, {
       agent: "hermes",
       sandboxName: HERMES_SANDBOX_NAME,

test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts

@@ -904,7 +904,7 @@ it("carries the generated planner matrix through the workflow output and PR repo
   } finally {
     fs.rmSync(directory, { force: true, recursive: true });
   }
-}, 30_000);
+}, 40_000);
 
 it("builds controller target matrices only from trusted runner mappings (#7031)", () => {
   const target = "ubuntu-repo-cloud-langchain-deepagents-code";