fix(rebuild): keep Hermes MCP runtime active after rebuild

test/e2e/live/mcp-bridge-hermes-http.ts

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { shellQuote } from "../../../src/lib/core/shell-quote";
+import { redactString } from "../fixtures/redaction.ts";
+
+export const HERMES_MCP_HTTP_STATUS_MARKER = "NEMOCLAW_HERMES_MCP_HTTP_STATUS=";
+export const HERMES_MCP_RESULT_TOKEN_MARKER = "NEMOCLAW_HERMES_MCP_RESULT_TOKEN=";
+export const HERMES_MCP_FAILURE_CAPTURE_BYTES = 4_096;
+export const HERMES_MCP_FAILURE_PREVIEW_CHARS = 1_024;
+export const HERMES_MCP_RESPONSE_FILE_BYTES = 65_536;
+
+interface HermesMcpCommandResult {
+  exitCode: number | null;
+  signal?: NodeJS.Signals | null;
+  stdout: string;
+  stderr: string;
+}
+
+const FAILURE_BODY_EMITTER = [
+  "import os, pathlib, sys",
+  "raw = pathlib.Path(sys.argv[1]).read_bytes()",
+  "secret = os.environ.get('API_SERVER_KEY', '').encode('utf-8')",
+  "replacement = b'[REDACTED]' if len(secret) >= 10 else b'*' * len(secret)",
+  "sys.stdout.buffer.write(raw.replace(secret, replacement) if secret else raw)",
+].join("\n");
+
+export function buildHermesMcpChatProbeScript(payload: string, resultToken: string): string {
+  return [
+    "set -eu",
+    "umask 077",
+    'response_file="$(mktemp /tmp/nemoclaw-hermes-mcp-chat.XXXXXX)"',
+    "trap 'rm -f \"$response_file\"' EXIT",
+    `set -- -sS --max-time 180 --max-filesize ${HERMES_MCP_RESPONSE_FILE_BYTES} -o "$response_file" -w '%{http_code}' http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json'`,
+    'if [ -n "${API_SERVER_KEY:-}" ]; then set -- "$@" -H "Authorization: Bearer ${API_SERVER_KEY}"; fi',
+    `emit_failure_body() { /usr/bin/python3 -I -S -c ${shellQuote(FAILURE_BODY_EMITTER)} "$response_file"; }`,
+    "set +e",
+    `status="$(curl "$@" --data-binary ${shellQuote(payload)})"`,
+    "curl_rc=$?",
+    "set -e",
+    `printf '\\n${HERMES_MCP_HTTP_STATUS_MARKER}%s\\n' "$status" >&2`,
+    'if [ "$curl_rc" -ne 0 ]; then exit "$curl_rc"; fi',
+    `case "$status" in 2??) if grep -Fq -- ${shellQuote(resultToken)} "$response_file"; then printf '${HERMES_MCP_RESULT_TOKEN_MARKER}present\\n' >&2; else printf '${HERMES_MCP_RESULT_TOKEN_MARKER}missing\\n' >&2; fi ;; *) emit_failure_body ;; esac`,
+  ].join("\n");
+}
+
+function sanitizedPreview(text: string, explicitValues: Iterable<string>): string {
+  const sanitized = redactString(text, explicitValues)
+    .replace(/\u001b\[[0-?]*[ -/]*[@-~]/gu, "")
+    .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, " ")
+    .trim();
+  if (!sanitized) return "<empty>";
+  const characters = [...sanitized];
+  if (characters.length <= HERMES_MCP_FAILURE_PREVIEW_CHARS) return sanitized;
+  const suffix = "… [truncated]";
+  return `${characters
+    .slice(0, HERMES_MCP_FAILURE_PREVIEW_CHARS - [...suffix].length)
+    .join("")}${suffix}`;
+}
+
+function parseHttpStatus(stderr: string): number | null {
+  const escapedMarker = HERMES_MCP_HTTP_STATUS_MARKER.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const matches = [...stderr.matchAll(new RegExp(`^${escapedMarker}([0-9]{3})$`, "gmu"))];
+  if (matches.length !== 1) return null;
+  return Number(matches[0]?.[1]);
+}
+
+function hasResultToken(stderr: string): boolean {
+  const marker = `${HERMES_MCP_RESULT_TOKEN_MARKER}present`;
+  return stderr.split(/\r?\n/u).filter((line) => line === marker).length === 1;
+}
+
+export function assertHermesMcpHttpResponse(
+  result: HermesMcpCommandResult,
+  explicitRedactionValues: Iterable<string>,
+): void {
+  const status = parseHttpStatus(result.stderr);
+  if (result.exitCode !== 0) {
+    const transport = result.signal
+      ? `signal=${result.signal}`
+      : `exit=${result.exitCode ?? "unknown"}`;
+    const detail = sanitizedPreview([result.stdout, result.stderr].filter(Boolean).join("\n"), [
+      ...explicitRedactionValues,
+    ]);
+    throw new Error(`Hermes real MCP tool call transport failed (${transport}): ${detail}`);
+  }
+  if (status === null) {
+    throw new Error("Hermes real MCP tool call did not report exactly one HTTP status marker");
+  }
+  if (status < 200 || status >= 300) {
+    const body = sanitizedPreview(result.stdout, explicitRedactionValues);
+    throw new Error(
+      `Hermes real MCP tool call failed: HTTP ${status}; redacted response body: ${body}`,
+    );
+  }
+  if (!hasResultToken(result.stderr)) {
+    throw new Error("Hermes real MCP tool call response did not contain the fixture result token");
+  }
+  if (result.stdout !== "") {
+    throw new Error("Hermes real MCP tool call success path emitted response contents");
+  }
+}

test/e2e/live/mcp-bridge.test.ts

@@ -26,6 +26,11 @@ import {
   type McpAdapter,
   removeMcpBridgeWithOneConcurrencyRetry,
 } from "./mcp-bridge-cleanup.ts";
+import {
+  assertHermesMcpHttpResponse,
+  buildHermesMcpChatProbeScript,
+  HERMES_MCP_FAILURE_CAPTURE_BYTES,
+} from "./mcp-bridge-hermes-http.ts";
 import {
   assertHermesConfig,
   assertHermesInspectionRejectsUnmanagedFields,
@@ -537,6 +542,17 @@ async function assertRealAdapterToolCall(
     messages: [{ role: "user", content: prompt }],
     max_tokens: 256,
   });
+  // ShellProbe redacts these values before it returns command output or writes
+  // artifacts. The HTTP assertion redacts them again before Vitest formats a
+  // bounded failure preview.
+  const hermesRedactionValues = [
+    HOST_SECRET,
+    ROTATED_HOST_SECRET,
+    COMPATIBLE_KEY,
+    TOOL_CHALLENGE,
+    prompt,
+    hermesPayload,
+  ];
   const command =
     options.agent === "openclaw"
       ? `nemoclaw-start mcporter call fake.fake_echo --args ${JSON.stringify(JSON.stringify({ challenge: TOOL_CHALLENGE }))} --output json`
@@ -545,7 +561,7 @@ async function assertRealAdapterToolCall(
             "set -a",
             "[ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env",
             "set +a",
-            `if [ -n "\${API_SERVER_KEY:-}" ]; then curl -fsS --max-time 180 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H "Authorization: Bearer \${API_SERVER_KEY}" --data-binary ${shellQuote(hermesPayload)}; else curl -fsS --max-time 180 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary ${shellQuote(hermesPayload)}; fi`,
+            buildHermesMcpChatProbeScript(hermesPayload, options.resultToken),
           ].join("\n")
         : `nemoclaw-start dcode -n ${JSON.stringify(prompt)}`;
   const result = await sandbox.execShell(
@@ -554,11 +570,19 @@ async function assertRealAdapterToolCall(
     {
       artifactName: options.artifactName,
       env: buildAvailabilityProbeEnv(),
+      captureLimitBytes: options.agent === "hermes" ? HERMES_MCP_FAILURE_CAPTURE_BYTES : undefined,
+      redactionValues: options.agent === "hermes" ? hermesRedactionValues : [],
       timeoutMs: 5 * 60_000,
     },
   );
-  expectExitZero(result, `${options.agent} real MCP tool call`);
-  expect(resultText(result)).toContain(options.resultToken);
+  const assertResponse =
+    options.agent === "hermes"
+      ? () => assertHermesMcpHttpResponse(result, hermesRedactionValues)
+      : () => {
+          expectExitZero(result, `${options.agent} real MCP tool call`);
+          expect(resultText(result)).toContain(options.resultToken);
+        };
+  assertResponse();
   const calls = fakeMcp.requests.filter((request) => request.rpcMethod === "tools/call");
   expect(calls).toHaveLength(before + 1);
   expect(calls.at(-1)).toMatchObject({

test/e2e/support/mcp-bridge-hermes-http.test.ts

@@ -0,0 +1,122 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import {
+  assertHermesMcpHttpResponse,
+  buildHermesMcpChatProbeScript,
+  HERMES_MCP_FAILURE_PREVIEW_CHARS,
+  HERMES_MCP_HTTP_STATUS_MARKER,
+  HERMES_MCP_RESULT_TOKEN_MARKER,
+} from "../live/mcp-bridge-hermes-http.ts";
+
+const TIMEOUT_MS = 5_000;
+const SYSTEM_PATH = "/usr/bin:/bin";
+
+function httpResult(status: number, body = "", result = "") {
+  return {
+    exitCode: 0,
+    signal: null,
+    stdout: body,
+    stderr: `${HERMES_MCP_HTTP_STATUS_MARKER}${status}\n${result}`,
+  };
+}
+
+describe("Hermes MCP HTTP failure diagnostics", () => {
+  it("sends one authenticated request without retrying and redacts its API key from failure output (#8697)", () => {
+    const token = "fixture-result-token";
+    const script = buildHermesMcpChatProbeScript('{"messages":[]}', token);
+    expect(script.match(/\bcurl "\$@"/gu)).toHaveLength(1);
+    expect(script).not.toMatch(/\bretry\b/iu);
+    expect(script).toContain("Authorization: Bearer ${API_SERVER_KEY}");
+
+    const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-hermes-mcp-http-"));
+    const bodyFile = path.join(directory, "body");
+    const countFile = path.join(directory, "count");
+    const curl = path.join(directory, "curl");
+    writeFileSync(
+      curl,
+      [
+        "#!/bin/sh",
+        "set -eu",
+        'all_args="$*"',
+        "output=",
+        'while [ "$#" -gt 0 ]; do',
+        '  if [ "$1" = "-o" ]; then output="$2"; shift 2; continue; fi',
+        "  shift",
+        "done",
+        'case "$all_args" in *"Authorization: Bearer $FAKE_API_KEY"*) ;; *) exit 67 ;; esac',
+        'cp "$FAKE_BODY_FILE" "$output"',
+        'printf "1\\n" >> "$FAKE_COUNT_FILE"',
+        'printf "%s" "$FAKE_STATUS"',
+      ].join("\n"),
+    );
+    chmodSync(curl, 0o755);
+
+    const apiKey = "fixture-api-key-value";
+    const run = (body: string, status: string) => {
+      writeFileSync(bodyFile, body);
+      return spawnSync("sh", ["-c", script], {
+        encoding: "utf8",
+        env: {
+          API_SERVER_KEY: apiKey,
+          FAKE_API_KEY: apiKey,
+          FAKE_BODY_FILE: bodyFile,
+          FAKE_COUNT_FILE: countFile,
+          FAKE_STATUS: status,
+          PATH: `${directory}:${SYSTEM_PATH}`,
+        },
+        killSignal: "SIGKILL",
+        timeout: TIMEOUT_MS,
+      });
+    };
+
+    try {
+      const failed = run(`failed with ${apiKey}`, "500");
+      expect(failed.status, failed.stderr).toBe(0);
+      expect(failed.stdout).toContain("[REDACTED]");
+      expect(failed.stdout).not.toContain(apiKey);
+      expect(failed.stderr).toContain(`${HERMES_MCP_HTTP_STATUS_MARKER}500`);
+
+      expect(readFileSync(countFile, "utf8")).toBe("1\n");
+    } finally {
+      rmSync(directory, { force: true, recursive: true });
+    }
+  });
+
+  it("rejects duplicate status markers, HTTP 500, and missing result tokens (#8697)", () => {
+    const secret = "fixture-diagnostic-secret";
+    const longBody = `${secret}\nAuthorization: Bearer another-secret\n${"x".repeat(
+      HERMES_MCP_FAILURE_PREVIEW_CHARS * 2,
+    )}`;
+    try {
+      assertHermesMcpHttpResponse(httpResult(500, longBody), [secret]);
+      throw new Error("expected the HTTP 500 response assertion to throw");
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      expect(message).toContain("HTTP 500");
+      expect(message).toContain("[REDACTED]");
+      expect(message).toContain("[truncated]");
+      expect(message).not.toContain(secret);
+    }
+
+    expect(() => assertHermesMcpHttpResponse(httpResult(200), [])).toThrowError(
+      /fixture result token/u,
+    );
+    expect(() =>
+      assertHermesMcpHttpResponse(httpResult(200, "", `${HERMES_MCP_HTTP_STATUS_MARKER}500\n`), []),
+    ).toThrowError(/exactly one HTTP status marker/u);
+    expect(() =>
+      assertHermesMcpHttpResponse(
+        httpResult(200, "", `${HERMES_MCP_RESULT_TOKEN_MARKER}present\n`),
+        [],
+      ),
+    ).not.toThrow();
+  });
+});