diff --git a/test/e2e/live/mcp-bridge-hermes-http.ts b/test/e2e/live/mcp-bridge-hermes-http.ts new file mode 100644 index 00000000000..ec0c76e18fc --- /dev/null +++ b/test/e2e/live/mcp-bridge-hermes-http.ts @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { redactString } from "../fixtures/redaction.ts"; + +export const HERMES_MCP_HTTP_STATUS_MARKER = "NEMOCLAW_HERMES_MCP_HTTP_STATUS="; +export const HERMES_MCP_RESULT_TOKEN_MARKER = "NEMOCLAW_HERMES_MCP_RESULT_TOKEN="; +export const HERMES_MCP_FAILURE_CAPTURE_BYTES = 4_096; +export const HERMES_MCP_FAILURE_PREVIEW_CHARS = 1_024; +export const HERMES_MCP_RESPONSE_FILE_BYTES = 65_536; + +interface HermesMcpCommandResult { + exitCode: number | null; + signal?: NodeJS.Signals | null; + stdout: string; + stderr: string; +} + +const FAILURE_BODY_EMITTER = [ + "import os, pathlib, sys", + "raw = pathlib.Path(sys.argv[1]).read_bytes()", + "secret = os.environ.get('API_SERVER_KEY', '').encode('utf-8')", + "replacement = b'[REDACTED]' if len(secret) >= 10 else b'*' * len(secret)", + "sys.stdout.buffer.write(raw.replace(secret, replacement) if secret else raw)", +].join("\n"); + +export function buildHermesMcpChatProbeScript(payload: string, resultToken: string): string { + return [ + "set -eu", + "umask 077", + 'response_file="$(mktemp /tmp/nemoclaw-hermes-mcp-chat.XXXXXX)"', + "trap 'rm -f \"$response_file\"' EXIT", + `set -- -sS --max-time 180 --max-filesize ${HERMES_MCP_RESPONSE_FILE_BYTES} -o "$response_file" -w '%{http_code}' http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json'`, + 'if [ -n "${API_SERVER_KEY:-}" ]; then set -- "$@" -H "Authorization: Bearer ${API_SERVER_KEY}"; fi', + `emit_failure_body() { /usr/bin/python3 -I -S -c ${shellQuote(FAILURE_BODY_EMITTER)} "$response_file"; }`, + "set +e", + `status="$(curl "$@" --data-binary ${shellQuote(payload)})"`, + "curl_rc=$?", + "set -e", + `printf '\\n${HERMES_MCP_HTTP_STATUS_MARKER}%s\\n' "$status" >&2`, + 'if [ "$curl_rc" -ne 0 ]; then exit "$curl_rc"; fi', + `case "$status" in 2??) if grep -Fq -- ${shellQuote(resultToken)} "$response_file"; then printf '${HERMES_MCP_RESULT_TOKEN_MARKER}present\\n' >&2; else printf '${HERMES_MCP_RESULT_TOKEN_MARKER}missing\\n' >&2; fi ;; *) emit_failure_body ;; esac`, + ].join("\n"); +} + +function sanitizedPreview(text: string, explicitValues: Iterable): string { + const sanitized = redactString(text, explicitValues) + .replace(/\u001b\[[0-?]*[ -/]*[@-~]/gu, "") + .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, " ") + .trim(); + if (!sanitized) return ""; + const characters = [...sanitized]; + if (characters.length <= HERMES_MCP_FAILURE_PREVIEW_CHARS) return sanitized; + const suffix = "… [truncated]"; + return `${characters + .slice(0, HERMES_MCP_FAILURE_PREVIEW_CHARS - [...suffix].length) + .join("")}${suffix}`; +} + +function parseHttpStatus(stderr: string): number | null { + const escapedMarker = HERMES_MCP_HTTP_STATUS_MARKER.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const matches = [...stderr.matchAll(new RegExp(`^${escapedMarker}([0-9]{3})$`, "gmu"))]; + if (matches.length !== 1) return null; + return Number(matches[0]?.[1]); +} + +function hasResultToken(stderr: string): boolean { + const marker = `${HERMES_MCP_RESULT_TOKEN_MARKER}present`; + return stderr.split(/\r?\n/u).filter((line) => line === marker).length === 1; +} + +export function assertHermesMcpHttpResponse( + result: HermesMcpCommandResult, + explicitRedactionValues: Iterable, +): void { + const status = parseHttpStatus(result.stderr); + if (result.exitCode !== 0) { + const transport = result.signal + ? `signal=${result.signal}` + : `exit=${result.exitCode ?? "unknown"}`; + const detail = sanitizedPreview([result.stdout, result.stderr].filter(Boolean).join("\n"), [ + ...explicitRedactionValues, + ]); + throw new Error(`Hermes real MCP tool call transport failed (${transport}): ${detail}`); + } + if (status === null) { + throw new Error("Hermes real MCP tool call did not report exactly one HTTP status marker"); + } + if (status < 200 || status >= 300) { + const body = sanitizedPreview(result.stdout, explicitRedactionValues); + throw new Error( + `Hermes real MCP tool call failed: HTTP ${status}; redacted response body: ${body}`, + ); + } + if (!hasResultToken(result.stderr)) { + throw new Error("Hermes real MCP tool call response did not contain the fixture result token"); + } + if (result.stdout !== "") { + throw new Error("Hermes real MCP tool call success path emitted response contents"); + } +} diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index a40359ef8c3..26466b0f737 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -26,6 +26,11 @@ import { type McpAdapter, removeMcpBridgeWithOneConcurrencyRetry, } from "./mcp-bridge-cleanup.ts"; +import { + assertHermesMcpHttpResponse, + buildHermesMcpChatProbeScript, + HERMES_MCP_FAILURE_CAPTURE_BYTES, +} from "./mcp-bridge-hermes-http.ts"; import { assertHermesConfig, assertHermesInspectionRejectsUnmanagedFields, @@ -537,6 +542,17 @@ async function assertRealAdapterToolCall( messages: [{ role: "user", content: prompt }], max_tokens: 256, }); + // ShellProbe redacts these values before it returns command output or writes + // artifacts. The HTTP assertion redacts them again before Vitest formats a + // bounded failure preview. + const hermesRedactionValues = [ + HOST_SECRET, + ROTATED_HOST_SECRET, + COMPATIBLE_KEY, + TOOL_CHALLENGE, + prompt, + hermesPayload, + ]; const command = options.agent === "openclaw" ? `nemoclaw-start mcporter call fake.fake_echo --args ${JSON.stringify(JSON.stringify({ challenge: TOOL_CHALLENGE }))} --output json` @@ -545,7 +561,7 @@ async function assertRealAdapterToolCall( "set -a", "[ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env", "set +a", - `if [ -n "\${API_SERVER_KEY:-}" ]; then curl -fsS --max-time 180 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H "Authorization: Bearer \${API_SERVER_KEY}" --data-binary ${shellQuote(hermesPayload)}; else curl -fsS --max-time 180 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary ${shellQuote(hermesPayload)}; fi`, + buildHermesMcpChatProbeScript(hermesPayload, options.resultToken), ].join("\n") : `nemoclaw-start dcode -n ${JSON.stringify(prompt)}`; const result = await sandbox.execShell( @@ -554,11 +570,19 @@ async function assertRealAdapterToolCall( { artifactName: options.artifactName, env: buildAvailabilityProbeEnv(), + captureLimitBytes: options.agent === "hermes" ? HERMES_MCP_FAILURE_CAPTURE_BYTES : undefined, + redactionValues: options.agent === "hermes" ? hermesRedactionValues : [], timeoutMs: 5 * 60_000, }, ); - expectExitZero(result, `${options.agent} real MCP tool call`); - expect(resultText(result)).toContain(options.resultToken); + const assertResponse = + options.agent === "hermes" + ? () => assertHermesMcpHttpResponse(result, hermesRedactionValues) + : () => { + expectExitZero(result, `${options.agent} real MCP tool call`); + expect(resultText(result)).toContain(options.resultToken); + }; + assertResponse(); const calls = fakeMcp.requests.filter((request) => request.rpcMethod === "tools/call"); expect(calls).toHaveLength(before + 1); expect(calls.at(-1)).toMatchObject({ diff --git a/test/e2e/support/mcp-bridge-hermes-http.test.ts b/test/e2e/support/mcp-bridge-hermes-http.test.ts new file mode 100644 index 00000000000..7814b71725c --- /dev/null +++ b/test/e2e/support/mcp-bridge-hermes-http.test.ts @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + assertHermesMcpHttpResponse, + buildHermesMcpChatProbeScript, + HERMES_MCP_FAILURE_PREVIEW_CHARS, + HERMES_MCP_HTTP_STATUS_MARKER, + HERMES_MCP_RESULT_TOKEN_MARKER, +} from "../live/mcp-bridge-hermes-http.ts"; + +const TIMEOUT_MS = 5_000; +const SYSTEM_PATH = "/usr/bin:/bin"; + +function httpResult(status: number, body = "", result = "") { + return { + exitCode: 0, + signal: null, + stdout: body, + stderr: `${HERMES_MCP_HTTP_STATUS_MARKER}${status}\n${result}`, + }; +} + +describe("Hermes MCP HTTP failure diagnostics", () => { + it("sends one authenticated request without retrying and redacts its API key from failure output (#8697)", () => { + const token = "fixture-result-token"; + const script = buildHermesMcpChatProbeScript('{"messages":[]}', token); + expect(script.match(/\bcurl "\$@"/gu)).toHaveLength(1); + expect(script).not.toMatch(/\bretry\b/iu); + expect(script).toContain("Authorization: Bearer ${API_SERVER_KEY}"); + + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-hermes-mcp-http-")); + const bodyFile = path.join(directory, "body"); + const countFile = path.join(directory, "count"); + const curl = path.join(directory, "curl"); + writeFileSync( + curl, + [ + "#!/bin/sh", + "set -eu", + 'all_args="$*"', + "output=", + 'while [ "$#" -gt 0 ]; do', + ' if [ "$1" = "-o" ]; then output="$2"; shift 2; continue; fi', + " shift", + "done", + 'case "$all_args" in *"Authorization: Bearer $FAKE_API_KEY"*) ;; *) exit 67 ;; esac', + 'cp "$FAKE_BODY_FILE" "$output"', + 'printf "1\\n" >> "$FAKE_COUNT_FILE"', + 'printf "%s" "$FAKE_STATUS"', + ].join("\n"), + ); + chmodSync(curl, 0o755); + + const apiKey = "fixture-api-key-value"; + const run = (body: string, status: string) => { + writeFileSync(bodyFile, body); + return spawnSync("sh", ["-c", script], { + encoding: "utf8", + env: { + API_SERVER_KEY: apiKey, + FAKE_API_KEY: apiKey, + FAKE_BODY_FILE: bodyFile, + FAKE_COUNT_FILE: countFile, + FAKE_STATUS: status, + PATH: `${directory}:${SYSTEM_PATH}`, + }, + killSignal: "SIGKILL", + timeout: TIMEOUT_MS, + }); + }; + + try { + const failed = run(`failed with ${apiKey}`, "500"); + expect(failed.status, failed.stderr).toBe(0); + expect(failed.stdout).toContain("[REDACTED]"); + expect(failed.stdout).not.toContain(apiKey); + expect(failed.stderr).toContain(`${HERMES_MCP_HTTP_STATUS_MARKER}500`); + + expect(readFileSync(countFile, "utf8")).toBe("1\n"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("rejects duplicate status markers, HTTP 500, and missing result tokens (#8697)", () => { + const secret = "fixture-diagnostic-secret"; + const longBody = `${secret}\nAuthorization: Bearer another-secret\n${"x".repeat( + HERMES_MCP_FAILURE_PREVIEW_CHARS * 2, + )}`; + try { + assertHermesMcpHttpResponse(httpResult(500, longBody), [secret]); + throw new Error("expected the HTTP 500 response assertion to throw"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + expect(message).toContain("HTTP 500"); + expect(message).toContain("[REDACTED]"); + expect(message).toContain("[truncated]"); + expect(message).not.toContain(secret); + } + + expect(() => assertHermesMcpHttpResponse(httpResult(200), [])).toThrowError( + /fixture result token/u, + ); + expect(() => + assertHermesMcpHttpResponse(httpResult(200, "", `${HERMES_MCP_HTTP_STATUS_MARKER}500\n`), []), + ).toThrowError(/exactly one HTTP status marker/u); + expect(() => + assertHermesMcpHttpResponse( + httpResult(200, "", `${HERMES_MCP_RESULT_TOKEN_MARKER}present\n`), + [], + ), + ).not.toThrow(); + }); +});