diff --git a/test/e2e/live/mcp-bridge-servers.ts b/test/e2e/live/mcp-bridge-servers.ts index ae36049794..b98e925b38 100644 --- a/test/e2e/live/mcp-bridge-servers.ts +++ b/test/e2e/live/mcp-bridge-servers.ts @@ -20,22 +20,31 @@ import type { TestProgress, TestProgressCapability } from "../fixtures/progress. type TestServer = http.Server | https.Server; +export const HERMES_DEFERRED_TOOL_SEARCH_MISS = + "Hermes tool_search did not return the deferred target"; + export interface StartedHttpServer { port: number; close(): Promise; } +export interface FakeMcpRequest { + method: string; + path: string; + auth: string; + body: string; + sessionId: string; + protocolVersion: string; + rpcMethod?: string; + responseStatus?: number; + responseHasResult?: boolean; + negotiatedSessionId?: string; + negotiatedProtocolVersion?: string; +} + export interface FakeMcpHttpsServer extends StartedHttpServer { setSecret(secret: string): void; - requests: Array<{ - method: string; - path: string; - auth: string; - body: string; - sessionId: string; - protocolVersion: string; - rpcMethod?: string; - }>; + requests: FakeMcpRequest[]; } export interface StartedPublicMcpTunnel { @@ -417,21 +426,27 @@ export async function startCompatibleMock(options: { return undefined; } }; - const hasExpectedHermesSearchResult = (toolName: string) => { - const parsed = parsedToolResult(0, "call_hermes_tool_search"); - return ( - Array.isArray(parsed?.matches) && - parsed.matches.some( - (match) => - match && - typeof match === "object" && - !Array.isArray(match) && - (match as Record).name === toolName, - ) + const classifyHermesSearchResult = ( + index: number, + toolName: string, + ): "target" | "miss" | "invalid" => { + const parsed = parsedToolResult(index, "call_hermes_tool_search"); + if (!Array.isArray(parsed?.matches)) return "invalid"; + const matches = parsed.matches; + const hasValidEntries = matches.every( + (match) => + match && + typeof match === "object" && + !Array.isArray(match) && + typeof (match as Record).name === "string", ); + if (!hasValidEntries) return "invalid"; + return matches.some((match) => (match as Record).name === toolName) + ? "target" + : "miss"; }; - const hasExpectedHermesDescription = (toolName: string) => { - const parsed = parsedToolResult(1, "call_hermes_tool_describe"); + const hasExpectedHermesDescription = (index: number, toolName: string) => { + const parsed = parsedToolResult(index, "call_hermes_tool_describe"); const parameters = parsed?.parameters; const properties = parameters && typeof parameters === "object" && !Array.isArray(parameters) @@ -490,17 +505,23 @@ export async function startCompatibleMock(options: { arguments: { query: options.deferredToolName }, }; } else if (toolResultCount === 1) { - if (hasExpectedHermesSearchResult(options.deferredToolName)) { + const searchResult = classifyHermesSearchResult(0, options.deferredToolName); + if (searchResult === "target") { plannedToolCall = { id: "call_hermes_tool_describe", name: "tool_describe", arguments: { name: options.deferredToolName }, }; + } else if (searchResult === "miss") { + protocolError = HERMES_DEFERRED_TOOL_SEARCH_MISS; } else { - protocolError = "Hermes tool_search did not return the deferred target"; + protocolError = "Hermes returned an unexpected deferred tool result sequence"; } - } else if (toolResultCount === 2) { - if (hasExpectedHermesDescription(options.deferredToolName)) { + } else if ( + toolResultCount === 2 && + toolResults.at(-1)?.tool_call_id === "call_hermes_tool_describe" + ) { + if (hasExpectedHermesDescription(1, options.deferredToolName)) { plannedToolCall = { id: "call_hermes_tool_call", name: "tool_call", @@ -513,7 +534,7 @@ export async function startCompatibleMock(options: { protocolError = "Hermes tool_describe did not return the deferred schema"; } } else { - protocolError = "Hermes returned an unexpected number of tool results"; + protocolError = "Hermes returned an unexpected deferred tool result sequence"; } } else if (!sawAuthenticatedToolResult) { const directToolName = [...visibleToolNames].find((name) => @@ -648,14 +669,7 @@ export async function startFakeMcpHttpsServer(options: { } return { cert: fs.readFileSync(certPath), key: fs.readFileSync(keyPath) }; })(); - const requests: Array<{ - method: string; - path: string; - auth: string; - body: string; - sessionId: string; - protocolVersion: string; - }> = []; + const requests: FakeMcpRequest[] = []; const server = https.createServer(tls, async (req, res) => { const requestPath = new URL(req.url ?? "/", "https://fake-mcp.local").pathname; const body = await readRequestBody(req); @@ -677,8 +691,9 @@ export async function startFakeMcpHttpsServer(options: { // The public quick-tunnel readiness probe uses HEAD /mcp. Keep it out of // the protocol request ledger so zero-upstream decoy and policy-denial // assertions continue to measure only attempted MCP traffic. + let recordedRequest: FakeMcpRequest | undefined; if (req.method !== "HEAD") { - requests.push({ + recordedRequest = { method: req.method ?? "", path: requestPath, auth, @@ -686,39 +701,54 @@ export async function startFakeMcpHttpsServer(options: { sessionId, protocolVersion, ...(typeof parsedPayload?.method === "string" ? { rpcMethod: parsedPayload.method } : {}), - }); + }; + requests.push(recordedRequest); } + const respondJson = (status: number, payload: unknown): void => { + if (recordedRequest) { + recordedRequest.responseStatus = status; + recordedRequest.responseHasResult = + typeof payload === "object" && + payload !== null && + Object.prototype.hasOwnProperty.call(payload, "result") && + !Object.prototype.hasOwnProperty.call(payload, "error"); + } + jsonResponse(res, status, payload); + }; + const respondEmpty = (status: number, headers?: http.OutgoingHttpHeaders): void => { + if (recordedRequest) recordedRequest.responseStatus = status; + res.writeHead(status, headers); + res.end(); + }; if (requestPath !== "/mcp") { - jsonResponse(res, 404, { error: { message: "not found" } }); + respondJson(404, { error: { message: "not found" } }); return; } if (req.method === "HEAD" || req.method === "GET") { - res.writeHead(405, { Allow: "POST" }); - res.end(); + respondEmpty(405, { Allow: "POST" }); return; } if (req.method !== "POST" && req.method !== "DELETE") { - jsonResponse(res, 405, { error: { message: "method not allowed" } }); + respondJson(405, { error: { message: "method not allowed" } }); return; } if (auth !== `Bearer ${expectedSecret}`) { - jsonResponse(res, 401, { error: { message: "missing rewritten bearer credential" } }); + respondJson(401, { error: { message: "missing rewritten bearer credential" } }); return; } if (req.method === "DELETE") { const negotiatedProtocolVersion = sessions.get(sessionId); if (!negotiatedProtocolVersion || protocolVersion !== negotiatedProtocolVersion) { - jsonResponse(res, 400, { error: { message: "missing negotiated MCP session metadata" } }); + respondJson(400, { error: { message: "missing negotiated MCP session metadata" } }); return; } sessions.delete(sessionId); - res.writeHead(204); - res.end(); + respondEmpty(204); return; } if (!parsedPayload) { - jsonResponse(res, 400, { error: { message: "invalid json" } }); + respondJson(400, { error: { message: "invalid json" } }); return; } // This shared fixture also serves intentional stateless policy probes. @@ -728,7 +758,7 @@ export async function startFakeMcpHttpsServer(options: { if (parsedPayload.method !== "initialize" && (sessionId !== "" || protocolVersion !== "")) { const negotiatedProtocolVersion = sessions.get(sessionId); if (!negotiatedProtocolVersion || protocolVersion !== negotiatedProtocolVersion) { - jsonResponse(res, 400, { error: { message: "missing negotiated MCP session metadata" } }); + respondJson(400, { error: { message: "missing negotiated MCP session metadata" } }); return; } } @@ -736,8 +766,7 @@ export async function startFakeMcpHttpsServer(options: { typeof parsedPayload.method === "string" && MCP_NOTIFICATION_METHODS.has(parsedPayload.method) ) { - res.writeHead(202); - res.end(); + respondEmpty(202); return; } let result: unknown; @@ -750,6 +779,10 @@ export async function startFakeMcpHttpsServer(options: { nextSessionId += 1; sessions.set(negotiatedSessionId, negotiatedProtocolVersion); res.setHeader("mcp-session-id", negotiatedSessionId); + if (recordedRequest) { + recordedRequest.negotiatedSessionId = negotiatedSessionId; + recordedRequest.negotiatedProtocolVersion = negotiatedProtocolVersion; + } result = { protocolVersion: negotiatedProtocolVersion, capabilities: { tools: {} }, @@ -783,7 +816,7 @@ export async function startFakeMcpHttpsServer(options: { ], }; } else { - jsonResponse(res, 200, { + respondJson(200, { jsonrpc: "2.0", id: parsedPayload.id ?? 1, error: { code: -32602, message: "invalid tools/list cursor" }, @@ -796,7 +829,7 @@ export async function startFakeMcpHttpsServer(options: { parsedPayload.params?.name !== "fake_echo" || (options.challenge !== undefined && challenge !== options.challenge) ) { - jsonResponse(res, 200, { + respondJson(200, { jsonrpc: "2.0", id: parsedPayload.id ?? 1, error: { code: -32602, message: "invalid fake_echo challenge" }, @@ -818,14 +851,14 @@ export async function startFakeMcpHttpsServer(options: { ) { result = MCP_EMPTY_RESULT_BY_METHOD[parsedPayload.method]; } else { - jsonResponse(res, 200, { + respondJson(200, { jsonrpc: "2.0", id: parsedPayload.id ?? 1, error: { code: -32601, message: "method not found" }, }); return; } - jsonResponse(res, 200, { + respondJson(200, { jsonrpc: "2.0", id: parsedPayload.id ?? 1, result, diff --git a/test/e2e/live/mcp-bridge-tool-discovery.ts b/test/e2e/live/mcp-bridge-tool-discovery.ts index 714f3559dc..1b1b4b5043 100644 --- a/test/e2e/live/mcp-bridge-tool-discovery.ts +++ b/test/e2e/live/mcp-bridge-tool-discovery.ts @@ -6,7 +6,69 @@ import { expect } from "vitest"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertExitZero } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; -import type { FakeMcpHttpsServer } from "./mcp-bridge-servers.ts"; +import type { FakeMcpHttpsServer, FakeMcpRequest } from "./mcp-bridge-servers.ts"; + +export interface AuthenticatedMcpDiscoveryTarget { + server: FakeMcpHttpsServer; + expectedSecret: string; + label: string; +} + +export async function assertAuthenticatedMcpRediscovery( + target: AuthenticatedMcpDiscoveryTarget | undefined, + requestOffset: number | undefined, +): Promise { + if (!target || requestOffset === undefined) return; + await assertAuthenticatedMcpDiscovery(target.server, { + requestOffset, + expectedSecret: target.expectedSecret, + label: target.label, + }); +} + +export function hasSuccessfulAuthenticatedMcpDiscovery( + requests: readonly FakeMcpRequest[], + expectedSecret: string, +): boolean { + const authenticatedRequests = requests.filter( + (request) => + request.method === "POST" && + request.path === "/mcp" && + request.auth === `Bearer ${expectedSecret}`, + ); + for (const [initializeIndex, initializeRequest] of authenticatedRequests.entries()) { + if ( + initializeRequest.rpcMethod !== "initialize" || + initializeRequest.responseStatus !== 200 || + initializeRequest.responseHasResult !== true || + !initializeRequest.negotiatedSessionId || + !initializeRequest.negotiatedProtocolVersion + ) { + continue; + } + const hasNegotiatedMetadata = (request: FakeMcpRequest) => + request.sessionId === initializeRequest.negotiatedSessionId && + request.protocolVersion === initializeRequest.negotiatedProtocolVersion; + const initializedIndex = authenticatedRequests.findIndex( + (request, requestIndex) => + requestIndex > initializeIndex && + request.rpcMethod === "notifications/initialized" && + request.responseStatus === 202 && + hasNegotiatedMetadata(request), + ); + if (initializedIndex === -1) continue; + const toolsListed = authenticatedRequests.some( + (request, requestIndex) => + requestIndex > initializedIndex && + request.rpcMethod === "tools/list" && + request.responseStatus === 200 && + request.responseHasResult === true && + hasNegotiatedMetadata(request), + ); + if (toolsListed) return true; + } + return false; +} export async function assertAuthenticatedMcpDiscovery( fakeMcp: FakeMcpHttpsServer, @@ -20,28 +82,25 @@ export async function assertAuthenticatedMcpDiscovery( .poll( () => { const requests = fakeMcp.requests.slice(options.requestOffset); - const observed = (rpcMethod: "initialize" | "tools/list") => - requests.some( - (request) => - request.method === "POST" && - request.path === "/mcp" && - request.rpcMethod === rpcMethod && - request.auth === `Bearer ${options.expectedSecret}`, - ); return { - initialized: observed("initialize"), - toolsListed: observed("tools/list"), + discovered: hasSuccessfulAuthenticatedMcpDiscovery(requests, options.expectedSecret), requests: requests.map((request) => ({ method: request.method, path: request.path, rpcMethod: request.rpcMethod, credentialRewritten: request.auth === `Bearer ${options.expectedSecret}`, + sessionId: request.sessionId, + protocolVersion: request.protocolVersion, + responseStatus: request.responseStatus, + responseHasResult: request.responseHasResult, + negotiatedSessionId: request.negotiatedSessionId, + negotiatedProtocolVersion: request.negotiatedProtocolVersion, })), }; }, { interval: 500, timeout: 90_000, message: options.label }, ) - .toMatchObject({ initialized: true, toolsListed: true }); + .toMatchObject({ discovered: true }); } export async function assertAuthenticatedMcpToolDiscovery( diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 16e22d8162..177165258c 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -49,6 +49,7 @@ import { } from "./mcp-bridge-servers.ts"; import { assertAuthenticatedMcpDiscovery, + assertAuthenticatedMcpRediscovery, assertAuthenticatedMcpToolDiscovery, } from "./mcp-bridge-tool-discovery.ts"; import { MCP_PROVIDER_REWRITE_PROBE_SOURCE } from "./mcp-provider-rewrite-probe.ts"; @@ -179,7 +180,6 @@ async function assertSecretAbsentFromSandbox( }); expectExitZero(result, "host MCP secret must not appear in sandbox files"); } - async function assertAdapterDnsRebindingDenied( host: HostCliClient, sandbox: SandboxClient, @@ -191,9 +191,7 @@ async function assertAdapterDnsRebindingDenied( secretPaths: string[]; }, ): Promise { - const rebindMcp = await startFakeMcpHttpsServer({ - secret: REBIND_HOST_SECRET, - }); + const rebindMcp = await startFakeMcpHttpsServer({ secret: REBIND_HOST_SECRET }); cleanup.add(`stop ${options.artifactPrefix} DNS rebinding fake MCP HTTPS server`, () => rebindMcp.close(), ); @@ -209,7 +207,6 @@ async function assertAdapterDnsRebindingDenied( cleanup.add(`restore ${options.artifactPrefix} DNS rebinding hosts fixture`, () => restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture), ); - await remapDnsRebindingHostname( host, options.sandboxName, @@ -242,7 +239,6 @@ async function assertAdapterDnsRebindingDenied( add, `${options.artifactPrefix} registers MCP route while its dedicated hostname resolves publicly`, ); - const status = await host.nemoclaw( [options.sandboxName, "mcp", "status", REBIND_SERVER_NAME, "--json"], { @@ -265,7 +261,6 @@ async function assertAdapterDnsRebindingDenied( policy: { gatewayPresent: true }, adapter: { registered: true }, }); - const policy = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-policy-pinned-public-ip`, env: buildAvailabilityProbeEnv(), @@ -289,11 +284,7 @@ async function assertAdapterDnsRebindingDenied( [REBIND_HOST_SECRET], `${options.artifactPrefix}-dns-rebinding-secret-absent-from-sandbox`, ); - - // If OpenShell resolved a second time after validating allowed_ips, this - // reachable runner address would receive the request. The pinned v0.0.72 - // implementation instead returns the one resolved-and-validated SocketAddr - // list directly to connect; see the exact proxy.rs citation in the helper. + // OpenShell connects to the address list resolved and validated against allowed_ips. const reboundAddress = await hostPrivateAddressForSandbox(host); expect(reboundAddress).not.toBe(REBIND_PUBLIC_IP); await remapDnsRebindingHostname( @@ -323,10 +314,7 @@ async function assertAdapterDnsRebindingDenied( rebindMcp.requests, `${options.artifactPrefix} rebound request must not reach the upstream MCP server`, ).toHaveLength(0); - - // Restore while the current sandbox container is stable. Removing the MCP - // route reloads policy and can restart the container first; the registered - // cleanup remains an idempotent fallback. + // Restore before removal can reload policy and restart the sandbox. await restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture); const remove = await host.nemoclaw([options.sandboxName, "mcp", "remove", REBIND_SERVER_NAME], { artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-remove`, @@ -335,7 +323,6 @@ async function assertAdapterDnsRebindingDenied( }); expectExitZero(remove, `${options.artifactPrefix} removes DNS rebinding route after proof`); } - async function addBridgeAndReadStatus( host: HostCliClient, options: { @@ -1252,25 +1239,26 @@ mcpBridgeShardTest("hermes")( "hermes-assert-secret-absent-after-add-gateway-restart", ); progress.phase("exercise lifecycle and confirm Hermes bridge removal"); + const survivingMcp = { + server: fakeMcp, + expectedSecret: HOST_SECRET, + label: "Hermes MCP rediscovery after explicit restart", + }; await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { adapter: "hermes-config", artifactPrefix: "hermes", sandboxName: HERMES_SANDBOX_NAME, secretPaths: ["/sandbox/.hermes"], }); - await assertRealAdapterToolCall(sandbox, fakeMcp, { - agent: "hermes", - sandboxName: HERMES_SANDBOX_NAME, - resultToken: hermesResult, - artifactName: "hermes-real-mcp-tool-call-initial", - }); + const survivingDiscoveryOffset = fakeMcp.requests.length; await restartBridgeWithoutHostSecret(host, HERMES_SANDBOX_NAME, "hermes"); await assertRealAdapterToolCall(sandbox, fakeMcp, { agent: "hermes", sandboxName: HERMES_SANDBOX_NAME, resultToken: hermesResult, - artifactName: "hermes-real-mcp-tool-call-after-restart", + artifactName: "hermes-real-mcp-tool-call-after-rediscovery-restart", }); + await assertAuthenticatedMcpRediscovery(survivingMcp, survivingDiscoveryOffset); fakeMcp.setSecret(ROTATED_HOST_SECRET); await rotateBridgeCredential(host, HERMES_SANDBOX_NAME, "hermes"); await assertRealAdapterToolCall(sandbox, fakeMcp, { diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts index 448a4bac35..4346ae4b62 100644 --- a/test/e2e/support/mcp-bridge-sandbox.test.ts +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -317,15 +317,30 @@ network_policies: expect(source).toContain(").toHaveLength(0);"); }); - it("restores the DNS fixture before MCP removal can restart the sandbox", () => { + it("captures the Hermes rediscovery offset after route removal and before restart", () => { const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); const denialProof = source.indexOf("rebound request must not reach the upstream MCP server"); const restore = source.indexOf("await restoreDnsRebindingHostsFixture", denialProof); const remove = source.indexOf("const remove = await host.nemoclaw", denialProof); + const hermesTest = source.indexOf('mcpBridgeShardTest("hermes")'); + const rebinding = source.indexOf("await assertAdapterDnsRebindingDenied", hermesTest); + const offset = source.indexOf( + "const survivingDiscoveryOffset = fakeMcp.requests.length", + rebinding, + ); + const restart = source.indexOf("await restartBridgeWithoutHostSecret", offset); + const toolCall = source.indexOf("await assertRealAdapterToolCall", restart); + const rediscovery = source.indexOf("await assertAuthenticatedMcpRediscovery", toolCall); expect(denialProof).toBeGreaterThanOrEqual(0); expect(restore).toBeGreaterThan(denialProof); expect(remove).toBeGreaterThan(restore); + expect(rebinding).toBeGreaterThan(hermesTest); + expect(offset).toBeGreaterThan(rebinding); + expect(restart).toBeGreaterThan(offset); + expect(toolCall).toBeGreaterThan(restart); + expect(rediscovery).toBeGreaterThan(toolCall); + expect(source).toContain("Hermes MCP rediscovery after explicit restart"); }); it("restores host DNS strictly while treating the ephemeral sandbox as best effort", async () => { diff --git a/test/e2e/support/mcp-bridge-tool-discovery.test.ts b/test/e2e/support/mcp-bridge-tool-discovery.test.ts new file mode 100644 index 0000000000..d3bfef5ca1 --- /dev/null +++ b/test/e2e/support/mcp-bridge-tool-discovery.test.ts @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it } from "vitest"; + +import { + type FakeMcpRequest, + HERMES_DEFERRED_TOOL_SEARCH_MISS, + type StartedHttpServer, + startCompatibleMock, +} from "../live/mcp-bridge-servers.ts"; +import { hasSuccessfulAuthenticatedMcpDiscovery } from "../live/mcp-bridge-tool-discovery.ts"; + +const EXPECTED_SECRET = "expected-secret"; +const EXPECTED_RESULT_TOKEN = "expected-result"; +const SESSION_ID = "fake-session-1"; +const PROTOCOL_VERSION = "2025-03-26"; + +function request(rpcMethod: string, overrides: Partial = {}): FakeMcpRequest { + return { + method: "POST", + path: "/mcp", + auth: `Bearer ${EXPECTED_SECRET}`, + body: "", + sessionId: SESSION_ID, + protocolVersion: PROTOCOL_VERSION, + rpcMethod, + responseStatus: rpcMethod === "notifications/initialized" ? 202 : 200, + responseHasResult: rpcMethod !== "notifications/initialized", + ...overrides, + }; +} + +function successfulInitialize(): FakeMcpRequest { + return request("initialize", { + sessionId: "", + protocolVersion: "", + negotiatedSessionId: SESSION_ID, + negotiatedProtocolVersion: PROTOCOL_VERSION, + }); +} + +interface CompatibleToolCall { + id: string; + function: { name: string; arguments: string }; +} + +interface CompatibleMessage { + role: string; + content: unknown; + tool_call_id?: string; + tool_calls?: CompatibleToolCall[]; +} + +const COMPATIBLE_API_KEY = "compatible-api-key"; +const COMPATIBLE_MODEL = "mock/mcp-bridge"; +const DEFERRED_TOOL_NAME = "mcp__fake__fake_echo"; +const TOOL_CHALLENGE = "deferred-tool-challenge"; +const BRIDGE_TOOLS = ["tool_search", "tool_describe", "tool_call"].map((name) => ({ + type: "function", + function: { name }, +})); + +let compatibleMock: StartedHttpServer | undefined; + +afterEach(async () => { + await compatibleMock?.close(); + compatibleMock = undefined; +}); + +async function startDeferredCompatibleMock(): Promise { + return startCompatibleMock({ + apiKey: COMPATIBLE_API_KEY, + model: COMPATIBLE_MODEL, + toolChallenge: TOOL_CHALLENGE, + toolResultToken: EXPECTED_RESULT_TOKEN, + deferredToolName: DEFERRED_TOOL_NAME, + }); +} + +async function requestCompatibleMessage( + server: StartedHttpServer, + messages: CompatibleMessage[], +): Promise { + const response = await fetch(`http://127.0.0.1:${server.port}/v1/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${COMPATIBLE_API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ model: COMPATIBLE_MODEL, messages, tools: BRIDGE_TOOLS }), + }); + expect(response.status).toBe(200); + const payload = (await response.json()) as { + choices?: Array<{ message?: CompatibleMessage }>; + }; + const message = payload.choices?.[0]?.message; + expect(message).toBeDefined(); + messages.push(message as CompatibleMessage); + return message as CompatibleMessage; +} + +function expectToolCall( + message: CompatibleMessage, + name: string, + expectedArguments: Record, +): CompatibleToolCall { + expect(message.tool_calls).toHaveLength(1); + const toolCall = message.tool_calls?.[0]; + expect(toolCall).toMatchObject({ function: { name } }); + expect(JSON.parse(toolCall?.function.arguments ?? "{}")).toEqual(expectedArguments); + return toolCall as CompatibleToolCall; +} + +function recordToolResult( + messages: CompatibleMessage[], + toolCall: CompatibleToolCall, + content: unknown, +): void { + messages.push({ + role: "tool", + content: JSON.stringify(content), + tool_call_id: toolCall.id, + }); +} + +describe("authenticated MCP rediscovery evidence", () => { + it("accepts successful tool discovery in one negotiated session", () => { + expect( + hasSuccessfulAuthenticatedMcpDiscovery( + [successfulInitialize(), request("notifications/initialized"), request("tools/list")], + EXPECTED_SECRET, + ), + ).toBe(true); + }); + + it("rejects tool discovery before session initialization completes", () => { + expect( + hasSuccessfulAuthenticatedMcpDiscovery( + [request("tools/list"), successfulInitialize(), request("notifications/initialized")], + EXPECTED_SECRET, + ), + ).toBe(false); + }); + + it("rejects tool discovery from a different negotiated session", () => { + expect( + hasSuccessfulAuthenticatedMcpDiscovery( + [ + successfulInitialize(), + request("notifications/initialized"), + request("tools/list", { sessionId: "fake-session-2" }), + ], + EXPECTED_SECRET, + ), + ).toBe(false); + }); + + it.each([ + ["an unsuccessful initialize HTTP response", 0, { responseStatus: 401 }], + ["an initialize response without a negotiated session ID", 0, { negotiatedSessionId: "" }], + [ + "an initialize response without a negotiated protocol version", + 0, + { negotiatedProtocolVersion: "" }, + ], + ["an initialized notification response with HTTP 200", 1, { responseStatus: 200 }], + ["a tools/list response without a JSON-RPC result", 2, { responseHasResult: false }], + ])("rejects %s", (_failure, failedRequestIndex, response) => { + const requests = [ + successfulInitialize(), + request("notifications/initialized"), + request("tools/list"), + ]; + Object.assign(requests[failedRequestIndex], response); + + expect(hasSuccessfulAuthenticatedMcpDiscovery(requests, EXPECTED_SECRET)).toBe(false); + }); +}); + +describe("Hermes deferred MCP tool discovery", () => { + it("uses one tool_search, tool_describe, and tool_call when the deferred target is present", async () => { + compatibleMock = await startDeferredCompatibleMock(); + const messages: CompatibleMessage[] = [{ role: "user", content: "call deferred tool" }]; + + const firstSearch = expectToolCall( + await requestCompatibleMessage(compatibleMock, messages), + "tool_search", + { query: DEFERRED_TOOL_NAME }, + ); + expect(firstSearch.id).toBe("call_hermes_tool_search"); + recordToolResult(messages, firstSearch, { matches: [{ name: DEFERRED_TOOL_NAME }] }); + + const description = expectToolCall( + await requestCompatibleMessage(compatibleMock, messages), + "tool_describe", + { name: DEFERRED_TOOL_NAME }, + ); + recordToolResult(messages, description, { + name: DEFERRED_TOOL_NAME, + parameters: { properties: { challenge: { type: "string" } } }, + }); + + const deferredCall = expectToolCall( + await requestCompatibleMessage(compatibleMock, messages), + "tool_call", + { + name: DEFERRED_TOOL_NAME, + arguments: { challenge: TOOL_CHALLENGE }, + }, + ); + recordToolResult(messages, deferredCall, EXPECTED_RESULT_TOKEN); + + const finalMessage = await requestCompatibleMessage(compatibleMock, messages); + expect(finalMessage).toMatchObject({ role: "assistant", content: EXPECTED_RESULT_TOKEN }); + expect(finalMessage.tool_calls).toBeUndefined(); + }); + + it("stops after one well-formed tool_search miss", async () => { + compatibleMock = await startDeferredCompatibleMock(); + const messages: CompatibleMessage[] = [{ role: "user", content: "call deferred tool" }]; + + const firstSearch = expectToolCall( + await requestCompatibleMessage(compatibleMock, messages), + "tool_search", + { query: DEFERRED_TOOL_NAME }, + ); + expect(firstSearch.id).toBe("call_hermes_tool_search"); + recordToolResult(messages, firstSearch, { matches: [] }); + + const terminalMessage = await requestCompatibleMessage(compatibleMock, messages); + expect(terminalMessage).toMatchObject({ + role: "assistant", + content: `mock protocol error: ${HERMES_DEFERRED_TOOL_SEARCH_MISS}`, + }); + expect(terminalMessage.tool_calls).toBeUndefined(); + }); + + it("rejects a malformed tool_search result without retrying", async () => { + compatibleMock = await startDeferredCompatibleMock(); + const messages: CompatibleMessage[] = [{ role: "user", content: "call deferred tool" }]; + + const firstSearch = expectToolCall( + await requestCompatibleMessage(compatibleMock, messages), + "tool_search", + { query: DEFERRED_TOOL_NAME }, + ); + expect(firstSearch.id).toBe("call_hermes_tool_search"); + recordToolResult(messages, firstSearch, { matches: [{ unexpected: true }] }); + + const terminalMessage = await requestCompatibleMessage(compatibleMock, messages); + expect(terminalMessage).toMatchObject({ + role: "assistant", + content: "mock protocol error: Hermes returned an unexpected deferred tool result sequence", + }); + expect(terminalMessage.tool_calls).toBeUndefined(); + }); +}); diff --git a/test/mcp-bridge-servers.test.ts b/test/mcp-bridge-servers.test.ts index 7c2cc8cca8..840a5befb7 100644 --- a/test/mcp-bridge-servers.test.ts +++ b/test/mcp-bridge-servers.test.ts @@ -13,6 +13,7 @@ import { MCP_BRIDGE_ALLOWED_METHODS } from "../src/lib/actions/sandbox/mcp-bridg import { startTestProgress } from "./e2e/fixtures/progress.ts"; import { buildCloudflaredQuickTunnelArgs, + HERMES_DEFERRED_TOOL_SEARCH_MISS, parseTryCloudflareOrigin, type StartedHttpServer, startCompatibleMock, @@ -36,7 +37,7 @@ type CompatibleToolCallResponse = { choices: Array<{ message: { content?: unknown; - tool_calls: Array<{ function: { name: string; arguments: string } }>; + tool_calls: Array<{ id: string; function: { name: string; arguments: string } }>; }; }>; }; @@ -588,6 +589,7 @@ describe("authenticated MCP live fixtures", () => { ).json()) as CompatibleToolCallResponse; const searchBody = await call([{ role: "user", content: "use the deferred tool" }]); expect(searchBody.choices[0].message.tool_calls[0]).toMatchObject({ + id: "call_hermes_tool_search", function: { name: "tool_search", arguments: JSON.stringify({ query: deferredToolName }), @@ -602,7 +604,11 @@ describe("authenticated MCP live fixtures", () => { ]); expect(missedSearch).toMatchObject({ choices: [ - { message: { content: expect.stringContaining("did not return the deferred target") } }, + { + message: { + content: `mock protocol error: ${HERMES_DEFERRED_TOOL_SEARCH_MISS}`, + }, + }, ], }); const echoedSearchQueryWithoutMatch = await call([ @@ -614,7 +620,11 @@ describe("authenticated MCP live fixtures", () => { ]); expect(echoedSearchQueryWithoutMatch).toMatchObject({ choices: [ - { message: { content: expect.stringContaining("did not return the deferred target") } }, + { + message: { + content: `mock protocol error: ${HERMES_DEFERRED_TOOL_SEARCH_MISS}`, + }, + }, ], }); const searchResult = {