test(e2e): restart Hermes before MCP rediscovery

test/e2e/live/mcp-bridge-servers.ts

@@ -20,22 +20,31 @@ import type { TestProgress, TestProgressCapability } from "../fixtures/progress.
 
 type TestServer = http.Server | https.Server;
 
+export const HERMES_DEFERRED_TOOL_SEARCH_MISS =
+  "Hermes tool_search did not return the deferred target";
+
 export interface StartedHttpServer {
   port: number;
   close(): Promise<void>;
 }
 
+export interface FakeMcpRequest {
+  method: string;
+  path: string;
+  auth: string;
+  body: string;
+  sessionId: string;
+  protocolVersion: string;
+  rpcMethod?: string;
+  responseStatus?: number;
+  responseHasResult?: boolean;
+  negotiatedSessionId?: string;
+  negotiatedProtocolVersion?: string;
+}
+
 export interface FakeMcpHttpsServer extends StartedHttpServer {
   setSecret(secret: string): void;
-  requests: Array<{
-    method: string;
-    path: string;
-    auth: string;
-    body: string;
-    sessionId: string;
-    protocolVersion: string;
-    rpcMethod?: string;
-  }>;
+  requests: FakeMcpRequest[];
 }
 
 export interface StartedPublicMcpTunnel {
@@ -417,21 +426,27 @@ export async function startCompatibleMock(options: {
           return undefined;
         }
       };
-      const hasExpectedHermesSearchResult = (toolName: string) => {
-        const parsed = parsedToolResult(0, "call_hermes_tool_search");
-        return (
-          Array.isArray(parsed?.matches) &&
-          parsed.matches.some(
-            (match) =>
-              match &&
-              typeof match === "object" &&
-              !Array.isArray(match) &&
-              (match as Record<string, unknown>).name === toolName,
-          )
+      const classifyHermesSearchResult = (
+        index: number,
+        toolName: string,
+      ): "target" | "miss" | "invalid" => {
+        const parsed = parsedToolResult(index, "call_hermes_tool_search");
+        if (!Array.isArray(parsed?.matches)) return "invalid";
+        const matches = parsed.matches;
+        const hasValidEntries = matches.every(
+          (match) =>
+            match &&
+            typeof match === "object" &&
+            !Array.isArray(match) &&
+            typeof (match as Record<string, unknown>).name === "string",
         );
+        if (!hasValidEntries) return "invalid";
+        return matches.some((match) => (match as Record<string, unknown>).name === toolName)
+          ? "target"
+          : "miss";
       };
-      const hasExpectedHermesDescription = (toolName: string) => {
-        const parsed = parsedToolResult(1, "call_hermes_tool_describe");
+      const hasExpectedHermesDescription = (index: number, toolName: string) => {
+        const parsed = parsedToolResult(index, "call_hermes_tool_describe");
         const parameters = parsed?.parameters;
         const properties =
           parameters && typeof parameters === "object" && !Array.isArray(parameters)
@@ -490,17 +505,23 @@ export async function startCompatibleMock(options: {
             arguments: { query: options.deferredToolName },
           };
         } else if (toolResultCount === 1) {
-          if (hasExpectedHermesSearchResult(options.deferredToolName)) {
+          const searchResult = classifyHermesSearchResult(0, options.deferredToolName);
+          if (searchResult === "target") {
             plannedToolCall = {
               id: "call_hermes_tool_describe",
               name: "tool_describe",
               arguments: { name: options.deferredToolName },
             };
+          } else if (searchResult === "miss") {
+            protocolError = HERMES_DEFERRED_TOOL_SEARCH_MISS;
           } else {
-            protocolError = "Hermes tool_search did not return the deferred target";
+            protocolError = "Hermes returned an unexpected deferred tool result sequence";
           }
-        } else if (toolResultCount === 2) {
-          if (hasExpectedHermesDescription(options.deferredToolName)) {
+        } else if (
+          toolResultCount === 2 &&
+          toolResults.at(-1)?.tool_call_id === "call_hermes_tool_describe"
+        ) {
+          if (hasExpectedHermesDescription(1, options.deferredToolName)) {
             plannedToolCall = {
               id: "call_hermes_tool_call",
               name: "tool_call",
@@ -513,7 +534,7 @@ export async function startCompatibleMock(options: {
             protocolError = "Hermes tool_describe did not return the deferred schema";
           }
         } else {
-          protocolError = "Hermes returned an unexpected number of tool results";
+          protocolError = "Hermes returned an unexpected deferred tool result sequence";
         }
       } else if (!sawAuthenticatedToolResult) {
         const directToolName = [...visibleToolNames].find((name) =>
@@ -648,14 +669,7 @@ export async function startFakeMcpHttpsServer(options: {
       }
       return { cert: fs.readFileSync(certPath), key: fs.readFileSync(keyPath) };
     })();
-  const requests: Array<{
-    method: string;
-    path: string;
-    auth: string;
-    body: string;
-    sessionId: string;
-    protocolVersion: string;
-  }> = [];
+  const requests: FakeMcpRequest[] = [];
   const server = https.createServer(tls, async (req, res) => {
     const requestPath = new URL(req.url ?? "/", "https://fake-mcp.local").pathname;
     const body = await readRequestBody(req);
@@ -677,48 +691,64 @@ export async function startFakeMcpHttpsServer(options: {
     // The public quick-tunnel readiness probe uses HEAD /mcp. Keep it out of
     // the protocol request ledger so zero-upstream decoy and policy-denial
     // assertions continue to measure only attempted MCP traffic.
+    let recordedRequest: FakeMcpRequest | undefined;
     if (req.method !== "HEAD") {
-      requests.push({
+      recordedRequest = {
         method: req.method ?? "",
         path: requestPath,
         auth,
         body,
         sessionId,
         protocolVersion,
         ...(typeof parsedPayload?.method === "string" ? { rpcMethod: parsedPayload.method } : {}),
-      });
+      };
+      requests.push(recordedRequest);
     }
+    const respondJson = (status: number, payload: unknown): void => {
+      if (recordedRequest) {
+        recordedRequest.responseStatus = status;
+        recordedRequest.responseHasResult =
+          typeof payload === "object" &&
+          payload !== null &&
+          Object.prototype.hasOwnProperty.call(payload, "result") &&
+          !Object.prototype.hasOwnProperty.call(payload, "error");
+      }
+      jsonResponse(res, status, payload);
+    };
+    const respondEmpty = (status: number, headers?: http.OutgoingHttpHeaders): void => {
+      if (recordedRequest) recordedRequest.responseStatus = status;
+      res.writeHead(status, headers);
+      res.end();
+    };
     if (requestPath !== "/mcp") {
-      jsonResponse(res, 404, { error: { message: "not found" } });
+      respondJson(404, { error: { message: "not found" } });
       return;
     }
     if (req.method === "HEAD" || req.method === "GET") {
-      res.writeHead(405, { Allow: "POST" });
-      res.end();
+      respondEmpty(405, { Allow: "POST" });
       return;
     }
     if (req.method !== "POST" && req.method !== "DELETE") {
-      jsonResponse(res, 405, { error: { message: "method not allowed" } });
+      respondJson(405, { error: { message: "method not allowed" } });
       return;
     }
     if (auth !== `Bearer ${expectedSecret}`) {
-      jsonResponse(res, 401, { error: { message: "missing rewritten bearer credential" } });
+      respondJson(401, { error: { message: "missing rewritten bearer credential" } });
       return;
     }
     if (req.method === "DELETE") {
       const negotiatedProtocolVersion = sessions.get(sessionId);
       if (!negotiatedProtocolVersion || protocolVersion !== negotiatedProtocolVersion) {
-        jsonResponse(res, 400, { error: { message: "missing negotiated MCP session metadata" } });
+        respondJson(400, { error: { message: "missing negotiated MCP session metadata" } });
         return;
       }
       sessions.delete(sessionId);
-      res.writeHead(204);
-      res.end();
+      respondEmpty(204);
       return;
     }
 
     if (!parsedPayload) {
-      jsonResponse(res, 400, { error: { message: "invalid json" } });
+      respondJson(400, { error: { message: "invalid json" } });
       return;
     }
     // This shared fixture also serves intentional stateless policy probes.
@@ -728,16 +758,15 @@ export async function startFakeMcpHttpsServer(options: {
     if (parsedPayload.method !== "initialize" && (sessionId !== "" || protocolVersion !== "")) {
       const negotiatedProtocolVersion = sessions.get(sessionId);
       if (!negotiatedProtocolVersion || protocolVersion !== negotiatedProtocolVersion) {
-        jsonResponse(res, 400, { error: { message: "missing negotiated MCP session metadata" } });
+        respondJson(400, { error: { message: "missing negotiated MCP session metadata" } });
         return;
       }
     }
     if (
       typeof parsedPayload.method === "string" &&
       MCP_NOTIFICATION_METHODS.has(parsedPayload.method)
     ) {
-      res.writeHead(202);
-      res.end();
+      respondEmpty(202);
       return;
     }
     let result: unknown;
@@ -750,6 +779,10 @@ export async function startFakeMcpHttpsServer(options: {
       nextSessionId += 1;
       sessions.set(negotiatedSessionId, negotiatedProtocolVersion);
       res.setHeader("mcp-session-id", negotiatedSessionId);
+      if (recordedRequest) {
+        recordedRequest.negotiatedSessionId = negotiatedSessionId;
+        recordedRequest.negotiatedProtocolVersion = negotiatedProtocolVersion;
+      }
       result = {
         protocolVersion: negotiatedProtocolVersion,
         capabilities: { tools: {} },
@@ -783,7 +816,7 @@ export async function startFakeMcpHttpsServer(options: {
           ],
         };
       } else {
-        jsonResponse(res, 200, {
+        respondJson(200, {
           jsonrpc: "2.0",
           id: parsedPayload.id ?? 1,
           error: { code: -32602, message: "invalid tools/list cursor" },
@@ -796,7 +829,7 @@ export async function startFakeMcpHttpsServer(options: {
         parsedPayload.params?.name !== "fake_echo" ||
         (options.challenge !== undefined && challenge !== options.challenge)
       ) {
-        jsonResponse(res, 200, {
+        respondJson(200, {
           jsonrpc: "2.0",
           id: parsedPayload.id ?? 1,
           error: { code: -32602, message: "invalid fake_echo challenge" },
@@ -818,14 +851,14 @@ export async function startFakeMcpHttpsServer(options: {
     ) {
       result = MCP_EMPTY_RESULT_BY_METHOD[parsedPayload.method];
     } else {
-      jsonResponse(res, 200, {
+      respondJson(200, {
         jsonrpc: "2.0",
         id: parsedPayload.id ?? 1,
         error: { code: -32601, message: "method not found" },
       });
       return;
     }
-    jsonResponse(res, 200, {
+    respondJson(200, {
       jsonrpc: "2.0",
       id: parsedPayload.id ?? 1,
       result,

test/e2e/live/mcp-bridge-tool-discovery.ts

@@ -6,7 +6,69 @@ import { expect } from "vitest";
 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
 import { assertExitZero } from "../fixtures/clients/command.ts";
 import type { HostCliClient } from "../fixtures/clients/host.ts";
-import type { FakeMcpHttpsServer } from "./mcp-bridge-servers.ts";
+import type { FakeMcpHttpsServer, FakeMcpRequest } from "./mcp-bridge-servers.ts";
+
+export interface AuthenticatedMcpDiscoveryTarget {
+  server: FakeMcpHttpsServer;
+  expectedSecret: string;
+  label: string;
+}
+
+export async function assertAuthenticatedMcpRediscovery(
+  target: AuthenticatedMcpDiscoveryTarget | undefined,
+  requestOffset: number | undefined,
+): Promise<void> {
+  if (!target || requestOffset === undefined) return;
+  await assertAuthenticatedMcpDiscovery(target.server, {
+    requestOffset,
+    expectedSecret: target.expectedSecret,
+    label: target.label,
+  });
+}
+
+export function hasSuccessfulAuthenticatedMcpDiscovery(
+  requests: readonly FakeMcpRequest[],
+  expectedSecret: string,
+): boolean {
+  const authenticatedRequests = requests.filter(
+    (request) =>
+      request.method === "POST" &&
+      request.path === "/mcp" &&
+      request.auth === `Bearer ${expectedSecret}`,
+  );
+  for (const [initializeIndex, initializeRequest] of authenticatedRequests.entries()) {
+    if (
+      initializeRequest.rpcMethod !== "initialize" ||
+      initializeRequest.responseStatus !== 200 ||
+      initializeRequest.responseHasResult !== true ||
+      !initializeRequest.negotiatedSessionId ||
+      !initializeRequest.negotiatedProtocolVersion
+    ) {
+      continue;
+    }
+    const hasNegotiatedMetadata = (request: FakeMcpRequest) =>
+      request.sessionId === initializeRequest.negotiatedSessionId &&
+      request.protocolVersion === initializeRequest.negotiatedProtocolVersion;
+    const initializedIndex = authenticatedRequests.findIndex(
+      (request, requestIndex) =>
+        requestIndex > initializeIndex &&
+        request.rpcMethod === "notifications/initialized" &&
+        request.responseStatus === 202 &&
+        hasNegotiatedMetadata(request),
+    );
+    if (initializedIndex === -1) continue;
+    const toolsListed = authenticatedRequests.some(
+      (request, requestIndex) =>
+        requestIndex > initializedIndex &&
+        request.rpcMethod === "tools/list" &&
+        request.responseStatus === 200 &&
+        request.responseHasResult === true &&
+        hasNegotiatedMetadata(request),
+    );
+    if (toolsListed) return true;
+  }
+  return false;
+}
 
 export async function assertAuthenticatedMcpDiscovery(
   fakeMcp: FakeMcpHttpsServer,
@@ -20,28 +82,25 @@ export async function assertAuthenticatedMcpDiscovery(
     .poll(
       () => {
         const requests = fakeMcp.requests.slice(options.requestOffset);
-        const observed = (rpcMethod: "initialize" | "tools/list") =>
-          requests.some(
-            (request) =>
-              request.method === "POST" &&
-              request.path === "/mcp" &&
-              request.rpcMethod === rpcMethod &&
-              request.auth === `Bearer ${options.expectedSecret}`,
-          );
         return {
-          initialized: observed("initialize"),
-          toolsListed: observed("tools/list"),
+          discovered: hasSuccessfulAuthenticatedMcpDiscovery(requests, options.expectedSecret),
           requests: requests.map((request) => ({
             method: request.method,
             path: request.path,
             rpcMethod: request.rpcMethod,
             credentialRewritten: request.auth === `Bearer ${options.expectedSecret}`,
+            sessionId: request.sessionId,
+            protocolVersion: request.protocolVersion,
+            responseStatus: request.responseStatus,
+            responseHasResult: request.responseHasResult,
+            negotiatedSessionId: request.negotiatedSessionId,
+            negotiatedProtocolVersion: request.negotiatedProtocolVersion,
           })),
         };
       },
       { interval: 500, timeout: 90_000, message: options.label },
     )
-    .toMatchObject({ initialized: true, toolsListed: true });
+    .toMatchObject({ discovered: true });
 }
 
 export async function assertAuthenticatedMcpToolDiscovery(