merge queue: checking release/v3.8.49 (7123236) and #7093 together

src/app/api/v1/relay/chat/completions/bifrost/route.ts

@@ -260,6 +260,8 @@ export async function POST(request: Request) {
       "x-relay-client-ip": clientIp,
       ...getProviderPluginManifestHeader(new URL(request.url).origin),
     };
+    const requestId = request.headers.get("x-request-id");
+    if (requestId) upstreamHeaders["x-request-id"] = requestId;
     if (BIFROST_API_KEY) {
       upstreamHeaders["Authorization"] = `Bearer ${BIFROST_API_KEY}`;
     }

src/app/api/v1/relay/chat/completions/route.ts

@@ -65,6 +65,7 @@ async function forwardToBifrost(
   body: unknown,
   token: RelayToken,
   config: BifrostRoutingConfig,
+  backend: ReturnType<typeof resolveRelayRoutingBackend>,
   startTime: number,
   clientIp: string,
   userAgent: string | null
@@ -77,6 +78,8 @@ async function forwardToBifrost(
     "x-relay-client-ip": clientIp,
     ...getProviderPluginManifestHeader(new URL(request.url).origin),
   };
+  const requestId = request.headers.get("x-request-id");
+  if (requestId) upstreamHeaders["x-request-id"] = requestId;
   if (config.apiKey) {
     upstreamHeaders.Authorization = `Bearer ${config.apiKey}`;
   }
@@ -95,7 +98,6 @@ async function forwardToBifrost(
       body: JSON.stringify(body),
       signal: ac.signal,
     });
-    clearTimeout(tid);
 
     const headers = new Headers(upstream.headers);
     headers.set("X-Routed-By", "bifrost");
@@ -107,14 +109,24 @@ async function forwardToBifrost(
 
     if (wantsStream && upstream.body) {
       const stream = finalizeReadableStream(upstream.body, (error) => {
+        clearTimeout(tid);
+        const statusCode = timedOut ? 504 : upstream.status;
+        if (error && backend === "auto") {
+          recordBifrostFailure(
+            config.baseUrl,
+            timedOut
+              ? `Bifrost sidecar stream timed out after ${config.timeoutMs}ms`
+              : "bifrost-stream-error"
+          );
+        }
         recordUsage(
           token.id,
           request,
           startTime,
           clientIp,
           userAgent,
-          error || upstream.status >= 500 ? "error" : "success",
-          upstream.status
+          error || statusCode >= 500 ? "error" : "success",
+          statusCode
         );
       });
 
@@ -124,6 +136,7 @@ async function forwardToBifrost(
       });
     }
 
+    clearTimeout(tid);
     recordUsage(
       token.id,
       request,
@@ -313,6 +326,7 @@ export async function POST(request: Request) {
             parsedBody,
             token,
             bifrostConfig,
+            backend,
             startTime,
             clientIp,
             userAgent

tests/unit/api/v1/bifrost-sidecar.test.ts

@@ -79,9 +79,8 @@ test("bifrost route: returns 503 + fallback header when BIFROST_BASE_URL is unse
   delete process.env.BIFROST_STREAMING_ENABLED;
 
   // Dynamic import after env is set so the module reads the empty value.
-  const { POST } = await import(
-    "../../../../src/app/api/v1/relay/chat/completions/bifrost/route.ts"
-  );
+  const { POST } =
+    await import("../../../../src/app/api/v1/relay/chat/completions/bifrost/route.ts");
 
   const req = new Request("http://localhost/api/v1/relay/chat/completions/bifrost", {
     method: "POST",
@@ -191,12 +190,14 @@ test("bifrost route: records relay usage after SSE stream completion", async ()
   delete process.env.BIFROST_STREAMING_ENABLED;
 
   const relayToken = seedRelayToken(`relay_bifrost_sse_${Date.now()}`);
+  let forwardedRequestId: string | null = null;
 
-  globalThis.fetch = async () =>
-    new Response(
+  globalThis.fetch = async (_input, init) => {
+    forwardedRequestId = new Headers(init?.headers).get("x-request-id");
+    return new Response(
       new ReadableStream<Uint8Array>({
         start(controller) {
-          controller.enqueue(new TextEncoder().encode("data: {\"delta\":\"hi\"}\n\n"));
+          controller.enqueue(new TextEncoder().encode('data: {"delta":"hi"}\n\n'));
           controller.close();
         },
       }),
@@ -205,6 +206,7 @@ test("bifrost route: records relay usage after SSE stream completion", async ()
         headers: { "content-type": "text/event-stream" },
       }
     );
+  };
 
   const { POST } = await import(
     `../../../../src/app/api/v1/relay/chat/completions/bifrost/route.ts?case=${Date.now()}-${Math.random()}`
@@ -227,6 +229,7 @@ test("bifrost route: records relay usage after SSE stream completion", async ()
   const res = await POST(req);
   assert.equal(res.status, 200);
   assert.equal(res.headers.get("X-Routed-By"), "bifrost");
+  assert.equal(forwardedRequestId, "bifrost-sse-lifecycle-test");
   assert.equal(getRelayLogs(relayToken.id, 10).length, 0);
 
   assert.match(await res.text(), /delta/);

tests/unit/api/v1/relay-routing-backend.test.ts

@@ -1,5 +1,6 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
 import {
   getBifrostRoutingConfig,
   getRoutingFallbackHeader,
@@ -152,3 +153,26 @@ test("relay routing backend strict bifrost bypasses manifest eligibility", () =>
     { tryBifrost: true }
   );
 });
+
+test("automatic relay keeps the Bifrost timeout active until an SSE stream finalizes", () => {
+  const routeSource = readFileSync(
+    new URL("../../../../src/app/api/v1/relay/chat/completions/route.ts", import.meta.url),
+    "utf8"
+  );
+  const forwardToBifrost = routeSource.slice(
+    routeSource.indexOf("async function forwardToBifrost"),
+    routeSource.indexOf("export async function OPTIONS")
+  );
+  const streamBranch = forwardToBifrost.slice(
+    forwardToBifrost.indexOf("if (wantsStream && upstream.body)"),
+    forwardToBifrost.indexOf("clearTimeout(tid);\n    recordUsage(")
+  );
+
+  assert.match(
+    streamBranch,
+    /finalizeReadableStream\(upstream\.body, \(error\) => \{\s*clearTimeout\(tid\)/
+  );
+  assert.match(streamBranch, /const statusCode = timedOut \? 504 : upstream\.status/);
+  assert.match(streamBranch, /error && backend === "auto"/);
+  assert.match(streamBranch, /recordBifrostFailure\(/);
+});