fix(chatgpt-web): render citations as markdown links

CHANGELOG.md

@@ -54,6 +54,7 @@ _Living section — bullets land here as PRs merge into `release/v3.8.47` (paral
 - **fix(cli):** compression CLI REST fallback now reads/writes the canonical `defaultMode` field (surfaced as `strategy`) instead of a nonexistent `engine` key, and table output renders nested objects as JSON instead of `[object Object]` (#6571 — thanks @charleszolot)
 - **fix(providers):** web-cookie providers without a `providerRegistry.ts` entry (`lmarena`, `gemini-business`, `poe-web`, `venice-web`, `v0-vercel-web`) now report `unsupported: true` instead of silently "OK" ([#6309](https://github.com/diegosouzapw/OmniRoute/pull/6309)) — `validateWebCookieProvider()` (`src/lib/providers/validation.ts`) previously required a registry entry and returned "Provider not found in registry" for these; a fallback to `WEB_COOKIE_PROVIDERS[provider].website` was proposed, but live verification showed the `${website}/models` probe does not reliably signal session validity for these providers (redirects/SPA 200s regardless of cookie validity — e.g. lmarena's real API is `arena.ai`, not `lmarena.ai`; Poe's real endpoint is a GraphQL POST, not a REST `/models`), so it would report an expired or garbage cookie as valid. Until each provider has a verified, side-effect-free auth probe against its real API host, the fallback now returns `unsupported` (no network call) instead of a false positive. Regression guard: `tests/unit/web-cookie-validation-fallback.test.ts`. (thanks @oyi77)
 - **fix(api):** `POST /api/middleware/hooks` and `PUT /api/middleware/hooks/[name]` no longer leak raw internal error messages in their 500 responses (#6645 — thanks @chirag127) — both catch blocks returned `error?.message` directly (Hard Rule #12), which could surface internal SQLite path fragments on a DB failure; both now route through `sanitizeErrorMessage()` from `open-sse/utils/error.ts`. Regression guard: `tests/unit/middleware-hooks-error-sanitization.test.ts`.
+- **fix(providers):** ChatGPT Web (`chatgpt-web`) responses rendered raw ChatGPT UI citation markup — private-use marker tokens (e.g. `citeturn0search0`) and `url…` inline-link markers — instead of real Markdown links, since these only ever get resolved client-side by chatgpt.com's own JS using `message.metadata.content_references` ([#6635](https://github.com/diegosouzapw/OmniRoute/pull/6635)) — `cleanChatGptText()` now resolves `content_references` (grouped webpages, footnote sources, inline `webpage`/`url` mentions) into `[label](url)` Markdown links for both the streaming and non-streaming response builders, and for the GPT-5.5 Pro `stream_handoff` polled-answer path, falling back to stripping any marker that has no resolvable source instead of leaking the raw private-use bytes. The citation parsing/rendering logic was extracted into a new pure sibling module (`open-sse/executors/chatgpt-web/citations.ts`) to keep the executor under the frozen file-size cap. Regression guard: `tests/unit/chatgpt-web-citations.test.ts` (non-streaming citation resolution, streaming marker buffering across split SSE chunks, and the Pro-handoff polled-answer path). (thanks @Thinkscape)
 
 ### 📝 Maintenance
 

config/quality/eslint-suppressions.json

@@ -737,11 +737,6 @@
       "count": 6
     }
   },
-  "tests/unit/chatgpt-web.test.ts": {
-    "@typescript-eslint/no-explicit-any": {
-      "count": 6
-    }
-  },
   "tests/unit/chipotle-executor.test.ts": {
     "@typescript-eslint/no-explicit-any": {
       "count": 3

open-sse/executors/chatgpt-web.ts

@@ -33,6 +33,7 @@ import {
   type ChatGptImageConversationContext,
 } from "../services/chatgptImageCache.ts";
 import { isThinkingCapableModel, resolveChatGptModel } from "./chatgpt-web/models.ts";
+import { cleanChatGptText } from "./chatgpt-web/citations.ts";
 
 // ─── Constants ──────────────────────────────────────────────────────────────
 
@@ -1165,6 +1166,7 @@ interface ContentChunk {
   answer?: string;
   conversationId?: string;
   messageId?: string;
+  metadata?: Record<string, unknown>;
   error?: string;
   done?: boolean;
   /** Image asset pointers seen on the current message (e.g. file-service://file-abc). */
@@ -1226,6 +1228,7 @@ async function* extractContent(
   let conversationId: string | null = null;
   let currentId: string | null = null;
   let currentParts = "";
+  let currentMetadata: Record<string, unknown> | undefined;
   let emittedLen = 0;
   let isLive = false;
   // Dedupe pointers across echoes / repeated events. Order-preserving Set.
@@ -1270,7 +1273,8 @@ async function* extractContent(
     // on a tool-role message (handled below).
     if (event.type === "server_ste_metadata") {
       const meta = (event as Record<string, unknown>).metadata as
-        Record<string, unknown> | undefined;
+        | Record<string, unknown>
+        | undefined;
       if (meta && meta.turn_use_case === "image gen") {
         imageGenAsync = true;
       }
@@ -1295,10 +1299,15 @@ async function* extractContent(
     if (id && id !== currentId) {
       currentId = id;
       currentParts = "";
+      currentMetadata = undefined;
       emittedLen = 0;
       isLive = false;
     }
 
+    if (m.metadata && typeof m.metadata === "object") {
+      currentMetadata = m.metadata;
+    }
+
     if (status === "in_progress") {
       isLive = true;
     }
@@ -1337,6 +1346,7 @@ async function* extractContent(
         answer: currentParts,
         conversationId: conversationId ?? undefined,
         messageId: currentId ?? undefined,
+        metadata: currentMetadata,
       };
     }
   }
@@ -1350,6 +1360,7 @@ async function* extractContent(
       answer: currentParts,
       conversationId: conversationId ?? undefined,
       messageId: currentId ?? undefined,
+      metadata: currentMetadata,
     };
   }
 
@@ -1358,6 +1369,7 @@ async function* extractContent(
     answer: currentParts,
     conversationId: conversationId ?? undefined,
     messageId: currentId ?? undefined,
+    metadata: currentMetadata,
     imagePointers: imagePointers.size > 0 ? Array.from(imagePointers.values()) : undefined,
     imageGenAsync,
     handoff,
@@ -1389,6 +1401,7 @@ interface ChatGptConversationDetail {
 interface FinalAssistantAnswer {
   text: string;
   messageId?: string;
+  metadata?: Record<string, unknown>;
   finished: boolean;
 }
 
@@ -1433,12 +1446,17 @@ function extractFinalAssistantAnswer(
       (finished && (!best.finished || sort >= best.sort)) ||
       (!finished && !best.finished && sort >= best.sort)
     ) {
-      best = { text, messageId: message.id, finished, sort };
+      best = { text, messageId: message.id, metadata: message.metadata, finished, sort };
     }
   }
 
   if (!best) return null;
-  return { text: best.text, messageId: best.messageId, finished: best.finished };
+  return {
+    text: best.text,
+    messageId: best.messageId,
+    metadata: best.metadata,
+    finished: best.finished,
+  };
 }
 
 function delayWithAbort(ms: number, signal?: AbortSignal | null): Promise<void> {
@@ -1587,10 +1605,7 @@ type ImageResolver = (
  * "no image was produced". Escalated mesh report: image visible in the ChatGPT
  * chat but returned to OmniRoute as a bare "completed without image markdown".
  */
-export function detectImageResolutionFailure(
-  pointerCount: number,
-  resolvedCount: number
-): boolean {
+export function detectImageResolutionFailure(pointerCount: number, resolvedCount: number): boolean {
   return pointerCount > 0 && resolvedCount === 0;
 }
 
@@ -1674,13 +1689,12 @@ function buildStreamingResponse(
           let imageGenAsync = false;
           let handoff = false;
           let emittedText = "";
-          let polledFinalAnswer = "";
+          let polledFinalAnswer: FinalAssistantAnswer | null = null;
           let parentCandidateMessageId: string | null = null;
 
-          const emitTextDelta = (content: string): void => {
-            const cleaned = cleanChatGptText(content);
-            if (!cleaned) return;
-            emittedText += cleaned;
+          const emitRenderedDelta = (content: string): void => {
+            if (!content) return;
+            emittedText += content;
             controller.enqueue(
               encoder.encode(
                 sseChunk({
@@ -1692,7 +1706,7 @@ function buildStreamingResponse(
                   choices: [
                     {
                       index: 0,
-                      delta: { content: cleaned },
+                      delta: { content },
                       finish_reason: null,
                       logprobs: null,
                     },
@@ -1702,14 +1716,29 @@ function buildStreamingResponse(
             );
           };
 
-          const appendFinalAnswer = (text: string): void => {
-            const cleaned = cleanChatGptText(text);
+          const emitRenderedAnswer = (
+            rawText: string,
+            metadata?: Record<string, unknown>
+          ): void => {
+            const rendered = cleanChatGptText(rawText, metadata);
+            if (!rendered || rendered.length <= emittedText.length) return;
+            if (!rendered.startsWith(emittedText)) {
+              // We cannot retract bytes already streamed. This should be rare;
+              // it mainly protects clients if ChatGPT rewrites earlier text.
+              const common = commonPrefixLength(rendered, emittedText);
+              if (common < emittedText.length) return;
+            }
+            emitRenderedDelta(rendered.slice(emittedText.length));
+          };
+
+          const appendFinalAnswer = (text: string, metadata?: Record<string, unknown>): void => {
+            const cleaned = cleanChatGptText(text, metadata);
             const finalTrimmed = cleaned.trim();
             if (!finalTrimmed) return;
             const emittedTrimmed = emittedText.trim();
             if (emittedTrimmed === finalTrimmed || emittedTrimmed.endsWith(finalTrimmed)) return;
             const prefix = emittedTrimmed && !emittedText.endsWith("\n") ? "\n\n" : "";
-            emitTextDelta(`${prefix}${cleaned}`);
+            emitRenderedDelta(`${prefix}${cleaned}`);
           };
 
           // Heartbeat: long async work (Pro polling, WebSocket image-gen,
@@ -1776,8 +1805,8 @@ function buildStreamingResponse(
               break;
             }
 
-            if (chunk.delta) {
-              emitTextDelta(chunk.delta);
+            if (chunk.answer) {
+              emitRenderedAnswer(chunk.answer, chunk.metadata);
             }
           }
 
@@ -1786,7 +1815,7 @@ function buildStreamingResponse(
             try {
               const polled = await pollFinalAnswer(conversationId);
               if (polled?.text) {
-                polledFinalAnswer = polled.text;
+                polledFinalAnswer = polled;
                 if (polled.messageId) parentCandidateMessageId = polled.messageId;
               }
             } finally {
@@ -1795,7 +1824,7 @@ function buildStreamingResponse(
           }
 
           if (polledFinalAnswer) {
-            appendFinalAnswer(polledFinalAnswer);
+            appendFinalAnswer(polledFinalAnswer.text, polledFinalAnswer.metadata);
           }
 
           // Async image_gen ends the SSE with a "Processing image..."
@@ -1971,6 +2000,7 @@ async function buildNonStreamingResponse(
   let imagePointers: ImagePointerRef[] | undefined;
   let imageGenAsync = false;
   let handoff = false;
+  let answerMetadata: Record<string, unknown> | undefined;
   let parentCandidateMessageId: string | null = null;
 
   for await (const chunk of extractContent(eventStream, signal)) {
@@ -1987,24 +2017,29 @@ async function buildNonStreamingResponse(
     }
     if (chunk.done) {
       fullAnswer = chunk.answer || fullAnswer;
+      answerMetadata = chunk.metadata ?? answerMetadata;
       imagePointers = chunk.imagePointers;
       imageGenAsync = chunk.imageGenAsync ?? false;
       handoff = handoff || (chunk.handoff ?? false);
       if (chunk.messageId) parentCandidateMessageId = chunk.messageId;
       break;
     }
-    if (chunk.answer) fullAnswer = chunk.answer;
+    if (chunk.answer) {
+      fullAnswer = chunk.answer;
+      answerMetadata = chunk.metadata ?? answerMetadata;
+    }
   }
 
   if (pollFinalAnswer && conversationId && (handoff || !fullAnswer.trim())) {
     const polled = await pollFinalAnswer(conversationId);
     if (polled?.text) {
       fullAnswer = polled.text;
+      answerMetadata = polled.metadata ?? answerMetadata;
       if (polled.messageId) parentCandidateMessageId = polled.messageId;
     }
   }
 
-  fullAnswer = cleanChatGptText(fullAnswer);
+  fullAnswer = cleanChatGptText(fullAnswer, answerMetadata);
 
   // Async image gen: SSE ended with "Processing image..." — poll for the
   // final pointer the same way the streaming path does.
@@ -2683,7 +2718,8 @@ export class ChatGptWebExecutor extends BaseExecutor {
     clientHeaders,
   }: ExecuteInput) {
     const messages = (body as Record<string, unknown> | null)?.messages as
-      Array<Record<string, unknown>> | undefined;
+      | Array<Record<string, unknown>>
+      | undefined;
     if (!messages || !Array.isArray(messages) || messages.length === 0) {
       return {
         response: errorResponse(400, "Missing or empty messages array"),
@@ -3073,15 +3109,11 @@ export class ChatGptWebExecutor extends BaseExecutor {
   }
 }
 
-// Strip ChatGPT's internal entity markup. The browser renders these as proper
-// inline citations / chips via JS; for a plain text completion we just want
-// the human-readable form.
-//   entity["city","Paris","capital of France"]  →  Paris
-//   entity["…","value", …]                       →  value
-const ENTITY_RE = /entity\["[^"]*","([^"]*)"[^\]]*\]/g;
-
-function cleanChatGptText(text: string): string {
-  return text.replace(ENTITY_RE, "$1");
+function commonPrefixLength(a: string, b: string): number {
+  const n = Math.min(a.length, b.length);
+  let i = 0;
+  while (i < n && a.charCodeAt(i) === b.charCodeAt(i)) i++;
+  return i;
 }
 
 function stringToStream(text: string): ReadableStream<Uint8Array> {