From cb63595a07049d9e357f55a48e5604c502a8591d Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 01:59:30 -0400 Subject: [PATCH 01/26] refactor(sse): move executeProviderRequest out of handleChatCore on current tip Lift the nested sender onto a leaf so the 497-line send block no longer lives inside handleChatCore. The leaf keeps tip queue-budget, post-translation system prompt, trusted effort context, and the Claude raw-passthrough execute marker. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 594 ++-------------- .../chatCore/executeProviderRequest.ts | 668 ++++++++++++++++++ .../chatcore/execute-provider-request.test.ts | 99 +++ 3 files changed, 820 insertions(+), 541 deletions(-) create mode 100644 open-sse/handlers/chatCore/executeProviderRequest.ts create mode 100644 tests/unit/chatcore/execute-provider-request.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 397dae6eaa38..6bcb97eabb26 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1,3 +1,4 @@ +import { executeProviderRequest as executeProviderRequestFromLeaf } from "./chatCore/executeProviderRequest.ts"; import { extractRequestToolIdentityMap, resolveResponseToolNameMap, @@ -76,7 +77,6 @@ import { getCombosCached } from "./chatCore/comboContextCache.ts"; export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comboContextCache.ts"; import { resolveAccountSemaphoreKey, - resolveAccountSemaphoreMaxConcurrency, buildClaudePromptCacheLogMeta, } from "./chatCore/executorHelpers.ts"; import { @@ -101,8 +101,6 @@ import { markCodexScopeRateLimited } from "./chatCore/codexFailover.ts"; import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity"; import { buildStreamingResponseHeaders, - materializeDeduplicatedExecutionResult, - stripNextMiddlewareControlHeaders, stripStaleForwardingHeaders, } from "./chatCore/responseHeaders.ts"; import { @@ -129,7 +127,6 @@ import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts" import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; import { injectCustomSystemPrompt, - injectSystemPromptPostTranslation, injectSystemPromptPreTranslation, } from "../services/systemPrompt.ts"; import { translateRequest, needsTranslation } from "../translator/index.ts"; @@ -219,33 +216,24 @@ import { STREAM_READINESS_MAX_TIMEOUT_MS, STREAM_READINESS_TIMEOUT_MS, ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE, - STREAM_RECOVERY, DEFAULT_MAX_TOKENS, STREAM_DISCONNECT_GRACE_PERIOD_MS, } from "../config/constants.ts"; import { applyStatusRestatement } from "../config/upstreamStatusRestatement.ts"; -import { createRecoverableStream, makeContinuationBody } from "../services/streamRecovery.ts"; -import { buildContinuationLogHooks } from "./chatCore/recoveryTraceLogging.ts"; -import { - resolveResilienceSettings, - isStreamRecoveryExplicitlyConfigured, -} from "@/lib/resilience/settings"; +import { resolveResilienceSettings } from "@/lib/resilience/settings"; import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts"; import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; import { wasRefreshTokenRotated } from "@omniroute/open-sse/services/refreshSerializer.ts"; import { connectionHasExtraKeys } from "../services/apiKeyRotator.ts"; import { recordKeyHealthStatus as recordKeyHealthStatusFor } from "./chatCore/keyHealth.ts"; import { getSkillsModelIdForFormat } from "./chatCore/skillsFormat.ts"; -import { readNonStreamingResponseBody } from "./chatCore/nonStreamingResponseBody.ts"; import { createSafeAbortError, createStreamingErrorResult, isSemaphoreCapacityError, - formatStreamRecoveryRetryWarning, getSafeErrorMetadata, getUpstreamErrorIdentifier, } from "./chatCore/streamErrorResult.ts"; -import { wrapReadableStreamWithFinalize } from "./chatCore/streamFinalize.ts"; import { buildCacheUsageLogMeta } from "./chatCore/cacheUsageMeta.ts"; import { buildExecutorClientHeaders } from "./chatCore/executorClientHeaders.ts"; import { getExecutionConnectionId } from "./chatCore/executionCredentials.ts"; @@ -312,11 +300,7 @@ import { recordStreamingUsageStats } from "./chatCore/streamingUsageStats.ts"; import { recordStreamingCost } from "./chatCore/streamingCost.ts"; import { isJsonRecord } from "./chatCore/nonStreamingResponseParse.ts"; import { recordNonStreamingUsageStats } from "./chatCore/nonStreamingUsageStats.ts"; -import { - normalizeExecutorResult, - executeWithUpstreamStartTimeout, - resolveConnectionTimeoutMs, -} from "./chatCore/upstreamTimeouts.ts"; +import { normalizeExecutorResult } from "./chatCore/upstreamTimeouts.ts"; import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; import { assertExclusiveConnectionLeaseFence } from "@/lib/db/exclusiveConnectionLeases"; @@ -345,25 +329,16 @@ import { requiresReasoningReplay, } from "../services/reasoningCache.ts"; import { isCompactResponsesEndpoint } from "../executors/codex.ts"; -import { persistCodexChildQuotaResponse } from "../services/codexAccount/index.ts"; -import { invalidateCodexQuotaCache } from "../services/codexQuotaFetcher.ts"; -import { invalidateGenericQuotaCacheOnStatus } from "../services/genericQuotaFetcher.ts"; import { translateNonStreamingResponse } from "./responseTranslator.ts"; import { extractToolSchemaMap } from "../translator/response/openai-responses/toolSchemas.ts"; import { extractUsageFromResponse } from "./usageExtractor.ts"; import { - withRateLimit, updateFromHeaders, updateFromResponseBody, initializeRateLimits, - resolveRequestQueueMaxWaitMs, } from "../services/rateLimitManager.ts"; import * as localLimiterErrors from "../services/rateLimitManager/errors.ts"; -import { rethrowAdmissionError, remainingQueueBudgetMs } from "./chatCore/queueBudget.ts"; -import { - acquireMany as acquireConcurrencyGates, - markBlocked as markAccountSemaphoreBlocked, -} from "../services/accountSemaphore.ts"; +import { markBlocked as markAccountSemaphoreBlocked } from "../services/accountSemaphore.ts"; import { lockModel, lockModelIfPerModelQuota, @@ -378,7 +353,6 @@ import { findLargerContextModel, getModelFamily, } from "../services/modelFamilyFallback.ts"; -import { computeRequestHash, deduplicate, shouldDeduplicate } from "../services/requestDedup.ts"; import { compressContext, estimateTokens, @@ -404,30 +378,17 @@ import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; import { extractFacts } from "@/lib/memory/extraction"; import { handleToolCallExecution } from "@/lib/skills/interception"; import { MEMORY_BUILTIN_TOOL_NAMES } from "@/lib/skills/memoryBuiltins"; -import { resolveProviderId } from "@/shared/constants/providers"; import { getClaudeCodeCompatibleRequestDefaults } from "@/lib/providers/requestDefaults"; import { buildClaudeCodeCompatibleRequest, resolveClaudeCodeCompatibleSessionId, } from "../services/claudeCodeCompatible.ts"; import { setGeminiThoughtSignatureMode } from "../services/geminiThoughtSignatureStore.ts"; -import { - classifyModelScope429, - getModelScopeRetryDelayMs, - isModelScopeProvider, -} from "../services/modelscopePolicy.ts"; -import { - incrementRequestCount, - incrementTokenUsage, - isTpmExhausted, -} from "../services/geminiRateLimitTracker.ts"; +import { classifyModelScope429, isModelScopeProvider } from "../services/modelscopePolicy.ts"; +import { incrementTokenUsage, isTpmExhausted } from "../services/geminiRateLimitTracker.ts"; +import { computeRequestHash, shouldDeduplicate } from "../services/requestDedup.ts"; import { getProactiveCompressionRatio } from "@/lib/db/compression"; -type ChatCoreExecutorResult = ReturnType & { - _executionCredentials?: Record; - _accountSemaphoreRelease?: () => void; -}; - /** * #12150 P1b: shape of handleChatCore's optional `videoBridgeLog` param — see * its destructure default below. `handleChatCore`'s own params object has no @@ -3060,502 +3021,53 @@ export async function handleChatCore({ ? computeRequestHash(dedupRequestBody, apiKeyInfo?.id, trustedEffortContext) : null; - const executeProviderRequest = async (modelToCall = effectiveModel, allowDedup = false) => { - const execute = async () => { - // Upstream body preparation extracted to chatCore/upstreamBody.ts (#3501 — first internal - // sub-slice of executeProviderRequest); produces the body sent upstream (payload rules + - // tool-limit truncation + prompt_cache_key injection). - let bodyToSend = await prepareUpstreamBody({ - translatedBody, - modelToCall, - ...trustedEffortContext, - provider, - targetFormat, - credentials: getExecutionCredentials(), - log, - bypassDefaultToolLimit: isOpencodeClient, - isOpencodeClient, - rawBody: body, - clientRawRequest, - }); - - // Global System Prompt — SINGLE injection point (post-translation) for - // carrier-ful targets. The old unconditional pre-translation pass - // (former chatCore injectSystemPrompt call) was removed: it chained - // with this pass to inject prefix/suffix 2-3x and dual-wrote - // body.system + messages[] on the claude path, which strict upstreams - // (HCP-Vision vLLM: "System message must be at the beginning") reject - // with 400. Format-aware via targetFormat: messages[] (openai/codex — - // prefix FIRST system, suffix LAST), claude `system` field, gemini - // `systemInstruction`, responses `instructions`. Carrier-less targets - // (kiro user-fold, antigravity Cloud Code envelope) are covered by the - // gated PRE-translation pass before translateRequest instead. - bodyToSend = injectSystemPromptPostTranslation(bodyToSend, { targetFormat }); - - updatePendingScope(pendingScope, { - providerRequest: bodyToSend, - stage: "payload_prepared", - }); - - let releaseRawResultAccountSemaphore = () => {}; - try { - const rawResult: ChatCoreExecutorResult = await (async () => { - let attempts = 0; - const isModelScopeForRequest = isModelScope(); - const maxAttempts = isModelScopeForRequest ? 3 : provider === "codex" ? 3 : 1; - - while (attempts < maxAttempts) { - trace("pre_executor", { attempt: attempts }); - updatePendingScope(pendingScope, { - stage: "sending_to_provider", - }); - const execCreds = getExecutionCredentials(); - const executionConnectionId = getExecutionConnectionId(execCreds); - const attemptConnectionId = executionConnectionId || connectionId; - const accountSemaphoreMaxConcurrency = resolveAccountSemaphoreMaxConcurrency(execCreds); - const accountSemaphoreKey = resolveAccountSemaphoreKey({ - provider, - model: modelToCall, - connectionId: attemptConnectionId, - credentials: execCreds, - }); - const canonicalProviderKey = resolveProviderId(String(provider).trim().toLowerCase()); - const providerConcurrency = - resilienceSettings.providerQuotaOverrides[canonicalProviderKey] - ?.providerConcurrency ?? 0; - - trace("pre_semaphore", { - semaphoreKey: accountSemaphoreKey, - max: accountSemaphoreMaxConcurrency, - }); - if (accountSemaphoreKey && accountSemaphoreMaxConcurrency != null) { - updatePendingScope(pendingScope, { - stage: "waiting_account_slot", - }); - } - const maxWaitMs = resolveRequestQueueMaxWaitMs( - provider, - undefined, - attemptConnectionId ?? undefined - ); - const gateStartedAt = Date.now(); - const releaseAccountSemaphore = await acquireConcurrencyGates( - [ - { - key: "global", - maxConcurrency: resilienceSettings.requestQueue.globalConcurrentRequests, - }, - { - key: `provider:${canonicalProviderKey}`, - maxConcurrency: providerConcurrency, - }, - { - key: accountSemaphoreKey || "", - maxConcurrency: accountSemaphoreKey ? accountSemaphoreMaxConcurrency : null, - }, - ], - { - timeoutMs: maxWaitMs, - maxQueueSize: resilienceSettings.requestQueue.maxQueueDepth, - signal: streamController.signal, - } - ).catch(rethrowAdmissionError); - const remainingAfterGate = remainingQueueBudgetMs(maxWaitMs, gateStartedAt); - trace("post_semaphore", { maxWaitMs, remainingAfterGate }); - updatePendingScope(pendingScope, { - stage: "waiting_rate_limit", - }); - - try { - trace("pre_rate_limit", { connectionId: attemptConnectionId }); - const rawExecutorResult = await withRateLimit( - provider, - attemptConnectionId, - modelToCall, - async () => { - trace("inside_rate_limit", { connectionId: attemptConnectionId }); - updatePendingScope(pendingScope, { - stage: "rate_limit_slot_acquired", - }); - assertManagedLeaseFence(attemptConnectionId); - return executeWithUpstreamStartTimeout({ - executor, - provider, - model: modelToCall, - connectionTimeoutMs: resolveConnectionTimeoutMs( - execCreds?.providerSpecificData - ), - signal: streamController.signal, - log, - execute: (signal) => - runWithCapture(providerRequestCapture, () => - executor.execute({ - model: modelToCall, - body: bodyToSend, - stream: upstreamStream, - credentials: execCreds, - signal, - log, - extendedContext, - upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall), - clientHeaders: buildExecutorClientHeaders( - clientRawRequest?.headers, - userAgent - ), - clientResponseFormat, - onCredentialsRefreshed, - skipUpstreamRetry, - contextEditing: { enabled: contextEditingEnabled }, - correlationId, - }) - ), - }); - }, - streamController.signal, - remainingAfterGate, - correlationId ?? undefined, - { - executor: executor as unknown as { getTimeoutMs?: () => unknown }, - providerSpecificData: execCreds?.providerSpecificData, - } - ); - const res = normalizeExecutorResult(rawExecutorResult); - trace("post_executor", { status: res?.response?.status }); - - if ( - provider === "codex" && - attemptConnectionId && - !(await shouldIsolateProbeFailures()) - ) { - try { - const persistedQuota = await persistCodexChildQuotaResponse({ - connectionId: String(attemptConnectionId), - model: modelToCall || model || requestedModel || "", - headers: normalizeHeaders(res.response.headers), - status: res.response.status, - }); - if (persistedQuota) { - execCreds.providerSpecificData = persistedQuota.providerSpecificData; - if (persistedQuota.exhaustionLog) { - log?.debug?.("CODEX", persistedQuota.exhaustionLog); - } - } - if (res.response.status === 429) { - invalidateCodexQuotaCache(String(attemptConnectionId)); - } - } catch (err) { - const errMessage = err instanceof Error ? err.message : String(err); - log?.debug?.("CODEX", `Failed to persist codex quota state: ${errMessage}`); - } - } else if (attemptConnectionId && res.response.status === 429) { - // Dropped generic quota cache after 429 - invalidateGenericQuotaCacheOnStatus({ - provider, - connectionId: String(attemptConnectionId), - status: res.response.status, - isolateProbe: await shouldIsolateProbeFailures(), - }); - } - - // Track Gemini RPM + RPD request counts for 429 classification - if (provider === "gemini") { - incrementRequestCount(modelToCall); - } - - updatePendingScope(pendingScope, { - stage: "provider_response_started", - }); - - if ( - stream && - (res.response.ok || - res.response.status === HTTP_STATUS.UNAUTHORIZED || - res.response.status === HTTP_STATUS.FORBIDDEN) && - executionConnectionId && - !(await shouldIsolateProbeFailures()) - ) { - const failureDetail = res.response.ok - ? "" - : await res.response - .clone() - .text() - .catch(() => ""); - recordKeyHealthStatus(res.response.status, execCreds, res.transport, failureDetail); - } - - if (isModelScope() && res.response.status === 429 && attempts < maxAttempts - 1) { - const bodyPeek = await res.response - .clone() - .text() - .catch(() => ""); - const normalizedHeaders = normalizeHeaders(res.response.headers); - const decision = classifyModelScope429(bodyPeek, normalizedHeaders); - if (decision.retryable) { - const delay = getModelScopeRetryDelayMs(normalizedHeaders, attempts); - log?.warn?.( - "MODELSCOPE_RETRY", - `429 ${decision.kind}; retrying in ${delay}ms (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"})` - ); - releaseAccountSemaphore(); - await new Promise((r) => setTimeout(r, delay)); - attempts++; - continue; - } - } - - // For streaming: release the semaphore when the client drains or cancels the stream. - // Non-2xx streams must drop the slot before returning so the pipeline can rotate - // accounts without holding the failed connection's concurrency gate. Do NOT - // cancel() the body here — the pipeline clones it (BYOP 422 / toOutcome). - if (stream) { - const originalBody = res.response.body; - const okStatus = res.response.status >= 200 && res.response.status < 300; - if (!originalBody || !okStatus) { - releaseAccountSemaphore(); - return { - ...res, - _executionCredentials: execCreds, - }; - } - - // Opt-in transparent stream recovery (free-claude-code port, default OFF). - // Only engages for a successful (2xx) stream — an error body must never be - // held or replayed. Setting is read once here from the cached resolved - // resilience settings; the default path is byte-for-byte unchanged. - let streamRecoveryEnabled = false; - let continueMidStreamEnabled = false; - let throughputWatchdog = - resolveResilienceSettings(null).streamRecovery.throughputWatchdog; - if (okStatus) { - try { - // Reuse the request-consolidated settings read (see line ~2076) — no - // second DB/cache hit. Default OFF when the setting is absent. - const sr = resolveResilienceSettings(settings).streamRecovery; - // Fail-closed: the agent-goal-policy heuristic may only ADD recovery - // when the operator has no explicit configuration. If the operator - // explicitly configured stream recovery (env var or DB/settings - // override), that value always wins — the goal policy must never - // re-enable recovery the operator explicitly turned off. - const operatorExplicit = isStreamRecoveryExplicitlyConfigured(settings); - const goalOverride = !operatorExplicit && agentGoalPolicy.streamRecoveryEnabled; - streamRecoveryEnabled = sr.enabled || goalOverride; - continueMidStreamEnabled = sr.continueMidStream === true; - throughputWatchdog = sr.throughputWatchdog; - if (goalOverride && !sr.enabled) { - log?.info?.( - "AGENT_GOAL", - `agentGoalPolicy override: stream recovery enabled for goal request requestId=${traceId} model=${modelToCall || model || requestedModel || "unknown"}` - ); - } - } catch { - streamRecoveryEnabled = false; - continueMidStreamEnabled = false; - throughputWatchdog = - resolveResilienceSettings(null).streamRecovery.throughputWatchdog; - } - } - - let clientBody: ReadableStream; - if (streamRecoveryEnabled || throughputWatchdog.enabled) { - // Run the SAME upstream (same account/creds) with a given body and return - // its 2xx stream, or null. Used both by the early-retry re-open (same body) - // and the mid-stream continuation (assistant-prefilled body). - const runUpstreamStream = async ( - body: unknown - ): Promise | null> => { - try { - assertManagedLeaseFence(attemptConnectionId); - const retryRaw = await executeWithUpstreamStartTimeout({ - executor, - provider, - model: modelToCall, - connectionTimeoutMs: resolveConnectionTimeoutMs( - execCreds?.providerSpecificData - ), - signal: streamController.signal, - log, - execute: (signal) => - runWithCapture(providerRequestCapture, () => - executor.execute({ - model: modelToCall, - body, - stream: upstreamStream, - credentials: execCreds, - signal, - log, - extendedContext, - upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall), - clientHeaders: buildExecutorClientHeaders( - clientRawRequest?.headers, - userAgent - ), - clientResponseFormat, - onCredentialsRefreshed, - skipUpstreamRetry, - contextEditing: { enabled: contextEditingEnabled }, - correlationId, - }) - ), - }); - const retryRes = normalizeExecutorResult(retryRaw); - const retryOk = - retryRes.response.status >= 200 && retryRes.response.status < 300; - if (retryOk && retryRes.response.body) { - return retryRes.response.body as ReadableStream; - } - await retryRes.response.body?.cancel().catch(() => {}); - return null; - } catch { - return null; - } - }; - - // Mid-stream continuation (Fase 4.4): re-request with the partial text as an - // assistant prefill. Gated by its own setting and only for OpenAI-compatible - // bodies (makeContinuationBody returns null otherwise). - const continueStream = continueMidStreamEnabled - ? (assistantSoFar: string) => { - const continuationBody = makeContinuationBody( - bodyToSend as Record, - assistantSoFar - ); - return continuationBody - ? runUpstreamStream(continuationBody) - : Promise.resolve(null); - } - : undefined; - - clientBody = createRecoverableStream( - originalBody as ReadableStream, - () => runUpstreamStream(bodyToSend), - { - finalize: releaseAccountSemaphore, - onRetry: (attempt, err) => - log?.warn?.( - "STREAM_RECOVERY", - formatStreamRecoveryRetryWarning( - attempt, - STREAM_RECOVERY.EARLY_RETRY_MAX, - err - ) - ), - continueStream, - ...buildContinuationLogHooks(log), - throughputWatchdog, - onWatchdogAbort: () => - log?.warn?.( - "STREAM_WATCHDOG", - "active upstream stream stayed below the configured useful-output rate" - ), - } - ); - } else { - clientBody = wrapReadableStreamWithFinalize( - originalBody, - releaseAccountSemaphore - ); - } - - return { - ...res, - _executionCredentials: execCreds, - response: new Response(clientBody, { - status: res.response.status, - statusText: res.response.statusText, - headers: new Headers(normalizeHeaders(res.response.headers)), - }), - }; - } - - return { - ...res, - _executionCredentials: execCreds, - _accountSemaphoreRelease: releaseAccountSemaphore, - }; - } catch (error) { - releaseAccountSemaphore(); - throw error; - } - } - })(); - - if (stream) { - return rawResult; - } - - // Non-stream: release semaphore immediately after reading full response body. - const status = rawResult.response.status; - - releaseRawResultAccountSemaphore = - typeof rawResult._accountSemaphoreRelease === "function" - ? rawResult._accountSemaphoreRelease - : () => {}; - - const statusText = rawResult.response.statusText; - const headersObj = normalizeHeaders(rawResult.response.headers); - const responseHeaders = new Headers(headersObj); - stripStaleForwardingHeaders(responseHeaders); - stripNextMiddlewareControlHeaders(responseHeaders); - // The upstream headers (turn-state included) are about to be committed - // to the client — record which connection minted the blob so a later - // cross-account echo can be stripped (Codex failover guard). - if (provider === "codex" && readCodexTurnStateHeader(responseHeaders)) { - noteCodexTurnStateProvenance( - getCodexClientSessionId(clientRawRequest?.headers), - rawResult._executionCredentials?.connectionId ?? credentials?.connectionId - ); - } - const contentType = (responseHeaders.get("content-type") || "").toLowerCase(); - const payload = await readNonStreamingResponseBody( - rawResult.response, - contentType, - upstreamStream - ); - // Use the exact execution credential selected for this request. Model capability - // failures stay in routing telemetry; authoritative success only recovers this key. - if ( - rawResult._executionCredentials?.connectionId && - (rawResult._executionCredentials.apiKey || rawResult._executionCredentials.accessToken) - ) { - recordKeyHealthStatus( - status, - rawResult._executionCredentials, - rawResult.transport, - status >= 400 ? payload : "" - ); - } - releaseRawResultAccountSemaphore(); - releaseRawResultAccountSemaphore = () => {}; - - return { - ...rawResult, - response: new Response(payload, { status, statusText, headers: responseHeaders }), - _dedupSnapshot: { - status, - statusText, - headers: (() => { - const arr: [string, string][] = []; - responseHeaders.forEach((v, k) => arr.push([k, v])); - return arr; - })(), - payload, - }, - }; - } catch (error) { - releaseRawResultAccountSemaphore(); - throw error; - } + const executeProviderRequest = (modelToCall = effectiveModel, allowDedup = false) => { + const sendDeps = { + agentGoalPolicy, + assertManagedLeaseFence, + buildUpstreamHeadersForExecute, + clientRawRequest, + clientResponseFormat, + connectionId, + correlationId, + contextEditingEnabled, + credentials, + dedupEnabled, + dedupHash, + effectiveModel, + executor, + extendedContext, + getExecutionCredentials, + isClaudePassthrough, + isModelScope, + isOpencodeClient, + log, + model, + onCredentialsRefreshed, + pendingScope, + provider, + providerRequestCapture, + rawBody: body, + recordKeyHealthStatus, + requestedModel, + resilienceSettings, + settings, + skipUpstreamRetry, + stream, + streamController, + targetFormat, + trace, + traceId, + translatedBody, + trustedEffortContext, + upstreamStream, + userAgent, }; - - if (allowDedup && dedupEnabled && dedupHash) { - const dedupResult = await deduplicate(dedupHash, execute); - if (dedupResult.wasDeduplicated) { - log?.debug?.("DEDUP", `Joined in-flight request hash=${dedupHash}`); - } - return materializeDeduplicatedExecutionResult(dedupResult.result); - } - - return execute(); + return executeProviderRequestFromLeaf( + sendDeps as unknown as import("./chatCore/executeProviderRequest.ts").ExecuteProviderRequestDeps, + modelToCall, + allowDedup + ); }; const registeredProviderRequest = diff --git a/open-sse/handlers/chatCore/executeProviderRequest.ts b/open-sse/handlers/chatCore/executeProviderRequest.ts new file mode 100644 index 000000000000..b7b064057aac --- /dev/null +++ b/open-sse/handlers/chatCore/executeProviderRequest.ts @@ -0,0 +1,668 @@ +/** + * Wire send for one provider attempt. Lifted from handleChatCore so the + * orchestrator no longer nests the sender. + */ + +import type { ChatCoreExecutorResult as PipelineChatCoreExecutorResult } from "./providerExecutionPipeline.ts"; +import type { AgentGoalPolicy } from "../../utils/agentGoalPolicy.ts"; +import type { BaseExecutor } from "../../executors/base.ts"; +import { prepareUpstreamBody } from "./upstreamBody.ts"; +import { getExecutionConnectionId } from "./executionCredentials.ts"; +import { + resolveAccountSemaphoreKey, + resolveAccountSemaphoreMaxConcurrency, +} from "./executorHelpers.ts"; +import { + materializeDeduplicatedExecutionResult, + stripNextMiddlewareControlHeaders, + stripStaleForwardingHeaders, +} from "./responseHeaders.ts"; +import { readNonStreamingResponseBody } from "./nonStreamingResponseBody.ts"; +import { wrapReadableStreamWithFinalize } from "./streamFinalize.ts"; +import { buildExecutorClientHeaders } from "./executorClientHeaders.ts"; +import { + normalizeExecutorResult, + executeWithUpstreamStartTimeout, + resolveConnectionTimeoutMs, +} from "./upstreamTimeouts.ts"; +import { getCodexClientSessionId } from "../../config/codexIdentity.ts"; +import { + noteCodexTurnStateProvenance, + readCodexTurnStateHeader, +} from "../../config/codexTurnState.ts"; +import { HTTP_STATUS, STREAM_RECOVERY } from "../../config/constants.ts"; +import { createRecoverableStream, makeContinuationBody } from "../../services/streamRecovery.ts"; +import { persistCodexChildQuotaResponse } from "../../services/codexAccount/index.ts"; +import { invalidateCodexQuotaCache } from "../../services/codexQuotaFetcher.ts"; +import { invalidateGenericQuotaCacheOnStatus } from "../../services/genericQuotaFetcher.ts"; +import { withRateLimit, resolveRequestQueueMaxWaitMs } from "../../services/rateLimitManager.ts"; +import { acquireMany as acquireConcurrencyGates } from "../../services/accountSemaphore.ts"; +import { rethrowAdmissionError, remainingQueueBudgetMs } from "./queueBudget.ts"; +import { deduplicate } from "../../services/requestDedup.ts"; +import { + classifyModelScope429, + getModelScopeRetryDelayMs, +} from "../../services/modelscopePolicy.ts"; +import { incrementRequestCount } from "../../services/geminiRateLimitTracker.ts"; +import { normalizeHeaders } from "../../utils/headers.ts"; +import { runWithCapture } from "../../utils/providerRequestLogging.ts"; +import type { Capture } from "../../utils/providerRequestLogging.ts"; +import { + resolveResilienceSettings, + isStreamRecoveryExplicitlyConfigured, + type ResilienceSettings, +} from "@/lib/resilience/settings"; +import { updatePendingScope, type PendingRequestScope } from "@/lib/usage/pendingRequestScope"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; +import { resolveProviderId } from "@/shared/constants/providers"; +import { buildContinuationLogHooks } from "./recoveryTraceLogging.ts"; +import { injectSystemPromptPostTranslation } from "../../services/systemPrompt.ts"; + +export type ChatCoreExecutorResult = PipelineChatCoreExecutorResult & { + _dedupSnapshot?: { + status: number; + statusText: string; + headers: [string, string][]; + payload: string; + }; +}; + +export type TrustedEffortContext = { + originModel?: string; + resolvedThinkingEffort?: string | null; + defaultThinkingEffort?: string | null; +}; + +export type ExecuteProviderRequestDeps = { + agentGoalPolicy: AgentGoalPolicy; + assertManagedLeaseFence: (attemptConnectionId: string | null | undefined) => void; + buildUpstreamHeadersForExecute: (modelToCall: string) => Record; + clientRawRequest: { + headers?: Headers | Record | null; + signal?: AbortSignal | null | undefined; + }; + clientResponseFormat: string | null | undefined; + connectionId: string | null | undefined; + correlationId: string | null | undefined; + contextEditingEnabled?: boolean; + credentials: Record | null | undefined; + dedupEnabled: boolean; + dedupHash: string | null; + effectiveModel: string; + executor: Pick; + extendedContext?: boolean; + getExecutionCredentials: () => Record; + isClaudePassthrough?: boolean; + isModelScope: boolean; + isOpencodeClient: boolean; + log: + | { + debug?: (...args: unknown[]) => void; + info?: (...args: unknown[]) => void; + warn?: (...args: unknown[]) => void; + error?: (...args: unknown[]) => void; + } + | null + | undefined; + model: string; + onCredentialsRefreshed: + ((next: Record) => void | Promise) | null | undefined; + pendingScope: PendingRequestScope; + provider: string; + providerRequestCapture: Capture; + rawBody: Record; + recordKeyHealthStatus: (...args: unknown[]) => void; + requestedModel: string; + resilienceSettings: ResilienceSettings; + settings: Record | null | undefined; + skipUpstreamRetry: boolean; + stream: boolean; + streamController: { signal: AbortSignal }; + targetFormat: string; + trace: (label: string, extra?: Record) => void; + traceId: string; + translatedBody: Record; + trustedEffortContext: TrustedEffortContext; + upstreamStream: boolean; + userAgent: string | undefined; +}; + +export async function executeProviderRequest( + deps: ExecuteProviderRequestDeps, + modelToCall: string = deps.effectiveModel, + allowDedup: boolean = false +): Promise { + const { + agentGoalPolicy, + assertManagedLeaseFence, + buildUpstreamHeadersForExecute, + clientRawRequest, + clientResponseFormat, + connectionId, + correlationId, + contextEditingEnabled, + credentials, + dedupEnabled, + dedupHash, + executor, + extendedContext, + getExecutionCredentials, + isClaudePassthrough, + isModelScope, + isOpencodeClient, + log, + model, + onCredentialsRefreshed, + pendingScope, + provider, + providerRequestCapture, + rawBody, + recordKeyHealthStatus, + requestedModel, + resilienceSettings, + settings, + skipUpstreamRetry, + stream, + streamController, + targetFormat, + trace, + traceId, + translatedBody, + trustedEffortContext, + upstreamStream, + userAgent, + } = deps; + const body = rawBody; + const execute = async () => { + // Upstream body preparation extracted to chatCore/upstreamBody.ts (#3501 — first internal + // sub-slice of executeProviderRequest); produces the body sent upstream (payload rules + + // tool-limit truncation + prompt_cache_key injection). + let bodyToSend = await prepareUpstreamBody({ + translatedBody, + modelToCall, + ...trustedEffortContext, + provider, + targetFormat, + credentials: getExecutionCredentials(), + log, + bypassDefaultToolLimit: isOpencodeClient, + isOpencodeClient, + rawBody: body, + clientRawRequest, + }); + + // Global System Prompt — SINGLE injection point (post-translation) for + // carrier-ful targets. The old unconditional pre-translation pass + // (former chatCore injectSystemPrompt call) was removed: it chained + // with this pass to inject prefix/suffix 2-3x and dual-wrote + // body.system + messages[] on the claude path, which strict upstreams + // (HCP-Vision vLLM: "System message must be at the beginning") reject + // with 400. Format-aware via targetFormat: messages[] (openai/codex — + // prefix FIRST system, suffix LAST), claude `system` field, gemini + // `systemInstruction`, responses `instructions`. Carrier-less targets + // (kiro user-fold, antigravity Cloud Code envelope) are covered by the + // gated PRE-translation pass before translateRequest instead. + bodyToSend = injectSystemPromptPostTranslation(bodyToSend, { targetFormat }); + + updatePendingScope(pendingScope, { + providerRequest: bodyToSend, + stage: "payload_prepared", + }); + + let releaseRawResultAccountSemaphore = () => {}; + try { + const rawResult: ChatCoreExecutorResult = await (async () => { + let attempts = 0; + const isModelScopeForRequest = isModelScope(); + const maxAttempts = isModelScopeForRequest ? 3 : provider === "codex" ? 3 : 1; + + while (attempts < maxAttempts) { + trace("pre_executor", { attempt: attempts }); + updatePendingScope(pendingScope, { + stage: "sending_to_provider", + }); + const execCreds = getExecutionCredentials(); + const executionConnectionId = getExecutionConnectionId(execCreds); + const attemptConnectionId = executionConnectionId || connectionId; + const accountSemaphoreMaxConcurrency = resolveAccountSemaphoreMaxConcurrency(execCreds); + const accountSemaphoreKey = resolveAccountSemaphoreKey({ + provider, + model: modelToCall, + connectionId: attemptConnectionId, + credentials: execCreds, + }); + const canonicalProviderKey = resolveProviderId(String(provider).trim().toLowerCase()); + const providerConcurrency = + resilienceSettings.providerQuotaOverrides[canonicalProviderKey]?.providerConcurrency ?? + 0; + + trace("pre_semaphore", { + semaphoreKey: accountSemaphoreKey, + max: accountSemaphoreMaxConcurrency, + }); + if (accountSemaphoreKey && accountSemaphoreMaxConcurrency != null) { + updatePendingScope(pendingScope, { + stage: "waiting_account_slot", + }); + } + const maxWaitMs = resolveRequestQueueMaxWaitMs( + provider, + undefined, + attemptConnectionId ?? undefined + ); + const gateStartedAt = Date.now(); + const releaseAccountSemaphore = await acquireConcurrencyGates( + [ + { + key: "global", + maxConcurrency: resilienceSettings.requestQueue.globalConcurrentRequests, + }, + { + key: `provider:${canonicalProviderKey}`, + maxConcurrency: providerConcurrency, + }, + { + key: accountSemaphoreKey || "", + maxConcurrency: accountSemaphoreKey ? accountSemaphoreMaxConcurrency : null, + }, + ], + { + timeoutMs: maxWaitMs, + maxQueueSize: resilienceSettings.requestQueue.maxQueueDepth, + signal: streamController.signal, + } + ).catch(rethrowAdmissionError); + const remainingAfterGate = remainingQueueBudgetMs(maxWaitMs, gateStartedAt); + trace("post_semaphore", { maxWaitMs, remainingAfterGate }); + updatePendingScope(pendingScope, { + stage: "waiting_rate_limit", + }); + + try { + trace("pre_rate_limit", { connectionId: attemptConnectionId }); + const rawExecutorResult = await withRateLimit( + provider, + attemptConnectionId, + modelToCall, + async () => { + trace("inside_rate_limit", { connectionId: attemptConnectionId }); + updatePendingScope(pendingScope, { + stage: "rate_limit_slot_acquired", + }); + assertManagedLeaseFence(attemptConnectionId); + return executeWithUpstreamStartTimeout({ + executor, + provider, + model: modelToCall, + connectionTimeoutMs: resolveConnectionTimeoutMs(execCreds?.providerSpecificData), + signal: streamController.signal, + log, + execute: (signal) => + runWithCapture(providerRequestCapture, () => + executor.execute({ + model: modelToCall, + body: bodyToSend, + stream: upstreamStream, + credentials: execCreds, + signal, + log, + extendedContext, + upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall), + clientHeaders: buildExecutorClientHeaders( + clientRawRequest?.headers, + userAgent + ), + clientResponseFormat, + onCredentialsRefreshed, + skipUpstreamRetry, + contextEditing: { enabled: contextEditingEnabled }, + isClaudePassthrough, + correlationId, + }) + ), + }); + }, + streamController.signal, + remainingAfterGate, + correlationId ?? undefined, + { + executor: executor as unknown as { getTimeoutMs?: () => unknown }, + providerSpecificData: execCreds?.providerSpecificData, + } + ); + const res = normalizeExecutorResult(rawExecutorResult); + trace("post_executor", { status: res?.response?.status }); + + if ( + provider === "codex" && + attemptConnectionId && + !(await shouldIsolateProbeFailures()) + ) { + try { + const persistedQuota = await persistCodexChildQuotaResponse({ + connectionId: String(attemptConnectionId), + model: modelToCall || model || requestedModel || "", + headers: normalizeHeaders(res.response.headers), + status: res.response.status, + }); + if (persistedQuota) { + execCreds.providerSpecificData = persistedQuota.providerSpecificData; + if (persistedQuota.exhaustionLog) { + log?.debug?.("CODEX", persistedQuota.exhaustionLog); + } + } + if (res.response.status === 429) { + invalidateCodexQuotaCache(String(attemptConnectionId)); + } + } catch (err) { + const errMessage = err instanceof Error ? err.message : String(err); + log?.debug?.("CODEX", `Failed to persist codex quota state: ${errMessage}`); + } + } else if (attemptConnectionId && res.response.status === 429) { + // Dropped generic quota cache after 429 + invalidateGenericQuotaCacheOnStatus({ + provider, + connectionId: String(attemptConnectionId), + status: res.response.status, + isolateProbe: await shouldIsolateProbeFailures(), + }); + } + + // Track Gemini RPM + RPD request counts for 429 classification + if (provider === "gemini") { + incrementRequestCount(modelToCall); + } + + updatePendingScope(pendingScope, { + stage: "provider_response_started", + }); + + if ( + stream && + (res.response.ok || + res.response.status === HTTP_STATUS.UNAUTHORIZED || + res.response.status === HTTP_STATUS.FORBIDDEN) && + executionConnectionId && + !(await shouldIsolateProbeFailures()) + ) { + const failureDetail = res.response.ok + ? "" + : await res.response + .clone() + .text() + .catch(() => ""); + recordKeyHealthStatus(res.response.status, execCreds, res.transport, failureDetail); + } + + if (isModelScope() && res.response.status === 429 && attempts < maxAttempts - 1) { + const bodyPeek = await res.response + .clone() + .text() + .catch(() => ""); + const normalizedHeaders = normalizeHeaders(res.response.headers); + const decision = classifyModelScope429(bodyPeek, normalizedHeaders); + if (decision.retryable) { + const delay = getModelScopeRetryDelayMs(normalizedHeaders, attempts); + log?.warn?.( + "MODELSCOPE_RETRY", + `429 ${decision.kind}; retrying in ${delay}ms (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"})` + ); + releaseAccountSemaphore(); + await new Promise((r) => setTimeout(r, delay)); + attempts++; + continue; + } + } + + // For streaming: release the semaphore when the client drains or cancels the stream. + // Non-2xx streams must drop the slot before returning so the pipeline can rotate + // accounts without holding the failed connection's concurrency gate. Do NOT + // cancel() the body here — the pipeline clones it (BYOP 422 / toOutcome). + if (stream) { + const originalBody = res.response.body; + const okStatus = res.response.status >= 200 && res.response.status < 300; + if (!originalBody || !okStatus) { + releaseAccountSemaphore(); + return { + ...res, + _executionCredentials: execCreds, + }; + } + + // Opt-in transparent stream recovery (free-claude-code port, default OFF). + // Only engages for a successful (2xx) stream — an error body must never be + // held or replayed. Setting is read once here from the cached resolved + // resilience settings; the default path is byte-for-byte unchanged. + let streamRecoveryEnabled = false; + let continueMidStreamEnabled = false; + let throughputWatchdog = + resolveResilienceSettings(null).streamRecovery.throughputWatchdog; + if (okStatus) { + try { + // Reuse the request-consolidated settings read (see line ~2076) — no + // second DB/cache hit. Default OFF when the setting is absent. + const sr = resolveResilienceSettings(settings).streamRecovery; + // Fail-closed: the agent-goal-policy heuristic may only ADD recovery + // when the operator has no explicit configuration. If the operator + // explicitly configured stream recovery (env var or DB/settings + // override), that value always wins — the goal policy must never + // re-enable recovery the operator explicitly turned off. + const operatorExplicit = isStreamRecoveryExplicitlyConfigured(settings); + const goalOverride = !operatorExplicit && agentGoalPolicy.streamRecoveryEnabled; + streamRecoveryEnabled = sr.enabled || goalOverride; + continueMidStreamEnabled = sr.continueMidStream === true; + throughputWatchdog = sr.throughputWatchdog; + if (goalOverride && !sr.enabled) { + log?.info?.( + "AGENT_GOAL", + `agentGoalPolicy override: stream recovery enabled for goal request requestId=${traceId} model=${modelToCall || model || requestedModel || "unknown"}` + ); + } + } catch { + streamRecoveryEnabled = false; + continueMidStreamEnabled = false; + throughputWatchdog = + resolveResilienceSettings(null).streamRecovery.throughputWatchdog; + } + } + + let clientBody: ReadableStream; + if (streamRecoveryEnabled || throughputWatchdog.enabled) { + // Run the SAME upstream (same account/creds) with a given body and return + // its 2xx stream, or null. Used both by the early-retry re-open (same body) + // and the mid-stream continuation (assistant-prefilled body). + const runUpstreamStream = async ( + body: unknown + ): Promise | null> => { + try { + assertManagedLeaseFence(attemptConnectionId); + const retryRaw = await executeWithUpstreamStartTimeout({ + executor, + provider, + model: modelToCall, + connectionTimeoutMs: resolveConnectionTimeoutMs( + execCreds?.providerSpecificData + ), + signal: streamController.signal, + log, + execute: (signal) => + runWithCapture(providerRequestCapture, () => + executor.execute({ + model: modelToCall, + body, + stream: upstreamStream, + credentials: execCreds, + signal, + log, + extendedContext, + upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall), + clientHeaders: buildExecutorClientHeaders( + clientRawRequest?.headers, + userAgent + ), + clientResponseFormat, + onCredentialsRefreshed, + skipUpstreamRetry, + contextEditing: { enabled: contextEditingEnabled }, + isClaudePassthrough, + correlationId, + }) + ), + }); + const retryRes = normalizeExecutorResult(retryRaw); + const retryOk = + retryRes.response.status >= 200 && retryRes.response.status < 300; + if (retryOk && retryRes.response.body) { + return retryRes.response.body as ReadableStream; + } + await retryRes.response.body?.cancel().catch(() => {}); + return null; + } catch { + return null; + } + }; + + // Mid-stream continuation (Fase 4.4): re-request with the partial text as an + // assistant prefill. Gated by its own setting and only for OpenAI-compatible + // bodies (makeContinuationBody returns null otherwise). + const continueStream = continueMidStreamEnabled + ? (assistantSoFar: string) => { + const continuationBody = makeContinuationBody( + bodyToSend as Record, + assistantSoFar + ); + return continuationBody + ? runUpstreamStream(continuationBody) + : Promise.resolve(null); + } + : undefined; + + clientBody = createRecoverableStream( + originalBody as ReadableStream, + () => runUpstreamStream(bodyToSend), + { + finalize: releaseAccountSemaphore, + onRetry: (attempt, err) => + log?.warn?.( + "STREAM_RECOVERY", + formatStreamRecoveryRetryWarning( + attempt, + STREAM_RECOVERY.EARLY_RETRY_MAX, + err + ) + ), + continueStream, + ...buildContinuationLogHooks(log), + throughputWatchdog, + onWatchdogAbort: () => + log?.warn?.( + "STREAM_WATCHDOG", + "active upstream stream stayed below the configured useful-output rate" + ), + } + ); + } else { + clientBody = wrapReadableStreamWithFinalize(originalBody, releaseAccountSemaphore); + } + + return { + ...res, + _executionCredentials: execCreds, + response: new Response(clientBody, { + status: res.response.status, + statusText: res.response.statusText, + headers: new Headers(normalizeHeaders(res.response.headers)), + }), + }; + } + + return { + ...res, + _executionCredentials: execCreds, + _accountSemaphoreRelease: releaseAccountSemaphore, + }; + } catch (error) { + releaseAccountSemaphore(); + throw error; + } + } + })(); + + if (stream) { + return rawResult; + } + + // Non-stream: release semaphore immediately after reading full response body. + const status = rawResult.response.status; + + releaseRawResultAccountSemaphore = + typeof rawResult._accountSemaphoreRelease === "function" + ? rawResult._accountSemaphoreRelease + : () => {}; + + const statusText = rawResult.response.statusText; + const headersObj = normalizeHeaders(rawResult.response.headers); + const responseHeaders = new Headers(headersObj); + stripStaleForwardingHeaders(responseHeaders); + stripNextMiddlewareControlHeaders(responseHeaders); + // The upstream headers (turn-state included) are about to be committed + // to the client — record which connection minted the blob so a later + // cross-account echo can be stripped (Codex failover guard). + if (provider === "codex" && readCodexTurnStateHeader(responseHeaders)) { + noteCodexTurnStateProvenance( + getCodexClientSessionId(clientRawRequest?.headers), + rawResult._executionCredentials?.connectionId ?? credentials?.connectionId + ); + } + const contentType = (responseHeaders.get("content-type") || "").toLowerCase(); + const payload = await readNonStreamingResponseBody( + rawResult.response, + contentType, + upstreamStream + ); + // Use the exact execution credential selected for this request. Model capability + // failures stay in routing telemetry; authoritative success only recovers this key. + if ( + rawResult._executionCredentials?.connectionId && + (rawResult._executionCredentials.apiKey || rawResult._executionCredentials.accessToken) + ) { + recordKeyHealthStatus( + status, + rawResult._executionCredentials, + rawResult.transport, + status >= 400 ? payload : "" + ); + } + releaseRawResultAccountSemaphore(); + releaseRawResultAccountSemaphore = () => {}; + + return { + ...rawResult, + response: new Response(payload, { status, statusText, headers: responseHeaders }), + _dedupSnapshot: { + status, + statusText, + headers: (() => { + const arr: [string, string][] = []; + responseHeaders.forEach((v, k) => arr.push([k, v])); + return arr; + })(), + payload, + }, + }; + } catch (error) { + releaseRawResultAccountSemaphore(); + throw error; + } + }; + + if (allowDedup && dedupEnabled && dedupHash) { + const dedupResult = await deduplicate(dedupHash, execute); + if (dedupResult.wasDeduplicated) { + log?.debug?.("DEDUP", `Joined in-flight request hash=${dedupHash}`); + } + return materializeDeduplicatedExecutionResult(dedupResult.result); + } + + return execute(); +} diff --git a/tests/unit/chatcore/execute-provider-request.test.ts b/tests/unit/chatcore/execute-provider-request.test.ts new file mode 100644 index 000000000000..e9a292f132d5 --- /dev/null +++ b/tests/unit/chatcore/execute-provider-request.test.ts @@ -0,0 +1,99 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); +const BARREL = path.join(ROOT, "open-sse/handlers/chatCore.ts"); +const LEAF = path.join(ROOT, "open-sse/handlers/chatCore/executeProviderRequest.ts"); + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-epr-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleChatCore } = await import("../../../open-sse/handlers/chatCore.ts"); + +function noopLog() { + return { debug() {}, info() {}, warn() {}, error() {} }; +} + +test("leaf executeProviderRequest.ts exists", () => { + assert.equal(fs.existsSync(LEAF), true, "executeProviderRequest.ts must exist"); +}); + +test("barrel no longer defines nested executeProviderRequest", () => { + const src = fs.readFileSync(BARREL, "utf8"); + assert.equal( + /const executeProviderRequest = async/.test(src), + false, + "nested const must move out of chatCore.ts" + ); +}); + +test("leaf send pins model via prepareUpstreamBody", async () => { + const originalFetch = globalThis.fetch; + const captured = []; + globalThis.fetch = async (_url, init = {}) => { + captured.push(init.body ? JSON.parse(String(init.body)) : null); + return new Response( + JSON.stringify({ + id: "chatcmpl-epr", + object: "chat.completion", + model: "gpt-4o-mini", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ); + }; + + const body = { + model: "gpt-4o-mini", + stream: false, + messages: [{ role: "user", content: "hello" }], + }; + + try { + const result = await handleChatCore({ + body: structuredClone(body), + modelInfo: { provider: "openai", model: "gpt-4o-mini", extendedContext: false }, + credentials: { apiKey: "sk-test", providerSpecificData: {} }, + log: noopLog(), + clientRawRequest: { + endpoint: "/v1/chat/completions", + body: structuredClone(body), + headers: new Headers({ accept: "application/json" }), + }, + userAgent: "unit-test", + }); + assert.equal(result.success, true, "non-stream request must succeed"); + assert.ok(captured.length >= 1, "upstream fetch must run"); + assert.equal(captured[0].model, "gpt-4o-mini", "prepareUpstreamBody pins the model"); + } finally { + globalThis.fetch = originalFetch; + } + + const leafSrc = fs.readFileSync(LEAF, "utf8"); + assert.equal( + leafSrc.includes("prepareUpstreamBody("), + true, + "leaf must call prepareUpstreamBody" + ); + const destructure = leafSrc.slice(leafSrc.indexOf("const {"), leafSrc.indexOf("} = deps;")); + assert.equal( + /(?:^|\n)\s*effectiveModel,/.test(destructure) || destructure.includes("\n effectiveModel,"), + false, + "effectiveModel is only the default for modelToCall, not a local" + ); +}); + +test.after(() => { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); From 603e18d442a2ab5032423563a3442785a9ccc4ed Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 07:42:41 -0400 Subject: [PATCH 02/26] refactor(sse): drive chatCore recovery from a decision table on current tip Lift the 404/403/429 decision table out of handleChatCore. Stream throws still terminate after getSafeErrorMetadata. Family fallback now reads onFailure instead of inlining isModelUnavailableError. The pipeline kept a string snapshot of the current model. Barrel setBodyAndModel only updates outer locals, so a 404 retried the same model. Write the sibling model back onto the pipeline wire after the callback. Probe isolation at the onFailure callsite now reads shouldIsolateProbeFailures instead of a hardcoded false. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 20 +- .../chatCore/providerExecutionPipeline.ts | 8 +- open-sse/handlers/chatCore/recoveryPolicy.ts | 116 +++++++++ tests/unit/chatcore/recovery-callsite.test.ts | 223 ++++++++++++++++++ tests/unit/chatcore/recovery-policy.test.ts | 132 +++++++++++ .../unit/provider-execution-pipeline.test.ts | 38 +++ 6 files changed, 533 insertions(+), 4 deletions(-) create mode 100644 open-sse/handlers/chatCore/recoveryPolicy.ts create mode 100644 tests/unit/chatcore/recovery-callsite.test.ts create mode 100644 tests/unit/chatcore/recovery-policy.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 6bcb97eabb26..f853ce0c6987 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -89,6 +89,7 @@ import { } from "./chatCore/passthroughHelpers.ts"; import { recoverAnthropicThinkingSignature } from "./chatCore/thinkingSignatureRecovery.ts"; import { runProviderExecutionPipeline } from "./chatCore/providerExecutionPipeline.ts"; +import { onFailure, onStreamThrow } from "./chatCore/recoveryPolicy.ts"; import { runNonStreamingProviderLeg } from "./chatCore/nonStreamingProviderLeg.ts"; import type { NonStreamingProviderLegResult } from "@/lib/skills/toolLoopTypes.ts"; import { @@ -347,7 +348,6 @@ import { } from "../services/accountFallback.ts"; import { saveIdempotency } from "@/lib/idempotencyLayer"; import { - isModelUnavailableError, getNextFamilyFallback, isContextOverflowError, findLargerContextModel, @@ -3734,6 +3734,7 @@ export async function handleChatCore({ // fail-open: saturation signal is best-effort } } catch (error) { + onStreamThrow(); trackPendingRequest(model, provider, connectionId, false); const errorMetadata = getSafeErrorMetadata(error); const managedLeaseFenceCode = getManagedLeaseFenceErrorCode(errorMetadata.code); @@ -4243,8 +4244,21 @@ export async function handleChatCore({ // Before returning a model-unavailable error upstream, try sibling models // from the same family. This keeps the request alive on the same account // instead of failing the entire combo. - if (!pipelineRecovered && isModelUnavailableError(statusCode, message, provider)) { - const nextModel = getNextFamilyFallback(currentModel, triedModels, provider); + const familyRecovery = onFailure({ + view: { kind: "pipeline" }, + status: statusCode, + message, + provider, + model: currentModel, + connectionId: String(getCurrentConnectionId() || connectionId || ""), + allowAccountRotation: !managedLease && comboStrategy !== "context-relay", + allowModelFallback: !pipelineRecovered, + isolateProbe: await shouldIsolateProbeFailures(), + nextModel: getNextFamilyFallback(currentModel, triedModels, provider), + canRefresh: false, + }); + if (!pipelineRecovered && familyRecovery.dispatch.action === "fallback-model") { + const nextModel = familyRecovery.dispatch.nextModel; if (nextModel) { triedModels.add(nextModel); currentModel = nextModel; diff --git a/open-sse/handlers/chatCore/providerExecutionPipeline.ts b/open-sse/handlers/chatCore/providerExecutionPipeline.ts index b50c17f91e64..50444ec29bfd 100644 --- a/open-sse/handlers/chatCore/providerExecutionPipeline.ts +++ b/open-sse/handlers/chatCore/providerExecutionPipeline.ts @@ -531,7 +531,13 @@ export async function runProviderExecutionPipeline( target.provider ); if (nextModel) { - wire.setBodyAndModel({ ...wire.body, model: nextModel }, nextModel); + const nextBody = { ...wire.body, model: nextModel }; + wire.setBodyAndModel(nextBody, nextModel); + // Barrel's setBodyAndModel only mutates outer locals. Keep the + // pipeline's own wire snapshot in lockstep so the next send uses + // the sibling model even when the callback does not write back. + wire.body = nextBody; + wire.currentModel = nextModel; modelFallbackPending = true; continue; } diff --git a/open-sse/handlers/chatCore/recoveryPolicy.ts b/open-sse/handlers/chatCore/recoveryPolicy.ts new file mode 100644 index 000000000000..15a52119210d --- /dev/null +++ b/open-sse/handlers/chatCore/recoveryPolicy.ts @@ -0,0 +1,116 @@ +/** + * Recovery decision table for one failed provider attempt. + * + * Side effects stay off the dispatch tag so a terminal 404 can lock + * later (in chatCore providerFailure) without skipping siblings. + * This module does not send, rotate, or lock; it only decides. + */ + +import { COOLDOWN_MS } from "../../config/errorConfig.ts"; +import { isModelUnavailableError } from "../../services/modelFamilyFallback.ts"; + +export type RecoveryView = { kind: "pipeline" }; + +export type RecoveryEffects = { + lockPreviousModel?: { + provider: string; + connectionId: string; + model: string; + reason: "model_not_found"; + cooldownMs: number; + }; + rateLimitUntil?: { connectionId: string; untilMs: number }; +}; + +export type RecoveryDispatch = + | { action: "retry-same"; nextBody?: unknown } + | { action: "rotate-account"; excludeConnectionId: string } + | { action: "fallback-model"; nextModel: string } + | { action: "refresh-credentials" } + | { action: "terminal" }; + +export type RecoveryDecision = { + effects: RecoveryEffects; + dispatch: RecoveryDispatch; +}; + +export type StreamThrowDecision = { action: "terminal" }; + +export type OnFailureInput = { + // view and model are unused in onFailure today. They stay on the + // input so a later lock fill in chatCore providerFailure can pass + // them through without widening this type. + view: RecoveryView; + status: number; + message: string; + provider: string; + model: string; + connectionId: string; + allowAccountRotation: boolean; + allowModelFallback: boolean; + isolateProbe: boolean; + nextModel: string | null; + canRefresh: boolean; + signatureNextBody?: unknown; +}; + +function canRotateAccount(input: OnFailureInput): boolean { + return input.allowAccountRotation && !input.isolateProbe; +} + +export function onFailure(input: OnFailureInput): RecoveryDecision { + if (canRotateAccount(input) && input.provider === "codex" && input.status === 429) { + return { + effects: {}, + dispatch: { action: "rotate-account", excludeConnectionId: input.connectionId }, + }; + } + + if ( + canRotateAccount(input) && + input.provider === "antigravity" && + input.status === 422 && + input.message.includes("gcp_project_required") + ) { + return { + effects: { + rateLimitUntil: { + connectionId: input.connectionId, + untilMs: Date.now() + (COOLDOWN_MS.gcpProjectRequired ?? 24 * 60 * 60 * 1000), + }, + }, + dispatch: { action: "rotate-account", excludeConnectionId: input.connectionId }, + }; + } + + if ((input.status === 401 || input.status === 403) && input.canRefresh) { + return { effects: {}, dispatch: { action: "refresh-credentials" } }; + } + + const not2xx = input.status < 200 || input.status >= 300; + if (input.signatureNextBody !== undefined && not2xx) { + return { + effects: {}, + dispatch: { action: "retry-same", nextBody: input.signatureNextBody }, + }; + } + + if ( + input.allowModelFallback && + isModelUnavailableError(input.status, input.message, input.provider) + ) { + if (input.nextModel) { + return { + effects: {}, + dispatch: { action: "fallback-model", nextModel: input.nextModel }, + }; + } + return { effects: {}, dispatch: { action: "terminal" } }; + } + + return { effects: {}, dispatch: { action: "terminal" } }; +} + +export function onStreamThrow(): StreamThrowDecision { + return { action: "terminal" }; +} diff --git a/tests/unit/chatcore/recovery-callsite.test.ts b/tests/unit/chatcore/recovery-callsite.test.ts new file mode 100644 index 000000000000..29f63c0408d7 --- /dev/null +++ b/tests/unit/chatcore/recovery-callsite.test.ts @@ -0,0 +1,223 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-recovery-callsite-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../../src/lib/db/core.ts"); +const accountFallback = await import("../../../open-sse/services/accountFallback.ts"); +const { runAsProbe } = await import("../../../src/shared/utils/probeOrigin.ts"); +const { handleChatCore } = await import("../../../open-sse/handlers/chatCore.ts"); + +const originalFetch = globalThis.fetch; +const PRIMARY_MODEL = "claude-sonnet-5"; +const CONN = "conn-t5"; + +function noopLog() { + return { debug() {}, info() {}, warn() {}, error() {} }; +} + +function requestedModel(init?: RequestInit): string { + try { + const body = JSON.parse(String(init?.body || "{}")); + return typeof body.model === "string" ? body.model : ""; + } catch { + return ""; + } +} + +function isPrimaryModel(model: string): boolean { + return model === PRIMARY_MODEL || model.endsWith(`/${PRIMARY_MODEL}`); +} + +function jsonError(status: number, message: string): Response { + return new Response(JSON.stringify({ error: { message, type: "invalid_request_error" } }), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +function claudeSseOk(model: string): Response { + return new Response( + [ + "event: message_start", + `data: ${JSON.stringify({ + type: "message_start", + message: { + id: "msg_ok", + type: "message", + role: "assistant", + model, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + })}`, + "", + "event: content_block_start", + `data: ${JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + })}`, + "", + "event: content_block_delta", + `data: ${JSON.stringify({ + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "ok" }, + })}`, + "", + "event: content_block_stop", + `data: ${JSON.stringify({ + type: "content_block_stop", + index: 0, + })}`, + "", + "event: message_delta", + `data: ${JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { output_tokens: 1 }, + })}`, + "", + "event: message_stop", + `data: ${JSON.stringify({ type: "message_stop" })}`, + "", + ].join("\n"), + { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + } + ); +} + +function buildRequest(stream: boolean) { + const body = { + model: PRIMARY_MODEL, + messages: [{ role: "user", content: "hello" }], + max_tokens: 16, + stream, + }; + return { + body: structuredClone(body), + modelInfo: { provider: "claude", model: PRIMARY_MODEL, extendedContext: false }, + credentials: { + apiKey: "test-key", + connectionId: CONN, + providerSpecificData: {}, + }, + log: noopLog(), + clientRawRequest: { + endpoint: "/v1/messages", + body: structuredClone(body), + headers: new Headers({ + accept: stream ? "text/event-stream" : "application/json", + "content-type": "application/json", + }), + }, + userAgent: "unit-test", + }; +} + +const FAMILY = [ + PRIMARY_MODEL, + "claude-sonnet-4-6", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-20250514", +]; + +function anyFamilyLocked(): boolean { + return FAMILY.some((m) => accountFallback.isModelLocked("claude", CONN, m)); +} + +test.afterEach(() => { + globalThis.fetch = originalFetch; + accountFallback.clearAllModelLockouts(); + core.resetDbInstance(); +}); + +test.after(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("stream 404 on primary retries the family sibling and returns 200", async () => { + let fetchCount = 0; + const sentModels: string[] = []; + globalThis.fetch = async (_url, init) => { + fetchCount += 1; + const model = requestedModel(init); + sentModels.push(model); + if (isPrimaryModel(model)) { + return jsonError(404, "model is not available"); + } + return claudeSseOk(model || "claude-sonnet-4-6"); + }; + + const result = await handleChatCore(buildRequest(true)); + assert.equal(fetchCount, 2); + assert.equal(result.success, true); + assert.equal(result.status ?? 200, 200); + assert.equal(isPrimaryModel(sentModels[0] || ""), true); + assert.equal(isPrimaryModel(sentModels[1] || ""), false); +}); + +test("stream 404 on primary and sibling locks the exhausted model once", async () => { + let fetchCount = 0; + globalThis.fetch = async () => { + fetchCount += 1; + return jsonError(404, "model is not available"); + }; + + const result = await handleChatCore(buildRequest(true)); + assert.equal(result.success, false); + assert.equal(result.status, 404); + assert.equal(fetchCount >= 2, true, "family fallback must send at least once more"); + assert.equal(anyFamilyLocked(), true); +}); + +test("probe isolation skips lock when every family member 404s", async () => { + let fetchCount = 0; + globalThis.fetch = async () => { + fetchCount += 1; + return jsonError(404, "model is not available"); + }; + + const result = await runAsProbe(() => handleChatCore(buildRequest(true))); + assert.equal(result.success, false); + assert.equal(result.status, 404); + assert.equal(fetchCount >= 1, true); + assert.equal(anyFamilyLocked(), false); +}); + +test("successful family fallback does not lock the primary model", async () => { + globalThis.fetch = async (_url, init) => { + const model = requestedModel(init); + if (isPrimaryModel(model)) { + return jsonError(404, "model is not available"); + } + return claudeSseOk(model || "claude-sonnet-4-6"); + }; + + const result = await handleChatCore(buildRequest(true)); + assert.equal(result.success, true); + assert.equal(anyFamilyLocked(), false); +}); + +test("abort on the first send does not rotate to a second account", async () => { + let fetchCount = 0; + globalThis.fetch = async () => { + fetchCount += 1; + const err = new Error("The operation was aborted."); + err.name = "AbortError"; + throw err; + }; + + const result = await handleChatCore(buildRequest(true)); + assert.equal(fetchCount, 1); + assert.equal(result.status, 499); + assert.equal(anyFamilyLocked(), false); +}); diff --git a/tests/unit/chatcore/recovery-policy.test.ts b/tests/unit/chatcore/recovery-policy.test.ts new file mode 100644 index 000000000000..d2c775831e9f --- /dev/null +++ b/tests/unit/chatcore/recovery-policy.test.ts @@ -0,0 +1,132 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { COOLDOWN_MS } from "../../../open-sse/config/errorConfig.ts"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-recovery-policy-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { onFailure, onStreamThrow } = + await import("../../../open-sse/handlers/chatCore/recoveryPolicy.ts"); + +function base(over = {}) { + return { + view: { kind: "pipeline" }, + status: 500, + message: "", + provider: "openai", + model: "gpt-5", + connectionId: "conn-1", + allowAccountRotation: true, + allowModelFallback: true, + isolateProbe: false, + nextModel: null, + canRefresh: false, + ...over, + }; +} + +test("onFailure is importable", async () => { + const mod = await import("../../../open-sse/handlers/chatCore/recoveryPolicy.ts"); + assert.equal(typeof mod.onFailure, "function"); + assert.equal(typeof mod.onStreamThrow, "function"); +}); + +test("404 with a sibling falls back and does not lock", () => { + const d = onFailure(base({ status: 404, nextModel: "b", model: "a" })); + assert.deepEqual(d, { + effects: {}, + dispatch: { action: "fallback-model", nextModel: "b" }, + }); + assert.equal("lockPreviousModel" in d.effects, false); +}); + +test("404 with no sibling is terminal and does not lock", () => { + const d = onFailure(base({ status: 404, nextModel: null, model: "a" })); + assert.deepEqual(d, { effects: {}, dispatch: { action: "terminal" } }); + assert.equal("lockPreviousModel" in d.effects, false); +}); + +test("Codex 429 with rotation allowed rotates the account", () => { + const d = onFailure(base({ provider: "codex", status: 429, connectionId: "codex-conn" })); + assert.deepEqual(d, { + effects: {}, + dispatch: { action: "rotate-account", excludeConnectionId: "codex-conn" }, + }); +}); + +test("Codex 429 under probe isolation is terminal", () => { + const d = onFailure(base({ provider: "codex", status: 429, isolateProbe: true })); + assert.equal(d.dispatch.action, "terminal"); +}); + +test("401 with refresh available refreshes credentials", () => { + const d = onFailure(base({ status: 401, canRefresh: true })); + assert.deepEqual(d, { effects: {}, dispatch: { action: "refresh-credentials" } }); +}); + +test("403 with refresh available refreshes credentials", () => { + const d = onFailure(base({ status: 403, canRefresh: true })); + assert.deepEqual(d, { effects: {}, dispatch: { action: "refresh-credentials" } }); +}); + +test("signature next body on a non-2xx retries the same account", () => { + const nextBody = { model: "claude", thinking: { type: "enabled" } }; + const d = onFailure(base({ status: 400, signatureNextBody: nextBody })); + assert.deepEqual(d, { + effects: {}, + dispatch: { action: "retry-same", nextBody }, + }); +}); + +test("onStreamThrow is always terminal", () => { + assert.deepEqual(onStreamThrow(), { action: "terminal" }); +}); + +test("antigravity 422 gcp_project_required rotates and cools the row", () => { + const before = Date.now(); + const d = onFailure( + base({ + provider: "antigravity", + status: 422, + message: "gcp_project_required for this account", + connectionId: "agy-1", + }) + ); + assert.equal(d.dispatch.action, "rotate-account"); + assert.equal(d.dispatch.excludeConnectionId, "agy-1"); + assert.equal(d.effects.rateLimitUntil?.connectionId, "agy-1"); + const cooldown = COOLDOWN_MS.gcpProjectRequired ?? 24 * 60 * 60 * 1000; + const until = d.effects.rateLimitUntil?.untilMs ?? 0; + const after = Date.now(); + assert.ok(until >= before + cooldown); + assert.ok(until <= after + cooldown); +}); + +test("onFailure fallback-model matches isModelUnavailableError", async () => { + const { isModelUnavailableError } = + await import("../../../open-sse/services/modelFamilyFallback.ts"); + const d = onFailure(base({ status: 404, nextModel: "b", model: "a", allowModelFallback: true })); + assert.equal(isModelUnavailableError(404, "", "openai"), true); + assert.equal(d.dispatch.action, "fallback-model"); + assert.equal(d.dispatch.nextModel, "b"); + + const notUnavailable = onFailure( + base({ status: 429, nextModel: "b", model: "a", allowModelFallback: true, provider: "openai" }) + ); + assert.equal(notUnavailable.dispatch.action === "fallback-model", false); +}); + +test("barrel onFailure isolateProbe comes from shouldIsolateProbeFailures", () => { + const src = fs.readFileSync( + new URL("../../../open-sse/handlers/chatCore.ts", import.meta.url), + "utf8" + ); + const idx = src.indexOf("const familyRecovery = onFailure({"); + assert.ok(idx >= 0, "familyRecovery callsite must exist"); + const slice = src.slice(idx, idx + 700); + assert.match(slice, /isolateProbe:\s*await shouldIsolateProbeFailures\(\)/); + assert.equal(/isolateProbe:\s*false/.test(slice), false); +}); diff --git a/tests/unit/provider-execution-pipeline.test.ts b/tests/unit/provider-execution-pipeline.test.ts index e0b584eef7af..e17992cf858b 100644 --- a/tests/unit/provider-execution-pipeline.test.ts +++ b/tests/unit/provider-execution-pipeline.test.ts @@ -450,6 +450,44 @@ test("initial model-unavailable falls back to sibling model", async () => { } }); +test("family fallback still switches when setBodyAndModel does not write currentModel", async () => { + const { runProviderExecutionPipeline } = + await import("../../open-sse/handlers/chatCore/providerExecutionPipeline.ts"); + const sentModels: string[] = []; + const input = makeInput({ + policy: { allowAccountRotation: true, allowModelFallback: true }, + provider: "claude", + model: "claude-sonnet-5", + connectionId: "conn-t5", + send: async (model) => { + sentModels.push(model); + assert.ok(sentModels.length <= 4, "family fallback must not resend the failed model"); + if (model === "claude-sonnet-5") { + return makeAttempt( + { error: { message: "model is not available", type: "invalid_request_error" } }, + 404 + ); + } + return makeAttempt({ id: "ok", content: [{ type: "text", text: "ok" }] }, 200); + }, + getNextFamilyFallback: (current) => + current === "claude-sonnet-5" ? "claude-sonnet-4-6" : null, + }); + // chatCore barrel captures currentModel as a string snapshot and only + // mutates its outer local. Pipeline must not rely on that write-back. + input.wire.setBodyAndModel = (body, nextModel) => { + input.wire.body = body; + input.wire.triedModels.add(nextModel); + }; + + const outcome = await runProviderExecutionPipeline(input); + assert.deepEqual(sentModels, ["claude-sonnet-5", "claude-sonnet-4-6"]); + assert.equal(outcome.kind, "response"); + if (outcome.kind === "response") { + assert.equal(outcome.model, "claude-sonnet-4-6"); + } +}); + test("follow-up allowModelFallback=false blocks model-unavailable fallback", async () => { const { runProviderExecutionPipeline } = await import("../../open-sse/handlers/chatCore/providerExecutionPipeline.ts"); From d3503158727e464c775dff741c5c9100bafb693f Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 09:07:27 -0400 Subject: [PATCH 03/26] refactor(sse): move onStreamComplete out of handleChatCore on current tip Lift the streaming completion callback onto streamMaterialize. Persist and semantic-cache arguments stay the tip shape: streamError || undefined, and bodyForCacheWrite without an empty-object fallback. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 322 +++---------- .../handlers/chatCore/streamMaterialize.ts | 424 ++++++++++++++++++ .../unit/chatcore/stream-materialize.test.ts | 33 ++ 3 files changed, 510 insertions(+), 269 deletions(-) create mode 100644 open-sse/handlers/chatCore/streamMaterialize.ts create mode 100644 tests/unit/chatcore/stream-materialize.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index f853ce0c6987..aba4340e669c 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -43,18 +43,14 @@ import { maybeWrapForcedNonStreamingResponsesJson } from "./chatCore/responsesJs import { enforceOutputTokenBudget } from "./chatCore/outputTokenBudget.ts"; import { maybeConvertJsonBodyToSse } from "./chatCore/jsonBodyToSse.ts"; import { assembleStreamingResponseHeaders } from "./chatCore/streamingResponseHeaders.ts"; -import { storeStreamingSemanticCacheResponse } from "./chatCore/streamingSemanticCacheStore.ts"; import { assembleStreamingPipeline } from "./chatCore/streamingPipeline.ts"; +import { makeOnStreamComplete } from "./chatCore/streamMaterialize.ts"; import { sanitizeChatRequestBody } from "./chatCore/sanitization.ts"; import { applyReasoningInputPolicy, resolveIncompatibleReasoningAction, } from "../services/reasoningInputPolicy.ts"; -import { - createRoutingEvent, - emitRoutingEvent, - outcomeFromStatus, -} from "../services/routing/index.ts"; +import { createRoutingEvent, emitRoutingEvent } from "../services/routing/index.ts"; import { routingFinishReason } from "./chatCore/routingFinishReason.ts"; import { @@ -292,13 +288,7 @@ import { recordCompressionCacheStats } from "./chatCore/compressionCacheStats.ts import { writeCavemanOutputAnalytics } from "./chatCore/cavemanOutputAnalytics.ts"; import { scheduleQuotaShareConsumption } from "./chatCore/quotaShareConsumption.ts"; import { emitRequestGamificationEvent } from "./chatCore/gamificationEvent.ts"; -import { - runPluginOnResponseHook, - runPluginOnStreamCompleteHook, -} from "./chatCore/pluginOnResponse.ts"; -import { scheduleStreamingQuotaShareConsumption } from "./chatCore/streamingQuotaShare.ts"; -import { recordStreamingUsageStats } from "./chatCore/streamingUsageStats.ts"; -import { recordStreamingCost } from "./chatCore/streamingCost.ts"; +import { runPluginOnResponseHook } from "./chatCore/pluginOnResponse.ts"; import { isJsonRecord } from "./chatCore/nonStreamingResponseParse.ts"; import { recordNonStreamingUsageStats } from "./chatCore/nonStreamingUsageStats.ts"; import { normalizeExecutorResult } from "./chatCore/upstreamTimeouts.ts"; @@ -325,13 +315,7 @@ import { resolveReportedServiceTier as resolveReportedServiceTierFor, type EffectiveServiceTier, } from "./chatCore/serviceTier.ts"; -import { - cacheReasoningFromAssistantMessage, - requiresReasoningReplay, -} from "../services/reasoningCache.ts"; import { isCompactResponsesEndpoint } from "../executors/codex.ts"; -import { translateNonStreamingResponse } from "./responseTranslator.ts"; -import { extractToolSchemaMap } from "../translator/response/openai-responses/toolSchemas.ts"; import { extractUsageFromResponse } from "./usageExtractor.ts"; import { updateFromHeaders, @@ -5379,257 +5363,57 @@ export async function handleChatCore({ let streamFailureCompletionRecorded = false; // Callback to save call log when stream completes (include responseBody when provided by stream) - const onStreamComplete = ({ - status: streamStatus, - usage: streamUsage, - responseBody: streamResponseBody, - providerPayload, - clientPayload, - error: streamError, - errorCode: streamErrorCode, - ttft, - itlMs: streamItlMs, - interrupted: _streamInterrupted, - }) => { - const normalizedStreamStatus = streamStatus || 200; - if (streamCompletionRecorded) return; - streamCompletionRecorded = true; - if (normalizedStreamStatus !== 200) { - if (streamFailureCompletionRecorded) return; - streamFailureCompletionRecorded = true; - } - const cacheUsageLogMeta = buildCacheUsageLogMeta(streamUsage); - const streamConnectionId = getCurrentConnectionId(); - - if (normalizedStreamStatus === 200) { - void maybeSyncClaudeExtraUsageState({ - provider, - connectionId: streamConnectionId, - providerSpecificData: credentials?.providerSpecificData, - log, - }); - } - - // Reasoning Replay Cache (#1628): Capture reasoning_content from streaming responses - // with tool_calls so it can be replayed on subsequent turns (DeepSeek V4, Kimi K2, etc.) - if (normalizedStreamStatus === 200 && streamResponseBody) { - try { - const streamBody = streamResponseBody as Record; - const cacheStreamBody = Array.isArray(streamBody.choices) - ? streamBody - : needsTranslation(clientResponseFormat, FORMATS.OPENAI) - ? (translateNonStreamingResponse( - streamBody, - clientResponseFormat, - FORMATS.OPENAI, - responseToolNameMap, - extractToolSchemaMap(finalBody || translatedBody || body) - ) as Record) - : streamBody; - const choices = cacheStreamBody.choices as - { message?: Record }[] | undefined; - const msg = choices?.[0]?.message; - // Responses-shaped bodies carry `input`, not `messages` — use the pivot - // transcript translateRequest reported so plain-turn keys match the read side. - const historyMessages = - (translatedBody as { messages?: unknown[] } | null | undefined)?.messages ?? - reasoningReplayHistory; - if (requiresReasoningReplay({ provider, model })) { - cacheReasoningFromAssistantMessage(msg, provider, model, { - scope: reasoningCacheScope, - historyMessages: Array.isArray(historyMessages) ? historyMessages : [], - }); - } - } catch { - // Cache capture is non-critical — never block the stream - } - } - effectiveServiceTier = resolveReportedServiceTier(streamResponseBody) ?? effectiveServiceTier; - - // Context Editing telemetry (streaming): the reconstructed stream body now carries - // context_management.applied_edits from the final message_delta snapshot. Mirror the - // non-streaming hook so streaming context-clear savings also surface under engine - // "context-editing" in compression analytics. Best-effort, Claude-only. - if (normalizedStreamStatus === 200) { - recordContextEditingTelemetryHook({ - contextEditingEnabled, - provider, - responseBody: streamResponseBody, - skillRequestId, - log, - }); - } - - streamFailure.finalizeStreamRequestLog({ - pendingRequestId, - model, - provider, - connectionId: streamConnectionId, - providerResponse: providerPayload ?? streamResponseBody ?? undefined, - clientResponse: clientPayload ?? streamResponseBody ?? undefined, - status: normalizedStreamStatus, - error: streamError, - errorCode: streamErrorCode, - }); - - // Track cache token metrics for streaming responses - if (streamUsage && typeof streamUsage === "object") { - attachCompressionUsageReceiptAfterAnalytics(streamUsage as Record, "stream"); - // Track Gemini token consumption for TPM rate-limit pre-check - if (provider === "gemini") { - const promptTokens = - typeof (streamUsage as Record).prompt_tokens === "number" - ? ((streamUsage as Record).prompt_tokens as number) - : 0; - if (promptTokens > 0) incrementTokenUsage(model, promptTokens); - } - } - recordStreamingUsageStats(streamUsage, { - provider, - model, - streamStatus: normalizedStreamStatus, - startTime, - ttft, - streamErrorCode, - connectionId: streamConnectionId, - apiKeyInfo, - effectiveServiceTier, - isCombo, - comboStrategy, - endpoint: endpointPath, - }); - - // Routing event (feedback foundation) — fire-and-forget, cheap, never blocks - // the stream. Feeds the quality tracker + optional OTel exporter. - void emitRoutingEvent( - createRoutingEvent({ - requestId: traceId || pendingRequestId || "unknown", - provider: provider || "unknown", - model: model || "unknown", - strategy: isCombo ? (comboStrategy ?? "combo") : "direct", - latencyMs: Date.now() - startTime, - ttftMs: typeof ttft === "number" && Number.isFinite(ttft) && ttft >= 0 ? ttft : null, - itlMs: - typeof streamItlMs === "number" && Number.isFinite(streamItlMs) && streamItlMs >= 0 - ? streamItlMs - : null, - inputTokens: - streamUsage && typeof streamUsage === "object" - ? (() => { - const promptTokens = (streamUsage as Record).prompt_tokens; - return typeof promptTokens === "number" && Number.isFinite(promptTokens) - ? promptTokens - : null; - })() - : null, - outputTokens: - streamUsage && typeof streamUsage === "object" - ? (() => { - const completionTokens = (streamUsage as Record).completion_tokens; - return typeof completionTokens === "number" && Number.isFinite(completionTokens) - ? completionTokens - : null; - })() - : null, - cost: null, - retries: 0, - fallbackUsed: false, // combo-level fallback tracked by decisionTrace - outcome: - normalizedStreamStatus === 200 - ? "success" - : streamErrorCode === "stream_interrupted" || streamErrorCode === "aborted" - ? "stream_interrupted" - : outcomeFromStatus(normalizedStreamStatus), - status: normalizedStreamStatus, - finishReason: routingFinishReason(streamResponseBody), - connectionId: streamConnectionId ?? credentials?.connectionId ?? null, - }) - ); - - persistAttemptLogs({ - status: normalizedStreamStatus, - error: streamError || undefined, - tokens: streamUsage || {}, - responseBody: streamResponseBody ?? undefined, - providerRequest: finalBody || translatedBody, - providerResponse: providerPayload, - clientResponse: clientPayload ?? streamResponseBody ?? undefined, - claudeCacheMeta: claudePromptCacheLogMeta, - claudeCacheUsageMeta: cacheUsageLogMeta, - cacheSource: "upstream", - }); - - recordStreamingCost({ - apiKeyId: apiKeyInfo?.id, - provider, - model, - streamUsage, - serviceTier: effectiveServiceTier, - calculateCost, - recordCost, - }); - - // === Quota Share POST-hook streaming (B/F7) — fire-and-forget, fail-open === - // Resolve the real per-request cost (calculateCost) so USD-unit pools accrue - // on streaming traffic too; this previously recorded usd:0 hardcoded, which - // meant DeepSeek-style `usd/monthly` shared pools never blocked on streams. - scheduleStreamingQuotaShareConsumption({ - apiKeyId: apiKeyInfo?.id, - connectionId: credentials?.connectionId, - provider, - model, - streamUsage, - streamStatus: normalizedStreamStatus, - serviceTier: effectiveServiceTier, - calculateCost, - log, - }); - // === /Quota Share POST-hook streaming === - - if (streamStatus === 200) { - // #12150 P1b surface 3 (fix round 1): see the matching non-streaming - // gate above — an observed request populates NO durable memory from - // either the request-derived text or this streamed response. - runMemoryExtractionGate({ - memoryOwnerId, - memorySettings, - videoBridgeObserved, - pipelineSessionId, - requestBody: body as Record, - responseBody: (streamResponseBody ?? null) as Record | null, - extractFacts, - log, - }); - } - - // Semantic cache: store assembled streaming response for future cache hits - storeStreamingSemanticCacheResponse({ - enabled: semanticCacheEnabled, - streamStatus, - streamResponseBody, - body: bodyForCacheWrite, - headers: clientRawRequest?.headers, - model, - provider, - apiKeyId: apiKeyInfo?.id ?? undefined, - streamUsage, - log, - }); + const onStreamComplete = makeOnStreamComplete({ + persistAttemptLogs, + getCurrentConnectionId, + provider, + model, + credentials, + log, + clientResponseFormat, + responseToolNameMap, + finalBody, + translatedBody, + body, + reasoningCacheScope, + contextEditingEnabled, + skillRequestId, + streamFailure, + pendingRequestId, + startTime, + apiKeyInfo, + isCombo, + comboStrategy, + endpointPath, + traceId, + calculateCost, + recordCost, + memoryOwnerId, + memorySettings, + videoBridgeObserved, + pipelineSessionId, + extractFacts, + semanticCacheEnabled, + bodyForCacheWrite, + clientRawRequest, + claudePromptCacheLogMeta, + resolveReportedServiceTier, + attachCompressionUsageReceiptAfterAnalytics, + routingFinishReason, + getStreamCompletionRecorded: () => streamCompletionRecorded, + setStreamCompletionRecorded: (v) => { + streamCompletionRecorded = v; + }, + getStreamFailureCompletionRecorded: () => streamFailureCompletionRecorded, + setStreamFailureCompletionRecorded: (v) => { + streamFailureCompletionRecorded = v; + }, + getEffectiveServiceTier: () => effectiveServiceTier, + setEffectiveServiceTier: (t) => { + effectiveServiceTier = t; + }, + }); - // Plugin onStreamComplete hook — fire-and-forget, fail-open (#9571) - // Pass traceId as requestId so plugins can correlate the stream-completion event - // with the originating request (the same id used for onRequest/onResponse). (#11825) - runPluginOnStreamCompleteHook({ - status: normalizedStreamStatus, - usage: streamUsage as Record | undefined, - ttft, - model, - provider, - errorCode: streamErrorCode, - startTime, - requestId: traceId, - }); - }; const streamFailureFinalizers = streamFailure.createStreamFailureFinalizers({ isFailureCompletionRecorded: () => streamFailureCompletionRecorded, diff --git a/open-sse/handlers/chatCore/streamMaterialize.ts b/open-sse/handlers/chatCore/streamMaterialize.ts new file mode 100644 index 000000000000..ab45f08a4dd8 --- /dev/null +++ b/open-sse/handlers/chatCore/streamMaterialize.ts @@ -0,0 +1,424 @@ +/** + * Streaming completion callback lifted out of handleChatCore. + * Logs, cache, cost, and plugin hooks after the upstream stream ends. + * Does not send or decide recovery. + */ + +import { FORMATS } from "../../translator/formats.ts"; +import { needsTranslation } from "../../translator/index.ts"; +import { extractToolSchemaMap } from "../../translator/response/openai-responses/toolSchemas.ts"; +import { + cacheReasoningFromAssistantMessage, + requiresReasoningReplay, +} from "../../services/reasoningCache.ts"; +import { incrementTokenUsage } from "../../services/geminiRateLimitTracker.ts"; +import { + createRoutingEvent, + emitRoutingEvent, + outcomeFromStatus, +} from "../../services/routing/index.ts"; +import { translateNonStreamingResponse } from "../responseTranslator.ts"; +import { buildCacheUsageLogMeta } from "./cacheUsageMeta.ts"; +import { maybeSyncClaudeExtraUsageState } from "./telemetryHelpers.ts"; +import { recordContextEditingTelemetryHook } from "./contextEditingTelemetry.ts"; +import { recordStreamingUsageStats } from "./streamingUsageStats.ts"; +import { recordStreamingCost } from "./streamingCost.ts"; +import { scheduleStreamingQuotaShareConsumption } from "./streamingQuotaShare.ts"; +import { runMemoryExtractionGate } from "./memoryExtraction.ts"; +import { storeStreamingSemanticCacheResponse } from "./streamingSemanticCacheStore.ts"; +import { runPluginOnStreamCompleteHook } from "./pluginOnResponse.ts"; +import type { PersistAttemptLogsArgs } from "./attemptLogging.ts"; +import type { EffectiveServiceTier } from "./serviceTier.ts"; + +export type StreamCompletePayload = { + status?: number; + usage?: unknown; + responseBody?: unknown; + providerPayload?: unknown; + clientPayload?: unknown; + error?: unknown; + errorCode?: string; + ttft?: number; + itlMs?: number; + interrupted?: unknown; +}; + +export type StreamMaterializeDeps = { + persistAttemptLogs: (args: PersistAttemptLogsArgs) => void; + getCurrentConnectionId: () => string | null; + provider: string | null | undefined; + model: string | null | undefined; + credentials: + | { + connectionId?: string | null; + providerSpecificData?: unknown; + } + | null + | undefined; + log: + | { + info?: (tag: string, msg: string) => void; + debug?: (tag: string, msg: string) => void; + warn?: (...args: unknown[]) => void; + } + | null + | undefined; + clientResponseFormat: string; + responseToolNameMap: Map | null | undefined; + finalBody: Record | null | undefined; + translatedBody: Record | null | undefined; + body: unknown; + reasoningCacheScope: string | null; + contextEditingEnabled: boolean; + skillRequestId: string; + streamFailure: { + finalizeStreamRequestLog: (args: Record) => void; + }; + pendingRequestId: string; + startTime: number; + apiKeyInfo: { id?: string | null } | undefined; + isCombo: boolean; + comboStrategy: string | null | undefined; + endpointPath: string | undefined; + traceId: string; + calculateCost: ( + provider: string, + model: string, + usage: Record | null | undefined, + options: { serviceTier?: string } + ) => Promise; + recordCost: (apiKeyId: string, cost: number) => void; + memoryOwnerId: string | null | undefined; + memorySettings: { enabled?: boolean | null; maxTokens?: number | null } | null | undefined; + videoBridgeObserved: boolean; + pipelineSessionId: string | null | undefined; + extractFacts: (text: string, memoryOwnerId: string, sessionId: string) => void; + semanticCacheEnabled: boolean; + bodyForCacheWrite: unknown; + clientRawRequest: { headers?: Headers | null | undefined }; + claudePromptCacheLogMeta: Record | null | undefined; + resolveReportedServiceTier: (payload?: unknown, maxDepth?: number) => EffectiveServiceTier | null; + attachCompressionUsageReceiptAfterAnalytics: ( + usage: Record, + source: "provider" | "estimated" | "stream" + ) => void; + routingFinishReason: (body: unknown) => string | null; + getStreamCompletionRecorded: () => boolean; + setStreamCompletionRecorded: (v: boolean) => void; + getStreamFailureCompletionRecorded: () => boolean; + setStreamFailureCompletionRecorded: (v: boolean) => void; + getEffectiveServiceTier: () => EffectiveServiceTier; + setEffectiveServiceTier: (t: EffectiveServiceTier) => void; +}; + +export function makeOnStreamComplete( + deps: StreamMaterializeDeps +): (payload: StreamCompletePayload) => void { + const { + persistAttemptLogs, + getCurrentConnectionId, + provider, + model, + credentials, + log, + clientResponseFormat, + responseToolNameMap, + finalBody, + translatedBody, + body, + reasoningCacheScope, + contextEditingEnabled, + skillRequestId, + streamFailure, + pendingRequestId, + startTime, + apiKeyInfo, + isCombo, + comboStrategy, + endpointPath, + traceId, + calculateCost, + recordCost, + memoryOwnerId, + memorySettings, + videoBridgeObserved, + pipelineSessionId, + extractFacts, + semanticCacheEnabled, + bodyForCacheWrite, + clientRawRequest, + claudePromptCacheLogMeta, + resolveReportedServiceTier, + attachCompressionUsageReceiptAfterAnalytics, + routingFinishReason, + getStreamCompletionRecorded, + setStreamCompletionRecorded, + getStreamFailureCompletionRecorded, + setStreamFailureCompletionRecorded, + getEffectiveServiceTier, + setEffectiveServiceTier, + } = deps; + + return ({ + status: streamStatus, + usage: streamUsage, + responseBody: streamResponseBody, + providerPayload, + clientPayload, + error: streamError, + errorCode: streamErrorCode, + ttft, + itlMs: streamItlMs, + interrupted: _streamInterrupted, + }: StreamCompletePayload) => { + const normalizedStreamStatus = streamStatus || 200; + if (getStreamCompletionRecorded()) return; + setStreamCompletionRecorded(true); + if (normalizedStreamStatus !== 200) { + if (getStreamFailureCompletionRecorded()) return; + setStreamFailureCompletionRecorded(true); + } + const cacheUsageLogMeta = buildCacheUsageLogMeta( + streamUsage && typeof streamUsage === "object" + ? (streamUsage as Record) + : null + ); + const streamConnectionId = getCurrentConnectionId(); + + if (normalizedStreamStatus === 200) { + void maybeSyncClaudeExtraUsageState({ + provider, + connectionId: streamConnectionId, + providerSpecificData: credentials?.providerSpecificData, + log, + }); + } + + // Reasoning Replay Cache (#1628): Capture reasoning_content from streaming responses + // with tool_calls so it can be replayed on subsequent turns (DeepSeek V4, Kimi K2, etc.) + if (normalizedStreamStatus === 200 && streamResponseBody) { + try { + const streamBody = streamResponseBody as Record; + const cacheStreamBody = Array.isArray(streamBody.choices) + ? streamBody + : needsTranslation(clientResponseFormat, FORMATS.OPENAI) + ? (translateNonStreamingResponse( + streamBody, + clientResponseFormat, + FORMATS.OPENAI, + responseToolNameMap, + extractToolSchemaMap(finalBody || translatedBody || body) + ) as Record) + : streamBody; + const choices = cacheStreamBody.choices as + { message?: Record }[] | undefined; + const msg = choices?.[0]?.message; + const historyMessages = (translatedBody as { messages?: unknown[] } | null | undefined) + ?.messages; + if (requiresReasoningReplay({ provider, model })) { + cacheReasoningFromAssistantMessage(msg, provider, model, { + scope: reasoningCacheScope, + historyMessages: Array.isArray(historyMessages) ? historyMessages : [], + }); + } + } catch { + // Cache capture is non-critical — never block the stream + } + } + const effectiveServiceTier = + resolveReportedServiceTier(streamResponseBody) ?? getEffectiveServiceTier(); + setEffectiveServiceTier(effectiveServiceTier); + + // Context Editing telemetry (streaming): the reconstructed stream body now carries + // context_management.applied_edits from the final message_delta snapshot. Mirror the + // non-streaming hook so streaming context-clear savings also surface under engine + // "context-editing" in compression analytics. Best-effort, Claude-only. + if (normalizedStreamStatus === 200) { + recordContextEditingTelemetryHook({ + contextEditingEnabled, + provider, + responseBody: streamResponseBody, + skillRequestId, + log, + }); + } + + streamFailure.finalizeStreamRequestLog({ + pendingRequestId, + model, + provider, + connectionId: streamConnectionId, + providerResponse: providerPayload ?? streamResponseBody ?? undefined, + clientResponse: clientPayload ?? streamResponseBody ?? undefined, + status: normalizedStreamStatus, + error: streamError, + errorCode: streamErrorCode, + }); + + // Track cache token metrics for streaming responses + if (streamUsage && typeof streamUsage === "object") { + attachCompressionUsageReceiptAfterAnalytics(streamUsage as Record, "stream"); + // Track Gemini token consumption for TPM rate-limit pre-check + if (provider === "gemini") { + const promptTokens = + typeof (streamUsage as Record).prompt_tokens === "number" + ? ((streamUsage as Record).prompt_tokens as number) + : 0; + if (promptTokens > 0) incrementTokenUsage(model, promptTokens); + } + } + recordStreamingUsageStats(streamUsage, { + provider, + model, + streamStatus: normalizedStreamStatus, + startTime, + ttft, + streamErrorCode, + connectionId: streamConnectionId, + apiKeyInfo, + effectiveServiceTier, + isCombo, + comboStrategy, + endpoint: endpointPath, + }); + + // Routing event (feedback foundation) — fire-and-forget, cheap, never blocks + // the stream. Feeds the quality tracker + optional OTel exporter. + void emitRoutingEvent( + createRoutingEvent({ + requestId: traceId || pendingRequestId || "unknown", + provider: provider || "unknown", + model: model || "unknown", + strategy: isCombo ? (comboStrategy ?? "combo") : "direct", + latencyMs: Date.now() - startTime, + ttftMs: typeof ttft === "number" && Number.isFinite(ttft) && ttft >= 0 ? ttft : null, + itlMs: + typeof streamItlMs === "number" && Number.isFinite(streamItlMs) && streamItlMs >= 0 + ? streamItlMs + : null, + inputTokens: + streamUsage && typeof streamUsage === "object" + ? (() => { + const promptTokens = (streamUsage as Record).prompt_tokens; + return typeof promptTokens === "number" && Number.isFinite(promptTokens) + ? promptTokens + : null; + })() + : null, + outputTokens: + streamUsage && typeof streamUsage === "object" + ? (() => { + const completionTokens = (streamUsage as Record).completion_tokens; + return typeof completionTokens === "number" && Number.isFinite(completionTokens) + ? completionTokens + : null; + })() + : null, + cost: null, + retries: 0, + fallbackUsed: false, // combo-level fallback tracked by decisionTrace + outcome: + normalizedStreamStatus === 200 + ? "success" + : streamErrorCode === "stream_interrupted" || streamErrorCode === "aborted" + ? "stream_interrupted" + : outcomeFromStatus(normalizedStreamStatus), + status: normalizedStreamStatus, + finishReason: routingFinishReason(streamResponseBody), + connectionId: streamConnectionId ?? credentials?.connectionId ?? null, + }) + ); + + persistAttemptLogs({ + status: normalizedStreamStatus, + error: streamError || undefined, + tokens: streamUsage || {}, + responseBody: streamResponseBody ?? undefined, + providerRequest: finalBody || translatedBody, + providerResponse: providerPayload, + clientResponse: clientPayload ?? streamResponseBody ?? undefined, + claudeCacheMeta: claudePromptCacheLogMeta, + claudeCacheUsageMeta: cacheUsageLogMeta, + cacheSource: "upstream", + }); + + recordStreamingCost({ + apiKeyId: apiKeyInfo?.id, + provider, + model, + streamUsage: + streamUsage && typeof streamUsage === "object" + ? (streamUsage as Record) + : null, + serviceTier: effectiveServiceTier, + calculateCost, + recordCost, + }); + + // === Quota Share POST-hook streaming (B/F7) — fire-and-forget, fail-open === + // Resolve the real per-request cost (calculateCost) so USD-unit pools accrue + // on streaming traffic too; this previously recorded usd:0 hardcoded, which + // meant DeepSeek-style `usd/monthly` shared pools never blocked on streams. + scheduleStreamingQuotaShareConsumption({ + apiKeyId: apiKeyInfo?.id, + connectionId: credentials?.connectionId, + provider, + model, + streamUsage, + streamStatus: normalizedStreamStatus, + serviceTier: effectiveServiceTier, + calculateCost, + log, + }); + // === /Quota Share POST-hook streaming === + + if (streamStatus === 200) { + // #12150 P1b surface 3 (fix round 1): see the matching non-streaming + // gate above — an observed request populates NO durable memory from + // either the request-derived text or this streamed response. + runMemoryExtractionGate({ + memoryOwnerId, + memorySettings, + videoBridgeObserved, + pipelineSessionId, + requestBody: body as Record, + responseBody: (streamResponseBody ?? null) as Record | null, + extractFacts, + log, + }); + } + + // Semantic cache: store assembled streaming response for future cache hits + storeStreamingSemanticCacheResponse({ + enabled: semanticCacheEnabled, + streamStatus, + streamResponseBody: streamResponseBody as Record | null | undefined, + body: bodyForCacheWrite as { + messages?: unknown; + input?: unknown; + temperature?: number; + top_p?: number; + }, + headers: clientRawRequest?.headers, + model, + provider, + apiKeyId: apiKeyInfo?.id ?? undefined, + streamUsage: streamUsage as Record | null | undefined, + log, + }); + + // Plugin onStreamComplete hook — fire-and-forget, fail-open (#9571) + // Pass traceId as requestId so plugins can correlate the stream-completion event + // with the originating request (the same id used for onRequest/onResponse). (#11825) + runPluginOnStreamCompleteHook({ + status: normalizedStreamStatus, + usage: streamUsage as Record | undefined, + ttft, + model, + provider, + errorCode: streamErrorCode, + startTime, + requestId: traceId, + }); + }; +} diff --git a/tests/unit/chatcore/stream-materialize.test.ts b/tests/unit/chatcore/stream-materialize.test.ts new file mode 100644 index 000000000000..f06b943857f9 --- /dev/null +++ b/tests/unit/chatcore/stream-materialize.test.ts @@ -0,0 +1,33 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const barrelPath = path.resolve(here, "../../../open-sse/handlers/chatCore.ts"); + +test("streamMaterialize leaf exports makeOnStreamComplete", async () => { + const mod = await import("../../../open-sse/handlers/chatCore/streamMaterialize.ts"); + assert.equal(typeof mod.makeOnStreamComplete, "function"); +}); + +test("chatCore barrel no longer nests const onStreamComplete", () => { + const src = fs.readFileSync(barrelPath, "utf8"); + assert.equal( + src.includes("const onStreamComplete = ({"), + false, + "nested const onStreamComplete = ({ must leave the barrel" + ); +}); + +test("streamMaterialize keeps tip persist and cache argument shape", () => { + const src = fs.readFileSync( + path.resolve(here, "../../../open-sse/handlers/chatCore/streamMaterialize.ts"), + "utf8" + ); + assert.match(src, /error: streamError \|\| undefined/); + assert.equal(src.includes('typeof streamError === "string"'), false); + assert.match(src, /body: bodyForCacheWrite as \{/); + assert.equal(src.includes('bodyForCacheWrite && typeof bodyForCacheWrite === "object"'), false); +}); From 204f3e69c617bae8d21f5bce6ecf71a979527097 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 12:16:10 -0400 Subject: [PATCH 04/26] refactor(sse): move request prelude out of handleChatCore on current tip Lift bypass, classifier, and lifecycle early-returns onto requestPrelude. Persist helpers are declared after endpointPath and reqLogger so they do not close over a TDZ binding. After translation the barrel rebinds persistAttemptLogs onto translatedBody and the live tokensCompressed. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 952 +++---------------- open-sse/handlers/chatCore/requestPrelude.ts | 945 ++++++++++++++++++ tests/unit/chatcore/request-prelude.test.ts | 66 ++ 3 files changed, 1158 insertions(+), 805 deletions(-) create mode 100644 open-sse/handlers/chatCore/requestPrelude.ts create mode 100644 tests/unit/chatcore/request-prelude.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index aba4340e669c..47ffdfa08970 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1,3 +1,4 @@ +import { runRequestPrelude } from "./chatCore/requestPrelude.ts"; import { executeProviderRequest as executeProviderRequestFromLeaf } from "./chatCore/executeProviderRequest.ts"; import { extractRequestToolIdentityMap, @@ -7,14 +8,13 @@ import { injectMemoryAndSkills, mergeInjectedFallbackOwnerNames, } from "./chatCore/memorySkillsInjection.ts"; -import { resolveChatCoreRequestSetup } from "./chatCore/requestSetup.ts"; import { normalizeOpenAICompatibleTools, shouldNormalizeFunctionToolsOnly, } from "./chatCore/openAICompatibleTools.ts"; import { - buildFailureUsageRecord, projectFailureUsageErrorCode, + buildFailureUsageRecord, type FailureUsageAggregate, } from "./chatCore/failureUsage.ts"; import { createTranslationFailureResult } from "./chatCore/translationFailure.ts"; @@ -27,15 +27,8 @@ export { extractSystemRoleMessages, relocateDirectiveOnlyMessages, } from "./chatCore/claudeSystemRole.ts"; -import { checkIdempotencyCache } from "./chatCore/idempotency.ts"; import { acquireTurnExecution, createTurnInProgressResult } from "./chatCore/turnExecutionGuard.ts"; import { checkSemanticCache } from "./chatCore/semanticCache.ts"; -import { checkLifecycle, resolveLifecycle } from "./chatCore/modelLifecyclePolicy.ts"; -import { - shouldDefaultAllowClassifier, - detectClassifierFormat, - buildDefaultAllowClaudeMessage, -} from "./chatCore/claudeClassifierCompat.ts"; import { buildPostCallGuardrailContext } from "./chatCore/postCallGuardrailContext.ts"; import { storeSemanticCacheResponse } from "./chatCore/semanticCacheStore.ts"; import { buildNonStreamingResponseHeaders } from "./chatCore/nonStreamingResponseHeaders.ts"; @@ -59,16 +52,11 @@ import { resolveCompressionHeader, } from "./chatCore/headers.ts"; -import { - getCodexClientSessionId, - isCodexOriginatedHeaders, - isClaudeCodeOriginatedHeaders, -} from "../config/codexIdentity.ts"; +import { getCodexClientSessionId } from "../config/codexIdentity.ts"; import { noteCodexTurnStateProvenance, readCodexTurnStateHeader, } from "../config/codexTurnState.ts"; -import { trackDevice, extractIpFromHeaders } from "../services/deviceTracker.ts"; import { getCombosCached } from "./chatCore/comboContextCache.ts"; export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comboContextCache.ts"; import { @@ -78,7 +66,6 @@ import { import { shouldUseNativeCodexPassthrough, shouldUseNativeXaiResponsesPassthrough, - shouldUseNativeOpenAICompatibleResponsesPassthrough, stampNativeResponsesPassthroughBody, redactPassthroughThinkingSignatures, isClaudeCodeSemanticPassthroughRequest, @@ -115,22 +102,15 @@ export { stripStaleForwardingHeaders, }; import { resolveMemoryOwnerId, runMemoryExtractionGate } from "./chatCore/memoryExtraction.ts"; -import { checkResourcePressureGuard } from "../utils/resourcePressure.ts"; import { normalizeHeaders } from "../utils/headers.ts"; -import { resolveChatCoreRequestFormat } from "./chatCore/requestFormat.ts"; -import { resolveChatCoreTargetFormat } from "./chatCore/targetFormat.ts"; import { resolveOmniGlyphTransport } from "../services/compression/imageTransportPolicy.ts"; import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts"; import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; -import { - injectCustomSystemPrompt, - injectSystemPromptPreTranslation, -} from "../services/systemPrompt.ts"; +import { injectSystemPromptPreTranslation } from "../services/systemPrompt.ts"; import { translateRequest, needsTranslation } from "../translator/index.ts"; import { applyReasoningRuleDirective } from "@/lib/reasoningRouting/policy"; import { withReasoningRuleContext } from "../utils/reasoningRuleContext.ts"; import { FORMATS } from "../translator/formats.ts"; -import { collectCustomToolNamesForSourceFormat } from "../translator/request/openai-responses/additionalTools.ts"; import { sanitizeKiroTools } from "../utils/kiroSanitizer.ts"; import { splitMisplacedToolResults } from "../translator/helpers/claudeHelper.ts"; import { ensureCacheControlOnLastUserMessage } from "../services/claudeCodeConstraints.ts"; @@ -140,9 +120,8 @@ import { COLORS, } from "../utils/stream.ts"; import { ensureStreamReadiness } from "../utils/streamReadiness.ts"; -import { resolveSuppressThinkClose, THINKING_MARKER_HEADER } from "../utils/thinkCloseMarker.ts"; +import { resolveSuppressThinkClose } from "../utils/thinkCloseMarker.ts"; import { resolveStreamReadinessTimeout } from "../utils/streamReadinessPolicy.ts"; -import { resolveAgentGoalPolicy } from "../utils/agentGoalPolicy.ts"; import { hasActiveClaudeThinking } from "../utils/thinkingBudget.ts"; import { createStreamController } from "../utils/streamHandler.ts"; import * as streamFailure from "../utils/streamFailureFinalization.ts"; @@ -153,11 +132,8 @@ import { runWithOnPersist, runWithCasGuard, } from "../services/tokenRefresh.ts"; -import { createRequestLogger } from "../utils/requestLogger.ts"; -import { createPreparedRequestLogger, runWithCapture } from "../utils/providerRequestLogging.ts"; -import { summarizeToolSources } from "../utils/toolSources.ts"; +import { runWithCapture } from "../utils/providerRequestLogging.ts"; import { applyResponsesPreviousResponseIdPolicy } from "../utils/responsesStatePolicy.ts"; -import { applyClaudeEffortVariant } from "./chatCore/claudeEffortVariant.ts"; import { DEFAULT_THINKING_CLAUDE_SIGNATURE } from "../config/defaultThinkingSignature.ts"; import { getStripTypesForProviderModel, @@ -165,7 +141,7 @@ import { } from "../services/modelStrip.ts"; import { shouldUseMidConversationSystem } from "../executors/claudeIdentity.ts"; import { echoModelInObject } from "../services/responseModelEcho.ts"; -import { getUnsupportedParams, REGISTRY } from "../config/providerRegistry.ts"; +import { getUnsupportedParams } from "../config/providerRegistry.ts"; import { checkToolCallingRequiredButUnsupported } from "./chatCore/toolCallingRequiredCheck.ts"; import { supportsMaxTokens, @@ -183,7 +159,6 @@ import { isFeatureFlagEnabled, isServerOwnedToolLoopEnabled, } from "@/shared/utils/featureFlags.ts"; -import { resolveNoAuthEchoModel } from "./chatCore/noAuthEchoModel.ts"; import { REASONING_BUFFER_MIN_TRIGGER, buildReasoningProbeTruncatedResponse, @@ -217,12 +192,10 @@ import { STREAM_DISCONNECT_GRACE_PERIOD_MS, } from "../config/constants.ts"; import { applyStatusRestatement } from "../config/upstreamStatusRestatement.ts"; -import { resolveResilienceSettings } from "@/lib/resilience/settings"; import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts"; import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; import { wasRefreshTokenRotated } from "@omniroute/open-sse/services/refreshSerializer.ts"; import { connectionHasExtraKeys } from "../services/apiKeyRotator.ts"; -import { recordKeyHealthStatus as recordKeyHealthStatusFor } from "./chatCore/keyHealth.ts"; import { getSkillsModelIdForFormat } from "./chatCore/skillsFormat.ts"; import { createSafeAbortError, @@ -242,23 +215,17 @@ import { persistAttemptLogs as persistAttemptLogsFor, type PersistAttemptLogsArgs, } from "./chatCore/attemptLogging.ts"; -import { stageTrace } from "./chatCore/stageTrace.ts"; import { attachCompressionUsageReceiptAfterAnalytics as attachCompressionUsageReceiptAfterAnalyticsFor } from "./chatCore/compressionUsageReceipt.ts"; import { prepareUpstreamBody } from "./chatCore/upstreamBody.ts"; import { getQuotaScopeLabelForProvider } from "../services/antigravityQuotaFamily.ts"; import { excludeConnectionForCooldown } from "./chatCore/connectionCooldown.ts"; import { handleRequestRejectedFailure } from "./chatCore/requestRejectedFailure.ts"; import { getKimiTemporaryRateLimitResetAt } from "./chatCore/kimiQuotaRecovery.ts"; -import { - getCallLogPipelineCaptureStreamChunks, - getCallLogPipelineMaxSizeBytes, -} from "@/lib/logEnv"; import { logAuditEvent } from "@/lib/compliance"; import { emit } from "@/lib/events/eventBus"; import { adaptBodyForCompression } from "../services/compression/bodyAdapter.ts"; import { ensureEngineBreakdown } from "../services/compression/engineBreakdown.ts"; -import { handleBypassRequest } from "../utils/bypassHandler.ts"; -import { saveRequestUsage, trackPendingRequest, appendRequestLog } from "@/lib/usageDb"; +import { trackPendingRequest, appendRequestLog, saveRequestUsage } from "@/lib/usageDb"; import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; import { recordCost } from "@/domain/costRules"; import { calculateCost } from "@/lib/usage/costCalculator"; @@ -282,7 +249,6 @@ import { writeCompressionAnalytics, writeCompressionSkip, } from "./chatCore/compressionAnalyticsWrite.ts"; -import { runPluginOnRequestHook } from "./chatCore/pluginOnRequest.ts"; import { recordContextEditingTelemetryHook } from "./chatCore/contextEditingTelemetry.ts"; import { recordCompressionCacheStats } from "./chatCore/compressionCacheStats.ts"; import { writeCavemanOutputAnalytics } from "./chatCore/cavemanOutputAnalytics.ts"; @@ -294,34 +260,16 @@ import { recordNonStreamingUsageStats } from "./chatCore/nonStreamingUsageStats. import { normalizeExecutorResult } from "./chatCore/upstreamTimeouts.ts"; import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; -import { assertExclusiveConnectionLeaseFence } from "@/lib/db/exclusiveConnectionLeases"; import { getCacheControlSettings } from "@/lib/cacheControlSettings"; import { guardrailRegistry } from "@/lib/guardrails"; import type { VideoBridgeLogRedactionEntry } from "@/lib/guardrails/videoBridge"; -import { - logClientRawRequestRedacted, - redactPendingBody, -} from "@/lib/guardrails/videoBridgeSnapshotRedaction"; import { shouldPreserveCacheControl, resolveConnectionCacheOverride, } from "../utils/cacheControlPolicy.ts"; -import { getCachedSettings } from "@/lib/db/readCache"; -import { applyCodexGlobalFastServiceTier } from "@/lib/providers/codexFastTier"; -import { buildUpstreamHeadersForExecute as buildUpstreamHeadersForExecuteFor } from "./chatCore/upstreamExecuteHeaders.ts"; -import { - resolveEffectiveServiceTier as resolveEffectiveServiceTierFor, - resolveReportedServiceTier as resolveReportedServiceTierFor, - type EffectiveServiceTier, -} from "./chatCore/serviceTier.ts"; -import { isCompactResponsesEndpoint } from "../executors/codex.ts"; import { extractUsageFromResponse } from "./usageExtractor.ts"; -import { - updateFromHeaders, - updateFromResponseBody, - initializeRateLimits, -} from "../services/rateLimitManager.ts"; +import { updateFromHeaders, updateFromResponseBody } from "../services/rateLimitManager.ts"; import * as localLimiterErrors from "../services/rateLimitManager/errors.ts"; import { markBlocked as markAccountSemaphoreBlocked } from "../services/accountSemaphore.ts"; import { @@ -344,18 +292,12 @@ import { getComboTargetTokenLimit, resolveComboContextLimit, } from "../services/contextManager.ts"; -import { resolveBackgroundTaskRedirect } from "./chatCore/backgroundRedirect.ts"; import type { CompressionConfig, CompressionPipelineStep, CompressionResult, } from "../services/compression/types.ts"; import { generateSessionId } from "../services/sessionManager.ts"; -import { prepareWebSearchFallbackBody } from "../services/webSearchFallback.ts"; -import { prepareWebFetchFallbackBody } from "../services/webFetchInterception.ts"; -import { resolveInterceptSearch, resolveInterceptFetch } from "@/lib/db/interceptionRules"; -import { resolveExplicitStreamAlias, resolveStreamFlag } from "../utils/aiSdkCompat.ts"; -import { generateRequestId } from "@/shared/utils/requestId"; import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; @@ -367,8 +309,7 @@ import { buildClaudeCodeCompatibleRequest, resolveClaudeCodeCompatibleSessionId, } from "../services/claudeCodeCompatible.ts"; -import { setGeminiThoughtSignatureMode } from "../services/geminiThoughtSignatureStore.ts"; -import { classifyModelScope429, isModelScopeProvider } from "../services/modelscopePolicy.ts"; +import { classifyModelScope429 } from "../services/modelscopePolicy.ts"; import { incrementTokenUsage, isTpmExhausted } from "../services/geminiRateLimitTracker.ts"; import { computeRequestHash, shouldDeduplicate } from "../services/requestDedup.ts"; import { getProactiveCompressionRatio } from "@/lib/db/compression"; @@ -380,8 +321,6 @@ import { getProactiveCompressionRatio } from "@/lib/db/compression"; * alias is applied via a local cast at each read site instead of widening * the whole destructure to a typed object. */ -type VideoBridgeLogParam = { observed: boolean; redaction: VideoBridgeLogRedactionEntry[] } | null; - /** * Core chat handler - shared between SSE and Worker * Returns { success, response, status, error } for caller to handle fallback @@ -443,183 +382,105 @@ export async function handleChatCore({ videoBridgeLog = undefined, fallbackAttempts = undefined, }) { - const { - model: originModel, - resolvedThinkingEffort, - defaultThinkingEffort, - } = modelInfo as typeof modelInfo & { - resolvedThinkingEffort?: string | null; - defaultThinkingEffort?: string | null; - }; - const trustedEffortContext = Object.freeze({ - originModel, - resolvedThinkingEffort, - defaultThinkingEffort, - }); - let { provider, model, extendedContext } = modelInfo; - // Keep the selected rule across format conversion, retries and refreshed credentials. - // Each combo leg gets its own execution context; nothing is written to shared accounts. - const reasoningRuleDirective = body?._omnirouteReasoningRule; - // #12150 P1b: true iff the video-bridge guardrail rendered >=1 transcript - // cue into a replaced part of this request. Gates both request- and - // response-derived Memory extraction - // (chatCore/memoryExtraction.ts::runMemoryExtractionGate). - const videoBridgeObserved: boolean = - (videoBridgeLog as VideoBridgeLogParam | undefined)?.observed === true; - const resilienceSettings = resolveResilienceSettings(cachedSettings); - if (!skipResourcePressureGuard) { - try { - const pressureGuard = checkResourcePressureGuard(); - if (pressureGuard) return pressureGuard; - } catch { - /* fail open */ - } - } - // Per-request model-routing metadata (first extracted slice of the request-setup phase). - const { apiFormat, customModelTargetFormat, requestedModel } = resolveChatCoreRequestSetup( - modelInfo, + const prelude = await runRequestPrelude({ body, - model - ); - const isModelScope = () => isModelScopeProvider(provider, credentials?.providerSpecificData); - const startTime = Date.now(); - // Per-request trace id + checkpoint helper. Lets us see exactly which await - // a hung request was sitting on in `[STAGE_TRACE]` log lines. Uses crypto RNG - // (not Math.random) purely to satisfy CodeQL js/insecure-randomness — this id - // is a log-correlation token, not a security secret. - const traceId = globalThis.crypto.randomUUID().slice(0, 6); - // Emit request.started event for real-time dashboard - setImmediate(() => { - emit("request.started", { - id: traceId, - model: model || "unknown", - provider: provider || "unknown", - timestamp: startTime, - comboName: comboName || undefined, - }); - }); - const traceEnabled = process.env.OMNIROUTE_TRACE === "true" || process.env.DEBUG === "true"; - // Stage trace extracted to chatCore/stageTrace.ts (#3501); bind the per-request inputs once so the - // call sites stay byte-identical. - const trace = (label: string, extra?: Record) => - stageTrace(label, extra, { traceEnabled, startTime, traceId, log }); - const getCurrentConnectionId = () => { - const credentialConnectionId = - typeof credentials?.connectionId === "string" && credentials.connectionId.trim().length > 0 - ? credentials.connectionId.trim() - : null; - return credentialConnectionId || connectionId || null; - }; - const assertManagedLeaseFence = (attemptConnectionId: string | null | undefined) => { - if (!managedLease) return; - if (!attemptConnectionId) { - throw Object.assign(new Error("Managed lease connection is unavailable"), { - code: "LEASE_CONNECTION_MISMATCH", - status: 409, - }); - } - const fence = assertExclusiveConnectionLeaseFence({ - leaseOwnerId: managedLease.context.leaseOwnerId, - generation: managedLease.context.generation, - apiKeyId: managedLease.apiKeyId, - connectionId: attemptConnectionId, - }); - if (fence.kind === "VALID") return; - const code = - fence.kind === "REQUIRED" - ? "LEASE_REQUIRED" - : fence.kind === "STALE" - ? "LEASE_FENCE_STALE" - : fence.kind === "AUTHORIZATION_MISMATCH" - ? "LEASE_AUTHORIZATION_MISMATCH" - : "LEASE_CONNECTION_MISMATCH"; - throw Object.assign(new Error("Managed lease request fence rejected the dispatch"), { - code, - status: 409, - }); - }; - const getManagedLeaseFenceErrorCode = (code: string | undefined): string | undefined => { - if (managedLease === null) return undefined; - return code?.startsWith("LEASE_") ? code : undefined; - }; - const managedLeaseFenceErrorResult = (code: string) => { - return { - ...createErrorResult(409, "Managed lease request fence rejected the dispatch", null, code), - errorType: "lease_error", - errorCode: code, - }; - }; - let tokensCompressed: number | null = null; - // ── Per-endpoint custom system prompt (port of upstream #2063) ── - // Reads from cachedSettings if available (passed in from combo/chat layer) - // to avoid an extra DB read on the hot path. Falls through to getCachedSettings() - // only when this function is called outside the normal chat dispatch. - { - const _s = cachedSettings ?? (await getCachedSettings()); - if ( - _s.customSystemPromptEnabled === true && - typeof _s.customSystemPrompt === "string" && - _s.customSystemPrompt - ) { - body = injectCustomSystemPrompt(body as Record, _s.customSystemPrompt); - log?.debug?.("CUSTOMSP", "custom system prompt injected"); - } - } - // ── Plugin onRequest hook ── - // Dynamic import cached by Node.js after first call — minimal overhead - const pluginGate = await runPluginOnRequestHook({ - requestId: traceId, - body, - model, - provider, - apiKeyInfo, - headers: clientRawRequest?.headers, + modelInfo, + credentials, log, + clientRawRequest, + connectionId, + apiKeyInfo, + userAgent, + comboName, + comboStrategy, + isCombo, + sessionAffinityKey, + comboStepId, + comboExecutionKey, + cachedSettings, + correlationId, + conversationId, + modelPinned, + skipResourcePressureGuard, + managedLease, + videoBridgeLog, }); - if (pluginGate.blocked === true) { - return { - success: false, - status: 403, - // Label the source: this 403 is our own policy decision, not the provider - // rejecting us. Unlabelled, it is indistinguishable from a real upstream 403 - // and gets the connection banned. Matches the type already sent to the client - // in pluginOnRequest.ts. - errorType: "plugin_block", - errorCode: "plugin_block", - error: "Request blocked by plugin", - response: pluginGate.response, - }; - } - if (pluginGate.body) { - body = pluginGate.body; - } - // Per-API-key device/connection tracking (port of upstream 9router#931, - // thanks @mugnimaestra). In-memory only, never blocks the request path. - if (apiKeyInfo?.id) { - trackDevice( - apiKeyInfo.id, - extractIpFromHeaders(clientRawRequest?.headers ?? null), - userAgent ?? null - ); - } - const agentGoalPolicy = resolveAgentGoalPolicy(body, clientRawRequest?.headers ?? null); - if (agentGoalPolicy.detected) { - log?.debug?.( - "AGENT_GOAL", - `long-running goal mode enabled: readinessMax=${agentGoalPolicy.readinessMaxTimeoutMs}ms streamRecovery=${agentGoalPolicy.streamRecoveryEnabled}` - ); - } - let effectiveServiceTier: EffectiveServiceTier = "standard"; - // Codex service-tier resolvers extracted to chatCore/serviceTier.ts (#3501); bind the per-request - // provider/credentials once and delegate so the existing call sites stay byte-identical. - const resolveEffectiveServiceTier = (requestBody?: unknown): EffectiveServiceTier => - resolveEffectiveServiceTierFor(provider, credentials?.providerSpecificData, requestBody); - const resolveReportedServiceTier = ( - payload?: unknown, - maxDepth = 3 - ): EffectiveServiceTier | null => resolveReportedServiceTierFor(provider, payload, maxDepth); - // Failure usage record building extracted to chatCore/failureUsage.ts (#3501); the handler keeps - // the fire-and-forget save + computes latencyMs, so the call sites stay byte-identical. + if (prelude.kind === "return") return prelude.value; + const c1 = prelude.continue; + body = c1.body; + credentials = c1.credentials; + let { tokensCompressed, effectiveServiceTier, compressionAnalyticsWritePromise } = c1; + const { + provider, + model, + extendedContext, + videoBridgeObserved, + resilienceSettings, + requestedModel, + isModelScope, + startTime, + traceId, + traceEnabled, + trace, + getCurrentConnectionId, + assertManagedLeaseFence, + getManagedLeaseFenceErrorCode, + managedLeaseFenceErrorResult, + agentGoalPolicy, + resolveEffectiveServiceTier, + resolveReportedServiceTier, + recordKeyHealthStatus, + idempotencyKey, + endpointPath, + sourceFormat, + isResponsesEndpoint, + nativeCodexPassthrough, + nativeXaiResponsesPassthrough, + isDroidCLI, + isOpencodeClient, + copilotCompatibleReasoning, + clientResponseFormat, + nativeOpenAICompatibleResponsesPassthrough, + customToolNames, + backgroundReason, + effectiveModel, + alias, + targetFormat, + nativeResponsesPassthrough, + pendingConnId, + pendingRequestId, + preConversionClientToolNames, + webSearchFallbackPlan, + clientRequestedResponsesStream, + webFetchFallbackPlan, + settings, + isCodexResponsesEcho, + echoModel, + skillRequestId, + pipelineSessionId, + reasoningCacheScope, + persistAttemptLogs: persistAttemptLogsPrelude, + detailedLoggingEnabled, + noLogEnabled, + explicitSessionIdHeader, + buildUpstreamHeadersForExecute, + streamUserAgent, + thinkingMarkerHeader, + providerRequiresStreaming, + stream, + semanticCacheEnabled, + reqLogger, + pendingScope, + providerRequestCapture, + bodyForCacheWrite, + trustedEffortContext, + reasoningRuleDirective, + } = c1; + // Normalized OpenAI transcript the reasoning replay pass digested for a + // Responses-API target (reported by translateRequest). A Responses body has + // `input`, not `messages`, so the replay-cache write side would otherwise digest + // an empty history and never match the read side for plain assistant turns. + let reasoningReplayHistory: unknown[] | null = null; const persistFailureUsage = ( statusCode: number, errorCode?: string | null, @@ -642,45 +503,14 @@ export async function handleChatCore({ }) ).catch(() => {}); }; - // Key-health updater extracted to chatCore/keyHealth.ts (#3501); bind the per-request log once - // and delegate so the existing call sites stay byte-identical. - const recordKeyHealthStatus = ( - status: number, - creds: Record | null | undefined, - transport?: string, - failureDetail?: string - ): void => recordKeyHealthStatusFor(status, creds, log, transport, failureDetail); - // Endpoint/format resolution extracted to chatCore/requestFormat.ts (#3501); pure derivation - // from the request. OUTSIDE the try below — persistFailureUsage closes over endpointPath. - const { - endpointPath, - sourceFormat, - isResponsesEndpoint, - nativeCodexPassthrough, - nativeXaiResponsesPassthrough, - isDroidCLI, - isOpencodeClient, - copilotCompatibleReasoning, - clientResponseFormat, - } = resolveChatCoreRequestFormat({ clientRawRequest, body, provider, userAgent }); - // ── Phase 9.2: Idempotency check ── - // Resolve the idempotency key once here and reuse it at the Phase 9.2 save site below, - // rather than re-deriving it. (#3821-review LEDGER-6) - const { hit: idempotencyHit, idempotencyKey } = await checkIdempotencyCache({ - clientRawRequest, - provider, - model, - // NEXA fusion-idempotency fix: body.messages feeds the key digest so combo-internal - // sub-requests (fusion panel + judge re-enter chatCore sharing the client's headers) - // can never collide on the raw Idempotency-Key/x-request-id header key. - body, - effectiveServiceTier, - startTime, - log, - }); - if (idempotencyHit) { - return idempotencyHit; - } + const attachCompressionUsageReceiptAfterAnalytics = ( + usage: Record, + source: "provider" | "estimated" | "stream" + ) => + attachCompressionUsageReceiptAfterAnalyticsFor(usage, source, { + pendingWrite: compressionAnalyticsWritePromise, + skillRequestId, + }); const turnExecution = acquireTurnExecution(idempotencyKey); if (turnExecution.acquired === false) { @@ -697,526 +527,6 @@ export async function handleChatCore({ // Preserve chatCore's canonical formatting while the guarded body remains byte-stable. // prettier-ignore try { - // T07: Inject connectionId into credentials so executors can rotate API keys - // using providerSpecificData.extraApiKeys (API Key Round-Robin feature) - if (connectionId && credentials && !credentials.connectionId) { - credentials.connectionId = connectionId; - } - let clientRequestedResponsesStream = false; - const nativeOpenAICompatibleResponsesPassthrough = - shouldUseNativeOpenAICompatibleResponsesPassthrough({ - provider, - sourceFormat, - endpointPath, - providerSpecificData: credentials?.providerSpecificData, - body, - }); - const responsesInputItems = Array.isArray(body?.input) ? body.input : []; - const customToolNames = collectCustomToolNamesForSourceFormat( - sourceFormat, - FORMATS.OPENAI_RESPONSES, - body?.tools, - responsesInputItems - ); - - const requestedLifecycleError = checkLifecycle(provider, model, log); - if (requestedLifecycleError) return requestedLifecycleError; - - // Check for bypass patterns (warmup, skip) - return fake response - const bypassResponse = handleBypassRequest(body, model, userAgent); - if (bypassResponse) { - return bypassResponse; - } - - // ── Claude Code auto-mode classifier compat (opt-in, default "off") ── - // Claude Code's `--permission-mode auto` sends an internal classifier request that - // requires the response to START with `no`/`yes`. - // When a combo/fallback route sends that call to a cheap model returning 200 with - // empty content, Claude Code fails closed on every gated action. Detect the - // classifier request and short-circuit with a synthetic ALLOW response, WITHOUT - // calling the upstream provider. See chatCore/claudeClassifierCompat.ts. - { - const classifierSettings = cachedSettings ?? (await getCachedSettings()); - if ( - shouldDefaultAllowClassifier( - sourceFormat, - body as Record, - classifierSettings.claudeClassifierCompat as string | undefined - ) - ) { - const classifierFormat = detectClassifierFormat(body as Record); - log?.warn?.( - "CHAT", - `classifier compat=${classifierSettings.claudeClassifierCompat} format=${classifierFormat} | short-circuit default-allow` - ); - return buildDefaultAllowClaudeMessage(requestedModel, classifierFormat); - } - } - - // Detect source format and get target format - // Model-specific targetFormat takes priority over provider default - - // ── Background Task Redirection (T41) — decision extracted to chatCore/backgroundRedirect.ts (#3501) - // backgroundReason is the detection signal (threaded into memory/skills injection below); redirect - // is the actual model downgrade to apply, if any. - const { backgroundReason, redirect: bgRedirect } = resolveBackgroundTaskRedirect({ - body, - headers: clientRawRequest?.headers, - model, - }); - if (bgRedirect) { - const originalModel = model; - log?.info?.( - "BACKGROUND", - `Background task redirect (${bgRedirect.reason}): ${originalModel} → ${bgRedirect.degradedModel}` - ); - model = bgRedirect.degradedModel; - if (body && typeof body === "object") { - body.model = model; - } - - logAuditEvent({ - action: "routing.background_task_redirect", - actor: apiKeyInfo?.name || "system", - target: connectionId || provider || "chat", - details: { - original_model: originalModel, - redirected_to: bgRedirect.degradedModel, - reason: bgRedirect.reason, - }, - }); - } - - // Custom aliases remain explicit; lifecycle replacements are advisory and never silently routed. - let [resolvedModel, effectiveModel, routedLifecycleError] = resolveLifecycle( - provider, - model, - log - ); - if (routedLifecycleError) return routedLifecycleError; - - // Effort-variant model ids: the Claude / Claude-Code model picker (e.g. VS Code's - // "Effort" slider) advertises claude-...-{low,medium,high,xhigh,max}. Anthropic has - // no such model, so the suffixed id 404s upstream. Strip it back to the real base id - // (forwarded as the upstream model via finalModelToUpstream below) and surface the - // level as reasoning_effort so the OpenAI→Claude translator / Claude-Code bridge turn - // it into Claude thinking/effort config. An explicit client-supplied effort always - // wins; native Claude passthrough is left untouched (it carries its own `thinking`), - // and non-thinking base models are cleaned up later by normalizeThinkingForModel(). - // Extracted to chatCore/claudeEffortVariant.ts (#3501); mutates body in place and returns the - // stripped model + an optional log line. The strip is unconditional (byte-identical to the - // original behavior) for the claude/Claude-Code-compatible lane; for any other provider it - // additionally requires isKnownClaudeEffortBaseModel(baseModel) to verify the base id is a - // real, effort-capable Claude model before stripping (vertex-claude-catalog-dispatch fix). - { - const effortVariant = applyClaudeEffortVariant({ - provider, - effectiveModel, - body, - sourceFormat, - }); - effectiveModel = effortVariant.effectiveModel; - if (effortVariant.log) { - log?.info?.("PARAMS", effortVariant.log); - } - } - - // Wire target-format resolution extracted to chatCore/targetFormat.ts (#3501); `alias` is reused - // downstream when stripping the alias/ prefix off the upstream model id. - const { alias, targetFormat } = resolveChatCoreTargetFormat({ - provider, - resolvedModel, - apiFormat, - sourceFormat, - customModelTargetFormat, - providerSpecificData: credentials?.providerSpecificData, - nativeXaiResponsesPassthrough, - nativeOpenAICompatibleResponsesPassthrough, - }); - const nativeResponsesPassthrough = - nativeCodexPassthrough || - nativeXaiResponsesPassthrough || - nativeOpenAICompatibleResponsesPassthrough; - - const initialProviderRequest = - body && typeof body === "object" && !Array.isArray(body) - ? { - ...(body as Record), - model: - typeof (body as Record).model === "string" - ? (body as Record).model - : effectiveModel, - } - : body; - - // Track pending requests before slower optional enrichment (settings, logging, - // compression) so internal usage/runtime counters stay accurate even when - // upstream never returns response headers. - // Use credentials.connectionId as a fallback so that requests without an - // explicit session-level connectionId still register in the pendingRequests map. - const pendingConnId = connectionId || credentials?.connectionId || null; - const pendingRequestId = - trackPendingRequest(model, provider, pendingConnId, true, { - clientEndpoint: clientRawRequest?.endpoint || "/v1/chat/completions", - clientRequest: redactPendingBody(clientRawRequest?.body ?? body, videoBridgeObserved), - providerRequest: initialProviderRequest, - stage: "registered", - correlationId, - sessionTag: conversationId || null, - }) || generateRequestId(); - - // Initialize rate limit settings from persisted DB (once, lazy) - await initializeRateLimits(); - - // #3384: per-model interception rule (src/lib/db/interceptionRules.ts) overrides the - // native-bypass defaults below when the operator explicitly configured it for this - // provider/model pair; undefined falls through to the existing bypass logic. - const interceptSearchOverride = resolveInterceptSearch(provider, effectiveModel); - - // Capture client tool names BEFORE fallback injection so the owner-provenance - // merge can distinguish tools the client already declared from synthetic tools - // added by the fallback preparer. Without this, a client function named - // `omniroute_web_search` (colliding with the fallback tool name) would be - // marked server-owned even though the client owns it. - const preConversionClientToolNames: string[] = ( - Array.isArray((body as Record).tools) - ? ((body as Record).tools as unknown[]) - : [] - ) - .map((tool) => { - if (!tool || typeof tool !== "object") return ""; - const record = tool as Record; - if (typeof record.name === "string") return record.name; - const fn = record.function; - if ( - fn && - typeof fn === "object" && - typeof (fn as Record).name === "string" - ) { - return (fn as Record).name as string; - } - return ""; - }) - .filter(Boolean); - - const { body: bodyWithWebSearchFallback, fallback: webSearchFallbackPlan } = - prepareWebSearchFallbackBody(body as Record, { - provider, - sourceFormat, - targetFormat, - nativeCodexPassthrough: nativeResponsesPassthrough, - interceptSearchOverride, - }); - if (webSearchFallbackPlan.enabled) { - body = bodyWithWebSearchFallback as typeof body; - // Server-side web-search execution cannot be injected into an arbitrary - // client SSE stream (streaming interception is not implemented — #9725), so - // a stream:true OpenAI Responses request whose web_search tool was converted - // to the fallback is executed non-streaming: the assembled response then - // carries the executed results (function_call_output + web_search_call) and - // JSON-tolerating Responses clients (pi-web-access) consume it directly. - if ( - sourceFormat === FORMATS.OPENAI_RESPONSES && - (body as Record).stream === true - ) { - clientRequestedResponsesStream = true; - (body as Record).stream = false; - log?.info?.("TOOLS", `web_search fallback forced non-streaming response for ${provider}`); - } - log?.info?.( - "TOOLS", - `Converted ${webSearchFallbackPlan.convertedToolCount} web_search tool(s) to OmniRoute fallback for ${provider}` - ); - } - // #7339: interceptFetch (Phase 3-4 of #3384) — same per-model rule + native-bypass - // pattern as interceptSearch directly above. - const interceptFetchOverride = resolveInterceptFetch(provider, effectiveModel); - const { body: bodyWithWebFetchFallback, fallback: webFetchFallbackPlan } = - prepareWebFetchFallbackBody(body as Record, { - provider, - sourceFormat, - targetFormat, - nativeCodexPassthrough: nativeResponsesPassthrough, - interceptFetchOverride, - }); - if (webFetchFallbackPlan.enabled) { - body = bodyWithWebFetchFallback as typeof body; - log?.info?.( - "TOOLS", - `Converted ${webFetchFallbackPlan.convertedToolCount} web_fetch tool(s) to OmniRoute fallback for ${provider}` - ); - } - const noLogEnabled = apiKeyInfo?.noLog === true; - // Consolidate settings reads — fetch once, reuse throughout the request - const settings = cachedSettings ?? (await getCachedSettings()); - // Opt-in tool-source diagnostics (#1825): summarize the request's tool definitions - // (count + MCP/hosted/client source breakdown + first names) as a single debug line. - if (settings.logToolSources === true) { - const toolSummary = summarizeToolSources((body as { tools?: unknown }).tools); - if (toolSummary) log?.debug?.("TOOLS", toolSummary); - } - // #1311 (opt-in): echo the client-requested alias/combo name in the response `model` - // field instead of the upstream model, so strict clients (Claude Desktop) that validate - // response.model === request.model stop rejecting alias/combo requests with a 401. - // #3697: always echo it for Codex CLI clients on the Responses API — regardless of the - // opt-in setting — since the Codex CLI status line/model button reads `response.model` - // to display the active model + reasoning effort (e.g. `gpt-5.5-xhigh`). Detection is by - // request headers (originator/User-Agent), not by the routed provider, so it still fires - // when `codex/gpt-5.5-xhigh` is routed through a combo to a non-codex upstream. - const isCodexResponsesEcho = - (isResponsesEndpoint || sourceFormat === FORMATS.OPENAI_RESPONSES) && - isCodexOriginatedHeaders(clientRawRequest?.headers); - - // Detect Claude Code CLI so we can auto-enable model echo — this prevents - // session restore failures when the resolved upstream model (e.g. - // `oc/nemotron-3-ultra-free`) is not recognized by the client on `--resume`. - const isClaudeCodeClient = isClaudeCodeOriginatedHeaders(clientRawRequest?.headers); - - let echoModel = - (settings.echoRequestedModelName === true || isCodexResponsesEcho || isClaudeCodeClient) && - typeof requestedModel === "string" && - requestedModel - ? requestedModel - : null; - // Auto-echo the listing-valid form for bare requests to noAuth catalog - // providers so clients validating response.model against /v1/models don't warn. - echoModel = resolveNoAuthEchoModel(requestedModel, provider) ?? echoModel; - const detailedLoggingEnabled = - !noLogEnabled && - (settings.call_log_pipeline_enabled === true || - settings.call_log_pipeline_enabled === "1" || - settings.call_log_pipeline_enabled === "true"); - const capturePipelineStreamChunks = - detailedLoggingEnabled && getCallLogPipelineCaptureStreamChunks(); - const skillRequestId = generateRequestId(); - let compressionAnalyticsWritePromise: Promise | null = null; - // Compression usage-receipt attachment extracted to chatCore/compressionUsageReceipt.ts (#3501); - // pass the in-flight analytics write + request id so behaviour stays byte-identical. - const attachCompressionUsageReceiptAfterAnalytics = ( - usage: Record, - source: "provider" | "estimated" | "stream" - ) => - attachCompressionUsageReceiptAfterAnalyticsFor(usage, source, { - pendingWrite: compressionAnalyticsWritePromise, - skillRequestId, - }); - // #8249: raw header value, kept separate from `pipelineSessionId`'s skillRequestId fallback - // below so call_logs.session_tag is only ever set when the caller explicitly supplied the - // header — never synthesized from the internal per-request skillRequestId. - const explicitSessionIdHeader = - (clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" - ? clientRawRequest.headers.get("x-omniroute-session-id") - : getHeaderValueCaseInsensitive( - clientRawRequest?.headers ?? null, - "x-omniroute-session-id" - )) || null; - const pipelineSessionId = explicitSessionIdHeader || skillRequestId; - const reasoningReplaySessionKey = sessionAffinityKey || explicitSessionIdHeader; - const reasoningCacheScope = reasoningReplaySessionKey - ? `api-key:${String(apiKeyInfo?.id ?? "local")}\x1f${String(reasoningReplaySessionKey)}` - : null; - // Normalized OpenAI transcript the reasoning replay pass digested for a - // Responses-API target (reported by translateRequest). A Responses body has - // `input`, not `messages`, so the replay-cache write side would otherwise digest - // an empty history and never match the read side for plain assistant turns. - let reasoningReplayHistory: unknown[] | null = null; - // persistAttemptLogs extracted to chatCore/attemptLogging.ts (#3501); bind the per-request context - // once so the 16 call sites keep passing only the per-attempt args (byte-identical). - const persistAttemptLogs = (args: PersistAttemptLogsArgs) => - persistAttemptLogsFor(args, { - traceId, - provider, - connectionId, - model, - skillRequestId, - detailedLoggingEnabled, - reqLogger, - pendingRequestId, - clientRawRequest, - requestedModel, - credentials, - startTime, - body, - sourceFormat, - targetFormat, - comboName, - comboStepId, - comboExecutionKey, - tokensCompressed, - apiKeyInfo, - noLogEnabled, - correlationId, - modelPinned, - // Resolved conversationId (open-sse/services/conversationTracker.ts) wins when - // present — it's populated for every request now, not just ones where the - // client explicitly sent x-omniroute-session-id. The raw header remains a - // fallback for any caller that somehow bypassed conversationId resolution. - sessionTag: conversationId || explicitSessionIdHeader, - // #12150 P1b surface 1: undefined for every non-video request (byte-identical - // to before this param existed) — see applyVideoBridgeLogRedaction. - videoBridgeLogRedaction: (videoBridgeLog as VideoBridgeLogParam | undefined)?.redaction, - // #12150 P2 surface 2: mark the persisted call_logs row so - // resolvePreviousResponseState refuses to rehydrate a snapshot whose video - // transcript was redacted. false for every non-video request. - videoContentRemoved: videoBridgeObserved, - }); - - // Primary path: merge client model id + alias target so config on either key applies; resolved - // id wins on same header name. T5 family fallback uses only (nextModel, resolveModelAlias(next)) - // so A-model headers are not sent to B — see buildUpstreamHeadersForExecute. - const connectionCustomUserAgent = - credentials?.providerSpecificData && - typeof credentials.providerSpecificData === "object" && - typeof credentials.providerSpecificData.customUserAgent === "string" - ? credentials.providerSpecificData.customUserAgent.trim() - : ""; - - // #8369: connection-level custom upstream headers from provider_specific_data. - const connectionCustomHeaders = - credentials?.providerSpecificData && - typeof credentials.providerSpecificData === "object" && - typeof credentials.providerSpecificData.customHeaders === "object" && - !Array.isArray(credentials.providerSpecificData.customHeaders) - ? (credentials.providerSpecificData.customHeaders as Record) - : undefined; - - // Upstream extra-header building extracted to chatCore/upstreamExecuteHeaders.ts (#3501); bind the - // per-request inputs once and delegate so the existing call sites stay byte-identical. - const buildUpstreamHeadersForExecute = (modelToCall: string): Record => - buildUpstreamHeadersForExecuteFor({ - modelToCall, - effectiveModel, - provider, - model, - resolvedModel, - sourceFormat, - connectionCustomUserAgent, - connectionCustomHeaders, - settings, - }); - - // Default to false unless client explicitly sets stream: true (OpenAI spec compliant) - const acceptHeader = - clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" - ? clientRawRequest.headers.get("accept") || clientRawRequest.headers.get("Accept") - : clientRawRequest?.headers?.["accept"] || clientRawRequest?.headers?.["Accept"]; - const streamUserAgent = [ - typeof userAgent === "string" ? userAgent : "", - getHeaderValueCaseInsensitive(clientRawRequest?.headers ?? null, "user-agent") || "", - ] - .filter(Boolean) - .join(" "); - - // Explicit per-request opt-in/out for the `` close marker - // (#5312 / #5245): `x-omniroute-thinking-marker: off` suppresses it for - // reasoning_content-native clients (e.g. Cursor's OpenAI path) that the UA - // allowlist does not cover; absent the header, the UA policy applies. - const thinkingMarkerHeader = getHeaderValueCaseInsensitive( - clientRawRequest?.headers ?? null, - THINKING_MARKER_HEADER - ); - - const explicitStreamAlias = resolveExplicitStreamAlias(body); - - // Remove non-standard non-stream aliases before provider translation/execution. - // They are accepted for compatibility at the OmniRoute API boundary only. - if (body && typeof body === "object") { - const b = body as Record; - if (explicitStreamAlias !== undefined) { - b.stream = explicitStreamAlias; - } - - delete b.non_stream; - delete b.disable_stream; - delete b.disable_streaming; - delete b.streaming; - } - - // Codex /responses/compact is JSON-only: Codex CLI does not send stream=false, - // so route shape must override the usual Accept/header fallback. - // sourceFormat="claude" applies the Anthropic Messages spec default (stream=false - // when body omits stream), preventing STREAM_EARLY_EOF on /v1/messages when - // clients send Accept: */* without an explicit stream flag. - // providerRequiresStreaming: providers with forceStream:true (cline/clinepass) - // only implement upstream streaming — a non-streaming request returns - // "generateText is not implemented" / an empty body. This flag forces the - // UPSTREAM request to stream (see `upstreamStream` below), but it MUST NOT - // force the client-facing `stream` flag: a stream:false client (e.g. the - // model-test button, plain JSON API callers) still expects a JSON response. - // The client-side `if (!stream)` branch drains the forced upstream SSE and - // converts it back to JSON via readNonStreamingResponseBody. Passing this - // flag into resolveStreamFlag would force `stream=true` and skip that - // conversion, yielding STREAM_EARLY_EOF for JSON callers. (#2081, #6126) - const providerRequiresStreaming = REGISTRY[provider]?.forceStream === true; - const stream = - nativeCodexPassthrough && isCompactResponsesEndpoint(endpointPath) - ? false - : resolveStreamFlag(body?.stream, acceptHeader, sourceFormat, { - userAgent: streamUserAgent, - streamDefaultMode: apiKeyInfo?.streamDefaultMode, - }); - - // `settings` is already consolidated once near the top of handleChatCore - // (the "fetch once, reuse" const). A second `const settings` here was a - // duplicate same-scope declaration that broke the esbuild/tsx transform - // ("settings has already been declared") and the production build. Reuse it. - credentials = applyCodexGlobalFastServiceTier(provider, credentials, settings, { - model: requestedModel, - body: body && typeof body === "object" ? (body as Record) : null, - }); - effectiveServiceTier = resolveEffectiveServiceTier(body); - setGeminiThoughtSignatureMode(settings.antigravitySignatureCacheMode); - const semanticCacheEnabled = settings.semanticCacheEnabled !== false; - - const reqLogger = await createRequestLogger(sourceFormat, targetFormat, model, { - enabled: detailedLoggingEnabled, - captureStreamChunks: capturePipelineStreamChunks, - maxStreamChunkBytes: getCallLogPipelineMaxSizeBytes(), - requestId: pendingRequestId, - model, - provider: provider || undefined, - connectionId: connectionId || credentials?.connectionId || undefined, - }); - const pendingScope = { id: pendingRequestId, model, provider, connectionId: pendingConnId }; - const providerRequestCapture = createPreparedRequestLogger(reqLogger, pendingScope); - // 0. Log client raw request (before format conversion) — redacts video transcript - // cues in the logged copy only; see videoBridgeSnapshotRedaction.ts. - logClientRawRequestRedacted(reqLogger, clientRawRequest, videoBridgeObserved); - const reasoningRouteDecision = - body && typeof body === "object" - ? (body as Record)._omnirouteReasoningRouteTrace - : null; - if (reasoningRouteDecision) { - reqLogger.logRouteDecision(reasoningRouteDecision); - body = { ...(body as Record) }; - delete (body as Record)._omnirouteReasoningRouteTrace; - } - - log?.debug?.("FORMAT", `${sourceFormat} → ${targetFormat} | stream=${stream}`); - - if (reasoningRuleDirective) { - // Cache identity must use the effective effort, not the overridden client value. - // Retain the directive for the translation step, where general thinking defaults run. - body = { - ...(applyReasoningRuleDirective( - body, - sourceFormat === FORMATS.OPENAI_RESPONSES - ? "openai-responses" - : sourceFormat === FORMATS.CLAUDE - ? "claude" - : undefined - ) as Record), - _omnirouteReasoningRule: reasoningRuleDirective, - }; - } - - // Preserve original body for cache signature — the body variable is mutated - // multiple times below (sanitization, memory/skills injection) before the - // cache store path runs at Phase 9.1 (non-streaming) / Phase 9.2 (streaming). - // Without this snapshot, the write-time signature differs from the read-time - // one, producing 0% hit rate. (#cache-signature-asymmetry) - const bodyForCacheWrite = body; - // ── Phase 9.1: Semantic cache check (temp=0, any streaming mode) ── const cacheHit = await checkSemanticCache({ semanticCacheEnabled, @@ -1230,7 +540,7 @@ export async function handleChatCore({ pendingScope, startTime, log, - persistAttemptLogs, + persistAttemptLogs: persistAttemptLogsPrelude, apiKeyId: apiKeyInfo?.id ?? undefined, cacheDefaultMode: (apiKeyInfo as { cacheDefaultMode?: "legacy" | "bypass" } | null) ?.cacheDefaultMode, @@ -3005,6 +2315,38 @@ export async function handleChatCore({ ? computeRequestHash(dedupRequestBody, apiKeyInfo?.id, trustedEffortContext) : null; + const persistAttemptLogs = (args: PersistAttemptLogsArgs) => + persistAttemptLogsFor(args, { + traceId, + provider, + connectionId, + model, + skillRequestId, + detailedLoggingEnabled, + reqLogger, + pendingRequestId, + clientRawRequest, + requestedModel, + credentials, + startTime, + body: translatedBody, + sourceFormat, + targetFormat, + comboName, + comboStepId, + comboExecutionKey, + tokensCompressed, + apiKeyInfo, + noLogEnabled, + correlationId, + modelPinned, + sessionTag: conversationId || explicitSessionIdHeader, + videoBridgeLogRedaction: ( + videoBridgeLog as { redaction?: VideoBridgeLogRedactionEntry[] } | undefined + )?.redaction, + videoContentRemoved: videoBridgeObserved, + }); + const executeProviderRequest = (modelToCall = effectiveModel, allowDedup = false) => { const sendDeps = { agentGoalPolicy, diff --git a/open-sse/handlers/chatCore/requestPrelude.ts b/open-sse/handlers/chatCore/requestPrelude.ts new file mode 100644 index 000000000000..c859e6d669b5 --- /dev/null +++ b/open-sse/handlers/chatCore/requestPrelude.ts @@ -0,0 +1,945 @@ +/** + * Request front door lifted out of handleChatCore. + * Bypass, classifier, and lifecycle early-returns live here. + * Search-tool fallback stays with that front door. + */ +import { resolveChatCoreRequestSetup } from "./requestSetup.ts"; +import { checkIdempotencyCache } from "./idempotency.ts"; +import { checkLifecycle, resolveLifecycle } from "./modelLifecyclePolicy.ts"; +import { + shouldDefaultAllowClassifier, + detectClassifierFormat, + buildDefaultAllowClaudeMessage, +} from "./claudeClassifierCompat.ts"; +import { getHeaderValueCaseInsensitive } from "./headers.ts"; +import { + isCodexOriginatedHeaders, + isClaudeCodeOriginatedHeaders, +} from "../../config/codexIdentity.ts"; +import { trackDevice, extractIpFromHeaders } from "../../services/deviceTracker.ts"; +import { shouldUseNativeOpenAICompatibleResponsesPassthrough } from "./passthroughHelpers.ts"; +import { checkResourcePressureGuard } from "../../utils/resourcePressure.ts"; +import { resolveChatCoreRequestFormat } from "./requestFormat.ts"; +import { resolveChatCoreTargetFormat } from "./targetFormat.ts"; +import { injectCustomSystemPrompt } from "../../services/systemPrompt.ts"; +import { FORMATS } from "../../translator/formats.ts"; +import { collectCustomToolNamesForSourceFormat } from "../../translator/request/openai-responses/additionalTools.ts"; +import { THINKING_MARKER_HEADER } from "../../utils/thinkCloseMarker.ts"; +import { resolveAgentGoalPolicy } from "../../utils/agentGoalPolicy.ts"; +import { createRequestLogger } from "../../utils/requestLogger.ts"; +import { createPreparedRequestLogger } from "../../utils/providerRequestLogging.ts"; +import { summarizeToolSources } from "../../utils/toolSources.ts"; +import { applyClaudeEffortVariant } from "./claudeEffortVariant.ts"; +import { REGISTRY } from "../../config/providerRegistry.ts"; +import { resolveNoAuthEchoModel } from "./noAuthEchoModel.ts"; +import { createErrorResult } from "../../utils/error.ts"; +import { resolveResilienceSettings } from "@/lib/resilience/settings"; +import { recordKeyHealthStatus as recordKeyHealthStatusFor } from "./keyHealth.ts"; +import { + persistAttemptLogs as persistAttemptLogsFor, + type PersistAttemptLogsArgs, +} from "./attemptLogging.ts"; +import { stageTrace } from "./stageTrace.ts"; +import { + getCallLogPipelineCaptureStreamChunks, + getCallLogPipelineMaxSizeBytes, +} from "@/lib/logEnv"; +import { logAuditEvent } from "@/lib/compliance"; +import { emit } from "@/lib/events/eventBus"; +import { handleBypassRequest } from "../../utils/bypassHandler.ts"; +import { trackPendingRequest } from "@/lib/usageDb"; +import { runPluginOnRequestHook } from "./pluginOnRequest.ts"; +import { assertExclusiveConnectionLeaseFence } from "@/lib/db/exclusiveConnectionLeases"; +import { + logClientRawRequestRedacted, + redactPendingBody, +} from "@/lib/guardrails/videoBridgeSnapshotRedaction"; +import { getCachedSettings } from "@/lib/db/readCache"; +import { applyCodexGlobalFastServiceTier } from "@/lib/providers/codexFastTier"; +import { buildUpstreamHeadersForExecute as buildUpstreamHeadersForExecuteFor } from "./upstreamExecuteHeaders.ts"; +import { + resolveEffectiveServiceTier as resolveEffectiveServiceTierFor, + resolveReportedServiceTier as resolveReportedServiceTierFor, + type EffectiveServiceTier, +} from "./serviceTier.ts"; +import { isCompactResponsesEndpoint } from "../../executors/codex.ts"; +import { initializeRateLimits } from "../../services/rateLimitManager.ts"; +import { resolveBackgroundTaskRedirect } from "./backgroundRedirect.ts"; +import { prepareWebSearchFallbackBody } from "../../services/webSearchFallback.ts"; +import { prepareWebFetchFallbackBody } from "../../services/webFetchInterception.ts"; +import { resolveInterceptSearch, resolveInterceptFetch } from "@/lib/db/interceptionRules"; +import { resolveExplicitStreamAlias, resolveStreamFlag } from "../../utils/aiSdkCompat.ts"; +import { generateRequestId } from "@/shared/utils/requestId"; +import { setGeminiThoughtSignatureMode } from "../../services/geminiThoughtSignatureStore.ts"; +import { isModelScopeProvider } from "../../services/modelscopePolicy.ts"; +import type { VideoBridgeLogRedactionEntry } from "@/lib/guardrails/videoBridge"; + +type VideoBridgeLogParam = { observed: boolean; redaction: VideoBridgeLogRedactionEntry[] } | null; + +import { applyReasoningRuleDirective } from "@/lib/reasoningRouting/policy"; +export async function runRequestPrelude({ + body, + modelInfo, + credentials, + log, + clientRawRequest, + connectionId, + apiKeyInfo = null, + userAgent, + comboName, + comboStrategy = null, + isCombo = false, + sessionAffinityKey = null, + comboStepId = null, + comboExecutionKey = null, + cachedSettings = null, + correlationId = null, + conversationId = null, + modelPinned = false, + skipResourcePressureGuard = false, + managedLease = null, + videoBridgeLog = undefined, +}) { + const { + model: originModel, + resolvedThinkingEffort, + defaultThinkingEffort, + } = modelInfo as typeof modelInfo & { + resolvedThinkingEffort?: string | null; + defaultThinkingEffort?: string | null; + }; + const trustedEffortContext = Object.freeze({ + originModel, + resolvedThinkingEffort, + defaultThinkingEffort, + }); + let { provider, model, extendedContext } = modelInfo; + // Keep the selected rule across format conversion, retries and refreshed credentials. + // Each combo leg gets its own execution context; nothing is written to shared accounts. + const reasoningRuleDirective = body?._omnirouteReasoningRule; + // #12150 P1b: true iff the video-bridge guardrail rendered >=1 transcript + // cue into a replaced part of this request. Gates both request- and + // response-derived Memory extraction + // (chatCore/memoryExtraction.ts::runMemoryExtractionGate). + const videoBridgeObserved: boolean = + (videoBridgeLog as VideoBridgeLogParam | undefined)?.observed === true; + const resilienceSettings = resolveResilienceSettings(cachedSettings); + if (!skipResourcePressureGuard) { + try { + const pressureGuard = checkResourcePressureGuard(); + if (pressureGuard) return { kind: "return" as const, value: pressureGuard }; + } catch { + /* fail open */ + } + } + // Per-request model-routing metadata (first extracted slice of the request-setup phase). + const { apiFormat, customModelTargetFormat, requestedModel } = resolveChatCoreRequestSetup( + modelInfo, + body, + model + ); + const isModelScope = () => isModelScopeProvider(provider, credentials?.providerSpecificData); + const startTime = Date.now(); + // Per-request trace id + checkpoint helper. Lets us see exactly which await + // a hung request was sitting on in `[STAGE_TRACE]` log lines. Uses crypto RNG + // (not Math.random) purely to satisfy CodeQL js/insecure-randomness — this id + // is a log-correlation token, not a security secret. + const traceId = globalThis.crypto.randomUUID().slice(0, 6); + // Emit request.started event for real-time dashboard + setImmediate(() => { + emit("request.started", { + id: traceId, + model: model || "unknown", + provider: provider || "unknown", + timestamp: startTime, + comboName: comboName || undefined, + }); + }); + const traceEnabled = process.env.OMNIROUTE_TRACE === "true" || process.env.DEBUG === "true"; + // Stage trace extracted to chatCore/stageTrace.ts (#3501); bind the per-request inputs once so the + // call sites stay byte-identical. + const trace = (label: string, extra?: Record) => + stageTrace(label, extra, { traceEnabled, startTime, traceId, log }); + const getCurrentConnectionId = () => { + const credentialConnectionId = + typeof credentials?.connectionId === "string" && credentials.connectionId.trim().length > 0 + ? credentials.connectionId.trim() + : null; + return credentialConnectionId || connectionId || null; + }; + const assertManagedLeaseFence = (attemptConnectionId: string | null | undefined) => { + if (!managedLease) return; + if (!attemptConnectionId) { + throw Object.assign(new Error("Managed lease connection is unavailable"), { + code: "LEASE_CONNECTION_MISMATCH", + status: 409, + }); + } + const fence = assertExclusiveConnectionLeaseFence({ + leaseOwnerId: managedLease.context.leaseOwnerId, + generation: managedLease.context.generation, + apiKeyId: managedLease.apiKeyId, + connectionId: attemptConnectionId, + }); + if (fence.kind === "VALID") return; + const code = + fence.kind === "REQUIRED" + ? "LEASE_REQUIRED" + : fence.kind === "STALE" + ? "LEASE_FENCE_STALE" + : fence.kind === "AUTHORIZATION_MISMATCH" + ? "LEASE_AUTHORIZATION_MISMATCH" + : "LEASE_CONNECTION_MISMATCH"; + throw Object.assign(new Error("Managed lease request fence rejected the dispatch"), { + code, + status: 409, + }); + }; + const getManagedLeaseFenceErrorCode = (code: string | undefined): string | undefined => { + if (managedLease === null) return undefined; + return code?.startsWith("LEASE_") ? code : undefined; + }; + const managedLeaseFenceErrorResult = (code: string) => { + return { + ...createErrorResult(409, "Managed lease request fence rejected the dispatch", null, code), + errorType: "lease_error", + errorCode: code, + }; + }; + let tokensCompressed: number | null = null; + // ── Per-endpoint custom system prompt (port of upstream #2063) ── + // Reads from cachedSettings if available (passed in from combo/chat layer) + // to avoid an extra DB read on the hot path. Falls through to getCachedSettings() + // only when this function is called outside the normal chat dispatch. + { + const _s = cachedSettings ?? (await getCachedSettings()); + if ( + _s.customSystemPromptEnabled === true && + typeof _s.customSystemPrompt === "string" && + _s.customSystemPrompt + ) { + body = injectCustomSystemPrompt(body as Record, _s.customSystemPrompt); + log?.debug?.("CUSTOMSP", "custom system prompt injected"); + } + } + // ── Plugin onRequest hook ── + // Dynamic import cached by Node.js after first call — minimal overhead + const pluginGate = await runPluginOnRequestHook({ + requestId: traceId, + body, + model, + provider, + apiKeyInfo, + headers: clientRawRequest?.headers, + log, + }); + if (pluginGate.blocked === true) { + return { + kind: "return" as const, + value: { + success: false, + status: 403, + // Label the source: this 403 is our own policy decision, not the provider + // rejecting us. Unlabelled, it is indistinguishable from a real upstream 403 + // and gets the connection banned. Matches the type already sent to the client + // in pluginOnRequest.ts. + errorType: "plugin_block", + errorCode: "plugin_block", + error: "Request blocked by plugin", + response: pluginGate.response, + }, + }; + } + if (pluginGate.body) { + body = pluginGate.body; + } + // Per-API-key device/connection tracking (port of upstream 9router#931, + // thanks @mugnimaestra). In-memory only, never blocks the request path. + if (apiKeyInfo?.id) { + trackDevice( + apiKeyInfo.id, + extractIpFromHeaders(clientRawRequest?.headers ?? null), + userAgent ?? null + ); + } + const agentGoalPolicy = resolveAgentGoalPolicy(body, clientRawRequest?.headers ?? null); + if (agentGoalPolicy.detected) { + log?.debug?.( + "AGENT_GOAL", + `long-running goal mode enabled: readinessMax=${agentGoalPolicy.readinessMaxTimeoutMs}ms streamRecovery=${agentGoalPolicy.streamRecoveryEnabled}` + ); + } + let effectiveServiceTier: EffectiveServiceTier = "standard"; + // Codex service-tier resolvers extracted to chatCore/serviceTier.ts (#3501); bind the per-request + // provider/credentials once and delegate so the existing call sites stay byte-identical. + const resolveEffectiveServiceTier = (requestBody?: unknown): EffectiveServiceTier => + resolveEffectiveServiceTierFor(provider, credentials?.providerSpecificData, requestBody); + const resolveReportedServiceTier = ( + payload?: unknown, + maxDepth = 3 + ): EffectiveServiceTier | null => resolveReportedServiceTierFor(provider, payload, maxDepth); + // Key-health updater extracted to chatCore/keyHealth.ts (#3501); bind the per-request log once + // and delegate so the existing call sites stay byte-identical. + const recordKeyHealthStatus = ( + status: number, + creds: Record | null | undefined, + transport?: string, + failureDetail?: string + ): void => recordKeyHealthStatusFor(status, creds, log, transport, failureDetail); + // ── Phase 9.2: Idempotency check ── + // Resolve the idempotency key once here and reuse it at the Phase 9.2 save site below, + // rather than re-deriving it. (#3821-review LEDGER-6) + const { hit: idempotencyHit, idempotencyKey } = await checkIdempotencyCache({ + clientRawRequest, + provider, + model, + // NEXA fusion-idempotency fix: body.messages feeds the key digest so combo-internal + // sub-requests (fusion panel + judge re-enter chatCore sharing the client's headers) + // can never collide on the raw Idempotency-Key/x-request-id header key. + body, + effectiveServiceTier, + startTime, + log, + }); + if (idempotencyHit) { + return { kind: "return" as const, value: idempotencyHit }; + } + // T07: Inject connectionId into credentials so executors can rotate API keys + // using providerSpecificData.extraApiKeys (API Key Round-Robin feature) + if (connectionId && credentials && !credentials.connectionId) { + credentials.connectionId = connectionId; + } + // Endpoint/format resolution extracted to chatCore/requestFormat.ts (#3501); pure derivation + // from the inbound request, destructured so every downstream use stays byte-identical. + const { + endpointPath, + sourceFormat, + isResponsesEndpoint, + nativeCodexPassthrough, + nativeXaiResponsesPassthrough, + isDroidCLI, + isOpencodeClient, + copilotCompatibleReasoning, + clientResponseFormat, + } = resolveChatCoreRequestFormat({ clientRawRequest, body, provider, userAgent }); + + // Failure usage record building extracted to chatCore/failureUsage.ts (#3501); the handler keeps + // the fire-and-forget save + computes latencyMs, so the call sites stay byte-identical. + const persistFailureUsage = ( + statusCode: number, + errorCode?: string | null, + aggregate?: FailureUsageAggregate | null + ) => { + saveRequestUsage( + buildFailureUsageRecord({ + provider, + model, + connectionId: getCurrentConnectionId(), + apiKeyInfo, + effectiveServiceTier, + isCombo, + comboStrategy, + statusCode, + errorCode, + latencyMs: Date.now() - startTime, + endpoint: endpointPath, + aggregate: aggregate ?? undefined, + }) + ).catch(() => {}); + }; + + let clientRequestedResponsesStream = false; + const nativeOpenAICompatibleResponsesPassthrough = + shouldUseNativeOpenAICompatibleResponsesPassthrough({ + provider, + sourceFormat, + endpointPath, + providerSpecificData: credentials?.providerSpecificData, + body, + }); + const responsesInputItems = Array.isArray(body?.input) ? body.input : []; + const customToolNames = collectCustomToolNamesForSourceFormat( + sourceFormat, + FORMATS.OPENAI_RESPONSES, + body?.tools, + responsesInputItems + ); + + const requestedLifecycleError = checkLifecycle(provider, model, log); + if (requestedLifecycleError) return { kind: "return" as const, value: requestedLifecycleError }; + + // Check for bypass patterns (warmup, skip) - return fake response + const bypassResponse = handleBypassRequest(body, model, userAgent); + if (bypassResponse) { + return { kind: "return" as const, value: bypassResponse }; + } + + // ── Claude Code auto-mode classifier compat (opt-in, default "off") ── + // Claude Code's `--permission-mode auto` sends an internal classifier request that + // requires the response to START with `no`/`yes`. + // When a combo/fallback route sends that call to a cheap model returning 200 with + // empty content, Claude Code fails closed on every gated action. Detect the + // classifier request and short-circuit with a synthetic ALLOW response, WITHOUT + // calling the upstream provider. See chatCore/claudeClassifierCompat.ts. + { + const classifierSettings = cachedSettings ?? (await getCachedSettings()); + if ( + shouldDefaultAllowClassifier( + sourceFormat, + body as Record, + classifierSettings.claudeClassifierCompat as string | undefined + ) + ) { + const classifierFormat = detectClassifierFormat(body as Record); + log?.warn?.( + "CHAT", + `classifier compat=${classifierSettings.claudeClassifierCompat} format=${classifierFormat} | short-circuit default-allow` + ); + return { + kind: "return" as const, + value: buildDefaultAllowClaudeMessage(requestedModel, classifierFormat), + }; + } + } + + // Detect source format and get target format + // Model-specific targetFormat takes priority over provider default + + // ── Background Task Redirection (T41) — decision extracted to chatCore/backgroundRedirect.ts (#3501) + // backgroundReason is the detection signal (threaded into memory/skills injection below); redirect + // is the actual model downgrade to apply, if any. + const { backgroundReason, redirect: bgRedirect } = resolveBackgroundTaskRedirect({ + body, + headers: clientRawRequest?.headers, + model, + }); + if (bgRedirect) { + const originalModel = model; + log?.info?.( + "BACKGROUND", + `Background task redirect (${bgRedirect.reason}): ${originalModel} → ${bgRedirect.degradedModel}` + ); + model = bgRedirect.degradedModel; + if (body && typeof body === "object") { + body.model = model; + } + + logAuditEvent({ + action: "routing.background_task_redirect", + actor: apiKeyInfo?.name || "system", + target: connectionId || provider || "chat", + details: { + original_model: originalModel, + redirected_to: bgRedirect.degradedModel, + reason: bgRedirect.reason, + }, + }); + } + + // Custom aliases remain explicit; lifecycle replacements are advisory and never silently routed. + let [resolvedModel, effectiveModel, routedLifecycleError] = resolveLifecycle( + provider, + model, + log + ); + if (routedLifecycleError) return { kind: "return" as const, value: routedLifecycleError }; + + // Effort-variant model ids: the Claude / Claude-Code model picker (e.g. VS Code's + // "Effort" slider) advertises claude-...-{low,medium,high,xhigh,max}. Anthropic has + // no such model, so the suffixed id 404s upstream. Strip it back to the real base id + // (forwarded as the upstream model via finalModelToUpstream below) and surface the + // level as reasoning_effort so the OpenAI→Claude translator / Claude-Code bridge turn + // it into Claude thinking/effort config. An explicit client-supplied effort always + // wins; native Claude passthrough is left untouched (it carries its own `thinking`), + // and non-thinking base models are cleaned up later by normalizeThinkingForModel(). + // Extracted to chatCore/claudeEffortVariant.ts (#3501); mutates body in place and returns the + // stripped model + an optional log line. The strip is unconditional (byte-identical to the + // original behavior) for the claude/Claude-Code-compatible lane; for any other provider it + // additionally requires isKnownClaudeEffortBaseModel(baseModel) to verify the base id is a + // real, effort-capable Claude model before stripping (vertex-claude-catalog-dispatch fix). + { + const effortVariant = applyClaudeEffortVariant({ + provider, + effectiveModel, + body, + sourceFormat, + }); + effectiveModel = effortVariant.effectiveModel; + if (effortVariant.log) { + log?.info?.("PARAMS", effortVariant.log); + } + } + + // Wire target-format resolution extracted to chatCore/targetFormat.ts (#3501); `alias` is reused + // downstream when stripping the alias/ prefix off the upstream model id. + const { alias, targetFormat } = resolveChatCoreTargetFormat({ + provider, + resolvedModel, + apiFormat, + sourceFormat, + customModelTargetFormat, + providerSpecificData: credentials?.providerSpecificData, + nativeXaiResponsesPassthrough, + nativeOpenAICompatibleResponsesPassthrough, + }); + const nativeResponsesPassthrough = + nativeCodexPassthrough || + nativeXaiResponsesPassthrough || + nativeOpenAICompatibleResponsesPassthrough; + + const initialProviderRequest = + body && typeof body === "object" && !Array.isArray(body) + ? { + ...(body as Record), + model: + typeof (body as Record).model === "string" + ? (body as Record).model + : effectiveModel, + } + : body; + + // Track pending requests before slower optional enrichment (settings, logging, + // compression) so internal usage/runtime counters stay accurate even when + // upstream never returns response headers. + // Use credentials.connectionId as a fallback so that requests without an + // explicit session-level connectionId still register in the pendingRequests map. + const pendingConnId = connectionId || credentials?.connectionId || null; + const pendingRequestId = + trackPendingRequest(model, provider, pendingConnId, true, { + clientEndpoint: clientRawRequest?.endpoint || "/v1/chat/completions", + clientRequest: redactPendingBody(clientRawRequest?.body ?? body, videoBridgeObserved), + providerRequest: initialProviderRequest, + stage: "registered", + correlationId, + sessionTag: conversationId || null, + }) || generateRequestId(); + + // Initialize rate limit settings from persisted DB (once, lazy) + await initializeRateLimits(); + + // #3384: per-model interception rule (src/lib/db/interceptionRules.ts) overrides the + // native-bypass defaults below when the operator explicitly configured it for this + // provider/model pair; undefined falls through to the existing bypass logic. + const interceptSearchOverride = resolveInterceptSearch(provider, effectiveModel); + + // Capture client tool names BEFORE fallback injection so the owner-provenance + // merge can distinguish tools the client already declared from synthetic tools + // added by the fallback preparer. Without this, a client function named + // `omniroute_web_search` (colliding with the fallback tool name) would be + // marked server-owned even though the client owns it. + const preConversionClientToolNames: string[] = ( + Array.isArray((body as Record).tools) + ? ((body as Record).tools as unknown[]) + : [] + ) + .map((tool) => { + if (!tool || typeof tool !== "object") return ""; + const record = tool as Record; + if (typeof record.name === "string") return record.name; + const fn = record.function; + if ( + fn && + typeof fn === "object" && + typeof (fn as Record).name === "string" + ) { + return (fn as Record).name as string; + } + return ""; + }) + .filter(Boolean); + + const { body: bodyWithWebSearchFallback, fallback: webSearchFallbackPlan } = + prepareWebSearchFallbackBody(body as Record, { + provider, + sourceFormat, + targetFormat, + nativeCodexPassthrough: nativeResponsesPassthrough, + interceptSearchOverride, + }); + if (webSearchFallbackPlan.enabled) { + body = bodyWithWebSearchFallback as typeof body; + // Server-side web-search execution cannot be injected into an arbitrary + // client SSE stream (streaming interception is not implemented — #9725), so + // a stream:true OpenAI Responses request whose web_search tool was converted + // to the fallback is executed non-streaming: the assembled response then + // carries the executed results (function_call_output + web_search_call) and + // JSON-tolerating Responses clients (pi-web-access) consume it directly. + if ( + sourceFormat === FORMATS.OPENAI_RESPONSES && + (body as Record).stream === true + ) { + clientRequestedResponsesStream = true; + (body as Record).stream = false; + log?.info?.("TOOLS", `web_search fallback forced non-streaming response for ${provider}`); + } + log?.info?.( + "TOOLS", + `Converted ${webSearchFallbackPlan.convertedToolCount} web_search tool(s) to OmniRoute fallback for ${provider}` + ); + } + // #7339: interceptFetch (Phase 3-4 of #3384) — same per-model rule + native-bypass + // pattern as interceptSearch directly above. + const interceptFetchOverride = resolveInterceptFetch(provider, effectiveModel); + const { body: bodyWithWebFetchFallback, fallback: webFetchFallbackPlan } = + prepareWebFetchFallbackBody(body as Record, { + provider, + sourceFormat, + targetFormat, + nativeCodexPassthrough: nativeResponsesPassthrough, + interceptFetchOverride, + }); + if (webFetchFallbackPlan.enabled) { + body = bodyWithWebFetchFallback as typeof body; + log?.info?.( + "TOOLS", + `Converted ${webFetchFallbackPlan.convertedToolCount} web_fetch tool(s) to OmniRoute fallback for ${provider}` + ); + } + const noLogEnabled = apiKeyInfo?.noLog === true; + // Consolidate settings reads — fetch once, reuse throughout the request + const settings = cachedSettings ?? (await getCachedSettings()); + // Opt-in tool-source diagnostics (#1825): summarize the request's tool definitions + // (count + MCP/hosted/client source breakdown + first names) as a single debug line. + if (settings.logToolSources === true) { + const toolSummary = summarizeToolSources((body as { tools?: unknown }).tools); + if (toolSummary) log?.debug?.("TOOLS", toolSummary); + } + // #1311 (opt-in): echo the client-requested alias/combo name in the response `model` + // field instead of the upstream model, so strict clients (Claude Desktop) that validate + // response.model === request.model stop rejecting alias/combo requests with a 401. + // #3697: always echo it for Codex CLI clients on the Responses API — regardless of the + // opt-in setting — since the Codex CLI status line/model button reads `response.model` + // to display the active model + reasoning effort (e.g. `gpt-5.5-xhigh`). Detection is by + // request headers (originator/User-Agent), not by the routed provider, so it still fires + // when `codex/gpt-5.5-xhigh` is routed through a combo to a non-codex upstream. + const isCodexResponsesEcho = + (isResponsesEndpoint || sourceFormat === FORMATS.OPENAI_RESPONSES) && + isCodexOriginatedHeaders(clientRawRequest?.headers); + + // Detect Claude Code CLI so we can auto-enable model echo — this prevents + // session restore failures when the resolved upstream model (e.g. + // `oc/nemotron-3-ultra-free`) is not recognized by the client on `--resume`. + const isClaudeCodeClient = isClaudeCodeOriginatedHeaders(clientRawRequest?.headers); + + let echoModel = + (settings.echoRequestedModelName === true || isCodexResponsesEcho || isClaudeCodeClient) && + typeof requestedModel === "string" && + requestedModel + ? requestedModel + : null; + // Auto-echo the listing-valid form for bare requests to noAuth catalog + // providers so clients validating response.model against /v1/models don't warn. + echoModel = resolveNoAuthEchoModel(requestedModel, provider) ?? echoModel; + const detailedLoggingEnabled = + !noLogEnabled && + (settings.call_log_pipeline_enabled === true || + settings.call_log_pipeline_enabled === "1" || + settings.call_log_pipeline_enabled === "true"); + const capturePipelineStreamChunks = + detailedLoggingEnabled && getCallLogPipelineCaptureStreamChunks(); + const skillRequestId = generateRequestId(); + let compressionAnalyticsWritePromise: Promise | null = null; + // Compression usage-receipt attachment extracted to chatCore/compressionUsageReceipt.ts (#3501); + // pass the in-flight analytics write + request id so behaviour stays byte-identical. + const attachCompressionUsageReceiptAfterAnalytics = ( + usage: Record, + source: "provider" | "estimated" | "stream" + ) => + attachCompressionUsageReceiptAfterAnalyticsFor(usage, source, { + pendingWrite: compressionAnalyticsWritePromise, + skillRequestId, + }); + // #8249: raw header value, kept separate from `pipelineSessionId`'s skillRequestId fallback + // below so call_logs.session_tag is only ever set when the caller explicitly supplied the + // header — never synthesized from the internal per-request skillRequestId. + const explicitSessionIdHeader = + (clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" + ? clientRawRequest.headers.get("x-omniroute-session-id") + : getHeaderValueCaseInsensitive( + clientRawRequest?.headers ?? null, + "x-omniroute-session-id" + )) || null; + const pipelineSessionId = explicitSessionIdHeader || skillRequestId; + const reasoningReplaySessionKey = sessionAffinityKey || explicitSessionIdHeader; + const reasoningCacheScope = reasoningReplaySessionKey + ? `api-key:${String(apiKeyInfo?.id ?? "local")}\x1f${String(reasoningReplaySessionKey)}` + : null; + + // Primary path: merge client model id + alias target so config on either key applies; resolved + // id wins on same header name. T5 family fallback uses only (nextModel, resolveModelAlias(next)) + // so A-model headers are not sent to B — see buildUpstreamHeadersForExecute. + const connectionCustomUserAgent = + credentials?.providerSpecificData && + typeof credentials.providerSpecificData === "object" && + typeof credentials.providerSpecificData.customUserAgent === "string" + ? credentials.providerSpecificData.customUserAgent.trim() + : ""; + + // #8369: connection-level custom upstream headers from provider_specific_data. + const connectionCustomHeaders = + credentials?.providerSpecificData && + typeof credentials.providerSpecificData === "object" && + typeof credentials.providerSpecificData.customHeaders === "object" && + !Array.isArray(credentials.providerSpecificData.customHeaders) + ? (credentials.providerSpecificData.customHeaders as Record) + : undefined; + + // Upstream extra-header building extracted to chatCore/upstreamExecuteHeaders.ts (#3501); bind the + // per-request inputs once and delegate so the existing call sites stay byte-identical. + const buildUpstreamHeadersForExecute = (modelToCall: string): Record => + buildUpstreamHeadersForExecuteFor({ + modelToCall, + effectiveModel, + provider, + model, + resolvedModel, + sourceFormat, + connectionCustomUserAgent, + connectionCustomHeaders, + settings, + }); + + // Default to false unless client explicitly sets stream: true (OpenAI spec compliant) + const acceptHeader = + clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" + ? clientRawRequest.headers.get("accept") || clientRawRequest.headers.get("Accept") + : clientRawRequest?.headers?.["accept"] || clientRawRequest?.headers?.["Accept"]; + const streamUserAgent = [ + typeof userAgent === "string" ? userAgent : "", + getHeaderValueCaseInsensitive(clientRawRequest?.headers ?? null, "user-agent") || "", + ] + .filter(Boolean) + .join(" "); + + // Explicit per-request opt-in/out for the `` close marker + // (#5312 / #5245): `x-omniroute-thinking-marker: off` suppresses it for + // reasoning_content-native clients (e.g. Cursor's OpenAI path) that the UA + // allowlist does not cover; absent the header, the UA policy applies. + const thinkingMarkerHeader = getHeaderValueCaseInsensitive( + clientRawRequest?.headers ?? null, + THINKING_MARKER_HEADER + ); + + const explicitStreamAlias = resolveExplicitStreamAlias(body); + + // Remove non-standard non-stream aliases before provider translation/execution. + // They are accepted for compatibility at the OmniRoute API boundary only. + if (body && typeof body === "object") { + const b = body as Record; + if (explicitStreamAlias !== undefined) { + b.stream = explicitStreamAlias; + } + + delete b.non_stream; + delete b.disable_stream; + delete b.disable_streaming; + delete b.streaming; + } + + // Codex /responses/compact is JSON-only: Codex CLI does not send stream=false, + // so route shape must override the usual Accept/header fallback. + // sourceFormat="claude" applies the Anthropic Messages spec default (stream=false + // when body omits stream), preventing STREAM_EARLY_EOF on /v1/messages when + // clients send Accept: */* without an explicit stream flag. + // providerRequiresStreaming: providers with forceStream:true (cline/clinepass) + // only implement upstream streaming — a non-streaming request returns + // "generateText is not implemented" / an empty body. This flag forces the + // UPSTREAM request to stream (see `upstreamStream` below), but it MUST NOT + // force the client-facing `stream` flag: a stream:false client (e.g. the + // model-test button, plain JSON API callers) still expects a JSON response. + // The client-side `if (!stream)` branch drains the forced upstream SSE and + // converts it back to JSON via readNonStreamingResponseBody. Passing this + // flag into resolveStreamFlag would force `stream=true` and skip that + // conversion, yielding STREAM_EARLY_EOF for JSON callers. (#2081, #6126) + const providerRequiresStreaming = REGISTRY[provider]?.forceStream === true; + const stream = + nativeCodexPassthrough && isCompactResponsesEndpoint(endpointPath) + ? false + : resolveStreamFlag(body?.stream, acceptHeader, sourceFormat, { + userAgent: streamUserAgent, + streamDefaultMode: apiKeyInfo?.streamDefaultMode, + }); + + // `settings` is already consolidated once near the top of handleChatCore + // (the "fetch once, reuse" const). A second `const settings` here was a + // duplicate same-scope declaration that broke the esbuild/tsx transform + // ("settings has already been declared") and the production build. Reuse it. + credentials = applyCodexGlobalFastServiceTier(provider, credentials, settings, { + model: requestedModel, + body: body && typeof body === "object" ? (body as Record) : null, + }); + effectiveServiceTier = resolveEffectiveServiceTier(body); + setGeminiThoughtSignatureMode(settings.antigravitySignatureCacheMode); + const semanticCacheEnabled = settings.semanticCacheEnabled !== false; + + const reqLogger = await createRequestLogger(sourceFormat, targetFormat, model, { + enabled: detailedLoggingEnabled, + captureStreamChunks: capturePipelineStreamChunks, + maxStreamChunkBytes: getCallLogPipelineMaxSizeBytes(), + requestId: pendingRequestId, + model, + provider: provider || undefined, + connectionId: connectionId || credentials?.connectionId || undefined, + }); + + // persistAttemptLogs extracted to chatCore/attemptLogging.ts (#3501); bind the per-request context + // once so the 16 call sites keep passing only the per-attempt args (byte-identical). + const persistAttemptLogs = (args: PersistAttemptLogsArgs) => + persistAttemptLogsFor(args, { + traceId, + provider, + connectionId, + model, + skillRequestId, + detailedLoggingEnabled, + reqLogger, + pendingRequestId, + clientRawRequest, + requestedModel, + credentials, + startTime, + body, + sourceFormat, + targetFormat, + comboName, + comboStepId, + comboExecutionKey, + tokensCompressed, + apiKeyInfo, + noLogEnabled, + correlationId, + modelPinned, + // Resolved conversationId (open-sse/services/conversationTracker.ts) wins when + // present — it's populated for every request now, not just ones where the + // client explicitly sent x-omniroute-session-id. The raw header remains a + // fallback for any caller that somehow bypassed conversationId resolution. + sessionTag: conversationId || explicitSessionIdHeader, + // #12150 P1b surface 1: undefined for every non-video request (byte-identical + // to before this param existed) — see applyVideoBridgeLogRedaction. + videoBridgeLogRedaction: (videoBridgeLog as VideoBridgeLogParam | undefined)?.redaction, + // #12150 P2 surface 2: mark the persisted call_logs row so + // resolvePreviousResponseState refuses to rehydrate a snapshot whose video + // transcript was redacted. false for every non-video request. + videoContentRemoved: videoBridgeObserved, + }); + + const pendingScope = { id: pendingRequestId, model, provider, connectionId: pendingConnId }; + const providerRequestCapture = createPreparedRequestLogger(reqLogger, pendingScope); + // 0. Log client raw request (before format conversion) — redacts video transcript + // cues in the logged copy only; see videoBridgeSnapshotRedaction.ts. + logClientRawRequestRedacted(reqLogger, clientRawRequest, videoBridgeObserved); + const reasoningRouteDecision = + body && typeof body === "object" + ? (body as Record)._omnirouteReasoningRouteTrace + : null; + if (reasoningRouteDecision) { + reqLogger.logRouteDecision(reasoningRouteDecision); + body = { ...(body as Record) }; + delete (body as Record)._omnirouteReasoningRouteTrace; + } + + log?.debug?.("FORMAT", `${sourceFormat} → ${targetFormat} | stream=${stream}`); + + if (reasoningRuleDirective) { + // Cache identity must use the effective effort, not the overridden client value. + // Retain the directive for the translation step, where general thinking defaults run. + body = { + ...(applyReasoningRuleDirective( + body, + sourceFormat === FORMATS.OPENAI_RESPONSES + ? "openai-responses" + : sourceFormat === FORMATS.CLAUDE + ? "claude" + : undefined + ) as Record), + _omnirouteReasoningRule: reasoningRuleDirective, + }; + } + + // Preserve original body for cache signature — the body variable is mutated + // multiple times below (sanitization, memory/skills injection) before the + // cache store path runs at Phase 9.1 (non-streaming) / Phase 9.2 (streaming). + // Without this snapshot, the write-time signature differs from the read-time + // one, producing 0% hit rate. (#cache-signature-asymmetry) + const bodyForCacheWrite = body; + + return { + kind: "continue" as const, + continue: { + body, + credentials, + provider, + model, + extendedContext, + videoBridgeObserved, + resilienceSettings, + requestedModel, + isModelScope, + startTime, + traceId, + traceEnabled, + trace, + getCurrentConnectionId, + assertManagedLeaseFence, + getManagedLeaseFenceErrorCode, + managedLeaseFenceErrorResult, + tokensCompressed, + agentGoalPolicy, + effectiveServiceTier, + resolveEffectiveServiceTier, + resolveReportedServiceTier, + recordKeyHealthStatus, + idempotencyKey, + endpointPath, + sourceFormat, + isResponsesEndpoint, + nativeCodexPassthrough, + nativeXaiResponsesPassthrough, + isDroidCLI, + isOpencodeClient, + copilotCompatibleReasoning, + clientResponseFormat, + nativeOpenAICompatibleResponsesPassthrough, + customToolNames, + backgroundReason, + effectiveModel, + alias, + targetFormat, + nativeResponsesPassthrough, + pendingConnId, + pendingRequestId, + preConversionClientToolNames, + webSearchFallbackPlan, + clientRequestedResponsesStream, + webFetchFallbackPlan, + settings, + isCodexResponsesEcho, + echoModel, + skillRequestId, + compressionAnalyticsWritePromise, + pipelineSessionId, + reasoningCacheScope, + persistAttemptLogs, + persistFailureUsage, + attachCompressionUsageReceiptAfterAnalytics, + detailedLoggingEnabled, + noLogEnabled, + explicitSessionIdHeader, + buildUpstreamHeadersForExecute, + streamUserAgent, + thinkingMarkerHeader, + providerRequiresStreaming, + stream, + semanticCacheEnabled, + reqLogger, + pendingScope, + providerRequestCapture, + bodyForCacheWrite, + trustedEffortContext, + reasoningRuleDirective, + }, + }; +} + +export type PreludeResult = Awaited>; +export type Continue1 = Extract["continue"]; diff --git a/tests/unit/chatcore/request-prelude.test.ts b/tests/unit/chatcore/request-prelude.test.ts new file mode 100644 index 000000000000..ff1301448def --- /dev/null +++ b/tests/unit/chatcore/request-prelude.test.ts @@ -0,0 +1,66 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const barrelPath = path.resolve(here, "../../../open-sse/handlers/chatCore.ts"); +const leafPath = path.resolve(here, "../../../open-sse/handlers/chatCore/requestPrelude.ts"); + +test("requestPrelude leaf exists", () => { + assert.equal(fs.existsSync(leafPath), true); +}); + +test("barrel no longer inlines handleBypassRequest", () => { + const src = fs.readFileSync(barrelPath, "utf8"); + assert.equal( + src.includes("handleBypassRequest(body, model, userAgent)"), + false, + "front-door bypass must leave barrel" + ); +}); + +test("prelude stamps clientRequestedResponsesStream before forcing stream false", () => { + const src = fs.readFileSync(leafPath, "utf8"); + assert.match(src, /let clientRequestedResponsesStream = false/); + const stamp = src.search(/^\s*clientRequestedResponsesStream = true;/m); + const force = src.indexOf(".stream = false", stamp + 1); + assert.ok(stamp >= 0 && force > stamp, "stamp must precede stream=false"); + assert.match(src, /clientRequestedResponsesStream,/); +}); + +test("barrel takes clientRequestedResponsesStream from prelude continue", () => { + const src = fs.readFileSync(barrelPath, "utf8"); + assert.match(src, /clientRequestedResponsesStream,/); + assert.equal(src.includes("let clientRequestedResponsesStream"), false); +}); + +test("prelude continue carries tip-only trustedEffortContext", () => { + const src = fs.readFileSync(leafPath, "utf8"); + assert.match(src, /trustedEffortContext,/); + assert.match(src, /getManagedLeaseFenceErrorCode,/); + assert.match(src, /reasoningRuleDirective,/); +}); + +test("barrel rebinds persistAttemptLogs after translation", () => { + const src = fs.readFileSync(barrelPath, "utf8"); + const rebind = src.indexOf("body: translatedBody"); + const send = src.indexOf("const executeProviderRequest ="); + assert.ok(rebind >= 0 && send > rebind, "persist rebind must precede send"); + assert.match(src, /persistAttemptLogsPrelude/); +}); + +test("persistFailureUsage is declared after endpointPath", () => { + const src = fs.readFileSync(leafPath, "utf8"); + const endpoint = src.search(/^\s*endpointPath,/m); + const helper = src.indexOf("const persistFailureUsage ="); + assert.ok(endpoint >= 0 && helper > endpoint, "persistFailureUsage must follow endpointPath"); +}); + +test("persistAttemptLogs is declared after reqLogger", () => { + const src = fs.readFileSync(leafPath, "utf8"); + const logger = src.indexOf("const reqLogger ="); + const helper = src.indexOf("const persistAttemptLogs ="); + assert.ok(logger >= 0 && helper > logger, "persistAttemptLogs must follow reqLogger"); +}); From 266098725e7cf3bfb11e93ee6078e7b28fb6ee06 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 14:44:33 -0400 Subject: [PATCH 05/26] refactor(sse): carry four upstream chatCore changes onto the extracted leaves The tip moved four times under this branch while the leaves were being split out, all four touching chatCore. Rebasing meant hand-resolving a 4971-line hunk, so each change is carried onto the leaf that now owns that code instead. Function-tools-only normalization becomes an allowlist so native hosted tools survive on Responses targets. The turn execution guard holds a slot for the whole handler body and hands it to the response stream on the streaming success path, so a client retry carrying the same idempotency key cannot open a second upstream execution. Reasoning replay reports the pivot transcript it digested, because a Responses body carries input rather than messages and the write side would otherwise digest a single assistant turn and miss every replay. Responses custom tool choice propagates through both provider legs. The stream wrapper releases the slot when the client stream drains, is cancelled, or errors. That last path had no coverage; it does now. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 23 +-- .../handlers/chatCore/streamMaterialize.ts | 7 +- .../chatcore/turn-execution-handoff.test.ts | 137 ++++++++++++++++++ 3 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 tests/unit/chatcore/turn-execution-handoff.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 47ffdfa08970..730fd8d23e7d 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -38,6 +38,8 @@ import { maybeConvertJsonBodyToSse } from "./chatCore/jsonBodyToSse.ts"; import { assembleStreamingResponseHeaders } from "./chatCore/streamingResponseHeaders.ts"; import { assembleStreamingPipeline } from "./chatCore/streamingPipeline.ts"; import { makeOnStreamComplete } from "./chatCore/streamMaterialize.ts"; +import { acquireTurnExecution, createTurnInProgressResult } from "./chatCore/turnExecutionGuard.ts"; +import { wrapReadableStreamWithFinalize } from "./chatCore/streamFinalize.ts"; import { sanitizeChatRequestBody } from "./chatCore/sanitization.ts"; import { applyReasoningInputPolicy, @@ -4718,6 +4720,7 @@ export async function handleChatCore({ translatedBody, body, reasoningCacheScope, + reasoningReplayHistory, contextEditingEnabled, skillRequestId, streamFailure, @@ -4898,16 +4901,16 @@ export async function handleChatCore({ // ── Gamification event (fire-and-forget) ── await emitRequestGamificationEvent({ apiKeyId: apiKeyInfo?.id, model, provider }); - // ── Plugin onResponse hook (fire-and-forget) ── - await runPluginOnResponseHook({ - requestId: traceId, - body, - model, - provider, - apiKeyInfo, - headers: clientRawRequest?.headers, - response: { status: 200, streamed: true }, - }); + // ── Plugin onResponse hook (fire-and-forget) ── + await runPluginOnResponseHook({ + requestId: traceId, + body, + model, + provider, + apiKeyInfo, + headers: clientRawRequest?.headers, + response: { status: 200, streamed: true }, + }); const response = new Response(clientFacingStream, { headers: responseHeaders, diff --git a/open-sse/handlers/chatCore/streamMaterialize.ts b/open-sse/handlers/chatCore/streamMaterialize.ts index ab45f08a4dd8..b75f85131d99 100644 --- a/open-sse/handlers/chatCore/streamMaterialize.ts +++ b/open-sse/handlers/chatCore/streamMaterialize.ts @@ -69,6 +69,7 @@ export type StreamMaterializeDeps = { translatedBody: Record | null | undefined; body: unknown; reasoningCacheScope: string | null; + reasoningReplayHistory?: unknown[] | null; contextEditingEnabled: boolean; skillRequestId: string; streamFailure: { @@ -127,6 +128,7 @@ export function makeOnStreamComplete( translatedBody, body, reasoningCacheScope, + reasoningReplayHistory, contextEditingEnabled, skillRequestId, streamFailure, @@ -213,8 +215,9 @@ export function makeOnStreamComplete( const choices = cacheStreamBody.choices as { message?: Record }[] | undefined; const msg = choices?.[0]?.message; - const historyMessages = (translatedBody as { messages?: unknown[] } | null | undefined) - ?.messages; + const historyMessages = + (translatedBody as { messages?: unknown[] } | null | undefined)?.messages ?? + reasoningReplayHistory; if (requiresReasoningReplay({ provider, model })) { cacheReasoningFromAssistantMessage(msg, provider, model, { scope: reasoningCacheScope, diff --git a/tests/unit/chatcore/turn-execution-handoff.test.ts b/tests/unit/chatcore/turn-execution-handoff.test.ts new file mode 100644 index 000000000000..903fc9d11689 --- /dev/null +++ b/tests/unit/chatcore/turn-execution-handoff.test.ts @@ -0,0 +1,137 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + acquireTurnExecution, + clearTurnExecutionsForTesting, + getTurnExecutionSnapshot, +} from "../../../open-sse/handlers/chatCore/turnExecutionGuard.ts"; +import { wrapReadableStreamWithFinalize } from "../../../open-sse/handlers/chatCore/streamFinalize.ts"; + +test.afterEach(() => { + clearTurnExecutionsForTesting(); +}); + +/** + * handleChatCore holds the turn slot across its whole body and hands it to the + * response stream only on the streaming success path. These tests pin that + * contract on the two primitives the handler composes, so a future refactor + * that drops the finally block or the stream wrapper leaks the slot loudly. + */ + +const KEY = "handoff-contract-key"; + +type GuardedRun = (ctx: { + markHandedOff: () => void; + release: () => void; +}) => ReadableStream | void; + +function simulateGuardedBody(run: GuardedRun): ReadableStream | void { + const turn = acquireTurnExecution(KEY); + assert.equal(turn.acquired, true); + const release = turn.release; + let handedOff = false; + try { + return run({ + markHandedOff: () => { + handedOff = true; + }, + release, + }); + } finally { + if (!handedOff) release(); + } +} + +test("early return inside the guarded body releases the turn slot", () => { + simulateGuardedBody(() => { + // cache hit / validation failure / non-streaming reply: never reaches handoff + return undefined; + }); + + assert.equal(getTurnExecutionSnapshot(KEY), null); +}); + +test("a throw inside the guarded body releases the turn slot", () => { + assert.throws(() => { + simulateGuardedBody(() => { + throw new Error("upstream exploded"); + }); + }, /upstream exploded/); + + assert.equal(getTurnExecutionSnapshot(KEY), null); +}); + +test("streaming success keeps the slot held until the client stream drains", async () => { + const stream = simulateGuardedBody(({ markHandedOff, release }) => { + assert.ok(getTurnExecutionSnapshot(KEY), "slot must still be held while assembling the stream"); + const source = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1])); + controller.close(); + }, + }); + const wrapped = wrapReadableStreamWithFinalize(source, release); + markHandedOff(); + return wrapped; + }); + + assert.ok(stream, "streaming path must return a stream"); + assert.ok(getTurnExecutionSnapshot(KEY), "slot stays held after the handler returns"); + + const reader = (stream as ReadableStream).getReader(); + while (!(await reader.read()).done) { + // drain + } + + assert.equal(getTurnExecutionSnapshot(KEY), null, "draining the stream releases the slot"); +}); + +test("cancelling the client stream also releases the turn slot", async () => { + const stream = simulateGuardedBody(({ markHandedOff, release }) => { + const source = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1])); + }, + }); + const wrapped = wrapReadableStreamWithFinalize(source, release); + markHandedOff(); + return wrapped; + }); + + assert.ok(stream); + assert.ok(getTurnExecutionSnapshot(KEY), "slot held while the client is still reading"); + + await (stream as ReadableStream).cancel("client disconnected"); + + assert.equal(getTurnExecutionSnapshot(KEY), null, "cancel releases the slot"); +}); + +test("an upstream stream error also releases the turn slot", async () => { + const stream = simulateGuardedBody(({ markHandedOff, release }) => { + const source = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1])); + }, + pull(controller) { + controller.error(new Error("upstream died mid-stream")); + }, + }); + const wrapped = wrapReadableStreamWithFinalize(source, release); + markHandedOff(); + return wrapped; + }); + + assert.ok(stream); + const reader = (stream as ReadableStream).getReader(); + await assert.rejects( + (async () => { + for (;;) { + const { done } = await reader.read(); + if (done) return; + } + })(), + /upstream died mid-stream/ + ); + + assert.equal(getTurnExecutionSnapshot(KEY), null, "a stream error releases the slot"); +}); From 230106ef92fd69ba5921dfd17bb70f7ed547ff6b Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 14:58:07 -0400 Subject: [PATCH 06/26] test(sse): pin the turn guard against a stale release from an abandoned stream The handler hands the release callback to the response stream before it awaits its response hooks. If a hook throws, the finally block frees the slot while the wrapped stream still holds a callback for the same key, so a retry can re-acquire that key and then have its slot yanked when the abandoned stream is finally drained. The guard already refuses that, by checking the map still holds the same entry before deleting it. Nothing pinned it, so deleting the check left the suite green. This covers it, along with the stream error path. Signed-off-by: Minxi Hou --- .../chatcore/turn-execution-handoff.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/unit/chatcore/turn-execution-handoff.test.ts b/tests/unit/chatcore/turn-execution-handoff.test.ts index 903fc9d11689..aaab492117e5 100644 --- a/tests/unit/chatcore/turn-execution-handoff.test.ts +++ b/tests/unit/chatcore/turn-execution-handoff.test.ts @@ -135,3 +135,40 @@ test("an upstream stream error also releases the turn slot", async () => { assert.equal(getTurnExecutionSnapshot(KEY), null, "a stream error releases the slot"); }); + +test("a late release from an abandoned stream cannot evict a newer turn", async () => { + // The handler wraps the stream before awaiting its response hooks. Those + // hooks and the stream wrapper hold two DIFFERENT release callbacks for the + // same key once a retry re-acquires it. The stale one must not evict the + // newer turn. + const first = acquireTurnExecution(KEY); + assert.equal(first.acquired, true); + + const source = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1])); + controller.close(); + }, + }); + // The stream keeps its own handle on the first turn's release. + const orphan = wrapReadableStreamWithFinalize(source, () => { + first.release(); + }); + + // The request fails before handoff; a different code path frees the slot. + clearTurnExecutionsForTesting(); + + const second = acquireTurnExecution(KEY); + assert.equal(second.acquired, true, "the key is free for a new turn"); + + const reader = orphan.getReader(); + while (!(await reader.read()).done) { + // drain the abandoned stream, firing its stale release + } + + assert.ok( + getTurnExecutionSnapshot(KEY), + "the stale release must not evict the newer turn holding the same key" + ); + second.release(); +}); From c0110927c3a1abeeb1be7cf19894d7f96c4e90d4 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 15:37:32 -0400 Subject: [PATCH 07/26] refactor(sse): lift prompt compression out of the chatCore barrel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compression and caveman-output block was 770 lines sitting in the middle of handleChatCore. It moves to chatCore/contextCompression.ts unchanged. The contract is the awkward part and it is now explicit. The block reads eighteen values and reassigns eleven of them, so everything it reads arrives as a parameter and everything it writes comes back in the result for the caller to reseat. nativeCodexPassthrough is the one asymmetric case: the block reads it to decide whether reactive compaction applies but never assigns it, and it is destructured as a const upstream, so returning it would throw at runtime while typechecking clean. A dropped write-back is silent — compression still runs, its output is just discarded — and none of the existing compression suites notice, because they exercise the block's internals rather than the barrel's plumbing. context-compression-carry-out.test.ts pins the plumbing: every returned field must be reseated, no write-back may cross-wire two fields, and nativeCodexPassthrough must stay out of the result. The codex passthrough guard reads source text to prove the gate still exists, so it follows the code: the threshold gate moved to the leaf, the overflow fail-fast stayed in the barrel, and the test now reads both. chatCore.ts 4901 -> 4175 lines. No behaviour change. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 862 +---------------- .../handlers/chatCore/contextCompression.ts | 915 ++++++++++++++++++ .../context-compression-carry-out.test.ts | 142 +++ ...dex-prompt-compression-passthrough.test.ts | 12 +- 4 files changed, 1116 insertions(+), 815 deletions(-) create mode 100644 open-sse/handlers/chatCore/contextCompression.ts create mode 100644 tests/unit/chatcore/context-compression-carry-out.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 730fd8d23e7d..ffe6e1a950fd 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -28,6 +28,7 @@ export { relocateDirectiveOnlyMessages, } from "./chatCore/claudeSystemRole.ts"; import { acquireTurnExecution, createTurnInProgressResult } from "./chatCore/turnExecutionGuard.ts"; +import { applyContextCompression } from "./chatCore/contextCompression.ts"; import { checkSemanticCache } from "./chatCore/semanticCache.ts"; import { buildPostCallGuardrailContext } from "./chatCore/postCallGuardrailContext.ts"; import { storeSemanticCacheResponse } from "./chatCore/semanticCacheStore.ts"; @@ -48,18 +49,13 @@ import { import { createRoutingEvent, emitRoutingEvent } from "../services/routing/index.ts"; import { routingFinishReason } from "./chatCore/routingFinishReason.ts"; -import { - getHeaderValueCaseInsensitive, - isNoMemoryRequested, - resolveCompressionHeader, -} from "./chatCore/headers.ts"; +import { isNoMemoryRequested } from "./chatCore/headers.ts"; import { getCodexClientSessionId } from "../config/codexIdentity.ts"; import { noteCodexTurnStateProvenance, readCodexTurnStateHeader, } from "../config/codexTurnState.ts"; -import { getCombosCached } from "./chatCore/comboContextCache.ts"; export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comboContextCache.ts"; import { resolveAccountSemaphoreKey, @@ -89,10 +85,7 @@ import { buildStreamingResponseHeaders, stripStaleForwardingHeaders, } from "./chatCore/responseHeaders.ts"; -import { - forwardDashboardEventToLiveWs, - maybeSyncClaudeExtraUsageState, -} from "./chatCore/telemetryHelpers.ts"; +import { maybeSyncClaudeExtraUsageState } from "./chatCore/telemetryHelpers.ts"; // Re-export the previously inline-defined helpers so existing importers of these // symbols from chatCore.ts (tests, sibling modules) keep resolving after the split. export { @@ -105,7 +98,6 @@ export { }; import { resolveMemoryOwnerId, runMemoryExtractionGate } from "./chatCore/memoryExtraction.ts"; import { normalizeHeaders } from "../utils/headers.ts"; -import { resolveOmniGlyphTransport } from "../services/compression/imageTransportPolicy.ts"; import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts"; import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; import { injectSystemPromptPreTranslation } from "../services/systemPrompt.ts"; @@ -224,9 +216,7 @@ import { excludeConnectionForCooldown } from "./chatCore/connectionCooldown.ts"; import { handleRequestRejectedFailure } from "./chatCore/requestRejectedFailure.ts"; import { getKimiTemporaryRateLimitResetAt } from "./chatCore/kimiQuotaRecovery.ts"; import { logAuditEvent } from "@/lib/compliance"; -import { emit } from "@/lib/events/eventBus"; import { adaptBodyForCompression } from "../services/compression/bodyAdapter.ts"; -import { ensureEngineBreakdown } from "../services/compression/engineBreakdown.ts"; import { trackPendingRequest, appendRequestLog, saveRequestUsage } from "@/lib/usageDb"; import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; import { recordCost } from "@/domain/costRules"; @@ -235,25 +225,9 @@ import { buildClaudePassthroughToolNameMap, mergeResponseToolNameMap, } from "./chatCore/passthroughToolNames.ts"; -import { - createDisabledCompressionConfig, - resolveCompressionSettings, -} from "./chatCore/compressionSettings.ts"; import type { EnforceDecision } from "@/lib/quota/types"; -import { isCompressionExcluded } from "../services/compression/exclusions.ts"; -import { - isBuiltinStackedPipeline, - isStackedCompressionCombo, - type RuntimeCompressionCombo, -} from "./chatCore/compressionComboPredicates.ts"; -import { emitOutputStyleTelemetry } from "./chatCore/outputStyleTelemetry.ts"; -import { - writeCompressionAnalytics, - writeCompressionSkip, -} from "./chatCore/compressionAnalyticsWrite.ts"; +import { writeCompressionAnalytics } from "./chatCore/compressionAnalyticsWrite.ts"; import { recordContextEditingTelemetryHook } from "./chatCore/contextEditingTelemetry.ts"; -import { recordCompressionCacheStats } from "./chatCore/compressionCacheStats.ts"; -import { writeCavemanOutputAnalytics } from "./chatCore/cavemanOutputAnalytics.ts"; import { scheduleQuotaShareConsumption } from "./chatCore/quotaShareConsumption.ts"; import { emitRequestGamificationEvent } from "./chatCore/gamificationEvent.ts"; import { runPluginOnResponseHook } from "./chatCore/pluginOnResponse.ts"; @@ -287,19 +261,8 @@ import { findLargerContextModel, getModelFamily, } from "../services/modelFamilyFallback.ts"; -import { - compressContext, - estimateTokens, - getTokenLimit, - getComboTargetTokenLimit, - resolveComboContextLimit, -} from "../services/contextManager.ts"; -import type { - CompressionConfig, - CompressionPipelineStep, - CompressionResult, -} from "../services/compression/types.ts"; -import { generateSessionId } from "../services/sessionManager.ts"; +import { compressContext, estimateTokens, getTokenLimit } from "../services/contextManager.ts"; +import type { CompressionResult } from "../services/compression/types.ts"; import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; @@ -314,7 +277,6 @@ import { import { classifyModelScope429 } from "../services/modelscopePolicy.ts"; import { incrementTokenUsage, isTpmExhausted } from "../services/geminiRateLimitTracker.ts"; import { computeRequestHash, shouldDeduplicate } from "../services/requestDedup.ts"; -import { getProactiveCompressionRatio } from "@/lib/db/compression"; /** * #12150 P1b: shape of handleChatCore's optional `videoBridgeLog` param — see @@ -647,776 +609,48 @@ export async function handleChatCore({ // further down — see #8378 (context limit resolved by the combo was silently // discarded because it only existed inside this `if` block). let contextLimit = getTokenLimit(provider, effectiveModel); - if (body && Array.isArray(allMessages) && allMessages.length > 0) { - let estimatedTokens = estimateTokens(allMessages); - const compressionSettingsResult = await resolveCompressionSettings(log); - const compressionSettings: CompressionConfig | null = compressionSettingsResult.settings; - // #8034 — operator-named model/endpoint exclusions bypass the whole pipeline, exactly - // like compression being globally disabled, so the body is provably byte-identical. - // Native Codex passthrough is deliberately NOT part of this exclusion: prompt - // compression runs through adaptBodyForCompression() (Responses input[] → messages - // → restore) with codex tool-output eligibility guards, so native contexts still - // compress (regression: #8933 introduced the passthrough bypass, landed on release - // via #11088, silencing codex analytics to skip_reason='excluded'). Reactive - // compaction + combo overflow fail-fast below still bypass native passthrough — - // intentionally left for follow-up. Operators who want byte-identical passthrough - // can add `codex/*` to the exclusions list. - const compressionExcluded = isCompressionExcluded( - { provider, model: effectiveModel }, - compressionSettings?.exclusions - ); - // A per-key opt-out is a request-scoped hard kill for prompt compression. It - // deliberately does not disable the independent reactive context-fit safety - // passes, matching the existing x-omniroute-compression: off contract. - const apiKeyCompressionEnabled = apiKeyInfo?.compressionEnabled !== false; - let promptCompressionEnabled = - compressionSettingsResult.enabled && !compressionExcluded && apiKeyCompressionEnabled; - reactiveContextCompactionEnabled = compressionSettingsResult.enabled && !compressionExcluded; - contextEditingEnabled = compressionSettingsResult.contextEditingEnabled; - if (!apiKeyCompressionEnabled) { - log?.debug?.("COMPRESSION", "Prompt compression disabled for this API key"); - } - if (compressionExcluded) { - void writeCompressionSkip( - { - stats: { - originalTokens: estimatedTokens, - compressedTokens: estimatedTokens, - savingsPercent: 0, - techniquesUsed: [], - mode: "off", - timestamp: Date.now(), - }, - provider, - effectiveModel, - effectiveServiceTier, - comboName, - mode: "off", - compressionComboId: null, - skillRequestId, - cavemanOutputModeApplied: false, - cavemanOutputModeIntensity: null, - log, - }, - "excluded" - ); - } - - // --- Modular Compression Pipeline (Phase 1 Lite + Phase 2 Standard/Caveman + Phase 3 Aggressive) --- - // Runs BEFORE the existing reactive compressContext() to proactively reduce tokens. - try { - const { - selectCompressionStrategy, - selectCompressionPlan, - enginesMapDerivesStackedPipeline, - activeComboResolves, - applyCompressionAsync, - resolveCacheAwareConfig, - formatCompressionMeta, - buildNamedComboLookup, - formatCompressionAnnotation, - } = await import("../services/compression/strategySelector.ts"); - const { trackCompressionStats } = await import("../services/compression/stats.ts"); - let config: CompressionConfig = compressionSettings ?? createDisabledCompressionConfig(); - if (compressionExcluded || !apiKeyCompressionEnabled) { - config = { ...config, enabled: false }; - } - if (!promptCompressionEnabled || !compressionSettings) { - log?.debug?.("COMPRESSION", "Prompt compression disabled or unavailable"); - } - let compressionComboKey = comboName ?? null; - let compressionComboApplied = false; - const applyCompressionComboConfig = ( - compressionCombo: RuntimeCompressionCombo | null, - routingOverrideIds: string[] = [] - ): boolean => { - if (!compressionCombo || compressionCombo.pipeline.length === 0) return false; - const comboLanguagePacks = [ - ...new Set( - compressionCombo.languagePacks - .map((pack) => pack.trim()) - .filter((pack) => pack.length > 0) - ), - ]; - const comboOutputIntensity = ( - ["lite", "full", "ultra"].includes(compressionCombo.outputModeIntensity) - ? compressionCombo.outputModeIntensity - : (config.cavemanOutputMode?.intensity ?? "full") - ) as "lite" | "full" | "ultra"; - const comboDefaultLanguage = - comboLanguagePacks.find((pack) => pack === config.languageConfig?.defaultLanguage) ?? - comboLanguagePacks[0] ?? - config.languageConfig?.defaultLanguage ?? - "en"; - const comboOverrides = { ...(config.comboOverrides ?? {}) }; - for (const id of routingOverrideIds) { - if (id) comboOverrides[id] = "stacked"; - } - config = { - ...config, - compressionComboId: compressionCombo.id, - stackedPipeline: compressionCombo.pipeline, - languageConfig: { - ...(config.languageConfig ?? { - enabled: false, - defaultLanguage: "en", - autoDetect: true, - enabledPacks: ["en"], - }), - enabled: true, - defaultLanguage: comboDefaultLanguage, - enabledPacks: - comboLanguagePacks.length > 0 - ? comboLanguagePacks - : (config.languageConfig?.enabledPacks ?? ["en"]), - }, - cavemanOutputMode: { - ...(config.cavemanOutputMode ?? { - enabled: false, - intensity: "full", - autoClarity: true, - }), - enabled: compressionCombo.outputMode, - intensity: comboOutputIntensity, - }, - comboOverrides, - }; - compressionComboApplied = true; - return true; - }; - if ((isCombo && comboName) || routingComboId) { - try { - const { getComboByName } = await import("@/lib/db/combos"); - let comboConfig = await getComboByName(comboName); - if (!comboConfig && comboName?.startsWith("combo/")) { - comboConfig = await getComboByName(comboName.substring(6)); - } - const comboRuntimeConfig = - comboConfig?.config && typeof comboConfig.config === "object" - ? (comboConfig.config as Record) - : {}; - const comboMode = - typeof comboRuntimeConfig.compressionMode === "string" - ? comboRuntimeConfig.compressionMode - : typeof comboConfig?.compressionOverride === "string" - ? comboConfig.compressionOverride - : null; - if ( - comboMode === "off" || - comboMode === "lite" || - comboMode === "standard" || - comboMode === "aggressive" || - comboMode === "ultra" || - comboMode === "rtk" || - comboMode === "stacked" - ) { - config = { - ...config, - comboOverrides: { - ...(config.comboOverrides ?? {}), - ...(comboName ? { [comboName]: comboMode } : {}), - ...(comboConfig?.id ? { [String(comboConfig.id)]: comboMode } : {}), - }, - }; - compressionComboKey = comboName; - } - const routingComboIds = [ - comboConfig?.id, - comboName, - routingComboId, - comboName?.startsWith("combo/") ? comboName.substring(6) : null, - ].filter((id): id is string => typeof id === "string" && id.length > 0); - if (routingComboIds.length > 0) { - const { getCompressionComboForRoutingCombo } = - await import("../../src/lib/db/compressionCombos.ts"); - const assignedCompressionCombo = - routingComboIds - .map((id) => getCompressionComboForRoutingCombo(id)) - .find((combo) => combo !== null) ?? null; - if ( - applyCompressionComboConfig( - assignedCompressionCombo as RuntimeCompressionCombo | null, - routingComboIds - ) - ) { - compressionComboKey = comboName; - } - } - } catch (err) { - log?.debug?.( - "COMPRESSION", - "Combo compression override lookup skipped: " + - (err instanceof Error ? err.message : String(err)) - ); - } - } - let namedCombos: Record = {}; - try { - const { listCompressionCombos } = await import("../../src/lib/db/compressionCombos.ts"); - namedCombos = buildNamedComboLookup(listCompressionCombos()); - } catch (err) { - log?.debug?.( - "COMPRESSION", - "Named combos load skipped: " + (err instanceof Error ? err.message : String(err)) - ); - } - // Phase 3: per-request override. Unknown values fall through in the resolver (never error). - const compressionHeader = resolveCompressionHeader(clientRawRequest?.headers ?? null); - if (compressionHeader) { - log?.debug?.("COMPRESSION", `x-omniroute-compression header: ${compressionHeader}`); - } - const connectionCacheOverride = resolveConnectionCacheOverride( - credentials?.providerSpecificData - ); - const modeBeforeOutputTransform = selectCompressionStrategy( - config, - compressionComboKey, - estimatedTokens, - body as Record, - { provider, targetFormat, model: effectiveModel, connectionCacheOverride }, - namedCombos, - compressionHeader - ); - if ( - modeBeforeOutputTransform === "stacked" && - !compressionComboApplied && - !config.compressionComboId && - isBuiltinStackedPipeline(config.stackedPipeline) && - // Don't let the legacy default combo override a panel-configured engines map: when the - // operator's explicit engines derive their own stacked pipeline, that pipeline (applied - // below from compressionPlan.stackedPipeline) is authoritative. Legacy/backfilled - // installs (enginesExplicit false) still fall through to the seeded default combo. - !enginesMapDerivesStackedPipeline(config) && - // Never let the legacy seeded default combo shadow the operator's active profile. - !activeComboResolves(config, namedCombos) - ) { - try { - const { getDefaultCompressionCombo } = - await import("../../src/lib/db/compressionCombos.ts"); - const defaultCompressionCombo = getDefaultCompressionCombo(); - if ( - isStackedCompressionCombo(defaultCompressionCombo as RuntimeCompressionCombo | null) && - applyCompressionComboConfig(defaultCompressionCombo as RuntimeCompressionCombo | null) - ) { - log?.debug?.( - "COMPRESSION", - `Default compression combo applied: ${defaultCompressionCombo?.id}` - ); - } - } catch (err) { - log?.debug?.( - "COMPRESSION", - "Default compression combo lookup skipped: " + - (err instanceof Error ? err.message : String(err)) - ); - } - } - // Phase 4A: unified output styles (supersedes cavemanOutputMode via the back-compat shim). - let outputStyleResult: - import("../services/compression/outputStyles/apply.ts").OutputStylesResult | null = null; - if (config.enabled && compressionHeader?.trim().toLowerCase() !== "off") { - try { - const { resolveOutputStyleSelection } = - await import("../services/compression/outputStyles/backCompat.ts"); - const selection = resolveOutputStyleSelection(config); - if (selection.length > 0) { - const { applyOutputStyles, resolveOutputStyleLanguage } = - await import("../services/compression/outputStyles/apply.ts"); - const outputStyleLanguage = resolveOutputStyleLanguage( - config.languageConfig, - body as Parameters[1] - ); - outputStyleResult = applyOutputStyles( - body as Parameters[0], - selection, - outputStyleLanguage - ); - if (outputStyleResult.applied) { - body = outputStyleResult.body as typeof body; - cavemanOutputModeApplied = true; - cavemanOutputModeIntensity = - outputStyleResult.appliedStyles?.map((s) => `${s.id}:${s.level}`).join(",") ?? null; - estimatedTokens = estimateTokens(body?.messages ?? body?.input ?? []); - log?.debug?.("COMPRESSION", "Output styles applied"); - } else if ( - outputStyleResult.skippedReason && - outputStyleResult.skippedReason !== "no_styles" - ) { - log?.debug?.( - "COMPRESSION", - `Output styles skipped: ${outputStyleResult.skippedReason}` - ); - } - } - } catch (err) { - log?.debug?.( - "COMPRESSION", - "Output styles skipped: " + (err instanceof Error ? err.message : String(err)) - ); - } - } - const compressionInputBody = body as Record; - // Adaptive context-budget (Sub-project C): model context window + request max_tokens drive - // the budget target. getTokenLimit is already imported; provider/effectiveModel resolved above. - const adaptiveModelContextLimit = - provider && effectiveModel ? getTokenLimit(provider, effectiveModel) : null; - const requestMaxTokens = - typeof (compressionInputBody as Record)?.max_tokens === "number" - ? ((compressionInputBody as Record).max_tokens as number) - : null; - let adaptiveTelemetry: - import("../services/compression/adaptiveCompression/types.ts").AdaptiveTelemetry | null = - null; - const compressionPlan = selectCompressionPlan( - config, - compressionComboKey, - estimatedTokens, - compressionInputBody, - { provider, targetFormat, model: effectiveModel, connectionCacheOverride }, - namedCombos, - compressionHeader, - { - modelContextLimit: adaptiveModelContextLimit, - requestMaxTokens: requestMaxTokens, - onAdaptive: (t) => { - adaptiveTelemetry = t; - }, - } - ); - const mode = compressionPlan.mode as CompressionConfig["defaultMode"]; - if (adaptiveTelemetry && adaptiveTelemetry.fit === false) { - log?.warn?.( - "COMPRESSION", - `adaptive budget-exceeded: target=${adaptiveTelemetry.target} headroomAfter=${adaptiveTelemetry.headroomAfter} stages=${adaptiveTelemetry.stagesApplied.join(",")} (best-effort plan sent, content preserved)` - ); - } - compressionResponseMeta = formatCompressionMeta(compressionPlan); - // When the per-engine toggle map derives a stacked pipeline (and no named/routing - // combo already set config.stackedPipeline), feed that derived pipeline through so - // applyCompressionAsync (which reads config.stackedPipeline for stacked mode) runs the - // engines the operator actually toggled on instead of the built-in rtk+caveman default. - if ( - mode === "stacked" && - compressionPlan.stackedPipeline.length > 0 && - !compressionComboApplied && - !config.compressionComboId - ) { - config = { - ...config, - stackedPipeline: compressionPlan.stackedPipeline as CompressionConfig["stackedPipeline"], - }; - } - let compressionAnalyticsRecorded = false; - if (mode !== "off") { - // #3890: in a caching context, never compress the system prompt (cacheable prefix) - // even if the operator disabled preserveSystemPrompt — honors the cache-aware flag - // that selectCompressionStrategy can only partially apply via the mode string. - const cacheCtx = { provider, targetFormat, model: effectiveModel, connectionCacheOverride }; - const compressionConfig = resolveCacheAwareConfig(config, compressionInputBody, cacheCtx); - const compressionPrincipalId = apiKeyInfo?.id ? String(apiKeyInfo.id) : undefined; - const compressionOptions = { - model: effectiveModel, - // #7237: feed the AUTHORITATIVE capability (model spec / models.dev sync / DB - // override, with the conservative model-id fragment heuristic only as its - // last-resort fallback) instead of calling the heuristic directly here. The - // heuristic alone wrongly returned false for e.g. gpt-5.5 (registered - // supportsVision:true in modelSpecs but absent from the deliberately-conservative - // fragment list), and lite.ts's gate (`supportsVision !== false`) treated that - // false as "strip every image_url block". Resolves to `null` for genuinely unknown - // models, which is intentionally NOT `false` so the gate still preserves images. - supportsVision: getResolvedModelCapabilities({ provider, model: effectiveModel }) - .supportsVision, - // OmniGlyph uses a measured provider/image-fidelity allowlist. Direct HTTP - // alone is not proof that a route preserves PNG bytes and dimensions. - ...resolveOmniGlyphTransport(provider), - // Sem o provider, a contabilidade do OmniGlyph cai para `unknown` e - // recusa deduzir a semântica de cache (Anthropic usa buckets disjuntos, - // OpenAI reporta cached como subconjunto do input). - provider, - sourceFormat, - targetFormat, - compressionStage: "pre-translation" as const, - config: compressionConfig, - cachingContext: cacheCtx, - principalId: compressionPrincipalId, - // F3.3: stream per-engine progress live (best-effort) before compression.completed. - onEngineStep: (s) => { - try { - const stepPayload = { - requestId: traceId, - comboId: null, - mode, - stepIndex: s.stepIndex, - totalSteps: s.totalSteps, - engine: s.engine, - state: s.state, - originalTokens: s.originalTokens, - compressedTokens: s.compressedTokens, - savingsPercent: s.savingsPercent, - ...(s.durationMs !== undefined ? { durationMs: s.durationMs } : {}), - timestamp: Date.now(), - }; - emit("compression.step", stepPayload); - void forwardDashboardEventToLiveWs("compression.step", stepPayload); - } catch (_stepErr) { - // best-effort live event — never fail the request - } - }, - }; - const runCompression = (input: Record) => - applyCompressionAsync(input, mode, compressionOptions); - const omniglyphSelected = - mode === "omniglyph" || - (mode === "stacked" && - Array.isArray(compressionConfig.stackedPipeline) && - compressionConfig.stackedPipeline.some((step) => - typeof step === "string" ? step === "omniglyph" : step.engine === "omniglyph" - )); - if ( - omniglyphSelected && - (targetFormat === FORMATS.CLAUDE || - targetFormat === FORMATS.OPENAI || - targetFormat === FORMATS.OPENAI_RESPONSES) && - !(sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE) - ) { - runPostTranslationCompression = (input) => - applyCompressionAsync(input, mode, { - ...compressionOptions, - compressionStage: "post-translation" as const, - }); - } - let result: CompressionResult; - if (compressionConfig.liveZone?.enabled === true) { - const { applyLiveZoneCompression } = await import("../services/compression/liveZone.ts"); - const explicitSessionId = - clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" - ? clientRawRequest.headers.get("x-omniroute-session-id") - : getHeaderValueCaseInsensitive( - clientRawRequest?.headers ?? null, - "x-omniroute-session-id" - ); - const liveZoneSessionId = - explicitSessionId || - generateSessionId(compressionInputBody, { - provider, - connectionId: getCurrentConnectionId() ?? undefined, - }) || - undefined; - result = await applyLiveZoneCompression( - compressionInputBody, - { - principalId: compressionPrincipalId, - sessionId: liveZoneSessionId, - variant: { - mode, - provider, - model: effectiveModel, - config: compressionConfig, - cachePrefix: { - system: compressionInputBody.system, - systemInstruction: compressionInputBody.systemInstruction, - system_instruction: compressionInputBody.system_instruction, - instructions: compressionInputBody.instructions, - tools: compressionInputBody.tools, - toolChoice: compressionInputBody.tool_choice, - }, - }, - ttlMinutes: compressionConfig.cacheMinutes, - }, - runCompression - ); - } else { - result = await runCompression(compressionInputBody); - } - if (result.stats) { - const annotation = formatCompressionAnnotation(result.stats); - if (annotation) { - compressionResponseMeta = `${compressionResponseMeta}; ${annotation}`; - } - if (result.compressed) { - body = result.body as typeof body; - estimatedTokens = result.stats.compressedTokens; - tokensCompressed = Math.max( - 0, - result.stats.originalTokens - result.stats.compressedTokens - ); - } - - // Fire-and-forget: emit live compression event for dashboard (U5). - // Guard: only emit when compression actually ran and produced stats. - if (result.compressed && result.stats) { - try { - const compressionCompletedPayload = { - requestId: traceId, - comboId: result.stats.compressionComboId ?? null, - mode, - originalTokens: result.stats.originalTokens, - compressedTokens: result.stats.compressedTokens, - savingsPercent: result.stats.savingsPercent, - // Single-engine modes leave engineBreakdown empty; synthesize a 1-entry - // breakdown so the studio shows a real engine node instead of an empty pipeline. - engineBreakdown: ensureEngineBreakdown(result.stats), - validationWarnings: result.stats.validationWarnings, - fallbackApplied: result.stats.fallbackApplied, - ...(adaptiveTelemetry ? { adaptive: adaptiveTelemetry } : {}), - timestamp: Date.now(), - }; - emit("compression.completed", compressionCompletedPayload); - void forwardDashboardEventToLiveWs( - "compression.completed", - compressionCompletedPayload - ); - } catch (_emitErr) { - // never propagate into the hot path — but log like the sibling - // fire-and-forget blocks so a throwing event bus isn't fully silent. - log?.debug?.( - "COMPRESSION", - "compression.completed emit skipped: " + - (_emitErr instanceof Error ? _emitErr.message : String(_emitErr)) - ); - } - } - - if (result.compressed || result.stats.fallbackApplied || cavemanOutputModeApplied) { - trackCompressionStats(result.stats); - compressionAnalyticsRecorded = true; - compressionAnalyticsWritePromise = writeCompressionAnalytics({ - stats: result.stats, - provider, - effectiveModel, - effectiveServiceTier, - comboName, - mode, - compressionComboId: config.compressionComboId, - skillRequestId, - cavemanOutputModeApplied, - cavemanOutputModeIntensity, - log, - }); - await compressionAnalyticsWritePromise; - } else { - // Compression was attempted (mode active, engines ran) but produced no - // recordable saving — e.g. a Stacked RTK→Caveman pipeline on already-compact - // context. Record a skip row so analytics can distinguish "ran but saved - // nothing" from "never ran" instead of dropping it silently (#4268). - compressionAnalyticsRecorded = true; - compressionAnalyticsWritePromise = writeCompressionSkip( - { - stats: result.stats, - provider, - effectiveModel, - effectiveServiceTier, - comboName, - mode, - compressionComboId: config.compressionComboId, - skillRequestId, - cavemanOutputModeApplied, - cavemanOutputModeIntensity, - log, - }, - "no_savings" - ); - await compressionAnalyticsWritePromise; - } - - if (result.compressed) { - recordCompressionCacheStats({ - compressionInputBody, - provider, - targetFormat, - effectiveModel, - mode, - stats: result.stats, - connectionCacheOverride, - log, - }); - log?.info?.( - "COMPRESSION", - `Prompt compressed (${mode}): ${result.stats.originalTokens} -> ${result.stats.compressedTokens} tokens (${result.stats.savingsPercent}% saved, techniques: ${result.stats.techniquesUsed.join(",")})` - ); - } - } - } - if (cavemanOutputModeApplied && !compressionAnalyticsRecorded) { - compressionAnalyticsWritePromise = writeCavemanOutputAnalytics({ - comboName, - provider, - compressionComboId: config.compressionComboId, - estimatedTokens, - skillRequestId, - cavemanOutputModeIntensity, - log, - }); - await compressionAnalyticsWritePromise; - } - emitOutputStyleTelemetry({ - outputStyleResult, - skillRequestId, - traceId, - effectiveModel, - provider, - compressionComboId: config.compressionComboId, - estimatedTokens, - log, - }); - } catch (err) { - log?.warn?.( - "COMPRESSION", - "Compression pipeline error (non-fatal): " + - (err instanceof Error ? err.message : String(err)) - ); - } - // --- End Modular Compression Pipeline --- - - if (!promptCompressionEnabled) { - log?.debug?.( - "CONTEXT", - reactiveContextCompactionEnabled - ? "Prompt Compression engines disabled; reactive context compaction still applies when over threshold" - : "Prompt Compression engines disabled; reactive context compaction is ALSO disabled — large histories will NOT be trimmed before reaching the upstream provider" - ); - } - if (isCombo && comboName) { - log?.info?.("CONTEXT", `Attempting to resolve combo limits for comboName=${comboName}`); - try { - const { getComboByName } = await import("@/lib/db/combos"); - const { resolveComboTargets } = await import("../services/combo.ts"); - let comboConfig = await getComboByName(comboName); - if (!comboConfig && comboName.startsWith("combo/")) { - comboConfig = await getComboByName(comboName.substring(6)); - } - let comboTargetLimits: number[] = []; - if (comboConfig) { - const allCombosData = await getCombosCached(); - const targets = resolveComboTargets( - comboConfig as unknown as { name: string; models: unknown[] }, - allCombosData as unknown as { name: string; models: unknown[] }[] - ); - // Fall back to ResolvedComboTarget.provider when modelStr lacks a - // provider/ prefix — parseModel alone returns provider:null (#8716). - comboTargetLimits = targets - .map((t: { modelStr?: string; provider?: string }) => - getComboTargetTokenLimit({ modelStr: t.modelStr, provider: t.provider }) - ) - .filter( - (limit): limit is number => - typeof limit === "number" && Number.isFinite(limit) && limit > 0 - ); - } - // chatCore executes per concrete target (handleSingleModel resolves - // provider/effectiveModel before delegating). Compress against THIS - // target's window; min(...allTargets) is only a defensive fallback — - // the old unconditional min compressed a 1M-target request at the - // smallest sibling's window ("agent keeps forgetting things"). - // An explicit `context_length` on the combo record (Agent Features → - // Context length) is an operator declaration and outranks the inferred - // per-target window — see resolveComboContextLimit(). - const rawComboContextLength = (comboConfig as { context_length?: unknown } | null) - ?.context_length; - const comboContextLength = - typeof rawComboContextLength === "number" && - Number.isFinite(rawComboContextLength) && - rawComboContextLength > 0 - ? rawComboContextLength - : null; - const resolved = resolveComboContextLimit({ - provider, - model: effectiveModel, - comboTargetLimits, - comboContextLength, - }); - contextLimit = resolved.limit; - log?.info?.( - "CONTEXT", - `Combo context limit: ${resolved.limit} (source=${resolved.source})` - ); - } catch (err) { - log?.warn?.("CONTEXT", "Failed to resolve combo limits for compression: " + err); - } - } - - const COMPRESSION_THRESHOLD = getProactiveCompressionRatio(); - let reservedTokens = 0; - if (Array.isArray(body.tools)) { - reservedTokens = estimateTokens(body.tools); - } - const threshold = Math.max( - 1, - Math.floor((Math.max(1, contextLimit) - reservedTokens) * COMPRESSION_THRESHOLD) - ); - - log?.debug?.( - "CONTEXT", - `Checking compression: ${estimatedTokens} tokens vs ${threshold} threshold (${contextLimit} limit, ${reservedTokens} reserved)` - ); - - // Capture pre-compression body so translators can access original message - // content even after compression alters it (e.g. stable Kiro conversationId). - preCompressionBody = body; - - // Reactive context compaction is independent of optional prompt-compression - // engines (Caveman/RTK). Codex Desktop / Responses clients need this path even - // when those engines are off, otherwise multi-turn image sessions hard-reject - // at the budget check below (#8560). - if ( - reactiveContextCompactionEnabled && - !nativeCodexPassthrough && - estimatedTokens > threshold - ) { - log?.info?.( - "CONTEXT", - `Proactive compression triggered: ${estimatedTokens} tokens > ${threshold} threshold (${contextLimit} limit)` - ); - - // Adapt Responses `input[]` → messages so compressContext can run, then restore. - const ctxAdapter = adaptBodyForCompression(body as Record); - const compressionResult = compressContext(ctxAdapter.body, { - provider, - model: effectiveModel, - maxTokens: threshold, - reserveTokens: 0, - }); - - if (compressionResult.compressed && compressionResult.body) { - body = ctxAdapter.adapted - ? ctxAdapter.restore(compressionResult.body as Record, { - dropMissingMappedItems: true, - }) - : compressionResult.body; - const stats = compressionResult.stats; - tokensCompressed = Math.max(0, (stats?.original ?? 0) - (stats?.final ?? 0)); - const layersInfo = - stats && "layers" in stats && Array.isArray(stats.layers) - ? ` (layers: ${stats.layers.map((l: { name: string }) => l.name).join(", ")})` - : ""; - - log?.info?.( - "CONTEXT", - `Context compressed: ${stats.original} → ${stats.final} tokens${layersInfo}` - ); - - logAuditEvent({ - action: "context.proactive_compression", - actor: apiKeyInfo?.name || "system", - target: connectionId || provider || "chat", - details: { - provider, - model: effectiveModel, - original_tokens: stats.original, - final_tokens: stats.final, - layers: "layers" in stats ? stats.layers : undefined, - }, - }); - } else { - log?.debug?.("CONTEXT", `Compression not applied: context already fits within target`); - } - } - } else { - log?.debug?.( - "CONTEXT", - `Skipping compression check: body=${!!body}, hasMessages=${Array.isArray(allMessages)}` - ); - } + const compressionOutcome = await applyContextCompression({ + body, + allMessages, + apiKeyInfo, + clientRawRequest, + comboName, + connectionId, + credentials, + effectiveModel, + effectiveServiceTier, + getCurrentConnectionId, + isCombo, + log, + provider, + routingComboId, + skillRequestId, + sourceFormat, + targetFormat, + traceId, + cavemanOutputModeApplied, + cavemanOutputModeIntensity, + compressionAnalyticsWritePromise, + compressionResponseMeta, + contextEditingEnabled, + contextLimit, + nativeCodexPassthrough, + preCompressionBody, + reactiveContextCompactionEnabled, + runPostTranslationCompression, + tokensCompressed, + }); + body = compressionOutcome.body; + cavemanOutputModeApplied = compressionOutcome.cavemanOutputModeApplied; + cavemanOutputModeIntensity = compressionOutcome.cavemanOutputModeIntensity; + compressionAnalyticsWritePromise = compressionOutcome.compressionAnalyticsWritePromise; + compressionResponseMeta = compressionOutcome.compressionResponseMeta; + contextEditingEnabled = compressionOutcome.contextEditingEnabled; + contextLimit = compressionOutcome.contextLimit; + preCompressionBody = compressionOutcome.preCompressionBody; + reactiveContextCompactionEnabled = compressionOutcome.reactiveContextCompactionEnabled; + runPostTranslationCompression = compressionOutcome.runPostTranslationCompression; + tokensCompressed = compressionOutcome.tokensCompressed; // Re-check the concrete target after all compression passes. Combo compatibility // filtering is advisory and may preserve an all-incompatible pool; this is the diff --git a/open-sse/handlers/chatCore/contextCompression.ts b/open-sse/handlers/chatCore/contextCompression.ts new file mode 100644 index 000000000000..17def628bd75 --- /dev/null +++ b/open-sse/handlers/chatCore/contextCompression.ts @@ -0,0 +1,915 @@ +import { adaptBodyForCompression } from "../../services/compression/bodyAdapter.ts"; +import { ensureEngineBreakdown } from "../../services/compression/engineBreakdown.ts"; +import { isCompressionExcluded } from "../../services/compression/exclusions.ts"; +import { resolveOmniGlyphTransport } from "../../services/compression/imageTransportPolicy.ts"; +import type { + CompressionConfig, + CompressionPipelineStep, + CompressionResult, +} from "../../services/compression/types.ts"; +import { + compressContext, + estimateTokens, + getComboTargetTokenLimit, + getTokenLimit, + resolveComboContextLimit, +} from "../../services/contextManager.ts"; +import { generateSessionId } from "../../services/sessionManager.ts"; +import { FORMATS } from "../../translator/formats.ts"; +import { resolveConnectionCacheOverride } from "../../utils/cacheControlPolicy.ts"; +import { writeCavemanOutputAnalytics } from "./cavemanOutputAnalytics.ts"; +import { getCombosCached } from "./comboContextCache.ts"; +import { writeCompressionAnalytics, writeCompressionSkip } from "./compressionAnalyticsWrite.ts"; +import { recordCompressionCacheStats } from "./compressionCacheStats.ts"; +import { + type RuntimeCompressionCombo, + isBuiltinStackedPipeline, + isStackedCompressionCombo, +} from "./compressionComboPredicates.ts"; +import { + createDisabledCompressionConfig, + resolveCompressionSettings, +} from "./compressionSettings.ts"; +import { getHeaderValueCaseInsensitive, resolveCompressionHeader } from "./headers.ts"; +import { emitOutputStyleTelemetry } from "./outputStyleTelemetry.ts"; +import { forwardDashboardEventToLiveWs } from "./telemetryHelpers.ts"; +import { logAuditEvent } from "@/lib/compliance"; +import { getProactiveCompressionRatio } from "@/lib/db/compression"; +import { emit } from "@/lib/events/eventBus"; +import { getResolvedModelCapabilities } from "@/lib/modelCapabilities.ts"; + +/** + * Prompt compression, reactive context compaction and caveman output styling. + * + * Lifted verbatim out of handleChatCore. The block reads a wide slice of the + * request state and writes back twelve values, so the contract is explicit: + * everything it reads arrives as a parameter, everything it reassigns comes + * back in the result and the caller reseats it. No behaviour change. + */ +type CompressionBody = Record; +type PostTranslationCompressor = + ((input: Record) => Promise) | null; + +export interface ApplyContextCompressionInput { + body: CompressionBody; + allMessages: unknown; + apiKeyInfo: Record | null; + clientRawRequest: Request | null | undefined; + comboName: string | null | undefined; + connectionId: string | null | undefined; + credentials: Record | null | undefined; + effectiveModel: string; + effectiveServiceTier: string | null | undefined; + getCurrentConnectionId: () => string | null | undefined; + isCombo: boolean; + log: Record | null | undefined; + provider: string; + routingComboId: string | null; + skillRequestId: string | null | undefined; + sourceFormat: string; + targetFormat: string; + traceId: string; + // mutable carry-ins + cavemanOutputModeApplied: boolean; + cavemanOutputModeIntensity: string | null; + compressionAnalyticsWritePromise: Promise | null; + compressionResponseMeta: string | null; + contextEditingEnabled: boolean; + contextLimit: number; + nativeCodexPassthrough: boolean; + preCompressionBody: CompressionBody | null; + reactiveContextCompactionEnabled: boolean; + runPostTranslationCompression: PostTranslationCompressor; + tokensCompressed: number; +} + +export interface ApplyContextCompressionResult { + body: CompressionBody; + cavemanOutputModeApplied: boolean; + cavemanOutputModeIntensity: string | null; + compressionAnalyticsWritePromise: Promise | null; + compressionResponseMeta: string | null; + contextEditingEnabled: boolean; + contextLimit: number; + preCompressionBody: CompressionBody | null; + reactiveContextCompactionEnabled: boolean; + runPostTranslationCompression: PostTranslationCompressor; + tokensCompressed: number; +} + +export async function applyContextCompression({ + body, + allMessages, + apiKeyInfo, + clientRawRequest, + comboName, + connectionId, + credentials, + effectiveModel, + effectiveServiceTier, + getCurrentConnectionId, + isCombo, + log, + provider, + routingComboId, + skillRequestId, + sourceFormat, + targetFormat, + traceId, + cavemanOutputModeApplied, + cavemanOutputModeIntensity, + compressionAnalyticsWritePromise, + compressionResponseMeta, + contextEditingEnabled, + contextLimit, + nativeCodexPassthrough, + preCompressionBody, + reactiveContextCompactionEnabled, + runPostTranslationCompression, + tokensCompressed, +}: ApplyContextCompressionInput): Promise { + // prettier-ignore + if (body && Array.isArray(allMessages) && allMessages.length > 0) { + let estimatedTokens = estimateTokens(allMessages); + const compressionSettingsResult = await resolveCompressionSettings(log); + const compressionSettings: CompressionConfig | null = compressionSettingsResult.settings; + // #8034 — operator-named model/endpoint exclusions bypass the whole pipeline, exactly + // like compression being globally disabled, so the body is provably byte-identical. + // Native Codex passthrough is deliberately NOT part of this exclusion: prompt + // compression runs through adaptBodyForCompression() (Responses input[] → messages + // → restore) with codex tool-output eligibility guards, so native contexts still + // compress (regression: #8933 introduced the passthrough bypass, landed on release + // via #11088, silencing codex analytics to skip_reason='excluded'). Reactive + // compaction + combo overflow fail-fast below still bypass native passthrough — + // intentionally left for follow-up. Operators who want byte-identical passthrough + // can add `codex/*` to the exclusions list. + const compressionExcluded = isCompressionExcluded( + { provider, model: effectiveModel }, + compressionSettings?.exclusions + ); + // A per-key opt-out is a request-scoped hard kill for prompt compression. It + // deliberately does not disable the independent reactive context-fit safety + // passes, matching the existing x-omniroute-compression: off contract. + const apiKeyCompressionEnabled = apiKeyInfo?.compressionEnabled !== false; + let promptCompressionEnabled = + compressionSettingsResult.enabled && !compressionExcluded && apiKeyCompressionEnabled; + reactiveContextCompactionEnabled = compressionSettingsResult.enabled && !compressionExcluded; + contextEditingEnabled = compressionSettingsResult.contextEditingEnabled; + if (!apiKeyCompressionEnabled) { + log?.debug?.("COMPRESSION", "Prompt compression disabled for this API key"); + } + if (compressionExcluded) { + void writeCompressionSkip( + { + stats: { + originalTokens: estimatedTokens, + compressedTokens: estimatedTokens, + savingsPercent: 0, + techniquesUsed: [], + mode: "off", + timestamp: Date.now(), + }, + provider, + effectiveModel, + effectiveServiceTier, + comboName, + mode: "off", + compressionComboId: null, + skillRequestId, + cavemanOutputModeApplied: false, + cavemanOutputModeIntensity: null, + log, + }, + "excluded" + ); + } + + // --- Modular Compression Pipeline (Phase 1 Lite + Phase 2 Standard/Caveman + Phase 3 Aggressive) --- + // Runs BEFORE the existing reactive compressContext() to proactively reduce tokens. + try { + const { + selectCompressionStrategy, + selectCompressionPlan, + enginesMapDerivesStackedPipeline, + activeComboResolves, + applyCompressionAsync, + resolveCacheAwareConfig, + formatCompressionMeta, + buildNamedComboLookup, + formatCompressionAnnotation, + } = await import("../services/compression/strategySelector.ts"); + const { trackCompressionStats } = await import("../services/compression/stats.ts"); + let config: CompressionConfig = compressionSettings ?? createDisabledCompressionConfig(); + if (compressionExcluded || !apiKeyCompressionEnabled) { + config = { ...config, enabled: false }; + } + if (!promptCompressionEnabled || !compressionSettings) { + log?.debug?.("COMPRESSION", "Prompt compression disabled or unavailable"); + } + let compressionComboKey = comboName ?? null; + let compressionComboApplied = false; + const applyCompressionComboConfig = ( + compressionCombo: RuntimeCompressionCombo | null, + routingOverrideIds: string[] = [] + ): boolean => { + if (!compressionCombo || compressionCombo.pipeline.length === 0) return false; + const comboLanguagePacks = [ + ...new Set( + compressionCombo.languagePacks + .map((pack) => pack.trim()) + .filter((pack) => pack.length > 0) + ), + ]; + const comboOutputIntensity = ( + ["lite", "full", "ultra"].includes(compressionCombo.outputModeIntensity) + ? compressionCombo.outputModeIntensity + : (config.cavemanOutputMode?.intensity ?? "full") + ) as "lite" | "full" | "ultra"; + const comboDefaultLanguage = + comboLanguagePacks.find((pack) => pack === config.languageConfig?.defaultLanguage) ?? + comboLanguagePacks[0] ?? + config.languageConfig?.defaultLanguage ?? + "en"; + const comboOverrides = { ...(config.comboOverrides ?? {}) }; + for (const id of routingOverrideIds) { + if (id) comboOverrides[id] = "stacked"; + } + config = { + ...config, + compressionComboId: compressionCombo.id, + stackedPipeline: compressionCombo.pipeline, + languageConfig: { + ...(config.languageConfig ?? { + enabled: false, + defaultLanguage: "en", + autoDetect: true, + enabledPacks: ["en"], + }), + enabled: true, + defaultLanguage: comboDefaultLanguage, + enabledPacks: + comboLanguagePacks.length > 0 + ? comboLanguagePacks + : (config.languageConfig?.enabledPacks ?? ["en"]), + }, + cavemanOutputMode: { + ...(config.cavemanOutputMode ?? { + enabled: false, + intensity: "full", + autoClarity: true, + }), + enabled: compressionCombo.outputMode, + intensity: comboOutputIntensity, + }, + comboOverrides, + }; + compressionComboApplied = true; + return true; + }; + if ((isCombo && comboName) || routingComboId) { + try { + const { getComboByName } = await import("@/lib/db/combos"); + let comboConfig = await getComboByName(comboName); + if (!comboConfig && comboName?.startsWith("combo/")) { + comboConfig = await getComboByName(comboName.substring(6)); + } + const comboRuntimeConfig = + comboConfig?.config && typeof comboConfig.config === "object" + ? (comboConfig.config as Record) + : {}; + const comboMode = + typeof comboRuntimeConfig.compressionMode === "string" + ? comboRuntimeConfig.compressionMode + : typeof comboConfig?.compressionOverride === "string" + ? comboConfig.compressionOverride + : null; + if ( + comboMode === "off" || + comboMode === "lite" || + comboMode === "standard" || + comboMode === "aggressive" || + comboMode === "ultra" || + comboMode === "rtk" || + comboMode === "stacked" + ) { + config = { + ...config, + comboOverrides: { + ...(config.comboOverrides ?? {}), + ...(comboName ? { [comboName]: comboMode } : {}), + ...(comboConfig?.id ? { [String(comboConfig.id)]: comboMode } : {}), + }, + }; + compressionComboKey = comboName; + } + const routingComboIds = [ + comboConfig?.id, + comboName, + routingComboId, + comboName?.startsWith("combo/") ? comboName.substring(6) : null, + ].filter((id): id is string => typeof id === "string" && id.length > 0); + if (routingComboIds.length > 0) { + const { getCompressionComboForRoutingCombo } = + await import("../../src/lib/db/compressionCombos.ts"); + const assignedCompressionCombo = + routingComboIds + .map((id) => getCompressionComboForRoutingCombo(id)) + .find((combo) => combo !== null) ?? null; + if ( + applyCompressionComboConfig( + assignedCompressionCombo as RuntimeCompressionCombo | null, + routingComboIds + ) + ) { + compressionComboKey = comboName; + } + } + } catch (err) { + log?.debug?.( + "COMPRESSION", + "Combo compression override lookup skipped: " + + (err instanceof Error ? err.message : String(err)) + ); + } + } + let namedCombos: Record = {}; + try { + const { listCompressionCombos } = await import("../../src/lib/db/compressionCombos.ts"); + namedCombos = buildNamedComboLookup(listCompressionCombos()); + } catch (err) { + log?.debug?.( + "COMPRESSION", + "Named combos load skipped: " + (err instanceof Error ? err.message : String(err)) + ); + } + // Phase 3: per-request override. Unknown values fall through in the resolver (never error). + const compressionHeader = resolveCompressionHeader(clientRawRequest?.headers ?? null); + if (compressionHeader) { + log?.debug?.("COMPRESSION", `x-omniroute-compression header: ${compressionHeader}`); + } + const connectionCacheOverride = resolveConnectionCacheOverride( + credentials?.providerSpecificData + ); + const modeBeforeOutputTransform = selectCompressionStrategy( + config, + compressionComboKey, + estimatedTokens, + body as Record, + { provider, targetFormat, model: effectiveModel, connectionCacheOverride }, + namedCombos, + compressionHeader + ); + if ( + modeBeforeOutputTransform === "stacked" && + !compressionComboApplied && + !config.compressionComboId && + isBuiltinStackedPipeline(config.stackedPipeline) && + // Don't let the legacy default combo override a panel-configured engines map: when the + // operator's explicit engines derive their own stacked pipeline, that pipeline (applied + // below from compressionPlan.stackedPipeline) is authoritative. Legacy/backfilled + // installs (enginesExplicit false) still fall through to the seeded default combo. + !enginesMapDerivesStackedPipeline(config) && + // Never let the legacy seeded default combo shadow the operator's active profile. + !activeComboResolves(config, namedCombos) + ) { + try { + const { getDefaultCompressionCombo } = + await import("../../src/lib/db/compressionCombos.ts"); + const defaultCompressionCombo = getDefaultCompressionCombo(); + if ( + isStackedCompressionCombo(defaultCompressionCombo as RuntimeCompressionCombo | null) && + applyCompressionComboConfig(defaultCompressionCombo as RuntimeCompressionCombo | null) + ) { + log?.debug?.( + "COMPRESSION", + `Default compression combo applied: ${defaultCompressionCombo?.id}` + ); + } + } catch (err) { + log?.debug?.( + "COMPRESSION", + "Default compression combo lookup skipped: " + + (err instanceof Error ? err.message : String(err)) + ); + } + } + // Phase 4A: unified output styles (supersedes cavemanOutputMode via the back-compat shim). + let outputStyleResult: + import("../services/compression/outputStyles/apply.ts").OutputStylesResult | null = null; + if (config.enabled && compressionHeader?.trim().toLowerCase() !== "off") { + try { + const { resolveOutputStyleSelection } = + await import("../services/compression/outputStyles/backCompat.ts"); + const selection = resolveOutputStyleSelection(config); + if (selection.length > 0) { + const { applyOutputStyles, resolveOutputStyleLanguage } = + await import("../services/compression/outputStyles/apply.ts"); + const outputStyleLanguage = resolveOutputStyleLanguage( + config.languageConfig, + body as Parameters[1] + ); + outputStyleResult = applyOutputStyles( + body as Parameters[0], + selection, + outputStyleLanguage + ); + if (outputStyleResult.applied) { + body = outputStyleResult.body as typeof body; + cavemanOutputModeApplied = true; + cavemanOutputModeIntensity = + outputStyleResult.appliedStyles?.map((s) => `${s.id}:${s.level}`).join(",") ?? null; + estimatedTokens = estimateTokens(body?.messages ?? body?.input ?? []); + log?.debug?.("COMPRESSION", "Output styles applied"); + } else if ( + outputStyleResult.skippedReason && + outputStyleResult.skippedReason !== "no_styles" + ) { + log?.debug?.( + "COMPRESSION", + `Output styles skipped: ${outputStyleResult.skippedReason}` + ); + } + } + } catch (err) { + log?.debug?.( + "COMPRESSION", + "Output styles skipped: " + (err instanceof Error ? err.message : String(err)) + ); + } + } + const compressionInputBody = body as Record; + // Adaptive context-budget (Sub-project C): model context window + request max_tokens drive + // the budget target. getTokenLimit is already imported; provider/effectiveModel resolved above. + const adaptiveModelContextLimit = + provider && effectiveModel ? getTokenLimit(provider, effectiveModel) : null; + const requestMaxTokens = + typeof (compressionInputBody as Record)?.max_tokens === "number" + ? ((compressionInputBody as Record).max_tokens as number) + : null; + let adaptiveTelemetry: + import("../services/compression/adaptiveCompression/types.ts").AdaptiveTelemetry | null = + null; + const compressionPlan = selectCompressionPlan( + config, + compressionComboKey, + estimatedTokens, + compressionInputBody, + { provider, targetFormat, model: effectiveModel, connectionCacheOverride }, + namedCombos, + compressionHeader, + { + modelContextLimit: adaptiveModelContextLimit, + requestMaxTokens: requestMaxTokens, + onAdaptive: (t) => { + adaptiveTelemetry = t; + }, + } + ); + const mode = compressionPlan.mode as CompressionConfig["defaultMode"]; + if (adaptiveTelemetry && adaptiveTelemetry.fit === false) { + log?.warn?.( + "COMPRESSION", + `adaptive budget-exceeded: target=${adaptiveTelemetry.target} headroomAfter=${adaptiveTelemetry.headroomAfter} stages=${adaptiveTelemetry.stagesApplied.join(",")} (best-effort plan sent, content preserved)` + ); + } + compressionResponseMeta = formatCompressionMeta(compressionPlan); + // When the per-engine toggle map derives a stacked pipeline (and no named/routing + // combo already set config.stackedPipeline), feed that derived pipeline through so + // applyCompressionAsync (which reads config.stackedPipeline for stacked mode) runs the + // engines the operator actually toggled on instead of the built-in rtk+caveman default. + if ( + mode === "stacked" && + compressionPlan.stackedPipeline.length > 0 && + !compressionComboApplied && + !config.compressionComboId + ) { + config = { + ...config, + stackedPipeline: compressionPlan.stackedPipeline as CompressionConfig["stackedPipeline"], + }; + } + let compressionAnalyticsRecorded = false; + if (mode !== "off") { + // #3890: in a caching context, never compress the system prompt (cacheable prefix) + // even if the operator disabled preserveSystemPrompt — honors the cache-aware flag + // that selectCompressionStrategy can only partially apply via the mode string. + const cacheCtx = { provider, targetFormat, model: effectiveModel, connectionCacheOverride }; + const compressionConfig = resolveCacheAwareConfig(config, compressionInputBody, cacheCtx); + const compressionPrincipalId = apiKeyInfo?.id ? String(apiKeyInfo.id) : undefined; + const compressionOptions = { + model: effectiveModel, + // #7237: feed the AUTHORITATIVE capability (model spec / models.dev sync / DB + // override, with the conservative model-id fragment heuristic only as its + // last-resort fallback) instead of calling the heuristic directly here. The + // heuristic alone wrongly returned false for e.g. gpt-5.5 (registered + // supportsVision:true in modelSpecs but absent from the deliberately-conservative + // fragment list), and lite.ts's gate (`supportsVision !== false`) treated that + // false as "strip every image_url block". Resolves to `null` for genuinely unknown + // models, which is intentionally NOT `false` so the gate still preserves images. + supportsVision: getResolvedModelCapabilities({ provider, model: effectiveModel }) + .supportsVision, + // OmniGlyph uses a measured provider/image-fidelity allowlist. Direct HTTP + // alone is not proof that a route preserves PNG bytes and dimensions. + ...resolveOmniGlyphTransport(provider), + // Sem o provider, a contabilidade do OmniGlyph cai para `unknown` e + // recusa deduzir a semântica de cache (Anthropic usa buckets disjuntos, + // OpenAI reporta cached como subconjunto do input). + provider, + sourceFormat, + targetFormat, + compressionStage: "pre-translation" as const, + config: compressionConfig, + cachingContext: cacheCtx, + principalId: compressionPrincipalId, + // F3.3: stream per-engine progress live (best-effort) before compression.completed. + onEngineStep: (s) => { + try { + const stepPayload = { + requestId: traceId, + comboId: null, + mode, + stepIndex: s.stepIndex, + totalSteps: s.totalSteps, + engine: s.engine, + state: s.state, + originalTokens: s.originalTokens, + compressedTokens: s.compressedTokens, + savingsPercent: s.savingsPercent, + ...(s.durationMs !== undefined ? { durationMs: s.durationMs } : {}), + timestamp: Date.now(), + }; + emit("compression.step", stepPayload); + void forwardDashboardEventToLiveWs("compression.step", stepPayload); + } catch (_stepErr) { + // best-effort live event — never fail the request + } + }, + }; + const runCompression = (input: Record) => + applyCompressionAsync(input, mode, compressionOptions); + const omniglyphSelected = + mode === "omniglyph" || + (mode === "stacked" && + Array.isArray(compressionConfig.stackedPipeline) && + compressionConfig.stackedPipeline.some((step) => + typeof step === "string" ? step === "omniglyph" : step.engine === "omniglyph" + )); + if ( + omniglyphSelected && + (targetFormat === FORMATS.CLAUDE || + targetFormat === FORMATS.OPENAI || + targetFormat === FORMATS.OPENAI_RESPONSES) && + !(sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE) + ) { + runPostTranslationCompression = (input) => + applyCompressionAsync(input, mode, { + ...compressionOptions, + compressionStage: "post-translation" as const, + }); + } + let result: CompressionResult; + if (compressionConfig.liveZone?.enabled === true) { + const { applyLiveZoneCompression } = await import("../services/compression/liveZone.ts"); + const explicitSessionId = + clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" + ? clientRawRequest.headers.get("x-omniroute-session-id") + : getHeaderValueCaseInsensitive( + clientRawRequest?.headers ?? null, + "x-omniroute-session-id" + ); + const liveZoneSessionId = + explicitSessionId || + generateSessionId(compressionInputBody, { + provider, + connectionId: getCurrentConnectionId() ?? undefined, + }) || + undefined; + result = await applyLiveZoneCompression( + compressionInputBody, + { + principalId: compressionPrincipalId, + sessionId: liveZoneSessionId, + variant: { + mode, + provider, + model: effectiveModel, + config: compressionConfig, + cachePrefix: { + system: compressionInputBody.system, + systemInstruction: compressionInputBody.systemInstruction, + system_instruction: compressionInputBody.system_instruction, + instructions: compressionInputBody.instructions, + tools: compressionInputBody.tools, + toolChoice: compressionInputBody.tool_choice, + }, + }, + ttlMinutes: compressionConfig.cacheMinutes, + }, + runCompression + ); + } else { + result = await runCompression(compressionInputBody); + } + if (result.stats) { + const annotation = formatCompressionAnnotation(result.stats); + if (annotation) { + compressionResponseMeta = `${compressionResponseMeta}; ${annotation}`; + } + if (result.compressed) { + body = result.body as typeof body; + estimatedTokens = result.stats.compressedTokens; + tokensCompressed = Math.max( + 0, + result.stats.originalTokens - result.stats.compressedTokens + ); + } + + // Fire-and-forget: emit live compression event for dashboard (U5). + // Guard: only emit when compression actually ran and produced stats. + if (result.compressed && result.stats) { + try { + const compressionCompletedPayload = { + requestId: traceId, + comboId: result.stats.compressionComboId ?? null, + mode, + originalTokens: result.stats.originalTokens, + compressedTokens: result.stats.compressedTokens, + savingsPercent: result.stats.savingsPercent, + // Single-engine modes leave engineBreakdown empty; synthesize a 1-entry + // breakdown so the studio shows a real engine node instead of an empty pipeline. + engineBreakdown: ensureEngineBreakdown(result.stats), + validationWarnings: result.stats.validationWarnings, + fallbackApplied: result.stats.fallbackApplied, + ...(adaptiveTelemetry ? { adaptive: adaptiveTelemetry } : {}), + timestamp: Date.now(), + }; + emit("compression.completed", compressionCompletedPayload); + void forwardDashboardEventToLiveWs( + "compression.completed", + compressionCompletedPayload + ); + } catch (_emitErr) { + // never propagate into the hot path — but log like the sibling + // fire-and-forget blocks so a throwing event bus isn't fully silent. + log?.debug?.( + "COMPRESSION", + "compression.completed emit skipped: " + + (_emitErr instanceof Error ? _emitErr.message : String(_emitErr)) + ); + } + } + + if (result.compressed || result.stats.fallbackApplied || cavemanOutputModeApplied) { + trackCompressionStats(result.stats); + compressionAnalyticsRecorded = true; + compressionAnalyticsWritePromise = writeCompressionAnalytics({ + stats: result.stats, + provider, + effectiveModel, + effectiveServiceTier, + comboName, + mode, + compressionComboId: config.compressionComboId, + skillRequestId, + cavemanOutputModeApplied, + cavemanOutputModeIntensity, + log, + }); + await compressionAnalyticsWritePromise; + } else { + // Compression was attempted (mode active, engines ran) but produced no + // recordable saving — e.g. a Stacked RTK→Caveman pipeline on already-compact + // context. Record a skip row so analytics can distinguish "ran but saved + // nothing" from "never ran" instead of dropping it silently (#4268). + compressionAnalyticsRecorded = true; + compressionAnalyticsWritePromise = writeCompressionSkip( + { + stats: result.stats, + provider, + effectiveModel, + effectiveServiceTier, + comboName, + mode, + compressionComboId: config.compressionComboId, + skillRequestId, + cavemanOutputModeApplied, + cavemanOutputModeIntensity, + log, + }, + "no_savings" + ); + await compressionAnalyticsWritePromise; + } + + if (result.compressed) { + recordCompressionCacheStats({ + compressionInputBody, + provider, + targetFormat, + effectiveModel, + mode, + stats: result.stats, + connectionCacheOverride, + log, + }); + log?.info?.( + "COMPRESSION", + `Prompt compressed (${mode}): ${result.stats.originalTokens} -> ${result.stats.compressedTokens} tokens (${result.stats.savingsPercent}% saved, techniques: ${result.stats.techniquesUsed.join(",")})` + ); + } + } + } + if (cavemanOutputModeApplied && !compressionAnalyticsRecorded) { + compressionAnalyticsWritePromise = writeCavemanOutputAnalytics({ + comboName, + provider, + compressionComboId: config.compressionComboId, + estimatedTokens, + skillRequestId, + cavemanOutputModeIntensity, + log, + }); + await compressionAnalyticsWritePromise; + } + emitOutputStyleTelemetry({ + outputStyleResult, + skillRequestId, + traceId, + effectiveModel, + provider, + compressionComboId: config.compressionComboId, + estimatedTokens, + log, + }); + } catch (err) { + log?.warn?.( + "COMPRESSION", + "Compression pipeline error (non-fatal): " + + (err instanceof Error ? err.message : String(err)) + ); + } + // --- End Modular Compression Pipeline --- + + if (!promptCompressionEnabled) { + log?.debug?.( + "CONTEXT", + reactiveContextCompactionEnabled + ? "Prompt Compression engines disabled; reactive context compaction still applies when over threshold" + : "Prompt Compression engines disabled; reactive context compaction is ALSO disabled — large histories will NOT be trimmed before reaching the upstream provider" + ); + } + if (isCombo && comboName) { + log?.info?.("CONTEXT", `Attempting to resolve combo limits for comboName=${comboName}`); + try { + const { getComboByName } = await import("@/lib/db/combos"); + const { resolveComboTargets } = await import("../services/combo.ts"); + let comboConfig = await getComboByName(comboName); + if (!comboConfig && comboName.startsWith("combo/")) { + comboConfig = await getComboByName(comboName.substring(6)); + } + let comboTargetLimits: number[] = []; + if (comboConfig) { + const allCombosData = await getCombosCached(); + const targets = resolveComboTargets( + comboConfig as unknown as { name: string; models: unknown[] }, + allCombosData as unknown as { name: string; models: unknown[] }[] + ); + // Fall back to ResolvedComboTarget.provider when modelStr lacks a + // provider/ prefix — parseModel alone returns provider:null (#8716). + comboTargetLimits = targets + .map((t: { modelStr?: string; provider?: string }) => + getComboTargetTokenLimit({ modelStr: t.modelStr, provider: t.provider }) + ) + .filter( + (limit): limit is number => + typeof limit === "number" && Number.isFinite(limit) && limit > 0 + ); + } + // chatCore executes per concrete target (handleSingleModel resolves + // provider/effectiveModel before delegating). Compress against THIS + // target's window; min(...allTargets) is only a defensive fallback — + // the old unconditional min compressed a 1M-target request at the + // smallest sibling's window ("agent keeps forgetting things"). + // An explicit `context_length` on the combo record (Agent Features → + // Context length) is an operator declaration and outranks the inferred + // per-target window — see resolveComboContextLimit(). + const rawComboContextLength = (comboConfig as { context_length?: unknown } | null) + ?.context_length; + const comboContextLength = + typeof rawComboContextLength === "number" && + Number.isFinite(rawComboContextLength) && + rawComboContextLength > 0 + ? rawComboContextLength + : null; + const resolved = resolveComboContextLimit({ + provider, + model: effectiveModel, + comboTargetLimits, + comboContextLength, + }); + contextLimit = resolved.limit; + log?.info?.( + "CONTEXT", + `Combo context limit: ${resolved.limit} (source=${resolved.source})` + ); + } catch (err) { + log?.warn?.("CONTEXT", "Failed to resolve combo limits for compression: " + err); + } + } + + const COMPRESSION_THRESHOLD = getProactiveCompressionRatio(); + let reservedTokens = 0; + if (Array.isArray(body.tools)) { + reservedTokens = estimateTokens(body.tools); + } + const threshold = Math.max( + 1, + Math.floor((Math.max(1, contextLimit) - reservedTokens) * COMPRESSION_THRESHOLD) + ); + + log?.debug?.( + "CONTEXT", + `Checking compression: ${estimatedTokens} tokens vs ${threshold} threshold (${contextLimit} limit, ${reservedTokens} reserved)` + ); + + // Capture pre-compression body so translators can access original message + // content even after compression alters it (e.g. stable Kiro conversationId). + preCompressionBody = body; + + // Reactive context compaction is independent of optional prompt-compression + // engines (Caveman/RTK). Codex Desktop / Responses clients need this path even + // when those engines are off, otherwise multi-turn image sessions hard-reject + // at the budget check below (#8560). + if ( + reactiveContextCompactionEnabled && + !nativeCodexPassthrough && + estimatedTokens > threshold + ) { + log?.info?.( + "CONTEXT", + `Proactive compression triggered: ${estimatedTokens} tokens > ${threshold} threshold (${contextLimit} limit)` + ); + + // Adapt Responses `input[]` → messages so compressContext can run, then restore. + const ctxAdapter = adaptBodyForCompression(body as Record); + const compressionResult = compressContext(ctxAdapter.body, { + provider, + model: effectiveModel, + maxTokens: threshold, + reserveTokens: 0, + }); + + if (compressionResult.compressed && compressionResult.body) { + body = ctxAdapter.adapted + ? ctxAdapter.restore(compressionResult.body as Record, { + dropMissingMappedItems: true, + }) + : compressionResult.body; + const stats = compressionResult.stats; + tokensCompressed = Math.max(0, (stats?.original ?? 0) - (stats?.final ?? 0)); + const layersInfo = + stats && "layers" in stats && Array.isArray(stats.layers) + ? ` (layers: ${stats.layers.map((l: { name: string }) => l.name).join(", ")})` + : ""; + + log?.info?.( + "CONTEXT", + `Context compressed: ${stats.original} → ${stats.final} tokens${layersInfo}` + ); + + logAuditEvent({ + action: "context.proactive_compression", + actor: apiKeyInfo?.name || "system", + target: connectionId || provider || "chat", + details: { + provider, + model: effectiveModel, + original_tokens: stats.original, + final_tokens: stats.final, + layers: "layers" in stats ? stats.layers : undefined, + }, + }); + } else { + log?.debug?.("CONTEXT", `Compression not applied: context already fits within target`); + } + } + } else { + log?.debug?.( + "CONTEXT", + `Skipping compression check: body=${!!body}, hasMessages=${Array.isArray(allMessages)}` + ); + } + return { + body, + cavemanOutputModeApplied, + cavemanOutputModeIntensity, + compressionAnalyticsWritePromise, + compressionResponseMeta, + contextEditingEnabled, + contextLimit, + preCompressionBody, + reactiveContextCompactionEnabled, + runPostTranslationCompression, + tokensCompressed, + }; +} diff --git a/tests/unit/chatcore/context-compression-carry-out.test.ts b/tests/unit/chatcore/context-compression-carry-out.test.ts new file mode 100644 index 000000000000..df6805f8b51f --- /dev/null +++ b/tests/unit/chatcore/context-compression-carry-out.test.ts @@ -0,0 +1,142 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import ts from "typescript"; + +// The compression block was lifted out of handleChatCore into its own leaf. +// Everything it reassigns now has to travel back through the result object and +// be reseated by the caller. Dropping one of those write-backs is silent: the +// compression still runs, its output is just thrown away, and every existing +// compression suite stays green because they exercise the leaf's internals +// rather than the barrel's plumbing. +// +// This pins the plumbing itself: every field the leaf returns must be reseated +// by the barrel. +// +// Both halves are read through the TypeScript parser rather than by regex. A +// text pattern here would encode today's formatting — indentation, whether the +// return object is split across lines — and go red on a reformat that changed +// no behaviour at all. + +const leafPath = new URL( + "../../../open-sse/handlers/chatCore/contextCompression.ts", + import.meta.url +); +const barrelPath = new URL("../../../open-sse/handlers/chatCore.ts", import.meta.url); + +function parse(path: URL) { + return ts.createSourceFile( + path.pathname, + readFileSync(path, "utf8"), + ts.ScriptTarget.ESNext, + true + ); +} + +/** Shorthand property names on the object literal `applyContextCompression` returns. */ +function returnedFields(): string[] { + const sf = parse(leafPath); + let fn: ts.FunctionDeclaration | null = null; + const findFn = (node: ts.Node) => { + if (ts.isFunctionDeclaration(node) && node.name?.text === "applyContextCompression") { + fn = node; + } + ts.forEachChild(node, findFn); + }; + findFn(sf); + assert.ok(fn, "applyContextCompression must exist in the leaf"); + + const body = (fn as ts.FunctionDeclaration).body; + assert.ok(body, "applyContextCompression must have a body"); + + const returns: ts.ReturnStatement[] = []; + const findReturns = (node: ts.Node) => { + // Nested functions carry their own returns; stop before descending into one. + if (ts.isFunctionLike(node) && node !== fn) return; + if (ts.isReturnStatement(node)) returns.push(node); + ts.forEachChild(node, findReturns); + }; + ts.forEachChild(body, findReturns); + + assert.equal(returns.length, 1, "the leaf must have exactly one exit so the contract is total"); + const expr = returns[0].expression; + assert.ok( + expr && ts.isObjectLiteralExpression(expr), + "the leaf must return an object literal carrying its results" + ); + return expr.properties + .map((prop) => (prop.name && ts.isIdentifier(prop.name) ? prop.name.text : null)) + .filter((name): name is string => name !== null) + .sort(); +} + +/** `x = compressionOutcome.y;` assignments in the barrel, as [target, source] pairs. */ +function barrelWriteBacks(): Array<[string, string]> { + const sf = parse(barrelPath); + const pairs: Array<[string, string]> = []; + const walk = (node: ts.Node) => { + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(node.left) && + ts.isPropertyAccessExpression(node.right) && + ts.isIdentifier(node.right.expression) && + node.right.expression.text === "compressionOutcome" + ) { + pairs.push([node.left.text, node.right.name.text]); + } + ts.forEachChild(node, walk); + }; + walk(sf); + return pairs; +} + +test("the leaf returns every value it reassigns", () => { + const fields = returnedFields(); + // Eleven: the twelve mutable carry-ins minus nativeCodexPassthrough, which is + // read-only. The count is deliberately exact. A new field on the result is + // only half a change — it also needs a write-back in the barrel — so growing + // the contract should break this and send you to the other test below. + assert.equal(fields.length, 11, `expected the full carry-out set, got ${fields.join(", ")}`); + assert.ok(fields.includes("body"), "body is reassigned inside the block"); + assert.ok(fields.includes("tokensCompressed")); + assert.ok(fields.includes("compressionResponseMeta")); +}); + +test("the barrel reseats every field the leaf returns", () => { + const reseated = new Set(barrelWriteBacks().map(([, source]) => source)); + const missing = returnedFields().filter((field) => !reseated.has(field)); + assert.deepEqual( + missing, + [], + `the barrel drops these compression results on the floor: ${missing.join(", ")}` + ); +}); + +test("the barrel does not reseat anything the leaf never returns", () => { + const returned = new Set(returnedFields()); + const pairs = barrelWriteBacks(); + assert.ok(pairs.length > 0, "the barrel must consume the compression result"); + for (const [target, source] of pairs) { + assert.equal(target, source, "a write-back must not cross-wire two fields"); + assert.ok( + returned.has(source), + `the barrel reads compressionOutcome.${source}, which the leaf does not return` + ); + } +}); + +test("nativeCodexPassthrough stays an input, never a carry-out", () => { + // It is destructured as a const from the request prelude. The block reads it + // to decide whether reactive compaction applies, but never assigns it, so + // putting it in the result would produce an assignment to a constant at + // runtime while typechecking clean. + assert.ok( + !returnedFields().includes("nativeCodexPassthrough"), + "nativeCodexPassthrough must not be returned" + ); + assert.ok( + !barrelWriteBacks().some(([, source]) => source === "nativeCodexPassthrough"), + "assigning it would throw: it is a const in the barrel" + ); +}); diff --git a/tests/unit/codex-prompt-compression-passthrough.test.ts b/tests/unit/codex-prompt-compression-passthrough.test.ts index 120c4e1b3562..912e373b8fcb 100644 --- a/tests/unit/codex-prompt-compression-passthrough.test.ts +++ b/tests/unit/codex-prompt-compression-passthrough.test.ts @@ -6,7 +6,15 @@ import { normalizeCompressionExclusions, } from "../../open-sse/services/compression/exclusions.ts"; +// The compression block moved out of chatCore.ts into its own leaf; the guard +// follows the code it guards. const chatCoreSource = readFileSync( + new URL("../../open-sse/handlers/chatCore/contextCompression.ts", import.meta.url), + "utf8" +); + +// The overflow fail-fast gate stayed behind in the barrel. +const barrelSource = readFileSync( new URL("../../open-sse/handlers/chatCore.ts", import.meta.url), "utf8" ); @@ -48,12 +56,14 @@ test("codex target is compressible by default; operators can still opt out via e test("prompt-only scope: reactive compaction still bypasses native passthrough", () => { // Deliberately left for follow-up (overflow fail-fast + history-rewriting safety). // If these gates are ever lifted, update the PR body notes, not just this test. + // The two gates now live in different files: the threshold gate moved out with + // the compression block, the overflow fail-fast stayed in the barrel. assert.match( chatCoreSource, /reactiveContextCompactionEnabled\s*&&\s*!nativeCodexPassthrough\s*&&\s*estimatedTokens\s*>\s*threshold/ ); assert.match( - chatCoreSource, + barrelSource, /reactiveContextCompactionEnabled\s*&&\s*!nativeCodexPassthrough\s*&&\s*finalEstimatedInputTokens\s*>=\s*finalContextLimit/ ); }); From fcc20d740e0f8da3a13d955075b7e1465c2184d3 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 16:54:32 -0400 Subject: [PATCH 08/26] refactor(sse): repair imports and dependency types across the chatCore leaves The extraction moved code into `open-sse/handlers/chatCore/`, one directory deeper than the barrel, and four bindings did not survive the move intact. Relative specifiers kept the barrel's depth. Nine `await import(...)` calls and two type-position `import("...")` annotations in contextCompression.ts still pointed at `../services/...` and `../../src/lib/...`, which resolve to nothing from the leaf directory. These are dynamic, so nothing failed until the branch that contains them actually ran: prompt compression with a combo, output styles, live-zone compression, and the compression-combo lookups. Four symbols were never imported into the leaves that use them: `formatStreamRecoveryRetryWarning` (executeProviderRequest.ts, stream recovery retry warning), `saveRequestUsage` and `buildFailureUsageRecord` plus the `FailureUsageAggregate` type (requestPrelude.ts, failure-usage persistence), and the aliased `attachCompressionUsageReceiptAfterAnalyticsFor` (requestPrelude.ts, compression usage receipts). Each one is a ReferenceError on the path that reaches it. Three dependency fields were typed wider than the values the barrel passes. `log` was `Record`, which makes `log?.debug?.(...)` a call on `unknown`; it now carries the same structured shape executeProviderRequest.ts already declares. `isModelScope` was `boolean` but is a predicate the barrel invokes. `apiKeyInfo` lost its `name`, and `compressionAnalyticsWritePromise` was widened to `Promise` while requestPrelude.ts births it as `Promise`. None of this was visible through `npm run typecheck:core`: that config is an explicit 28-file allowlist, and neither chatCore.ts nor any leaf is on it. Checked against the branch's own files with the repository's compiler settings, chatCore.ts and contextCompression.ts now report zero errors, down from the 36 that had accumulated. Two errors remain in executeProviderRequest.ts, both `isClaudePassthrough` missing from `ExecuteInput`. That field arrives with the Claude OAuth raw passthrough pin and predates this extraction, so it is left for that line of work rather than folded in here. Signed-off-by: Minxi Hou --- .../handlers/chatCore/contextCompression.ts | 38 +++++++++++-------- .../chatCore/executeProviderRequest.ts | 3 +- open-sse/handlers/chatCore/requestPrelude.ts | 4 ++ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/open-sse/handlers/chatCore/contextCompression.ts b/open-sse/handlers/chatCore/contextCompression.ts index 17def628bd75..959eb34a444d 100644 --- a/open-sse/handlers/chatCore/contextCompression.ts +++ b/open-sse/handlers/chatCore/contextCompression.ts @@ -53,7 +53,7 @@ type PostTranslationCompressor = export interface ApplyContextCompressionInput { body: CompressionBody; allMessages: unknown; - apiKeyInfo: Record | null; + apiKeyInfo: ({ name?: string } & Record) | null; clientRawRequest: Request | null | undefined; comboName: string | null | undefined; connectionId: string | null | undefined; @@ -62,7 +62,15 @@ export interface ApplyContextCompressionInput { effectiveServiceTier: string | null | undefined; getCurrentConnectionId: () => string | null | undefined; isCombo: boolean; - log: Record | null | undefined; + log: + | { + debug?: (...args: unknown[]) => void; + info?: (...args: unknown[]) => void; + warn?: (...args: unknown[]) => void; + error?: (...args: unknown[]) => void; + } + | null + | undefined; provider: string; routingComboId: string | null; skillRequestId: string | null | undefined; @@ -72,7 +80,7 @@ export interface ApplyContextCompressionInput { // mutable carry-ins cavemanOutputModeApplied: boolean; cavemanOutputModeIntensity: string | null; - compressionAnalyticsWritePromise: Promise | null; + compressionAnalyticsWritePromise: Promise | null; compressionResponseMeta: string | null; contextEditingEnabled: boolean; contextLimit: number; @@ -87,7 +95,7 @@ export interface ApplyContextCompressionResult { body: CompressionBody; cavemanOutputModeApplied: boolean; cavemanOutputModeIntensity: string | null; - compressionAnalyticsWritePromise: Promise | null; + compressionAnalyticsWritePromise: Promise | null; compressionResponseMeta: string | null; contextEditingEnabled: boolean; contextLimit: number; @@ -197,8 +205,8 @@ export async function applyContextCompression({ formatCompressionMeta, buildNamedComboLookup, formatCompressionAnnotation, - } = await import("../services/compression/strategySelector.ts"); - const { trackCompressionStats } = await import("../services/compression/stats.ts"); + } = await import("../../services/compression/strategySelector.ts"); + const { trackCompressionStats } = await import("../../services/compression/stats.ts"); let config: CompressionConfig = compressionSettings ?? createDisabledCompressionConfig(); if (compressionExcluded || !apiKeyCompressionEnabled) { config = { ...config, enabled: false }; @@ -310,7 +318,7 @@ export async function applyContextCompression({ ].filter((id): id is string => typeof id === "string" && id.length > 0); if (routingComboIds.length > 0) { const { getCompressionComboForRoutingCombo } = - await import("../../src/lib/db/compressionCombos.ts"); + await import("../../../src/lib/db/compressionCombos.ts"); const assignedCompressionCombo = routingComboIds .map((id) => getCompressionComboForRoutingCombo(id)) @@ -334,7 +342,7 @@ export async function applyContextCompression({ } let namedCombos: Record = {}; try { - const { listCompressionCombos } = await import("../../src/lib/db/compressionCombos.ts"); + const { listCompressionCombos } = await import("../../../src/lib/db/compressionCombos.ts"); namedCombos = buildNamedComboLookup(listCompressionCombos()); } catch (err) { log?.debug?.( @@ -374,7 +382,7 @@ export async function applyContextCompression({ ) { try { const { getDefaultCompressionCombo } = - await import("../../src/lib/db/compressionCombos.ts"); + await import("../../../src/lib/db/compressionCombos.ts"); const defaultCompressionCombo = getDefaultCompressionCombo(); if ( isStackedCompressionCombo(defaultCompressionCombo as RuntimeCompressionCombo | null) && @@ -395,15 +403,15 @@ export async function applyContextCompression({ } // Phase 4A: unified output styles (supersedes cavemanOutputMode via the back-compat shim). let outputStyleResult: - import("../services/compression/outputStyles/apply.ts").OutputStylesResult | null = null; + import("../../services/compression/outputStyles/apply.ts").OutputStylesResult | null = null; if (config.enabled && compressionHeader?.trim().toLowerCase() !== "off") { try { const { resolveOutputStyleSelection } = - await import("../services/compression/outputStyles/backCompat.ts"); + await import("../../services/compression/outputStyles/backCompat.ts"); const selection = resolveOutputStyleSelection(config); if (selection.length > 0) { const { applyOutputStyles, resolveOutputStyleLanguage } = - await import("../services/compression/outputStyles/apply.ts"); + await import("../../services/compression/outputStyles/apply.ts"); const outputStyleLanguage = resolveOutputStyleLanguage( config.languageConfig, body as Parameters[1] @@ -447,7 +455,7 @@ export async function applyContextCompression({ ? ((compressionInputBody as Record).max_tokens as number) : null; let adaptiveTelemetry: - import("../services/compression/adaptiveCompression/types.ts").AdaptiveTelemetry | null = + import("../../services/compression/adaptiveCompression/types.ts").AdaptiveTelemetry | null = null; const compressionPlan = selectCompressionPlan( config, @@ -569,7 +577,7 @@ export async function applyContextCompression({ } let result: CompressionResult; if (compressionConfig.liveZone?.enabled === true) { - const { applyLiveZoneCompression } = await import("../services/compression/liveZone.ts"); + const { applyLiveZoneCompression } = await import("../../services/compression/liveZone.ts"); const explicitSessionId = clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" ? clientRawRequest.headers.get("x-omniroute-session-id") @@ -762,7 +770,7 @@ export async function applyContextCompression({ log?.info?.("CONTEXT", `Attempting to resolve combo limits for comboName=${comboName}`); try { const { getComboByName } = await import("@/lib/db/combos"); - const { resolveComboTargets } = await import("../services/combo.ts"); + const { resolveComboTargets } = await import("../../services/combo.ts"); let comboConfig = await getComboByName(comboName); if (!comboConfig && comboName.startsWith("combo/")) { comboConfig = await getComboByName(comboName.substring(6)); diff --git a/open-sse/handlers/chatCore/executeProviderRequest.ts b/open-sse/handlers/chatCore/executeProviderRequest.ts index b7b064057aac..9fc50e0858f8 100644 --- a/open-sse/handlers/chatCore/executeProviderRequest.ts +++ b/open-sse/handlers/chatCore/executeProviderRequest.ts @@ -4,6 +4,7 @@ */ import type { ChatCoreExecutorResult as PipelineChatCoreExecutorResult } from "./providerExecutionPipeline.ts"; +import { formatStreamRecoveryRetryWarning } from "./streamErrorResult.ts"; import type { AgentGoalPolicy } from "../../utils/agentGoalPolicy.ts"; import type { BaseExecutor } from "../../executors/base.ts"; import { prepareUpstreamBody } from "./upstreamBody.ts"; @@ -93,7 +94,7 @@ export type ExecuteProviderRequestDeps = { extendedContext?: boolean; getExecutionCredentials: () => Record; isClaudePassthrough?: boolean; - isModelScope: boolean; + isModelScope: () => boolean; isOpencodeClient: boolean; log: | { diff --git a/open-sse/handlers/chatCore/requestPrelude.ts b/open-sse/handlers/chatCore/requestPrelude.ts index c859e6d669b5..669770f2474a 100644 --- a/open-sse/handlers/chatCore/requestPrelude.ts +++ b/open-sse/handlers/chatCore/requestPrelude.ts @@ -4,6 +4,10 @@ * Search-tool fallback stays with that front door. */ import { resolveChatCoreRequestSetup } from "./requestSetup.ts"; +import { attachCompressionUsageReceiptAfterAnalytics as attachCompressionUsageReceiptAfterAnalyticsFor } from "./compressionUsageReceipt.ts"; +import { buildFailureUsageRecord } from "./failureUsage.ts"; +import type { FailureUsageAggregate } from "./failureUsage.ts"; +import { saveRequestUsage } from "@/lib/usage/usageHistory"; import { checkIdempotencyCache } from "./idempotency.ts"; import { checkLifecycle, resolveLifecycle } from "./modelLifecyclePolicy.ts"; import { From 1b5771bd37fc23af1da9069c1f9e229f4bd2bf76 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 17:19:16 -0400 Subject: [PATCH 09/26] fix(sse): narrow the stream-complete error field to what the sink accepts `StreamCompletePayload.error` was declared `unknown` when the leaf was split out of the barrel. The barrel inferred a concrete type for `streamError` from its enclosing scope, so the same expression type-checked there; once the payload became a parameter, the inference was gone and the field fell back to `unknown`. That value flows straight into `persistAttemptLogs`, whose `error` field is `string | null`. The mismatch was invisible because `tsconfig.typecheck-core.json` is an explicit 28-file allowlist and none of these files are on it. Compiling the branch's own files directly drops from nine errors to eight, with `streamMaterialize.ts` clearing entirely. The eight that remain are out of scope here: - four `Transformer.cancel` entries in responsesTransformer, progressTracker, sseHeartbeat and stream, none of which this branch touches - two `isClaudePassthrough` entries that arrive with the Claude OAuth raw passthrough pin; the base barrel already carries three references to the field while `ExecuteInput` does not declare it - two on the `materializeDeduplicatedExecutionResult` call, which exists verbatim in the base barrel at line 3528 67 tests pass. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore/streamMaterialize.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-sse/handlers/chatCore/streamMaterialize.ts b/open-sse/handlers/chatCore/streamMaterialize.ts index b75f85131d99..794cfcf59a14 100644 --- a/open-sse/handlers/chatCore/streamMaterialize.ts +++ b/open-sse/handlers/chatCore/streamMaterialize.ts @@ -36,7 +36,7 @@ export type StreamCompletePayload = { responseBody?: unknown; providerPayload?: unknown; clientPayload?: unknown; - error?: unknown; + error?: string | null; errorCode?: string; ttft?: number; itlMs?: number; From 3e909642883eaeae1bd34cca4f698a3b0bfb44b5 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 18:08:56 -0400 Subject: [PATCH 10/26] refactor(sse): lift the non-streaming leg out of the chatCore barrel The `if (!stream)` block was 798 lines inside an already oversized function. It moves out verbatim to `chatCore/nonStreamingLeg.ts`. The contract is simpler than the compression leaf's. Every path out of this block terminates -- the try ends in a return, the catch ends in a throw -- so nothing the block assigns is ever read afterwards. That makes it a plain call with a return value and no carry-out, where the compression leaf had to hand eleven reassigned values back. Nine of the 73 inputs are reassigned inside the block. Those writes stay local for the same reason: the barrel cannot observe them. `pipelineRecovered` is the clearest case -- the block sets it on the recovery path, and the reads that matter live in the streaming leg, which never enters here. Types come from the source of truth rather than hand-written approximations: `ReturnType` for the request capture, and the exported `MemorySkillsInjectionResult`, `WebFetchFallbackPlan` and `WebSearchFallbackPlan` for the three inputs whose shapes a narrower guess got wrong. `finalBody` keeps its implicit `any` with an explicit lint exemption: the barrel declares it as a bare `let`, one branch stores a captured request body and another an executor result, and the two do not unify. Annotating it would change behaviour. Thirty-two imports the barrel no longer uses are dropped with the block. Barrel: 4136 -> 3383 lines. --- open-sse/handlers/chatCore.ts | 911 ++------------ open-sse/handlers/chatCore/nonStreamingLeg.ts | 1116 +++++++++++++++++ 2 files changed, 1194 insertions(+), 833 deletions(-) create mode 100644 open-sse/handlers/chatCore/nonStreamingLeg.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index ffe6e1a950fd..ea0e54e92fb3 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -30,10 +30,6 @@ export { import { acquireTurnExecution, createTurnInProgressResult } from "./chatCore/turnExecutionGuard.ts"; import { applyContextCompression } from "./chatCore/contextCompression.ts"; import { checkSemanticCache } from "./chatCore/semanticCache.ts"; -import { buildPostCallGuardrailContext } from "./chatCore/postCallGuardrailContext.ts"; -import { storeSemanticCacheResponse } from "./chatCore/semanticCacheStore.ts"; -import { buildNonStreamingResponseHeaders } from "./chatCore/nonStreamingResponseHeaders.ts"; -import { maybeWrapForcedNonStreamingResponsesJson } from "./chatCore/responsesJsonToSse.ts"; import { enforceOutputTokenBudget } from "./chatCore/outputTokenBudget.ts"; import { maybeConvertJsonBodyToSse } from "./chatCore/jsonBodyToSse.ts"; import { assembleStreamingResponseHeaders } from "./chatCore/streamingResponseHeaders.ts"; @@ -46,7 +42,6 @@ import { applyReasoningInputPolicy, resolveIncompatibleReasoningAction, } from "../services/reasoningInputPolicy.ts"; -import { createRoutingEvent, emitRoutingEvent } from "../services/routing/index.ts"; import { routingFinishReason } from "./chatCore/routingFinishReason.ts"; import { isNoMemoryRequested } from "./chatCore/headers.ts"; @@ -71,21 +66,13 @@ import { import { recoverAnthropicThinkingSignature } from "./chatCore/thinkingSignatureRecovery.ts"; import { runProviderExecutionPipeline } from "./chatCore/providerExecutionPipeline.ts"; import { onFailure, onStreamThrow } from "./chatCore/recoveryPolicy.ts"; -import { runNonStreamingProviderLeg } from "./chatCore/nonStreamingProviderLeg.ts"; -import type { NonStreamingProviderLegResult } from "@/lib/skills/toolLoopTypes.ts"; -import { - applyServerOwnedToolLoopIfNeeded, - derivePostInjectionRequestIdentity, - followUpLegInput, -} from "./chatCore/serverOwnedToolLoopWire.ts"; -import { finalizeToolLoopError } from "./chatCore/nonStreamingFinalization.ts"; +import { runNonStreamingLeg } from "./chatCore/nonStreamingLeg.ts"; import { markCodexScopeRateLimited } from "./chatCore/codexFailover.ts"; import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity"; import { buildStreamingResponseHeaders, stripStaleForwardingHeaders, } from "./chatCore/responseHeaders.ts"; -import { maybeSyncClaudeExtraUsageState } from "./chatCore/telemetryHelpers.ts"; // Re-export the previously inline-defined helpers so existing importers of these // symbols from chatCore.ts (tests, sibling modules) keep resolving after the split. export { @@ -96,7 +83,7 @@ export { buildStreamingResponseHeaders, stripStaleForwardingHeaders, }; -import { resolveMemoryOwnerId, runMemoryExtractionGate } from "./chatCore/memoryExtraction.ts"; +import { resolveMemoryOwnerId } from "./chatCore/memoryExtraction.ts"; import { normalizeHeaders } from "../utils/headers.ts"; import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts"; import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; @@ -119,7 +106,6 @@ import { resolveStreamReadinessTimeout } from "../utils/streamReadinessPolicy.ts import { hasActiveClaudeThinking } from "../utils/thinkingBudget.ts"; import { createStreamController } from "../utils/streamHandler.ts"; import * as streamFailure from "../utils/streamFailureFinalization.ts"; -import { normalizeUsage } from "../utils/usageTracking.ts"; import { refreshWithRetry, isUnrecoverableRefreshError, @@ -134,7 +120,6 @@ import { stripIncompatibleMessageContent, } from "../services/modelStrip.ts"; import { shouldUseMidConversationSystem } from "../executors/claudeIdentity.ts"; -import { echoModelInObject } from "../services/responseModelEcho.ts"; import { getUnsupportedParams } from "../config/providerRegistry.ts"; import { checkToolCallingRequiredButUnsupported } from "./chatCore/toolCallingRequiredCheck.ts"; import { @@ -151,7 +136,6 @@ import { import { areContextWindowChecksDisabled, isFeatureFlagEnabled, - isServerOwnedToolLoopEnabled, } from "@/shared/utils/featureFlags.ts"; import { REASONING_BUFFER_MIN_TRIGGER, @@ -169,11 +153,6 @@ import { sanitizeErrorMessage, sanitizeUpstreamDetails, } from "../utils/error.ts"; -import { - reportMalformed200, - detectMalformedNonStream, - describeMalformedNonStream, -} from "../utils/diagnostics.ts"; import { checkTokenLimits } from "@omniroute/open-sse/services/tokenLimitCounter.ts"; import { COOLDOWN_MS, @@ -190,7 +169,6 @@ import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorCl import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; import { wasRefreshTokenRotated } from "@omniroute/open-sse/services/refreshSerializer.ts"; import { connectionHasExtraKeys } from "../services/apiKeyRotator.ts"; -import { getSkillsModelIdForFormat } from "./chatCore/skillsFormat.ts"; import { createSafeAbortError, createStreamingErrorResult, @@ -198,7 +176,6 @@ import { getSafeErrorMetadata, getUpstreamErrorIdentifier, } from "./chatCore/streamErrorResult.ts"; -import { buildCacheUsageLogMeta } from "./chatCore/cacheUsageMeta.ts"; import { buildExecutorClientHeaders } from "./chatCore/executorClientHeaders.ts"; import { getExecutionConnectionId } from "./chatCore/executionCredentials.ts"; import { resolveExecutionCredentials as resolveExecutionCredentialsFor } from "./chatCore/executionCredentials.ts"; @@ -227,24 +204,18 @@ import { } from "./chatCore/passthroughToolNames.ts"; import type { EnforceDecision } from "@/lib/quota/types"; import { writeCompressionAnalytics } from "./chatCore/compressionAnalyticsWrite.ts"; -import { recordContextEditingTelemetryHook } from "./chatCore/contextEditingTelemetry.ts"; -import { scheduleQuotaShareConsumption } from "./chatCore/quotaShareConsumption.ts"; import { emitRequestGamificationEvent } from "./chatCore/gamificationEvent.ts"; import { runPluginOnResponseHook } from "./chatCore/pluginOnResponse.ts"; -import { isJsonRecord } from "./chatCore/nonStreamingResponseParse.ts"; -import { recordNonStreamingUsageStats } from "./chatCore/nonStreamingUsageStats.ts"; import { normalizeExecutorResult } from "./chatCore/upstreamTimeouts.ts"; import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; import { getCacheControlSettings } from "@/lib/cacheControlSettings"; -import { guardrailRegistry } from "@/lib/guardrails"; import type { VideoBridgeLogRedactionEntry } from "@/lib/guardrails/videoBridge"; import { shouldPreserveCacheControl, resolveConnectionCacheOverride, } from "../utils/cacheControlPolicy.ts"; -import { extractUsageFromResponse } from "./usageExtractor.ts"; import { updateFromHeaders, updateFromResponseBody } from "../services/rateLimitManager.ts"; import * as localLimiterErrors from "../services/rateLimitManager/errors.ts"; import { markBlocked as markAccountSemaphoreBlocked } from "../services/accountSemaphore.ts"; @@ -254,7 +225,6 @@ import { recordCoreOwnedAntigravityQuotaState, shouldDeferAntigravityQuotaStateToCaller, } from "../services/accountFallback.ts"; -import { saveIdempotency } from "@/lib/idempotencyLayer"; import { getNextFamilyFallback, isContextOverflowError, @@ -267,15 +237,13 @@ import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; import { extractFacts } from "@/lib/memory/extraction"; -import { handleToolCallExecution } from "@/lib/skills/interception"; -import { MEMORY_BUILTIN_TOOL_NAMES } from "@/lib/skills/memoryBuiltins"; import { getClaudeCodeCompatibleRequestDefaults } from "@/lib/providers/requestDefaults"; import { buildClaudeCodeCompatibleRequest, resolveClaudeCodeCompatibleSessionId, } from "../services/claudeCodeCompatible.ts"; import { classifyModelScope429 } from "../services/modelscopePolicy.ts"; -import { incrementTokenUsage, isTpmExhausted } from "../services/geminiRateLimitTracker.ts"; +import { isTpmExhausted } from "../services/geminiRateLimitTracker.ts"; import { computeRequestHash, shouldDeduplicate } from "../services/requestDedup.ts"; /** @@ -3033,805 +3001,82 @@ export async function handleChatCore({ // Non-streaming response if (!stream) { - try { - const runNonStreamingPipeline = async ({ - policy, - model: pipelineModel, - translatedBody: wireBody, - }) => { - translatedBody = wireBody as typeof translatedBody; - currentModel = pipelineModel; - triedModels.add(pipelineModel); - return runProviderExecutionPipeline({ - policy, - target: { - provider, - requestedModel: pipelineModel, - sourceFormat, - targetFormat, - stream: false, - }, - connection: { - initialConnectionId: String(getCurrentConnectionId() || connectionId || ""), - getCurrentConnectionId: () => getCurrentConnectionId() || undefined, - getCredentials: () => (credentials || {}) as Record, - replaceCredentials: (next) => { - Object.assign(credentials, next); - }, - onCredentialsRefreshed: handleCredentialsRefreshed, - refreshCredentials: executeRefreshCredentials, - assertManagedLeaseFence: (id) => { - assertManagedLeaseFence(id); - }, - getProviderCredentials, - }, - wire: { - body: translatedBody as Record, - currentModel, - triedModels, - setBodyAndModel: (nextBody, nextModel) => { - translatedBody = nextBody as typeof translatedBody; - currentModel = nextModel; - triedModels.add(nextModel); - }, - }, - state: { - updatePendingStage: (stage, data) => { - updatePendingScope(pendingScope, { stage, ...(data || {}) }); - }, - recordRateLimitHeaders: updateFromHeaders, - recordRateLimitBody: updateFromResponseBody, - writeTerminalStatus, - persistConnectionPatch: updateProviderConnection, - setConnectionRateLimitedUntil: async (id, untilMs) => { - const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); - setConnectionRateLimitUntil(id, untilMs); - }, - lockModel, - recordAntigravityQuotaState: recordCoreOwnedAntigravityQuotaState, - markAccountSemaphoreBlocked: (key) => { - markAccountSemaphoreBlocked(key, Date.now() + 60_000); - }, - isolateProbeFailures: () => shouldIsolateProbeFailures(), - onCodexScopeRateLimited: async (params) => { - await markCodexScopeRateLimited({ - failedConnectionId: params.failedConnectionId, - model: params.model, - rateLimitedUntil: params.rateLimitedUntil, - credentials: (params.credentials || credentials) as { - connectionId?: string | null; - providerSpecificData?: unknown; - }, - }); - }, - onClearSessionAffinity: () => { - const key = - sessionAffinityKey || - extractSessionAffinityKey(body, clientRawRequest?.headers) || - null; - if (!key) return; - try { - deleteSessionAccountAffinity(key, "codex"); - } catch { - // best-effort - } - }, - onAuditAccountRotation: (params) => { - logAuditEvent({ - action: params.action, - actor: apiKeyInfo?.name || "system", - target: params.newConnectionId, - details: { - failed_connection_id: params.failedConnectionId, - new_connection_id: params.newConnectionId, - attempt: params.attempt, - retry_after_ms: params.retryAfterMs, - }, - }); - }, - }, - sendProviderAttempt: (modelToCall, allowDedup) => - executeProviderRequest(modelToCall, allowDedup), - }); - }; - - let toolLoopRan = false; - let toolLoopUsage = null; - let legResult = await runNonStreamingProviderLeg({ - phase: "initial", - sourceBody: (body || {}) as Record, - expectedConnectionId: managedLease - ? String(getCurrentConnectionId() || connectionId || "") || undefined - : undefined, - allowAccountRotation: !managedLease && comboStrategy !== "context-relay", - allowModelFallback: true, - executeProviderRequest: (modelToCall, allowDedup) => - executeProviderRequest(modelToCall, allowDedup), - runProviderExecution: runNonStreamingPipeline, - setRequestWireState: ({ translatedBody: nextBody, effectiveModel: nextModel }) => { - translatedBody = nextBody as typeof translatedBody; - currentModel = nextModel; - triedModels.add(nextModel); - }, - sourceFormat, - targetFormat, - clientResponseFormat, - provider, - model: effectiveModel, - connectionId: String(getCurrentConnectionId() || connectionId || ""), - getCurrentConnectionId: () => getCurrentConnectionId() || undefined, - effectiveModel: currentModel, - translatedBody: translatedBody as Record, - toolNameMap, - customToolNames, - requestToolIdentityMap, - reasoningCacheScope, - reasoningReplayHistory, - clientHeaders: clientRawRequest?.headers ?? null, - isClaudeCodeCompatible, - log, - }); - - if (legResult.kind === "error") { - const err = legResult.result; - const errMessage = - err?.rawMessage || - (err?.originalError instanceof Error ? err.originalError.message : err?.error) || - ""; - const errHeaders = err?.upstreamHeaders || err?.response?.headers; - const errUpstreamBody = err?.upstreamErrorBody; - if (err) { - await applyProviderFailureClassification({ - statusCode: err.status, - message: errMessage, - headers: errHeaders, - upstreamErrorBody: errUpstreamBody, - retryAfterMs: err.retryAfterMs ?? null, - targetModel: currentModel, - }); - } - - const captured = providerRequestCapture.latest?.() ?? null; - finalBody = captured?.body ?? finalBody ?? translatedBody; - if (captured) { - reqLogger.logTargetRequest(captured.url, captured.headers, captured.body); - } - reqLogger.logError(new Error(err.error || "Provider request failed"), finalBody); - const isNetworkThrow = Boolean(err.originalError); - if (err.response && !isNetworkThrow) { - reqLogger.logProviderResponse( - err.status, - err.response.statusText || "Error", - err.response.headers, - err.response - ); - } - appendRequestLog({ - model, - provider, - connectionId, - status: `FAILED ${err.status}`, - }).catch(() => {}); - persistAttemptLogs({ - status: err.status, - error: err.error || "Provider request failed", - providerRequest: finalBody || translatedBody, - providerResponse: isNetworkThrow ? undefined : err.response, - // On a client abort the client already disconnected before we got here, so this - // body is what we WOULD have sent, not what was delivered. The dashboard reads - // `clientResponse` as "what the client received", so logging it misleads — - // `error` above already records the reason. The pre-#12867 path omitted it here; - // the leg-based path must keep doing so. - clientResponse: isLocalStreamLifecycleError(err.originalError) - ? undefined - : buildErrorBody(err.status, err.error || "Provider request failed"), - cacheSource: "upstream", - }); - persistFailureUsage(err.status, err.errorCode || `upstream_${err.status}`); - trackPendingRequest(model, provider, connectionId, false); - return err; - } - - pipelineRecovered = true; - const expectedConn = managedLease - ? String(getCurrentConnectionId() || connectionId || "") || undefined - : undefined; - // The identity is the tool loop's execution fence key, and deriveToolRequestIdentity - // canonicalizes the body — which by design rejects Dates, Maps and class instances. - // It was computed eagerly, so a body carrying any of those threw on EVERY - // non-streaming request even with SERVER_OWNED_TOOL_LOOP_ENABLED off (the default). - // Derive it only when the loop can run, and fail closed rather than crash: no - // identity means no fence, and without a fence the loop must not run. - let toolLoopEnabled = isServerOwnedToolLoopEnabled(); - let postInjectionRequestIdentity = ""; - if (toolLoopEnabled) { - try { - postInjectionRequestIdentity = derivePostInjectionRequestIdentity({ - apiKeyId: memoryOwnerId || "local", - headers: clientRawRequest?.headers ?? null, - skillRequestId, - postInjectionBody: (body || {}) as Record, - }); - } catch (identityError) { - log?.warn?.( - "SERVER_OWNED_TOOL_LOOP", - `request body is not canonicalizable, skipping the loop: ${ - identityError instanceof Error ? identityError.message : "unknown" - }` - ); - toolLoopEnabled = false; - } - } - const loopApply = await applyServerOwnedToolLoopIfNeeded({ - enabled: toolLoopEnabled, - stream, - isResponsesEndpoint, - sourceFormat, - initialLeg: legResult, - sourceBody: (body || {}) as Record, - skillsModelId: getSkillsModelIdForFormat(sourceFormat), - executionContext: { - apiKeyId: memoryOwnerId || "local", - sessionId: pipelineSessionId, - requestId: skillRequestId, - requestIdentity: postInjectionRequestIdentity, - builtinToolNames: injectionResult.builtinToolNames, - injectedCustomSkillNames: injectionResult.injectedCustomSkillNames, - customSkillExecutionEnabled: - Boolean(memoryOwnerId) && memorySettings?.skillsEnabled === true, - executionFenceEnabled: true, - provider, - model: effectiveModel, - }, - abortSignal: clientRawRequest?.signal, - expectedConnectionId: expectedConn, - followUpLeg: async (nextSourceBody) => { - translatedBody = translateRequest( - sourceFormat, - targetFormat, - model, - { ...nextSourceBody }, - false, - credentials, - provider, - reqLogger, - { - normalizeToolCallId: getModelNormalizeToolCallId( - provider || "", - model || "", - sourceFormat - ), - preserveDeveloperRole: getModelPreserveOpenAIDeveloperRole( - provider || "", - model || "", - sourceFormat - ), - preserveCacheControl, - signatureNamespace: connectionId, - copilotClient: copilotCompatibleReasoning, - reasoningCacheScope, - onReasoningReplayHistory: (messages) => { - reasoningReplayHistory = messages; - }, - } - ); - return runNonStreamingProviderLeg( - followUpLegInput( - { - executeProviderRequest: (modelToCall, allowDedup) => - executeProviderRequest(modelToCall, allowDedup), - runProviderExecution: runNonStreamingPipeline, - setRequestWireState: ({ translatedBody: nextBody, effectiveModel: nextModel }) => { - translatedBody = nextBody as typeof translatedBody; - currentModel = nextModel; - triedModels.add(nextModel); - }, - sourceFormat, - targetFormat, - clientResponseFormat, - provider, - model: effectiveModel, - connectionId: String(getCurrentConnectionId() || connectionId || ""), - getCurrentConnectionId: () => getCurrentConnectionId() || undefined, - effectiveModel: currentModel, - translatedBody: translatedBody as Record, - toolNameMap, - customToolNames, - requestToolIdentityMap, - reasoningCacheScope, - reasoningReplayHistory, - clientHeaders: clientRawRequest?.headers ?? null, - isClaudeCodeCompatible, - log, - }, - nextSourceBody, - expectedConn - ) - ); - }, - logReceipt: (receipt) => reqLogger.logToolLoopReceipt(receipt), - }); - if (loopApply.kind === "error") { - return await finalizeToolLoopError({ - loop: loopApply.loop, - model, - provider, - connectionId, - providerRequest: loopApply.loop.finalProviderRequest || finalBody || translatedBody, - persistFailureUsage, - persistAttemptLogs, - trackPendingRequest, - }); - } - // `legResult` is declared as the full NonStreamingProviderLegResult union. The - // `kind === "error"` guard above narrows it to the ok variant, but the conditional - // reassignment below widens it back to the declared type, so every field read past - // this point lost the narrowing — 13 TS2339 diagnostics under - // tsconfig.typecheck-api.json, which pulls chatCore.ts in through the route while - // tsconfig.typecheck-core.json does not. Pin the ok variant in its own binding: - // `loopApply.leg` is already `NonStreamingProviderLegResult & { kind: "ok" }`, - // so no cast is involved. - let okLeg: NonStreamingProviderLegResult & { kind: "ok" } = legResult; - if (loopApply.kind === "ok") { - toolLoopRan = true; - toolLoopUsage = loopApply.usage; - okLeg = loopApply.leg; - } - - if (okLeg.upstreamResponse) { - providerResponse = okLeg.upstreamResponse; - providerHeaders = normalizeHeaders(okLeg.upstreamResponse.headers); - } else { - providerResponse = new Response(null, { - status: 200, - headers: okLeg.headers, - }); - providerHeaders = normalizeHeaders(okLeg.headers); - } - finalBody = providerRequestCapture.body(okLeg.providerRequest || translatedBody); - // Built inside executeProviderRequest on the pre-#12867 path. The leg now owns the - // first non-streaming send, so that assignment never runs here and the meta stayed - // null — `_omniroute.claudePromptCache` silently vanished from every call log on - // this path. Same inputs, same helper, at the point where they are available. - claudePromptCacheLogMeta = buildClaudePromptCacheLogMeta( - targetFormat, - finalBody, - providerHeaders, - clientRawRequest?.headers - ); - const capturedOk = providerRequestCapture.latest?.(); - reqLogger.logTargetRequest( - okLeg.requestUrl || capturedOk?.url || "", - okLeg.requestHeaders || capturedOk?.headers || {}, - capturedOk?.body ?? finalBody - ); - const responseBody = okLeg.providerBody; - const responsePayloadFormat = okLeg.responsePayloadFormat; - const looksLikeSSE = okLeg.looksLikeSSE; - let translatedResponse = okLeg.response; - const memoryExtractionResponse = okLeg.responseForMemoryExtraction; - reqLogger.logProviderResponse( - 200, - "OK", - providerResponse.headers, - looksLikeSSE - ? { _streamed: true, _format: "sse-json", summary: responseBody } - : responseBody - ); - effectiveServiceTier = resolveReportedServiceTier(responseBody) ?? effectiveServiceTier; - if (onRequestSuccess) { - await onRequestSuccess(); - } - const successConnectionId = getCurrentConnectionId(); - await maybeSyncClaudeExtraUsageState({ - provider, - connectionId: successConnectionId, - providerSpecificData: credentials?.providerSpecificData, - log, - }); - const usage = toolLoopUsage ?? extractUsageFromResponse(responseBody, provider); - const cacheUsageLogMeta = buildCacheUsageLogMeta(usage); - if (usage && typeof usage === "object") { - attachCompressionUsageReceiptAfterAnalytics(usage as Record, "provider"); - if (provider === "gemini") { - const promptTokens = - typeof (usage as Record).prompt_tokens === "number" - ? ((usage as Record).prompt_tokens as number) - : 0; - if (promptTokens > 0) incrementTokenUsage(model, promptTokens); - } - } - recordContextEditingTelemetryHook({ - contextEditingEnabled, - provider, - responseBody, - skillRequestId, - log, - }); - appendRequestLog({ - model, - provider, - connectionId: successConnectionId, - tokens: usage, - status: "200 OK", - }).catch(() => {}); - recordNonStreamingUsageStats(usage, { - traceEnabled, - provider, - connectionId: successConnectionId, - model, - startTime, - apiKeyInfo, - effectiveServiceTier, - isCombo, - comboStrategy, - endpoint: endpointPath, - }); - - // #12150 P1b surface 3 (fix round 1): a video-bridge-observed request's - // request- AND response-derived text both carry the full transcript (the - // flattened description on the request side, the model's own reply on - // the response side) — neither may populate durable Memory. See - // runMemoryExtractionGate for the shared gate + extraction wiring, unit - // tested directly in tests/unit/video-bridge-memory-suppression.test.ts. - runMemoryExtractionGate({ - memoryOwnerId, - memorySettings, - videoBridgeObserved, - pipelineSessionId, - requestBody: body as Record, - responseBody: memoryExtractionResponse as Record | null, - extractFacts, - log, - }); - - const customSkillExecutionEnabled = - Boolean(memoryOwnerId) && memorySettings?.skillsEnabled === true; - const builtinToolNames = [ - webSearchFallbackPlan.toolName, - webFetchFallbackPlan.toolName, - ...(memoryOwnerId && memorySettings?.enabled ? MEMORY_BUILTIN_TOOL_NAMES : []), - ].filter((name): name is string => Boolean(name)); - if (!toolLoopRan && (customSkillExecutionEnabled || builtinToolNames.length > 0)) { - const skillSessionId = pipelineSessionId; - - translatedResponse = await handleToolCallExecution( - translatedResponse, - getSkillsModelIdForFormat(sourceFormat), - { - apiKeyId: memoryOwnerId || "local", - sessionId: skillSessionId, - requestId: skillRequestId, - builtinToolNames, - customSkillExecutionEnabled, - provider, - model: effectiveModel, - } - ); - } - - const guardrailContext = buildPostCallGuardrailContext({ - apiKeyInfo, - body, - clientRawRequest, - log, - model, - provider, - responsePayloadFormat, - clientResponseFormat, - }); - const postCallGuardrails = await guardrailRegistry.runPostCallHooks( - translatedResponse, - guardrailContext - ); - translatedResponse = postCallGuardrails.response; - - const responseUsage = isJsonRecord(usage) - ? usage - : isJsonRecord(translatedResponse.usage) - ? translatedResponse.usage - : null; - const costUsage = normalizeUsage(responseUsage); - const estimatedCost = costUsage - ? await calculateCost(provider, model, costUsage, { serviceTier: effectiveServiceTier }) - : 0; - - if (postCallGuardrails.blocked) { - const guardrailMessage = postCallGuardrails.message || "Response blocked by guardrail"; - persistAttemptLogs({ - status: HTTP_STATUS.BAD_REQUEST, - tokens: usage, - responseBody, - providerRequest: finalBody || translatedBody, - providerResponse: looksLikeSSE - ? { - _streamed: true, - _format: "sse-json", - summary: responseBody, - } - : responseBody, - clientResponse: buildErrorBody(HTTP_STATUS.BAD_REQUEST, guardrailMessage), - claudeCacheMeta: claudePromptCacheLogMeta, - claudeCacheUsageMeta: cacheUsageLogMeta, - cacheSource: "upstream", - }); - if (apiKeyInfo?.id && estimatedCost > 0) { - recordCost(apiKeyInfo.id, estimatedCost); - } - log?.warn?.( - "GUARDRAIL", - `Response blocked by ${postCallGuardrails.guardrail || "guardrail"}: ${guardrailMessage}` - ); - finalizePendingScope(pendingScope, { - providerResponse: responseBody, - clientResponse: translatedResponse, - }); - return createErrorResult(HTTP_STATUS.BAD_REQUEST, guardrailMessage); - } - - // Validate the *translated* response actually carries client-usable output. - // isEmptyContentResponse (above) runs on the raw responseBody before translation; - // this check runs after translation + sanitization + tool-call execution to catch - // cases where a provider returns a structurally valid raw body that translates into - // choices:[] or output:[] with no usable content (Responses API shape included). - const malformedTranslatedReason = detectMalformedNonStream(translatedResponse, provider); - if (malformedTranslatedReason) { - const totalLatency = Date.now() - startTime; - const rawBytes = (() => { - try { - return JSON.stringify(responseBody || {}).length; - } catch { - return -1; - } - })(); - reportMalformed200({ - mode: "nonstream", - provider, - model, - connectionId, - reason: malformedTranslatedReason, - recvBytes: rawBytes, - recvLines: -1, - emitted: -1, - events: {}, - ttftMs: totalLatency, - elapsedMs: totalLatency, - }); - appendRequestLog({ - model, - provider, - connectionId, - status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`, - }).catch(() => {}); - const malformed = describeMalformedNonStream(translatedResponse, malformedTranslatedReason); - const malformedMessage = `[${provider}/${model}] ${malformed.message}`; - const malformedClientBody = buildErrorBody( - HTTP_STATUS.BAD_GATEWAY, - malformedMessage, - undefined, - { code: malformed.code, type: malformed.type } - ); - const sanitizedMalformedResponse = sanitizeUpstreamDetails(responseBody); - const sanitizedMalformedProviderResponse = looksLikeSSE - ? { _streamed: true, _format: "sse-json", summary: sanitizedMalformedResponse } - : sanitizedMalformedResponse; - persistAttemptLogs({ - status: HTTP_STATUS.BAD_GATEWAY, - tokens: usage, - responseBody: sanitizedMalformedResponse, - providerRequest: finalBody || translatedBody, - providerResponse: sanitizedMalformedProviderResponse, - clientResponse: malformedClientBody, - claudeCacheMeta: claudePromptCacheLogMeta, - claudeCacheUsageMeta: cacheUsageLogMeta, - cacheSource: "upstream", - }); - persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "malformed_translated_response"); - trackPendingRequest(model, provider, pendingConnId, false); - // Routing event (feedback foundation) — record the malformed outcome so - // the quality tracker de-prioritizes this model over time. - void emitRoutingEvent( - createRoutingEvent({ - requestId: traceId || pendingRequestId || "unknown", - provider: provider || "unknown", - model: model || "unknown", - strategy: isCombo ? (comboStrategy ?? "combo") : "direct", - latencyMs: Date.now() - startTime, - ttftMs: null, - inputTokens: null, - outputTokens: null, - cost: null, - retries: 0, - fallbackUsed: false, // combo-level fallback tracked by decisionTrace - outcome: "malformed", - status: HTTP_STATUS.BAD_GATEWAY, - finishReason: routingFinishReason(translatedResponse), - connectionId: credentials?.connectionId ?? null, - }) - ); - return createErrorResult( - HTTP_STATUS.BAD_GATEWAY, - malformedMessage, - null, - malformed.code, - malformed.type - ); - } - - // ── Phase 9.1: Cache store (non-streaming, temp=0) ── - storeSemanticCacheResponse({ - enabled: semanticCacheEnabled, - body: bodyForCacheWrite, - headers: clientRawRequest?.headers, - translatedResponse, - model, - // The dual-layer manager scopes entries per provider (cacheByProvider); - // lookup passes the resolved provider, so the write must too (#14159). - provider, - apiKeyId: apiKeyInfo?.id ?? undefined, - usage, - log, - }); - - // ── Phase 9.2: Save for idempotency ── - // Reuse the key resolved by checkIdempotencyCache() above (single derivation per - // request). (#3821-review LEDGER-6) - saveIdempotency(idempotencyKey, translatedResponse, 200); - reqLogger.logConvertedResponse(translatedResponse); - persistAttemptLogs({ - status: 200, - tokens: usage, - responseBody, - providerRequest: finalBody || translatedBody, - providerResponse: looksLikeSSE - ? { - _streamed: true, - _format: "sse-json", - summary: responseBody, - } - : responseBody, - clientResponse: translatedResponse, - claudeCacheMeta: claudePromptCacheLogMeta, - claudeCacheUsageMeta: cacheUsageLogMeta, - cacheSource: "upstream", - }); - if (apiKeyInfo?.id && estimatedCost > 0) { - recordCost(apiKeyInfo.id, estimatedCost); - } - - // === Quota Share POST-hook (B/F7) — fire-and-forget, fail-open === - await scheduleQuotaShareConsumption({ - apiKeyId: apiKeyInfo?.id, - connectionId: credentials?.connectionId, - provider, - model, - usage, - estimatedCost, - log, - }); - // === /Quota Share POST-hook === - - // ── Gamification event (fire-and-forget) ── - await emitRequestGamificationEvent({ apiKeyId: apiKeyInfo?.id, model, provider }); - - finalizePendingScope(pendingScope, { - providerResponse: responseBody, - clientResponse: translatedResponse, - }); - const responseHeaders = buildNonStreamingResponseHeaders({ - provider, - model, - startTime, - responseUsage, - estimatedCost, - requestId: skillRequestId, - compressionResponseMeta, - comboStrategy, - fallbackAttempts, - }); - // #6426: align response body `model` with the `X-OmniRoute-Model` header - // (both must be the resolved backend model). Some upstreams (notably legacy - // /v1/completions text-completion path) return a body `model` field that - // differs from the resolved backend id we advertised in the header, leaving - // strict clients unable to reconcile the two. Rewrite body.model to `model` - // FIRST, then let #1311 echo override it when the opt-in setting is on. - if (typeof model === "string" && model) echoModelInObject(translatedResponse, model); - // #1311: echo the requested alias/combo name in the non-streaming response model. - if (echoModel) echoModelInObject(translatedResponse, echoModel); - - // ── Plugin onResponse hook (fire-and-forget) ── - // #8395: the streaming branch below already calls this; the non-streaming - // (stream:false) branch returned without it, so onResponse never fired for - // non-streaming requests at all. - await runPluginOnResponseHook({ - requestId: traceId, - body, - model, - provider, - apiKeyInfo, - headers: clientRawRequest?.headers, - response: { status: 200, data: translatedResponse }, - }); - - // Routing event (feedback foundation) — fire-and-forget, cheap. - void emitRoutingEvent( - createRoutingEvent({ - requestId: traceId || pendingRequestId || "unknown", - provider: provider || "unknown", - model: model || "unknown", - strategy: isCombo ? (comboStrategy ?? "combo") : "direct", - latencyMs: Date.now() - startTime, - ttftMs: null, - inputTokens: - usage && typeof usage === "object" - ? (() => { - const promptTokens = (usage as Record).prompt_tokens; - return typeof promptTokens === "number" && Number.isFinite(promptTokens) - ? promptTokens - : null; - })() - : null, - outputTokens: - usage && typeof usage === "object" - ? (() => { - const completionTokens = (usage as Record).completion_tokens; - return typeof completionTokens === "number" && Number.isFinite(completionTokens) - ? completionTokens - : null; - })() - : null, - cost: Number.isFinite(estimatedCost) ? estimatedCost : null, - retries: 0, - fallbackUsed: false, // combo-level fallback tracked by decisionTrace - outcome: "success", - status: 200, - finishReason: routingFinishReason(translatedResponse), - connectionId: credentials?.connectionId ?? null, - }) - ); + return await runNonStreamingLeg({ + apiKeyInfo, + applyProviderFailureClassification, + assertManagedLeaseFence, + attachCompressionUsageReceiptAfterAnalytics, + body, + bodyForCacheWrite, + claudePromptCacheLogMeta, + clientRawRequest, + clientRequestedResponsesStream, + clientResponseFormat, + comboStrategy, + compressionResponseMeta, + connectionId, + contextEditingEnabled, + copilotCompatibleReasoning, + credentials, + currentModel, + customToolNames, + echoModel, + effectiveModel, + effectiveServiceTier, + endpointPath, + executeProviderRequest, + executeRefreshCredentials, + fallbackAttempts, + finalBody, + getCurrentConnectionId, + getManagedLeaseFenceErrorCode, + handleCredentialsRefreshed, + idempotencyKey, + injectionResult, + isClaudeCodeCompatible, + isCombo, + isResponsesEndpoint, + log, + managedLease, + managedLeaseFenceErrorResult, + memoryOwnerId, + memorySettings, + model, + onRequestSuccess, + pendingConnId, + pendingRequestId, + pendingScope, + persistAttemptLogs, + persistFailureUsage, + pipelineRecovered, + pipelineSessionId, + preserveCacheControl, + provider, + providerHeaders, + providerRequestCapture, + providerResponse, + reasoningCacheScope, + reasoningReplayHistory, + reqLogger, + requestToolIdentityMap, + resolveReportedServiceTier, + semanticCacheEnabled, + sessionAffinityKey, + skillRequestId, + sourceFormat, + startTime, + stream, + targetFormat, + toolNameMap, + traceEnabled, + traceId, + translatedBody, + triedModels, + videoBridgeObserved, + webFetchFallbackPlan, + webSearchFallbackPlan, + }); - return { - success: true, - response: maybeWrapForcedNonStreamingResponsesJson({ - clientRequestedResponsesStream, - body: translatedResponse, - headers: responseHeaders, - }), - }; - } catch (error) { - trackPendingRequest(model, provider, connectionId, false); - const errorMetadata = getSafeErrorMetadata(error); - const managedLeaseFenceCode = getManagedLeaseFenceErrorCode(errorMetadata.code); - if (managedLeaseFenceCode) return managedLeaseFenceErrorResult(managedLeaseFenceCode); - // isSemaphoreCapacityError already reads the code through getSafeErrorMetadata, - // so a hostile rejection cannot escape this classification. - if (isSemaphoreCapacityError(error)) { - const semaphoreCode = errorMetadata.code as string; - appendRequestLog({ - model, - provider, - connectionId, - status: `FAILED ${semaphoreCode}`, - }).catch(() => {}); - const failureMessage = sanitizeErrorMessage(errorMetadata.message) || "Semaphore timeout"; - persistAttemptLogs({ - status: HTTP_STATUS.RATE_LIMITED, - error: failureMessage, - providerRequest: finalBody || translatedBody, - clientResponse: buildErrorBody(HTTP_STATUS.RATE_LIMITED, failureMessage), - claudeCacheMeta: claudePromptCacheLogMeta, - cacheSource: "upstream", - }); - persistFailureUsage(HTTP_STATUS.RATE_LIMITED, semaphoreCode); - const result = createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); - return { - ...result, - errorType: "account_semaphore_capacity", - errorCode: semaphoreCode, - }; - } - throw error; - } } // Streaming response diff --git a/open-sse/handlers/chatCore/nonStreamingLeg.ts b/open-sse/handlers/chatCore/nonStreamingLeg.ts new file mode 100644 index 000000000000..c1918a5e0e94 --- /dev/null +++ b/open-sse/handlers/chatCore/nonStreamingLeg.ts @@ -0,0 +1,1116 @@ +/** + * The non-streaming leg of handleChatCore, lifted verbatim out of the barrel. + * + * Every path through this block terminates -- the try ends in a return, the + * catch ends in a throw -- so nothing downstream observes what this code + * assigns. The contract is therefore a plain return value with no carry-out, + * unlike the compression leaf which hands eleven reassigned values back. + */ +import { buildPostCallGuardrailContext } from "./postCallGuardrailContext.ts"; +import { storeSemanticCacheResponse } from "./semanticCacheStore.ts"; +import { buildNonStreamingResponseHeaders } from "./nonStreamingResponseHeaders.ts"; +import { maybeWrapForcedNonStreamingResponsesJson } from "./responsesJsonToSse.ts"; +import { createRoutingEvent, emitRoutingEvent } from "../../services/routing/index.ts"; +import { routingFinishReason } from "./routingFinishReason.ts"; +import { buildClaudePromptCacheLogMeta } from "./executorHelpers.ts"; +import { runProviderExecutionPipeline } from "./providerExecutionPipeline.ts"; +import { runNonStreamingProviderLeg } from "./nonStreamingProviderLeg.ts"; +import type { NonStreamingProviderLegResult } from "@/lib/skills/toolLoopTypes.ts"; +import { + applyServerOwnedToolLoopIfNeeded, + derivePostInjectionRequestIdentity, + followUpLegInput, +} from "./serverOwnedToolLoopWire.ts"; +import { finalizeToolLoopError } from "./nonStreamingFinalization.ts"; +import { markCodexScopeRateLimited } from "./codexFailover.ts"; +import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity"; +import { maybeSyncClaudeExtraUsageState } from "./telemetryHelpers.ts"; +import { runMemoryExtractionGate } from "./memoryExtraction.ts"; +import { normalizeHeaders } from "../../utils/headers.ts"; +import { translateRequest } from "../../translator/index.ts"; +import { normalizeUsage } from "../../utils/usageTracking.ts"; +import { echoModelInObject } from "../../services/responseModelEcho.ts"; +import { isServerOwnedToolLoopEnabled } from "@/shared/utils/featureFlags.ts"; +import { + buildErrorBody, + createErrorResult, + sanitizeErrorMessage, + sanitizeUpstreamDetails, +} from "../../utils/error.ts"; +import { + reportMalformed200, + detectMalformedNonStream, + describeMalformedNonStream, +} from "../../utils/diagnostics.ts"; +import { HTTP_STATUS } from "../../config/constants.ts"; +import { updateProviderConnection } from "@/lib/db/providers"; +import { getSkillsModelIdForFormat } from "./skillsFormat.ts"; +import { isSemaphoreCapacityError, getSafeErrorMetadata } from "./streamErrorResult.ts"; +import { buildCacheUsageLogMeta } from "./cacheUsageMeta.ts"; +import { logAuditEvent } from "@/lib/compliance"; +import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb"; +import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; +import { recordCost } from "@/domain/costRules"; +import { calculateCost } from "@/lib/usage/costCalculator"; +import { recordContextEditingTelemetryHook } from "./contextEditingTelemetry.ts"; +import { scheduleQuotaShareConsumption } from "./quotaShareConsumption.ts"; +import { emitRequestGamificationEvent } from "./gamificationEvent.ts"; +import { runPluginOnResponseHook } from "./pluginOnResponse.ts"; +import { isJsonRecord } from "./nonStreamingResponseParse.ts"; +import { recordNonStreamingUsageStats } from "./nonStreamingUsageStats.ts"; +import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; +import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; +import { guardrailRegistry } from "@/lib/guardrails"; +import { extractUsageFromResponse } from "../usageExtractor.ts"; +import { updateFromHeaders, updateFromResponseBody } from "../../services/rateLimitManager.ts"; +import { markBlocked as markAccountSemaphoreBlocked } from "../../services/accountSemaphore.ts"; +import { lockModel, recordCoreOwnedAntigravityQuotaState } from "../../services/accountFallback.ts"; +import { saveIdempotency } from "@/lib/idempotencyLayer"; +import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; +import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; +import { extractFacts } from "@/lib/memory/extraction"; +import { handleToolCallExecution } from "@/lib/skills/interception"; +import { MEMORY_BUILTIN_TOOL_NAMES } from "@/lib/skills/memoryBuiltins"; +import { incrementTokenUsage } from "../../services/geminiRateLimitTracker.ts"; + +import type { PersistAttemptLogsArgs } from "./attemptLogging.ts"; +import type { EffectiveServiceTier } from "./serviceTier.ts"; +import type { ChatCoreExecutorResult } from "./executeProviderRequest.ts"; +import type { MemorySkillsInjectionResult } from "./memorySkillsInjection.ts"; +import type { WebFetchFallbackPlan } from "../../services/webFetchInterception.ts"; +import type { WebSearchFallbackPlan } from "../../services/webSearchFallback.ts"; +import type { createPreparedRequestLogger } from "../../utils/providerRequestLogging.ts"; +import type { FailureUsageAggregate } from "./failureUsage.ts"; + +/** + * The block calls `log?.warn` directly and passes `log` on to helpers that + * reach for debug/info/error, so the shape has to cover all four. + */ +type LoggerLike = + | { + warn?: (...args: unknown[]) => void; + debug?: (...args: unknown[]) => void; + info?: (...args: unknown[]) => void; + error?: (...args: unknown[]) => void; + } + | null + | undefined; + +type RequestLogger = { + logConvertedResponse?: (...args: unknown[]) => unknown; + logError?: (...args: unknown[]) => unknown; + logProviderResponse?: (...args: unknown[]) => unknown; + logTargetRequest?: (...args: unknown[]) => unknown; + logToolLoopReceipt?: (...args: unknown[]) => unknown; +}; + +export interface NonStreamingLegDeps { + apiKeyInfo: ({ id?: string | null; name?: string } & Record) | null; + applyProviderFailureClassification: ({ + statusCode, + message, + headers, + upstreamErrorBody, + retryAfterMs, + targetModel, + }: { + statusCode: number; + message: string; + headers?: Headers | null; + upstreamErrorBody?: unknown; + retryAfterMs?: number | null; + targetModel: string; + }) => Promise; + assertManagedLeaseFence: (attemptConnectionId: string | null | undefined) => void; + attachCompressionUsageReceiptAfterAnalytics: ( + usage: Record, + source: "provider" | "estimated" | "stream" + ) => void; + body: Record; + bodyForCacheWrite: Record | null | undefined; + claudePromptCacheLogMeta: Record | null | undefined; + clientRawRequest: + | { + headers?: Record; + signal?: AbortSignal; + } + | null + | undefined; + clientRequestedResponsesStream: boolean; + clientResponseFormat: string; + comboStrategy: string | null | undefined; + compressionResponseMeta: string; + connectionId: string | null | undefined; + contextEditingEnabled: boolean; + copilotCompatibleReasoning: boolean; + credentials: ({ connectionId?: string | null } & Record) | null | undefined; + currentModel: string; + customToolNames: Set; + echoModel: string; + effectiveModel: string; + effectiveServiceTier: EffectiveServiceTier; + endpointPath: string; + executeProviderRequest: ( + modelToCall?: string, + allowDedup?: boolean + ) => Promise; + executeRefreshCredentials: ( + currentCreds: Record + ) => Promise | null>; + fallbackAttempts: number; + // The barrel declares this as a bare `let`, i.e. implicit any: one branch + // stores a captured request body, another an executor result, and the two + // shapes do not unify. Any narrower annotation here would change behaviour, + // so the original looseness is preserved deliberately. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + finalBody: any; + getCurrentConnectionId: () => string | null | undefined; + getManagedLeaseFenceErrorCode: (code: string | undefined) => string | undefined; + handleCredentialsRefreshed: (refreshed: Record) => Promise; + idempotencyKey: string; + injectionResult: MemorySkillsInjectionResult; + isClaudeCodeCompatible: boolean; + isCombo: boolean; + isResponsesEndpoint: boolean; + log: LoggerLike | null | undefined; + managedLease: { connectionId?: string | null } | null | undefined; + managedLeaseFenceErrorResult: (code: string) => { + errorType: string; + errorCode: string; + success: false; + status: number; + error: string; + rawMessage: string; + response: Response; + retryAfterMs?: number; + }; + memoryOwnerId: string; + memorySettings: { enabled: boolean; skillsEnabled: boolean; maxTokens: number }; + model: string | null | undefined; + onRequestSuccess: ((...args: never[]) => unknown) | null | undefined; + pendingConnId: string | null | undefined; + pendingRequestId: string; + pendingScope: { + id: string; + model: string | null | undefined; + provider: string | null | undefined; + connectionId: string | null | undefined; + }; + persistAttemptLogs: (args: PersistAttemptLogsArgs) => void; + persistFailureUsage: ( + statusCode: number, + errorCode?: string | null, + aggregate?: FailureUsageAggregate | null + ) => void; + /** + * Seed value for the recovery flag. The leg assigns to its local copy on the + * recovery path, and nothing reads it afterwards -- every route out of here + * returns or throws, so the assignment cannot escape. The reads that matter + * live in the streaming leg, which never enters this function. Kept in the + * contract so the lifted assignment stays verbatim. + */ + pipelineRecovered: boolean; + pipelineSessionId: string | null | undefined; + preserveCacheControl: boolean; + provider: string | null | undefined; + providerHeaders: Record | Headers | null | undefined; + providerRequestCapture: ReturnType; + providerResponse: (Response & { body?: unknown }) | null | undefined; + reasoningCacheScope: string; + reasoningReplayHistory: unknown[]; + reqLogger: RequestLogger; + requestToolIdentityMap: Map; + resolveReportedServiceTier: (responseBody?: unknown) => EffectiveServiceTier | null | undefined; + semanticCacheEnabled: boolean; + sessionAffinityKey: string | null | undefined; + skillRequestId: `${string}-${string}-${string}-${string}-${string}`; + sourceFormat: string; + startTime: number; + stream: boolean; + targetFormat: string; + toolNameMap: Map; + traceEnabled: boolean; + traceId: string; + translatedBody: Record; + triedModels: Set; + videoBridgeObserved: boolean; + webFetchFallbackPlan: WebFetchFallbackPlan | null | undefined; + webSearchFallbackPlan: WebSearchFallbackPlan | null | undefined; +} + +export async function runNonStreamingLeg(deps: NonStreamingLegDeps) { + // Nine of these are reassigned inside the block. Those writes stay local: + // every path returns or throws, so the barrel never reads them back. + let { + apiKeyInfo, + applyProviderFailureClassification, + assertManagedLeaseFence, + attachCompressionUsageReceiptAfterAnalytics, + body, + bodyForCacheWrite, + claudePromptCacheLogMeta, + clientRawRequest, + clientRequestedResponsesStream, + clientResponseFormat, + comboStrategy, + compressionResponseMeta, + connectionId, + contextEditingEnabled, + copilotCompatibleReasoning, + credentials, + currentModel, + customToolNames, + echoModel, + effectiveModel, + effectiveServiceTier, + endpointPath, + executeProviderRequest, + executeRefreshCredentials, + fallbackAttempts, + finalBody, + getCurrentConnectionId, + getManagedLeaseFenceErrorCode, + handleCredentialsRefreshed, + idempotencyKey, + injectionResult, + isClaudeCodeCompatible, + isCombo, + isResponsesEndpoint, + log, + managedLease, + managedLeaseFenceErrorResult, + memoryOwnerId, + memorySettings, + model, + onRequestSuccess, + pendingConnId, + pendingRequestId, + pendingScope, + persistAttemptLogs, + persistFailureUsage, + pipelineRecovered: _pipelineRecovered, + pipelineSessionId, + preserveCacheControl, + provider, + providerHeaders, + providerRequestCapture, + providerResponse, + reasoningCacheScope, + reasoningReplayHistory, + reqLogger, + requestToolIdentityMap, + resolveReportedServiceTier, + semanticCacheEnabled, + sessionAffinityKey, + skillRequestId, + sourceFormat, + startTime, + stream, + targetFormat, + toolNameMap, + traceEnabled, + traceId, + translatedBody, + triedModels, + videoBridgeObserved, + webFetchFallbackPlan, + webSearchFallbackPlan, + } = deps; + + try { + const runNonStreamingPipeline = async ({ + policy, + model: pipelineModel, + translatedBody: wireBody, + }) => { + translatedBody = wireBody as typeof translatedBody; + currentModel = pipelineModel; + triedModels.add(pipelineModel); + return runProviderExecutionPipeline({ + policy, + target: { + provider, + requestedModel: pipelineModel, + sourceFormat, + targetFormat, + stream: false, + }, + connection: { + initialConnectionId: String(getCurrentConnectionId() || connectionId || ""), + getCurrentConnectionId: () => getCurrentConnectionId() || undefined, + getCredentials: () => (credentials || {}) as Record, + replaceCredentials: (next) => { + Object.assign(credentials, next); + }, + onCredentialsRefreshed: handleCredentialsRefreshed, + refreshCredentials: executeRefreshCredentials, + assertManagedLeaseFence: (id) => { + assertManagedLeaseFence(id); + }, + getProviderCredentials, + }, + wire: { + body: translatedBody as Record, + currentModel, + triedModels, + setBodyAndModel: (nextBody, nextModel) => { + translatedBody = nextBody as typeof translatedBody; + currentModel = nextModel; + triedModels.add(nextModel); + }, + }, + state: { + updatePendingStage: (stage, data) => { + updatePendingScope(pendingScope, { stage, ...(data || {}) }); + }, + recordRateLimitHeaders: updateFromHeaders, + recordRateLimitBody: updateFromResponseBody, + writeTerminalStatus, + persistConnectionPatch: updateProviderConnection, + setConnectionRateLimitedUntil: async (id, untilMs) => { + const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); + setConnectionRateLimitUntil(id, untilMs); + }, + lockModel, + recordAntigravityQuotaState: recordCoreOwnedAntigravityQuotaState, + markAccountSemaphoreBlocked: (key) => { + markAccountSemaphoreBlocked(key, Date.now() + 60_000); + }, + isolateProbeFailures: () => shouldIsolateProbeFailures(), + onCodexScopeRateLimited: async (params) => { + await markCodexScopeRateLimited({ + failedConnectionId: params.failedConnectionId, + model: params.model, + rateLimitedUntil: params.rateLimitedUntil, + credentials: (params.credentials || credentials) as { + connectionId?: string | null; + providerSpecificData?: unknown; + }, + }); + }, + onClearSessionAffinity: () => { + const key = + sessionAffinityKey || + extractSessionAffinityKey(body, clientRawRequest?.headers) || + null; + if (!key) return; + try { + deleteSessionAccountAffinity(key, "codex"); + } catch { + // best-effort + } + }, + onAuditAccountRotation: (params) => { + logAuditEvent({ + action: params.action, + actor: apiKeyInfo?.name || "system", + target: params.newConnectionId, + details: { + failed_connection_id: params.failedConnectionId, + new_connection_id: params.newConnectionId, + attempt: params.attempt, + retry_after_ms: params.retryAfterMs, + }, + }); + }, + }, + sendProviderAttempt: (modelToCall, allowDedup) => + executeProviderRequest(modelToCall, allowDedup), + }); + }; + + let toolLoopRan = false; + let toolLoopUsage = null; + let legResult = await runNonStreamingProviderLeg({ + phase: "initial", + sourceBody: (body || {}) as Record, + expectedConnectionId: managedLease + ? String(getCurrentConnectionId() || connectionId || "") || undefined + : undefined, + allowAccountRotation: !managedLease && comboStrategy !== "context-relay", + allowModelFallback: true, + executeProviderRequest: (modelToCall, allowDedup) => + executeProviderRequest(modelToCall, allowDedup), + runProviderExecution: runNonStreamingPipeline, + setRequestWireState: ({ translatedBody: nextBody, effectiveModel: nextModel }) => { + translatedBody = nextBody as typeof translatedBody; + currentModel = nextModel; + triedModels.add(nextModel); + }, + sourceFormat, + targetFormat, + clientResponseFormat, + provider, + model: effectiveModel, + connectionId: String(getCurrentConnectionId() || connectionId || ""), + getCurrentConnectionId: () => getCurrentConnectionId() || undefined, + effectiveModel: currentModel, + translatedBody: translatedBody as Record, + toolNameMap, + customToolNames, + requestToolIdentityMap, + reasoningCacheScope, + reasoningReplayHistory, + clientHeaders: clientRawRequest?.headers ?? null, + isClaudeCodeCompatible, + log, + }); + + if (legResult.kind === "error") { + const err = legResult.result; + const errMessage = + err?.rawMessage || + (err?.originalError instanceof Error ? err.originalError.message : err?.error) || + ""; + const errHeaders = err?.upstreamHeaders || err?.response?.headers; + const errUpstreamBody = err?.upstreamErrorBody; + if (err) { + await applyProviderFailureClassification({ + statusCode: err.status, + message: errMessage, + headers: errHeaders, + upstreamErrorBody: errUpstreamBody, + retryAfterMs: err.retryAfterMs ?? null, + targetModel: currentModel, + }); + } + + const captured = providerRequestCapture.latest?.() ?? null; + finalBody = captured?.body ?? finalBody ?? translatedBody; + if (captured) { + reqLogger.logTargetRequest(captured.url, captured.headers, captured.body); + } + reqLogger.logError(new Error(err.error || "Provider request failed"), finalBody); + const isNetworkThrow = Boolean(err.originalError); + if (err.response && !isNetworkThrow) { + reqLogger.logProviderResponse( + err.status, + err.response.statusText || "Error", + err.response.headers, + err.response + ); + } + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${err.status}`, + }).catch(() => {}); + persistAttemptLogs({ + status: err.status, + error: err.error || "Provider request failed", + providerRequest: finalBody || translatedBody, + providerResponse: isNetworkThrow ? undefined : err.response, + // On a client abort the client already disconnected before we got here, so this + // body is what we WOULD have sent, not what was delivered. The dashboard reads + // `clientResponse` as "what the client received", so logging it misleads — + // `error` above already records the reason. The pre-#12867 path omitted it here; + // the leg-based path must keep doing so. + clientResponse: isLocalStreamLifecycleError(err.originalError) + ? undefined + : buildErrorBody(err.status, err.error || "Provider request failed"), + cacheSource: "upstream", + }); + persistFailureUsage(err.status, err.errorCode || `upstream_${err.status}`); + trackPendingRequest(model, provider, connectionId, false); + return err; + } + + _pipelineRecovered = true; + const expectedConn = managedLease + ? String(getCurrentConnectionId() || connectionId || "") || undefined + : undefined; + // The identity is the tool loop's execution fence key, and deriveToolRequestIdentity + // canonicalizes the body — which by design rejects Dates, Maps and class instances. + // It was computed eagerly, so a body carrying any of those threw on EVERY + // non-streaming request even with SERVER_OWNED_TOOL_LOOP_ENABLED off (the default). + // Derive it only when the loop can run, and fail closed rather than crash: no + // identity means no fence, and without a fence the loop must not run. + let toolLoopEnabled = isServerOwnedToolLoopEnabled(); + let postInjectionRequestIdentity = ""; + if (toolLoopEnabled) { + try { + postInjectionRequestIdentity = derivePostInjectionRequestIdentity({ + apiKeyId: memoryOwnerId || "local", + headers: clientRawRequest?.headers ?? null, + skillRequestId, + postInjectionBody: (body || {}) as Record, + }); + } catch (identityError) { + log?.warn?.( + "SERVER_OWNED_TOOL_LOOP", + `request body is not canonicalizable, skipping the loop: ${ + identityError instanceof Error ? identityError.message : "unknown" + }` + ); + toolLoopEnabled = false; + } + } + const loopApply = await applyServerOwnedToolLoopIfNeeded({ + enabled: toolLoopEnabled, + stream, + isResponsesEndpoint, + sourceFormat, + initialLeg: legResult, + sourceBody: (body || {}) as Record, + skillsModelId: getSkillsModelIdForFormat(sourceFormat), + executionContext: { + apiKeyId: memoryOwnerId || "local", + sessionId: pipelineSessionId, + requestId: skillRequestId, + requestIdentity: postInjectionRequestIdentity, + builtinToolNames: injectionResult.builtinToolNames, + injectedCustomSkillNames: injectionResult.injectedCustomSkillNames, + customSkillExecutionEnabled: + Boolean(memoryOwnerId) && memorySettings?.skillsEnabled === true, + executionFenceEnabled: true, + provider, + model: effectiveModel, + }, + abortSignal: clientRawRequest?.signal, + expectedConnectionId: expectedConn, + followUpLeg: async (nextSourceBody) => { + translatedBody = translateRequest( + sourceFormat, + targetFormat, + model, + { ...nextSourceBody }, + false, + credentials, + provider, + reqLogger, + { + normalizeToolCallId: getModelNormalizeToolCallId( + provider || "", + model || "", + sourceFormat + ), + preserveDeveloperRole: getModelPreserveOpenAIDeveloperRole( + provider || "", + model || "", + sourceFormat + ), + preserveCacheControl, + signatureNamespace: connectionId, + copilotClient: copilotCompatibleReasoning, + reasoningCacheScope, + onReasoningReplayHistory: (messages) => { + reasoningReplayHistory = messages; + }, + } + ); + return runNonStreamingProviderLeg( + followUpLegInput( + { + executeProviderRequest: (modelToCall, allowDedup) => + executeProviderRequest(modelToCall, allowDedup), + runProviderExecution: runNonStreamingPipeline, + setRequestWireState: ({ translatedBody: nextBody, effectiveModel: nextModel }) => { + translatedBody = nextBody as typeof translatedBody; + currentModel = nextModel; + triedModels.add(nextModel); + }, + sourceFormat, + targetFormat, + clientResponseFormat, + provider, + model: effectiveModel, + connectionId: String(getCurrentConnectionId() || connectionId || ""), + getCurrentConnectionId: () => getCurrentConnectionId() || undefined, + effectiveModel: currentModel, + translatedBody: translatedBody as Record, + toolNameMap, + customToolNames, + requestToolIdentityMap, + reasoningCacheScope, + reasoningReplayHistory, + clientHeaders: clientRawRequest?.headers ?? null, + isClaudeCodeCompatible, + log, + }, + nextSourceBody, + expectedConn + ) + ); + }, + logReceipt: (receipt) => reqLogger.logToolLoopReceipt(receipt), + }); + if (loopApply.kind === "error") { + return await finalizeToolLoopError({ + loop: loopApply.loop, + model, + provider, + connectionId, + providerRequest: loopApply.loop.finalProviderRequest || finalBody || translatedBody, + persistFailureUsage, + persistAttemptLogs, + trackPendingRequest, + }); + } + // `legResult` is declared as the full NonStreamingProviderLegResult union. The + // `kind === "error"` guard above narrows it to the ok variant, but the conditional + // reassignment below widens it back to the declared type, so every field read past + // this point lost the narrowing — 13 TS2339 diagnostics under + // tsconfig.typecheck-api.json, which pulls chatCore.ts in through the route while + // tsconfig.typecheck-core.json does not. Pin the ok variant in its own binding: + // `loopApply.leg` is already `NonStreamingProviderLegResult & { kind: "ok" }`, + // so no cast is involved. + let okLeg: NonStreamingProviderLegResult & { kind: "ok" } = legResult; + if (loopApply.kind === "ok") { + toolLoopRan = true; + toolLoopUsage = loopApply.usage; + okLeg = loopApply.leg; + } + + if (okLeg.upstreamResponse) { + providerResponse = okLeg.upstreamResponse; + providerHeaders = normalizeHeaders(okLeg.upstreamResponse.headers); + } else { + providerResponse = new Response(null, { + status: 200, + headers: okLeg.headers, + }); + providerHeaders = normalizeHeaders(okLeg.headers); + } + finalBody = providerRequestCapture.body(okLeg.providerRequest || translatedBody); + // Built inside executeProviderRequest on the pre-#12867 path. The leg now owns the + // first non-streaming send, so that assignment never runs here and the meta stayed + // null — `_omniroute.claudePromptCache` silently vanished from every call log on + // this path. Same inputs, same helper, at the point where they are available. + claudePromptCacheLogMeta = buildClaudePromptCacheLogMeta( + targetFormat, + finalBody, + providerHeaders, + clientRawRequest?.headers + ); + const capturedOk = providerRequestCapture.latest?.(); + reqLogger.logTargetRequest( + okLeg.requestUrl || capturedOk?.url || "", + okLeg.requestHeaders || capturedOk?.headers || {}, + capturedOk?.body ?? finalBody + ); + const responseBody = okLeg.providerBody; + const responsePayloadFormat = okLeg.responsePayloadFormat; + const looksLikeSSE = okLeg.looksLikeSSE; + let translatedResponse = okLeg.response; + const memoryExtractionResponse = okLeg.responseForMemoryExtraction; + reqLogger.logProviderResponse( + 200, + "OK", + providerResponse.headers, + looksLikeSSE ? { _streamed: true, _format: "sse-json", summary: responseBody } : responseBody + ); + effectiveServiceTier = resolveReportedServiceTier(responseBody) ?? effectiveServiceTier; + if (onRequestSuccess) { + await onRequestSuccess(); + } + const successConnectionId = getCurrentConnectionId(); + await maybeSyncClaudeExtraUsageState({ + provider, + connectionId: successConnectionId, + providerSpecificData: credentials?.providerSpecificData, + log, + }); + const usage = toolLoopUsage ?? extractUsageFromResponse(responseBody, provider); + const cacheUsageLogMeta = buildCacheUsageLogMeta(usage); + if (usage && typeof usage === "object") { + attachCompressionUsageReceiptAfterAnalytics(usage as Record, "provider"); + if (provider === "gemini") { + const promptTokens = + typeof (usage as Record).prompt_tokens === "number" + ? ((usage as Record).prompt_tokens as number) + : 0; + if (promptTokens > 0) incrementTokenUsage(model, promptTokens); + } + } + recordContextEditingTelemetryHook({ + contextEditingEnabled, + provider, + responseBody, + skillRequestId, + log, + }); + appendRequestLog({ + model, + provider, + connectionId: successConnectionId, + tokens: usage, + status: "200 OK", + }).catch(() => {}); + recordNonStreamingUsageStats(usage, { + traceEnabled, + provider, + connectionId: successConnectionId, + model, + startTime, + apiKeyInfo, + effectiveServiceTier, + isCombo, + comboStrategy, + endpoint: endpointPath, + }); + + // #12150 P1b surface 3 (fix round 1): a video-bridge-observed request's + // request- AND response-derived text both carry the full transcript (the + // flattened description on the request side, the model's own reply on + // the response side) — neither may populate durable Memory. See + // runMemoryExtractionGate for the shared gate + extraction wiring, unit + // tested directly in tests/unit/video-bridge-memory-suppression.test.ts. + runMemoryExtractionGate({ + memoryOwnerId, + memorySettings, + videoBridgeObserved, + pipelineSessionId, + requestBody: body as Record, + responseBody: memoryExtractionResponse as Record | null, + extractFacts, + log, + }); + + const customSkillExecutionEnabled = + Boolean(memoryOwnerId) && memorySettings?.skillsEnabled === true; + const builtinToolNames = [ + webSearchFallbackPlan.toolName, + webFetchFallbackPlan.toolName, + ...(memoryOwnerId && memorySettings?.enabled ? MEMORY_BUILTIN_TOOL_NAMES : []), + ].filter((name): name is string => Boolean(name)); + if (!toolLoopRan && (customSkillExecutionEnabled || builtinToolNames.length > 0)) { + const skillSessionId = pipelineSessionId; + + translatedResponse = await handleToolCallExecution( + translatedResponse, + getSkillsModelIdForFormat(sourceFormat), + { + apiKeyId: memoryOwnerId || "local", + sessionId: skillSessionId, + requestId: skillRequestId, + builtinToolNames, + customSkillExecutionEnabled, + provider, + model: effectiveModel, + } + ); + } + + const guardrailContext = buildPostCallGuardrailContext({ + apiKeyInfo, + body, + clientRawRequest, + log, + model, + provider, + responsePayloadFormat, + clientResponseFormat, + }); + const postCallGuardrails = await guardrailRegistry.runPostCallHooks( + translatedResponse, + guardrailContext + ); + translatedResponse = postCallGuardrails.response; + + const responseUsage = isJsonRecord(usage) + ? usage + : isJsonRecord(translatedResponse.usage) + ? translatedResponse.usage + : null; + const costUsage = normalizeUsage(responseUsage); + const estimatedCost = costUsage + ? await calculateCost(provider, model, costUsage, { serviceTier: effectiveServiceTier }) + : 0; + + if (postCallGuardrails.blocked) { + const guardrailMessage = postCallGuardrails.message || "Response blocked by guardrail"; + persistAttemptLogs({ + status: HTTP_STATUS.BAD_REQUEST, + tokens: usage, + responseBody, + providerRequest: finalBody || translatedBody, + providerResponse: looksLikeSSE + ? { + _streamed: true, + _format: "sse-json", + summary: responseBody, + } + : responseBody, + clientResponse: buildErrorBody(HTTP_STATUS.BAD_REQUEST, guardrailMessage), + claudeCacheMeta: claudePromptCacheLogMeta, + claudeCacheUsageMeta: cacheUsageLogMeta, + cacheSource: "upstream", + }); + if (apiKeyInfo?.id && estimatedCost > 0) { + recordCost(apiKeyInfo.id, estimatedCost); + } + log?.warn?.( + "GUARDRAIL", + `Response blocked by ${postCallGuardrails.guardrail || "guardrail"}: ${guardrailMessage}` + ); + finalizePendingScope(pendingScope, { + providerResponse: responseBody, + clientResponse: translatedResponse, + }); + return createErrorResult(HTTP_STATUS.BAD_REQUEST, guardrailMessage); + } + + // Validate the *translated* response actually carries client-usable output. + // isEmptyContentResponse (above) runs on the raw responseBody before translation; + // this check runs after translation + sanitization + tool-call execution to catch + // cases where a provider returns a structurally valid raw body that translates into + // choices:[] or output:[] with no usable content (Responses API shape included). + const malformedTranslatedReason = detectMalformedNonStream(translatedResponse, provider); + if (malformedTranslatedReason) { + const totalLatency = Date.now() - startTime; + const rawBytes = (() => { + try { + return JSON.stringify(responseBody || {}).length; + } catch { + return -1; + } + })(); + reportMalformed200({ + mode: "nonstream", + provider, + model, + connectionId, + reason: malformedTranslatedReason, + recvBytes: rawBytes, + recvLines: -1, + emitted: -1, + events: {}, + ttftMs: totalLatency, + elapsedMs: totalLatency, + }); + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`, + }).catch(() => {}); + const malformed = describeMalformedNonStream(translatedResponse, malformedTranslatedReason); + const malformedMessage = `[${provider}/${model}] ${malformed.message}`; + const malformedClientBody = buildErrorBody( + HTTP_STATUS.BAD_GATEWAY, + malformedMessage, + undefined, + { code: malformed.code, type: malformed.type } + ); + const sanitizedMalformedResponse = sanitizeUpstreamDetails(responseBody); + const sanitizedMalformedProviderResponse = looksLikeSSE + ? { _streamed: true, _format: "sse-json", summary: sanitizedMalformedResponse } + : sanitizedMalformedResponse; + persistAttemptLogs({ + status: HTTP_STATUS.BAD_GATEWAY, + tokens: usage, + responseBody: sanitizedMalformedResponse, + providerRequest: finalBody || translatedBody, + providerResponse: sanitizedMalformedProviderResponse, + clientResponse: malformedClientBody, + claudeCacheMeta: claudePromptCacheLogMeta, + claudeCacheUsageMeta: cacheUsageLogMeta, + cacheSource: "upstream", + }); + persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "malformed_translated_response"); + trackPendingRequest(model, provider, pendingConnId, false); + // Routing event (feedback foundation) — record the malformed outcome so + // the quality tracker de-prioritizes this model over time. + void emitRoutingEvent( + createRoutingEvent({ + requestId: traceId || pendingRequestId || "unknown", + provider: provider || "unknown", + model: model || "unknown", + strategy: isCombo ? (comboStrategy ?? "combo") : "direct", + latencyMs: Date.now() - startTime, + ttftMs: null, + inputTokens: null, + outputTokens: null, + cost: null, + retries: 0, + fallbackUsed: false, // combo-level fallback tracked by decisionTrace + outcome: "malformed", + status: HTTP_STATUS.BAD_GATEWAY, + finishReason: routingFinishReason(translatedResponse), + connectionId: credentials?.connectionId ?? null, + }) + ); + return createErrorResult( + HTTP_STATUS.BAD_GATEWAY, + malformedMessage, + null, + malformed.code, + malformed.type + ); + } + + // ── Phase 9.1: Cache store (non-streaming, temp=0) ── + storeSemanticCacheResponse({ + enabled: semanticCacheEnabled, + body: bodyForCacheWrite, + headers: clientRawRequest?.headers, + translatedResponse, + model, + provider, + apiKeyId: apiKeyInfo?.id ?? undefined, + usage, + log, + }); + + // ── Phase 9.2: Save for idempotency ── + // Reuse the key resolved by checkIdempotencyCache() above (single derivation per + // request). (#3821-review LEDGER-6) + saveIdempotency(idempotencyKey, translatedResponse, 200); + reqLogger.logConvertedResponse(translatedResponse); + persistAttemptLogs({ + status: 200, + tokens: usage, + responseBody, + providerRequest: finalBody || translatedBody, + providerResponse: looksLikeSSE + ? { + _streamed: true, + _format: "sse-json", + summary: responseBody, + } + : responseBody, + clientResponse: translatedResponse, + claudeCacheMeta: claudePromptCacheLogMeta, + claudeCacheUsageMeta: cacheUsageLogMeta, + cacheSource: "upstream", + }); + if (apiKeyInfo?.id && estimatedCost > 0) { + recordCost(apiKeyInfo.id, estimatedCost); + } + + // === Quota Share POST-hook (B/F7) — fire-and-forget, fail-open === + await scheduleQuotaShareConsumption({ + apiKeyId: apiKeyInfo?.id, + connectionId: credentials?.connectionId, + provider, + model, + usage, + estimatedCost, + log, + }); + // === /Quota Share POST-hook === + + // ── Gamification event (fire-and-forget) ── + await emitRequestGamificationEvent({ apiKeyId: apiKeyInfo?.id, model, provider }); + + finalizePendingScope(pendingScope, { + providerResponse: responseBody, + clientResponse: translatedResponse, + }); + const responseHeaders = buildNonStreamingResponseHeaders({ + provider, + model, + startTime, + responseUsage, + estimatedCost, + requestId: skillRequestId, + compressionResponseMeta, + comboStrategy, + fallbackAttempts, + }); + // #6426: align response body `model` with the `X-OmniRoute-Model` header + // (both must be the resolved backend model). Some upstreams (notably legacy + // /v1/completions text-completion path) return a body `model` field that + // differs from the resolved backend id we advertised in the header, leaving + // strict clients unable to reconcile the two. Rewrite body.model to `model` + // FIRST, then let #1311 echo override it when the opt-in setting is on. + if (typeof model === "string" && model) echoModelInObject(translatedResponse, model); + // #1311: echo the requested alias/combo name in the non-streaming response model. + if (echoModel) echoModelInObject(translatedResponse, echoModel); + + // ── Plugin onResponse hook (fire-and-forget) ── + // #8395: the streaming branch below already calls this; the non-streaming + // (stream:false) branch returned without it, so onResponse never fired for + // non-streaming requests at all. + await runPluginOnResponseHook({ + requestId: traceId, + body, + model, + provider, + apiKeyInfo, + headers: clientRawRequest?.headers, + response: { status: 200, data: translatedResponse }, + }); + + // Routing event (feedback foundation) — fire-and-forget, cheap. + void emitRoutingEvent( + createRoutingEvent({ + requestId: traceId || pendingRequestId || "unknown", + provider: provider || "unknown", + model: model || "unknown", + strategy: isCombo ? (comboStrategy ?? "combo") : "direct", + latencyMs: Date.now() - startTime, + ttftMs: null, + inputTokens: + usage && typeof usage === "object" + ? (() => { + const promptTokens = (usage as Record).prompt_tokens; + return typeof promptTokens === "number" && Number.isFinite(promptTokens) + ? promptTokens + : null; + })() + : null, + outputTokens: + usage && typeof usage === "object" + ? (() => { + const completionTokens = (usage as Record).completion_tokens; + return typeof completionTokens === "number" && Number.isFinite(completionTokens) + ? completionTokens + : null; + })() + : null, + cost: Number.isFinite(estimatedCost) ? estimatedCost : null, + retries: 0, + fallbackUsed: false, // combo-level fallback tracked by decisionTrace + outcome: "success", + status: 200, + finishReason: routingFinishReason(translatedResponse), + connectionId: credentials?.connectionId ?? null, + }) + ); + + return { + success: true, + response: maybeWrapForcedNonStreamingResponsesJson({ + clientRequestedResponsesStream, + body: translatedResponse, + headers: responseHeaders, + }), + }; + } catch (error) { + trackPendingRequest(model, provider, connectionId, false); + const errorMetadata = getSafeErrorMetadata(error); + const managedLeaseFenceCode = getManagedLeaseFenceErrorCode(errorMetadata.code); + if (managedLeaseFenceCode) return managedLeaseFenceErrorResult(managedLeaseFenceCode); + // isSemaphoreCapacityError already reads the code through getSafeErrorMetadata, + // so a hostile rejection cannot escape this classification. + if (isSemaphoreCapacityError(error)) { + const semaphoreCode = errorMetadata.code as string; + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${semaphoreCode}`, + }).catch(() => {}); + const failureMessage = sanitizeErrorMessage(errorMetadata.message) || "Semaphore timeout"; + persistAttemptLogs({ + status: HTTP_STATUS.RATE_LIMITED, + error: failureMessage, + providerRequest: finalBody || translatedBody, + clientResponse: buildErrorBody(HTTP_STATUS.RATE_LIMITED, failureMessage), + claudeCacheMeta: claudePromptCacheLogMeta, + cacheSource: "upstream", + }); + persistFailureUsage(HTTP_STATUS.RATE_LIMITED, semaphoreCode); + const result = createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); + return { + ...result, + errorType: "account_semaphore_capacity", + errorCode: semaphoreCode, + }; + } + throw error; + } +} From 71d8b106001c16b475eae13a895528fe0be35df2 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 19:38:11 -0400 Subject: [PATCH 11/26] refactor(sse): lift the streaming leg out of the chatCore barrel This is the companion to the non-streaming lift, and the two legs differ in one way that shapes the contract. The non-streaming block owned every exit, so it could be a plain call. This one cannot: six values it assigns are read again by the streaming response section that runs after it -- claudePromptCacheLogMeta, effectiveServiceTier, finalBody, providerHeaders, providerResponse and translatedBody. That section is only reachable when stream is true, because the non-streaming leg has already returned by then, so those writes are a real data flow rather than leftovers. Rather than let a shared mutable closure carry them, runStreamingLeg returns them in a `carry` object and the barrel writes them back at the callsite. The handoff is visible in the diff instead of hiding in scope. Control flow transfers unchanged. All thirteen exits return a value, so they become `{ kind: "returned", value, carry }` and the barrel re-returns it. The `providerFailure` label spans 2588..2971 of the old barrel and both of its breaks sit inside that range, so the jumps never crossed the boundary being cut. The function-level try/finally that releases turn execution stays in the barrel where it belongs -- it wraps far more than this block. Types come from the barrel's own checker, and where a field already had a shape in the non-streaming leaf that shape is reused so the two legs agree. Two exceptions are deliberate. ErrorResponseBody is declared but not exported by open-sse/utils/error.ts, so its shape is mirrored locally rather than widening the export surface of a file this commit has no business touching. finalBody stays untyped for the reason the sibling leaf already documents: the barrel's implicit any holds two incompatible shapes and naming either one would narrow what upstream left open. One test moved with the code. recovery-policy.test.ts asserts on source text, looking for the familyRecovery callsite in chatCore.ts to prove isolateProbe is resolved rather than hard-coded. The callsite is now in the leaf, so the assertion reads the leaf. Hard-coding isolateProbe: false still turns it red, which is the regression it exists to catch. Barrel drops from 3382 to 2504 lines. Suite is 60 green, typecheck:core clean, lint clean. Injecting a throw at the top of runStreamingLeg turns a test red, confirming the leaf is on the live path and not merely compiled. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 1020 +-------------- open-sse/handlers/chatCore/streamingLeg.ts | 1286 +++++++++++++++++++ tests/unit/chatcore/recovery-policy.test.ts | 6 +- 3 files changed, 1361 insertions(+), 951 deletions(-) create mode 100644 open-sse/handlers/chatCore/streamingLeg.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index ea0e54e92fb3..e908e5050c7e 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -12,11 +12,7 @@ import { normalizeOpenAICompatibleTools, shouldNormalizeFunctionToolsOnly, } from "./chatCore/openAICompatibleTools.ts"; -import { - projectFailureUsageErrorCode, - buildFailureUsageRecord, - type FailureUsageAggregate, -} from "./chatCore/failureUsage.ts"; +import { buildFailureUsageRecord, type FailureUsageAggregate } from "./chatCore/failureUsage.ts"; import { createTranslationFailureResult } from "./chatCore/translationFailure.ts"; import { estimateFinalInputTokens } from "./chatCore/contextEstimation.ts"; import { @@ -52,10 +48,7 @@ import { readCodexTurnStateHeader, } from "../config/codexTurnState.ts"; export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comboContextCache.ts"; -import { - resolveAccountSemaphoreKey, - buildClaudePromptCacheLogMeta, -} from "./chatCore/executorHelpers.ts"; +import { resolveAccountSemaphoreKey } from "./chatCore/executorHelpers.ts"; import { shouldUseNativeCodexPassthrough, shouldUseNativeXaiResponsesPassthrough, @@ -63,12 +56,8 @@ import { redactPassthroughThinkingSignatures, isClaudeCodeSemanticPassthroughRequest, } from "./chatCore/passthroughHelpers.ts"; -import { recoverAnthropicThinkingSignature } from "./chatCore/thinkingSignatureRecovery.ts"; -import { runProviderExecutionPipeline } from "./chatCore/providerExecutionPipeline.ts"; -import { onFailure, onStreamThrow } from "./chatCore/recoveryPolicy.ts"; import { runNonStreamingLeg } from "./chatCore/nonStreamingLeg.ts"; -import { markCodexScopeRateLimited } from "./chatCore/codexFailover.ts"; -import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity"; +import { runStreamingLeg } from "./chatCore/streamingLeg.ts"; import { buildStreamingResponseHeaders, stripStaleForwardingHeaders, @@ -98,7 +87,6 @@ import { ensureCacheControlOnLastUserMessage } from "../services/claudeCodeConst import { createSSETransformStreamWithLogger, createPassthroughStreamWithLogger, - COLORS, } from "../utils/stream.ts"; import { ensureStreamReadiness } from "../utils/streamReadiness.ts"; import { resolveSuppressThinkClose } from "../utils/thinkCloseMarker.ts"; @@ -106,13 +94,7 @@ import { resolveStreamReadinessTimeout } from "../utils/streamReadinessPolicy.ts import { hasActiveClaudeThinking } from "../utils/thinkingBudget.ts"; import { createStreamController } from "../utils/streamHandler.ts"; import * as streamFailure from "../utils/streamFailureFinalization.ts"; -import { - refreshWithRetry, - isUnrecoverableRefreshError, - runWithOnPersist, - runWithCasGuard, -} from "../services/tokenRefresh.ts"; -import { runWithCapture } from "../utils/providerRequestLogging.ts"; +import { refreshWithRetry, runWithOnPersist, runWithCasGuard } from "../services/tokenRefresh.ts"; import { applyResponsesPreviousResponseIdPolicy } from "../utils/responsesStatePolicy.ts"; import { DEFAULT_THINKING_CLAUDE_SIGNATURE } from "../config/defaultThinkingSignature.ts"; import { @@ -137,22 +119,8 @@ import { areContextWindowChecksDisabled, isFeatureFlagEnabled, } from "@/shared/utils/featureFlags.ts"; -import { - REASONING_BUFFER_MIN_TRIGGER, - buildReasoningProbeTruncatedResponse, - isEmptyContentUpstreamFailure, - isTinyBudgetReasoningProbe, - toPositiveInteger, -} from "../services/reasoningTokenBuffer.ts"; -import { - buildErrorBody, - createErrorResult, - parseUpstreamError, - formatProviderError, - projectPublicErrorIdentifier, - sanitizeErrorMessage, - sanitizeUpstreamDetails, -} from "../utils/error.ts"; +import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts"; +import { buildErrorBody, createErrorResult, sanitizeErrorMessage } from "../utils/error.ts"; import { checkTokenLimits } from "@omniroute/open-sse/services/tokenLimitCounter.ts"; import { COOLDOWN_MS, @@ -160,24 +128,12 @@ import { PROVIDER_MAX_TOKENS, STREAM_READINESS_MAX_TIMEOUT_MS, STREAM_READINESS_TIMEOUT_MS, - ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE, DEFAULT_MAX_TOKENS, STREAM_DISCONNECT_GRACE_PERIOD_MS, } from "../config/constants.ts"; -import { applyStatusRestatement } from "../config/upstreamStatusRestatement.ts"; import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts"; import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; -import { wasRefreshTokenRotated } from "@omniroute/open-sse/services/refreshSerializer.ts"; import { connectionHasExtraKeys } from "../services/apiKeyRotator.ts"; -import { - createSafeAbortError, - createStreamingErrorResult, - isSemaphoreCapacityError, - getSafeErrorMetadata, - getUpstreamErrorIdentifier, -} from "./chatCore/streamErrorResult.ts"; -import { buildExecutorClientHeaders } from "./chatCore/executorClientHeaders.ts"; -import { getExecutionConnectionId } from "./chatCore/executionCredentials.ts"; import { resolveExecutionCredentials as resolveExecutionCredentialsFor } from "./chatCore/executionCredentials.ts"; import { resolveExecutorWithProxy as resolveExecutorWithProxyFor } from "./chatCore/executorProxy.ts"; import type { ClaudeMessage } from "./chatCore/claudeMessageTypes.ts"; @@ -187,12 +143,10 @@ import { type PersistAttemptLogsArgs, } from "./chatCore/attemptLogging.ts"; import { attachCompressionUsageReceiptAfterAnalytics as attachCompressionUsageReceiptAfterAnalyticsFor } from "./chatCore/compressionUsageReceipt.ts"; -import { prepareUpstreamBody } from "./chatCore/upstreamBody.ts"; import { getQuotaScopeLabelForProvider } from "../services/antigravityQuotaFamily.ts"; import { excludeConnectionForCooldown } from "./chatCore/connectionCooldown.ts"; import { handleRequestRejectedFailure } from "./chatCore/requestRejectedFailure.ts"; import { getKimiTemporaryRateLimitResetAt } from "./chatCore/kimiQuotaRecovery.ts"; -import { logAuditEvent } from "@/lib/compliance"; import { adaptBodyForCompression } from "../services/compression/bodyAdapter.ts"; import { trackPendingRequest, appendRequestLog, saveRequestUsage } from "@/lib/usageDb"; import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; @@ -206,9 +160,7 @@ import type { EnforceDecision } from "@/lib/quota/types"; import { writeCompressionAnalytics } from "./chatCore/compressionAnalyticsWrite.ts"; import { emitRequestGamificationEvent } from "./chatCore/gamificationEvent.ts"; import { runPluginOnResponseHook } from "./chatCore/pluginOnResponse.ts"; -import { normalizeExecutorResult } from "./chatCore/upstreamTimeouts.ts"; import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; -import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; import { getCacheControlSettings } from "@/lib/cacheControlSettings"; import type { VideoBridgeLogRedactionEntry } from "@/lib/guardrails/videoBridge"; @@ -217,7 +169,6 @@ import { resolveConnectionCacheOverride, } from "../utils/cacheControlPolicy.ts"; import { updateFromHeaders, updateFromResponseBody } from "../services/rateLimitManager.ts"; -import * as localLimiterErrors from "../services/rateLimitManager/errors.ts"; import { markBlocked as markAccountSemaphoreBlocked } from "../services/accountSemaphore.ts"; import { lockModel, @@ -225,15 +176,8 @@ import { recordCoreOwnedAntigravityQuotaState, shouldDeferAntigravityQuotaStateToCaller, } from "../services/accountFallback.ts"; -import { - getNextFamilyFallback, - isContextOverflowError, - findLargerContextModel, - getModelFamily, -} from "../services/modelFamilyFallback.ts"; import { compressContext, estimateTokens, getTokenLimit } from "../services/contextManager.ts"; import type { CompressionResult } from "../services/compression/types.ts"; -import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; import { extractFacts } from "@/lib/memory/extraction"; @@ -2109,893 +2053,71 @@ export async function handleChatCore({ let pipelineRecovered = false; if (stream) { - try { - const pipelineOutcome = await runProviderExecutionPipeline({ - policy: { - allowAccountRotation: !managedLease && comboStrategy !== "context-relay", - allowModelFallback: true, - expectedConnectionId: managedLease - ? String(getCurrentConnectionId() || connectionId || "") || undefined - : undefined, - }, - target: { - provider, - requestedModel: effectiveModel, - sourceFormat, - targetFormat, - stream, - }, - connection: { - initialConnectionId: String(getCurrentConnectionId() || connectionId || ""), - getCurrentConnectionId: () => getCurrentConnectionId() || undefined, - getCredentials: () => (credentials || {}) as Record, - replaceCredentials: (next) => { - Object.assign(credentials, next); - }, - onCredentialsRefreshed: handleCredentialsRefreshed, - refreshCredentials: executeRefreshCredentials, - assertManagedLeaseFence: (id) => { - assertManagedLeaseFence(id); - }, - getProviderCredentials, - }, - wire: { - body: translatedBody as Record, - currentModel, - triedModels, - setBodyAndModel: (body, model) => { - translatedBody = body as typeof translatedBody; - currentModel = model; - triedModels.add(model); - }, - }, - state: { - updatePendingStage: (stage, data) => { - updatePendingScope(pendingScope, { stage, ...(data || {}) }); - }, - recordRateLimitHeaders: updateFromHeaders, - recordRateLimitBody: updateFromResponseBody, - writeTerminalStatus, - persistConnectionPatch: updateProviderConnection, - setConnectionRateLimitedUntil: async (id, untilMs) => { - const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); - setConnectionRateLimitUntil(id, untilMs); - }, - lockModel, - recordAntigravityQuotaState: recordCoreOwnedAntigravityQuotaState, - markAccountSemaphoreBlocked: (key) => { - markAccountSemaphoreBlocked(key, Date.now() + 60_000); - }, - isolateProbeFailures: () => shouldIsolateProbeFailures(), - onCodexScopeRateLimited: async (params) => { - await markCodexScopeRateLimited({ - failedConnectionId: params.failedConnectionId, - model: params.model, - rateLimitedUntil: params.rateLimitedUntil, - credentials: (params.credentials || credentials) as { - connectionId?: string | null; - providerSpecificData?: unknown; - }, - }); - }, - onClearSessionAffinity: () => { - const key = - sessionAffinityKey || - extractSessionAffinityKey(body, clientRawRequest?.headers) || - null; - if (!key) return; - try { - deleteSessionAccountAffinity(key, "codex"); - } catch { - // best-effort - } - }, - onAuditAccountRotation: (params) => { - logAuditEvent({ - action: params.action, - actor: apiKeyInfo?.name || "system", - target: params.newConnectionId, - details: { - failed_connection_id: params.failedConnectionId, - new_connection_id: params.newConnectionId, - attempt: params.attempt, - retry_after_ms: params.retryAfterMs, - }, - }); - }, - }, - sendProviderAttempt: (modelToCall, allowDedup) => - executeProviderRequest(modelToCall, allowDedup), - }); - - pipelineRecovered = true; - currentModel = pipelineOutcome.model; - if (pipelineOutcome.kind === "error") { - providerResponse = pipelineOutcome.result.response; - providerUrl = ""; - providerHeaders = normalizeHeaders(pipelineOutcome.result.response.headers); - finalBody = translatedBody; - } else { - const result = { - response: pipelineOutcome.response, - url: pipelineOutcome.url, - headers: pipelineOutcome.headers, - transformedBody: pipelineOutcome.transformedBody, - }; - providerResponse = result.response; - providerUrl = result.url; - providerHeaders = result.headers; - finalBody = providerRequestCapture.body(result.transformedBody); - } - const responseConnectionId = getCurrentConnectionId(); - effectiveServiceTier = resolveEffectiveServiceTier(finalBody); - claudePromptCacheLogMeta = buildClaudePromptCacheLogMeta( - targetFormat, - finalBody, - providerHeaders, - clientRawRequest?.headers - ); - - // Log target request (final request to provider) - reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); - updatePendingScope(pendingScope, { - providerRequest: finalBody, - providerUrl, - stage: "provider_response_started", - }); - // Update rate limiter from response headers (learn limits dynamically) - updateFromHeaders( - provider, - responseConnectionId, - providerResponse.headers, - providerResponse.status, - model - ); - - // Store rate-limit headers for quota saturation signals - try { - const { storeRateLimitHeaders } = await import("@/lib/quota/saturationSignals"); - storeRateLimitHeaders( - responseConnectionId, - provider, - providerResponse.headers as Record - ); - } catch { - // fail-open: saturation signal is best-effort - } - } catch (error) { - onStreamThrow(); - trackPendingRequest(model, provider, connectionId, false); - const errorMetadata = getSafeErrorMetadata(error); - const managedLeaseFenceCode = getManagedLeaseFenceErrorCode(errorMetadata.code); - if (managedLeaseFenceCode) return managedLeaseFenceErrorResult(managedLeaseFenceCode); - // isSemaphoreCapacityError already reads the code through getSafeErrorMetadata, - // so a hostile rejection cannot escape this classification. - if (isSemaphoreCapacityError(error)) { - const semaphoreCode = errorMetadata.code as string; - appendRequestLog({ - model, - provider, - connectionId, - status: `FAILED ${semaphoreCode}`, - }).catch(() => {}); - const failureMessage = sanitizeErrorMessage(errorMetadata.message) || "Semaphore timeout"; - persistAttemptLogs({ - status: HTTP_STATUS.RATE_LIMITED, - error: failureMessage, - providerRequest: finalBody || translatedBody, - clientResponse: buildErrorBody(HTTP_STATUS.RATE_LIMITED, failureMessage), - claudeCacheMeta: claudePromptCacheLogMeta, - cacheSource: "upstream", - }); - persistFailureUsage(HTTP_STATUS.RATE_LIMITED, semaphoreCode); - const result = stream - ? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, semaphoreCode) - : createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); - return { - ...result, - errorType: "account_semaphore_capacity", - errorCode: semaphoreCode, - }; - } - // abort(reason) can reject with a raw string lacking `name`/`status`; classify - // it through isLocalStreamLifecycleError so it maps to 499 rather than the - // 502 provider-failure default. - let isRequestAborted = errorMetadata.name === "AbortError"; - if (!isRequestAborted) { - try { - isRequestAborted = isLocalStreamLifecycleError(error); - } catch { - // A hostile Proxy must not escape the provider-error boundary during classification. - } - } - // #8376: proxyFetch tags unreachable transport failures so they remain - // distinguishable from ordinary provider 5xx responses. - const isProxyUnreachableFailure = - !isRequestAborted && errorMetadata.errorCode === "proxy_unreachable"; - const errorCode = errorMetadata.code; - const localRateLimitFailure = localLimiterErrors.getClientSafeLocalRateLimitError(error); - const failureStatus = isRequestAborted - ? 499 - : isProxyUnreachableFailure - ? HTTP_STATUS.BAD_GATEWAY - : localRateLimitFailure - ? localRateLimitFailure.status - : errorMetadata.name === "TimeoutError" || errorMetadata.name === "BodyTimeoutError" - ? HTTP_STATUS.GATEWAY_TIMEOUT - : errorMetadata.status - ? errorMetadata.status - : HTTP_STATUS.BAD_GATEWAY; - const failureMessage = isRequestAborted - ? "Request aborted" - : (() => { - try { - return formatProviderError( - localRateLimitFailure ?? error, - provider, - model, - failureStatus - ); - } catch { - // Formatting is diagnostic only; hostile rejection metadata falls back safely. - return errorMetadata.message || "Upstream provider error"; - } - })(); - const safeFailureMessage = sanitizeErrorMessage(failureMessage) || "Upstream provider error"; - const upstreamErrorCode = - localRateLimitFailure?.code ?? - (isProxyUnreachableFailure ? "proxy_unreachable" : errorCode); - // Tag our own deadline timeouts (fetch-start TimeoutError / body BodyTimeoutError, - // both surfaced as a 504) as "upstream_timeout" so the cooldown layer can tell a - // slow-but-not-failed request apart from a real provider 5xx. (Antigravity already - // tags its pre-response timeout via the code below.) - const isOwnDeadlineTimeout = - failureStatus === HTTP_STATUS.GATEWAY_TIMEOUT && - (errorMetadata.name === "TimeoutError" || errorMetadata.name === "BodyTimeoutError"); - const upstreamErrorType = - upstreamErrorCode === ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE || isOwnDeadlineTimeout - ? "upstream_timeout" - : failureStatus === 401 - ? "authentication_error" - : undefined; - appendRequestLog({ - model, - provider, - connectionId, - status: `FAILED ${failureStatus}`, - }).catch(() => {}); - persistAttemptLogs({ - status: failureStatus, - error: safeFailureMessage, - providerRequest: finalBody || translatedBody, - // On a client-abort (AbortError), the client already disconnected before - // we ever got here — this body is what we WOULD have sent, not what was - // actually delivered. Logging it as `clientResponse` is misleading (the - // dashboard reads that field as "what the client received"), so omit it - // for this case; `error` above already records the failure reason. - clientResponse: - errorMetadata.name === "AbortError" - ? undefined - : buildErrorBody(failureStatus, failureMessage), - claudeCacheMeta: claudePromptCacheLogMeta, - cacheSource: "upstream", - }); - if (isRequestAborted) { - streamController.handleError(createSafeAbortError()); - return createErrorResult(499, "Request aborted"); - } - const persistentErrorCode = projectFailureUsageErrorCode({ - statusCode: failureStatus, - message: failureMessage, - errorCode: projectPublicErrorIdentifier( - upstreamErrorCode || errorMetadata.name, - "upstream_error" - ), - errorType: upstreamErrorType, - }); - persistFailureUsage(failureStatus, persistentErrorCode); - console.log(`${COLORS.red}[ERROR] ${safeFailureMessage}${COLORS.reset}`); - if (stream && upstreamErrorCode) { - const result = createStreamingErrorResult( - failureStatus, - failureMessage, - upstreamErrorCode, - upstreamErrorType - ); - localLimiterErrors.markTrustedLocalRateLimitResponse(result.response, error); - return { - ...result, - errorType: upstreamErrorType, - errorCode: upstreamErrorCode, - }; - } - const result = createErrorResult( - failureStatus, - failureMessage, - null, - upstreamErrorCode, - upstreamErrorType - ); - localLimiterErrors.markTrustedLocalRateLimitResponse(result.response, error); - return result; - } - let upstreamErrorParsed = false; - let parsedStatusCode = providerResponse.status; - let parsedMessage = ""; - let parsedRetryAfterMs: number | null = null; - let upstreamErrorBody: unknown = null; - - // Track whether stream_options was present and stripped — if so, 401/403 after - // that may be from the modification rather than a genuine auth failure, so we - // skip the credential refresh attempt in that case. - const hadStreamOptions = - targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody; - if (hadStreamOptions) { - delete translatedBody.stream_options; - } - - // Handle 401/403 - try token refresh using executor - // T-PROBE: probe-origin failures never attempt the refresh — a probe must - // not consume a rotating refresh token nor persist an "expired" - // deactivation on refresh failure (#9817). The 401/403 then flows into - // the normal providerFailure classification (record-only in probe mode). - if ( - (providerResponse.status === HTTP_STATUS.UNAUTHORIZED || - providerResponse.status === HTTP_STATUS.FORBIDDEN) && - !hadStreamOptions && // Skip refresh if failure may be from stream_options removal, not auth - !(await shouldIsolateProbeFailures()) - ) { - // Fix A: wrap refreshCredentials in runWithOnPersist so the persist callback - // executes INSIDE the per-connection mutex held by getAccessToken. This makes - // [network refresh + DB write + outer-state mutation] one atomic step and - // prevents concurrent requests from reading a stale refreshToken before the - // DB has been updated (refresh_token_reused on Codex/OpenAI). - // - // Not every executor routes refresh through getAccessToken (e.g. github.ts - // calls refreshCopilotToken directly). When the persistFn doesn't fire from - // inside getAccessToken, we still need to do the credentials mutation + user - // callback after refreshCredentials returns. The `persistFnRan` flag tracks - // which path executed so we don't double-fire (race-prone) or skip (regression). - // Front 3: remember the refresh_token we are about to present so that, if the - // refresh fails as unrecoverable, we can tell a genuine death apart from a - // stale-token reuse that a concurrent/sibling refresh already rotated past. - const attemptedRefreshToken = - typeof credentials?.refreshToken === "string" ? credentials.refreshToken : null; - let persistFnRan = false; - const persistFn = onCredentialsRefreshed - ? async (refreshResult: Record) => { - persistFnRan = true; - // Mutate the shared credentials object so subsequent executor calls - // in this request see the new tokens. Runs INSIDE the mutex. - Object.assign(credentials, refreshResult); - await onCredentialsRefreshed(refreshResult); - } - : undefined; - - // #4038: build a compare-and-swap reread so getAccessToken can skip the persist if a - // concurrent writer (sibling request / HealthCheck / replica) already rotated this - // connection's refresh_token past the one we presented — overwriting would revert it - // and revoke the token family. No connectionId ⇒ no guard (behavior unchanged). - const casConnectionId = - typeof credentials?.connectionId === "string" ? credentials.connectionId.trim() : ""; - const casReread = casConnectionId - ? async () => { - const latest = await getProviderConnectionById(casConnectionId); - return typeof latest?.refreshToken === "string" ? latest.refreshToken : null; - } - : null; - - const newCredentials = (await refreshWithRetry( - () => - runWithCasGuard( - casReread ? { expectedRefreshToken: attemptedRefreshToken, reread: casReread } : null, - () => runWithOnPersist(persistFn, () => executor.refreshCredentials(credentials, log)) - ), - 3, - log, - provider // Explicitly pass the provider to avoid universally tripping the "unknown" circuit breaker - )) as null | { - accessToken?: string; - copilotToken?: string; - }; - - if (newCredentials?.accessToken || newCredentials?.copilotToken) { - log?.info?.("TOKEN", `${provider?.toUpperCase()} | refreshed`); - - // Fall back to post-mutex mutation only for executors that don't route - // through getAccessToken (and therefore never fire onPersist). For - // executors that DO route through it (Codex, Claude, Gemini, etc.) the - // mutation already happened atomically inside the mutex. - if (!persistFnRan) { - Object.assign(credentials, newCredentials); - if (onCredentialsRefreshed) { - await onCredentialsRefreshed(newCredentials); - } - } - - // Retry with new credentials — model + extra headers follow translatedBody.model so they - // stay aligned if this block ever runs after a path that mutates body.model (e.g. fallback). - try { - const retryModelId = String(translatedBody.model || effectiveModel); - const retryBody = await prepareUpstreamBody({ - translatedBody, - modelToCall: retryModelId, - ...trustedEffortContext, - provider, - targetFormat, - credentials: getExecutionCredentials(), - log, - bypassDefaultToolLimit: isOpencodeClient, - isOpencodeClient, - rawBody: body, - clientRawRequest, - }); - assertManagedLeaseFence(getExecutionConnectionId(getExecutionCredentials())); - const retryResult = normalizeExecutorResult( - await runWithCapture(providerRequestCapture, () => - executor.execute({ - model: retryModelId, - body: retryBody, - stream: upstreamStream, - credentials: getExecutionCredentials(), - signal: streamController.signal, - log, - extendedContext, - upstreamExtraHeaders: buildUpstreamHeadersForExecute(retryModelId), - clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), - clientResponseFormat, - onCredentialsRefreshed, - skipUpstreamRetry: isCombo, - contextEditing: { enabled: contextEditingEnabled }, - correlationId, - }) - ) - ); - - if (retryResult.response.ok) { - providerResponse = retryResult.response; - providerUrl = retryResult.url; - providerHeaders = new Headers(retryResult.headers || {}); - finalBody = providerRequestCapture.body(retryResult.transformedBody); - reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); - updatePendingScope(pendingScope, { - providerRequest: finalBody, - providerUrl, - stage: "provider_response_started", - }); - upstreamErrorParsed = false; // Reset since new response is OK - } else { - providerResponse = retryResult.response; - upstreamErrorParsed = false; // Let it be parsed downstream - } - } catch (retryErr) { - const retryLeaseFenceCode = getManagedLeaseFenceErrorCode( - getUpstreamErrorIdentifier(retryErr) - ); - if (retryLeaseFenceCode) return managedLeaseFenceErrorResult(retryLeaseFenceCode); - // Refresh succeeded but the retry leg failed (network blip, AbortError, - // executor throw). Don't swallow — the operator-visible signal "the user - // saw 401 even though auth was actually fixed" is much more confusing - // than the original 401 alone. Surface at error level with sanitization. - log?.error?.( - "TOKEN", - `${provider?.toUpperCase()} | retry after refresh failed: ${sanitizeErrorMessage(retryErr)}` - ); - } - } else { - log?.warn?.("TOKEN", `${provider?.toUpperCase()} | refresh failed`); - if (isUnrecoverableRefreshError(newCredentials) && onCredentialsRefreshed) { - // Front 3 (reuse-race tolerance): before deactivating, re-read the DB. - // If a sibling/concurrent refresh already rotated this connection's - // refresh_token (common for Codex/OpenAI under one shared Auth0 client), - // the failure we saw was a stale-token reuse — the account is healthy - // with the newer token, so keep it active instead of killing it. - let alreadyRotated = false; - if (typeof connectionId === "string" && connectionId && attemptedRefreshToken) { - try { - const latest = await getProviderConnectionById(connectionId); - if (wasRefreshTokenRotated(attemptedRefreshToken, latest?.refreshToken)) { - alreadyRotated = true; - log?.warn?.( - "TOKEN", - `${provider.toUpperCase()} | refresh_token already rotated by a concurrent refresh — keeping connection active` - ); - } - } catch { - // DB read failed — fall through to the safe default (deactivate). - } - } - if (!alreadyRotated) { - await onCredentialsRefreshed({ testStatus: "expired", isActive: false }); - } - } - } - } - - // Check provider response - return error info for fallback handling - providerFailure: if (!providerResponse.ok) { - trackPendingRequest(model, provider, connectionId, false); - - let statusCode = providerResponse.status; - let message = ""; - let retryAfterMs: number | null = null; - let upstreamErrorCode: string | undefined; - let upstreamErrorType: string | undefined; - - if (upstreamErrorParsed) { - statusCode = parsedStatusCode; - message = parsedMessage; - retryAfterMs = parsedRetryAfterMs; - } else { - const details = await parseUpstreamError(providerResponse, provider); - statusCode = details.statusCode; - message = details.message; - retryAfterMs = details.retryAfterMs; - upstreamErrorBody = details.responseBody; - upstreamErrorCode = typeof details.errorCode === "string" ? details.errorCode : undefined; - upstreamErrorType = typeof details.errorType === "string" ? details.errorType : undefined; - } - - // Gateways like agentrouter misstate temporary quota exhaustion as 403/400, - // which downstream classification treats as AUTH_ERROR and clients like - // Claude Code treat as permanent. Restate to 429 (+ synthetic Retry-After) - // BEFORE any classification so both the fallback engine and the surfaced - // client status see a retryable error. Registry-scoped per provider. - const restatement = applyStatusRestatement({ - provider, - status: statusCode, - message, - body: upstreamErrorBody, - retryAfterMs, - }); - if (restatement.ruleId) { - statusCode = restatement.status; - retryAfterMs = restatement.retryAfterMs; - log?.info?.( - "STATUS_RESTATE", - `${provider} ${restatement.fromStatus}→${statusCode} (${restatement.ruleId})` - ); - } - - const signatureRecovery = pipelineRecovered - ? { attempted: false, succeeded: false, execution: null, error: null, recoveryBody: null } - : await recoverAnthropicThinkingSignature({ - provider, - statusCode, - message, - body: translatedBody, - execute: async (recoveryBody) => { - translatedBody = recoveryBody as typeof translatedBody; - return executeProviderRequest(currentModel, false); - }, - parseError: (response) => parseUpstreamError(response, provider), - }); - if (!pipelineRecovered && signatureRecovery.attempted && signatureRecovery.execution) { - providerResponse = signatureRecovery.execution.response; - if (signatureRecovery.succeeded) { - providerUrl = signatureRecovery.execution.url; - providerHeaders = signatureRecovery.execution.headers; - finalBody = providerRequestCapture.body(signatureRecovery.execution.transformedBody); - reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); - updatePendingScope(pendingScope, { - providerRequest: finalBody, - providerUrl, - stage: "provider_response_started", - }); - log?.info?.( - "THINKING_SIGNATURE", - `Recovered ${provider}/${currentModel} after one historical-thinking retry` - ); - } else if (signatureRecovery.error) { - statusCode = signatureRecovery.error.statusCode; - message = signatureRecovery.error.message; - retryAfterMs = signatureRecovery.error.retryAfterMs; - upstreamErrorBody = signatureRecovery.error.responseBody; - upstreamErrorCode = - typeof signatureRecovery.error.errorCode === "string" - ? signatureRecovery.error.errorCode - : undefined; - upstreamErrorType = - typeof signatureRecovery.error.errorType === "string" - ? signatureRecovery.error.errorType - : undefined; - } - } - - if (signatureRecovery.succeeded) break providerFailure; - - // #10281 — tiny-budget reasoning probes (e.g. Claude Code's `/model` check - // sends `max_tokens: 1`): the model burns the whole budget on thinking, and - // some upstreams (e.g. api.cline.bot for deepseek-v4-flash) answer the empty - // outcome with a 5xx ("empty response content") instead of a truncated 200. - // Answer such probes with a valid truncated response rather than relaying the - // upstream failure — which would also mark the connection unavailable and - // poison fallback/cooldown bookkeeping for a request that is only a probe. - if ( - !stream && - isTinyBudgetReasoningProbe({ model: currentModel, body: finalBody || translatedBody }) && - isEmptyContentUpstreamFailure(statusCode, message) - ) { - providerResponse = buildReasoningProbeTruncatedResponse({ - model: currentModel, - maxTokens: toPositiveInteger( - (finalBody || translatedBody)?.max_tokens ?? - (finalBody || translatedBody)?.max_completion_tokens - ), - requestId: skillRequestId, - }); - log?.warn?.( - "PROBE", - `Reasoning probe (max_tokens < ${REASONING_BUFFER_MIN_TRIGGER}) answered with truncated 200 — upstream reported "${message}"` - ); - break providerFailure; - } - - const errorConnectionId = getCurrentConnectionId() || connectionId; - await applyProviderFailureClassification({ - statusCode, - message, - headers: providerResponse.headers, - upstreamErrorBody, - retryAfterMs, - targetModel: currentModel, - }); - - appendRequestLog({ - model, - provider, - connectionId: errorConnectionId, - status: `FAILED ${statusCode}`, - }).catch(() => {}); - - const errMsg = formatProviderError(new Error(message), provider, model, statusCode); - const safeErrMsg = sanitizeErrorMessage(errMsg) || "Upstream provider error"; - const safeUpstreamErrorBody = sanitizeUpstreamDetails(upstreamErrorBody); - console.log(`${COLORS.red}[ERROR] ${safeErrMsg}${COLORS.reset}`); - - // Log Antigravity retry time if available - if (retryAfterMs && provider === "antigravity") { - const retrySeconds = Math.ceil(retryAfterMs / 1000); - log?.debug?.("RETRY", `Antigravity quota reset in ${retrySeconds}s (${retryAfterMs}ms)`); - } - - // Log error with full request body for debugging - reqLogger.logError(new Error(message), finalBody || translatedBody); - reqLogger.logProviderResponse( - providerResponse.status, - providerResponse.statusText, - providerResponse.headers, - safeUpstreamErrorBody - ); - - // Rate limiter updated in applyProviderFailureClassification - - // ── T5: Intra-family model fallback ────────────────────────────────────── - // Before returning a model-unavailable error upstream, try sibling models - // from the same family. This keeps the request alive on the same account - // instead of failing the entire combo. - const familyRecovery = onFailure({ - view: { kind: "pipeline" }, - status: statusCode, - message, - provider, - model: currentModel, - connectionId: String(getCurrentConnectionId() || connectionId || ""), - allowAccountRotation: !managedLease && comboStrategy !== "context-relay", - allowModelFallback: !pipelineRecovered, - isolateProbe: await shouldIsolateProbeFailures(), - nextModel: getNextFamilyFallback(currentModel, triedModels, provider), - canRefresh: false, - }); - if (!pipelineRecovered && familyRecovery.dispatch.action === "fallback-model") { - const nextModel = familyRecovery.dispatch.nextModel; - if (nextModel) { - triedModels.add(nextModel); - currentModel = nextModel; - translatedBody.model = nextModel; - log?.info?.( - "MODEL_FALLBACK", - `${model} unavailable (${statusCode}) → trying ${nextModel}` - ); - // Re-execute with the fallback model - try { - const fallbackResult = await executeProviderRequest(nextModel, false); - if (fallbackResult.response.ok) { - providerResponse = fallbackResult.response; - providerUrl = fallbackResult.url; - providerHeaders = fallbackResult.headers; - finalBody = providerRequestCapture.body(fallbackResult.transformedBody); - reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); - updatePendingScope(pendingScope, { - providerRequest: finalBody, - providerUrl, - stage: "provider_response_started", - }); - // Continue processing with the fallback response — skip error return - log?.info?.("MODEL_FALLBACK", `Serving ${nextModel} as fallback for ${model}`); - // Jump to streaming/non-streaming handling below - // We fall through by NOT returning here - } else { - // Fallback also failed — return original error - persistAttemptLogs({ - status: statusCode, - error: safeErrMsg, - providerRequest: finalBody || translatedBody, - providerResponse: safeUpstreamErrorBody, - clientResponse: buildErrorBody(statusCode, errMsg), - cacheSource: "upstream", - }); - persistFailureUsage(statusCode, "model_unavailable"); - return createErrorResult( - statusCode, - errMsg, - retryAfterMs, - upstreamErrorCode, - upstreamErrorType, - upstreamErrorBody, - { passthrough: sourceFormat === FORMATS.CLAUDE } - ); - } - } catch { - persistAttemptLogs({ - status: statusCode, - error: safeErrMsg, - providerRequest: finalBody || translatedBody, - providerResponse: safeUpstreamErrorBody, - clientResponse: buildErrorBody(statusCode, errMsg), - cacheSource: "upstream", - }); - persistFailureUsage(statusCode, "model_unavailable"); - return createErrorResult( - statusCode, - errMsg, - retryAfterMs, - upstreamErrorCode, - upstreamErrorType, - upstreamErrorBody, - { passthrough: sourceFormat === FORMATS.CLAUDE } - ); - } - } else { - persistAttemptLogs({ - status: statusCode, - error: safeErrMsg, - providerRequest: finalBody || translatedBody, - providerResponse: safeUpstreamErrorBody, - clientResponse: buildErrorBody(statusCode, errMsg), - cacheSource: "upstream", - }); - persistFailureUsage(statusCode, "model_unavailable"); - return createErrorResult( - statusCode, - errMsg, - retryAfterMs, - upstreamErrorCode, - upstreamErrorType, - upstreamErrorBody, - { passthrough: sourceFormat === FORMATS.CLAUDE } - ); - } - } else if (isContextOverflowError(statusCode, message)) { - const familyCandidates = getModelFamily(currentModel, provider).filter( - (m) => m !== currentModel && !triedModels.has(m) - ); - const nextModel = - findLargerContextModel(currentModel, familyCandidates, provider) ?? - getNextFamilyFallback(currentModel, triedModels, provider); - if (nextModel) { - triedModels.add(nextModel); - currentModel = nextModel; - translatedBody.model = nextModel; - log?.info?.( - "CONTEXT_OVERFLOW_FALLBACK", - `${model} context overflow → trying ${nextModel}` - ); - try { - const fallbackResult = await executeProviderRequest(nextModel, false); - if (fallbackResult.response.ok) { - providerResponse = fallbackResult.response; - providerUrl = fallbackResult.url; - providerHeaders = fallbackResult.headers; - finalBody = providerRequestCapture.body(fallbackResult.transformedBody); - reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); - updatePendingScope(pendingScope, { - providerRequest: finalBody, - providerUrl, - stage: "provider_response_started", - }); - log?.info?.( - "CONTEXT_OVERFLOW_FALLBACK", - `Serving ${nextModel} as fallback for ${model}` - ); - } else { - persistAttemptLogs({ - status: statusCode, - error: safeErrMsg, - providerRequest: finalBody || translatedBody, - providerResponse: safeUpstreamErrorBody, - clientResponse: buildErrorBody(statusCode, errMsg), - cacheSource: "upstream", - }); - persistFailureUsage(statusCode, "context_overflow"); - return createErrorResult( - statusCode, - errMsg, - retryAfterMs, - upstreamErrorCode, - upstreamErrorType, - upstreamErrorBody, - { passthrough: sourceFormat === FORMATS.CLAUDE } - ); - } - } catch { - persistAttemptLogs({ - status: statusCode, - error: safeErrMsg, - providerRequest: finalBody || translatedBody, - providerResponse: safeUpstreamErrorBody, - clientResponse: buildErrorBody(statusCode, errMsg), - cacheSource: "upstream", - }); - persistFailureUsage(statusCode, "context_overflow"); - return createErrorResult( - statusCode, - errMsg, - retryAfterMs, - upstreamErrorCode, - upstreamErrorType, - upstreamErrorBody, - { passthrough: sourceFormat === FORMATS.CLAUDE } - ); - } - } else { - persistAttemptLogs({ - status: statusCode, - error: safeErrMsg, - providerRequest: finalBody || translatedBody, - providerResponse: safeUpstreamErrorBody, - clientResponse: buildErrorBody(statusCode, errMsg), - cacheSource: "upstream", - }); - persistFailureUsage(statusCode, "context_overflow"); - return createErrorResult( - statusCode, - errMsg, - retryAfterMs, - upstreamErrorCode, - upstreamErrorType, - upstreamErrorBody, - { passthrough: sourceFormat === FORMATS.CLAUDE } - ); - } - } else { - persistAttemptLogs({ - status: statusCode, - error: safeErrMsg, - providerRequest: finalBody || translatedBody, - providerResponse: safeUpstreamErrorBody, - clientResponse: buildErrorBody(statusCode, errMsg), - cacheSource: "upstream", - }); - persistFailureUsage(statusCode, `upstream_${statusCode}`); - - // Emergency budget fallback is orchestrated exclusively by the routing layer - // (src/sse/handlers/chat.ts), which resolves credentials FOR the emergency - // provider through account selection. The executor-level hop that used to - // live here re-sent the FAILING provider's credentials to the emergency - // provider's endpoint (e.g. the OpenAI API key to integrate.api.nvidia.com) - // — a cross-provider credential leak that also never succeeded upstream. - return createErrorResult( - statusCode, - errMsg, - retryAfterMs, - upstreamErrorCode, - upstreamErrorType, - upstreamErrorBody, - { passthrough: sourceFormat === FORMATS.CLAUDE } - ); - } - // ── End T5 ─────────────────────────────────────────────────────────────── + const streamingOutcome = await runStreamingLeg({ + apiKeyInfo, + applyProviderFailureClassification, + assertManagedLeaseFence, + body, + buildErrorBody, + buildUpstreamHeadersForExecute, + claudePromptCacheLogMeta, + clientRawRequest, + clientResponseFormat, + comboStrategy, + connectionId, + contextEditingEnabled, + correlationId, + credentials, + currentModel, + effectiveModel, + effectiveServiceTier, + executeProviderRequest, + executeRefreshCredentials, + executor, + extendedContext, + finalBody, + getCurrentConnectionId, + getExecutionCredentials, + getManagedLeaseFenceErrorCode, + handleCredentialsRefreshed, + isCombo, + isOpencodeClient, + log, + managedLease, + managedLeaseFenceErrorResult, + model, + onCredentialsRefreshed, + pendingScope, + persistAttemptLogs, + persistFailureUsage, + pipelineRecovered, + provider, + providerHeaders, + providerRequestCapture, + providerResponse, + providerUrl, + reqLogger, + resolveEffectiveServiceTier, + sessionAffinityKey, + skillRequestId, + sourceFormat, + stream, + streamController, + targetFormat, + translatedBody, + triedModels, + trustedEffortContext, + upstreamStream, + userAgent, + }); + claudePromptCacheLogMeta = streamingOutcome.carry.claudePromptCacheLogMeta; + effectiveServiceTier = streamingOutcome.carry.effectiveServiceTier; + finalBody = streamingOutcome.carry.finalBody; + providerHeaders = streamingOutcome.carry.providerHeaders; + providerResponse = streamingOutcome.carry.providerResponse; + translatedBody = streamingOutcome.carry.translatedBody; + if (streamingOutcome.kind === "returned") { + return streamingOutcome.value; } } diff --git a/open-sse/handlers/chatCore/streamingLeg.ts b/open-sse/handlers/chatCore/streamingLeg.ts new file mode 100644 index 000000000000..20b24055a94a --- /dev/null +++ b/open-sse/handlers/chatCore/streamingLeg.ts @@ -0,0 +1,1286 @@ +import { projectFailureUsageErrorCode, type FailureUsageAggregate } from "./failureUsage.ts"; +import { buildClaudePromptCacheLogMeta } from "./executorHelpers.ts"; +import { recoverAnthropicThinkingSignature } from "./thinkingSignatureRecovery.ts"; +import { runProviderExecutionPipeline } from "./providerExecutionPipeline.ts"; +import { onFailure, onStreamThrow } from "./recoveryPolicy.ts"; +import { markCodexScopeRateLimited } from "./codexFailover.ts"; +import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity"; +import { normalizeHeaders } from "../../utils/headers.ts"; +import { FORMATS } from "../../translator/formats.ts"; +import { COLORS } from "../../utils/stream.ts"; +import { + refreshWithRetry, + isUnrecoverableRefreshError, + runWithOnPersist, + runWithCasGuard, +} from "../../services/tokenRefresh.ts"; +import { runWithCapture } from "../../utils/providerRequestLogging.ts"; +import type { PersistAttemptLogsArgs } from "./attemptLogging.ts"; +import { + REASONING_BUFFER_MIN_TRIGGER, + buildReasoningProbeTruncatedResponse, + isEmptyContentUpstreamFailure, + isTinyBudgetReasoningProbe, + toPositiveInteger, +} from "../../services/reasoningTokenBuffer.ts"; +import { + createErrorResult, + parseUpstreamError, + formatProviderError, + projectPublicErrorIdentifier, + sanitizeErrorMessage, + sanitizeUpstreamDetails, +} from "../../utils/error.ts"; +import { HTTP_STATUS, ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE } from "../../config/constants.ts"; +import { applyStatusRestatement } from "../../config/upstreamStatusRestatement.ts"; +import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; +import { wasRefreshTokenRotated } from "@omniroute/open-sse/services/refreshSerializer.ts"; +import { + createSafeAbortError, + createStreamingErrorResult, + isSemaphoreCapacityError, + getSafeErrorMetadata, + getUpstreamErrorIdentifier, +} from "./streamErrorResult.ts"; +import { buildExecutorClientHeaders } from "./executorClientHeaders.ts"; +import { getExecutionConnectionId } from "./executionCredentials.ts"; +import { prepareUpstreamBody } from "./upstreamBody.ts"; +import { logAuditEvent } from "@/lib/compliance"; +import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb"; +import { updatePendingScope } from "@/lib/usage/pendingRequestScope"; +import { normalizeExecutorResult } from "./upstreamTimeouts.ts"; +import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; +import { updateFromHeaders, updateFromResponseBody } from "../../services/rateLimitManager.ts"; +import * as localLimiterErrors from "../../services/rateLimitManager/errors.ts"; +import { markBlocked as markAccountSemaphoreBlocked } from "../../services/accountSemaphore.ts"; +import { lockModel, recordCoreOwnedAntigravityQuotaState } from "../../services/accountFallback.ts"; +import { + getNextFamilyFallback, + isContextOverflowError, + findLargerContextModel, + getModelFamily, +} from "../../services/modelFamilyFallback.ts"; +import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; +import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; + +type LoggerLike = + | { + warn?: (...args: unknown[]) => void; + debug?: (...args: unknown[]) => void; + info?: (...args: unknown[]) => void; + error?: (...args: unknown[]) => void; + } + | null + | undefined; + +/** + * Mirrors the unexported `ErrorResponseBody` from `open-sse/utils/error.ts:17`. + * That interface is module-private; widening its export surface is not this + * commit's business, so the shape is restated here and nowhere else. + */ +interface ErrorResponseBody { + error: { + message: string; + type?: string; + code?: string; + reason?: string; + }; + upstream_details?: Record | null; +} + +/** + * The streaming half of `handleChatCore`, lifted verbatim out of the barrel. + * + * Unlike the non-streaming leg, this one does not own every exit: six values it + * assigns are read again by the streaming response section that follows the + * call. They come back through {@link StreamingLegOutcome.carry} so the data + * flow stays visible instead of travelling through a shared mutable closure. + */ +export interface StreamingLegDeps { + body: Record; + model: string | null | undefined; + apiKeyInfo: ({ id?: string | null; name?: string } & Record) | null; + applyProviderFailureClassification: ({ + statusCode, + message, + headers, + upstreamErrorBody, + retryAfterMs, + targetModel, + }: { + statusCode: number; + message: string; + headers?: Headers | null; + upstreamErrorBody?: unknown; + retryAfterMs?: number | null; + targetModel: string; + }) => Promise; + assertManagedLeaseFence: (attemptConnectionId: string | null | undefined) => void; + buildErrorBody: ( + statusCode: number, + message: string, + upstreamDetails?: unknown, + classification?: import("../../utils/error.ts").ErrorBodyClassification + ) => ErrorResponseBody; + buildUpstreamHeadersForExecute: (modelToCall: string) => Record; + claudePromptCacheLogMeta: Record | null | undefined; + clientRawRequest: + | { + headers?: Record; + signal?: AbortSignal; + } + | null + | undefined; + clientResponseFormat: string; + comboStrategy: unknown; + connectionId: string | null | undefined; + contextEditingEnabled: boolean; + correlationId: unknown; + credentials: ({ connectionId?: string | null } & Record) | null | undefined; + currentModel: string; + effectiveModel: string; + effectiveServiceTier: import("./serviceTier.js").EffectiveServiceTier; + executeProviderRequest: ( + modelToCall?: string, + allowDedup?: boolean + ) => Promise; + executeRefreshCredentials: ( + currentCreds: Record + ) => Promise | null>; + executor: { + execute: (args: Record) => Promise; + refreshCredentials?: (credentials: unknown, log?: LoggerLike | null) => Promise; + }; + extendedContext: unknown; + // The barrel declares `let finalBody;` -- implicit any under strict:false -- so it + // can hold either a translated request Record or a ChatCoreExecutorResult. Naming + // either one here would narrow what upstream deliberately left open. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + finalBody: any; + getCurrentConnectionId: () => string | null | undefined; + getExecutionCredentials: () => Record & { + providerSpecificData: Record; + }; + getManagedLeaseFenceErrorCode: (code: string | undefined) => string | undefined; + handleCredentialsRefreshed: (refreshed: Record) => Promise; + isCombo: boolean; + isOpencodeClient: boolean; + log: LoggerLike | null | undefined; + managedLease: unknown; + managedLeaseFenceErrorResult: (code: string) => { + errorType: string; + errorCode: string; + success: false; + status: number; + error: string; + rawMessage: string; + response: Response; + retryAfterMs?: number; + }; + onCredentialsRefreshed: (refreshed: unknown) => void | Promise; + pendingScope: { + id: string; + model: string | null | undefined; + provider: string | null | undefined; + connectionId: string | null | undefined; + }; + persistAttemptLogs: (args: PersistAttemptLogsArgs) => void; + persistFailureUsage: ( + statusCode: number, + errorCode?: string | null, + aggregate?: FailureUsageAggregate | null + ) => void; + pipelineRecovered: boolean; + provider: string | null | undefined; + providerHeaders: Record | Headers | null | undefined; + providerRequestCapture: import("../../utils/providerRequestLogging.ts").Capture; + providerResponse: (Response & { body?: unknown }) | null | undefined; + providerUrl: string; + reqLogger: { + sessionPath: null; + logClientRawRequest: ( + endpoint: unknown, + body: unknown, + headers?: + Headers | Record | { entries?: () => IterableIterator<[string, string]> }, + effectiveInput?: unknown + ) => void; + logRouteDecision: (decision: unknown) => void; + logOpenAIRequest: (body: unknown) => void; + logTargetRequest: ( + url: unknown, + headers: + Headers | Record | { entries?: () => IterableIterator<[string, string]> }, + body: unknown + ) => void; + logProviderResponse: ( + status: unknown, + statusText: unknown, + headers: + Headers | Record | { entries?: () => IterableIterator<[string, string]> }, + body: unknown + ) => void; + appendProviderChunk: (chunk: string) => void; + appendOpenAIChunk: (chunk: string) => void; + logConvertedResponse: (body: unknown) => void; + appendConvertedChunk: (chunk: string) => void; + logError: (error: unknown, requestBody?: unknown) => void; + logToolLoopReceipt: (receipt: unknown) => void; + getPipelinePayloads: () => + import("../../utils/requestLogger.js").RequestPipelinePayloads | null; + }; + resolveEffectiveServiceTier: ( + requestBody?: unknown + ) => import("./serviceTier.js").EffectiveServiceTier; + sessionAffinityKey: string | null | undefined; + skillRequestId: `${string}-${string}-${string}-${string}-${string}`; + sourceFormat: string; + stream: boolean; + streamController: { + signal: AbortSignal; + startTime: number; + isConnected: () => boolean; + handleDisconnect: (reason?: string) => void; + handleComplete: () => void; + markClientTerminalSeen: () => void; + markCompletedToolHandoffSeen: () => void; + registerCompletedToolHandoffDrain: (drain: () => void) => void; + shouldDeferCompletedToolHandoff: () => boolean; + handleError: (error: unknown) => void; + abort: () => void; + clientResponseFormat: string; + clientDisconnectGracePeriodMs: number; + }; + targetFormat: string; + translatedBody: Record; + triedModels: Set; + trustedEffortContext: Readonly<{ + originModel: string | null | undefined; + resolvedThinkingEffort: string | null | undefined; + defaultThinkingEffort: string | null | undefined; + }>; + upstreamStream: boolean; + userAgent: string | null | undefined; +} + +/** Values the barrel must write back before continuing down the streaming path. */ +export interface StreamingLegCarry { + claudePromptCacheLogMeta: Record | null | undefined; + effectiveServiceTier: import("./serviceTier.js").EffectiveServiceTier; + // The barrel declares `let finalBody;` -- implicit any under strict:false -- so it + // can hold either a translated request Record or a ChatCoreExecutorResult. Naming + // either one here would narrow what upstream deliberately left open. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + finalBody: any; + providerHeaders: Record | Headers | null | undefined; + providerResponse: (Response & { body?: unknown }) | null | undefined; + translatedBody: Record; +} + +export type StreamingLegOutcome = + | { kind: "returned"; value: unknown; carry: StreamingLegCarry } + | { kind: "fellThrough"; carry: StreamingLegCarry }; + +export async function runStreamingLeg(deps: StreamingLegDeps): Promise { + let { + body, + model, + apiKeyInfo, + applyProviderFailureClassification, + assertManagedLeaseFence, + buildErrorBody, + buildUpstreamHeadersForExecute, + claudePromptCacheLogMeta, + clientRawRequest, + clientResponseFormat, + comboStrategy, + connectionId, + contextEditingEnabled, + correlationId, + credentials, + currentModel, + effectiveModel, + effectiveServiceTier, + executeProviderRequest, + executeRefreshCredentials, + executor, + extendedContext, + finalBody, + getCurrentConnectionId, + getExecutionCredentials, + getManagedLeaseFenceErrorCode, + handleCredentialsRefreshed, + isCombo, + isOpencodeClient, + log, + managedLease, + managedLeaseFenceErrorResult, + onCredentialsRefreshed, + pendingScope, + persistAttemptLogs, + persistFailureUsage, + pipelineRecovered, + provider, + providerHeaders, + providerRequestCapture, + providerResponse, + providerUrl, + reqLogger, + resolveEffectiveServiceTier, + sessionAffinityKey, + skillRequestId, + sourceFormat, + stream, + streamController, + targetFormat, + translatedBody, + triedModels, + trustedEffortContext, + upstreamStream, + userAgent, + } = deps; + + const carry = (): StreamingLegCarry => ({ + claudePromptCacheLogMeta, + effectiveServiceTier, + finalBody, + providerHeaders, + providerResponse, + translatedBody, + }); + + try { + const pipelineOutcome = await runProviderExecutionPipeline({ + policy: { + allowAccountRotation: !managedLease && comboStrategy !== "context-relay", + allowModelFallback: true, + expectedConnectionId: managedLease + ? String(getCurrentConnectionId() || connectionId || "") || undefined + : undefined, + }, + target: { + provider, + requestedModel: effectiveModel, + sourceFormat, + targetFormat, + stream, + }, + connection: { + initialConnectionId: String(getCurrentConnectionId() || connectionId || ""), + getCurrentConnectionId: () => getCurrentConnectionId() || undefined, + getCredentials: () => (credentials || {}) as Record, + replaceCredentials: (next) => { + Object.assign(credentials, next); + }, + onCredentialsRefreshed: handleCredentialsRefreshed, + refreshCredentials: executeRefreshCredentials, + assertManagedLeaseFence: (id) => { + assertManagedLeaseFence(id); + }, + getProviderCredentials, + }, + wire: { + body: translatedBody as Record, + currentModel, + triedModels, + setBodyAndModel: (body, model) => { + translatedBody = body as typeof translatedBody; + currentModel = model; + triedModels.add(model); + }, + }, + state: { + updatePendingStage: (stage, data) => { + updatePendingScope(pendingScope, { stage, ...(data || {}) }); + }, + recordRateLimitHeaders: updateFromHeaders, + recordRateLimitBody: updateFromResponseBody, + writeTerminalStatus, + persistConnectionPatch: updateProviderConnection, + setConnectionRateLimitedUntil: async (id, untilMs) => { + const { setConnectionRateLimitUntil } = await import("@/lib/db/providers"); + setConnectionRateLimitUntil(id, untilMs); + }, + lockModel, + recordAntigravityQuotaState: recordCoreOwnedAntigravityQuotaState, + markAccountSemaphoreBlocked: (key) => { + markAccountSemaphoreBlocked(key, Date.now() + 60_000); + }, + isolateProbeFailures: () => shouldIsolateProbeFailures(), + onCodexScopeRateLimited: async (params) => { + await markCodexScopeRateLimited({ + failedConnectionId: params.failedConnectionId, + model: params.model, + rateLimitedUntil: params.rateLimitedUntil, + credentials: (params.credentials || credentials) as { + connectionId?: string | null; + providerSpecificData?: unknown; + }, + }); + }, + onClearSessionAffinity: () => { + const key = + sessionAffinityKey || + extractSessionAffinityKey(body, clientRawRequest?.headers) || + null; + if (!key) return; + try { + deleteSessionAccountAffinity(key, "codex"); + } catch { + // best-effort + } + }, + onAuditAccountRotation: (params) => { + logAuditEvent({ + action: params.action, + actor: apiKeyInfo?.name || "system", + target: params.newConnectionId, + details: { + failed_connection_id: params.failedConnectionId, + new_connection_id: params.newConnectionId, + attempt: params.attempt, + retry_after_ms: params.retryAfterMs, + }, + }); + }, + }, + sendProviderAttempt: (modelToCall, allowDedup) => + executeProviderRequest(modelToCall, allowDedup), + }); + + pipelineRecovered = true; + currentModel = pipelineOutcome.model; + if (pipelineOutcome.kind === "error") { + providerResponse = pipelineOutcome.result.response; + providerUrl = ""; + providerHeaders = normalizeHeaders(pipelineOutcome.result.response.headers); + finalBody = translatedBody; + } else { + const result = { + response: pipelineOutcome.response, + url: pipelineOutcome.url, + headers: pipelineOutcome.headers, + transformedBody: pipelineOutcome.transformedBody, + }; + providerResponse = result.response; + providerUrl = result.url; + providerHeaders = result.headers; + finalBody = providerRequestCapture.body(result.transformedBody); + } + const responseConnectionId = getCurrentConnectionId(); + effectiveServiceTier = resolveEffectiveServiceTier(finalBody); + claudePromptCacheLogMeta = buildClaudePromptCacheLogMeta( + targetFormat, + finalBody, + providerHeaders, + clientRawRequest?.headers + ); + + // Log target request (final request to provider) + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + updatePendingScope(pendingScope, { + providerRequest: finalBody, + providerUrl, + stage: "provider_response_started", + }); + // Update rate limiter from response headers (learn limits dynamically) + updateFromHeaders( + provider, + responseConnectionId, + providerResponse.headers, + providerResponse.status, + model + ); + + // Store rate-limit headers for quota saturation signals + try { + const { storeRateLimitHeaders } = await import("@/lib/quota/saturationSignals"); + storeRateLimitHeaders( + responseConnectionId, + provider, + providerResponse.headers as unknown as Record + ); + } catch { + // fail-open: saturation signal is best-effort + } + } catch (error) { + onStreamThrow(); + trackPendingRequest(model, provider, connectionId, false); + const errorMetadata = getSafeErrorMetadata(error); + const managedLeaseFenceCode = getManagedLeaseFenceErrorCode(errorMetadata.code); + if (managedLeaseFenceCode) + return { + kind: "returned" as const, + value: managedLeaseFenceErrorResult(managedLeaseFenceCode), + carry: carry(), + }; + // isSemaphoreCapacityError already reads the code through getSafeErrorMetadata, + // so a hostile rejection cannot escape this classification. + if (isSemaphoreCapacityError(error)) { + const semaphoreCode = errorMetadata.code as string; + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${semaphoreCode}`, + }).catch(() => {}); + const failureMessage = sanitizeErrorMessage(errorMetadata.message) || "Semaphore timeout"; + persistAttemptLogs({ + status: HTTP_STATUS.RATE_LIMITED, + error: failureMessage, + providerRequest: finalBody || translatedBody, + clientResponse: buildErrorBody(HTTP_STATUS.RATE_LIMITED, failureMessage), + claudeCacheMeta: claudePromptCacheLogMeta, + cacheSource: "upstream", + }); + persistFailureUsage(HTTP_STATUS.RATE_LIMITED, semaphoreCode); + const result = stream + ? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, semaphoreCode) + : createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage); + return { + kind: "returned" as const, + value: { + ...result, + errorType: "account_semaphore_capacity", + errorCode: semaphoreCode, + }, + carry: carry(), + }; + } + // abort(reason) can reject with a raw string lacking `name`/`status`; classify + // it through isLocalStreamLifecycleError so it maps to 499 rather than the + // 502 provider-failure default. + let isRequestAborted = errorMetadata.name === "AbortError"; + if (!isRequestAborted) { + try { + isRequestAborted = isLocalStreamLifecycleError(error); + } catch { + // A hostile Proxy must not escape the provider-error boundary during classification. + } + } + // #8376: proxyFetch tags unreachable transport failures so they remain + // distinguishable from ordinary provider 5xx responses. + const isProxyUnreachableFailure = + !isRequestAborted && errorMetadata.errorCode === "proxy_unreachable"; + const errorCode = errorMetadata.code; + const localRateLimitFailure = localLimiterErrors.getClientSafeLocalRateLimitError(error); + const failureStatus = isRequestAborted + ? 499 + : isProxyUnreachableFailure + ? HTTP_STATUS.BAD_GATEWAY + : localRateLimitFailure + ? localRateLimitFailure.status + : errorMetadata.name === "TimeoutError" || errorMetadata.name === "BodyTimeoutError" + ? HTTP_STATUS.GATEWAY_TIMEOUT + : errorMetadata.status + ? errorMetadata.status + : HTTP_STATUS.BAD_GATEWAY; + const failureMessage = isRequestAborted + ? "Request aborted" + : (() => { + try { + return formatProviderError( + localRateLimitFailure ?? error, + provider, + model, + failureStatus + ); + } catch { + // Formatting is diagnostic only; hostile rejection metadata falls back safely. + return errorMetadata.message || "Upstream provider error"; + } + })(); + const safeFailureMessage = sanitizeErrorMessage(failureMessage) || "Upstream provider error"; + const upstreamErrorCode = + localRateLimitFailure?.code ?? (isProxyUnreachableFailure ? "proxy_unreachable" : errorCode); + // Tag our own deadline timeouts (fetch-start TimeoutError / body BodyTimeoutError, + // both surfaced as a 504) as "upstream_timeout" so the cooldown layer can tell a + // slow-but-not-failed request apart from a real provider 5xx. (Antigravity already + // tags its pre-response timeout via the code below.) + const isOwnDeadlineTimeout = + failureStatus === HTTP_STATUS.GATEWAY_TIMEOUT && + (errorMetadata.name === "TimeoutError" || errorMetadata.name === "BodyTimeoutError"); + const upstreamErrorType = + upstreamErrorCode === ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE || isOwnDeadlineTimeout + ? "upstream_timeout" + : failureStatus === 401 + ? "authentication_error" + : undefined; + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${failureStatus}`, + }).catch(() => {}); + persistAttemptLogs({ + status: failureStatus, + error: safeFailureMessage, + providerRequest: finalBody || translatedBody, + // On a client-abort (AbortError), the client already disconnected before + // we ever got here — this body is what we WOULD have sent, not what was + // actually delivered. Logging it as `clientResponse` is misleading (the + // dashboard reads that field as "what the client received"), so omit it + // for this case; `error` above already records the failure reason. + clientResponse: + errorMetadata.name === "AbortError" + ? undefined + : buildErrorBody(failureStatus, failureMessage), + claudeCacheMeta: claudePromptCacheLogMeta, + cacheSource: "upstream", + }); + if (isRequestAborted) { + streamController.handleError(createSafeAbortError()); + return { + kind: "returned" as const, + value: createErrorResult(499, "Request aborted"), + carry: carry(), + }; + } + const persistentErrorCode = projectFailureUsageErrorCode({ + statusCode: failureStatus, + message: failureMessage, + errorCode: projectPublicErrorIdentifier( + upstreamErrorCode || errorMetadata.name, + "upstream_error" + ), + errorType: upstreamErrorType, + }); + persistFailureUsage(failureStatus, persistentErrorCode); + console.log(`${COLORS.red}[ERROR] ${safeFailureMessage}${COLORS.reset}`); + if (stream && upstreamErrorCode) { + const result = createStreamingErrorResult( + failureStatus, + failureMessage, + upstreamErrorCode, + upstreamErrorType + ); + localLimiterErrors.markTrustedLocalRateLimitResponse(result.response, error); + return { + kind: "returned" as const, + value: { + ...result, + errorType: upstreamErrorType, + errorCode: upstreamErrorCode, + }, + carry: carry(), + }; + } + const result = createErrorResult( + failureStatus, + failureMessage, + null, + upstreamErrorCode, + upstreamErrorType + ); + localLimiterErrors.markTrustedLocalRateLimitResponse(result.response, error); + return { kind: "returned" as const, value: result, carry: carry() }; + } + let upstreamErrorParsed = false; + let parsedStatusCode = providerResponse.status; + let parsedMessage = ""; + let parsedRetryAfterMs: number | null = null; + let upstreamErrorBody: unknown = null; + + // Track whether stream_options was present and stripped — if so, 401/403 after + // that may be from the modification rather than a genuine auth failure, so we + // skip the credential refresh attempt in that case. + const hadStreamOptions = + targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody; + if (hadStreamOptions) { + delete translatedBody.stream_options; + } + + // Handle 401/403 - try token refresh using executor + // T-PROBE: probe-origin failures never attempt the refresh — a probe must + // not consume a rotating refresh token nor persist an "expired" + // deactivation on refresh failure (#9817). The 401/403 then flows into + // the normal providerFailure classification (record-only in probe mode). + if ( + (providerResponse.status === HTTP_STATUS.UNAUTHORIZED || + providerResponse.status === HTTP_STATUS.FORBIDDEN) && + !hadStreamOptions && // Skip refresh if failure may be from stream_options removal, not auth + !(await shouldIsolateProbeFailures()) + ) { + // Fix A: wrap refreshCredentials in runWithOnPersist so the persist callback + // executes INSIDE the per-connection mutex held by getAccessToken. This makes + // [network refresh + DB write + outer-state mutation] one atomic step and + // prevents concurrent requests from reading a stale refreshToken before the + // DB has been updated (refresh_token_reused on Codex/OpenAI). + // + // Not every executor routes refresh through getAccessToken (e.g. github.ts + // calls refreshCopilotToken directly). When the persistFn doesn't fire from + // inside getAccessToken, we still need to do the credentials mutation + user + // callback after refreshCredentials returns. The `persistFnRan` flag tracks + // which path executed so we don't double-fire (race-prone) or skip (regression). + // Front 3: remember the refresh_token we are about to present so that, if the + // refresh fails as unrecoverable, we can tell a genuine death apart from a + // stale-token reuse that a concurrent/sibling refresh already rotated past. + const attemptedRefreshToken = + typeof credentials?.refreshToken === "string" ? credentials.refreshToken : null; + let persistFnRan = false; + const persistFn = onCredentialsRefreshed + ? async (refreshResult: Record) => { + persistFnRan = true; + // Mutate the shared credentials object so subsequent executor calls + // in this request see the new tokens. Runs INSIDE the mutex. + Object.assign(credentials, refreshResult); + await onCredentialsRefreshed(refreshResult); + } + : undefined; + + // #4038: build a compare-and-swap reread so getAccessToken can skip the persist if a + // concurrent writer (sibling request / HealthCheck / replica) already rotated this + // connection's refresh_token past the one we presented — overwriting would revert it + // and revoke the token family. No connectionId ⇒ no guard (behavior unchanged). + const casConnectionId = + typeof credentials?.connectionId === "string" ? credentials.connectionId.trim() : ""; + const casReread = casConnectionId + ? async () => { + const latest = await getProviderConnectionById(casConnectionId); + return typeof latest?.refreshToken === "string" ? latest.refreshToken : null; + } + : null; + + const newCredentials = (await refreshWithRetry( + () => + runWithCasGuard( + casReread ? { expectedRefreshToken: attemptedRefreshToken, reread: casReread } : null, + () => runWithOnPersist(persistFn, () => executor.refreshCredentials(credentials, log)) + ), + 3, + log, + provider // Explicitly pass the provider to avoid universally tripping the "unknown" circuit breaker + )) as null | { + accessToken?: string; + copilotToken?: string; + }; + + if (newCredentials?.accessToken || newCredentials?.copilotToken) { + log?.info?.("TOKEN", `${provider?.toUpperCase()} | refreshed`); + + // Fall back to post-mutex mutation only for executors that don't route + // through getAccessToken (and therefore never fire onPersist). For + // executors that DO route through it (Codex, Claude, Gemini, etc.) the + // mutation already happened atomically inside the mutex. + if (!persistFnRan) { + Object.assign(credentials, newCredentials); + if (onCredentialsRefreshed) { + await onCredentialsRefreshed(newCredentials); + } + } + + // Retry with new credentials — model + extra headers follow translatedBody.model so they + // stay aligned if this block ever runs after a path that mutates body.model (e.g. fallback). + try { + const retryModelId = String(translatedBody.model || effectiveModel); + const retryBody = await prepareUpstreamBody({ + translatedBody, + modelToCall: retryModelId, + ...trustedEffortContext, + provider, + targetFormat, + credentials: getExecutionCredentials(), + log, + bypassDefaultToolLimit: isOpencodeClient, + isOpencodeClient, + rawBody: body, + clientRawRequest, + }); + assertManagedLeaseFence(getExecutionConnectionId(getExecutionCredentials())); + const retryResult = normalizeExecutorResult( + await runWithCapture(providerRequestCapture, () => + executor.execute({ + model: retryModelId, + body: retryBody, + stream: upstreamStream, + credentials: getExecutionCredentials(), + signal: streamController.signal, + log, + extendedContext, + upstreamExtraHeaders: buildUpstreamHeadersForExecute(retryModelId), + clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), + clientResponseFormat, + onCredentialsRefreshed, + skipUpstreamRetry: isCombo, + contextEditing: { enabled: contextEditingEnabled }, + correlationId, + }) + ) + ); + + if (retryResult.response.ok) { + providerResponse = retryResult.response; + providerUrl = retryResult.url; + providerHeaders = new Headers(retryResult.headers || {}); + finalBody = providerRequestCapture.body(retryResult.transformedBody); + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + updatePendingScope(pendingScope, { + providerRequest: finalBody, + providerUrl, + stage: "provider_response_started", + }); + upstreamErrorParsed = false; // Reset since new response is OK + } else { + providerResponse = retryResult.response; + upstreamErrorParsed = false; // Let it be parsed downstream + } + } catch (retryErr) { + const retryLeaseFenceCode = getManagedLeaseFenceErrorCode( + getUpstreamErrorIdentifier(retryErr) + ); + if (retryLeaseFenceCode) + return { + kind: "returned" as const, + value: managedLeaseFenceErrorResult(retryLeaseFenceCode), + carry: carry(), + }; + // Refresh succeeded but the retry leg failed (network blip, AbortError, + // executor throw). Don't swallow — the operator-visible signal "the user + // saw 401 even though auth was actually fixed" is much more confusing + // than the original 401 alone. Surface at error level with sanitization. + log?.error?.( + "TOKEN", + `${provider?.toUpperCase()} | retry after refresh failed: ${sanitizeErrorMessage(retryErr)}` + ); + } + } else { + log?.warn?.("TOKEN", `${provider?.toUpperCase()} | refresh failed`); + if (isUnrecoverableRefreshError(newCredentials) && onCredentialsRefreshed) { + // Front 3 (reuse-race tolerance): before deactivating, re-read the DB. + // If a sibling/concurrent refresh already rotated this connection's + // refresh_token (common for Codex/OpenAI under one shared Auth0 client), + // the failure we saw was a stale-token reuse — the account is healthy + // with the newer token, so keep it active instead of killing it. + let alreadyRotated = false; + if (typeof connectionId === "string" && connectionId && attemptedRefreshToken) { + try { + const latest = await getProviderConnectionById(connectionId); + if (wasRefreshTokenRotated(attemptedRefreshToken, latest?.refreshToken)) { + alreadyRotated = true; + log?.warn?.( + "TOKEN", + `${provider.toUpperCase()} | refresh_token already rotated by a concurrent refresh — keeping connection active` + ); + } + } catch { + // DB read failed — fall through to the safe default (deactivate). + } + } + if (!alreadyRotated) { + await onCredentialsRefreshed({ testStatus: "expired", isActive: false }); + } + } + } + } + + // Check provider response - return error info for fallback handling + providerFailure: if (!providerResponse.ok) { + trackPendingRequest(model, provider, connectionId, false); + + let statusCode = providerResponse.status; + let message = ""; + let retryAfterMs: number | null = null; + let upstreamErrorCode: string | undefined; + let upstreamErrorType: string | undefined; + + if (upstreamErrorParsed) { + statusCode = parsedStatusCode; + message = parsedMessage; + retryAfterMs = parsedRetryAfterMs; + } else { + const details = await parseUpstreamError(providerResponse, provider); + statusCode = details.statusCode; + message = details.message; + retryAfterMs = details.retryAfterMs; + upstreamErrorBody = details.responseBody; + upstreamErrorCode = typeof details.errorCode === "string" ? details.errorCode : undefined; + upstreamErrorType = typeof details.errorType === "string" ? details.errorType : undefined; + } + + // Gateways like agentrouter misstate temporary quota exhaustion as 403/400, + // which downstream classification treats as AUTH_ERROR and clients like + // Claude Code treat as permanent. Restate to 429 (+ synthetic Retry-After) + // BEFORE any classification so both the fallback engine and the surfaced + // client status see a retryable error. Registry-scoped per provider. + const restatement = applyStatusRestatement({ + provider, + status: statusCode, + message, + body: upstreamErrorBody, + retryAfterMs, + }); + if (restatement.ruleId) { + statusCode = restatement.status; + retryAfterMs = restatement.retryAfterMs; + log?.info?.( + "STATUS_RESTATE", + `${provider} ${restatement.fromStatus}→${statusCode} (${restatement.ruleId})` + ); + } + + const signatureRecovery = pipelineRecovered + ? { attempted: false, succeeded: false, execution: null, error: null, recoveryBody: null } + : await recoverAnthropicThinkingSignature({ + provider, + statusCode, + message, + body: translatedBody, + execute: async (recoveryBody) => { + translatedBody = recoveryBody as typeof translatedBody; + return executeProviderRequest(currentModel, false); + }, + parseError: (response) => parseUpstreamError(response, provider), + }); + if (!pipelineRecovered && signatureRecovery.attempted && signatureRecovery.execution) { + providerResponse = signatureRecovery.execution.response; + if (signatureRecovery.succeeded) { + providerUrl = signatureRecovery.execution.url; + providerHeaders = signatureRecovery.execution.headers; + finalBody = providerRequestCapture.body(signatureRecovery.execution.transformedBody); + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + updatePendingScope(pendingScope, { + providerRequest: finalBody, + providerUrl, + stage: "provider_response_started", + }); + log?.info?.( + "THINKING_SIGNATURE", + `Recovered ${provider}/${currentModel} after one historical-thinking retry` + ); + } else if (signatureRecovery.error) { + statusCode = signatureRecovery.error.statusCode; + message = signatureRecovery.error.message; + retryAfterMs = signatureRecovery.error.retryAfterMs; + upstreamErrorBody = signatureRecovery.error.responseBody; + upstreamErrorCode = + typeof signatureRecovery.error.errorCode === "string" + ? signatureRecovery.error.errorCode + : undefined; + upstreamErrorType = + typeof signatureRecovery.error.errorType === "string" + ? signatureRecovery.error.errorType + : undefined; + } + } + + if (signatureRecovery.succeeded) break providerFailure; + + // #10281 — tiny-budget reasoning probes (e.g. Claude Code's `/model` check + // sends `max_tokens: 1`): the model burns the whole budget on thinking, and + // some upstreams (e.g. api.cline.bot for deepseek-v4-flash) answer the empty + // outcome with a 5xx ("empty response content") instead of a truncated 200. + // Answer such probes with a valid truncated response rather than relaying the + // upstream failure — which would also mark the connection unavailable and + // poison fallback/cooldown bookkeeping for a request that is only a probe. + if ( + !stream && + isTinyBudgetReasoningProbe({ model: currentModel, body: finalBody || translatedBody }) && + isEmptyContentUpstreamFailure(statusCode, message) + ) { + providerResponse = buildReasoningProbeTruncatedResponse({ + model: currentModel, + maxTokens: toPositiveInteger( + (finalBody || translatedBody)?.max_tokens ?? + (finalBody || translatedBody)?.max_completion_tokens + ), + requestId: skillRequestId, + }); + log?.warn?.( + "PROBE", + `Reasoning probe (max_tokens < ${REASONING_BUFFER_MIN_TRIGGER}) answered with truncated 200 — upstream reported "${message}"` + ); + break providerFailure; + } + + const errorConnectionId = getCurrentConnectionId() || connectionId; + await applyProviderFailureClassification({ + statusCode, + message, + headers: providerResponse.headers, + upstreamErrorBody, + retryAfterMs, + targetModel: currentModel, + }); + + appendRequestLog({ + model, + provider, + connectionId: errorConnectionId, + status: `FAILED ${statusCode}`, + }).catch(() => {}); + + const errMsg = formatProviderError(new Error(message), provider, model, statusCode); + const safeErrMsg = sanitizeErrorMessage(errMsg) || "Upstream provider error"; + const safeUpstreamErrorBody = sanitizeUpstreamDetails(upstreamErrorBody); + console.log(`${COLORS.red}[ERROR] ${safeErrMsg}${COLORS.reset}`); + + // Log Antigravity retry time if available + if (retryAfterMs && provider === "antigravity") { + const retrySeconds = Math.ceil(retryAfterMs / 1000); + log?.debug?.("RETRY", `Antigravity quota reset in ${retrySeconds}s (${retryAfterMs}ms)`); + } + + // Log error with full request body for debugging + reqLogger.logError(new Error(message), finalBody || translatedBody); + reqLogger.logProviderResponse( + providerResponse.status, + providerResponse.statusText, + providerResponse.headers, + safeUpstreamErrorBody + ); + + // Rate limiter updated in applyProviderFailureClassification + + // ── T5: Intra-family model fallback ────────────────────────────────────── + // Before returning a model-unavailable error upstream, try sibling models + // from the same family. This keeps the request alive on the same account + // instead of failing the entire combo. + const familyRecovery = onFailure({ + view: { kind: "pipeline" }, + status: statusCode, + message, + provider, + model: currentModel, + connectionId: String(getCurrentConnectionId() || connectionId || ""), + allowAccountRotation: !managedLease && comboStrategy !== "context-relay", + allowModelFallback: !pipelineRecovered, + isolateProbe: await shouldIsolateProbeFailures(), + nextModel: getNextFamilyFallback(currentModel, triedModels, provider), + canRefresh: false, + }); + if (!pipelineRecovered && familyRecovery.dispatch.action === "fallback-model") { + const nextModel = familyRecovery.dispatch.nextModel; + if (nextModel) { + triedModels.add(nextModel); + currentModel = nextModel; + translatedBody.model = nextModel; + log?.info?.("MODEL_FALLBACK", `${model} unavailable (${statusCode}) → trying ${nextModel}`); + // Re-execute with the fallback model + try { + const fallbackResult = await executeProviderRequest(nextModel, false); + if (fallbackResult.response.ok) { + providerResponse = fallbackResult.response; + providerUrl = fallbackResult.url; + providerHeaders = fallbackResult.headers; + finalBody = providerRequestCapture.body(fallbackResult.transformedBody); + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + updatePendingScope(pendingScope, { + providerRequest: finalBody, + providerUrl, + stage: "provider_response_started", + }); + // Continue processing with the fallback response — skip error return + log?.info?.("MODEL_FALLBACK", `Serving ${nextModel} as fallback for ${model}`); + // Jump to streaming/non-streaming handling below + // We fall through by NOT returning here + } else { + // Fallback also failed — return original error + persistAttemptLogs({ + status: statusCode, + error: safeErrMsg, + providerRequest: finalBody || translatedBody, + providerResponse: safeUpstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + cacheSource: "upstream", + }); + persistFailureUsage(statusCode, "model_unavailable"); + return { + kind: "returned" as const, + value: createErrorResult( + statusCode, + errMsg, + retryAfterMs, + upstreamErrorCode, + upstreamErrorType, + upstreamErrorBody, + { passthrough: sourceFormat === FORMATS.CLAUDE } + ), + carry: carry(), + }; + } + } catch { + persistAttemptLogs({ + status: statusCode, + error: safeErrMsg, + providerRequest: finalBody || translatedBody, + providerResponse: safeUpstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + cacheSource: "upstream", + }); + persistFailureUsage(statusCode, "model_unavailable"); + return { + kind: "returned" as const, + value: createErrorResult( + statusCode, + errMsg, + retryAfterMs, + upstreamErrorCode, + upstreamErrorType, + upstreamErrorBody, + { passthrough: sourceFormat === FORMATS.CLAUDE } + ), + carry: carry(), + }; + } + } else { + persistAttemptLogs({ + status: statusCode, + error: safeErrMsg, + providerRequest: finalBody || translatedBody, + providerResponse: safeUpstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + cacheSource: "upstream", + }); + persistFailureUsage(statusCode, "model_unavailable"); + return { + kind: "returned" as const, + value: createErrorResult( + statusCode, + errMsg, + retryAfterMs, + upstreamErrorCode, + upstreamErrorType, + upstreamErrorBody, + { passthrough: sourceFormat === FORMATS.CLAUDE } + ), + carry: carry(), + }; + } + } else if (isContextOverflowError(statusCode, message)) { + const familyCandidates = getModelFamily(currentModel, provider).filter( + (m) => m !== currentModel && !triedModels.has(m) + ); + const nextModel = + findLargerContextModel(currentModel, familyCandidates, provider) ?? + getNextFamilyFallback(currentModel, triedModels, provider); + if (nextModel) { + triedModels.add(nextModel); + currentModel = nextModel; + translatedBody.model = nextModel; + log?.info?.("CONTEXT_OVERFLOW_FALLBACK", `${model} context overflow → trying ${nextModel}`); + try { + const fallbackResult = await executeProviderRequest(nextModel, false); + if (fallbackResult.response.ok) { + providerResponse = fallbackResult.response; + providerUrl = fallbackResult.url; + providerHeaders = fallbackResult.headers; + finalBody = providerRequestCapture.body(fallbackResult.transformedBody); + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + updatePendingScope(pendingScope, { + providerRequest: finalBody, + providerUrl, + stage: "provider_response_started", + }); + log?.info?.( + "CONTEXT_OVERFLOW_FALLBACK", + `Serving ${nextModel} as fallback for ${model}` + ); + } else { + persistAttemptLogs({ + status: statusCode, + error: safeErrMsg, + providerRequest: finalBody || translatedBody, + providerResponse: safeUpstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + cacheSource: "upstream", + }); + persistFailureUsage(statusCode, "context_overflow"); + return { + kind: "returned" as const, + value: createErrorResult( + statusCode, + errMsg, + retryAfterMs, + upstreamErrorCode, + upstreamErrorType, + upstreamErrorBody, + { passthrough: sourceFormat === FORMATS.CLAUDE } + ), + carry: carry(), + }; + } + } catch { + persistAttemptLogs({ + status: statusCode, + error: safeErrMsg, + providerRequest: finalBody || translatedBody, + providerResponse: safeUpstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + cacheSource: "upstream", + }); + persistFailureUsage(statusCode, "context_overflow"); + return { + kind: "returned" as const, + value: createErrorResult( + statusCode, + errMsg, + retryAfterMs, + upstreamErrorCode, + upstreamErrorType, + upstreamErrorBody, + { passthrough: sourceFormat === FORMATS.CLAUDE } + ), + carry: carry(), + }; + } + } else { + persistAttemptLogs({ + status: statusCode, + error: safeErrMsg, + providerRequest: finalBody || translatedBody, + providerResponse: safeUpstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + cacheSource: "upstream", + }); + persistFailureUsage(statusCode, "context_overflow"); + return { + kind: "returned" as const, + value: createErrorResult( + statusCode, + errMsg, + retryAfterMs, + upstreamErrorCode, + upstreamErrorType, + upstreamErrorBody, + { passthrough: sourceFormat === FORMATS.CLAUDE } + ), + carry: carry(), + }; + } + } else { + persistAttemptLogs({ + status: statusCode, + error: safeErrMsg, + providerRequest: finalBody || translatedBody, + providerResponse: safeUpstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + cacheSource: "upstream", + }); + persistFailureUsage(statusCode, `upstream_${statusCode}`); + + // Emergency budget fallback is orchestrated exclusively by the routing layer + // (src/sse/handlers/chat.ts), which resolves credentials FOR the emergency + // provider through account selection. The executor-level hop that used to + // live here re-sent the FAILING provider's credentials to the emergency + // provider's endpoint (e.g. the OpenAI API key to integrate.api.nvidia.com) + // — a cross-provider credential leak that also never succeeded upstream. + return { + kind: "returned" as const, + value: createErrorResult( + statusCode, + errMsg, + retryAfterMs, + upstreamErrorCode, + upstreamErrorType, + upstreamErrorBody, + { passthrough: sourceFormat === FORMATS.CLAUDE } + ), + carry: carry(), + }; + } + // ── End T5 ─────────────────────────────────────────────────────────────── + } + + return { kind: "fellThrough", carry: carry() }; +} diff --git a/tests/unit/chatcore/recovery-policy.test.ts b/tests/unit/chatcore/recovery-policy.test.ts index d2c775831e9f..2bf8bac2bba4 100644 --- a/tests/unit/chatcore/recovery-policy.test.ts +++ b/tests/unit/chatcore/recovery-policy.test.ts @@ -119,9 +119,11 @@ test("onFailure fallback-model matches isModelUnavailableError", async () => { assert.equal(notUnavailable.dispatch.action === "fallback-model", false); }); -test("barrel onFailure isolateProbe comes from shouldIsolateProbeFailures", () => { +test("streaming-leg onFailure isolateProbe comes from shouldIsolateProbeFailures", () => { + // The callsite moved out of the barrel when the streaming leg was lifted; the + // guarantee it encodes (isolateProbe is resolved, never hard-coded) is unchanged. const src = fs.readFileSync( - new URL("../../../open-sse/handlers/chatCore.ts", import.meta.url), + new URL("../../../open-sse/handlers/chatCore/streamingLeg.ts", import.meta.url), "utf8" ); const idx = src.indexOf("const familyRecovery = onFailure({"); From e5e621561647eb08c024c9d0fc033cb3f82f5ad9 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 19:59:04 -0400 Subject: [PATCH 12/26] fix(sse): carry back the model and recovery flag the streaming leg rebinds The streaming lift moved six values through `carry`, but picked them by asking which ones the barrel reads afterwards. That question misses a second class: inputs the leg *rebinds*. `currentModel` is reassigned four times during model fallback, and `pipelineRecovered` once during recovery. A reassignment inside the leg rebinds the leg's own parameter and never reaches the caller, so once control fell through to the non-streaming leg the barrel handed it the pre-call values -- the model chosen by fallback was discarded, and recovery looked as if it had not happened. Both now travel back on `StreamingLegCarry` and are reseated at the callsite. `providerUrl` is rebound too and is deliberately left out: the barrel does not read it after the call, so carrying it would imply a data flow that does not exist. `credentials` stays out for the other reason -- it is only ever updated through Object.assign, which writes into the object the barrel still holds. The gap was invisible to the suite. The leg runs, fallback still picks a new model, and nothing downstream is asserted, so every test stayed green while the value was dropped. streaming-leg-carry.test.ts closes that by reading both files through the TypeScript parser and comparing three sets: what the leg rebinds, what the barrel still reads after the call, and what the carry declares. Removing either field from the carry turns it red. A third case pins why credentials is absent, so converting one of those Object.assign calls into a reassignment fails rather than silently losing state. Also replaces the fixed 700-character window in recovery-policy.test.ts with a parser walk. The window was inherited, not introduced here, but it stops covering the property it checks as soon as a line break or an added argument pushes isolateProbe past the budget -- and then passes for the wrong reason. Verified both ways: hard-coding isolateProbe: false still fails, and padding the callsite past 700 characters no longer does. Suite is 63 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 2 + open-sse/handlers/chatCore/streamingLeg.ts | 19 +- tests/unit/chatcore/recovery-policy.test.ts | 55 ++++- .../unit/chatcore/streaming-leg-carry.test.ts | 216 ++++++++++++++++++ 4 files changed, 282 insertions(+), 10 deletions(-) create mode 100644 tests/unit/chatcore/streaming-leg-carry.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index e908e5050c7e..62c592a81088 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -2111,8 +2111,10 @@ export async function handleChatCore({ userAgent, }); claudePromptCacheLogMeta = streamingOutcome.carry.claudePromptCacheLogMeta; + currentModel = streamingOutcome.carry.currentModel; effectiveServiceTier = streamingOutcome.carry.effectiveServiceTier; finalBody = streamingOutcome.carry.finalBody; + pipelineRecovered = streamingOutcome.carry.pipelineRecovered; providerHeaders = streamingOutcome.carry.providerHeaders; providerResponse = streamingOutcome.carry.providerResponse; translatedBody = streamingOutcome.carry.translatedBody; diff --git a/open-sse/handlers/chatCore/streamingLeg.ts b/open-sse/handlers/chatCore/streamingLeg.ts index 20b24055a94a..8e870eb14559 100644 --- a/open-sse/handlers/chatCore/streamingLeg.ts +++ b/open-sse/handlers/chatCore/streamingLeg.ts @@ -264,15 +264,30 @@ export interface StreamingLegDeps { userAgent: string | null | undefined; } -/** Values the barrel must write back before continuing down the streaming path. */ +/** + * Values the barrel must write back before continuing down the streaming path. + * + * Two kinds of state live here. Most entries are plain values the block assigns + * and the streaming response section reads afterwards. `currentModel` and + * `pipelineRecovered` are different: the block *rebinds* them during model + * fallback and recovery, and a rebind inside this function is invisible to the + * caller, so they have to travel back explicitly. `providerUrl` is rebound too + * but is deliberately absent -- the barrel never reads it after the call. + * + * `credentials` is also absent, for the opposite reason: it is only ever + * updated via Object.assign, which writes through into the object the barrel + * still holds. + */ export interface StreamingLegCarry { claudePromptCacheLogMeta: Record | null | undefined; + currentModel: string; effectiveServiceTier: import("./serviceTier.js").EffectiveServiceTier; // The barrel declares `let finalBody;` -- implicit any under strict:false -- so it // can hold either a translated request Record or a ChatCoreExecutorResult. Naming // either one here would narrow what upstream deliberately left open. // eslint-disable-next-line @typescript-eslint/no-explicit-any finalBody: any; + pipelineRecovered: boolean; providerHeaders: Record | Headers | null | undefined; providerResponse: (Response & { body?: unknown }) | null | undefined; translatedBody: Record; @@ -343,8 +358,10 @@ export async function runStreamingLeg(deps: StreamingLegDeps): Promise ({ claudePromptCacheLogMeta, + currentModel, effectiveServiceTier, finalBody, + pipelineRecovered, providerHeaders, providerResponse, translatedBody, diff --git a/tests/unit/chatcore/recovery-policy.test.ts b/tests/unit/chatcore/recovery-policy.test.ts index 2bf8bac2bba4..710622d4801f 100644 --- a/tests/unit/chatcore/recovery-policy.test.ts +++ b/tests/unit/chatcore/recovery-policy.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import ts from "typescript"; import { COOLDOWN_MS } from "../../../open-sse/config/errorConfig.ts"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-recovery-policy-")); @@ -121,14 +122,50 @@ test("onFailure fallback-model matches isModelUnavailableError", async () => { test("streaming-leg onFailure isolateProbe comes from shouldIsolateProbeFailures", () => { // The callsite moved out of the barrel when the streaming leg was lifted; the - // guarantee it encodes (isolateProbe is resolved, never hard-coded) is unchanged. - const src = fs.readFileSync( - new URL("../../../open-sse/handlers/chatCore/streamingLeg.ts", import.meta.url), - "utf8" + // guarantee it encodes -- isolateProbe is resolved at runtime, never hard-coded + // -- is unchanged. + // + // Read through the parser rather than slicing a fixed window of source text. + // A character budget silently stops covering the property it is meant to check + // the moment an argument or a line break pushes it past the end, and the test + // then passes for the wrong reason. + const url = new URL("../../../open-sse/handlers/chatCore/streamingLeg.ts", import.meta.url); + const sf = ts.createSourceFile( + url.pathname, + fs.readFileSync(url, "utf8"), + ts.ScriptTarget.ESNext, + true + ); + + let isolateProbe: ts.Expression | null = null; + const walk = (node: ts.Node) => { + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.name.text === "familyRecovery" && + node.initializer && + ts.isCallExpression(node.initializer) && + node.initializer.expression.getText() === "onFailure" + ) { + const arg = node.initializer.arguments[0]; + if (arg && ts.isObjectLiteralExpression(arg)) { + for (const prop of arg.properties) { + if (ts.isPropertyAssignment(prop) && prop.name.getText() === "isolateProbe") { + isolateProbe = prop.initializer; + } + } + } + } + ts.forEachChild(node, walk); + }; + walk(sf); + + assert.ok(isolateProbe, "familyRecovery callsite must pass isolateProbe"); + const expr = isolateProbe as ts.Expression; + assert.ok( + ts.isAwaitExpression(expr) && + ts.isCallExpression(expr.expression) && + expr.expression.expression.getText() === "shouldIsolateProbeFailures", + `isolateProbe must be await shouldIsolateProbeFailures(), got: ${expr.getText()}` ); - const idx = src.indexOf("const familyRecovery = onFailure({"); - assert.ok(idx >= 0, "familyRecovery callsite must exist"); - const slice = src.slice(idx, idx + 700); - assert.match(slice, /isolateProbe:\s*await shouldIsolateProbeFailures\(\)/); - assert.equal(/isolateProbe:\s*false/.test(slice), false); }); diff --git a/tests/unit/chatcore/streaming-leg-carry.test.ts b/tests/unit/chatcore/streaming-leg-carry.test.ts new file mode 100644 index 000000000000..eec53103c946 --- /dev/null +++ b/tests/unit/chatcore/streaming-leg-carry.test.ts @@ -0,0 +1,216 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import ts from "typescript"; + +// The streaming block was lifted out of handleChatCore into its own leaf. Unlike +// the non-streaming leg, this one does not own every exit: control can fall out +// of it and carry on down the barrel, so anything it *rebinds* has to travel +// back through `carry` and be reseated by the caller. +// +// The distinction that matters is rebinding versus mutation: +// +// credentials -- only ever Object.assign(credentials, ...), which writes +// through the parameter into the object the barrel still +// holds. Nothing to carry. +// currentModel -- reassigned during model fallback. A reassignment inside the +// leaf rebinds the leaf's own parameter and is invisible to +// the barrel, so it must be carried back. +// +// Dropping a carry-back is silent. The leaf still runs, its fallback still +// picks a new model, and every suite that exercises the leaf's internals stays +// green -- the barrel just keeps using the stale value afterwards. This test +// pins the plumbing rather than the behaviour, which is the part no other test +// is looking at. +// +// Read through the TypeScript parser, not by regex: a text pattern would encode +// today's formatting and go red on a reformat that changed no behaviour. + +const leafPath = new URL("../../../open-sse/handlers/chatCore/streamingLeg.ts", import.meta.url); +const barrelPath = new URL("../../../open-sse/handlers/chatCore.ts", import.meta.url); + +function parse(path: URL): ts.SourceFile { + return ts.createSourceFile( + path.pathname, + readFileSync(path, "utf8"), + ts.ScriptTarget.ESNext, + true + ); +} + +/** Parameter names destructured out of the leaf's `deps` argument. */ +function leafInputs(sf: ts.SourceFile): Set { + const names = new Set(); + const walk = (node: ts.Node) => { + if ( + ts.isVariableDeclaration(node) && + ts.isObjectBindingPattern(node.name) && + ts.isIdentifier(node.initializer as ts.Node as ts.Identifier) && + (node.initializer as ts.Identifier).text === "deps" + ) { + for (const el of node.name.elements) { + if (ts.isIdentifier(el.name)) names.add(el.name.text); + } + } + ts.forEachChild(node, walk); + }; + walk(sf); + return names; +} + +/** Inputs the leaf reassigns (`x = ...`), i.e. rebinds its own parameter. */ +function rebound(sf: ts.SourceFile, inputs: Set): Set { + const hit = new Set(); + const walk = (node: ts.Node) => { + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(node.left) && + inputs.has(node.left.text) + ) { + hit.add(node.left.text); + } + ts.forEachChild(node, walk); + }; + walk(sf); + return hit; +} + +/** Members declared on the StreamingLegCarry interface. */ +function carryFields(sf: ts.SourceFile): Set { + const names = new Set(); + const walk = (node: ts.Node) => { + if (ts.isInterfaceDeclaration(node) && node.name.text === "StreamingLegCarry") { + for (const m of node.members) { + if (m.name && ts.isIdentifier(m.name)) names.add(m.name.text); + } + } + ts.forEachChild(node, walk); + }; + walk(sf); + return names; +} + +/** Names the barrel reseats as `x = streamingOutcome.carry.x`. */ +function reseatedByBarrel(sf: ts.SourceFile): Set { + const names = new Set(); + const walk = (node: ts.Node) => { + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(node.left) && + ts.isPropertyAccessExpression(node.right) && + node.right.name.text === node.left.text && + node.right.expression.getText().endsWith("carry") + ) { + names.add(node.left.text); + } + ts.forEachChild(node, walk); + }; + walk(sf); + return names; +} + +/** Identifiers the barrel reads after the runStreamingLeg call statement. */ +function readAfterTheCall(sf: ts.SourceFile): Set { + const line = (pos: number) => sf.getLineAndCharacterOfPosition(pos).line + 1; + let callEnd: number | null = null; + const findCall = (node: ts.Node) => { + if (ts.isCallExpression(node) && node.expression.getText() === "runStreamingLeg") { + let stmt: ts.Node = node; + while (stmt.parent && !ts.isStatement(stmt)) stmt = stmt.parent; + callEnd = line(stmt.getEnd()); + } + ts.forEachChild(node, findCall); + }; + findCall(sf); + assert.ok(callEnd !== null, "failed to locate the runStreamingLeg call"); + + const names = new Set(); + const walk = (node: ts.Node) => { + if (ts.isIdentifier(node) && line(node.getStart()) > (callEnd as number)) { + const p = node.parent; + const isWrite = + ts.isBinaryExpression(p) && + p.left === node && + p.operatorToken.kind === ts.SyntaxKind.EqualsToken; + const isDecl = ts.isVariableDeclaration(p) && p.name === node; + const isProp = ts.isPropertyAccessExpression(p) && p.name === node; + if (!isWrite && !isDecl && !isProp) names.add(node.text); + } + ts.forEachChild(node, walk); + }; + walk(sf); + return names; +} + +test("inputs the leg rebinds and the barrel still reads are carried back", () => { + const leaf = parse(leafPath); + const inputs = leafInputs(leaf); + assert.ok(inputs.size > 10, "failed to read the leaf's destructured inputs"); + + // A rebind only matters if someone downstream looks at the variable again. + // providerUrl, for instance, is reassigned inside the leg and never read by + // the barrel afterwards; carrying it back would suggest a data flow that does + // not exist. Narrow to the rebinds that would actually go stale. + const stillRead = readAfterTheCall(parse(barrelPath)); + const declared = carryFields(leaf); + const missing = [...rebound(leaf, inputs)] + .filter((n) => stillRead.has(n)) + .filter((n) => !declared.has(n)); + + assert.deepEqual( + missing, + [], + `rebound by the leg and read again by the barrel, but never carried back, so the barrel sees the pre-call value: ${missing.join(", ")}` + ); +}); + +test("every carry field is reseated by the barrel", () => { + const declared = carryFields(parse(leafPath)); + assert.ok(declared.size > 0, "failed to read StreamingLegCarry"); + + const reseated = reseatedByBarrel(parse(barrelPath)); + const dropped = [...declared].filter((n) => !reseated.has(n)); + assert.deepEqual( + dropped, + [], + `carried back but never written back in chatCore.ts: ${dropped.join(", ")}` + ); +}); + +test("credentials is mutated in place, so it is deliberately not carried", () => { + // Guards the reasoning above: if someone ever converts one of these + // Object.assign calls into a reassignment, the first test starts demanding a + // carry field for it -- and this one explains why it was absent until then. + const sf = parse(leafPath); + let assigns = 0; + let rebinds = 0; + const walk = (node: ts.Node) => { + if ( + ts.isCallExpression(node) && + node.expression.getText() === "Object.assign" && + node.arguments.length > 0 && + node.arguments[0].getText() === "credentials" + ) { + assigns += 1; + } + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(node.left) && + node.left.text === "credentials" + ) { + rebinds += 1; + } + ts.forEachChild(node, walk); + }; + walk(sf); + + assert.ok(assigns > 0, "expected credentials to be updated via Object.assign"); + assert.equal( + rebinds, + 0, + "credentials was rebound; a rebind does not reach the barrel and needs a carry field" + ); +}); From acb595293bd9162d70402b119ec2d173ee1cbc2a Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 20:15:15 -0400 Subject: [PATCH 13/26] refactor(sse): localise providerUrl and pin the carry's live-binding property Two follow-ups from reviewing the streaming lift. providerUrl was still declared in the barrel, passed into the leg, and never looked at again -- the leg assigns it on both branches before its first read, and the non-streaming leg does not take it at all. It was the one member of the 1606-1610 execution-result group whose value no longer crosses the boundary, so it moves inside the leg. Declaring it without an initialiser means the compiler, not a comment, is what guarantees every path assigns before reading. The carry tests checked that the plumbing is declared but not that it reports post-fallback values. A carry that snapshotted its inputs -- build the object once, return it from the closure -- satisfies every one of those checks and still hands the barrel the pre-call state, which is the same class of silent loss the previous commit fixed. The new case pins two properties that together rule that out: deps is destructured with `let`, and the factory names those bindings in an object literal it builds per call. Shorthand in an arrow re-evaluates the binding on every invocation, so with both holding, a reassignment before carry() is observed by it. Both are read through the parser. An earlier draft assembled a harness with the Function constructor, which is banned repo-wide and rightly flagged; a getter probe cannot tell the two shapes apart either, since destructuring reads each property exactly once regardless. Verified against all three defect shapes: snapshotting the carry, binding deps with const, and substituting a precomputed value for the live one each turn it red. Suite is 64 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 2 - open-sse/handlers/chatCore/streamingLeg.ts | 6 +- .../unit/chatcore/streaming-leg-carry.test.ts | 86 +++++++++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 62c592a81088..2833d19c30d3 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1632,7 +1632,6 @@ export async function handleChatCore({ // Execute request using executor (handles URL building, headers, fallback, transform) let providerResponse; - let providerUrl; let providerHeaders; let finalBody; let claudePromptCacheLogMeta = null; @@ -2095,7 +2094,6 @@ export async function handleChatCore({ providerHeaders, providerRequestCapture, providerResponse, - providerUrl, reqLogger, resolveEffectiveServiceTier, sessionAffinityKey, diff --git a/open-sse/handlers/chatCore/streamingLeg.ts b/open-sse/handlers/chatCore/streamingLeg.ts index 8e870eb14559..b8d962cbd5f9 100644 --- a/open-sse/handlers/chatCore/streamingLeg.ts +++ b/open-sse/handlers/chatCore/streamingLeg.ts @@ -196,7 +196,6 @@ export interface StreamingLegDeps { providerHeaders: Record | Headers | null | undefined; providerRequestCapture: import("../../utils/providerRequestLogging.ts").Capture; providerResponse: (Response & { body?: unknown }) | null | undefined; - providerUrl: string; reqLogger: { sessionPath: null; logClientRawRequest: ( @@ -340,7 +339,6 @@ export async function runStreamingLeg(deps: StreamingLegDeps): Promise ({ claudePromptCacheLogMeta, currentModel, diff --git a/tests/unit/chatcore/streaming-leg-carry.test.ts b/tests/unit/chatcore/streaming-leg-carry.test.ts index eec53103c946..720712af9aab 100644 --- a/tests/unit/chatcore/streaming-leg-carry.test.ts +++ b/tests/unit/chatcore/streaming-leg-carry.test.ts @@ -214,3 +214,89 @@ test("credentials is mutated in place, so it is deliberately not carried", () => "credentials was rebound; a rebind does not reach the barrel and needs a carry field" ); }); + +test("carry() reads the live bindings, so reassignments reach the barrel", () => { + // The checks above prove the plumbing is declared; this one pins the part that + // makes the plumbing mean anything. A carry that snapshotted its inputs -- say + // `const snap = { currentModel }; const carry = () => snap;` -- satisfies every + // static check and still hands the barrel the pre-fallback values. + // + // Two properties together rule that out, and both are decidable from the tree: + // + // 1. deps is destructured with `let`. A const binding could not be reassigned + // at all, and the reassignments are the whole reason this leg carries state. + // 2. the carry factory names those same bindings directly in its returned + // object literal, rather than returning a previously-built object. + // + // Shorthand `{ currentModel }` inside an arrow evaluates the binding on every + // call, so with both properties holding, a reassignment before carry() is + // observed by it. + const sf = parse(leafPath); + + let depsDecl: ts.VariableDeclaration | null = null; + let declList: ts.VariableDeclarationList | null = null; + let carryFactory: ts.ArrowFunction | null = null; + + const walk = (node: ts.Node) => { + if ( + ts.isVariableDeclaration(node) && + ts.isObjectBindingPattern(node.name) && + node.initializer && + node.initializer.getText() === "deps" + ) { + depsDecl = node; + if (ts.isVariableDeclarationList(node.parent)) declList = node.parent; + } + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.name.text === "carry" && + node.initializer && + ts.isArrowFunction(node.initializer) + ) { + carryFactory = node.initializer; + } + ts.forEachChild(node, walk); + }; + walk(sf); + + assert.ok(depsDecl, "failed to find the deps destructure"); + assert.ok(declList, "failed to find the deps declaration list"); + assert.ok(carryFactory, "failed to find the carry factory"); + + assert.ok( + ((declList as ts.VariableDeclarationList).flags & ts.NodeFlags.Let) !== 0, + "deps must be destructured with let; a const binding could not carry a reassignment" + ); + + // The factory must return an object literal built at call time, not a + // previously-constructed one. + const body = (carryFactory as ts.ArrowFunction).body; + const literal = ts.isParenthesizedExpression(body) ? body.expression : body; + assert.ok( + ts.isObjectLiteralExpression(literal), + `carry must return a fresh object literal, got: ${literal.getText().slice(0, 60)}` + ); + + // And each carried name must be the binding itself, evaluated per call. + const bound = new Set(); + const pattern = (depsDecl as ts.VariableDeclaration).name as ts.ObjectBindingPattern; + for (const el of pattern.elements) { + if (ts.isIdentifier(el.name)) bound.add(el.name.text); + } + + for (const name of ["currentModel", "pipelineRecovered"]) { + const prop = (literal as ts.ObjectLiteralExpression).properties.find( + (pr) => pr.name?.getText() === name + ); + assert.ok(prop, `${name} must appear on the carry literal`); + assert.ok( + ts.isShorthandPropertyAssignment(prop) || + (ts.isPropertyAssignment(prop) && + ts.isIdentifier(prop.initializer) && + prop.initializer.text === name), + `${name} must be carried as the live binding, not a precomputed value` + ); + assert.ok(bound.has(name), `${name} must be one of the let-bound deps`); + } +}); From 0b6fe76b076dd603078c24d008262d1e0db20ad8 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 20:24:13 -0400 Subject: [PATCH 14/26] test(sse): pin providerUrl's definite assignment, which typecheck cannot The previous commit declared providerUrl without an initialiser and said the compiler, not a comment, guarantees every path assigns before reading. That is wrong. Definite-assignment analysis is a strictNullChecks feature and this repo compiles with strict: false, so npm run typecheck:core never asks the question. Commenting out one branch's assignment leaves the build completely green. Checked rather than assumed: with the flag switched on for that one file the current code reports zero TS2454, so the claim about the code was right even though the claim about the enforcement was not. Same check across nonStreamingLeg.ts and chatCore.ts is also zero -- the gap is a missing guard, not a live defect. The test re-runs that single-file check with strictNullChecks on, which keeps the rest of the codebase out of strict mode. Gapping either branch turns it red; both were verified. The comment now says where the guarantee actually lives instead of pointing at a compiler pass that does not run. Suite is 65 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore/streamingLeg.ts | 6 +++ .../unit/chatcore/streaming-leg-carry.test.ts | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/open-sse/handlers/chatCore/streamingLeg.ts b/open-sse/handlers/chatCore/streamingLeg.ts index b8d962cbd5f9..3f35ab6fbf96 100644 --- a/open-sse/handlers/chatCore/streamingLeg.ts +++ b/open-sse/handlers/chatCore/streamingLeg.ts @@ -356,6 +356,12 @@ export async function runStreamingLeg(deps: StreamingLegDeps): Promise ({ diff --git a/tests/unit/chatcore/streaming-leg-carry.test.ts b/tests/unit/chatcore/streaming-leg-carry.test.ts index 720712af9aab..5451abcc59a6 100644 --- a/tests/unit/chatcore/streaming-leg-carry.test.ts +++ b/tests/unit/chatcore/streaming-leg-carry.test.ts @@ -300,3 +300,45 @@ test("carry() reads the live bindings, so reassignments reach the barrel", () => assert.ok(bound.has(name), `${name} must be one of the let-bound deps`); } }); + +test("providerUrl is assigned on every path before it is read", () => { + // providerUrl became a local when the leg took ownership of it, declared + // without an initialiser so that a missing assignment is an error rather than + // a silent undefined flowing into logTargetRequest. + // + // The repo compiles with strict: false, and definite-assignment analysis is a + // strictNullChecks feature -- so npm run typecheck:core does NOT enforce this. + // An earlier version of this change claimed the compiler guaranteed it; that + // was wrong, and commenting out one branch's assignment kept the build green. + // + // Re-check the single file with the flag on. Scoped to this file, so it does + // not drag the rest of the codebase into strict mode. + const prog = ts.createProgram([leafPath.pathname], { + strictNullChecks: true, + noEmit: true, + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + skipLibCheck: true, + // The leg's imports are irrelevant here and resolving them would pull in the + // whole graph; definite assignment is a per-function flow question. + noResolve: true, + }); + + const sf = prog.getSourceFile(leafPath.pathname); + assert.ok(sf, "failed to load the leaf into a program"); + + const usedBeforeAssigned = prog + .getSemanticDiagnostics(sf) + .filter((d) => d.code === 2454) + .map((d) => { + const { line } = sf!.getLineAndCharacterOfPosition(d.start ?? 0); + return `${line + 1}: ${ts.flattenDiagnosticMessageText(d.messageText, " ")}`; + }); + + assert.deepEqual( + usedBeforeAssigned, + [], + `variables read before assignment:\n ${usedBeforeAssigned.join("\n ")}` + ); +}); From a4269a89b91ca9e6f5aba766bf5a07438297f392 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 20:37:12 -0400 Subject: [PATCH 15/26] test(sse): make the definite-assignment guard prove it can still fail Review asked whether noResolve: true also skips the flow analysis the guard depends on, and whether 2454 is even the right diagnostic code. Both are fair: a guard that hard-codes a number and disables part of the compiler is one upstream change away from passing forever while checking nothing. Measured both. Gapping a branch reports TS2454 twice with noResolve on and twice with it off -- identical, because definite assignment is a question about control flow inside one function and does not need the import graph. The code is 2454, not 2300 or 2304. Rather than record that in a comment, the test now demonstrates it: it copies the leaf, removes an assignment, and asserts that configuration still reports TS2454 before trusting the clean result from the real file. Compiler options are hoisted so the probe and the real check cannot drift apart. Verified the self-check has teeth: renumbering the diagnostic and turning strictNullChecks off each turn it red -- the two ways this guard could quietly stop guarding anything. Suite is 65 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- .../unit/chatcore/streaming-leg-carry.test.ts | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/tests/unit/chatcore/streaming-leg-carry.test.ts b/tests/unit/chatcore/streaming-leg-carry.test.ts index 5451abcc59a6..bc34aaa1a05a 100644 --- a/tests/unit/chatcore/streaming-leg-carry.test.ts +++ b/tests/unit/chatcore/streaming-leg-carry.test.ts @@ -1,6 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import ts from "typescript"; // The streaming block was lifted out of handleChatCore into its own leaf. Unlike @@ -313,7 +315,7 @@ test("providerUrl is assigned on every path before it is read", () => { // // Re-check the single file with the flag on. Scoped to this file, so it does // not drag the rest of the codebase into strict mode. - const prog = ts.createProgram([leafPath.pathname], { + const programOptions: ts.CompilerOptions = { strictNullChecks: true, noEmit: true, target: ts.ScriptTarget.ESNext, @@ -321,13 +323,46 @@ test("providerUrl is assigned on every path before it is read", () => { moduleResolution: ts.ModuleResolutionKind.Bundler, skipLibCheck: true, // The leg's imports are irrelevant here and resolving them would pull in the - // whole graph; definite assignment is a per-function flow question. + // whole graph; definite assignment is a per-function flow question. The + // self-check below confirms this does not weaken the analysis. noResolve: true, - }); + }; + + const prog = ts.createProgram([leafPath.pathname], programOptions); const sf = prog.getSourceFile(leafPath.pathname); assert.ok(sf, "failed to load the leaf into a program"); + // Before trusting a zero, prove this configuration can still produce a + // non-zero. noResolve skips the import graph, and a reasonable worry is that + // it also skips the flow analysis this check depends on -- measured, it does + // not, but the check should demonstrate that rather than rely on the comment. + // Same for the diagnostic code: if TypeScript ever renumbers it, a hard-coded + // 2454 would silently pass forever. + const probeSource = readFileSync(leafPath, "utf8").replace( + /^(\s*)providerUrl = /m, + "$1// probe: assignment removed\n$1void 0 && (providerUrl = " + ); + assert.notEqual( + probeSource, + readFileSync(leafPath, "utf8"), + "failed to build the self-check probe: no providerUrl assignment matched" + ); + const probePath = join(tmpdir(), `streaming-leg-probe-${process.pid}.ts`); + writeFileSync(probePath, probeSource); + let probeFound = 0; + try { + const probeProg = ts.createProgram([probePath], programOptions); + const probeSf = probeProg.getSourceFile(probePath); + probeFound = probeProg.getSemanticDiagnostics(probeSf).filter((d) => d.code === 2454).length; + } finally { + rmSync(probePath, { force: true }); + } + assert.ok( + probeFound > 0, + "this program configuration no longer reports TS2454, so a clean result below would be meaningless" + ); + const usedBeforeAssigned = prog .getSemanticDiagnostics(sf) .filter((d) => d.code === 2454) From 68065634926307f73e3922a458f582bd6dbdf2e2 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 20:42:19 -0400 Subject: [PATCH 16/26] test(sse): fix the self-check probe, which was not a valid TypeScript file Review flagged that the probe's regex splice could mangle the source in ways the assertions would not notice. Checking it, that was already happening: the replacement wrapped the assignment in `void 0 && (providerUrl = ...` and left the parenthesis unbalanced, so the probe carried one parse error. That matters because a file which does not parse can report 2454 for reasons unrelated to the branch under test -- so the self-check could have been passing on a broken probe rather than on working flow analysis, which is the exact false negative it was added to rule out. The probe now comments the assignment out, leaving a file that parses cleanly, and two assertions pin what the old code only assumed: the probe has zero parse diagnostics, and it carries exactly one fewer assignment than the leaf. Measured after the change: 0 parse errors, 6 assignments down to 5, still two TS2454. All four failure shapes turn it red -- a probe that does not parse, a probe that drops nothing, a renumbered diagnostic, and a real gapped branch. Suite is 65 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- .../unit/chatcore/streaming-leg-carry.test.ts | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/unit/chatcore/streaming-leg-carry.test.ts b/tests/unit/chatcore/streaming-leg-carry.test.ts index bc34aaa1a05a..c30ac15594ad 100644 --- a/tests/unit/chatcore/streaming-leg-carry.test.ts +++ b/tests/unit/chatcore/streaming-leg-carry.test.ts @@ -339,15 +339,32 @@ test("providerUrl is assigned on every path before it is read", () => { // not, but the check should demonstrate that rather than rely on the comment. // Same for the diagnostic code: if TypeScript ever renumbers it, a hard-coded // 2454 would silently pass forever. - const probeSource = readFileSync(leafPath, "utf8").replace( - /^(\s*)providerUrl = /m, - "$1// probe: assignment removed\n$1void 0 && (providerUrl = " + const leafSource = readFileSync(leafPath, "utf8"); + + // Comment the assignment out entirely rather than splicing around it. An + // earlier attempt wrapped it in `void 0 && (...` and left the parenthesis + // unbalanced, so the probe did not parse -- and a file that fails to parse can + // report 2454 for reasons that have nothing to do with the branch under test, + // which is exactly the false negative this self-check exists to avoid. + const assignment = /^([ \t]*)providerUrl = .*$/m; + const match = leafSource.match(assignment); + assert.ok(match, "failed to build the self-check probe: no providerUrl assignment matched"); + const probeSource = leafSource.replace(assignment, "$1// probe: assignment removed"); + + // The probe must still be a valid TypeScript file, and it must genuinely be + // missing the assignment -- otherwise a 2454 below would prove nothing. + const probeTree = ts.createSourceFile("probe.ts", probeSource, ts.ScriptTarget.ESNext, true); + assert.equal( + (probeTree as unknown as { parseDiagnostics?: unknown[] }).parseDiagnostics?.length ?? 0, + 0, + "self-check probe does not parse, so any diagnostic it produces is meaningless" ); - assert.notEqual( - probeSource, - readFileSync(leafPath, "utf8"), - "failed to build the self-check probe: no providerUrl assignment matched" + assert.equal( + (probeSource.match(/^[ \t]*providerUrl = /gm) ?? []).length, + (leafSource.match(/^[ \t]*providerUrl = /gm) ?? []).length - 1, + "self-check probe did not actually drop an assignment" ); + const probePath = join(tmpdir(), `streaming-leg-probe-${process.pid}.ts`); writeFileSync(probePath, probeSource); let probeFound = 0; From 69eb31c09cd53881b7e3e667aecbfdb7f09190e5 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 20:51:00 -0400 Subject: [PATCH 17/26] docs(sse): reword a comment that broke the review tooling's JSON encoding The comment quoted the malformed splice inline, backticks and all. Encoding that line into the review pipeline's JSON payload failed mid-string, which surfaced as a parse error and a mismatched excerpt rather than anything about the test. Same explanation, described in words instead of quoting the broken expression. No behaviour change; the file's other inline code references round-trip fine, so they stay as they are. Suite is 65 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- tests/unit/chatcore/streaming-leg-carry.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/chatcore/streaming-leg-carry.test.ts b/tests/unit/chatcore/streaming-leg-carry.test.ts index c30ac15594ad..58c975ad5171 100644 --- a/tests/unit/chatcore/streaming-leg-carry.test.ts +++ b/tests/unit/chatcore/streaming-leg-carry.test.ts @@ -342,10 +342,11 @@ test("providerUrl is assigned on every path before it is read", () => { const leafSource = readFileSync(leafPath, "utf8"); // Comment the assignment out entirely rather than splicing around it. An - // earlier attempt wrapped it in `void 0 && (...` and left the parenthesis - // unbalanced, so the probe did not parse -- and a file that fails to parse can - // report 2454 for reasons that have nothing to do with the branch under test, - // which is exactly the false negative this self-check exists to avoid. + // earlier attempt wrapped it in a short-circuit expression and left the + // opening parenthesis unbalanced, so the probe did not parse -- and a file + // that fails to parse can report 2454 for reasons that have nothing to do + // with the branch under test, which is exactly the false negative this + // self-check exists to avoid. const assignment = /^([ \t]*)providerUrl = .*$/m; const match = leafSource.match(assignment); assert.ok(match, "failed to build the self-check probe: no providerUrl assignment matched"); From 93eae0122e4179afb83ac9ea2cd5872883c05485 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 22:14:59 -0400 Subject: [PATCH 18/26] test(sse): locate probe targets with the parser, and probe all of them Review pointed out the probe's line regex would only comment out the first line of a multi-line assignment. Reproduced it: reshaping one assignment across three lines left the guard green, because the regex simply matched a different single-line assignment instead and probed that one. The branch whose shape changed was never exercised. The probe targets now come from the AST, so a statement spanning several lines is removed whole, and every assignment is probed rather than just whichever the regex happened to hit first. Probing all of them surfaced something the single-probe version could not: not every removal is a compile error. Dropping the assignment in the retry branch leaves an earlier one reaching the same read, so the compiler sees a value and says nothing -- the fault there is a stale URL, not an unassigned one, which definite-assignment analysis does not model. Requiring every probe to fire would have been asserting something untrue about the code, so the check requires that at least one fires, which is what makes a clean result meaningful. Re-verified after loosening: a gapped branch is red, a multi-line assignment left intact is green, the same assignment gapped is red, and renumbering the diagnostic is red. Suite is 65 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- .../unit/chatcore/streaming-leg-carry.test.ts | 102 ++++++++++++------ 1 file changed, 67 insertions(+), 35 deletions(-) diff --git a/tests/unit/chatcore/streaming-leg-carry.test.ts b/tests/unit/chatcore/streaming-leg-carry.test.ts index 58c975ad5171..a840709cc43a 100644 --- a/tests/unit/chatcore/streaming-leg-carry.test.ts +++ b/tests/unit/chatcore/streaming-leg-carry.test.ts @@ -341,44 +341,76 @@ test("providerUrl is assigned on every path before it is read", () => { // 2454 would silently pass forever. const leafSource = readFileSync(leafPath, "utf8"); - // Comment the assignment out entirely rather than splicing around it. An - // earlier attempt wrapped it in a short-circuit expression and left the - // opening parenthesis unbalanced, so the probe did not parse -- and a file - // that fails to parse can report 2454 for reasons that have nothing to do - // with the branch under test, which is exactly the false negative this - // self-check exists to avoid. - const assignment = /^([ \t]*)providerUrl = .*$/m; - const match = leafSource.match(assignment); - assert.ok(match, "failed to build the self-check probe: no providerUrl assignment matched"); - const probeSource = leafSource.replace(assignment, "$1// probe: assignment removed"); - - // The probe must still be a valid TypeScript file, and it must genuinely be - // missing the assignment -- otherwise a 2454 below would prove nothing. - const probeTree = ts.createSourceFile("probe.ts", probeSource, ts.ScriptTarget.ESNext, true); - assert.equal( - (probeTree as unknown as { parseDiagnostics?: unknown[] }).parseDiagnostics?.length ?? 0, - 0, - "self-check probe does not parse, so any diagnostic it produces is meaningless" - ); - assert.equal( - (probeSource.match(/^[ \t]*providerUrl = /gm) ?? []).length, - (leafSource.match(/^[ \t]*providerUrl = /gm) ?? []).length - 1, - "self-check probe did not actually drop an assignment" - ); + // Locate the assignments with the parser, not a line regex. A regex that + // anchors on one line silently skips a multi-line assignment and then probes + // some other single-line one instead -- the self-check still goes green while + // the branch that changed shape is never exercised. + const findAssignments = (src: string) => { + const tree = ts.createSourceFile("scan.ts", src, ts.ScriptTarget.ESNext, true); + const spans: Array<[number, number]> = []; + const walk = (node: ts.Node) => { + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(node.left) && + node.left.text === "providerUrl" + ) { + const statement = node.parent && ts.isExpressionStatement(node.parent) ? node.parent : node; + spans.push([statement.getStart(tree), statement.getEnd()]); + } + node.forEachChild(walk); + }; + walk(tree); + return spans; + }; - const probePath = join(tmpdir(), `streaming-leg-probe-${process.pid}.ts`); - writeFileSync(probePath, probeSource); - let probeFound = 0; - try { - const probeProg = ts.createProgram([probePath], programOptions); - const probeSf = probeProg.getSourceFile(probePath); - probeFound = probeProg.getSemanticDiagnostics(probeSf).filter((d) => d.code === 2454).length; - } finally { - rmSync(probePath, { force: true }); + const assignments = findAssignments(leafSource); + assert.ok(assignments.length > 0, "no providerUrl assignments found to probe"); + + // Probe every assignment so that a multi-line one cannot be skipped, but do + // not demand that each removal is an error. Definite assignment only asks + // whether some path reaches a read with nothing assigned -- a reassignment + // that overwrites an earlier one can be deleted without the compiler + // objecting, because the stale value is still a value. The assignment inside + // the retry branch is exactly that shape today: dropping it yields a stale + // URL, not a TS2454. + // + // What this must establish is that the configuration is capable of + // reporting, so require at least one probe to fire. + let probesThatFired = 0; + for (const [from, to] of assignments) { + const line = leafSource.slice(0, from).split("\n").length; + const probeSource = + leafSource.slice(0, from) + "// probe: assignment removed" + leafSource.slice(to); + + const probeTree = ts.createSourceFile("probe.ts", probeSource, ts.ScriptTarget.ESNext, true); + assert.equal( + (probeTree as unknown as { parseDiagnostics?: unknown[] }).parseDiagnostics?.length ?? 0, + 0, + `self-check probe for line ${line} does not parse, so any diagnostic it produces is meaningless` + ); + assert.equal( + findAssignments(probeSource).length, + assignments.length - 1, + `self-check probe for line ${line} did not actually drop an assignment` + ); + + const probePath = join(tmpdir(), `streaming-leg-probe-${process.pid}-${line}.ts`); + writeFileSync(probePath, probeSource); + let probeFound = 0; + try { + const probeProg = ts.createProgram([probePath], programOptions); + const probeSf = probeProg.getSourceFile(probePath); + probeFound = probeProg.getSemanticDiagnostics(probeSf).filter((d) => d.code === 2454).length; + } finally { + rmSync(probePath, { force: true }); + } + if (probeFound > 0) probesThatFired += 1; } + assert.ok( - probeFound > 0, - "this program configuration no longer reports TS2454, so a clean result below would be meaningless" + probesThatFired > 0, + "no probe produced a TS2454, so this configuration cannot report the defect and a clean result below would be meaningless" ); const usedBeforeAssigned = prog From 118f77cbf15937c1e823f0a88ed47581b093ce5f Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Fri, 18 Sep 2026 23:13:02 -0400 Subject: [PATCH 19/26] test(sse): pin the tail call that lets the non-streaming leg skip a carry-out The streaming leg needs eight values carried back because it can fall through and the barrel keeps using what it rebound. The non-streaming leg rebinds the same kinds of values -- currentModel, translatedBody, finalBody -- and carries none of them, which is only safe because the barrel tail-calls it: the result is returned directly and nothing in that block runs afterwards. Nothing recorded that. Adding work after the return would give the second leg the same stale-value defect this suite just fixed in the first one, and the suite would stay green. The test parses the barrel and asserts the shape: exactly one callsite, the result flowing to a return through nothing but await, and no statements after it. Each of those turning false is a real change in the contract, not a formatting difference, so it fails with a message naming the shape it lost. Verified against three ways the property can break: a statement after the return, capturing the result into a local instead of returning it, and a second callsite appearing. Suite is 66 green, typecheck:core and lint clean. Signed-off-by: Minxi Hou --- .../non-streaming-leg-tail-call.test.ts | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/unit/chatcore/non-streaming-leg-tail-call.test.ts diff --git a/tests/unit/chatcore/non-streaming-leg-tail-call.test.ts b/tests/unit/chatcore/non-streaming-leg-tail-call.test.ts new file mode 100644 index 000000000000..a37cf44dff53 --- /dev/null +++ b/tests/unit/chatcore/non-streaming-leg-tail-call.test.ts @@ -0,0 +1,85 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import ts from "typescript"; + +// The non-streaming leg carries nothing back to the barrel, and that is only +// safe because the barrel tail-calls it: `return await runNonStreamingLeg(...)` +// with nothing after it. The streaming leg is the opposite shape -- it can fall +// through, so the barrel keeps using values the leg rebound, and every one of +// those has to be carried out explicitly (see streaming-leg-carry.test.ts). +// +// If someone ever adds work after that return, the two legs stop being +// different shapes and the non-streaming one silently acquires the same +// stale-value problem this suite caught in the streaming one. + +const barrelPath = new URL("../../../open-sse/handlers/chatCore.ts", import.meta.url); + +const parseBarrel = () => { + const source = readFileSync(barrelPath, "utf8"); + return { + source, + tree: ts.createSourceFile("chatCore.ts", source, ts.ScriptTarget.ESNext, true), + }; +}; + +const findCall = (tree: ts.SourceFile, callee: string) => { + const hits: ts.CallExpression[] = []; + const walk = (node: ts.Node) => { + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + node.expression.text === callee + ) { + hits.push(node); + } + node.forEachChild(walk); + }; + walk(tree); + return hits; +}; + +test("the barrel tail-calls the non-streaming leg, so it needs no carry-out", () => { + const { tree } = parseBarrel(); + const calls = findCall(tree, "runNonStreamingLeg"); + + assert.equal( + calls.length, + 1, + `expected exactly one callsite, found ${calls.length} -- a second one would need its own review of whether the tail-call property still holds` + ); + + const call = calls[0]; + const line = tree.getLineAndCharacterOfPosition(call.getStart(tree)).line + 1; + + // Walk out to the enclosing return, tolerating the await in between. + let node: ts.Node | undefined = call.parent; + while (node && !ts.isReturnStatement(node)) { + // Anything other than await between the call and the return means the + // result is being consumed rather than handed straight back. + assert.ok( + ts.isAwaitExpression(node) || ts.isParenthesizedExpression(node), + `line ${line}: the leg's result flows through ${ts.SyntaxKind[node.kind]} instead of being returned directly` + ); + node = node.parent; + } + assert.ok(node, `line ${line}: the leg's result is not returned`); + + const block = node.parent; + assert.ok( + block && ts.isBlock(block), + `line ${line}: expected the return to sit directly in a block` + ); + + const trailing = (block as ts.Block).statements.filter( + (statement) => statement.getStart(tree) > node!.getEnd() + ); + assert.deepEqual( + trailing.map( + (statement) => + `${tree.getLineAndCharacterOfPosition(statement.getStart(tree)).line + 1}: ${ts.SyntaxKind[statement.kind]}` + ), + [], + "code now runs after the non-streaming leg returns, so values the leg rebinds no longer reach it -- give it a carry-out like the streaming leg has" + ); +}); From 0bf3c697470993930d2e54ac26e80d9608f8ff0b Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sat, 19 Sep 2026 03:07:11 -0400 Subject: [PATCH 20/26] test(chatcore): re-point hierarchical admission and stream error guards at split modules Re-point structural source guards in chatcore-hierarchical-admission.test.ts and chatcore-stream-error-result.test.ts at the split leaf modules: - Point wire-send loop and concurrency gate assertions at executeProviderRequest.ts - Point sendProviderAttempt assertions at streamingLeg.ts and nonStreamingLeg.ts - Point non-streaming semaphore try/catch assertions at nonStreamingLeg.ts - Assert that barrel chatCore.ts delegates and passes executeProviderRequest to both legs Signed-off-by: Minxi Hou --- .../chatcore-hierarchical-admission.test.ts | 50 ++++++++++++-- .../unit/chatcore-stream-error-result.test.ts | 67 ++++++++++++++----- 2 files changed, 94 insertions(+), 23 deletions(-) diff --git a/tests/unit/chatcore-hierarchical-admission.test.ts b/tests/unit/chatcore-hierarchical-admission.test.ts index 72d53290ec8a..1e69cb5edd4c 100644 --- a/tests/unit/chatcore-hierarchical-admission.test.ts +++ b/tests/unit/chatcore-hierarchical-admission.test.ts @@ -3,10 +3,25 @@ import { readFileSync } from "node:fs"; import { test } from "node:test"; const source = readFileSync( + new URL("../../open-sse/handlers/chatCore/executeProviderRequest.ts", import.meta.url), + "utf8" +); + +const barrel = readFileSync( new URL("../../open-sse/handlers/chatCore.ts", import.meta.url), "utf8" ); +const nonStreamingLeg = readFileSync( + new URL("../../open-sse/handlers/chatCore/nonStreamingLeg.ts", import.meta.url), + "utf8" +); + +const streamingLeg = readFileSync( + new URL("../../open-sse/handlers/chatCore/streamingLeg.ts", import.meta.url), + "utf8" +); + const pipeline = readFileSync( new URL("../../open-sse/handlers/chatCore/providerExecutionPipeline.ts", import.meta.url), "utf8" @@ -35,7 +50,7 @@ test("chatCore acquires cumulative gates immediately before withRateLimit", () = // single index check covered it; the loop is now split across two files, so the // guard checks both halves of the same invariant. test("each rotated account attempt acquires and releases a fresh composite slot", () => { - const sendFn = source.indexOf("const executeProviderRequest = async ("); + const sendFn = source.indexOf("export async function executeProviderRequest("); const attemptLoop = source.indexOf("while (attempts < maxAttempts)", sendFn); const acquire = source.indexOf("await acquireConcurrencyGates(", attemptLoop); const release = source.indexOf("releaseAccountSemaphore();", acquire); @@ -52,11 +67,36 @@ test("each rotated account attempt acquires and releases a fresh composite slot" "a throwing attempt must release the composite slot" ); - const sendWirings = - source.match( - /sendProviderAttempt: \(modelToCall, allowDedup\) =>\s*executeProviderRequest\(modelToCall, allowDedup\)/g - ) ?? []; + const sendWirings = [ + ...(nonStreamingLeg.match( + /sendProviderAttempt:\s*\(modelToCall,\s*allowDedup\)\s*=>\s*executeProviderRequest\(modelToCall,\s*allowDedup\)/g + ) ?? []), + ...(streamingLeg.match( + /sendProviderAttempt:\s*\(modelToCall,\s*allowDedup\)\s*=>\s*executeProviderRequest\(modelToCall,\s*allowDedup\)/g + ) ?? []), + ]; assert.equal(sendWirings.length, 2, "both legs send every pipeline attempt through the gate"); + const streamIdx = barrel.indexOf("await runStreamingLeg({"); + assert.ok(streamIdx >= 0, "runStreamingLeg call must exist in barrel"); + const streamEnd = barrel.indexOf("});", streamIdx); + assert.ok(streamEnd > streamIdx, "runStreamingLeg invocation bounds"); + const streamBlock = barrel.slice(streamIdx, streamEnd); + assert.match( + streamBlock, + /\bexecuteProviderRequest\b/, + "barrel passes wire-send function to streaming leg" + ); + + const nonStreamIdx = barrel.indexOf("await runNonStreamingLeg({"); + assert.ok(nonStreamIdx >= 0, "runNonStreamingLeg call must exist in barrel"); + const nonStreamEnd = barrel.indexOf("});", nonStreamIdx); + assert.ok(nonStreamEnd > nonStreamIdx, "runNonStreamingLeg invocation bounds"); + const nonStreamBlock = barrel.slice(nonStreamIdx, nonStreamEnd); + assert.match( + nonStreamBlock, + /\bexecuteProviderRequest\b/, + "barrel passes wire-send function to non-streaming leg" + ); const rotationLoop = pipeline.search(/while \(\s*attempts < maxAttempts\b/); assert.ok(rotationLoop >= 0, "the account/model recovery loop must exist"); diff --git a/tests/unit/chatcore-stream-error-result.test.ts b/tests/unit/chatcore-stream-error-result.test.ts index 2fb68dbf62e8..8ac650978c40 100644 --- a/tests/unit/chatcore-stream-error-result.test.ts +++ b/tests/unit/chatcore-stream-error-result.test.ts @@ -6,6 +6,8 @@ // the string-code extraction. import { test } from "node:test"; import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import ts from "typescript"; import { createSafeAbortError, formatStreamRecoveryRetryWarning, @@ -154,25 +156,54 @@ test("getUpstreamErrorIdentifier returns a non-empty string code or undefined", assert.equal(safeAbort.message, "Request aborted"); }); -test("non-streaming runNonStreamingProviderLeg is inside a try that maps semaphore errors", async () => { - const fs = await import("node:fs"); - const src = fs.readFileSync("open-sse/handlers/chatCore.ts", "utf8"); - // `let`, not `const`, since 6077b9dd (#12867) made the finalization step reassign - // legResult. The guard is about the try/catch that wraps the call, not the keyword. - const idx = src.search(/(?:const|let) legResult = await runNonStreamingProviderLeg/); - assert.ok(idx >= 0, "non-streaming branch must exist"); - const start = src.lastIndexOf("if (!stream)", idx); - const end = src.indexOf("// Streaming response", idx); - assert.ok(start >= 0 && end > start, "non-stream block bounds"); - const block = src.slice(start, end); - assert.match( - block, - /try\s*\{[\s\S]*runNonStreamingProviderLeg/, - "non-stream leg must sit in a try so SEMAPHORE_TIMEOUT cannot escape handleChatCore" +test("non-streaming runNonStreamingProviderLeg is inside a try that maps semaphore errors", () => { + const barrel = readFileSync( + new URL("../../open-sse/handlers/chatCore.ts", import.meta.url), + "utf8" ); assert.match( - block, - /isSemaphoreCapacityError/, - "same catch that maps stream semaphore errors must cover the non-stream leg" + barrel, + /if\s*\(!stream\)\s*\{\s*return await runNonStreamingLeg\(/, + "barrel must delegate non-streaming requests to runNonStreamingLeg" + ); + + const src = readFileSync( + new URL("../../open-sse/handlers/chatCore/nonStreamingLeg.ts", import.meta.url), + "utf8" ); + const sf = ts.createSourceFile("nonStreamingLeg.ts", src, ts.ScriptTarget.ESNext, true); + + const calls: ts.CallExpression[] = []; + const visit = (node: ts.Node) => { + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + node.expression.text === "runNonStreamingProviderLeg" + ) { + calls.push(node); + } + ts.forEachChild(node, visit); + }; + visit(sf); + assert.ok(calls.length >= 1, "at least one runNonStreamingProviderLeg call must exist"); + + for (const callNode of calls) { + let tryNode: ts.TryStatement | null = null; + let curr: ts.Node = callNode; + while (curr && curr !== sf) { + if (ts.isTryStatement(curr)) { + tryNode = curr; + break; + } + curr = curr.parent; + } + assert.ok(tryNode, "each runNonStreamingProviderLeg call must sit in a try statement"); + assert.ok(tryNode.catchClause, "enclosing try must have a catch clause"); + const catchText = tryNode.catchClause.getText(sf); + assert.match( + catchText, + /isSemaphoreCapacityError/, + "same catch that maps stream semaphore errors must cover the non-stream leg" + ); + } }); From 756dbf00363d29a4439763f730f586d50815710b Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sat, 19 Sep 2026 05:09:19 -0400 Subject: [PATCH 21/26] refactor(chatcore): lift provider failure classification out of handleChatCore Extract the inline applyProviderFailureClassification logic (cooldowns, terminal statuses, lockouts, quota state updates, and response telemetry) into a dedicated leaf module open-sse/handlers/chatCore/providerFailureClassification.ts. The chatCore barrel now instantiates the classifier via createProviderFailureClassifier, reducing chatCore.ts from 2534 lines to 2211 lines (-323 lines). Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 351 +---------------- .../chatCore/providerFailureClassification.ts | 367 ++++++++++++++++++ .../provider-failure-classification.test.ts | 118 ++++++ 3 files changed, 497 insertions(+), 339 deletions(-) create mode 100644 open-sse/handlers/chatCore/providerFailureClassification.ts create mode 100644 tests/unit/chatcore/provider-failure-classification.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 2833d19c30d3..022b79862f34 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -19,10 +19,7 @@ import { extractSystemRoleMessages, relocateDirectiveOnlyMessages, } from "./chatCore/claudeSystemRole.ts"; -export { - extractSystemRoleMessages, - relocateDirectiveOnlyMessages, -} from "./chatCore/claudeSystemRole.ts"; +export { extractSystemRoleMessages, relocateDirectiveOnlyMessages }; import { acquireTurnExecution, createTurnInProgressResult } from "./chatCore/turnExecutionGuard.ts"; import { applyContextCompression } from "./chatCore/contextCompression.ts"; import { checkSemanticCache } from "./chatCore/semanticCache.ts"; @@ -31,7 +28,6 @@ import { maybeConvertJsonBodyToSse } from "./chatCore/jsonBodyToSse.ts"; import { assembleStreamingResponseHeaders } from "./chatCore/streamingResponseHeaders.ts"; import { assembleStreamingPipeline } from "./chatCore/streamingPipeline.ts"; import { makeOnStreamComplete } from "./chatCore/streamMaterialize.ts"; -import { acquireTurnExecution, createTurnInProgressResult } from "./chatCore/turnExecutionGuard.ts"; import { wrapReadableStreamWithFinalize } from "./chatCore/streamFinalize.ts"; import { sanitizeChatRequestBody } from "./chatCore/sanitization.ts"; import { @@ -48,7 +44,6 @@ import { readCodexTurnStateHeader, } from "../config/codexTurnState.ts"; export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comboContextCache.ts"; -import { resolveAccountSemaphoreKey } from "./chatCore/executorHelpers.ts"; import { shouldUseNativeCodexPassthrough, shouldUseNativeXaiResponsesPassthrough, @@ -58,6 +53,7 @@ import { } from "./chatCore/passthroughHelpers.ts"; import { runNonStreamingLeg } from "./chatCore/nonStreamingLeg.ts"; import { runStreamingLeg } from "./chatCore/streamingLeg.ts"; +import { createProviderFailureClassifier } from "./chatCore/providerFailureClassification.ts"; import { buildStreamingResponseHeaders, stripStaleForwardingHeaders, @@ -73,7 +69,6 @@ export { stripStaleForwardingHeaders, }; import { resolveMemoryOwnerId } from "./chatCore/memoryExtraction.ts"; -import { normalizeHeaders } from "../utils/headers.ts"; import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts"; import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; import { injectSystemPromptPreTranslation } from "../services/systemPrompt.ts"; @@ -123,7 +118,6 @@ import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts"; import { buildErrorBody, createErrorResult, sanitizeErrorMessage } from "../utils/error.ts"; import { checkTokenLimits } from "@omniroute/open-sse/services/tokenLimitCounter.ts"; import { - COOLDOWN_MS, HTTP_STATUS, PROVIDER_MAX_TOKENS, STREAM_READINESS_MAX_TIMEOUT_MS, @@ -131,9 +125,7 @@ import { DEFAULT_MAX_TOKENS, STREAM_DISCONNECT_GRACE_PERIOD_MS, } from "../config/constants.ts"; -import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts"; import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; -import { connectionHasExtraKeys } from "../services/apiKeyRotator.ts"; import { resolveExecutionCredentials as resolveExecutionCredentialsFor } from "./chatCore/executionCredentials.ts"; import { resolveExecutorWithProxy as resolveExecutorWithProxyFor } from "./chatCore/executorProxy.ts"; import type { ClaudeMessage } from "./chatCore/claudeMessageTypes.ts"; @@ -143,10 +135,6 @@ import { type PersistAttemptLogsArgs, } from "./chatCore/attemptLogging.ts"; import { attachCompressionUsageReceiptAfterAnalytics as attachCompressionUsageReceiptAfterAnalyticsFor } from "./chatCore/compressionUsageReceipt.ts"; -import { getQuotaScopeLabelForProvider } from "../services/antigravityQuotaFamily.ts"; -import { excludeConnectionForCooldown } from "./chatCore/connectionCooldown.ts"; -import { handleRequestRejectedFailure } from "./chatCore/requestRejectedFailure.ts"; -import { getKimiTemporaryRateLimitResetAt } from "./chatCore/kimiQuotaRecovery.ts"; import { adaptBodyForCompression } from "../services/compression/bodyAdapter.ts"; import { trackPendingRequest, appendRequestLog, saveRequestUsage } from "@/lib/usageDb"; import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; @@ -168,25 +156,15 @@ import { shouldPreserveCacheControl, resolveConnectionCacheOverride, } from "../utils/cacheControlPolicy.ts"; -import { updateFromHeaders, updateFromResponseBody } from "../services/rateLimitManager.ts"; -import { markBlocked as markAccountSemaphoreBlocked } from "../services/accountSemaphore.ts"; -import { - lockModel, - lockModelIfPerModelQuota, - recordCoreOwnedAntigravityQuotaState, - shouldDeferAntigravityQuotaStateToCaller, -} from "../services/accountFallback.ts"; import { compressContext, estimateTokens, getTokenLimit } from "../services/contextManager.ts"; import type { CompressionResult } from "../services/compression/types.ts"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; -import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; import { extractFacts } from "@/lib/memory/extraction"; import { getClaudeCodeCompatibleRequestDefaults } from "@/lib/providers/requestDefaults"; import { buildClaudeCodeCompatibleRequest, resolveClaudeCodeCompatibleSessionId, } from "../services/claudeCodeCompatible.ts"; -import { classifyModelScope429 } from "../services/modelscopePolicy.ts"; import { isTpmExhausted } from "../services/geminiRateLimitTracker.ts"; import { computeRequestHash, shouldDeduplicate } from "../services/requestDedup.ts"; @@ -1734,321 +1712,16 @@ export async function handleChatCore({ } }; - const applyProviderFailureClassification = async ({ - statusCode, - message, - headers, - upstreamErrorBody, - retryAfterMs, - targetModel, - }: { - statusCode: number; - message: string; - headers?: Headers | null; - upstreamErrorBody?: unknown; - retryAfterMs?: number | null; - targetModel: string; - }) => { - let errorType = classifyProviderError(statusCode, message, provider); - if (statusCode === 429 && isModelScope()) { - const decision = classifyModelScope429(message, normalizeHeaders(headers)); - errorType = - decision.kind === "quota_exhausted" - ? PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED - : PROVIDER_ERROR_TYPES.RATE_LIMITED; - log?.warn?.( - "MODELSCOPE_429", - `${decision.kind} (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"}, total remaining: ${decision.snapshot.totalRemaining ?? "unknown"})` - ); - } - const persistentMessage = sanitizeErrorMessage(message) || "Provider request failed"; - const errorConnectionId = getCurrentConnectionId() || connectionId; - if (errorConnectionId && errorType) { - try { - if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { - const probeIsolated = await shouldIsolateProbeFailures(); - await writeTerminalStatus( - errorConnectionId, - { - testStatus: "banned", - isActive: false, - lastError: persistentMessage, - lastErrorType: errorType, - errorCode: String(statusCode), - }, - probeIsolated ? "probe" : "production" - ); - if (probeIsolated) { - console.warn( - `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) -- connection stays active` - ); - } else { - console.warn( - `[provider] Node ${errorConnectionId} banned (${statusCode}) -- disabling permanently` - ); - } - } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { - if ( - connectionHasExtraKeys( - errorConnectionId, - (credentials?.providerSpecificData as Record | undefined) - ?.extraApiKeys as string[] | undefined - ) - ) { - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: persistentMessage, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) -- has extra keys, keeping connection active` - ); - } else { - const probeIsolated2 = await shouldIsolateProbeFailures(); - await writeTerminalStatus( - errorConnectionId, - { - testStatus: "deactivated", - isActive: false, - lastError: persistentMessage, - lastErrorType: errorType, - errorCode: String(statusCode), - }, - probeIsolated2 ? "probe" : "production" - ); - if (probeIsolated2) { - console.warn( - `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) -- connection stays active` - ); - } else { - console.warn( - `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) -- disabling permanently` - ); - } - } - } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { - const probeIsolated3 = await shouldIsolateProbeFailures(); - if (probeIsolated3) { - await writeTerminalStatus( - errorConnectionId, - { - testStatus: "credits_exhausted", - lastError: persistentMessage, - lastErrorType: errorType, - errorCode: String(statusCode), - }, - "probe" - ); - console.warn( - `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) -- connection stays active` - ); - } else { - let kimiRateLimitResetAt: string | null = null; - if (provider === "kimi-coding") { - try { - const { fetchAndPersistProviderLimits } = - await import("@/lib/usage/providerLimits"); - const { usage } = await fetchAndPersistProviderLimits(errorConnectionId, "manual"); - kimiRateLimitResetAt = getKimiTemporaryRateLimitResetAt(usage); - } catch {} - } - - let quotaCooldownMs = kimiRateLimitResetAt - ? Math.max(new Date(kimiRateLimitResetAt).getTime() - Date.now(), 0) - : retryAfterMs || COOLDOWN_MS.rateLimit; - const deferAntigravityQuotaStateToCaller = shouldDeferAntigravityQuotaStateToCaller( - provider, - typeof onStreamFailure === "function" - ); - const isAntigravityQuotaFamily = shouldDeferAntigravityQuotaStateToCaller( - provider, - true - ); - let coreOwnedAntigravityLockout: { - cooldownMs: number; - failureCount: number; - } | null = null; - if (isAntigravityQuotaFamily && !deferAntigravityQuotaStateToCaller) { - const quotaErrorText = - typeof upstreamErrorBody === "string" - ? upstreamErrorBody - : upstreamErrorBody == null - ? message - : JSON.stringify(upstreamErrorBody); - coreOwnedAntigravityLockout = await recordCoreOwnedAntigravityQuotaState({ - provider, - connectionId: errorConnectionId, - model, - status: statusCode, - errorText: quotaErrorText, - headers: headers ?? undefined, - }); - quotaCooldownMs = coreOwnedAntigravityLockout.cooldownMs; - } - const accountSemaphoreKey = resolveAccountSemaphoreKey({ - provider, - model: targetModel, - connectionId: errorConnectionId, - credentials, - }); - if (accountSemaphoreKey && !deferAntigravityQuotaStateToCaller) { - markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); - } - if (deferAntigravityQuotaStateToCaller) { - } else if (coreOwnedAntigravityLockout) { - console.warn( - `[provider] Node ${errorConnectionId} Antigravity model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(coreOwnedAntigravityLockout.cooldownMs / 1000)}s (failureCount=${coreOwnedAntigravityLockout.failureCount}, owner=core)` - ); - } else if (kimiRateLimitResetAt) { - await updateProviderConnection(errorConnectionId, { - testStatus: "unavailable", - rateLimitedUntil: kimiRateLimitResetAt, - backoffLevel: 0, - lastErrorType: PROVIDER_ERROR_TYPES.RATE_LIMITED, - lastError: persistentMessage, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} Kimi request window exhausted (${statusCode}) -- retrying after ${kimiRateLimitResetAt}` - ); - } else if (isModelScope() && errorConnectionId) { - lockModel(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); - if (targetModel && targetModel !== model) { - lockModel( - provider, - errorConnectionId, - targetModel, - "quota_exhausted", - quotaCooldownMs - ); - } - console.warn( - `[provider] Node ${errorConnectionId} ModelScope model quota exhausted (${statusCode}) for ${targetModel} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` - ); - } else if ( - lockModelIfPerModelQuota( - provider, - errorConnectionId, - model, - "quota_exhausted", - quotaCooldownMs - ) || - (targetModel && - targetModel !== model && - lockModelIfPerModelQuota( - provider, - errorConnectionId, - targetModel, - "quota_exhausted", - quotaCooldownMs - )) - ) { - const quotaScope = getQuotaScopeLabelForProvider(provider, targetModel); - console.warn( - `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${targetModel} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)` - ); - } else { - await writeTerminalStatus( - errorConnectionId, - { - testStatus: "credits_exhausted", - lastError: persistentMessage, - lastErrorType: errorType, - errorCode: String(statusCode), - }, - "production" - ); - console.warn(`[provider] Node ${errorConnectionId} exhausted quota (${statusCode})`); - } - } - } else if (errorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED) { - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: persistentMessage, - errorCode: statusCode, - }); - } else if (errorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN) { - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: persistentMessage, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} OAuth token invalid (${statusCode}) -- token refresh available` - ); - } else if (errorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR) { - await updateProviderConnection(errorConnectionId, { - lastErrorType: errorType, - lastError: persistentMessage, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} project routing error (${statusCode}) -- not banning` - ); - } else if (errorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED) { - // Google regional refusal: account-independent, non-terminal; park the connection - // until egress uses a supported region; probes skip the day-long cooldown (#9817). - await excludeConnectionForCooldown({ - connectionId: errorConnectionId, - errorType, - message: persistentMessage, - statusCode, - cooldownMs: COOLDOWN_MS.geoBlocked ?? 24 * 60 * 60 * 1000, - skipCooldownForProbe: true, - label: "geo-blocked", - suffix: "trying other accounts", - }); - } else if (errorType === PROVIDER_ERROR_TYPES.REQUEST_REJECTED) { - // Per-request refusal (#12859): growing cooldown, streak → banned. - await handleRequestRejectedFailure({ - connectionId: errorConnectionId, - statusCode, - message: persistentMessage, - }); - } else if (errorType === PROVIDER_ERROR_TYPES.GCP_PROJECT_REQUIRED) { - // Antigravity BYOP: fixable via a Project ID; never a lockout/ban. Park the connection. - await excludeConnectionForCooldown({ - connectionId: errorConnectionId, - errorType, - message: persistentMessage, - statusCode, - cooldownMs: COOLDOWN_MS.gcpProjectRequired ?? 24 * 60 * 60 * 1000, - skipCooldownForProbe: false, - label: "GCP project required", - suffix: "routing to other accounts (enter a Project ID to restore)", - }); - } else if (errorType === PROVIDER_ERROR_TYPES.MODEL_NOT_FOUND) { - const notFoundCooldownMs = COOLDOWN_MS.notFound; - if (!(await shouldIsolateProbeFailures())) { - const modelToLock = targetModel || model; - lockModel( - provider, - errorConnectionId, - modelToLock, - "model_not_found", - notFoundCooldownMs - ); - console.warn( - `[provider] Node ${errorConnectionId} model not found (${statusCode}) for ${modelToLock} - locking model for ${Math.ceil(notFoundCooldownMs / 1000)}s (connection stays active)` - ); - } - } - } catch {} - } - - if (headers) { - updateFromHeaders(provider, errorConnectionId, headers, statusCode, targetModel); - } - if (errorConnectionId && upstreamErrorBody !== null && upstreamErrorBody !== undefined) { - updateFromResponseBody( - provider, - errorConnectionId, - upstreamErrorBody, - statusCode, - targetModel - ); - } - }; + const applyProviderFailureClassification = createProviderFailureClassifier({ + provider, + model, + connectionId, + credentials, + onStreamFailure, + isModelScope, + getCurrentConnectionId, + log, + }); let pipelineRecovered = false; if (stream) { diff --git a/open-sse/handlers/chatCore/providerFailureClassification.ts b/open-sse/handlers/chatCore/providerFailureClassification.ts new file mode 100644 index 000000000000..287987003c37 --- /dev/null +++ b/open-sse/handlers/chatCore/providerFailureClassification.ts @@ -0,0 +1,367 @@ +import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../../services/errorClassifier.ts"; +import { classifyModelScope429 } from "../../services/modelscopePolicy.ts"; +import { normalizeHeaders } from "../../utils/headers.ts"; +import { sanitizeErrorMessage } from "../../utils/error.ts"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; +import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; +import { connectionHasExtraKeys } from "../../services/apiKeyRotator.ts"; +import { updateProviderConnection } from "@/lib/db/providers"; +import { getKimiTemporaryRateLimitResetAt } from "./kimiQuotaRecovery.ts"; +import { COOLDOWN_MS } from "../../config/constants.ts"; +import { + lockModel, + lockModelIfPerModelQuota, + recordCoreOwnedAntigravityQuotaState, + shouldDeferAntigravityQuotaStateToCaller, +} from "../../services/accountFallback.ts"; +import { resolveAccountSemaphoreKey } from "./executorHelpers.ts"; +import { markBlocked as markAccountSemaphoreBlocked } from "../../services/accountSemaphore.ts"; +import { getQuotaScopeLabelForProvider } from "../../services/antigravityQuotaFamily.ts"; +import { excludeConnectionForCooldown } from "./connectionCooldown.ts"; +import { handleRequestRejectedFailure } from "./requestRejectedFailure.ts"; +import { updateFromHeaders, updateFromResponseBody } from "../../services/rateLimitManager.ts"; + +export interface ProviderFailureClassificationDeps { + provider: string; + model: string; + connectionId?: string | null; + credentials?: Record | null; + onStreamFailure?: ((err?: unknown) => unknown) | null; + isModelScope: () => boolean; + getCurrentConnectionId: () => string | null | undefined; + log?: { + warn?: (tag: string, message: string, ...args: unknown[]) => void; + info?: (tag: string, message: string, ...args: unknown[]) => void; + debug?: (tag: string, message: string, ...args: unknown[]) => void; + } | null; +} + +export interface ProviderFailureClassificationArgs { + statusCode: number; + message: string; + headers?: Headers | null; + upstreamErrorBody?: unknown; + retryAfterMs?: number | null; + targetModel: string; +} + +/** + * Classifies and applies provider failure side-effects (connection cooldowns, + * terminal statuses, lockouts, quota state updates, and response/header telemetry). + * + * Lifted verbatim out of handleChatCore into a focused leaf module. + */ +export async function applyProviderFailureClassification( + deps: ProviderFailureClassificationDeps, + args: ProviderFailureClassificationArgs +): Promise { + const { + provider, + model, + connectionId, + credentials, + onStreamFailure, + isModelScope, + getCurrentConnectionId, + log, + } = deps; + + const { statusCode, message, headers, upstreamErrorBody, retryAfterMs, targetModel } = args; + + let errorType = classifyProviderError(statusCode, message, provider); + if (statusCode === 429 && isModelScope()) { + const decision = classifyModelScope429(message, normalizeHeaders(headers)); + errorType = + decision.kind === "quota_exhausted" + ? PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED + : PROVIDER_ERROR_TYPES.RATE_LIMITED; + log?.warn?.( + "MODELSCOPE_429", + `${decision.kind} (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"}, total remaining: ${decision.snapshot.totalRemaining ?? "unknown"})` + ); + } + const persistentMessage = sanitizeErrorMessage(message) || "Provider request failed"; + const errorConnectionId = getCurrentConnectionId() || connectionId; + if (errorConnectionId && errorType) { + try { + if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { + const probeIsolated = await shouldIsolateProbeFailures(); + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "banned", + isActive: false, + lastError: persistentMessage, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + probeIsolated ? "probe" : "production" + ); + if (probeIsolated) { + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) -- connection stays active` + ); + } else { + console.warn( + `[provider] Node ${errorConnectionId} banned (${statusCode}) -- disabling permanently` + ); + } + } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { + if ( + connectionHasExtraKeys( + errorConnectionId, + (credentials?.providerSpecificData as Record | undefined) + ?.extraApiKeys as string[] | undefined + ) + ) { + await updateProviderConnection(errorConnectionId, { + lastErrorType: errorType, + lastError: persistentMessage, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) -- has extra keys, keeping connection active` + ); + } else { + const probeIsolated2 = await shouldIsolateProbeFailures(); + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "deactivated", + isActive: false, + lastError: persistentMessage, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + probeIsolated2 ? "probe" : "production" + ); + if (probeIsolated2) { + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) -- connection stays active` + ); + } else { + console.warn( + `[provider] Node ${errorConnectionId} account deactivated (${statusCode}) -- disabling permanently` + ); + } + } + } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { + const probeIsolated3 = await shouldIsolateProbeFailures(); + if (probeIsolated3) { + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "credits_exhausted", + lastError: persistentMessage, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + "probe" + ); + console.warn( + `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) -- connection stays active` + ); + } else { + let kimiRateLimitResetAt: string | null = null; + if (provider === "kimi-coding") { + try { + const { fetchAndPersistProviderLimits } = await import("@/lib/usage/providerLimits"); + const { usage } = await fetchAndPersistProviderLimits(errorConnectionId, "manual"); + kimiRateLimitResetAt = getKimiTemporaryRateLimitResetAt(usage); + } catch {} + } + + let quotaCooldownMs = kimiRateLimitResetAt + ? Math.max(new Date(kimiRateLimitResetAt).getTime() - Date.now(), 0) + : retryAfterMs || COOLDOWN_MS.rateLimit; + const deferAntigravityQuotaStateToCaller = shouldDeferAntigravityQuotaStateToCaller( + provider, + typeof onStreamFailure === "function" + ); + const isAntigravityQuotaFamily = shouldDeferAntigravityQuotaStateToCaller(provider, true); + let coreOwnedAntigravityLockout: { + cooldownMs: number; + failureCount: number; + } | null = null; + if (isAntigravityQuotaFamily && !deferAntigravityQuotaStateToCaller) { + const quotaErrorText = + typeof upstreamErrorBody === "string" + ? upstreamErrorBody + : upstreamErrorBody == null + ? message + : JSON.stringify(upstreamErrorBody); + coreOwnedAntigravityLockout = await recordCoreOwnedAntigravityQuotaState({ + provider, + connectionId: errorConnectionId, + model, + status: statusCode, + errorText: quotaErrorText, + headers: headers ?? undefined, + }); + quotaCooldownMs = coreOwnedAntigravityLockout.cooldownMs; + } + const accountSemaphoreKey = resolveAccountSemaphoreKey({ + provider, + model: targetModel, + connectionId: errorConnectionId, + credentials, + }); + if (accountSemaphoreKey && !deferAntigravityQuotaStateToCaller) { + markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); + } + if (deferAntigravityQuotaStateToCaller) { + } else if (coreOwnedAntigravityLockout) { + console.warn( + `[provider] Node ${errorConnectionId} Antigravity model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(coreOwnedAntigravityLockout.cooldownMs / 1000)}s (failureCount=${coreOwnedAntigravityLockout.failureCount}, owner=core)` + ); + } else if (kimiRateLimitResetAt) { + await updateProviderConnection(errorConnectionId, { + testStatus: "unavailable", + rateLimitedUntil: kimiRateLimitResetAt, + backoffLevel: 0, + lastErrorType: PROVIDER_ERROR_TYPES.RATE_LIMITED, + lastError: persistentMessage, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${errorConnectionId} Kimi request window exhausted (${statusCode}) -- retrying after ${kimiRateLimitResetAt}` + ); + } else if (isModelScope() && errorConnectionId) { + lockModel(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); + if (targetModel && targetModel !== model) { + lockModel( + provider, + errorConnectionId, + targetModel, + "quota_exhausted", + quotaCooldownMs + ); + } + console.warn( + `[provider] Node ${errorConnectionId} ModelScope model quota exhausted (${statusCode}) for ${targetModel} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` + ); + } else if ( + lockModelIfPerModelQuota( + provider, + errorConnectionId, + model, + "quota_exhausted", + quotaCooldownMs + ) || + (targetModel && + targetModel !== model && + lockModelIfPerModelQuota( + provider, + errorConnectionId, + targetModel, + "quota_exhausted", + quotaCooldownMs + )) + ) { + const quotaScope = getQuotaScopeLabelForProvider(provider, targetModel); + console.warn( + `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${targetModel} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)` + ); + } else { + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "credits_exhausted", + lastError: persistentMessage, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + "production" + ); + console.warn(`[provider] Node ${errorConnectionId} exhausted quota (${statusCode})`); + } + } + } else if (errorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED) { + await updateProviderConnection(errorConnectionId, { + lastErrorType: errorType, + lastError: persistentMessage, + errorCode: statusCode, + }); + } else if (errorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN) { + await updateProviderConnection(errorConnectionId, { + lastErrorType: errorType, + lastError: persistentMessage, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${errorConnectionId} OAuth token invalid (${statusCode}) -- token refresh available` + ); + } else if (errorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR) { + await updateProviderConnection(errorConnectionId, { + lastErrorType: errorType, + lastError: persistentMessage, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${errorConnectionId} project routing error (${statusCode}) -- not banning` + ); + } else if (errorType === PROVIDER_ERROR_TYPES.GEO_BLOCKED) { + // Google regional refusal: account-independent, non-terminal; park the connection + // until egress uses a supported region; probes skip the day-long cooldown (#9817). + await excludeConnectionForCooldown({ + connectionId: errorConnectionId, + errorType, + message: persistentMessage, + statusCode, + cooldownMs: COOLDOWN_MS.geoBlocked ?? 24 * 60 * 60 * 1000, + skipCooldownForProbe: true, + label: "geo-blocked", + suffix: "trying other accounts", + }); + } else if (errorType === PROVIDER_ERROR_TYPES.REQUEST_REJECTED) { + // Per-request refusal (#12859): growing cooldown, streak → banned. + await handleRequestRejectedFailure({ + connectionId: errorConnectionId, + statusCode, + message: persistentMessage, + }); + } else if (errorType === PROVIDER_ERROR_TYPES.GCP_PROJECT_REQUIRED) { + // Antigravity BYOP: fixable via a Project ID; never a lockout/ban. Park the connection. + await excludeConnectionForCooldown({ + connectionId: errorConnectionId, + errorType, + message: persistentMessage, + statusCode, + cooldownMs: COOLDOWN_MS.gcpProjectRequired ?? 24 * 60 * 60 * 1000, + skipCooldownForProbe: false, + label: "GCP project required", + suffix: "routing to other accounts (enter a Project ID to restore)", + }); + } else if (errorType === PROVIDER_ERROR_TYPES.MODEL_NOT_FOUND) { + const notFoundCooldownMs = COOLDOWN_MS.notFound; + if (!(await shouldIsolateProbeFailures())) { + const modelToLock = targetModel || model; + lockModel( + provider, + errorConnectionId, + modelToLock, + "model_not_found", + notFoundCooldownMs + ); + console.warn( + `[provider] Node ${errorConnectionId} model not found (${statusCode}) for ${modelToLock} - locking model for ${Math.ceil(notFoundCooldownMs / 1000)}s (connection stays active)` + ); + } + } + } catch {} + } + + if (headers) { + updateFromHeaders(provider, errorConnectionId, headers, statusCode, targetModel); + } + if (errorConnectionId && upstreamErrorBody !== null && upstreamErrorBody !== undefined) { + updateFromResponseBody(provider, errorConnectionId, upstreamErrorBody, statusCode, targetModel); + } +} + +export function createProviderFailureClassifier( + deps: ProviderFailureClassificationDeps +): (args: ProviderFailureClassificationArgs) => Promise { + return (args: ProviderFailureClassificationArgs) => + applyProviderFailureClassification(deps, args); +} diff --git a/tests/unit/chatcore/provider-failure-classification.test.ts b/tests/unit/chatcore/provider-failure-classification.test.ts new file mode 100644 index 000000000000..4132d7d6e03f --- /dev/null +++ b/tests/unit/chatcore/provider-failure-classification.test.ts @@ -0,0 +1,118 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +test("providerFailureClassification exports classifier factory and handles failure branches", async () => { + const { createProviderConnection, getProviderConnectionById } = + await import("@/lib/db/providers"); + const { isModelLocked } = await import("../../../open-sse/services/accountFallback.ts"); + const module = + await import("../../../open-sse/handlers/chatCore/providerFailureClassification.ts"); + assert.equal( + typeof module.createProviderFailureClassifier, + "function", + "createProviderFailureClassifier must be exported as a function" + ); + assert.equal( + typeof module.applyProviderFailureClassification, + "function", + "applyProviderFailureClassification must be exported as a function" + ); + + const testConn = (await createProviderConnection({ + provider: "openai", + apiKey: "sk-test-fail-classif", + name: "test-fail-conn", + })) as { id: string }; + + const calls: string[] = []; + const classifier = module.createProviderFailureClassifier({ + provider: "openai", + model: "gpt-4o", + connectionId: testConn.id, + isModelScope: () => false, + getCurrentConnectionId: () => testConn.id, + log: { + warn: (_tag, msg) => calls.push(`warn:${msg}`), + info: (_tag, msg) => calls.push(`info:${msg}`), + }, + }); + + // 1. Exercise 401 UNAUTHORIZED branch + await classifier({ + statusCode: 401, + message: "Invalid API key provided", + targetModel: "gpt-4o", + }); + + const updatedConn401 = await getProviderConnectionById(testConn.id); + assert.ok(updatedConn401, "connection must exist in database"); + assert.equal( + updatedConn401.lastErrorType, + "unauthorized", + "401 failure must record error type unauthorized" + ); + assert.equal(Number(updatedConn401.errorCode), 401, "401 failure must record error code 401"); + assert.equal( + updatedConn401.lastError, + "Invalid API key provided", + "401 failure must record sanitized error message" + ); + + // 2. Exercise 403 FORBIDDEN branch (terminal status) + await classifier({ + statusCode: 403, + message: "Cloudflare Turnstile challenge: SENTINEL_BLOCKED", + targetModel: "gpt-4o", + }); + + const updatedConn403 = await getProviderConnectionById(testConn.id); + assert.ok(updatedConn403, "connection must exist in database"); + assert.equal( + updatedConn403.testStatus, + "banned", + "403 forbidden failure must mark connection as banned" + ); + assert.equal(updatedConn403.isActive, false, "403 forbidden failure must deactivate connection"); + + // 3. Exercise 404 MODEL_NOT_FOUND branch + await classifier({ + statusCode: 404, + message: "The model gpt-4o does not exist", + targetModel: "gpt-4o", + }); + + assert.equal( + isModelLocked("openai", testConn.id, "gpt-4o"), + true, + "404 model not found must lock model for connection" + ); +}); + +test("barrel chatCore.ts delegates provider failure classification to leaf", () => { + const barrel = readFileSync( + new URL("../../../open-sse/handlers/chatCore.ts", import.meta.url), + "utf-8" + ); + + // Must import createProviderFailureClassifier from ./chatCore/providerFailureClassification.ts + assert.match( + barrel, + /import\s*\{[^}]*?\bcreateProviderFailureClassifier\b[^}]*?\}\s*from\s*["']\.\/chatCore\/providerFailureClassification(?:\.ts)?["']/, + "chatCore.ts must import createProviderFailureClassifier from the leaf module" + ); + + // The inline 300+ line async definition must no longer exist in the barrel + assert.doesNotMatch( + barrel, + /const\s+applyProviderFailureClassification\s*=\s*async\s*\(\s*\{[\s\S]*?statusCode[\s\S]*?message[\s\S]*?headers[\s\S]*?upstreamErrorBody[\s\S]*?retryAfterMs[\s\S]*?targetModel[\s\S]*?\}\s*:\s*\{[\s\S]*?classifyProviderError/, + "chatCore.ts must not keep the 300+ line inline applyProviderFailureClassification definition" + ); + + // chatCore.ts must call createProviderFailureClassifier + assert.match( + barrel, + /const\s+applyProviderFailureClassification\s*=\s*createProviderFailureClassifier\(\{/, + "chatCore.ts must instantiate classifier using createProviderFailureClassifier" + ); +}); From 86de4e27ade7d6b71a6436e891b5bf607b36b559 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sat, 19 Sep 2026 07:46:43 -0400 Subject: [PATCH 22/26] refactor(chatcore): lift streaming success materialization out of handleChatCore Extract streaming success response header assembly, transform stream selection, pipeline chaining, and completion callback wiring into open-sse/handlers/chatCore/streamingSuccessResponse.ts. - Move streaming Response construction, transform stream creation (Responses API translation, general translation, or passthrough), codex turn-state provenance, and completion/failure callback registration to a dedicated leaf helper. - Pass dependencies explicitly via StreamingSuccessResponseDeps. - chatCore barrel delegates streaming completion directly to the leaf, reducing handleChatCore tail complexity and shrinking the barrel from 2211 to 2011 lines. - Add unit characterization test with defect injection verification. Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 272 ++--------- .../chatCore/streamingSuccessResponse.ts | 435 ++++++++++++++++++ .../streaming-success-response.test.ts | 120 +++++ 3 files changed, 593 insertions(+), 234 deletions(-) create mode 100644 open-sse/handlers/chatCore/streamingSuccessResponse.ts create mode 100644 tests/unit/chatcore/streaming-success-response.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 022b79862f34..edc9619f7d00 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1,3 +1,4 @@ +import { materializeStreamingSuccessResponse } from "./chatCore/streamingSuccessResponse.ts"; import { runRequestPrelude } from "./chatCore/requestPrelude.ts"; import { executeProviderRequest as executeProviderRequestFromLeaf } from "./chatCore/executeProviderRequest.ts"; import { @@ -25,10 +26,6 @@ import { applyContextCompression } from "./chatCore/contextCompression.ts"; import { checkSemanticCache } from "./chatCore/semanticCache.ts"; import { enforceOutputTokenBudget } from "./chatCore/outputTokenBudget.ts"; import { maybeConvertJsonBodyToSse } from "./chatCore/jsonBodyToSse.ts"; -import { assembleStreamingResponseHeaders } from "./chatCore/streamingResponseHeaders.ts"; -import { assembleStreamingPipeline } from "./chatCore/streamingPipeline.ts"; -import { makeOnStreamComplete } from "./chatCore/streamMaterialize.ts"; -import { wrapReadableStreamWithFinalize } from "./chatCore/streamFinalize.ts"; import { sanitizeChatRequestBody } from "./chatCore/sanitization.ts"; import { applyReasoningInputPolicy, @@ -38,11 +35,6 @@ import { import { routingFinishReason } from "./chatCore/routingFinishReason.ts"; import { isNoMemoryRequested } from "./chatCore/headers.ts"; -import { getCodexClientSessionId } from "../config/codexIdentity.ts"; -import { - noteCodexTurnStateProvenance, - readCodexTurnStateHeader, -} from "../config/codexTurnState.ts"; export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comboContextCache.ts"; import { shouldUseNativeCodexPassthrough, @@ -72,21 +64,15 @@ import { resolveMemoryOwnerId } from "./chatCore/memoryExtraction.ts"; import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts"; import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; import { injectSystemPromptPreTranslation } from "../services/systemPrompt.ts"; -import { translateRequest, needsTranslation } from "../translator/index.ts"; +import { translateRequest } from "../translator/index.ts"; import { applyReasoningRuleDirective } from "@/lib/reasoningRouting/policy"; import { withReasoningRuleContext } from "../utils/reasoningRuleContext.ts"; import { FORMATS } from "../translator/formats.ts"; import { sanitizeKiroTools } from "../utils/kiroSanitizer.ts"; import { splitMisplacedToolResults } from "../translator/helpers/claudeHelper.ts"; import { ensureCacheControlOnLastUserMessage } from "../services/claudeCodeConstraints.ts"; -import { - createSSETransformStreamWithLogger, - createPassthroughStreamWithLogger, -} from "../utils/stream.ts"; import { ensureStreamReadiness } from "../utils/streamReadiness.ts"; -import { resolveSuppressThinkClose } from "../utils/thinkCloseMarker.ts"; import { resolveStreamReadinessTimeout } from "../utils/streamReadinessPolicy.ts"; -import { hasActiveClaudeThinking } from "../utils/thinkingBudget.ts"; import { createStreamController } from "../utils/streamHandler.ts"; import * as streamFailure from "../utils/streamFailureFinalization.ts"; import { refreshWithRetry, runWithOnPersist, runWithCasGuard } from "../services/tokenRefresh.ts"; @@ -140,14 +126,9 @@ import { trackPendingRequest, appendRequestLog, saveRequestUsage } from "@/lib/u import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; import { recordCost } from "@/domain/costRules"; import { calculateCost } from "@/lib/usage/costCalculator"; -import { - buildClaudePassthroughToolNameMap, - mergeResponseToolNameMap, -} from "./chatCore/passthroughToolNames.ts"; +import { buildClaudePassthroughToolNameMap } from "./chatCore/passthroughToolNames.ts"; import type { EnforceDecision } from "@/lib/quota/types"; import { writeCompressionAnalytics } from "./chatCore/compressionAnalyticsWrite.ts"; -import { emitRequestGamificationEvent } from "./chatCore/gamificationEvent.ts"; -import { runPluginOnResponseHook } from "./chatCore/pluginOnResponse.ts"; import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; import { getCacheControlSettings } from "@/lib/cacheControlSettings"; @@ -1944,65 +1925,38 @@ export async function handleChatCore({ } providerResponse = streamReadiness.response; - // Notify success - caller can clear error status if needed - if (onRequestSuccess) { - await onRequestSuccess(); - } - - const responseHeaders = assembleStreamingResponseHeaders({ - providerHeaders: providerResponse.headers, + const response = await materializeStreamingSuccessResponse({ + providerResponse, + onRequestSuccess, provider, model, + connectionId, + credentials, pendingRequestId, compressionResponseMeta, comboStrategy, fallbackAttempts, - }); - - // The streaming headers (turn-state included, when present) are committed to - // the client from here on — record which connection minted the blob so a - // later cross-account echo can be stripped (Codex failover guard). The - // in-place failover update means `credentials` is the winning account. - if (provider === "codex" && readCodexTurnStateHeader(providerResponse.headers)) { - noteCodexTurnStateProvenance( - getCodexClientSessionId(clientRawRequest?.headers), - credentials?.connectionId - ); - } - - // Create transform stream with logger for streaming response - let transformStream; - const responseToolNameMap = mergeResponseToolNameMap( + clientRawRequest, toolNameMap, - (finalBody as Record | null | undefined) ?? null - ); - - let streamCompletionRecorded = false; - let streamFailureCompletionRecorded = false; - - // Callback to save call log when stream completes (include responseBody when provided by stream) - const onStreamComplete = makeOnStreamComplete({ + finalBody, + translatedBody, + body, persistAttemptLogs, getCurrentConnectionId, - provider, - model, - credentials, log, + reqLogger, clientResponseFormat, - responseToolNameMap, - finalBody, - translatedBody, - body, + targetFormat, + isResponsesEndpoint, + isDroidCLI, reasoningCacheScope, reasoningReplayHistory, contextEditingEnabled, skillRequestId, streamFailure, - pendingRequestId, startTime, apiKeyInfo, isCombo, - comboStrategy, endpointPath, traceId, calculateCost, @@ -2014,186 +1968,36 @@ export async function handleChatCore({ extractFacts, semanticCacheEnabled, bodyForCacheWrite, - clientRawRequest, claudePromptCacheLogMeta, resolveReportedServiceTier, attachCompressionUsageReceiptAfterAnalytics, routingFinishReason, - getStreamCompletionRecorded: () => streamCompletionRecorded, - setStreamCompletionRecorded: (v) => { - streamCompletionRecorded = v; - }, - getStreamFailureCompletionRecorded: () => streamFailureCompletionRecorded, - setStreamFailureCompletionRecorded: (v) => { - streamFailureCompletionRecorded = v; - }, - getEffectiveServiceTier: () => effectiveServiceTier, - setEffectiveServiceTier: (t) => { - effectiveServiceTier = t; - }, - }); - - - const streamFailureFinalizers = streamFailure.createStreamFailureFinalizers({ - isFailureCompletionRecorded: () => streamFailureCompletionRecorded, - isStreamCompletionRecorded: () => streamCompletionRecorded, - onStreamComplete, + effectiveServiceTier, persistFailureUsage, onStreamFailure, - }); - const handleStreamFailure = streamFailureFinalizers.handleStreamFailure; - onPipelineStreamError = streamFailureFinalizers.onPipelineStreamError; - // #9653: gives a genuine, race-delayed completion a chance to land (see - // createClientDisconnectGraceHandler's doc comment) before persisting a false - // 499/0-tokens for a request that actually delivered its full response. - onClientDisconnectFinalize = streamFailure.createClientDisconnectGraceHandler({ - isStreamCompletionRecorded: () => streamCompletionRecorded, - gracePeriodMs: STREAM_DISCONNECT_GRACE_PERIOD_MS, - finalize: (event) => - handleStreamFailure({ - status: 499, - message: `Client disconnected: ${event.reason}`, - code: "client_disconnected", - type: "client_disconnected", - }), + setOnPipelineStreamError: (fn) => { + onPipelineStreamError = fn; + }, + setOnClientDisconnectFinalize: (fn) => { + onClientDisconnectFinalize = fn; + }, + streamUserAgent, + thinkingMarkerHeader, + copilotCompatibleReasoning, + customToolNames, + requestToolIdentityMap, + streamController, + createPiiTransform, + echoModel, + streamReadinessPolicy, + releaseTurnExecution, }); - // For providers using Responses API format, translate stream back to openai (Chat Completions) format - // UNLESS client is Droid CLI which expects openai-responses format back - const needsResponsesTranslation = - targetFormat === FORMATS.OPENAI_RESPONSES && - clientResponseFormat === FORMATS.OPENAI && - !isResponsesEndpoint && - !isDroidCLI; - const streamStateBody = finalBody || body; - - // Client's explicit thinking intent (Anthropic Messages shape). Claude Code - // sends `{type:"enabled"}` or `{type:"adaptive"}` to opt into relaying - // upstream reasoning_content as Claude thinking blocks; `{type:"disabled"}` - // or an omitted `thinking` field opts out. Kept false for every other - // client schema (OpenAI / Responses), which never express intent through - // `body.thinking`. Mirrors hasActiveClaudeThinking() so the request and - // response sides agree on what counts as "thinking requested" — a prior - // inline `=== "enabled"` check silently suppressed `adaptive` (the intent - // Claude Code actually sends), leaking the mismatch as a broken tool-call - // turn (call log 1787566395384-bab9ab: reasoning dropped → model emitted - // DSML tool-call markers as plain text → incomplete `stop` finish). - const requestedThinking = hasActiveClaudeThinking((body ?? {}) as Record); - - if (needsResponsesTranslation) { - // Provider returns openai-responses, translate to openai (Chat Completions) that clients expect - log?.debug?.("STREAM", `Responses translation mode: openai-responses → openai`); - transformStream = createSSETransformStreamWithLogger( - "openai-responses", - "openai", - provider, - reqLogger, - responseToolNameMap, - model, - connectionId, - streamStateBody, - onStreamComplete, - apiKeyInfo, - handleStreamFailure, - copilotCompatibleReasoning, - false, - requestedThinking, - customToolNames, - // openai-responses → openai translation still wants the namespace identity - // map for #7936-style round-trip closure when the client also speaks - // Responses (Codex CLI). - requestToolIdentityMap - ); - } else if (needsTranslation(targetFormat, clientResponseFormat)) { - // Standard translation for other providers - log?.debug?.("STREAM", `Translation mode: ${targetFormat} → ${clientResponseFormat}`); - transformStream = createSSETransformStreamWithLogger( - targetFormat, - clientResponseFormat, - provider, - reqLogger, - responseToolNameMap, - model, - connectionId, - streamStateBody, - onStreamComplete, - apiKeyInfo, - handleStreamFailure, - copilotCompatibleReasoning, - // Suppress the `` close marker for clients that render it verbatim - // (e.g. OpenCode by UA; any client via `x-omniroute-thinking-marker: off`); - // preserved for Claude Code / Cursor and unknown clients by default (#5245 / - // #5312). Responses API clients always suppress it (structured reasoning - // items make the marker meaningless); otherwise the header wins over the - // UA allowlist. - resolveSuppressThinkClose({ - userAgent: streamUserAgent, - thinkingMarkerHeader, - clientResponseFormat, - }), - requestedThinking, - customToolNames, - requestToolIdentityMap - ); - } else { - log?.debug?.("STREAM", `Standard passthrough mode`); - transformStream = createPassthroughStreamWithLogger( - provider, - reqLogger, - responseToolNameMap, - model, - connectionId, - streamStateBody, - onStreamComplete, - apiKeyInfo, - handleStreamFailure, - clientResponseFormat, - requestToolIdentityMap - ); - } - - const finalStream = assembleStreamingPipeline({ - providerResponse, - transformStream, - streamController, - createPiiTransform, - clientRawRequestHeaders: clientRawRequest?.headers, - clientResponseFormat, - echoModel, - responseHeaders, - // Same adaptive budget the pre-handoff readiness gate above just used — - // reasoning models that legitimately take a while to say anything keep - // that same patience for their first REAL content, not just their first - // lifecycle frame. See pipeWithDisconnect's own doc comment. - contentStallTimeoutMs: streamReadinessPolicy.timeoutMs, - }); - const clientFacingStream = wrapReadableStreamWithFinalize( - finalStream, - releaseTurnExecution - ); - - // ── Gamification event (fire-and-forget) ── - await emitRequestGamificationEvent({ apiKeyId: apiKeyInfo?.id, model, provider }); - - // ── Plugin onResponse hook (fire-and-forget) ── - await runPluginOnResponseHook({ - requestId: traceId, - body, - model, - provider, - apiKeyInfo, - headers: clientRawRequest?.headers, - response: { status: 200, streamed: true }, - }); - - const response = new Response(clientFacingStream, { - headers: responseHeaders, - }); - turnExecutionHandedOffToStream = true; - return { - success: true, - response, - }; + turnExecutionHandedOffToStream = true; + return { + success: true, + response, + }; } finally { if (!turnExecutionHandedOffToStream) { releaseTurnExecution(); diff --git a/open-sse/handlers/chatCore/streamingSuccessResponse.ts b/open-sse/handlers/chatCore/streamingSuccessResponse.ts new file mode 100644 index 000000000000..235c26229cab --- /dev/null +++ b/open-sse/handlers/chatCore/streamingSuccessResponse.ts @@ -0,0 +1,435 @@ +/** + * Streaming success response materialization lifted out of handleChatCore. + * + * Orchestrates: + * 1. Success notification (onRequestSuccess) + * 2. Streaming response header assembly + * 3. Codex turn-state provenance tracking + * 4. Completion and failure finalizers (onStreamComplete, streamFailureFinalizers, disconnect grace handler) + * 5. Transform stream creation (Responses translation, standard translation, or passthrough) + * 6. Streaming pipeline assembly (disconnect pipe, PII transform, progress, heartbeat, model echo) + * 7. Release turn execution wrapping + * 8. Gamification & plugin onResponse hooks (fire-and-forget) + * 9. Constructing and returning the client-facing Response + */ + +import { FORMATS } from "../../translator/formats.ts"; +import { needsTranslation } from "../../translator/index.ts"; +import { mergeResponseToolNameMap } from "./passthroughToolNames.ts"; +import { getCodexClientSessionId } from "../../config/codexIdentity.ts"; +import { + noteCodexTurnStateProvenance, + readCodexTurnStateHeader, +} from "../../config/codexTurnState.ts"; +import { STREAM_DISCONNECT_GRACE_PERIOD_MS } from "../../config/constants.ts"; +import { hasActiveClaudeThinking } from "../../utils/thinkingBudget.ts"; +import { + createSSETransformStreamWithLogger, + createPassthroughStreamWithLogger, +} from "../../utils/stream.ts"; +import { resolveSuppressThinkClose } from "../../utils/thinkCloseMarker.ts"; +import { emitRequestGamificationEvent } from "./gamificationEvent.ts"; +import { assembleStreamingResponseHeaders } from "./streamingResponseHeaders.ts"; +import { makeOnStreamComplete } from "./streamMaterialize.ts"; +import { assembleStreamingPipeline } from "./streamingPipeline.ts"; +import { wrapReadableStreamWithFinalize } from "./streamFinalize.ts"; +import { runPluginOnResponseHook } from "./pluginOnResponse.ts"; +import type { PersistAttemptLogsArgs } from "./attemptLogging.ts"; +import type { EffectiveServiceTier } from "./serviceTier.ts"; +import * as defaultStreamFailure from "../../utils/streamFailureFinalization.ts"; + +export interface StreamingSuccessResponseDeps { + providerResponse: Response; + onRequestSuccess?: (() => Promise | void) | null; + provider: string; + model: string; + connectionId?: string | null; + credentials?: { connectionId?: string | null; providerSpecificData?: unknown } | null; + pendingRequestId: string; + compressionResponseMeta?: string | null; + comboStrategy?: string | null; + fallbackAttempts?: number; + clientRawRequest?: { headers?: Headers | null; signal?: AbortSignal } | null; + toolNameMap?: Map | null; + finalBody?: Record | null; + translatedBody?: Record | null; + body: unknown; + persistAttemptLogs: (args: PersistAttemptLogsArgs) => void; + getCurrentConnectionId: () => string | null; + log?: { + info?: (tag: string, msg: string) => void; + debug?: (tag: string, msg: string) => void; + warn?: (...args: unknown[]) => void; + } | null; + reqLogger?: Parameters[3]; + clientResponseFormat: string; + targetFormat: string; + isResponsesEndpoint: boolean; + isDroidCLI: boolean; + reasoningCacheScope: string | null; + reasoningReplayHistory?: unknown[] | null; + contextEditingEnabled: boolean; + skillRequestId: string; + streamFailure?: typeof defaultStreamFailure; + startTime: number; + apiKeyInfo?: { id?: string | null }; + isCombo: boolean; + endpointPath?: string; + traceId: string; + calculateCost: ( + provider: string, + model: string, + usage: Record | null | undefined, + options: { serviceTier?: string } + ) => Promise; + recordCost: (apiKeyId: string, cost: number) => void; + memoryOwnerId?: string | null; + memorySettings?: { enabled?: boolean | null; maxTokens?: number | null } | null; + videoBridgeObserved: boolean; + pipelineSessionId?: string | null; + extractFacts: (text: string, memoryOwnerId: string, sessionId: string) => void; + semanticCacheEnabled: boolean; + bodyForCacheWrite: unknown; + claudePromptCacheLogMeta?: Record | null; + resolveReportedServiceTier: (payload?: unknown, maxDepth?: number) => EffectiveServiceTier | null; + attachCompressionUsageReceiptAfterAnalytics: ( + usage: Record, + source: "provider" | "estimated" | "stream" + ) => void; + routingFinishReason: (body: unknown) => string | null; + effectiveServiceTier: EffectiveServiceTier; + persistFailureUsage: ( + statusCode: number, + errorCode?: string | null, + aggregate?: unknown + ) => Promise | void; + onStreamFailure?: ((err: unknown) => Promise | void) | null; + setOnPipelineStreamError: (fn: defaultStreamFailure.PipelineStreamErrorHandler | null) => void; + setOnClientDisconnectFinalize: ( + fn: ((event: { reason: string; duration: number }) => boolean) | null + ) => void; + streamUserAgent?: string; + thinkingMarkerHeader?: string | null; + copilotCompatibleReasoning?: boolean; + customToolNames?: ReadonlySet; + requestToolIdentityMap?: Map | null; + streamController: Parameters[0]["streamController"]; + createPiiTransform?: unknown; + echoModel?: string; + streamReadinessPolicy: { timeoutMs: number }; + releaseTurnExecution: () => void; +} + +export async function materializeStreamingSuccessResponse( + deps: StreamingSuccessResponseDeps +): Promise { + const { + providerResponse, + onRequestSuccess, + provider, + model, + connectionId, + credentials, + pendingRequestId, + compressionResponseMeta, + comboStrategy, + fallbackAttempts, + clientRawRequest, + toolNameMap, + finalBody, + translatedBody, + body, + persistAttemptLogs, + getCurrentConnectionId, + log, + reqLogger, + clientResponseFormat, + targetFormat, + isResponsesEndpoint, + isDroidCLI, + reasoningCacheScope, + reasoningReplayHistory, + contextEditingEnabled, + skillRequestId, + streamFailure = defaultStreamFailure, + startTime, + apiKeyInfo, + isCombo, + endpointPath, + traceId, + calculateCost, + recordCost, + memoryOwnerId, + memorySettings, + videoBridgeObserved, + pipelineSessionId, + extractFacts, + semanticCacheEnabled, + bodyForCacheWrite, + claudePromptCacheLogMeta, + resolveReportedServiceTier, + attachCompressionUsageReceiptAfterAnalytics, + routingFinishReason, + effectiveServiceTier: initialServiceTier, + persistFailureUsage, + onStreamFailure, + setOnPipelineStreamError, + setOnClientDisconnectFinalize, + streamUserAgent, + thinkingMarkerHeader, + copilotCompatibleReasoning, + customToolNames, + requestToolIdentityMap, + streamController, + createPiiTransform, + echoModel, + streamReadinessPolicy, + releaseTurnExecution, + } = deps; + + // Notify success - caller can clear error status if needed + if (onRequestSuccess) { + await onRequestSuccess(); + } + + const responseHeaders = assembleStreamingResponseHeaders({ + providerHeaders: providerResponse.headers, + provider, + model, + pendingRequestId, + compressionResponseMeta, + comboStrategy, + fallbackAttempts, + }); + + // The streaming headers (turn-state included, when present) are committed to + // the client from here on — record which connection minted the blob so a + // later cross-account echo can be stripped (Codex failover guard). The + // in-place failover update means `credentials` is the winning account. + if (provider === "codex" && readCodexTurnStateHeader(providerResponse.headers)) { + noteCodexTurnStateProvenance( + getCodexClientSessionId(clientRawRequest?.headers), + credentials?.connectionId + ); + } + + // Create transform stream with logger for streaming response + let transformStream: Parameters[0]["transformStream"]; + const responseToolNameMap = mergeResponseToolNameMap( + toolNameMap, + (finalBody as Record | null | undefined) ?? null + ); + + let streamCompletionRecorded = false; + let streamFailureCompletionRecorded = false; + let currentServiceTier = initialServiceTier; + + // Callback to save call log when stream completes (include responseBody when provided by stream) + const onStreamComplete = makeOnStreamComplete({ + persistAttemptLogs, + getCurrentConnectionId, + provider, + model, + credentials, + log, + clientResponseFormat, + responseToolNameMap, + finalBody, + translatedBody, + body, + reasoningCacheScope, + reasoningReplayHistory, + contextEditingEnabled, + skillRequestId, + streamFailure, + pendingRequestId, + startTime, + apiKeyInfo, + isCombo, + comboStrategy, + endpointPath, + traceId, + calculateCost, + recordCost, + memoryOwnerId, + memorySettings, + videoBridgeObserved, + pipelineSessionId, + extractFacts, + semanticCacheEnabled, + bodyForCacheWrite, + clientRawRequest: clientRawRequest ?? {}, + claudePromptCacheLogMeta, + resolveReportedServiceTier, + attachCompressionUsageReceiptAfterAnalytics, + routingFinishReason, + getStreamCompletionRecorded: () => streamCompletionRecorded, + setStreamCompletionRecorded: (v) => { + streamCompletionRecorded = v; + }, + getStreamFailureCompletionRecorded: () => streamFailureCompletionRecorded, + setStreamFailureCompletionRecorded: (v) => { + streamFailureCompletionRecorded = v; + }, + getEffectiveServiceTier: () => currentServiceTier, + setEffectiveServiceTier: (t) => { + currentServiceTier = t; + }, + }); + + const streamFailureFinalizers = streamFailure.createStreamFailureFinalizers({ + isFailureCompletionRecorded: () => streamFailureCompletionRecorded, + isStreamCompletionRecorded: () => streamCompletionRecorded, + onStreamComplete, + persistFailureUsage, + onStreamFailure, + }); + const handleStreamFailure = streamFailureFinalizers.handleStreamFailure; + setOnPipelineStreamError(streamFailureFinalizers.onPipelineStreamError); + + // #9653: gives a genuine, race-delayed completion a chance to land (see + // createClientDisconnectGraceHandler's doc comment) before persisting a false + // 499/0-tokens for a request that actually delivered its full response. + setOnClientDisconnectFinalize( + streamFailure.createClientDisconnectGraceHandler({ + isStreamCompletionRecorded: () => streamCompletionRecorded, + gracePeriodMs: STREAM_DISCONNECT_GRACE_PERIOD_MS, + finalize: (event) => + handleStreamFailure({ + status: 499, + message: `Client disconnected: ${event.reason}`, + code: "client_disconnected", + type: "client_disconnected", + }), + }) + ); + + // For providers using Responses API format, translate stream back to openai (Chat Completions) format + // UNLESS client is Droid CLI which expects openai-responses format back + const needsResponsesTranslation = + targetFormat === FORMATS.OPENAI_RESPONSES && + clientResponseFormat === FORMATS.OPENAI && + !isResponsesEndpoint && + !isDroidCLI; + const streamStateBody = (finalBody || body) as Record; + + // Client's explicit thinking intent (Anthropic Messages shape). Claude Code + // sends `{type:"enabled"}` or `{type:"adaptive"}` to opt into relaying + // upstream reasoning_content as Claude thinking blocks; `{type:"disabled"}` + // or an omitted `thinking` field opts out. Kept false for every other + // client schema (OpenAI / Responses), which never express intent through + // `body.thinking`. Mirrors hasActiveClaudeThinking() so the request and + // response sides agree on what counts as "thinking requested" — a prior + // inline `=== "enabled"` check silently suppressed `adaptive` (the intent + // Claude Code actually sends), leaking the mismatch as a broken tool-call + // turn (call log 1787566395384-bab9ab: reasoning dropped → model emitted + // DSML tool-call markers as plain text → incomplete `stop` finish). + const requestedThinking = hasActiveClaudeThinking((body ?? {}) as Record); + + if (needsResponsesTranslation) { + // Provider returns openai-responses, translate to openai (Chat Completions) that clients expect + log?.debug?.("STREAM", `Responses translation mode: openai-responses → openai`); + transformStream = createSSETransformStreamWithLogger( + "openai-responses", + "openai", + provider, + reqLogger ?? null, + responseToolNameMap, + model, + connectionId, + streamStateBody, + onStreamComplete, + apiKeyInfo, + handleStreamFailure, + copilotCompatibleReasoning, + false, + requestedThinking, + customToolNames, + // openai-responses → openai translation still wants the namespace identity + // map for #7936-style round-trip closure when the client also speaks + // Responses (Codex CLI). + requestToolIdentityMap + ); + } else if (needsTranslation(targetFormat, clientResponseFormat)) { + // Standard translation for other providers + log?.debug?.("STREAM", `Translation mode: ${targetFormat} → ${clientResponseFormat}`); + transformStream = createSSETransformStreamWithLogger( + targetFormat, + clientResponseFormat, + provider, + reqLogger ?? null, + responseToolNameMap, + model, + connectionId, + streamStateBody, + onStreamComplete, + apiKeyInfo, + handleStreamFailure, + copilotCompatibleReasoning, + // Suppress the `` close marker for clients that render it verbatim + // (e.g. OpenCode by UA; any client via `x-omniroute-thinking-marker: off`); + // preserved for Claude Code / Cursor and unknown clients by default (#5245 / + // #5312). Responses API clients always suppress it (structured reasoning + // items make the marker meaningless); otherwise the header wins over the + // UA allowlist. + resolveSuppressThinkClose({ + userAgent: streamUserAgent, + thinkingMarkerHeader, + clientResponseFormat, + }), + requestedThinking, + customToolNames, + requestToolIdentityMap + ); + } else { + log?.debug?.("STREAM", `Standard passthrough mode`); + transformStream = createPassthroughStreamWithLogger( + provider, + reqLogger ?? null, + responseToolNameMap, + model, + connectionId, + streamStateBody, + onStreamComplete, + apiKeyInfo, + handleStreamFailure, + clientResponseFormat, + requestToolIdentityMap + ); + } + + const finalStream = assembleStreamingPipeline({ + providerResponse, + transformStream, + streamController, + createPiiTransform, + clientRawRequestHeaders: clientRawRequest?.headers, + clientResponseFormat, + echoModel, + responseHeaders, + // Same adaptive budget the pre-handoff readiness gate above just used — + // reasoning models that legitimately take a while to say anything keep + // that same patience for their first REAL content, not just their first + // lifecycle frame. See pipeWithDisconnect's own doc comment. + contentStallTimeoutMs: streamReadinessPolicy.timeoutMs, + }); + const clientFacingStream = wrapReadableStreamWithFinalize(finalStream, releaseTurnExecution); + + // ── Gamification event (fire-and-forget) ── + await emitRequestGamificationEvent({ apiKeyId: apiKeyInfo?.id, model, provider }); + + // ── Plugin onResponse hook (fire-and-forget) ── + await runPluginOnResponseHook({ + requestId: traceId, + body, + model, + provider, + apiKeyInfo, + headers: clientRawRequest?.headers, + response: { status: 200, streamed: true }, + }); + + return new Response(clientFacingStream, { + headers: responseHeaders, + }); +} diff --git a/tests/unit/chatcore/streaming-success-response.test.ts b/tests/unit/chatcore/streaming-success-response.test.ts new file mode 100644 index 000000000000..600a60ce7d6e --- /dev/null +++ b/tests/unit/chatcore/streaming-success-response.test.ts @@ -0,0 +1,120 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const barrelPath = path.resolve(here, "../../../open-sse/handlers/chatCore.ts"); + +test("streamingSuccessResponse exports materializeStreamingSuccessResponse", async () => { + const mod = await import("../../../open-sse/handlers/chatCore/streamingSuccessResponse.ts"); + assert.equal(typeof mod.materializeStreamingSuccessResponse, "function"); +}); + +test("chatCore barrel delegates streaming success materialization", () => { + const src = fs.readFileSync(barrelPath, "utf8"); + assert.equal( + src.includes("materializeStreamingSuccessResponse"), + true, + "chatCore barrel must import and call materializeStreamingSuccessResponse" + ); + assert.equal( + src.includes("const onStreamComplete = makeOnStreamComplete({"), + false, + "makeOnStreamComplete call must move out of chatCore barrel" + ); + assert.equal( + src.includes("const finalStream = assembleStreamingPipeline({"), + false, + "assembleStreamingPipeline call must move out of chatCore barrel" + ); +}); + +test("materializeStreamingSuccessResponse builds streaming pipeline and returns Response", async () => { + const { materializeStreamingSuccessResponse } = + await import("../../../open-sse/handlers/chatCore/streamingSuccessResponse.ts"); + + let successCalled = false; + let releaseCalled = false; + let pipelineErrorHandler: unknown = null; + let clientDisconnectHandler: unknown = null; + + const mockUpstreamBody = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"test":1}\n\n')); + controller.close(); + }, + }); + const mockUpstreamResponse = new Response(mockUpstreamBody, { + status: 200, + headers: new Headers({ "content-type": "text/event-stream" }), + }); + + const { createStreamController } = await import("../../../open-sse/utils/streamHandler.ts"); + const streamController = createStreamController(); + + const res = await materializeStreamingSuccessResponse({ + providerResponse: mockUpstreamResponse, + onRequestSuccess: () => { + successCalled = true; + }, + provider: "openai", + model: "gpt-4o", + pendingRequestId: "req-test-12345", + body: { model: "gpt-4o", messages: [{ role: "user", content: "hi" }] }, + persistAttemptLogs: () => {}, + getCurrentConnectionId: () => "conn-123", + clientResponseFormat: "openai", + targetFormat: "openai", + isResponsesEndpoint: false, + isDroidCLI: false, + reasoningCacheScope: null, + contextEditingEnabled: false, + skillRequestId: "skill-1", + startTime: Date.now(), + isCombo: false, + traceId: "trace-123", + calculateCost: async () => 0, + recordCost: () => {}, + videoBridgeObserved: false, + extractFacts: () => {}, + semanticCacheEnabled: false, + bodyForCacheWrite: null, + resolveReportedServiceTier: () => null, + attachCompressionUsageReceiptAfterAnalytics: () => {}, + routingFinishReason: () => null, + effectiveServiceTier: "standard", + persistFailureUsage: () => {}, + setOnPipelineStreamError: (fn) => { + pipelineErrorHandler = fn; + }, + setOnClientDisconnectFinalize: (fn) => { + clientDisconnectHandler = fn; + }, + streamController, + streamReadinessPolicy: { timeoutMs: 30000 }, + releaseTurnExecution: () => { + releaseCalled = true; + }, + }); + + assert.ok(res instanceof Response, "must return a Response instance"); + assert.equal(res.status, 200); + assert.equal(res.headers.get("x-omniroute-request-id"), "req-test-12345"); + assert.equal(successCalled, true, "onRequestSuccess must be called"); + assert.equal(typeof pipelineErrorHandler, "function", "pipelineErrorHandler must be registered"); + assert.equal( + typeof clientDisconnectHandler, + "function", + "clientDisconnectHandler must be registered" + ); + + // Drain the response stream to trigger releaseTurnExecution + const reader = res.body!.getReader(); + while (true) { + const { done } = await reader.read(); + if (done) break; + } + assert.equal(releaseCalled, true, "releaseTurnExecution must be invoked when stream finishes"); +}); From 94e3110c6f2558a0ea088bcf5d40bc60e76bb1e6 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sat, 19 Sep 2026 10:05:16 -0400 Subject: [PATCH 23/26] refactor(chatcore): lift semantic cache check and context compression into leaf module Lifts semantic cache check (temp=0 early return), reasoning input policy, body sanitization, memory/skills injection, proactive context compression, and output token budget enforcement out of handleChatCore into open-sse/handlers/chatCore/cacheAndCompress.ts. - Replaces inline Phases 9.1 through 9.4 with a typed runCacheAndCompress delegate - Reduces chatCore.ts barrel from 2011 to 1785 lines (-226 lines, cumulative -71.2%) - Passes all 685 unit tests with verified AST/runtime defect injections Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 296 ++---------- .../handlers/chatCore/cacheAndCompress.ts | 431 ++++++++++++++++++ .../unit/chatcore-cache-hit-callsite.test.ts | 79 ++++ .../unit/chatcore/cache-and-compress.test.ts | 64 +++ .../context-compression-carry-out.test.ts | 29 +- 5 files changed, 624 insertions(+), 275 deletions(-) create mode 100644 open-sse/handlers/chatCore/cacheAndCompress.ts create mode 100644 tests/unit/chatcore-cache-hit-callsite.test.ts create mode 100644 tests/unit/chatcore/cache-and-compress.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index edc9619f7d00..fd014c491af8 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1,3 +1,4 @@ +import { runCacheAndCompress } from "./chatCore/cacheAndCompress.ts"; import { materializeStreamingSuccessResponse } from "./chatCore/streamingSuccessResponse.ts"; import { runRequestPrelude } from "./chatCore/requestPrelude.ts"; import { executeProviderRequest as executeProviderRequestFromLeaf } from "./chatCore/executeProviderRequest.ts"; @@ -5,35 +6,21 @@ import { extractRequestToolIdentityMap, resolveResponseToolNameMap, } from "./chatCore/requestToolIdentity.ts"; -import { - injectMemoryAndSkills, - mergeInjectedFallbackOwnerNames, -} from "./chatCore/memorySkillsInjection.ts"; import { normalizeOpenAICompatibleTools, shouldNormalizeFunctionToolsOnly, } from "./chatCore/openAICompatibleTools.ts"; import { buildFailureUsageRecord, type FailureUsageAggregate } from "./chatCore/failureUsage.ts"; import { createTranslationFailureResult } from "./chatCore/translationFailure.ts"; -import { estimateFinalInputTokens } from "./chatCore/contextEstimation.ts"; import { extractSystemRoleMessages, relocateDirectiveOnlyMessages, } from "./chatCore/claudeSystemRole.ts"; export { extractSystemRoleMessages, relocateDirectiveOnlyMessages }; import { acquireTurnExecution, createTurnInProgressResult } from "./chatCore/turnExecutionGuard.ts"; -import { applyContextCompression } from "./chatCore/contextCompression.ts"; -import { checkSemanticCache } from "./chatCore/semanticCache.ts"; -import { enforceOutputTokenBudget } from "./chatCore/outputTokenBudget.ts"; import { maybeConvertJsonBodyToSse } from "./chatCore/jsonBodyToSse.ts"; -import { sanitizeChatRequestBody } from "./chatCore/sanitization.ts"; -import { - applyReasoningInputPolicy, - resolveIncompatibleReasoningAction, -} from "../services/reasoningInputPolicy.ts"; import { routingFinishReason } from "./chatCore/routingFinishReason.ts"; -import { isNoMemoryRequested } from "./chatCore/headers.ts"; export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comboContextCache.ts"; import { @@ -60,7 +47,6 @@ export { buildStreamingResponseHeaders, stripStaleForwardingHeaders, }; -import { resolveMemoryOwnerId } from "./chatCore/memoryExtraction.ts"; import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts"; import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; import { injectSystemPromptPreTranslation } from "../services/systemPrompt.ts"; @@ -85,22 +71,13 @@ import { import { shouldUseMidConversationSystem } from "../executors/claudeIdentity.ts"; import { getUnsupportedParams } from "../config/providerRegistry.ts"; import { checkToolCallingRequiredButUnsupported } from "./chatCore/toolCallingRequiredCheck.ts"; -import { - supportsMaxTokens, - getResolvedModelCapabilities, - getExplicitModelOutputCap, - resolveInputTokenCapForGate, -} from "@/lib/modelCapabilities.ts"; +import { supportsMaxTokens, getResolvedModelCapabilities } from "@/lib/modelCapabilities.ts"; import { checkRequestCapabilityFit, deriveRequestCapabilityRequirements, buildCapabilityMismatchMessage, } from "@/shared/constants/capabilities/capabilityFilter.ts"; -import { - areContextWindowChecksDisabled, - isFeatureFlagEnabled, -} from "@/shared/utils/featureFlags.ts"; -import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts"; +import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts"; import { buildErrorBody, createErrorResult, sanitizeErrorMessage } from "../utils/error.ts"; import { checkTokenLimits } from "@omniroute/open-sse/services/tokenLimitCounter.ts"; import { @@ -108,7 +85,6 @@ import { PROVIDER_MAX_TOKENS, STREAM_READINESS_MAX_TIMEOUT_MS, STREAM_READINESS_TIMEOUT_MS, - DEFAULT_MAX_TOKENS, STREAM_DISCONNECT_GRACE_PERIOD_MS, } from "../config/constants.ts"; import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; @@ -121,7 +97,6 @@ import { type PersistAttemptLogsArgs, } from "./chatCore/attemptLogging.ts"; import { attachCompressionUsageReceiptAfterAnalytics as attachCompressionUsageReceiptAfterAnalyticsFor } from "./chatCore/compressionUsageReceipt.ts"; -import { adaptBodyForCompression } from "../services/compression/bodyAdapter.ts"; import { trackPendingRequest, appendRequestLog, saveRequestUsage } from "@/lib/usageDb"; import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; import { recordCost } from "@/domain/costRules"; @@ -137,8 +112,6 @@ import { shouldPreserveCacheControl, resolveConnectionCacheOverride, } from "../utils/cacheControlPolicy.ts"; -import { compressContext, estimateTokens, getTokenLimit } from "../services/contextManager.ts"; -import type { CompressionResult } from "../services/compression/types.ts"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { extractFacts } from "@/lib/memory/extraction"; import { getClaudeCodeCompatibleRequestDefaults } from "@/lib/providers/requestDefaults"; @@ -362,258 +335,57 @@ export async function handleChatCore({ // Preserve chatCore's canonical formatting while the guarded body remains byte-stable. // prettier-ignore try { - // ── Phase 9.1: Semantic cache check (temp=0, any streaming mode) ── - const cacheHit = await checkSemanticCache({ - semanticCacheEnabled, + const cacheAndCompressResult = await runCacheAndCompress({ body, + semanticCacheEnabled, + stream: !!stream, clientRawRequest, model, + effectiveModel, provider, - stream: !!stream, - reqLogger, effectiveServiceTier, pendingScope, + reqLogger, startTime, log, - persistAttemptLogs: persistAttemptLogsPrelude, - apiKeyId: apiKeyInfo?.id ?? undefined, - cacheDefaultMode: (apiKeyInfo as { cacheDefaultMode?: "legacy" | "bypass" } | null) - ?.cacheDefaultMode, - }); - if (cacheHit) { - return cacheHit; - } - - const reasoningInputFormat = - sourceFormat === FORMATS.OPENAI_RESPONSES - ? "responses" - : sourceFormat === FORMATS.OPENAI - ? "chat" - : null; - if (reasoningInputFormat && body && typeof body === "object") { - const policy = applyReasoningInputPolicy( - body as Record, - reasoningInputFormat, - { - provider, - preserveEncryptedReasoning: - credentials?.providerSpecificData?.preserveEncryptedReasoning === true, - onIncompatibleReasoning: resolveIncompatibleReasoningAction({ - reasoningTransportFallback, - // #11178 regressed combo steps whose combo record carries no explicit - // stepId/executionKey (plain model-list combos): their explicit - // `reasoningTransportFallback: "skip"` config was silently degraded to - // "drop". `isCombo` is the combo marker; step ids are optional - // finer-grained metadata that plain combos never set. - isComboStep: Boolean(isCombo) || Boolean(comboStepId || comboExecutionKey), - headers: clientRawRequest?.headers ?? null, - }), - } - ); - if (policy.incompatibleReasoning) { - trackPendingRequest(model, provider, connectionId, false); - return createErrorResult( - HTTP_STATUS.BAD_REQUEST, - "Reasoning continuation is not compatible with the selected target" - ); - } - } - - body = sanitizeChatRequestBody(body, sourceFormat, targetFormat); - // Per-request opt-out: clients that manage their own context send - // `x-omniroute-no-memory: true` to skip memory+skills injection (a null owner - // disables both branches in injectMemoryAndSkills). See PRD-2026-06-19-no-memory-header. - const memoryOwnerId = isNoMemoryRequested(clientRawRequest?.headers ?? null) - ? null - : resolveMemoryOwnerId(apiKeyInfo as Record | null); - const injectionResult = await injectMemoryAndSkills({ - body, - memoryOwnerId, - provider, - effectiveModel, + persistAttemptLogsPrelude, + apiKeyInfo, sourceFormat, targetFormat, + credentials, + reasoningTransportFallback, + isCombo, + comboStepId, + comboExecutionKey, + connectionId, backgroundReason, - log, - }); - body = injectionResult.body; - const memorySettings = injectionResult.memorySettings; - - // Merge web-search/web-fetch fallback tool names into the builtin owner set. - // injectMemoryAndSkills only tracks memory tools; the fallback names were - // injected into body.tools by prepareWebSearchFallbackBody/prepareWebFetchFallbackBody - // above, so they must be carried into the owner provenance chain here. - const mergedOwnerNames = mergeInjectedFallbackOwnerNames( - injectionResult, - [webSearchFallbackPlan, webFetchFallbackPlan], - preConversionClientToolNames - ); - injectionResult.builtinToolNames = mergedOwnerNames.builtinToolNames; - - // Translate request (pass reqLogger for intermediate logging) - // ── Proactive Context Compression (Phase 4) ── - // Check if context exceeds 70% of limit and compress proactively before sending to provider. - // This prevents "prompt too long" errors for large-but-not-full contexts. - const compressionBody = body - ? adaptBodyForCompression(body as Record).body - : null; - const allMessages = compressionBody?.messages || body?.contents || body?.request?.contents || []; - let cavemanOutputModeApplied = false; - let cavemanOutputModeIntensity: string | null = null; - let preCompressionBody: typeof body | null = null; - let compressionResponseMeta: string | null = null; - // OmniGlyph 1.3.x has native OpenAI Chat/Responses transformers. When the - // inbound protocol differs from the provider wire, defer only that engine to - // the post-translation body; the text engines still run in their legacy lane. - let runPostTranslationCompression: - ((input: Record) => Promise) | null = null; - // Delegated Context Editing (Claude only): captured at the canonical compression - // settings read below, then threaded to executor.execute() further down. Lives at - // function scope because the read happens inside the per-message compression block. - let contextEditingEnabled = false; - // The dashboard's global compression switch must also control the built-in - // reactive and last-resort compaction passes. Otherwise an operator selecting - // "off" still has large histories rewritten by trim_tools/purify_history. - let reactiveContextCompactionEnabled = false; - // Hoisted to function scope (not just the compression-block scope below) so the - // combo-resolved override survives to the final enforceOutputTokenBudget() call - // further down — see #8378 (context limit resolved by the combo was silently - // discarded because it only existed inside this `if` block). - let contextLimit = getTokenLimit(provider, effectiveModel); - const compressionOutcome = await applyContextCompression({ - body, - allMessages, - apiKeyInfo, - clientRawRequest, + webSearchFallbackPlan, + webFetchFallbackPlan, + preConversionClientToolNames, comboName, - connectionId, - credentials, - effectiveModel, - effectiveServiceTier, getCurrentConnectionId, - isCombo, - log, - provider, routingComboId, skillRequestId, - sourceFormat, - targetFormat, traceId, - cavemanOutputModeApplied, - cavemanOutputModeIntensity, - compressionAnalyticsWritePromise, - compressionResponseMeta, - contextEditingEnabled, - contextLimit, nativeCodexPassthrough, - preCompressionBody, - reactiveContextCompactionEnabled, - runPostTranslationCompression, tokensCompressed, + compressionAnalyticsWritePromise, }); - body = compressionOutcome.body; - cavemanOutputModeApplied = compressionOutcome.cavemanOutputModeApplied; - cavemanOutputModeIntensity = compressionOutcome.cavemanOutputModeIntensity; - compressionAnalyticsWritePromise = compressionOutcome.compressionAnalyticsWritePromise; - compressionResponseMeta = compressionOutcome.compressionResponseMeta; - contextEditingEnabled = compressionOutcome.contextEditingEnabled; - contextLimit = compressionOutcome.contextLimit; - preCompressionBody = compressionOutcome.preCompressionBody; - reactiveContextCompactionEnabled = compressionOutcome.reactiveContextCompactionEnabled; - runPostTranslationCompression = compressionOutcome.runPostTranslationCompression; - tokensCompressed = compressionOutcome.tokensCompressed; - - // Re-check the concrete target after all compression passes. Combo compatibility - // filtering is advisory and may preserve an all-incompatible pool; this is the - // hard boundary that prevents a too-large prompt (or a negative token budget) - // from reaching an OpenAI-compatible upstream such as NVIDIA NIM. - let finalEstimatedInputTokens = estimateFinalInputTokens(body as Record); - // Reuse the already-resolved `contextLimit` (may have been narrowed to the - // per-target combo window above, resolveComboContextLimit) instead of a bare - // getTokenLimit(provider, effectiveModel) re-fetch, which would silently - // discard that combo-aware override and re-widen the last-resort budget. - const finalContextLimit = contextLimit; - const toolsReserve = Array.isArray(body?.tools) ? estimateTokens(body.tools) : 0; - - // Last-resort compaction against the concrete input budget (not the 70% threshold). - // Covers cases where the proactive pass was skipped or still left the request oversized (#8560). - if ( - reactiveContextCompactionEnabled && - !nativeCodexPassthrough && - finalEstimatedInputTokens >= finalContextLimit && - body - ) { - const lastResortTarget = Math.max(1, finalContextLimit - toolsReserve - 1); - const lastResortAdapter = adaptBodyForCompression(body as Record); - const lastResortResult = compressContext(lastResortAdapter.body, { - provider, - model: effectiveModel, - maxTokens: lastResortTarget, - reserveTokens: 0, - }); - if (lastResortResult.compressed && lastResortResult.body) { - body = lastResortAdapter.adapted - ? lastResortAdapter.restore(lastResortResult.body as Record, { - dropMissingMappedItems: true, - }) - : lastResortResult.body; - finalEstimatedInputTokens = estimateFinalInputTokens(body as Record); - log?.info?.( - "CONTEXT", - `Last-resort context compaction: ${lastResortResult.stats?.original} → ${lastResortResult.stats?.final} tokens ` + - `(re-estimated input ${finalEstimatedInputTokens}, limit ${finalContextLimit})` - ); - } - } - - const modelOutputCap = toPositiveInteger( - getExplicitModelOutputCap({ provider, model: effectiveModel }) - ); - const contextWindowChecksDisabled = areContextWindowChecksDisabled(); - const outputBudget = enforceOutputTokenBudget( - body as Record, - finalEstimatedInputTokens, - contextWindowChecksDisabled ? Number.MAX_SAFE_INTEGER : finalContextLimit, - targetFormat === FORMATS.CLAUDE && sourceFormat !== FORMATS.CLAUDE ? DEFAULT_MAX_TOKENS : 0, - modelOutputCap, - contextWindowChecksDisabled - ? null - : toPositiveInteger( - resolveInputTokenCapForGate({ provider, model: effectiveModel }, { isCombo }) - ) - ); - if (outputBudget.ok === false) { - const exceededInputCap = outputBudget.maxInputTokens !== undefined; - const message = - `Input exceeds ${exceededInputCap ? "maximum input tokens" : "context window"} for ${provider}/${effectiveModel}: ` + - `estimated ${outputBudget.estimatedInputTokens} input tokens, ${exceededInputCap ? `max input ${outputBudget.maxInputTokens}` : `limit ${outputBudget.contextLimit}`}. ` + - `Reduce the prompt or route to a model with a larger ${exceededInputCap ? "input limit" : "context window"}.`; - log?.warn?.("CONTEXT", message); - trackPendingRequest(model, provider, connectionId, false); - return createErrorResult( - HTTP_STATUS.BAD_REQUEST, - message, - null, - "context_length_exceeded", - "invalid_request_error" - ); - } - if (outputBudget.adjustedFields.length > 0) { - // A field can also be adjusted by *removal* (invalid/non-positive value), which - // the cap did not cause — so state the ceiling in effect rather than claiming - // the cap drove this particular adjustment. - const modelCapIsBinding = - modelOutputCap != null && modelOutputCap < outputBudget.availableOutputTokens; - log?.info?.( - "CONTEXT", - `Adjusted invalid or oversized output token fields (${outputBudget.adjustedFields.join(", ")}); ` + - `${outputBudget.availableOutputTokens} tokens remain for output` + - (modelCapIsBinding - ? ` (output ceiling in effect: ${modelOutputCap}, ${provider}/${effectiveModel}'s own cap)` - : "") - ); + if (cacheAndCompressResult.kind === "return") { + return cacheAndCompressResult.response as Response | Record; } - body = outputBudget.body; + body = cacheAndCompressResult.body; + const memoryOwnerId = cacheAndCompressResult.memoryOwnerId; + const memorySettings = cacheAndCompressResult.memorySettings; + const injectionResult = cacheAndCompressResult.injectionResult; + compressionAnalyticsWritePromise = cacheAndCompressResult.compressionAnalyticsWritePromise; + let compressionResponseMeta = cacheAndCompressResult.compressionResponseMeta; + let contextEditingEnabled = cacheAndCompressResult.contextEditingEnabled; + let preCompressionBody = cacheAndCompressResult.preCompressionBody; + // Re-seat post-translation compression callback onto the caller's local variable. + // This is invoked downstream (around line 780) on the post-translation body. + let runPostTranslationCompression = cacheAndCompressResult.runPostTranslationCompression; + tokensCompressed = cacheAndCompressResult.tokensCompressed; let translatedBody = body; const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE; diff --git a/open-sse/handlers/chatCore/cacheAndCompress.ts b/open-sse/handlers/chatCore/cacheAndCompress.ts new file mode 100644 index 000000000000..9a70d6caf6bb --- /dev/null +++ b/open-sse/handlers/chatCore/cacheAndCompress.ts @@ -0,0 +1,431 @@ +/** + * Context compression and semantic cache checking lifted out of handleChatCore. + * + * Orchestrates: + * 1. Semantic cache check (temp=0, early return on hit) + * 2. Reasoning input policy validation + * 3. Body sanitization & memory/skills injection + * 4. Proactive context compression & last-resort compaction + * 5. Output token budget enforcement + */ + +import { checkSemanticCache } from "./semanticCache.ts"; +import { FORMATS } from "../../translator/formats.ts"; +import { + applyReasoningInputPolicy, + resolveIncompatibleReasoningAction, +} from "../../services/reasoningInputPolicy.ts"; +import { trackPendingRequest } from "@/lib/usageDb"; +import { HTTP_STATUS, DEFAULT_MAX_TOKENS } from "../../config/constants.ts"; +import { createErrorResult } from "../../utils/error.ts"; +import { sanitizeChatRequestBody } from "./sanitization.ts"; +import { isNoMemoryRequested } from "./headers.ts"; +import { resolveMemoryOwnerId } from "./memoryExtraction.ts"; +import { injectMemoryAndSkills, mergeInjectedFallbackOwnerNames } from "./memorySkillsInjection.ts"; +import { adaptBodyForCompression } from "../../services/compression/bodyAdapter.ts"; +import { applyContextCompression } from "./contextCompression.ts"; +import { estimateFinalInputTokens } from "./contextEstimation.ts"; +import { compressContext, estimateTokens, getTokenLimit } from "../../services/contextManager.ts"; +import { toPositiveInteger } from "../../services/reasoningTokenBuffer.ts"; +import { getExplicitModelOutputCap, resolveInputTokenCapForGate } from "@/lib/modelCapabilities"; +import { areContextWindowChecksDisabled } from "@/shared/utils/featureFlags.ts"; +import { enforceOutputTokenBudget } from "./outputTokenBudget.ts"; +import type { CompressionResult } from "../../services/compression/types.ts"; + +export type CacheAndCompressDeps = { + semanticCacheEnabled: boolean; + body: Record | null; + clientRawRequest: Parameters[0]["clientRawRequest"]; + model: string; + provider: string; + stream: boolean; + reqLogger?: Parameters[0]["reqLogger"]; + effectiveServiceTier?: string; + pendingScope?: Parameters[0]["pendingScope"]; + startTime: number; + log?: { + debug?: (...args: unknown[]) => void; + info?: (...args: unknown[]) => void; + warn?: (...args: unknown[]) => void; + error?: (...args: unknown[]) => void; + } | null; + persistAttemptLogsPrelude?: Parameters[0]["persistAttemptLogs"]; + apiKeyInfo?: Record | null; + sourceFormat: string; + targetFormat: string; + credentials?: { + providerSpecificData?: { preserveEncryptedReasoning?: boolean } & Record; + } | null; + reasoningTransportFallback?: string; + isCombo: boolean; + comboStepId?: string | null; + comboExecutionKey?: string | null; + connectionId?: string | null; + effectiveModel: string; + backgroundReason?: unknown; + webSearchFallbackPlan?: { + enabled?: boolean; + convertedToolCount?: number; + toolName?: string; + } | null; + webFetchFallbackPlan?: { + enabled?: boolean; + convertedToolCount?: number; + toolName?: string; + } | null; + preConversionClientToolNames?: string[] | null; + comboName?: string | null; + getCurrentConnectionId?: () => string | null | undefined; + routingComboId?: string | null; + skillRequestId?: string | null; + traceId?: string; + nativeCodexPassthrough?: boolean; + tokensCompressed: number; + compressionAnalyticsWritePromise?: Promise | null; +}; + +export type CacheAndCompressResult = + | { kind: "return"; response: Response | Record } + | { + kind: "continue"; + body: Record | null; + memoryOwnerId: string | null; + memorySettings: unknown; + injectionResult: unknown; + compressionAnalyticsWritePromise: Promise | null; + compressionResponseMeta: string | null; + contextEditingEnabled: boolean; + preCompressionBody: Record | null; + runPostTranslationCompression: + ((input: Record) => Promise) | null; + tokensCompressed: number; + cavemanOutputModeApplied: boolean; + cavemanOutputModeIntensity: string | null; + contextLimit: number | null; + reactiveContextCompactionEnabled: boolean; + }; + +export async function runCacheAndCompress( + deps: CacheAndCompressDeps +): Promise { + const { + semanticCacheEnabled, + clientRawRequest, + model, + provider, + stream, + reqLogger, + effectiveServiceTier, + pendingScope, + startTime, + log, + persistAttemptLogsPrelude, + apiKeyInfo, + sourceFormat, + targetFormat, + credentials, + reasoningTransportFallback, + isCombo, + comboStepId, + comboExecutionKey, + connectionId, + effectiveModel, + backgroundReason, + webSearchFallbackPlan, + webFetchFallbackPlan, + preConversionClientToolNames, + comboName, + getCurrentConnectionId, + routingComboId, + skillRequestId, + traceId, + nativeCodexPassthrough, + } = deps; + + let body = deps.body; + let tokensCompressed = deps.tokensCompressed; + let compressionAnalyticsWritePromise = deps.compressionAnalyticsWritePromise ?? null; + + // -- Phase 9.1: Semantic cache check (temp=0, any streaming mode) -- + const cacheHit = await checkSemanticCache({ + semanticCacheEnabled, + body, + clientRawRequest, + model, + provider, + stream: !!stream, + reqLogger, + effectiveServiceTier, + pendingScope, + startTime, + log, + persistAttemptLogs: persistAttemptLogsPrelude, + apiKeyId: apiKeyInfo?.id ?? undefined, + cacheDefaultMode: (apiKeyInfo as { cacheDefaultMode?: "legacy" | "bypass" } | null) + ?.cacheDefaultMode, + }); + if (cacheHit) { + return { kind: "return", response: cacheHit as Response }; + } + + const reasoningInputFormat = + sourceFormat === FORMATS.OPENAI_RESPONSES + ? "responses" + : sourceFormat === FORMATS.OPENAI + ? "chat" + : null; + if (reasoningInputFormat && body && typeof body === "object") { + const policy = applyReasoningInputPolicy( + body as Record, + reasoningInputFormat, + { + provider, + preserveEncryptedReasoning: + credentials?.providerSpecificData?.preserveEncryptedReasoning === true, + onIncompatibleReasoning: resolveIncompatibleReasoningAction({ + reasoningTransportFallback, + // #11178 regressed combo steps whose combo record carries no explicit + // stepId/executionKey (plain model-list combos): their explicit + // `reasoningTransportFallback: "skip"` config was silently degraded to + // "drop". `isCombo` is the combo marker; step ids are optional + // finer-grained metadata that plain combos never set. + isComboStep: Boolean(isCombo) || Boolean(comboStepId || comboExecutionKey), + headers: clientRawRequest?.headers ?? null, + }), + } + ); + if (policy.incompatibleReasoning) { + trackPendingRequest(model, provider, connectionId, false); + return { + kind: "return", + response: createErrorResult( + HTTP_STATUS.BAD_REQUEST, + "Reasoning continuation is not compatible with the selected target" + ), + }; + } + } + + body = sanitizeChatRequestBody(body, sourceFormat, targetFormat); + // Per-request opt-out: clients that manage their own context send + // `x-omniroute-no-memory: true` to skip memory+skills injection (a null owner + // disables both branches in injectMemoryAndSkills). See PRD-2026-06-19-no-memory-header. + const memoryOwnerId = isNoMemoryRequested(clientRawRequest?.headers ?? null) + ? null + : resolveMemoryOwnerId(apiKeyInfo as Record | null); + const injectionResult = await injectMemoryAndSkills({ + body, + memoryOwnerId, + provider, + effectiveModel, + sourceFormat, + targetFormat, + backgroundReason, + log, + }); + body = injectionResult.body; + const memorySettings = injectionResult.memorySettings; + + // Merge web-search/web-fetch fallback tool names into the builtin owner set. + // injectMemoryAndSkills only tracks memory tools; the fallback names were + // injected into body.tools by prepareWebSearchFallbackBody/prepareWebFetchFallbackBody + // above, so they must be carried into the owner provenance chain here. + const mergedOwnerNames = mergeInjectedFallbackOwnerNames( + injectionResult, + [webSearchFallbackPlan, webFetchFallbackPlan], + preConversionClientToolNames + ); + injectionResult.builtinToolNames = mergedOwnerNames.builtinToolNames; + + // Translate request (pass reqLogger for intermediate logging) + // -- Proactive Context Compression (Phase 4) -- + // Check if context exceeds 70% of limit and compress proactively before sending to provider. + // This prevents "prompt too long" errors for large-but-not-full contexts. + const compressionBody = body + ? adaptBodyForCompression(body as Record).body + : null; + const allMessages = compressionBody?.messages || body?.contents || body?.request?.contents || []; + let cavemanOutputModeApplied = false; + let cavemanOutputModeIntensity: string | null = null; + let preCompressionBody: typeof body | null = null; + let compressionResponseMeta: string | null = null; + // OmniGlyph 1.3.x has native OpenAI Chat/Responses transformers. When the + // inbound protocol differs from the provider wire, defer only that engine to + // the post-translation body; the text engines still run in their legacy lane. + let runPostTranslationCompression: + ((input: Record) => Promise) | null = null; + // Delegated Context Editing (Claude only): captured at the canonical compression + // settings read below, then threaded to executor.execute() further down. Lives at + // function scope because the read happens inside the per-message compression block. + let contextEditingEnabled = false; + // The dashboard's global compression switch must also control the built-in + // reactive and last-resort compaction passes. Otherwise an operator selecting + // "off" still has large histories rewritten by trim_tools/purify_history. + let reactiveContextCompactionEnabled = false; + // Hoisted to function scope (not just the compression-block scope below) so the + // combo-resolved override survives to the final enforceOutputTokenBudget() call + // further down -- see #8378 (context limit resolved by the combo was silently + // discarded because it only existed inside this `if` block). + let contextLimit = getTokenLimit(provider, effectiveModel); + const compressionOutcome = await applyContextCompression({ + body, + allMessages, + apiKeyInfo, + clientRawRequest, + comboName, + connectionId, + credentials, + effectiveModel, + effectiveServiceTier, + getCurrentConnectionId, + isCombo, + log, + provider, + routingComboId, + skillRequestId, + sourceFormat, + targetFormat, + traceId, + cavemanOutputModeApplied, + cavemanOutputModeIntensity, + compressionAnalyticsWritePromise, + compressionResponseMeta, + contextEditingEnabled, + contextLimit, + nativeCodexPassthrough, + preCompressionBody, + reactiveContextCompactionEnabled, + runPostTranslationCompression, + tokensCompressed, + }); + body = compressionOutcome.body; + cavemanOutputModeApplied = compressionOutcome.cavemanOutputModeApplied; + cavemanOutputModeIntensity = compressionOutcome.cavemanOutputModeIntensity; + compressionAnalyticsWritePromise = compressionOutcome.compressionAnalyticsWritePromise; + compressionResponseMeta = compressionOutcome.compressionResponseMeta; + contextEditingEnabled = compressionOutcome.contextEditingEnabled; + contextLimit = compressionOutcome.contextLimit; + preCompressionBody = compressionOutcome.preCompressionBody; + reactiveContextCompactionEnabled = compressionOutcome.reactiveContextCompactionEnabled; + runPostTranslationCompression = compressionOutcome.runPostTranslationCompression; + tokensCompressed = compressionOutcome.tokensCompressed; + + // Re-check the concrete target after all compression passes. Combo compatibility + // filtering is advisory and may preserve an all-incompatible pool; this is the + // hard boundary that prevents a too-large prompt (or a negative token budget) + // from reaching an OpenAI-compatible upstream such as NVIDIA NIM. + let finalEstimatedInputTokens = estimateFinalInputTokens(body as Record); + // Reuse the already-resolved `contextLimit` (may have been narrowed to the + // per-target combo window above, resolveComboContextLimit) instead of a bare + // getTokenLimit(provider, effectiveModel) re-fetch, which would silently + // discard that combo-aware override and re-widen the last-resort budget. + const finalContextLimit = contextLimit; + const toolsReserve = Array.isArray(body?.tools) ? estimateTokens(body.tools) : 0; + + // Last-resort compaction against the concrete input budget (not the 70% threshold). + // Covers cases where the proactive pass was skipped or still left the request oversized (#8560). + if ( + reactiveContextCompactionEnabled && + !nativeCodexPassthrough && + finalEstimatedInputTokens >= finalContextLimit && + body + ) { + const lastResortTarget = Math.max(1, finalContextLimit - toolsReserve - 1); + const lastResortAdapter = adaptBodyForCompression(body as Record); + const lastResortResult = compressContext(lastResortAdapter.body, { + provider, + model: effectiveModel, + maxTokens: lastResortTarget, + reserveTokens: 0, + }); + if (lastResortResult.compressed && lastResortResult.body) { + body = lastResortAdapter.adapted + ? lastResortAdapter.restore(lastResortResult.body as Record, { + dropMissingMappedItems: true, + }) + : lastResortResult.body; + finalEstimatedInputTokens = estimateFinalInputTokens(body as Record); + if (log && typeof log.info === "function") { + log.info( + "CONTEXT", + `Last-resort context compaction: ${lastResortResult.stats?.original} -> ${lastResortResult.stats?.final} tokens ` + + `(re-estimated input ${finalEstimatedInputTokens}, limit ${finalContextLimit})` + ); + } + } + } + + const modelOutputCap = toPositiveInteger( + getExplicitModelOutputCap({ provider, model: effectiveModel }) + ); + const contextWindowChecksDisabled = areContextWindowChecksDisabled(); + const outputBudget = enforceOutputTokenBudget( + body as Record, + finalEstimatedInputTokens, + contextWindowChecksDisabled ? Number.MAX_SAFE_INTEGER : finalContextLimit, + targetFormat === FORMATS.CLAUDE && sourceFormat !== FORMATS.CLAUDE ? DEFAULT_MAX_TOKENS : 0, + modelOutputCap, + contextWindowChecksDisabled + ? null + : toPositiveInteger( + resolveInputTokenCapForGate({ provider, model: effectiveModel }, { isCombo }) + ) + ); + if (outputBudget.ok === false) { + const exceededInputCap = outputBudget.maxInputTokens !== undefined; + const message = + `Input exceeds ${exceededInputCap ? "maximum input tokens" : "context window"} for ${provider}/${effectiveModel}: ` + + `estimated ${outputBudget.estimatedInputTokens} input tokens, ${exceededInputCap ? `max input ${outputBudget.maxInputTokens}` : `limit ${outputBudget.contextLimit}`}. ` + + `Reduce the prompt or route to a model with a larger ${exceededInputCap ? "input limit" : "context window"}.`; + if (log && typeof log.warn === "function") { + log.warn("CONTEXT", message); + } + trackPendingRequest(model, provider, connectionId, false); + return { + kind: "return", + response: createErrorResult( + HTTP_STATUS.BAD_REQUEST, + message, + null, + "context_length_exceeded", + "invalid_request_error" + ), + }; + } + if (outputBudget.adjustedFields.length > 0) { + // A field can also be adjusted by *removal* (invalid/non-positive value), which + // the cap did not cause -- so state the ceiling in effect rather than claiming + // the cap drove this particular adjustment. + const modelCapIsBinding = + modelOutputCap != null && modelOutputCap < outputBudget.availableOutputTokens; + if (log && typeof log.info === "function") { + log.info( + "CONTEXT", + `Adjusted invalid or oversized output token fields (${outputBudget.adjustedFields.join(", ")}); ` + + `${outputBudget.availableOutputTokens} tokens remain for output` + + (modelCapIsBinding + ? ` (output ceiling in effect: ${modelOutputCap}, ${provider}/${effectiveModel}'s own cap)` + : "") + ); + } + } + body = outputBudget.body; + + return { + kind: "continue", + body: body as Record, + memoryOwnerId, + memorySettings, + injectionResult, + compressionAnalyticsWritePromise, + compressionResponseMeta, + contextEditingEnabled, + preCompressionBody, + runPostTranslationCompression, + tokensCompressed, + cavemanOutputModeApplied, + cavemanOutputModeIntensity, + contextLimit, + reactiveContextCompactionEnabled, + }; +} diff --git a/tests/unit/chatcore-cache-hit-callsite.test.ts b/tests/unit/chatcore-cache-hit-callsite.test.ts new file mode 100644 index 000000000000..2e0ab61fa382 --- /dev/null +++ b/tests/unit/chatcore-cache-hit-callsite.test.ts @@ -0,0 +1,79 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-sem-cache-callsite-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts"); +const { generateSignature, setCachedResponse, clearCache } = + await import("../../src/lib/semanticCache.ts"); +const { OMNIROUTE_RESPONSE_HEADERS } = await import("../../src/shared/constants/headers.ts"); +const core = await import("../../src/lib/db/core.ts"); + +test.after(async () => { + await new Promise((r) => setTimeout(r, 100)); + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +function noopLog() { + return { + debug() {}, + info() {}, + warn() {}, + error() {}, + }; +} + +test("handleChatCore semantic cache HIT short-circuits with cache headers", async () => { + clearCache(); + const model = "gpt-4o"; + const messages = [{ role: "user", content: "ping cache hit test" }]; + const temperature = 0; + const signature = generateSignature(model, messages, temperature, undefined, undefined); + const cachedPayload = { + id: "chatcmpl-cached-t0", + choices: [ + { + index: 0, + message: { role: "assistant", content: "pong cached" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 8, completion_tokens: 4, total_tokens: 12 }, + }; + setCachedResponse(signature, model, cachedPayload); + + const originalFetch = globalThis.fetch; + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + throw new Error("upstream fetch must not run on semantic cache HIT"); + }) as typeof fetch; + + try { + const result = await handleChatCore({ + body: { model, messages, temperature, stream: false }, + modelInfo: { provider: "openai", model, extendedContext: false }, + credentials: { apiKey: "test-sk", providerSpecificData: {} }, + log: noopLog(), + clientRawRequest: { + endpoint: "/v1/chat/completions", + headers: new Headers({ accept: "application/json" }), + }, + skipResourcePressureGuard: true, + }); + assert.equal(result.success, true); + assert.equal(fetchCalls, 0, "HIT must not call upstream"); + const res = (result as { response: Response }).response; + assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT"); + assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cacheHit), "true"); + const json = await res.json(); + assert.equal(json.id, cachedPayload.id); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/chatcore/cache-and-compress.test.ts b/tests/unit/chatcore/cache-and-compress.test.ts new file mode 100644 index 000000000000..ad12a289b713 --- /dev/null +++ b/tests/unit/chatcore/cache-and-compress.test.ts @@ -0,0 +1,64 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const barrelPath = path.resolve(here, "../../../open-sse/handlers/chatCore.ts"); +const leafPath = path.resolve(here, "../../../open-sse/handlers/chatCore/cacheAndCompress.ts"); + +test("cacheAndCompress leaf exists", () => { + assert.equal(fs.existsSync(leafPath), true, "cacheAndCompress.ts must exist"); +}); + +test("barrel no longer inlines checkSemanticCache call", () => { + const src = fs.readFileSync(barrelPath, "utf8"); + assert.equal( + src.includes("const cacheHit = await checkSemanticCache({"), + false, + "checkSemanticCache must move out of chatCore.ts barrel" + ); +}); + +test("runCacheAndCompress returns continue with processed body and memory settings", async () => { + const { runCacheAndCompress } = + await import("../../../open-sse/handlers/chatCore/cacheAndCompress.ts"); + const result = await runCacheAndCompress({ + semanticCacheEnabled: false, + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "hello world" }], + }, + clientRawRequest: { headers: {} }, + model: "gpt-4o", + provider: "openai", + stream: false, + startTime: Date.now(), + sourceFormat: "openai", + targetFormat: "openai", + isCombo: false, + effectiveModel: "gpt-4o", + tokensCompressed: 0, + traceId: "test-trace-id", + nativeCodexPassthrough: false, + webSearchFallbackPlan: { enabled: false }, + webFetchFallbackPlan: { enabled: false }, + }); + + assert.equal(result.kind, "continue"); + if (result.kind === "continue") { + assert.ok(result.body); + assert.equal(typeof result.tokensCompressed, "number"); + assert.equal(typeof result.contextEditingEnabled, "boolean"); + assert.ok("preCompressionBody" in result, "preCompressionBody key must exist in result"); + assert.ok( + "compressionResponseMeta" in result, + "compressionResponseMeta key must exist in result" + ); + assert.ok( + "runPostTranslationCompression" in result, + "runPostTranslationCompression key must exist in result" + ); + } +}); diff --git a/tests/unit/chatcore/context-compression-carry-out.test.ts b/tests/unit/chatcore/context-compression-carry-out.test.ts index df6805f8b51f..279d19b02e97 100644 --- a/tests/unit/chatcore/context-compression-carry-out.test.ts +++ b/tests/unit/chatcore/context-compression-carry-out.test.ts @@ -22,7 +22,10 @@ const leafPath = new URL( "../../../open-sse/handlers/chatCore/contextCompression.ts", import.meta.url ); -const barrelPath = new URL("../../../open-sse/handlers/chatCore.ts", import.meta.url); +const parentLeafPath = new URL( + "../../../open-sse/handlers/chatCore/cacheAndCompress.ts", + import.meta.url +); function parse(path: URL) { return ts.createSourceFile( @@ -70,9 +73,9 @@ function returnedFields(): string[] { .sort(); } -/** `x = compressionOutcome.y;` assignments in the barrel, as [target, source] pairs. */ -function barrelWriteBacks(): Array<[string, string]> { - const sf = parse(barrelPath); +/** `x = compressionOutcome.y;` assignments in the parent leaf / caller, as [target, source] pairs. */ +function callerWriteBacks(): Array<[string, string]> { + const sf = parse(parentLeafPath); const pairs: Array<[string, string]> = []; const walk = (node: ts.Node) => { if ( @@ -103,25 +106,25 @@ test("the leaf returns every value it reassigns", () => { assert.ok(fields.includes("compressionResponseMeta")); }); -test("the barrel reseats every field the leaf returns", () => { - const reseated = new Set(barrelWriteBacks().map(([, source]) => source)); +test("the caller reseats every field the leaf returns", () => { + const reseated = new Set(callerWriteBacks().map(([, source]) => source)); const missing = returnedFields().filter((field) => !reseated.has(field)); assert.deepEqual( missing, [], - `the barrel drops these compression results on the floor: ${missing.join(", ")}` + "the caller drops these compression results on the floor: " + missing.join(", ") ); }); -test("the barrel does not reseat anything the leaf never returns", () => { +test("the caller does not reseat anything the leaf never returns", () => { const returned = new Set(returnedFields()); - const pairs = barrelWriteBacks(); - assert.ok(pairs.length > 0, "the barrel must consume the compression result"); + const pairs = callerWriteBacks(); + assert.ok(pairs.length > 0, "the caller must consume the compression result"); for (const [target, source] of pairs) { assert.equal(target, source, "a write-back must not cross-wire two fields"); assert.ok( returned.has(source), - `the barrel reads compressionOutcome.${source}, which the leaf does not return` + `the caller reads compressionOutcome.${source}, which the leaf does not return` ); } }); @@ -136,7 +139,7 @@ test("nativeCodexPassthrough stays an input, never a carry-out", () => { "nativeCodexPassthrough must not be returned" ); assert.ok( - !barrelWriteBacks().some(([, source]) => source === "nativeCodexPassthrough"), - "assigning it would throw: it is a const in the barrel" + !callerWriteBacks().some(([, source]) => source === "nativeCodexPassthrough"), + "assigning it would throw: it is a const in the caller" ); }); From 65512fb90f2db75f6a5f13a08e904340f11e4afa Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sat, 19 Sep 2026 21:53:08 -0400 Subject: [PATCH 24/26] refactor(chatcore): lift request translation and request dedup into leaf module Lifts request translation (format conversion, system prompt injection, tool normalization, message stripping), stream controller setup, executor resolution, credential binding, Claude passthrough handling, and request deduplication hashing out of handleChatCore into open-sse/handlers/chatCore/translateAndDedup.ts. - Replaces inline translation and dedup block with runTranslateAndDedup delegate - Drops chatCore.ts barrel from 1785 to 993 lines (-792 lines, cumulative -84.0%) - Adds tests/unit/chatcore/translate-and-dedup.test.ts verifying structural seams - Passes all 685 unit tests with verified runtime defect injections Signed-off-by: Minxi Hou --- open-sse/handlers/chatCore.ts | 929 ++------------- .../handlers/chatCore/translateAndDedup.ts | 1041 +++++++++++++++++ .../unit/chatcore/translate-and-dedup.test.ts | 114 ++ 3 files changed, 1224 insertions(+), 860 deletions(-) create mode 100644 open-sse/handlers/chatCore/translateAndDedup.ts create mode 100644 tests/unit/chatcore/translate-and-dedup.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index fd014c491af8..afdb1fc65851 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1,17 +1,9 @@ +import { runTranslateAndDedup } from "./chatCore/translateAndDedup.ts"; import { runCacheAndCompress } from "./chatCore/cacheAndCompress.ts"; import { materializeStreamingSuccessResponse } from "./chatCore/streamingSuccessResponse.ts"; import { runRequestPrelude } from "./chatCore/requestPrelude.ts"; import { executeProviderRequest as executeProviderRequestFromLeaf } from "./chatCore/executeProviderRequest.ts"; -import { - extractRequestToolIdentityMap, - resolveResponseToolNameMap, -} from "./chatCore/requestToolIdentity.ts"; -import { - normalizeOpenAICompatibleTools, - shouldNormalizeFunctionToolsOnly, -} from "./chatCore/openAICompatibleTools.ts"; import { buildFailureUsageRecord, type FailureUsageAggregate } from "./chatCore/failureUsage.ts"; -import { createTranslationFailureResult } from "./chatCore/translationFailure.ts"; import { extractSystemRoleMessages, relocateDirectiveOnlyMessages, @@ -26,7 +18,6 @@ export { clearCombosCache, clearUpstreamProxyConfigCache } from "./chatCore/comb import { shouldUseNativeCodexPassthrough, shouldUseNativeXaiResponsesPassthrough, - stampNativeResponsesPassthroughBody, redactPassthroughThinkingSignatures, isClaudeCodeSemanticPassthroughRequest, } from "./chatCore/passthroughHelpers.ts"; @@ -47,80 +38,33 @@ export { buildStreamingResponseHeaders, stripStaleForwardingHeaders, }; -import { stripStore, usesClaudeBridge } from "./chatCore/agentRouterProtocol.ts"; -import { normalizeClaudeToolsForDispatch } from "./chatCore/claudeToolDefaults.ts"; -import { injectSystemPromptPreTranslation } from "../services/systemPrompt.ts"; -import { translateRequest } from "../translator/index.ts"; -import { applyReasoningRuleDirective } from "@/lib/reasoningRouting/policy"; -import { withReasoningRuleContext } from "../utils/reasoningRuleContext.ts"; import { FORMATS } from "../translator/formats.ts"; -import { sanitizeKiroTools } from "../utils/kiroSanitizer.ts"; -import { splitMisplacedToolResults } from "../translator/helpers/claudeHelper.ts"; -import { ensureCacheControlOnLastUserMessage } from "../services/claudeCodeConstraints.ts"; import { ensureStreamReadiness } from "../utils/streamReadiness.ts"; import { resolveStreamReadinessTimeout } from "../utils/streamReadinessPolicy.ts"; -import { createStreamController } from "../utils/streamHandler.ts"; import * as streamFailure from "../utils/streamFailureFinalization.ts"; import { refreshWithRetry, runWithOnPersist, runWithCasGuard } from "../services/tokenRefresh.ts"; -import { applyResponsesPreviousResponseIdPolicy } from "../utils/responsesStatePolicy.ts"; -import { DEFAULT_THINKING_CLAUDE_SIGNATURE } from "../config/defaultThinkingSignature.ts"; -import { - getStripTypesForProviderModel, - stripIncompatibleMessageContent, -} from "../services/modelStrip.ts"; -import { shouldUseMidConversationSystem } from "../executors/claudeIdentity.ts"; -import { getUnsupportedParams } from "../config/providerRegistry.ts"; -import { checkToolCallingRequiredButUnsupported } from "./chatCore/toolCallingRequiredCheck.ts"; -import { supportsMaxTokens, getResolvedModelCapabilities } from "@/lib/modelCapabilities.ts"; -import { - checkRequestCapabilityFit, - deriveRequestCapabilityRequirements, - buildCapabilityMismatchMessage, -} from "@/shared/constants/capabilities/capabilityFilter.ts"; -import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts"; -import { buildErrorBody, createErrorResult, sanitizeErrorMessage } from "../utils/error.ts"; +import { buildErrorBody, createErrorResult } from "../utils/error.ts"; import { checkTokenLimits } from "@omniroute/open-sse/services/tokenLimitCounter.ts"; import { HTTP_STATUS, - PROVIDER_MAX_TOKENS, STREAM_READINESS_MAX_TIMEOUT_MS, STREAM_READINESS_TIMEOUT_MS, - STREAM_DISCONNECT_GRACE_PERIOD_MS, } from "../config/constants.ts"; import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; -import { resolveExecutionCredentials as resolveExecutionCredentialsFor } from "./chatCore/executionCredentials.ts"; -import { resolveExecutorWithProxy as resolveExecutorWithProxyFor } from "./chatCore/executorProxy.ts"; -import type { ClaudeMessage } from "./chatCore/claudeMessageTypes.ts"; -import { normalizeClaudeUpstreamMessages as normalizeClaudeUpstreamMessagesFor } from "./chatCore/claudeUpstreamMessages.ts"; import { persistAttemptLogs as persistAttemptLogsFor, type PersistAttemptLogsArgs, } from "./chatCore/attemptLogging.ts"; import { attachCompressionUsageReceiptAfterAnalytics as attachCompressionUsageReceiptAfterAnalyticsFor } from "./chatCore/compressionUsageReceipt.ts"; import { trackPendingRequest, appendRequestLog, saveRequestUsage } from "@/lib/usageDb"; -import { finalizePendingScope, updatePendingScope } from "@/lib/usage/pendingRequestScope"; +import { updatePendingScope } from "@/lib/usage/pendingRequestScope"; import { recordCost } from "@/domain/costRules"; import { calculateCost } from "@/lib/usage/costCalculator"; -import { buildClaudePassthroughToolNameMap } from "./chatCore/passthroughToolNames.ts"; -import type { EnforceDecision } from "@/lib/quota/types"; -import { writeCompressionAnalytics } from "./chatCore/compressionAnalyticsWrite.ts"; -import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; -import { getCacheControlSettings } from "@/lib/cacheControlSettings"; import type { VideoBridgeLogRedactionEntry } from "@/lib/guardrails/videoBridge"; -import { - shouldPreserveCacheControl, - resolveConnectionCacheOverride, -} from "../utils/cacheControlPolicy.ts"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { extractFacts } from "@/lib/memory/extraction"; -import { getClaudeCodeCompatibleRequestDefaults } from "@/lib/providers/requestDefaults"; -import { - buildClaudeCodeCompatibleRequest, - resolveClaudeCodeCompatibleSessionId, -} from "../services/claudeCodeCompatible.ts"; import { isTpmExhausted } from "../services/geminiRateLimitTracker.ts"; -import { computeRequestHash, shouldDeduplicate } from "../services/requestDedup.ts"; /** * #12150 P1b: shape of handleChatCore's optional `videoBridgeLog` param — see @@ -387,812 +331,77 @@ export async function handleChatCore({ let runPostTranslationCompression = cacheAndCompressResult.runPostTranslationCompression; tokensCompressed = cacheAndCompressResult.tokensCompressed; - let translatedBody = body; - const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE; - const isClaudeCodeCompatible = usesClaudeBridge(provider, targetFormat, credentials); - const isClaudeCodeSemanticPassthrough = isClaudeCodeSemanticPassthroughRequest({ - provider, - sourceFormat, - targetFormat, - headers: clientRawRequest?.headers, - userAgent, - }); - // `forceStream` providers (e.g. Cline / ClinePass) only implement upstream - // streaming — a non-streaming request returns "generateText is not implemented" - // / an empty body. Force the upstream request to stream even when the client - // wants JSON; the non-streaming branch below accumulates the SSE and converts - // it back to JSON (same mechanism already used for Claude-Code-compatible - // providers via isClaudeCodeCompatible). - const upstreamStream = stream || isClaudeCodeCompatible || providerRequiresStreaming; - let ccSessionId: string | null = null; - const stripTypes = getStripTypesForProviderModel(provider || "", model || ""); - - if (Array.isArray(translatedBody?.messages) && stripTypes.length > 0) { - const stripResult = stripIncompatibleMessageContent(translatedBody.messages, stripTypes); - if (stripResult.removedParts > 0) { - translatedBody = { - ...translatedBody, - messages: stripResult.messages, - }; - log?.warn?.( - "CONTENT", - `Stripped ${stripResult.removedParts} incompatible content part(s) for ${provider}/${model}` - ); - } - } - - // Determine if we should preserve client-side cache_control headers - // Fetch settings from DB to get user preference - const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const); - const connectionCacheOverride = resolveConnectionCacheOverride(credentials?.providerSpecificData); - const preserveCacheControl = shouldPreserveCacheControl({ + const translated = await runTranslateAndDedup({ + body, + tokensCompressed, + compressionAnalyticsWritePromise, + compressionResponseMeta, + runPostTranslationCompression, + preCompressionBody, + credentials, + log, + clientRawRequest, + connectionId, + apiKeyInfo, userAgent, - isCombo, + comboName, comboStrategy, - targetProvider: provider, - targetFormat, - settings: { alwaysPreserveClientCache: cacheControlMode }, - connectionCacheOverride, - }); - - if (preserveCacheControl) { - log?.debug?.( - "CACHE", - `Preserving client cache_control (client=${userAgent?.substring(0, 20)}, combo=${isCombo}, strategy=${comboStrategy}, provider=${provider})` - ); - } - - // extractSystemMessagesToBody + normalizeClaudeUpstreamMessages extracted to - // chatCore/claudeUpstreamMessages.ts (#3501); bind `log` once so the call sites stay byte-identical. - const normalizeClaudeUpstreamMessages = ( - payload: Record, - options?: { preserveToolResultBlocks?: boolean } - ) => normalizeClaudeUpstreamMessagesFor(payload, options, log); - - try { - if (nativeResponsesPassthrough) { - translatedBody = stampNativeResponsesPassthroughBody( - applyReasoningRuleDirective(body, "openai-responses") as Record, - nativeCodexPassthrough - ? "codex" - : nativeXaiResponsesPassthrough - ? "xai" - : "openai-compatible" - ); - log?.debug?.( - "FORMAT", - nativeCodexPassthrough - ? "native codex passthrough enabled" - : nativeXaiResponsesPassthrough - ? "native xAI Responses Agent Tools passthrough enabled" - : "native openai-compatible Responses passthrough enabled" - ); - } else if (isClaudeCodeCompatible) { - let normalizedForCc = { ...body }; - - // Claude Code-compatible providers expect Anthropic Messages-shaped payloads, - // but we extract only role/text/max_tokens/effort from an OpenAI-like view first. - if (sourceFormat === FORMATS.CLAUDE && isClaudeCodeSemanticPassthrough) { - normalizedForCc = applyReasoningRuleDirective( - normalizedForCc, - "claude" - ) as typeof normalizedForCc; - log?.debug?.("FORMAT", "claude-code semantic passthrough enabled for compatible bridge"); - } else if (sourceFormat !== FORMATS.OPENAI) { - const normalizeToolCallId = getModelNormalizeToolCallId( - provider || "", - model || "", - sourceFormat - ); - const preserveDeveloperRole = getModelPreserveOpenAIDeveloperRole( - provider || "", - model || "", - sourceFormat - ); - normalizedForCc = translateRequest( - sourceFormat, - FORMATS.OPENAI, - model, - { ...body }, - stream, - credentials, - provider, - reqLogger, - { - normalizeToolCallId, - preserveDeveloperRole, - preserveCacheControl, - copilotClient: copilotCompatibleReasoning, - reasoningCacheScope, - } - ); - } - - ccSessionId = resolveClaudeCodeCompatibleSessionId(clientRawRequest?.headers); - const ccRequestDefaults = getClaudeCodeCompatibleRequestDefaults( - credentials?.providerSpecificData - ); - // OpenAI-shaped bridge requests skip translateRequest too. - if (sourceFormat === FORMATS.OPENAI) { - normalizedForCc = applyReasoningRuleDirective(normalizedForCc) as typeof normalizedForCc; - } - translatedBody = buildClaudeCodeCompatibleRequest({ - sourceBody: body, - normalizedBody: normalizedForCc, - claudeBody: sourceFormat === FORMATS.CLAUDE ? body : null, - model, - stream: upstreamStream, - sessionId: ccSessionId, - cwd: process.cwd(), - now: new Date(), - preserveCacheControl, - preserveClaudeMessages: sourceFormat === FORMATS.CLAUDE && isClaudeCodeSemanticPassthrough, - summarizeThinking: ccRequestDefaults.summarizeThinking === true, - }); - log?.debug?.("FORMAT", "claude-code-compatible bridge enabled"); - - if (isClaudeCodeSemanticPassthrough) { - // Semantic passthrough: only lift system/developer role messages - // without converting file/document blocks, tool history, etc. - extractSystemRoleMessages(translatedBody); - } else { - // Non-CC path: full normalization including content type conversion. - // Preserve tool_result blocks only when the upstream target speaks the - // Anthropic Messages format — OpenAI-compatible gateways reject them - // and return 503. See issue #13971. - normalizeClaudeUpstreamMessages(translatedBody, { - preserveToolResultBlocks: targetFormat === FORMATS.CLAUDE, - }); - } - } else if (isClaudePassthrough) { - // Pure passthrough: forward the body as-is without OpenAI round-trip. - // The Claude→OpenAI→Claude double translation was lossy and corrupted - // payloads at high context (150+ msgs, 100+ tools). Fix: #1359. - // Claude Code sends well-formed Messages API payloads — trust them - // regardless of combo strategy or cache_control settings. - translatedBody = applyReasoningRuleDirective({ ...body }, "claude"); - translatedBody._disableToolPrefix = true; - - // Sanitize historical thinking-block signatures for Anthropic-native Claude OAuth. - // Only Anthropic's first-party API validates these signatures (token-bound); third-party - // Claude-shape providers do not. See redactPassthroughThinkingSignatures + issue #2454. - if (provider === "claude") { - translatedBody.messages = redactPassthroughThinkingSignatures( - translatedBody.messages, - DEFAULT_THINKING_CLAUDE_SIGNATURE - ) as typeof translatedBody.messages; - - // Anthropic API rejects requests with both temperature and top_p. - // VS Code Claude extension and similar clients send both; strip top_p. - if (translatedBody.temperature !== undefined && translatedBody.top_p !== undefined) { - delete translatedBody.top_p; - } - } - - // Legacy models reject role:"system" messages. Supported models accept - // them behind a beta, and hoisting them breaks the prompt cache prefix. - if (isClaudeCodeSemanticPassthrough) { - if ( - provider !== "claude" || - !shouldUseMidConversationSystem(translatedBody, effectiveModel) - ) { - extractSystemRoleMessages(translatedBody); - } else { - // The mid-conversation-system path keeps system-role messages inside - // messages[], but a directive-only message (content: [] + - // output_config) at messages[0] is rejected by Anthropic. Move it past - // the first real turn; Anthropic accepts the form at any other position. - relocateDirectiveOnlyMessages(translatedBody); - } - if (Array.isArray(translatedBody.messages)) { - translatedBody.messages = splitMisplacedToolResults( - translatedBody.messages as ClaudeMessage[] - ) as typeof translatedBody.messages; - } - if (provider === "claude") { - ensureCacheControlOnLastUserMessage(translatedBody); - } - } else { - // Same guard as the CC-bridge path: only preserve tool_result blocks - // for Anthropic-native targets. See issue #13971. This branch only runs - // under isClaudePassthrough (sourceFormat === targetFormat === CLAUDE, - // defined above), so targetFormat === FORMATS.CLAUDE always holds here — - // the guard is a no-op on this call site, kept for symmetry with the - // CC-bridge one above rather than a change to code the issue said not - // to touch. - normalizeClaudeUpstreamMessages(translatedBody, { - preserveToolResultBlocks: targetFormat === FORMATS.CLAUDE, - }); - } - - log?.debug?.("FORMAT", `claude passthrough (preserveCache=${preserveCacheControl})`); - - // Migrate deprecated top-level `output_format` → `output_config.format`. - // Anthropic returns a 400 on the legacy field; some clients (e.g. ForgeCode) - // still emit it. Preserves an existing output_config.format if present. - if (translatedBody.output_format !== undefined) { - const oc = - translatedBody.output_config && typeof translatedBody.output_config === "object" - ? (translatedBody.output_config as Record) - : {}; - if (oc.format === undefined) oc.format = translatedBody.output_format; - translatedBody.output_config = oc; - delete translatedBody.output_format; - } - - // Fix #1719: Strip output_config.format for non-Anthropic Claude-compatible providers. - // Third-party Claude endpoints (MiniMax, DeepSeek via aggregators) reject this field - // with 400 errors since they don't support Anthropic's structured output / json_schema. - if ( - provider !== "claude" && - translatedBody.output_config && - typeof translatedBody.output_config === "object" - ) { - const oc = translatedBody.output_config as Record; - delete oc.format; - if (Object.keys(oc).length === 0) { - delete translatedBody.output_config; - } - } - } else { - translatedBody = { ...body }; - - // Issue #199 + #618: Always disable tool name prefix in Claude passthrough. - // The proxy_ prefix was designed for OpenAI→Claude translation to avoid - // conflicts with Claude OAuth tools, but in the passthrough path the tools - // are already in Claude format. Applying the prefix turns "Bash" into - // "proxy_Bash", which Claude rejects ("No such tool available: proxy_Bash"). - // - // #618's actual traffic was real Claude Code talking to first-party Anthropic - // (provider "claude") reaching this fallback branch instead of the dedicated - // Claude Code bridge/passthrough branches above. Scoping the disable to - // `provider === "claude"` keeps that fix intact while no longer blanket-applying - // it to every other provider that merely targets Claude's wire format — a - // third-party provider's own ordinary (non-Claude-native) tool names, e.g. - // GitHub Copilot's own client-executed "web_fetch" tool, were passing through - // unprefixed here and colliding with Claude's reserved tool namespace, since - // they were never "already in Claude format" the way this comment assumes. - // See #13835. - if (targetFormat === FORMATS.CLAUDE) { - if (provider === "claude") { - translatedBody._disableToolPrefix = true; - } - normalizeClaudeUpstreamMessages(translatedBody); - } - - // OpenAI-compatible providers only support function tools. - // Non-function tool types (computer, mcp, web_search, custom, etc.) are handled: - // - tools with a name → converted to function format in-place before translation - // - tools without a name AND without .function → dropped (unconvertible) - // This must happen before translateRequest, which validates and throws on unknown types. - // Skip normalization when we are in native openai-compatible Responses passthrough mode - // to preserve native tool definitions (exec with lark grammar, collaboration namespace, etc.). - // #13789: built-in providers observed to reject non-function tool types (agentrouter GLM: - // `400 tools[0].type:type is illegal`) are normalized too, via a conservative allowlist - // in shouldNormalizeFunctionToolsOnly that keeps openai's own `custom` tools untouched. - if ( - !nativeOpenAICompatibleResponsesPassthrough && - shouldNormalizeFunctionToolsOnly(provider, targetFormat) && - Array.isArray(translatedBody.tools) - ) { - const normalized = normalizeOpenAICompatibleTools( - translatedBody.tools as Record[], - sourceFormat - ); - translatedBody.tools = normalized.tools; - const { dropped } = normalized; - if (dropped > 0) { - log?.debug?.( - "TOOLS", - `Dropped ${dropped} unconvertible tool(s) for ${provider} (function-tools-only)` - ); - } - } - - const normalizeToolCallId = getModelNormalizeToolCallId( - provider || "", - model || "", - sourceFormat - ); - const preserveDeveloperRole = getModelPreserveOpenAIDeveloperRole( - provider || "", - model || "", - sourceFormat - ); - // Carrier-less targets (kiro / antigravity) have no post-translation - // system carrier for the single pass at ~3068 to write into — inject - // into the client body BEFORE translation so their user-merge / - // relocation paths carry the global prompt (baseline coverage of the - // removed pre-translation pass). The gate writes ONE carrier only. - translatedBody = injectSystemPromptPreTranslation(translatedBody, { targetFormat }); - translatedBody = translateRequest( - sourceFormat, - targetFormat, - model, - translatedBody, - stream, - credentials, - provider, - reqLogger, - { - normalizeToolCallId, - preserveDeveloperRole, - preserveCacheControl, - signatureNamespace: connectionId, - copilotClient: copilotCompatibleReasoning, - reasoningCacheScope, - onReasoningReplayHistory: (messages) => { - reasoningReplayHistory = messages; - }, - ...(preCompressionBody ? { preCompressionBody } : {}), - } - ); - } - } catch (error) { - // ── Plugin onError hook ── - try { - const { runOnError } = await import("@/lib/plugins/hooks"); - await runOnError( - { requestId: traceId, body, model, provider, apiKeyInfo, metadata: {} }, - error instanceof Error ? error : new Error(String(error)) - ); - } catch (pluginErr) { - const pluginErrorMessage = sanitizeErrorMessage(pluginErr) || "Plugin onError hook failed"; - log?.debug?.("PLUGIN", `onError hook error (non-fatal): ${pluginErrorMessage}`); - } - - let parsedStatus = Number.NaN; - try { - parsedStatus = Number(error?.statusCode); - } catch { - // Hostile thrown values may expose Symbols or throwing status accessors. - } - const statusCode = - Number.isInteger(parsedStatus) && parsedStatus >= 400 && parsedStatus <= 599 - ? parsedStatus - : HTTP_STATUS.SERVER_ERROR; - let message = "Invalid request"; - try { - const candidate = error?.message; - message = - (typeof candidate === "string" ? candidate : sanitizeErrorMessage(candidate)) || message; - } catch { - // Hostile thrown values may expose throwing property accessors. - } - let errorType: string | null = null; - try { - const candidate = error?.errorType; - errorType = typeof candidate === "string" ? candidate : null; - } catch { - // Hostile thrown values may expose throwing classification accessors. - } - const result = createTranslationFailureResult(statusCode, message, errorType); - log?.warn?.("TRANSLATE", `Request translation failed: ${result.error}`); - - trackPendingRequest(model, provider, connectionId, false); - return result; - } - - // The latest OmniGlyph release has protocol-native OpenAI transforms. Run - // the deferred stage only after translation so Chat/Responses receives the - // exact provider wire shape (and so a source→target conversion never embeds - // Anthropic image blocks into an OpenAI request, or vice versa). - if (runPostTranslationCompression && translatedBody && typeof translatedBody === "object") { - const transientFields = new Map(); - const postInput = { ...(translatedBody as Record) }; - for (const [key, value] of Object.entries(postInput)) { - // Translators keep response-side aliases in Maps under private keys. They - // are not JSON request fields and would otherwise be stringified to `{}` - // by the OmniGlyph library wrapper; restore them after the wire transform. - if (key.startsWith("_") && value instanceof Map) { - transientFields.set(key, value); - delete postInput[key]; - } - } - try { - const [{ formatCompressionAnnotation }, { trackCompressionStats }] = await Promise.all([ - import("../services/compression/strategySelector.ts"), - import("../services/compression/stats.ts"), - ]); - const postResult = await runPostTranslationCompression(postInput); - if (postResult.compressed) { - translatedBody = { - ...(postResult.body as typeof translatedBody), - ...Object.fromEntries(transientFields), - }; - tokensCompressed += Math.max( - 0, - (postResult.stats?.originalTokens ?? 0) - (postResult.stats?.compressedTokens ?? 0) - ); - if (postResult.stats) { - const annotation = formatCompressionAnnotation(postResult.stats); - if (annotation) { - compressionResponseMeta = compressionResponseMeta - ? `${compressionResponseMeta}; ${annotation}` - : annotation; - } - trackCompressionStats(postResult.stats); - compressionAnalyticsWritePromise = writeCompressionAnalytics({ - stats: postResult.stats, - provider, - effectiveModel, - effectiveServiceTier, - comboName, - mode: postResult.stats.mode, - compressionComboId: postResult.stats.compressionComboId ?? null, - skillRequestId, - cavemanOutputModeApplied: false, - cavemanOutputModeIntensity: null, - log, - }); - await compressionAnalyticsWritePromise; - } - log?.info?.( - "COMPRESSION", - `Post-translation OmniGlyph applied (${sourceFormat} → ${targetFormat})` - ); - } - } catch (error) { - // Compression is deliberately fail-open. A provider-shaped transform - // must never turn an otherwise valid translated request into a 500. - log?.warn?.( - "COMPRESSION", - "Post-translation OmniGlyph skipped: " + - (error instanceof Error ? error.message : String(error)) - ); - } - } - - trace("post_translation"); - - // Keep the request translator's namespace identities separate from toolNameMap: - // the latter is a Kiro/Claude passthrough alias channel with string values, - // while namespace identities carry `{namespace, name}` for the #7936 response - // seam. Extract first because Kiro merge may reuse `_toolNameMap` below. - const requestToolIdentityMap = extractRequestToolIdentityMap(translatedBody); - - // Kiro: sanitize tool schemas before dispatch. Kiro returns 400 "Improperly - // formed request" for unsupported JSON-Schema keywords (anyOf/$ref/if-then, - // etc.) and tool names >64 chars. Strip those keys and hash-truncate long - // names; merge the truncated→original nameMap into the existing - // `_toolNameMap` so kiro-to-openai maps streamed tool-call names back (#1375). - if (targetFormat === FORMATS.KIRO) { - const kiroTools = - translatedBody?.conversationState?.currentMessage?.userInputMessage?.userInputMessageContext - ?.tools; - if (kiroTools) { - const { tools: sanitizedKiroTools, nameMap: kiroNameMap } = sanitizeKiroTools(kiroTools); - translatedBody.conversationState.currentMessage.userInputMessage.userInputMessageContext.tools = - sanitizedKiroTools; - if (kiroNameMap.size > 0) { - const existing = - translatedBody._toolNameMap instanceof Map - ? translatedBody._toolNameMap - : new Map(); - kiroNameMap.forEach((original, truncated) => existing.set(truncated, original)); - translatedBody._toolNameMap = existing; - } - } - } - - // Claude: strict Anthropic-compatible gateways (e.g. MiniMax) reject tool - // definitions that omit the required `type` discriminator with HTTP 400. Default - // a missing `type` to "custom" before dispatch, mirroring Anthropic's own - // inference, so legacy Claude-format tool payloads survive strict gateways (#2195). - // AgentRouter is the opposite quirk: its Rust deserializer only accepts versioned - // tool types and 400s on `type: "custom"` — there the discriminator is stripped - // instead (see claudeToolDefaults.ts). - if (targetFormat === FORMATS.CLAUDE && Array.isArray(translatedBody.tools)) { - translatedBody.tools = normalizeClaudeToolsForDispatch( - translatedBody.tools, - provider - ) as typeof translatedBody.tools; - } - - // Extract toolNameMap for response translation (Claude OAuth) - const translatedToolNameMap = translatedBody._toolNameMap; - const nativeClaudeToolNameMap = isClaudePassthrough - ? buildClaudePassthroughToolNameMap(body) - : null; - // Resolution order matters: `_toolNameMap` was already deleted by - // `extractRequestToolIdentityMap`, so Gemini/Antigravity depend on the - // `requestToolIdentityMap` fallback inside this helper (#9568 / #7936). - const toolNameMap = resolveResponseToolNameMap( - translatedToolNameMap, - nativeClaudeToolNameMap, - requestToolIdentityMap - ); - delete translatedBody._toolNameMap; - delete translatedBody._disableToolPrefix; - - // Update model in body — use resolved alias so the provider gets the correct model ID (#472) - // Strip provider/alias prefix if it exactly matches the routing prefix so upstream receives the raw model name (#1261) - let finalModelToUpstream = effectiveModel; - // Defense-in-depth: only string-strip when effectiveModel is actually a string. - // The API guards `model` via Zod (z.string()), but internal callers could pass a - // non-string and a bare `.startsWith` would crash with `startsWith is not a - // function` (same class as #2359 / #2463). Mirrors 9router's `?.startsWith?.()`. - if (typeof finalModelToUpstream === "string") { - if (finalModelToUpstream.startsWith(`${provider}/`)) { - finalModelToUpstream = finalModelToUpstream.slice(provider.length + 1); - } else if (alias && finalModelToUpstream.startsWith(`${alias}/`)) { - finalModelToUpstream = finalModelToUpstream.slice(alias.length + 1); - } - } - translatedBody.model = finalModelToUpstream; - - const previousResponseIdPolicy = applyResponsesPreviousResponseIdPolicy(translatedBody, { - mode: settings.responsesPreviousResponseIdMode, - provider, - sourceFormat, - targetFormat, - credentials, - }); - translatedBody = previousResponseIdPolicy.body as typeof translatedBody; - - // #1789: Prevent output_config.effort from overriding effort encoded in model name (Codex) - if (provider === "codex" || provider?.startsWith("codex")) { - const hasEffortSuffix = finalModelToUpstream.match(/-(low|medium|high|xhigh)$/i); - if ( - hasEffortSuffix && - translatedBody.output_config && - typeof translatedBody.output_config === "object" - ) { - const oc = translatedBody.output_config as Record; - if (oc.effort) { - log?.warn?.( - "PARAMS", - `Stripped output_config.effort="${oc.effort}" because model "${finalModelToUpstream}" already encodes effort` - ); - delete oc.effort; - if (Object.keys(oc).length === 0) { - delete translatedBody.output_config; - } - } - } - } - - // Strip unsupported parameters for reasoning models (o1, o3, etc.) and any - // provider that can't accept them at all (e.g. AI Horde's raw completion - // backends). When "tools" is among them, also flattens leftover - // tool_calls/tool-result messages in history (from a combo failover away - // from a tool-capable model) — those message shapes break non-tool-calling - // backends just as much as a live `tools` param does. - const unsupported = getUnsupportedParams(provider, model); - - // Direct/pinned requests (isCombo: false) have no other target to fail - // over to. Combo requests are already kept off a tool-incapable target by - // filterTargetsByRequestCompatibility before ever reaching this point, so - // this only fires for the case that filter can't protect: a client - // explicitly asking for this exact model. A clear error beats a 200 that - // silently can't do what was asked (the model narrates a fake tool call - // instead — live incident: AI Horde/Behemoth-X-123B). - const toolCallingCheck = checkToolCallingRequiredButUnsupported( - translatedBody, - unsupported, isCombo, - model - ); - if (toolCallingCheck.blocked) { - trackPendingRequest(model, provider, connectionId, false); - return createErrorResult(400, toolCallingCheck.message!, null, "tool_calling_not_supported"); - } - - // Rename max_tokens to max_completion_tokens if not supported (#1961) - if (!supportsMaxTokens({ provider, model })) { - if (translatedBody.max_tokens !== undefined) { - if (translatedBody.max_completion_tokens === undefined) { - translatedBody.max_completion_tokens = translatedBody.max_tokens; - } - delete translatedBody.max_tokens; - log?.debug?.("PARAMS", `Renamed max_tokens to max_completion_tokens for ${model}`); - } - } else if (translatedBody.max_completion_tokens !== undefined) { - // Symmetric case (#6912): some providers/models (e.g. Volcengine Ark / - // DeepSeek) only document the legacy `max_tokens` field and silently - // ignore an unrecognized `max_completion_tokens`, so a client sending the - // newer field alone would have it dropped upstream with no cap applied. - if (translatedBody.max_tokens === undefined) { - translatedBody.max_tokens = translatedBody.max_completion_tokens; - } - delete translatedBody.max_completion_tokens; - log?.debug?.("PARAMS", `Renamed max_completion_tokens to max_tokens for ${model}`); - } - - stripStore( - translatedBody, - provider, - targetFormat, - credentials?.providerSpecificData as Record | null | undefined - ); - - // Chat clients may send stream_options.include_usage, but OpenAI Responses - // upstreams (including Azure AI Foundry /responses) reject stream_options. - if (targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody) { - delete translatedBody.stream_options; - } - - // Provider-specific max_tokens caps (#711) - // Some providers reject requests when max_tokens exceeds their API limit. - // Cap before sending to avoid upstream HTTP 400 errors. - const providerCap = PROVIDER_MAX_TOKENS[provider]; - if (providerCap) { - for (const field of ["max_tokens", "max_completion_tokens"] as const) { - if (typeof translatedBody[field] === "number" && translatedBody[field] > providerCap) { - log?.debug?.( - "PARAMS", - `Capping ${field} from ${translatedBody[field]} to ${providerCap} for ${provider}` - ); - translatedBody[field] = providerCap; - } - } - } - - // Resolve executor with optional upstream proxy (CLIProxyAPI) routing. - // mode="native" (default): returns the native executor unchanged. - // mode="cliproxyapi": returns the CLIProxyAPI executor instead. - // mode="fallback": returns a wrapper that tries native first, falls back to CLIProxyAPI on 5xx/network errors. - - // #6339: pass the resolved connection's providerSpecificData so a per-connection - // cliproxyapiMode="claude-native" override can deep-route this single connection - // through CLIProxyAPI regardless of the provider-level upstream_proxy_config mode. - const resolveExecutorWithProxy = (prov: string) => - resolveExecutorWithProxyFor( - prov, - log, - (credentials?.providerSpecificData as Record | null | undefined) ?? null - ); - - // === Quota Share enforcement PRE-hook (B/F7) === - // Runs after provider/model/credentials/apiKeyInfo are fully resolved, - // before dispatcher. Fail-open per B16: errors → allow. - let quotaSoftDeprioritize = false; - if (apiKeyInfo?.id && credentials?.connectionId) { - try { - const { enforceQuotaShare } = await import("@/lib/quota/enforce"); - const decision = await enforceQuotaShare({ - apiKeyId: apiKeyInfo.id, - connectionId: credentials.connectionId, - provider: provider ?? "unknown", - // Resolved model id (post background-redirect / alias) — the same scope the - // router/log use. Operators configure per-(key,model) caps against THIS id. - model: model || undefined, - estimatedCost: {}, - }).catch((err: unknown): EnforceDecision => { - log?.warn?.( - "QUOTA_SHARE", - `enforceQuotaShare failed; fail-open: ${err instanceof Error ? err.message : String(err)}` - ); - return { kind: "allow" as const }; - }); - - if (decision.kind === "block") { - const { buildErrorBody } = await import("../utils/error.ts"); - log?.warn?.( - "QUOTA_SHARE", - `[quotaShare] blocked apiKeyId=${apiKeyInfo.id} provider=${provider ?? "unknown"}: ${decision.reason}` - ); - // Finalize the pending-request slot registered at handler entry — this - // return path never reaches the upstream, and without the decrement the - // pending detail lingers as an orphaned status-0 call-log row until the - // reaper sweeps it (mirrors the other pre-upstream error returns). - trackPendingRequest( - model, - provider, - connectionId || credentials?.connectionId || null, - false - ); - const headers: Record = { "Content-Type": "application/json" }; - if (decision.retryAfterSeconds) { - headers["Retry-After"] = String(decision.retryAfterSeconds); - } - return new Response(JSON.stringify(buildErrorBody(429, decision.reason)), { - status: 429, - headers, - }); - } - - if (decision.kind === "allow" && decision.deprioritize) { - quotaSoftDeprioritize = true; - log?.info?.( - "QUOTA_SHARE", - `[quotaShare] soft deprioritize active for apiKeyId=${apiKeyInfo.id} provider=${provider ?? "unknown"}` - ); - } - } catch (err) { - // Outer fail-open guard — should not be reached (inner .catch covers it) - log?.warn?.( - "QUOTA_SHARE", - `[quotaShare] enforceQuotaShare unexpected error; fail-open: ${err instanceof Error ? err.message : String(err)}` - ); - } - } - // G2: Propagate soft penalty to the current candidate so combo scoring can deprioritize. - if (quotaSoftDeprioritize && isCombo && comboStepId) { - try { - const { setCandidateQuotaSoftPenalty } = await import("../services/combo"); - setCandidateQuotaSoftPenalty(comboExecutionKey, comboStepId, true); - } catch (err) { - log?.warn?.( - "QUOTA_SHARE", - `[quotaShare] could not set soft penalty on candidate: ${err instanceof Error ? err.message : String(err)}` - ); - } - } - // === /Quota Share enforcement PRE-hook === - if (isFeatureFlagEnabled("CAPABILITY_FILTER_ENABLED")) { - const fit = checkRequestCapabilityFit( - getResolvedModelCapabilities({ provider, model: effectiveModel }), - deriveRequestCapabilityRequirements(body as Record), - provider - ); - if (!fit.compatible) { - const msg = buildCapabilityMismatchMessage(fit.terminalReason!, provider, effectiveModel); - log?.warn?.("CAPABILITY", msg); - trackPendingRequest(model, provider, connectionId, false); - return createErrorResult(400, msg, null, fit.terminalReason, "invalid_request_error"); - } - } - // Get executor for this provider (with optional upstream proxy routing) - const executor = await resolveExecutorWithProxy(provider); - const getExecutionCredentials = () => - withReasoningRuleContext( - resolveExecutionCredentialsFor({ - credentials, - nativeCodexPassthrough: nativeResponsesPassthrough, - endpointPath, - targetFormat, - provider, - ccSessionId, - modelInfo, - }), - reasoningRuleDirective - ); - - let onPipelineStreamError: streamFailure.PipelineStreamErrorHandler | null = null; - let onClientDisconnectFinalize: - ((event: { reason: string; duration: number }) => boolean) | null = null; - - // Create stream controller for disconnect detection - const streamController = createStreamController({ - onDisconnect: (event) => { - let finalized = false; - try { - finalized = onClientDisconnectFinalize?.(event) === true; - } catch {} - if (!finalized) { - try { - finalizePendingScope(pendingScope, { - status: 499, - error: `Client disconnected: ${event.reason}`, - errorCode: "client_disconnected", - }); - finalized = true; - } catch {} - } - try { - onDisconnect?.(event); - } catch {} - return finalized; - }, - onError: (event) => onPipelineStreamError?.(event), + comboStepId, + comboExecutionKey, provider, model, - connectionId, + modelInfo, + effectiveModel, + alias, + sourceFormat, + targetFormat, + stream, clientResponseFormat, - clientAbortSignal: clientRawRequest?.signal, - allowCompletedToolHandoffGrace: isCodexResponsesEcho, - clientDisconnectGracePeriodMs: STREAM_DISCONNECT_GRACE_PERIOD_MS, + providerRequiresStreaming, + nativeCodexPassthrough, + nativeXaiResponsesPassthrough, + nativeResponsesPassthrough, + nativeOpenAICompatibleResponsesPassthrough, + isCodexResponsesEcho, + copilotCompatibleReasoning, + reasoningCacheScope, + reqLogger, + trace, + traceId, + skillRequestId, + pendingScope, + endpointPath, + effectiveServiceTier, + onDisconnect, + settings, + reasoningRuleDirective, + trustedEffortContext, }); - - const dedupRequestBody = { ...translatedBody, model: `${provider}/${model}`, stream }; - const dedupEnabled = shouldDeduplicate(dedupRequestBody); - // Namespaced by the calling API key: dedup hands the SAME response object to - // every joiner, so a shared hash across keys is a cross-principal response - // leak (GHSA-6c7w-56xp-wpc6). - const dedupHash = dedupEnabled - ? computeRequestHash(dedupRequestBody, apiKeyInfo?.id, trustedEffortContext) - : null; + if (translated.kind === "return") return translated.value; + const c3 = translated.continue; + ({ + tokensCompressed, + compressionAnalyticsWritePromise, + compressionResponseMeta, + reasoningReplayHistory, + } = c3); + let { translatedBody } = c3; + const { + dedupEnabled, + dedupHash, + toolNameMap, + requestToolIdentityMap, + streamController, + executor, + getExecutionCredentials, + upstreamStream, + isClaudeCodeCompatible, + isClaudePassthrough, + preserveCacheControl, + bindPipelineStreamError, + bindClientDisconnectFinalize, + } = c3; const persistAttemptLogs = (args: PersistAttemptLogsArgs) => persistAttemptLogsFor(args, { @@ -1748,10 +957,10 @@ export async function handleChatCore({ persistFailureUsage, onStreamFailure, setOnPipelineStreamError: (fn) => { - onPipelineStreamError = fn; + bindPipelineStreamError(fn); }, setOnClientDisconnectFinalize: (fn) => { - onClientDisconnectFinalize = fn; + bindClientDisconnectFinalize(fn); }, streamUserAgent, thinkingMarkerHeader, diff --git a/open-sse/handlers/chatCore/translateAndDedup.ts b/open-sse/handlers/chatCore/translateAndDedup.ts new file mode 100644 index 000000000000..03d8253bd9ce --- /dev/null +++ b/open-sse/handlers/chatCore/translateAndDedup.ts @@ -0,0 +1,1041 @@ +/** + * Request translation, tool maps, and dedup hash lifted out of handleChatCore. + * Translation / tool / capability / quota early-returns stay here. + */ +import { + extractRequestToolIdentityMap, + resolveResponseToolNameMap, +} from "./requestToolIdentity.ts"; +import { + normalizeOpenAICompatibleTools, + shouldNormalizeFunctionToolsOnly, +} from "./openAICompatibleTools.ts"; +import { createTranslationFailureResult } from "./translationFailure.ts"; +import { extractSystemRoleMessages, relocateDirectiveOnlyMessages } from "./claudeSystemRole.ts"; +import { + stampNativeResponsesPassthroughBody, + redactPassthroughThinkingSignatures, + isClaudeCodeSemanticPassthroughRequest, +} from "./passthroughHelpers.ts"; +import { stripStore, usesClaudeBridge } from "./agentRouterProtocol.ts"; +import { normalizeClaudeToolsForDispatch } from "./claudeToolDefaults.ts"; +import { injectSystemPromptPreTranslation } from "../../services/systemPrompt.ts"; +import { translateRequest } from "../../translator/index.ts"; +import { applyReasoningRuleDirective } from "@/lib/reasoningRouting/policy"; +import { withReasoningRuleContext } from "../../utils/reasoningRuleContext.ts"; +import { FORMATS } from "../../translator/formats.ts"; +import { sanitizeKiroTools } from "../../utils/kiroSanitizer.ts"; +import { splitMisplacedToolResults } from "../../translator/helpers/claudeHelper.ts"; +import { ensureCacheControlOnLastUserMessage } from "../../services/claudeCodeConstraints.ts"; +import { createStreamController } from "../../utils/streamHandler.ts"; +import * as streamFailure from "../../utils/streamFailureFinalization.ts"; +import { applyResponsesPreviousResponseIdPolicy } from "../../utils/responsesStatePolicy.ts"; +import { DEFAULT_THINKING_CLAUDE_SIGNATURE } from "../../config/defaultThinkingSignature.ts"; +import { + getStripTypesForProviderModel, + stripIncompatibleMessageContent, +} from "../../services/modelStrip.ts"; +import { shouldUseMidConversationSystem } from "../../executors/claudeIdentity.ts"; +import { checkToolCallingRequiredButUnsupported } from "./toolCallingRequiredCheck.ts"; +import { getUnsupportedParams } from "../../config/providerRegistry.ts"; +import { buildClaudePassthroughToolNameMap } from "./passthroughToolNames.ts"; +import { createErrorResult, sanitizeErrorMessage } from "../../utils/error.ts"; +import { + HTTP_STATUS, + PROVIDER_MAX_TOKENS, + STREAM_DISCONNECT_GRACE_PERIOD_MS, +} from "../../config/constants.ts"; +import { trackPendingRequest } from "@/lib/usageDb"; +import type { EnforceDecision } from "@/lib/quota/types"; +import { writeCompressionAnalytics } from "./compressionAnalyticsWrite.ts"; +import { getCacheControlSettings } from "@/lib/cacheControlSettings"; +import { + shouldPreserveCacheControl, + resolveConnectionCacheOverride, +} from "../../utils/cacheControlPolicy.ts"; +import { getModelNormalizeToolCallId, getModelPreserveOpenAIDeveloperRole } from "@/lib/db/models"; +import { getClaudeCodeCompatibleRequestDefaults } from "@/lib/providers/requestDefaults"; +import { + buildClaudeCodeCompatibleRequest, + resolveClaudeCodeCompatibleSessionId, +} from "../../services/claudeCodeCompatible.ts"; +import { computeRequestHash, shouldDeduplicate } from "../../services/requestDedup.ts"; +import { finalizePendingScope } from "@/lib/usage/pendingRequestScope"; +import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts"; +import { + checkRequestCapabilityFit, + deriveRequestCapabilityRequirements, + buildCapabilityMismatchMessage, +} from "@/shared/constants/capabilities/capabilityFilter.ts"; +import { supportsMaxTokens, getResolvedModelCapabilities } from "@/lib/modelCapabilities.ts"; +import type { ClaudeMessage } from "./claudeMessageTypes.ts"; +import { normalizeClaudeUpstreamMessages as normalizeClaudeUpstreamMessagesFor } from "./claudeUpstreamMessages.ts"; +import { resolveExecutionCredentials as resolveExecutionCredentialsFor } from "./executionCredentials.ts"; +import { resolveExecutorWithProxy as resolveExecutorWithProxyFor } from "./executorProxy.ts"; + +export interface TranslateAndDedupOptions { + body: Record; + tokensCompressed: number; + compressionAnalyticsWritePromise: Promise | null; + compressionResponseMeta: string | null; + runPostTranslationCompression?: + | ((input: Record) => Promise<{ + compressedBody: Record; + tokensSaved: number; + annotation?: string; + analyticsPromise?: Promise; + }>) + | null; + preCompressionBody: Record; + credentials: Record | null | undefined; + log?: { + debug?: (category: string, message: string) => void; + info?: (category: string, message: string) => void; + warn?: (category: string, message: string) => void; + error?: (category: string, message: string) => void; + } | null; + clientRawRequest?: { + headers?: Headers | Record; + signal?: AbortSignal; + } | null; + connectionId: string | null; + apiKeyInfo: Record | null | undefined; + userAgent?: string | null; + comboName?: string | null; + comboStrategy?: string | null; + isCombo?: boolean; + comboStepId?: string | null; + comboExecutionKey?: string | null; + provider: string; + model: string; + modelInfo?: Record | null; + effectiveModel: string; + alias?: string | null; + sourceFormat: string; + targetFormat: string; + stream?: boolean; + clientResponseFormat?: string | null; + providerRequiresStreaming?: boolean; + nativeCodexPassthrough?: boolean; + nativeXaiResponsesPassthrough?: boolean; + nativeResponsesPassthrough?: boolean; + nativeOpenAICompatibleResponsesPassthrough?: boolean; + isCodexResponsesEcho?: boolean; + copilotCompatibleReasoning?: boolean; + reasoningCacheScope?: string | null; + reqLogger?: unknown; + trace?: ((stage: string) => void) | null; + traceId: string; + skillRequestId?: string | null; + pendingScope?: unknown; + endpointPath?: string; + effectiveServiceTier?: string | null; + onDisconnect?: (event: { reason: string; duration: number }) => void; + settings: Record; + reasoningRuleDirective?: unknown; + trustedEffortContext?: unknown; +} + +export async function runTranslateAndDedup({ + body, + tokensCompressed: tokensCompressedIn, + compressionAnalyticsWritePromise: compressionAnalyticsWritePromiseIn, + compressionResponseMeta: compressionResponseMetaIn, + runPostTranslationCompression, + preCompressionBody, + credentials, + log, + clientRawRequest, + connectionId, + apiKeyInfo, + userAgent, + comboName, + comboStrategy, + isCombo, + comboStepId, + comboExecutionKey, + provider, + model, + modelInfo, + effectiveModel, + alias, + sourceFormat, + targetFormat, + stream, + clientResponseFormat, + providerRequiresStreaming, + nativeCodexPassthrough, + nativeXaiResponsesPassthrough, + nativeResponsesPassthrough, + nativeOpenAICompatibleResponsesPassthrough, + isCodexResponsesEcho, + copilotCompatibleReasoning, + reasoningCacheScope, + reqLogger, + trace, + traceId, + skillRequestId, + pendingScope, + endpointPath, + effectiveServiceTier, + onDisconnect, + settings, + reasoningRuleDirective, + trustedEffortContext, +}: TranslateAndDedupOptions) { + let tokensCompressed = tokensCompressedIn; + let compressionAnalyticsWritePromise = compressionAnalyticsWritePromiseIn; + let compressionResponseMeta = compressionResponseMetaIn; + let reasoningReplayHistory: unknown[] | null = null; + let translatedBody = body; + const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE; + const isClaudeCodeCompatible = usesClaudeBridge(provider, targetFormat, credentials); + const isClaudeCodeSemanticPassthrough = isClaudeCodeSemanticPassthroughRequest({ + provider, + sourceFormat, + targetFormat, + headers: clientRawRequest?.headers, + userAgent, + }); + // `forceStream` providers (e.g. Cline / ClinePass) only implement upstream + // streaming -- a non-streaming request returns "generateText is not implemented" + // / an empty body. Force the upstream request to stream even when the client + // wants JSON; the non-streaming branch below accumulates the SSE and converts + // it back to JSON (same mechanism already used for Claude-Code-compatible + // providers via isClaudeCodeCompatible). + const upstreamStream = stream || isClaudeCodeCompatible || providerRequiresStreaming; + let ccSessionId: string | null = null; + const stripTypes = getStripTypesForProviderModel(provider || "", model || ""); + + if (Array.isArray(translatedBody?.messages) && stripTypes.length > 0) { + const stripResult = stripIncompatibleMessageContent(translatedBody.messages, stripTypes); + if (stripResult.removedParts > 0) { + translatedBody = { + ...translatedBody, + messages: stripResult.messages, + }; + log?.warn?.( + "CONTENT", + `Stripped ${stripResult.removedParts} incompatible content part(s) for ${provider}/${model}` + ); + } + } + + // Determine if we should preserve client-side cache_control headers + // Fetch settings from DB to get user preference + const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const); + const connectionCacheOverride = resolveConnectionCacheOverride(credentials?.providerSpecificData); + const preserveCacheControl = shouldPreserveCacheControl({ + userAgent, + isCombo, + comboStrategy, + targetProvider: provider, + targetFormat, + settings: { alwaysPreserveClientCache: cacheControlMode }, + connectionCacheOverride, + }); + + if (preserveCacheControl) { + log?.debug?.( + "CACHE", + `Preserving client cache_control (client=${userAgent?.substring(0, 20)}, combo=${isCombo}, strategy=${comboStrategy}, provider=${provider})` + ); + } + + // extractSystemMessagesToBody + normalizeClaudeUpstreamMessages extracted to + // chatCore/claudeUpstreamMessages.ts (#3501); bind `log` once so the call sites stay byte-identical. + const normalizeClaudeUpstreamMessages = ( + payload: Record, + options?: { preserveToolResultBlocks?: boolean } + ) => normalizeClaudeUpstreamMessagesFor(payload, options, log); + + try { + if (nativeResponsesPassthrough) { + translatedBody = stampNativeResponsesPassthroughBody( + applyReasoningRuleDirective(body, "openai-responses") as Record, + nativeCodexPassthrough + ? "codex" + : nativeXaiResponsesPassthrough + ? "xai" + : "openai-compatible" + ); + log?.debug?.( + "FORMAT", + nativeCodexPassthrough + ? "native codex passthrough enabled" + : nativeXaiResponsesPassthrough + ? "native xAI Responses Agent Tools passthrough enabled" + : "native openai-compatible Responses passthrough enabled" + ); + } else if (isClaudeCodeCompatible) { + let normalizedForCc = { ...body }; + + // Claude Code-compatible providers expect Anthropic Messages-shaped payloads, + // but we extract only role/text/max_tokens/effort from an OpenAI-like view first. + if (sourceFormat === FORMATS.CLAUDE && isClaudeCodeSemanticPassthrough) { + normalizedForCc = applyReasoningRuleDirective( + normalizedForCc, + "claude" + ) as typeof normalizedForCc; + log?.debug?.("FORMAT", "claude-code semantic passthrough enabled for compatible bridge"); + } else if (sourceFormat !== FORMATS.OPENAI) { + const normalizeToolCallId = getModelNormalizeToolCallId( + provider || "", + model || "", + sourceFormat + ); + const preserveDeveloperRole = getModelPreserveOpenAIDeveloperRole( + provider || "", + model || "", + sourceFormat + ); + normalizedForCc = translateRequest( + sourceFormat, + FORMATS.OPENAI, + model, + { ...body }, + stream, + credentials, + provider, + reqLogger, + { + normalizeToolCallId, + preserveDeveloperRole, + preserveCacheControl, + copilotClient: copilotCompatibleReasoning, + reasoningCacheScope, + } + ); + } + + ccSessionId = resolveClaudeCodeCompatibleSessionId(clientRawRequest?.headers); + const ccRequestDefaults = getClaudeCodeCompatibleRequestDefaults( + credentials?.providerSpecificData + ); + // OpenAI-shaped bridge requests skip translateRequest too. + if (sourceFormat === FORMATS.OPENAI) { + normalizedForCc = applyReasoningRuleDirective(normalizedForCc) as typeof normalizedForCc; + } + translatedBody = buildClaudeCodeCompatibleRequest({ + sourceBody: body, + normalizedBody: normalizedForCc, + claudeBody: sourceFormat === FORMATS.CLAUDE ? body : null, + model, + stream: upstreamStream, + sessionId: ccSessionId, + cwd: process.cwd(), + now: new Date(), + preserveCacheControl, + preserveClaudeMessages: sourceFormat === FORMATS.CLAUDE && isClaudeCodeSemanticPassthrough, + summarizeThinking: ccRequestDefaults.summarizeThinking === true, + }); + log?.debug?.("FORMAT", "claude-code-compatible bridge enabled"); + + if (isClaudeCodeSemanticPassthrough) { + // Semantic passthrough: only lift system/developer role messages + // without converting file/document blocks, tool history, etc. + extractSystemRoleMessages(translatedBody); + } else { + // Non-CC path: full normalization including content type conversion. + // Preserve tool_result blocks only when the upstream target speaks the + // Anthropic Messages format -- OpenAI-compatible gateways reject them + // and return 503. See issue #13971. + normalizeClaudeUpstreamMessages(translatedBody, { + preserveToolResultBlocks: targetFormat === FORMATS.CLAUDE, + }); + } + } else if (isClaudePassthrough) { + // Pure passthrough: forward the body as-is without OpenAI round-trip. + // The Claude->OpenAI->Claude double translation was lossy and corrupted + // payloads at high context (150+ msgs, 100+ tools). Fix: #1359. + // Claude Code sends well-formed Messages API payloads -- trust them + // regardless of combo strategy or cache_control settings. + translatedBody = applyReasoningRuleDirective({ ...body }, "claude"); + translatedBody._disableToolPrefix = true; + + // Sanitize historical thinking-block signatures for Anthropic-native Claude OAuth. + // Only Anthropic's first-party API validates these signatures (token-bound); third-party + // Claude-shape providers do not. See redactPassthroughThinkingSignatures + issue #2454. + if (provider === "claude") { + translatedBody.messages = redactPassthroughThinkingSignatures( + translatedBody.messages, + DEFAULT_THINKING_CLAUDE_SIGNATURE + ) as typeof translatedBody.messages; + + // Anthropic API rejects requests with both temperature and top_p. + // VS Code Claude extension and similar clients send both; strip top_p. + if (translatedBody.temperature !== undefined && translatedBody.top_p !== undefined) { + delete translatedBody.top_p; + } + } + + // Legacy models reject role:"system" messages. Supported models accept + // them behind a beta, and hoisting them breaks the prompt cache prefix. + if (isClaudeCodeSemanticPassthrough) { + if ( + provider !== "claude" || + !shouldUseMidConversationSystem(translatedBody, effectiveModel) + ) { + extractSystemRoleMessages(translatedBody); + } else { + // The mid-conversation-system path keeps system-role messages inside + // messages[], but a directive-only message (content: [] + + // output_config) at messages[0] is rejected by Anthropic. Move it past + // the first real turn; Anthropic accepts the form at any other position. + relocateDirectiveOnlyMessages(translatedBody); + } + if (Array.isArray(translatedBody.messages)) { + translatedBody.messages = splitMisplacedToolResults( + translatedBody.messages as ClaudeMessage[] + ) as typeof translatedBody.messages; + } + if (provider === "claude") { + ensureCacheControlOnLastUserMessage(translatedBody); + } + } else { + // Same guard as the CC-bridge path: only preserve tool_result blocks + // for Anthropic-native targets. See issue #13971. This branch only runs + // under isClaudePassthrough (sourceFormat === targetFormat === CLAUDE, + // defined above), so targetFormat === FORMATS.CLAUDE always holds here -- + // the guard is a no-op on this call site, kept for symmetry with the + // CC-bridge one above rather than a change to code the issue said not + // to touch. + normalizeClaudeUpstreamMessages(translatedBody, { + preserveToolResultBlocks: targetFormat === FORMATS.CLAUDE, + }); + } + + log?.debug?.("FORMAT", `claude passthrough (preserveCache=${preserveCacheControl})`); + + // Migrate deprecated top-level `output_format` -> `output_config.format`. + // Anthropic returns a 400 on the legacy field; some clients (e.g. ForgeCode) + // still emit it. Preserves an existing output_config.format if present. + if (translatedBody.output_format !== undefined) { + const oc = + translatedBody.output_config && typeof translatedBody.output_config === "object" + ? (translatedBody.output_config as Record) + : {}; + if (oc.format === undefined) oc.format = translatedBody.output_format; + translatedBody.output_config = oc; + delete translatedBody.output_format; + } + + // Fix #1719: Strip output_config.format for non-Anthropic Claude-compatible providers. + // Third-party Claude endpoints (MiniMax, DeepSeek via aggregators) reject this field + // with 400 errors since they don't support Anthropic's structured output / json_schema. + if ( + provider !== "claude" && + translatedBody.output_config && + typeof translatedBody.output_config === "object" + ) { + const oc = translatedBody.output_config as Record; + delete oc.format; + if (Object.keys(oc).length === 0) { + delete translatedBody.output_config; + } + } + } else { + translatedBody = { ...body }; + + // Issue #199 + #618: Always disable tool name prefix in Claude passthrough. + // The proxy_ prefix was designed for OpenAI->Claude translation to avoid + // conflicts with Claude OAuth tools, but in the passthrough path the tools + // are already in Claude format. Applying the prefix turns "Bash" into + // "proxy_Bash", which Claude rejects ("No such tool available: proxy_Bash"). + // + // #618's actual traffic was real Claude Code talking to first-party Anthropic + // (provider "claude") reaching this fallback branch instead of the dedicated + // Claude Code bridge/passthrough branches above. Scoping the disable to + // `provider === "claude"` keeps that fix intact while no longer blanket-applying + // it to every other provider that merely targets Claude's wire format -- a + // third-party provider's own ordinary (non-Claude-native) tool names, e.g. + // GitHub Copilot's own client-executed "web_fetch" tool, were passing through + // unprefixed here and colliding with Claude's reserved tool namespace, since + // they were never "already in Claude format" the way this comment assumes. + // See #13835. + if (targetFormat === FORMATS.CLAUDE) { + if (provider === "claude") { + translatedBody._disableToolPrefix = true; + } + normalizeClaudeUpstreamMessages(translatedBody); + } + + // OpenAI-compatible providers only support function tools. + // Non-function tool types (computer, mcp, web_search, custom, etc.) are handled: + // - tools with a name -> converted to function format in-place before translation + // - tools without a name AND without .function -> dropped (unconvertible) + // This must happen before translateRequest, which validates and throws on unknown types. + // Skip normalization when we are in native openai-compatible Responses passthrough mode + // to preserve native tool definitions (exec with lark grammar, collaboration namespace, etc.). + // #13789: built-in providers observed to reject non-function tool types (agentrouter GLM: + // `400 tools[0].type:type is illegal`) are normalized too, via a conservative allowlist + // in shouldNormalizeFunctionToolsOnly that keeps openai's own `custom` tools untouched. + if ( + !nativeOpenAICompatibleResponsesPassthrough && + shouldNormalizeFunctionToolsOnly(provider, targetFormat) && + Array.isArray(translatedBody.tools) + ) { + const normalized = normalizeOpenAICompatibleTools( + translatedBody.tools as Record[], + sourceFormat + ); + translatedBody.tools = normalized.tools; + const { dropped } = normalized; + if (dropped > 0) { + log?.debug?.( + "TOOLS", + `Dropped ${dropped} unconvertible tool(s) for ${provider} (function-tools-only)` + ); + } + } + + const normalizeToolCallId = getModelNormalizeToolCallId( + provider || "", + model || "", + sourceFormat + ); + const preserveDeveloperRole = getModelPreserveOpenAIDeveloperRole( + provider || "", + model || "", + sourceFormat + ); + // Carrier-less targets (kiro / antigravity) have no post-translation + // system carrier for the single pass at ~3068 to write into -- inject + // into the client body BEFORE translation so their user-merge / + // relocation paths carry the global prompt (baseline coverage of the + // removed pre-translation pass). The gate writes ONE carrier only. + translatedBody = injectSystemPromptPreTranslation(translatedBody, { targetFormat }); + translatedBody = translateRequest( + sourceFormat, + targetFormat, + model, + translatedBody, + stream, + credentials, + provider, + reqLogger, + { + normalizeToolCallId, + preserveDeveloperRole, + preserveCacheControl, + signatureNamespace: connectionId, + copilotClient: copilotCompatibleReasoning, + reasoningCacheScope, + onReasoningReplayHistory: (messages) => { + reasoningReplayHistory = messages; + }, + ...(preCompressionBody ? { preCompressionBody } : {}), + } + ); + } + } catch (error) { + // -- Plugin onError hook -- + try { + const { runOnError } = await import("@/lib/plugins/hooks"); + await runOnError( + { requestId: traceId, body, model, provider, apiKeyInfo, metadata: {} }, + error instanceof Error ? error : new Error(String(error)) + ); + } catch (pluginErr) { + const pluginErrorMessage = sanitizeErrorMessage(pluginErr) || "Plugin onError hook failed"; + log?.debug?.("PLUGIN", `onError hook error (non-fatal): ${pluginErrorMessage}`); + } + + let parsedStatus = Number.NaN; + try { + parsedStatus = Number(error?.statusCode); + } catch { + // Hostile thrown values may expose Symbols or throwing status accessors. + } + const statusCode = + Number.isInteger(parsedStatus) && parsedStatus >= 400 && parsedStatus <= 599 + ? parsedStatus + : HTTP_STATUS.SERVER_ERROR; + let message = "Invalid request"; + try { + const candidate = error?.message; + message = + (typeof candidate === "string" ? candidate : sanitizeErrorMessage(candidate)) || message; + } catch { + // Hostile thrown values may expose throwing property accessors. + } + let errorType: string | null = null; + try { + const candidate = error?.errorType; + errorType = typeof candidate === "string" ? candidate : null; + } catch { + // Hostile thrown values may expose throwing classification accessors. + } + const result = createTranslationFailureResult(statusCode, message, errorType); + log?.warn?.("TRANSLATE", `Request translation failed: ${result.error}`); + + trackPendingRequest(model, provider, connectionId, false); + return { kind: "return" as const, value: result }; + } + + // The latest OmniGlyph release has protocol-native OpenAI transforms. Run + // the deferred stage only after translation so Chat/Responses receives the + // exact provider wire shape (and so a source->target conversion never embeds + // Anthropic image blocks into an OpenAI request, or vice versa). + if (runPostTranslationCompression && translatedBody && typeof translatedBody === "object") { + const transientFields = new Map(); + const postInput = { ...(translatedBody as Record) }; + for (const [key, value] of Object.entries(postInput)) { + // Translators keep response-side aliases in Maps under private keys. They + // are not JSON request fields and would otherwise be stringified to `{}` + // by the OmniGlyph library wrapper; restore them after the wire transform. + if (key.startsWith("_") && value instanceof Map) { + transientFields.set(key, value); + delete postInput[key]; + } + } + try { + const [{ formatCompressionAnnotation }, { trackCompressionStats }] = await Promise.all([ + import("../services/compression/strategySelector.ts"), + import("../services/compression/stats.ts"), + ]); + const postResult = await runPostTranslationCompression(postInput); + if (postResult.compressed) { + translatedBody = { + ...(postResult.body as typeof translatedBody), + ...Object.fromEntries(transientFields), + }; + tokensCompressed += Math.max( + 0, + (postResult.stats?.originalTokens ?? 0) - (postResult.stats?.compressedTokens ?? 0) + ); + if (postResult.stats) { + const annotation = formatCompressionAnnotation(postResult.stats); + if (annotation) { + compressionResponseMeta = compressionResponseMeta + ? `${compressionResponseMeta}; ${annotation}` + : annotation; + } + trackCompressionStats(postResult.stats); + compressionAnalyticsWritePromise = writeCompressionAnalytics({ + stats: postResult.stats, + provider, + effectiveModel, + effectiveServiceTier, + comboName, + mode: postResult.stats.mode, + compressionComboId: postResult.stats.compressionComboId ?? null, + skillRequestId, + cavemanOutputModeApplied: false, + cavemanOutputModeIntensity: null, + log, + }); + await compressionAnalyticsWritePromise; + } + log?.info?.( + "COMPRESSION", + `Post-translation OmniGlyph applied (${sourceFormat} -> ${targetFormat})` + ); + } + } catch (error) { + // Compression is deliberately fail-open. A provider-shaped transform + // must never turn an otherwise valid translated request into a 500. + log?.warn?.( + "COMPRESSION", + "Post-translation OmniGlyph skipped: " + + (error instanceof Error ? error.message : String(error)) + ); + } + } + + trace?.("post_translation"); + + // Keep the request translator's namespace identities separate from toolNameMap: + // the latter is a Kiro/Claude passthrough alias channel with string values, + // while namespace identities carry `{namespace, name}` for the #7936 response + // seam. Extract first because Kiro merge may reuse `_toolNameMap` below. + const requestToolIdentityMap = extractRequestToolIdentityMap(translatedBody); + + // Kiro: sanitize tool schemas before dispatch. Kiro returns 400 "Improperly + // formed request" for unsupported JSON-Schema keywords (anyOf/$ref/if-then, + // etc.) and tool names >64 chars. Strip those keys and hash-truncate long + // names; merge the truncated->original nameMap into the existing + // `_toolNameMap` so kiro-to-openai maps streamed tool-call names back (#1375). + if (targetFormat === FORMATS.KIRO) { + const kiroTools = + translatedBody?.conversationState?.currentMessage?.userInputMessage?.userInputMessageContext + ?.tools; + if (kiroTools) { + const { tools: sanitizedKiroTools, nameMap: kiroNameMap } = sanitizeKiroTools(kiroTools); + translatedBody.conversationState.currentMessage.userInputMessage.userInputMessageContext.tools = + sanitizedKiroTools; + if (kiroNameMap.size > 0) { + const existing = + translatedBody._toolNameMap instanceof Map + ? translatedBody._toolNameMap + : new Map(); + kiroNameMap.forEach((original, truncated) => existing.set(truncated, original)); + translatedBody._toolNameMap = existing; + } + } + } + + // Claude: strict Anthropic-compatible gateways (e.g. MiniMax) reject tool + // definitions that omit the required `type` discriminator with HTTP 400. Default + // a missing `type` to "custom" before dispatch, mirroring Anthropic's own + // inference, so legacy Claude-format tool payloads survive strict gateways (#2195). + // AgentRouter is the opposite quirk: its Rust deserializer only accepts versioned + // tool types and 400s on `type: "custom"` -- there the discriminator is stripped + // instead (see claudeToolDefaults.ts). + if (targetFormat === FORMATS.CLAUDE && Array.isArray(translatedBody.tools)) { + translatedBody.tools = normalizeClaudeToolsForDispatch( + translatedBody.tools, + provider + ) as typeof translatedBody.tools; + } + + // Extract toolNameMap for response translation (Claude OAuth) + const translatedToolNameMap = translatedBody._toolNameMap; + const nativeClaudeToolNameMap = isClaudePassthrough + ? buildClaudePassthroughToolNameMap(body) + : null; + // Resolution order matters: `_toolNameMap` was already deleted by + // `extractRequestToolIdentityMap`, so Gemini/Antigravity depend on the + // `requestToolIdentityMap` fallback inside this helper (#9568 / #7936). + const toolNameMap = resolveResponseToolNameMap( + translatedToolNameMap, + nativeClaudeToolNameMap, + requestToolIdentityMap + ); + delete translatedBody._toolNameMap; + delete translatedBody._disableToolPrefix; + + // Update model in body -- use resolved alias so the provider gets the correct model ID (#472) + // Strip provider/alias prefix if it exactly matches the routing prefix so upstream receives the raw model name (#1261) + let finalModelToUpstream = effectiveModel; + // Defense-in-depth: only string-strip when effectiveModel is actually a string. + // The API guards `model` via Zod (z.string()), but internal callers could pass a + // non-string and a bare `.startsWith` would crash with `startsWith is not a + // function` (same class as #2359 / #2463). Mirrors 9router's `?.startsWith?.()`. + if (typeof finalModelToUpstream === "string") { + if (finalModelToUpstream.startsWith(`${provider}/`)) { + finalModelToUpstream = finalModelToUpstream.slice(provider.length + 1); + } else if (alias && finalModelToUpstream.startsWith(`${alias}/`)) { + finalModelToUpstream = finalModelToUpstream.slice(alias.length + 1); + } + } + translatedBody.model = finalModelToUpstream; + + const previousResponseIdPolicy = applyResponsesPreviousResponseIdPolicy(translatedBody, { + mode: settings.responsesPreviousResponseIdMode, + provider, + sourceFormat, + targetFormat, + credentials, + }); + translatedBody = previousResponseIdPolicy.body as typeof translatedBody; + + // #1789: Prevent output_config.effort from overriding effort encoded in model name (Codex) + if (provider === "codex" || provider?.startsWith("codex")) { + const hasEffortSuffix = finalModelToUpstream.match(/-(low|medium|high|xhigh)$/i); + if ( + hasEffortSuffix && + translatedBody.output_config && + typeof translatedBody.output_config === "object" + ) { + const oc = translatedBody.output_config as Record; + if (oc.effort) { + log?.warn?.( + "PARAMS", + `Stripped output_config.effort="${oc.effort}" because model "${finalModelToUpstream}" already encodes effort` + ); + delete oc.effort; + if (Object.keys(oc).length === 0) { + delete translatedBody.output_config; + } + } + } + } + + // Strip unsupported parameters for reasoning models (o1, o3, etc.) and any + // provider that can't accept them at all (e.g. AI Horde's raw completion + // backends). When "tools" is among them, also flattens leftover + // tool_calls/tool-result messages in history (from a combo failover away + // from a tool-capable model) -- those message shapes break non-tool-calling + // backends just as much as a live `tools` param does. + const unsupported = getUnsupportedParams(provider, model); + + // Direct/pinned requests (isCombo: false) have no other target to fail + // over to. Combo requests are already kept off a tool-incapable target by + // filterTargetsByRequestCompatibility before ever reaching this point, so + // this only fires for the case that filter can't protect: a client + // explicitly asking for this exact model. A clear error beats a 200 that + // silently can't do what was asked (the model narrates a fake tool call + // instead -- live incident: AI Horde/Behemoth-X-123B). + const toolCallingCheck = checkToolCallingRequiredButUnsupported( + translatedBody, + unsupported, + isCombo, + model + ); + if (toolCallingCheck.blocked) { + trackPendingRequest(model, provider, connectionId, false); + return { + kind: "return" as const, + value: createErrorResult(400, toolCallingCheck.message!, null, "tool_calling_not_supported"), + }; + } + + // Rename max_tokens to max_completion_tokens if not supported (#1961) + if (!supportsMaxTokens({ provider, model })) { + if (translatedBody.max_tokens !== undefined) { + if (translatedBody.max_completion_tokens === undefined) { + translatedBody.max_completion_tokens = translatedBody.max_tokens; + } + delete translatedBody.max_tokens; + log?.debug?.("PARAMS", `Renamed max_tokens to max_completion_tokens for ${model}`); + } + } else if (translatedBody.max_completion_tokens !== undefined) { + // Symmetric case (#6912): some providers/models (e.g. Volcengine Ark / + // DeepSeek) only document the legacy `max_tokens` field and silently + // ignore an unrecognized `max_completion_tokens`, so a client sending the + // newer field alone would have it dropped upstream with no cap applied. + if (translatedBody.max_tokens === undefined) { + translatedBody.max_tokens = translatedBody.max_completion_tokens; + } + delete translatedBody.max_completion_tokens; + log?.debug?.("PARAMS", `Renamed max_completion_tokens to max_tokens for ${model}`); + } + + stripStore( + translatedBody, + provider, + targetFormat, + credentials?.providerSpecificData as Record | null | undefined + ); + + // Chat clients may send stream_options.include_usage, but OpenAI Responses + // upstreams (including Azure AI Foundry /responses) reject stream_options. + if (targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody) { + delete translatedBody.stream_options; + } + + // Provider-specific max_tokens caps (#711) + // Some providers reject requests when max_tokens exceeds their API limit. + // Cap before sending to avoid upstream HTTP 400 errors. + const providerCap = PROVIDER_MAX_TOKENS[provider]; + if (providerCap) { + for (const field of ["max_tokens", "max_completion_tokens"] as const) { + if (typeof translatedBody[field] === "number" && translatedBody[field] > providerCap) { + log?.debug?.( + "PARAMS", + `Capping ${field} from ${translatedBody[field]} to ${providerCap} for ${provider}` + ); + translatedBody[field] = providerCap; + } + } + } + + // Resolve executor with optional upstream proxy (CLIProxyAPI) routing. + // mode="native" (default): returns the native executor unchanged. + // mode="cliproxyapi": returns the CLIProxyAPI executor instead. + // mode="fallback": returns a wrapper that tries native first, falls back to CLIProxyAPI on 5xx/network errors. + + // #6339: pass the resolved connection's providerSpecificData so a per-connection + // cliproxyapiMode="claude-native" override can deep-route this single connection + // through CLIProxyAPI regardless of the provider-level upstream_proxy_config mode. + const resolveExecutorWithProxy = (prov: string) => + resolveExecutorWithProxyFor( + prov, + log, + (credentials?.providerSpecificData as Record | null | undefined) ?? null + ); + + // === Quota Share enforcement PRE-hook (B/F7) === + // Runs after provider/model/credentials/apiKeyInfo are fully resolved, + // before dispatcher. Fail-open per B16: errors -> allow. + let quotaSoftDeprioritize = false; + if (apiKeyInfo?.id && credentials?.connectionId) { + try { + const { enforceQuotaShare } = await import("@/lib/quota/enforce"); + const decision = await enforceQuotaShare({ + apiKeyId: apiKeyInfo.id, + connectionId: credentials.connectionId, + provider: provider ?? "unknown", + // Resolved model id (post background-redirect / alias) -- the same scope the + // router/log use. Operators configure per-(key,model) caps against THIS id. + model: model || undefined, + estimatedCost: {}, + }).catch((err: unknown): EnforceDecision => { + log?.warn?.( + "QUOTA_SHARE", + `enforceQuotaShare failed; fail-open: ${err instanceof Error ? err.message : String(err)}` + ); + return { kind: "allow" as const }; + }); + + if (decision.kind === "block") { + const { buildErrorBody } = await import("../utils/error.ts"); + log?.warn?.( + "QUOTA_SHARE", + `[quotaShare] blocked apiKeyId=${apiKeyInfo.id} provider=${provider ?? "unknown"}: ${decision.reason}` + ); + // Finalize the pending-request slot registered at handler entry -- this + // return path never reaches the upstream, and without the decrement the + // pending detail lingers as an orphaned status-0 call-log row until the + // reaper sweeps it (mirrors the other pre-upstream error returns). + trackPendingRequest( + model, + provider, + connectionId || credentials?.connectionId || null, + false + ); + const headers: Record = { "Content-Type": "application/json" }; + if (decision.retryAfterSeconds) { + headers["Retry-After"] = String(decision.retryAfterSeconds); + } + return { + kind: "return" as const, + value: new Response(JSON.stringify(buildErrorBody(429, decision.reason)), { + status: 429, + headers, + }), + }; + } + + if (decision.kind === "allow" && decision.deprioritize) { + quotaSoftDeprioritize = true; + log?.info?.( + "QUOTA_SHARE", + `[quotaShare] soft deprioritize active for apiKeyId=${apiKeyInfo.id} provider=${provider ?? "unknown"}` + ); + } + } catch (err) { + // Outer fail-open guard -- should not be reached (inner .catch covers it) + log?.warn?.( + "QUOTA_SHARE", + `[quotaShare] enforceQuotaShare unexpected error; fail-open: ${err instanceof Error ? err.message : String(err)}` + ); + } + } + // G2: Propagate soft penalty to the current candidate so combo scoring can deprioritize. + if (quotaSoftDeprioritize && isCombo && comboStepId) { + try { + const { setCandidateQuotaSoftPenalty } = await import("../services/combo"); + setCandidateQuotaSoftPenalty(comboExecutionKey, comboStepId, true); + } catch (err) { + log?.warn?.( + "QUOTA_SHARE", + `[quotaShare] could not set soft penalty on candidate: ${err instanceof Error ? err.message : String(err)}` + ); + } + } + // === /Quota Share enforcement PRE-hook === + if (isFeatureFlagEnabled("CAPABILITY_FILTER_ENABLED")) { + const fit = checkRequestCapabilityFit( + getResolvedModelCapabilities({ provider, model: effectiveModel }), + deriveRequestCapabilityRequirements(body as Record), + provider + ); + if (!fit.compatible) { + const msg = buildCapabilityMismatchMessage(fit.terminalReason!, provider, effectiveModel); + log?.warn?.("CAPABILITY", msg); + trackPendingRequest(model, provider, connectionId, false); + return { + kind: "return" as const, + value: createErrorResult(400, msg, null, fit.terminalReason, "invalid_request_error"), + }; + } + } + // Get executor for this provider (with optional upstream proxy routing) + const executor = await resolveExecutorWithProxy(provider); + const getExecutionCredentials = () => + withReasoningRuleContext( + resolveExecutionCredentialsFor({ + credentials, + nativeCodexPassthrough: nativeResponsesPassthrough, + endpointPath, + targetFormat, + provider, + ccSessionId, + modelInfo, + }), + reasoningRuleDirective + ); + + let onPipelineStreamError: streamFailure.PipelineStreamErrorHandler | null = null; + let onClientDisconnectFinalize: + ((event: { reason: string; duration: number }) => boolean) | null = null; + + const bindPipelineStreamError = (handler: streamFailure.PipelineStreamErrorHandler | null) => { + onPipelineStreamError = handler; + }; + const bindClientDisconnectFinalize = ( + handler: ((event: { reason: string; duration: number }) => boolean) | null + ) => { + onClientDisconnectFinalize = handler; + }; + + // Create stream controller for disconnect detection + const streamController = createStreamController({ + onDisconnect: (event) => { + let finalized = false; + try { + finalized = onClientDisconnectFinalize?.(event) === true; + } catch {} + if (!finalized) { + try { + finalizePendingScope(pendingScope, { + status: 499, + error: `Client disconnected: ${event.reason}`, + errorCode: "client_disconnected", + }); + finalized = true; + } catch {} + } + try { + onDisconnect?.(event); + } catch {} + return finalized; + }, + onError: (event) => onPipelineStreamError?.(event), + provider, + model, + connectionId, + clientResponseFormat, + clientAbortSignal: clientRawRequest?.signal, + allowCompletedToolHandoffGrace: isCodexResponsesEcho, + clientDisconnectGracePeriodMs: STREAM_DISCONNECT_GRACE_PERIOD_MS, + }); + + const dedupRequestBody = { ...translatedBody, model: `${provider}/${model}`, stream }; + const dedupEnabled = shouldDeduplicate(dedupRequestBody); + // Namespaced by the calling API key: dedup hands the SAME response object to + // every joiner, so a shared hash across keys is a cross-principal response + // leak (GHSA-6c7w-56xp-wpc6). + const dedupHash = dedupEnabled + ? computeRequestHash(dedupRequestBody, apiKeyInfo?.id, trustedEffortContext) + : null; + + return { + kind: "continue" as const, + continue: { + translatedBody, + tokensCompressed, + compressionAnalyticsWritePromise, + compressionResponseMeta, + reasoningReplayHistory, + dedupEnabled, + dedupHash, + toolNameMap, + requestToolIdentityMap, + streamController, + executor, + getExecutionCredentials, + upstreamStream, + isClaudeCodeCompatible, + isClaudePassthrough, + preserveCacheControl, + bindPipelineStreamError, + bindClientDisconnectFinalize, + }, + }; +} + +export type TranslateDedupResult = Awaited>; +export type Continue3 = Extract["continue"]; diff --git a/tests/unit/chatcore/translate-and-dedup.test.ts b/tests/unit/chatcore/translate-and-dedup.test.ts new file mode 100644 index 000000000000..bed142205810 --- /dev/null +++ b/tests/unit/chatcore/translate-and-dedup.test.ts @@ -0,0 +1,114 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const barrelPath = path.resolve(here, "../../../open-sse/handlers/chatCore.ts"); +const leafPath = path.resolve(here, "../../../open-sse/handlers/chatCore/translateAndDedup.ts"); + +function readBarrelSource(): string { + assert.equal(fs.existsSync(barrelPath), true, "chatCore.ts barrel must exist"); + return fs.readFileSync(barrelPath, "utf8"); +} + +test("translateAndDedup leaf exists", () => { + assert.equal(fs.existsSync(leafPath), true, "translateAndDedup.ts must exist"); +}); + +test("barrel file exists", () => { + assert.equal(fs.existsSync(barrelPath), true, "chatCore.ts barrel must exist"); +}); + +test("barrel imports and calls runTranslateAndDedup", () => { + const src = readBarrelSource(); + assert.equal( + src.includes('import { runTranslateAndDedup } from "./chatCore/translateAndDedup.ts";'), + true, + "barrel must import runTranslateAndDedup" + ); + assert.equal( + src.includes("const translated = await runTranslateAndDedup("), + true, + "barrel must invoke runTranslateAndDedup" + ); +}); + +test("barrel no longer inlines translatedBody assignment", () => { + const src = readBarrelSource(); + assert.equal( + src.includes("let translatedBody = body;"), + false, + "request translation must leave barrel" + ); +}); + +test("barrel rebinds persistAttemptLogs with compressed tokens before send", () => { + const src = readBarrelSource(); + const sliceCall = src.indexOf("await runTranslateAndDedup({"); + const sendIdx = src.indexOf("const executeProviderRequest = ("); + assert.notEqual(sliceCall, -1, "barrel must call runTranslateAndDedup"); + assert.notEqual(sendIdx, -1, "executeProviderRequest wrapper must remain"); + assert.ok(sliceCall < sendIdx, "translation must run before send"); + const between = src.slice(sliceCall, sendIdx); + assert.equal( + between.includes("persistAttemptLogsFor"), + true, + "rebind must call persistAttemptLogsFor after translation" + ); + assert.equal( + between.includes("tokensCompressed"), + true, + "rebind must capture live tokensCompressed" + ); +}); + +test("runTranslateAndDedup returns continue with translated payload and controllers", async () => { + const { runTranslateAndDedup } = + await import("../../../open-sse/handlers/chatCore/translateAndDedup.ts"); + const result = await runTranslateAndDedup({ + body: { + model: "gpt-4o", + messages: [{ role: "user", content: "hello world" }], + }, + tokensCompressed: 0, + compressionAnalyticsWritePromise: null, + compressionResponseMeta: null, + preCompressionBody: {}, + credentials: { apiKey: "sk-test" }, + clientRawRequest: { headers: {} }, + connectionId: "conn-1", + apiKeyInfo: { id: "key-1" }, + provider: "openai", + model: "gpt-4o", + effectiveModel: "gpt-4o", + sourceFormat: "openai", + targetFormat: "openai", + stream: false, + traceId: "test-trace-id", + settings: {}, + }); + + assert.equal(result.kind, "continue"); + if (result.kind === "continue") { + assert.ok(result.continue.translatedBody, "translatedBody must be truthy"); + assert.equal( + (result.continue.translatedBody as Record).model, + "gpt-4o", + "translatedBody.model must match input model" + ); + assert.deepEqual( + (result.continue.translatedBody as Record).messages, + [{ role: "user", content: "hello world" }], + "translatedBody.messages must be preserved" + ); + assert.equal(typeof result.continue.dedupEnabled, "boolean"); + assert.ok(result.continue.streamController); + assert.equal(typeof result.continue.tokensCompressed, "number"); + assert.ok("compressionAnalyticsWritePromise" in result.continue); + assert.equal(typeof result.continue.getExecutionCredentials, "function"); + assert.equal(typeof result.continue.bindPipelineStreamError, "function"); + assert.equal(typeof result.continue.bindClientDisconnectFinalize, "function"); + } +}); From a340acf6709a247ca465538cc3d9ea23a14f5c03 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Tue, 29 Sep 2026 05:50:41 -0300 Subject: [PATCH 25/26] test: point compression regression guard at cacheAndCompress leaf The overflow fail-fast gate (finalEstimatedInputTokens >= finalContextLimit) moved from chatCore.ts into chatCore/cacheAndCompress.ts with the response-path decomposition, but the #12793 regression guard's barrelSource still read chatCore.ts, silently asserting against a file that no longer contains the gate's logic. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- tests/unit/codex-prompt-compression-passthrough.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/codex-prompt-compression-passthrough.test.ts b/tests/unit/codex-prompt-compression-passthrough.test.ts index 912e373b8fcb..7ce36f77bfa5 100644 --- a/tests/unit/codex-prompt-compression-passthrough.test.ts +++ b/tests/unit/codex-prompt-compression-passthrough.test.ts @@ -13,9 +13,11 @@ const chatCoreSource = readFileSync( "utf8" ); -// The overflow fail-fast gate stayed behind in the barrel. +// The overflow fail-fast gate moved into the cache/compress leaf with the +// chatCore decomposition (#13065) — it never lived alongside the threshold +// gate above, but the two used to share a file. const barrelSource = readFileSync( - new URL("../../open-sse/handlers/chatCore.ts", import.meta.url), + new URL("../../open-sse/handlers/chatCore/cacheAndCompress.ts", import.meta.url), "utf8" ); From 9b78184f421dae791a1de83fa39c3255a9b7bd2b Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Tue, 29 Sep 2026 06:17:33 -0300 Subject: [PATCH 26/26] refactor(sse): extract credential-refresh-retry sub-slice from streamingLeg streamingLeg.ts was 1311 lines, over the 1200-line cap for new files (config/quality/file-size-baseline.json). Lift the self-contained 401/403 credential-refresh-and-retry block into its own leaf, mirroring the deps-bag pattern already used across chatCore's other extractions. No behavior change: the block moves verbatim, taking its declared-but-unused-until-later state (upstreamErrorParsed, parsedStatusCode, parsedMessage, parsedRetryAfterMs, upstreamErrorBody) along as return values instead of shared closure locals. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .../chatCore/credentialRefreshRetry.ts | 343 ++++++++++++++++++ open-sse/handlers/chatCore/streamingLeg.ts | 262 +++---------- 2 files changed, 395 insertions(+), 210 deletions(-) create mode 100644 open-sse/handlers/chatCore/credentialRefreshRetry.ts diff --git a/open-sse/handlers/chatCore/credentialRefreshRetry.ts b/open-sse/handlers/chatCore/credentialRefreshRetry.ts new file mode 100644 index 000000000000..500aad3f34de --- /dev/null +++ b/open-sse/handlers/chatCore/credentialRefreshRetry.ts @@ -0,0 +1,343 @@ +import { FORMATS } from "../../translator/formats.ts"; +import { + refreshWithRetry, + isUnrecoverableRefreshError, + runWithOnPersist, + runWithCasGuard, +} from "../../services/tokenRefresh.ts"; +import { runWithCapture } from "../../utils/providerRequestLogging.ts"; +import { sanitizeErrorMessage } from "../../utils/error.ts"; +import { HTTP_STATUS } from "../../config/constants.ts"; +import { getProviderConnectionById } from "@/lib/db/providers"; +import { wasRefreshTokenRotated } from "@omniroute/open-sse/services/refreshSerializer.ts"; +import { getUpstreamErrorIdentifier } from "./streamErrorResult.ts"; +import { buildExecutorClientHeaders } from "./executorClientHeaders.ts"; +import { getExecutionConnectionId } from "./executionCredentials.ts"; +import { prepareUpstreamBody } from "./upstreamBody.ts"; +import { updatePendingScope } from "@/lib/usage/pendingRequestScope"; +import { normalizeExecutorResult } from "./upstreamTimeouts.ts"; +import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; +import type { StreamingLegDeps } from "./streamingLeg.ts"; + +/** + * The 401/403 credential-refresh-and-retry sub-slice of {@link runStreamingLeg}, + * lifted out verbatim to keep the response-path leaf under the new-file line cap. + * + * Runs only when the initial provider dispatch came back unauthorized/forbidden + * and the failure isn't attributable to a stripped `stream_options` (which can + * masquerade as an auth error). On a successful refresh it retries the request + * once with the rotated credentials; the caller inherits whatever this returns + * (a fresh response on success, the original one otherwise). + */ +export interface CredentialRefreshRetryDeps extends Pick< + StreamingLegDeps, + | "targetFormat" + | "credentials" + | "provider" + | "log" + | "onCredentialsRefreshed" + | "connectionId" + | "effectiveModel" + | "trustedEffortContext" + | "isOpencodeClient" + | "body" + | "clientRawRequest" + | "getExecutionCredentials" + | "assertManagedLeaseFence" + | "executor" + | "streamController" + | "extendedContext" + | "buildUpstreamHeadersForExecute" + | "userAgent" + | "clientResponseFormat" + | "isCombo" + | "contextEditingEnabled" + | "correlationId" + | "reqLogger" + | "pendingScope" + | "getManagedLeaseFenceErrorCode" + | "managedLeaseFenceErrorResult" + | "providerRequestCapture" + | "translatedBody" + | "upstreamStream" +> { + providerResponse: Response & { body?: unknown }; + providerUrl: string; + providerHeaders: Record | Headers | null | undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- mirrors StreamingLegDeps.finalBody + finalBody: any; +} + +export interface CredentialRefreshRetryOutcome { + /** Set when the lease fence rejected the retry; the caller returns this immediately. */ + earlyReturn: ReturnType | null; + providerResponse: Response & { body?: unknown }; + providerUrl: string; + providerHeaders: Record | Headers | null | undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- mirrors StreamingLegDeps.finalBody + finalBody: any; + upstreamErrorParsed: boolean; + parsedStatusCode: number; + parsedMessage: string; + parsedRetryAfterMs: number | null; + upstreamErrorBody: unknown; +} + +export async function runCredentialRefreshRetry( + deps: CredentialRefreshRetryDeps +): Promise { + const { + targetFormat, + credentials, + provider, + log, + onCredentialsRefreshed, + connectionId, + effectiveModel, + trustedEffortContext, + isOpencodeClient, + body, + clientRawRequest, + getExecutionCredentials, + assertManagedLeaseFence, + executor, + streamController, + extendedContext, + buildUpstreamHeadersForExecute, + userAgent, + clientResponseFormat, + isCombo, + contextEditingEnabled, + correlationId, + reqLogger, + pendingScope, + getManagedLeaseFenceErrorCode, + managedLeaseFenceErrorResult, + providerRequestCapture, + translatedBody, + upstreamStream, + } = deps; + let providerResponse = deps.providerResponse; + let providerUrl = deps.providerUrl; + let providerHeaders = deps.providerHeaders; + let finalBody = deps.finalBody; + + let upstreamErrorParsed = false; + const parsedStatusCode = providerResponse.status; + const parsedMessage = ""; + const parsedRetryAfterMs: number | null = null; + const upstreamErrorBody: unknown = null; + + // Track whether stream_options was present and stripped — if so, 401/403 after + // that may be from the modification rather than a genuine auth failure, so we + // skip the credential refresh attempt in that case. + const hadStreamOptions = + targetFormat === FORMATS.OPENAI_RESPONSES && "stream_options" in translatedBody; + if (hadStreamOptions) { + delete translatedBody.stream_options; + } + + // Handle 401/403 - try token refresh using executor + // T-PROBE: probe-origin failures never attempt the refresh — a probe must + // not consume a rotating refresh token nor persist an "expired" + // deactivation on refresh failure (#9817). The 401/403 then flows into + // the normal providerFailure classification (record-only in probe mode). + if ( + (providerResponse.status === HTTP_STATUS.UNAUTHORIZED || + providerResponse.status === HTTP_STATUS.FORBIDDEN) && + !hadStreamOptions && // Skip refresh if failure may be from stream_options removal, not auth + !(await shouldIsolateProbeFailures()) + ) { + // Fix A: wrap refreshCredentials in runWithOnPersist so the persist callback + // executes INSIDE the per-connection mutex held by getAccessToken. This makes + // [network refresh + DB write + outer-state mutation] one atomic step and + // prevents concurrent requests from reading a stale refreshToken before the + // DB has been updated (refresh_token_reused on Codex/OpenAI). + // + // Not every executor routes refresh through getAccessToken (e.g. github.ts + // calls refreshCopilotToken directly). When the persistFn doesn't fire from + // inside getAccessToken, we still need to do the credentials mutation + user + // callback after refreshCredentials returns. The `persistFnRan` flag tracks + // which path executed so we don't double-fire (race-prone) or skip (regression). + // Front 3: remember the refresh_token we are about to present so that, if the + // refresh fails as unrecoverable, we can tell a genuine death apart from a + // stale-token reuse that a concurrent/sibling refresh already rotated past. + const attemptedRefreshToken = + typeof credentials?.refreshToken === "string" ? credentials.refreshToken : null; + let persistFnRan = false; + const persistFn = onCredentialsRefreshed + ? async (refreshResult: Record) => { + persistFnRan = true; + // Mutate the shared credentials object so subsequent executor calls + // in this request see the new tokens. Runs INSIDE the mutex. + Object.assign(credentials, refreshResult); + await onCredentialsRefreshed(refreshResult); + } + : undefined; + + // #4038: build a compare-and-swap reread so getAccessToken can skip the persist if a + // concurrent writer (sibling request / HealthCheck / replica) already rotated this + // connection's refresh_token past the one we presented — overwriting would revert it + // and revoke the token family. No connectionId ⇒ no guard (behavior unchanged). + const casConnectionId = + typeof credentials?.connectionId === "string" ? credentials.connectionId.trim() : ""; + const casReread = casConnectionId + ? async () => { + const latest = await getProviderConnectionById(casConnectionId); + return typeof latest?.refreshToken === "string" ? latest.refreshToken : null; + } + : null; + + const newCredentials = (await refreshWithRetry( + () => + runWithCasGuard( + casReread ? { expectedRefreshToken: attemptedRefreshToken, reread: casReread } : null, + () => runWithOnPersist(persistFn, () => executor.refreshCredentials(credentials, log)) + ), + 3, + log, + provider // Explicitly pass the provider to avoid universally tripping the "unknown" circuit breaker + )) as null | { + accessToken?: string; + copilotToken?: string; + }; + + if (newCredentials?.accessToken || newCredentials?.copilotToken) { + log?.info?.("TOKEN", `${provider?.toUpperCase()} | refreshed`); + + // Fall back to post-mutex mutation only for executors that don't route + // through getAccessToken (and therefore never fire onPersist). For + // executors that DO route through it (Codex, Claude, Gemini, etc.) the + // mutation already happened atomically inside the mutex. + if (!persistFnRan) { + Object.assign(credentials, newCredentials); + if (onCredentialsRefreshed) { + await onCredentialsRefreshed(newCredentials); + } + } + + // Retry with new credentials — model + extra headers follow translatedBody.model so they + // stay aligned if this block ever runs after a path that mutates body.model (e.g. fallback). + try { + const retryModelId = String(translatedBody.model || effectiveModel); + const retryBody = await prepareUpstreamBody({ + translatedBody, + modelToCall: retryModelId, + ...trustedEffortContext, + provider, + targetFormat, + credentials: getExecutionCredentials(), + log, + bypassDefaultToolLimit: isOpencodeClient, + isOpencodeClient, + rawBody: body, + clientRawRequest, + }); + assertManagedLeaseFence(getExecutionConnectionId(getExecutionCredentials())); + const retryResult = normalizeExecutorResult( + await runWithCapture(providerRequestCapture, () => + executor.execute({ + model: retryModelId, + body: retryBody, + stream: upstreamStream, + credentials: getExecutionCredentials(), + signal: streamController.signal, + log, + extendedContext, + upstreamExtraHeaders: buildUpstreamHeadersForExecute(retryModelId), + clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), + clientResponseFormat, + onCredentialsRefreshed, + skipUpstreamRetry: isCombo, + contextEditing: { enabled: contextEditingEnabled }, + correlationId, + }) + ) + ); + + if (retryResult.response.ok) { + providerResponse = retryResult.response; + providerUrl = retryResult.url; + providerHeaders = new Headers(retryResult.headers || {}); + finalBody = providerRequestCapture.body(retryResult.transformedBody); + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + updatePendingScope(pendingScope, { + providerRequest: finalBody, + providerUrl, + stage: "provider_response_started", + }); + upstreamErrorParsed = false; // Reset since new response is OK + } else { + providerResponse = retryResult.response; + upstreamErrorParsed = false; // Let it be parsed downstream + } + } catch (retryErr) { + const retryLeaseFenceCode = getManagedLeaseFenceErrorCode( + getUpstreamErrorIdentifier(retryErr) + ); + if (retryLeaseFenceCode) { + return { + earlyReturn: managedLeaseFenceErrorResult(retryLeaseFenceCode), + providerResponse, + providerUrl, + providerHeaders, + finalBody, + upstreamErrorParsed, + parsedStatusCode, + parsedMessage, + parsedRetryAfterMs, + upstreamErrorBody, + }; + } + // Refresh succeeded but the retry leg failed (network blip, AbortError, + // executor throw). Don't swallow — the operator-visible signal "the user + // saw 401 even though auth was actually fixed" is much more confusing + // than the original 401 alone. Surface at error level with sanitization. + log?.error?.( + "TOKEN", + `${provider?.toUpperCase()} | retry after refresh failed: ${sanitizeErrorMessage(retryErr)}` + ); + } + } else { + log?.warn?.("TOKEN", `${provider?.toUpperCase()} | refresh failed`); + if (isUnrecoverableRefreshError(newCredentials) && onCredentialsRefreshed) { + // Front 3 (reuse-race tolerance): before deactivating, re-read the DB. + // If a sibling/concurrent refresh already rotated this connection's + // refresh_token (common for Codex/OpenAI under one shared Auth0 client), + // the failure we saw was a stale-token reuse — the account is healthy + // with the newer token, so keep it active instead of killing it. + let alreadyRotated = false; + if (typeof connectionId === "string" && connectionId && attemptedRefreshToken) { + try { + const latest = await getProviderConnectionById(connectionId); + if (wasRefreshTokenRotated(attemptedRefreshToken, latest?.refreshToken)) { + alreadyRotated = true; + log?.warn?.( + "TOKEN", + `${provider.toUpperCase()} | refresh_token already rotated by a concurrent refresh — keeping connection active` + ); + } + } catch { + // DB read failed — fall through to the safe default (deactivate). + } + } + if (!alreadyRotated) { + await onCredentialsRefreshed({ testStatus: "expired", isActive: false }); + } + } + } + } + + return { + earlyReturn: null, + providerResponse, + providerUrl, + providerHeaders, + finalBody, + upstreamErrorParsed, + parsedStatusCode, + parsedMessage, + parsedRetryAfterMs, + upstreamErrorBody, + }; +} diff --git a/open-sse/handlers/chatCore/streamingLeg.ts b/open-sse/handlers/chatCore/streamingLeg.ts index 3f35ab6fbf96..9ba92497560a 100644 --- a/open-sse/handlers/chatCore/streamingLeg.ts +++ b/open-sse/handlers/chatCore/streamingLeg.ts @@ -2,19 +2,13 @@ import { projectFailureUsageErrorCode, type FailureUsageAggregate } from "./fail import { buildClaudePromptCacheLogMeta } from "./executorHelpers.ts"; import { recoverAnthropicThinkingSignature } from "./thinkingSignatureRecovery.ts"; import { runProviderExecutionPipeline } from "./providerExecutionPipeline.ts"; +import { runCredentialRefreshRetry } from "./credentialRefreshRetry.ts"; import { onFailure, onStreamThrow } from "./recoveryPolicy.ts"; import { markCodexScopeRateLimited } from "./codexFailover.ts"; import { deleteSessionAccountAffinity } from "@/lib/db/sessionAccountAffinity"; import { normalizeHeaders } from "../../utils/headers.ts"; import { FORMATS } from "../../translator/formats.ts"; import { COLORS } from "../../utils/stream.ts"; -import { - refreshWithRetry, - isUnrecoverableRefreshError, - runWithOnPersist, - runWithCasGuard, -} from "../../services/tokenRefresh.ts"; -import { runWithCapture } from "../../utils/providerRequestLogging.ts"; import type { PersistAttemptLogsArgs } from "./attemptLogging.ts"; import { REASONING_BUFFER_MIN_TRIGGER, @@ -33,22 +27,16 @@ import { } from "../../utils/error.ts"; import { HTTP_STATUS, ANTIGRAVITY_PRE_RESPONSE_TIMEOUT_CODE } from "../../config/constants.ts"; import { applyStatusRestatement } from "../../config/upstreamStatusRestatement.ts"; -import { updateProviderConnection, getProviderConnectionById } from "@/lib/db/providers"; -import { wasRefreshTokenRotated } from "@omniroute/open-sse/services/refreshSerializer.ts"; +import { updateProviderConnection } from "@/lib/db/providers"; import { createSafeAbortError, createStreamingErrorResult, isSemaphoreCapacityError, getSafeErrorMetadata, - getUpstreamErrorIdentifier, } from "./streamErrorResult.ts"; -import { buildExecutorClientHeaders } from "./executorClientHeaders.ts"; -import { getExecutionConnectionId } from "./executionCredentials.ts"; -import { prepareUpstreamBody } from "./upstreamBody.ts"; import { logAuditEvent } from "@/lib/compliance"; import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb"; import { updatePendingScope } from "@/lib/usage/pendingRequestScope"; -import { normalizeExecutorResult } from "./upstreamTimeouts.ts"; import { getProviderCredentials, extractSessionAffinityKey } from "@/sse/services/auth"; import { updateFromHeaders, updateFromResponseBody } from "../../services/rateLimitManager.ts"; import * as localLimiterErrors from "../../services/rateLimitManager/errors.ts"; @@ -64,7 +52,7 @@ import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; import { writeTerminalStatus } from "@/shared/utils/terminalStatus"; -type LoggerLike = +export type LoggerLike = | { warn?: (...args: unknown[]) => void; debug?: (...args: unknown[]) => void; @@ -701,203 +689,57 @@ export async function runStreamingLeg(deps: StreamingLegDeps): Promise) => { - persistFnRan = true; - // Mutate the shared credentials object so subsequent executor calls - // in this request see the new tokens. Runs INSIDE the mutex. - Object.assign(credentials, refreshResult); - await onCredentialsRefreshed(refreshResult); - } - : undefined; - - // #4038: build a compare-and-swap reread so getAccessToken can skip the persist if a - // concurrent writer (sibling request / HealthCheck / replica) already rotated this - // connection's refresh_token past the one we presented — overwriting would revert it - // and revoke the token family. No connectionId ⇒ no guard (behavior unchanged). - const casConnectionId = - typeof credentials?.connectionId === "string" ? credentials.connectionId.trim() : ""; - const casReread = casConnectionId - ? async () => { - const latest = await getProviderConnectionById(casConnectionId); - return typeof latest?.refreshToken === "string" ? latest.refreshToken : null; - } - : null; - - const newCredentials = (await refreshWithRetry( - () => - runWithCasGuard( - casReread ? { expectedRefreshToken: attemptedRefreshToken, reread: casReread } : null, - () => runWithOnPersist(persistFn, () => executor.refreshCredentials(credentials, log)) - ), - 3, - log, - provider // Explicitly pass the provider to avoid universally tripping the "unknown" circuit breaker - )) as null | { - accessToken?: string; - copilotToken?: string; + const refreshRetryOutcome = await runCredentialRefreshRetry({ + body, + buildUpstreamHeadersForExecute, + clientRawRequest, + clientResponseFormat, + connectionId, + contextEditingEnabled, + correlationId, + credentials, + effectiveModel, + executor, + extendedContext, + finalBody, + getExecutionCredentials, + getManagedLeaseFenceErrorCode, + isCombo, + isOpencodeClient, + log, + managedLeaseFenceErrorResult, + onCredentialsRefreshed, + pendingScope, + provider, + providerHeaders, + providerRequestCapture, + providerResponse, + providerUrl, + reqLogger, + streamController, + targetFormat, + translatedBody, + trustedEffortContext, + upstreamStream, + userAgent, + assertManagedLeaseFence, + }); + if (refreshRetryOutcome.earlyReturn) { + return { + kind: "returned" as const, + value: refreshRetryOutcome.earlyReturn, + carry: carry(), }; - - if (newCredentials?.accessToken || newCredentials?.copilotToken) { - log?.info?.("TOKEN", `${provider?.toUpperCase()} | refreshed`); - - // Fall back to post-mutex mutation only for executors that don't route - // through getAccessToken (and therefore never fire onPersist). For - // executors that DO route through it (Codex, Claude, Gemini, etc.) the - // mutation already happened atomically inside the mutex. - if (!persistFnRan) { - Object.assign(credentials, newCredentials); - if (onCredentialsRefreshed) { - await onCredentialsRefreshed(newCredentials); - } - } - - // Retry with new credentials — model + extra headers follow translatedBody.model so they - // stay aligned if this block ever runs after a path that mutates body.model (e.g. fallback). - try { - const retryModelId = String(translatedBody.model || effectiveModel); - const retryBody = await prepareUpstreamBody({ - translatedBody, - modelToCall: retryModelId, - ...trustedEffortContext, - provider, - targetFormat, - credentials: getExecutionCredentials(), - log, - bypassDefaultToolLimit: isOpencodeClient, - isOpencodeClient, - rawBody: body, - clientRawRequest, - }); - assertManagedLeaseFence(getExecutionConnectionId(getExecutionCredentials())); - const retryResult = normalizeExecutorResult( - await runWithCapture(providerRequestCapture, () => - executor.execute({ - model: retryModelId, - body: retryBody, - stream: upstreamStream, - credentials: getExecutionCredentials(), - signal: streamController.signal, - log, - extendedContext, - upstreamExtraHeaders: buildUpstreamHeadersForExecute(retryModelId), - clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), - clientResponseFormat, - onCredentialsRefreshed, - skipUpstreamRetry: isCombo, - contextEditing: { enabled: contextEditingEnabled }, - correlationId, - }) - ) - ); - - if (retryResult.response.ok) { - providerResponse = retryResult.response; - providerUrl = retryResult.url; - providerHeaders = new Headers(retryResult.headers || {}); - finalBody = providerRequestCapture.body(retryResult.transformedBody); - reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); - updatePendingScope(pendingScope, { - providerRequest: finalBody, - providerUrl, - stage: "provider_response_started", - }); - upstreamErrorParsed = false; // Reset since new response is OK - } else { - providerResponse = retryResult.response; - upstreamErrorParsed = false; // Let it be parsed downstream - } - } catch (retryErr) { - const retryLeaseFenceCode = getManagedLeaseFenceErrorCode( - getUpstreamErrorIdentifier(retryErr) - ); - if (retryLeaseFenceCode) - return { - kind: "returned" as const, - value: managedLeaseFenceErrorResult(retryLeaseFenceCode), - carry: carry(), - }; - // Refresh succeeded but the retry leg failed (network blip, AbortError, - // executor throw). Don't swallow — the operator-visible signal "the user - // saw 401 even though auth was actually fixed" is much more confusing - // than the original 401 alone. Surface at error level with sanitization. - log?.error?.( - "TOKEN", - `${provider?.toUpperCase()} | retry after refresh failed: ${sanitizeErrorMessage(retryErr)}` - ); - } - } else { - log?.warn?.("TOKEN", `${provider?.toUpperCase()} | refresh failed`); - if (isUnrecoverableRefreshError(newCredentials) && onCredentialsRefreshed) { - // Front 3 (reuse-race tolerance): before deactivating, re-read the DB. - // If a sibling/concurrent refresh already rotated this connection's - // refresh_token (common for Codex/OpenAI under one shared Auth0 client), - // the failure we saw was a stale-token reuse — the account is healthy - // with the newer token, so keep it active instead of killing it. - let alreadyRotated = false; - if (typeof connectionId === "string" && connectionId && attemptedRefreshToken) { - try { - const latest = await getProviderConnectionById(connectionId); - if (wasRefreshTokenRotated(attemptedRefreshToken, latest?.refreshToken)) { - alreadyRotated = true; - log?.warn?.( - "TOKEN", - `${provider.toUpperCase()} | refresh_token already rotated by a concurrent refresh — keeping connection active` - ); - } - } catch { - // DB read failed — fall through to the safe default (deactivate). - } - } - if (!alreadyRotated) { - await onCredentialsRefreshed({ testStatus: "expired", isActive: false }); - } - } - } } + providerResponse = refreshRetryOutcome.providerResponse; + providerUrl = refreshRetryOutcome.providerUrl; + providerHeaders = refreshRetryOutcome.providerHeaders; + finalBody = refreshRetryOutcome.finalBody; + let upstreamErrorParsed = refreshRetryOutcome.upstreamErrorParsed; + let parsedStatusCode = refreshRetryOutcome.parsedStatusCode; + let parsedMessage = refreshRetryOutcome.parsedMessage; + let parsedRetryAfterMs = refreshRetryOutcome.parsedRetryAfterMs; + let upstreamErrorBody: unknown = refreshRetryOutcome.upstreamErrorBody; // Check provider response - return error info for fallback handling providerFailure: if (!providerResponse.ok) {