diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index ad216dfcbfe..902b196038d 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -23,7 +23,11 @@ import { formatProviderError, } from "../utils/error.ts"; import { HTTP_STATUS, PROVIDER_MAX_TOKENS } from "../config/constants.ts"; -import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts"; +import { + classifyProviderError, + PROVIDER_ERROR_TYPES, + isEmptyContentResponse, +} from "../services/errorClassifier.ts"; import { updateProviderConnection } from "@/lib/db/providers"; import { isDetailedLoggingEnabled } from "@/lib/db/detailedLogs"; import { logAuditEvent } from "@/lib/compliance"; @@ -82,7 +86,12 @@ import { } from "@/lib/semanticCache"; import { getIdempotencyKey, checkIdempotency, saveIdempotency } from "@/lib/idempotencyLayer"; import { createProgressTransform, wantsProgress } from "../utils/progressTracker.ts"; -import { isModelUnavailableError, getNextFamilyFallback } from "../services/modelFamilyFallback.ts"; +import { + isModelUnavailableError, + getNextFamilyFallback, + isContextOverflowError, + findLargerContextModel, +} from "../services/modelFamilyFallback.ts"; import { computeRequestHash, deduplicate, shouldDeduplicate } from "../services/requestDedup.ts"; import { getBackgroundTaskReason, @@ -1527,6 +1536,16 @@ export async function handleChatCore({ lastError: message, errorCode: statusCode, }); + } else if (errorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN) { + // OAuth 401 with invalid credentials - token refresh can recover + await updateProviderConnection(connectionId, { + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${connectionId} OAuth token invalid (${statusCode}) — token refresh available` + ); } else if (errorType === PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR) { // Cloud Code 403 with stale project: not a ban, keep account active. await updateProviderConnection(connectionId, { @@ -1626,6 +1645,58 @@ export async function handleChatCore({ persistFailureUsage(statusCode, "model_unavailable"); return createErrorResult(statusCode, errMsg, retryAfterMs); } + } else if (isContextOverflowError(statusCode, message)) { + const nextModel = getNextFamilyFallback(currentModel, triedModels); + if (nextModel) { + triedModels.add(nextModel); + currentModel = nextModel; + translatedBody.model = nextModel; + log?.info?.("CONTEXT_OVERFLOW_FALLBACK", `${model} context overflow → trying ${nextModel}`); + try { + const fallbackResult = await executeProviderRequest(nextModel, false); + if (fallbackResult.response.ok) { + providerResponse = fallbackResult.response; + providerUrl = fallbackResult.url; + providerHeaders = fallbackResult.headers; + finalBody = fallbackResult.transformedBody; + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + log?.info?.( + "CONTEXT_OVERFLOW_FALLBACK", + `Serving ${nextModel} as fallback for ${model}` + ); + } else { + persistAttemptLogs({ + status: statusCode, + error: errMsg, + providerRequest: finalBody || translatedBody, + providerResponse: upstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + }); + persistFailureUsage(statusCode, "context_overflow"); + return createErrorResult(statusCode, errMsg, retryAfterMs); + } + } catch { + persistAttemptLogs({ + status: statusCode, + error: errMsg, + providerRequest: finalBody || translatedBody, + providerResponse: upstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + }); + persistFailureUsage(statusCode, "context_overflow"); + return createErrorResult(statusCode, errMsg, retryAfterMs); + } + } else { + persistAttemptLogs({ + status: statusCode, + error: errMsg, + providerRequest: finalBody || translatedBody, + providerResponse: upstreamErrorBody, + clientResponse: buildErrorBody(statusCode, errMsg), + }); + persistFailureUsage(statusCode, "context_overflow"); + return createErrorResult(statusCode, errMsg, retryAfterMs); + } } else { persistAttemptLogs({ status: statusCode, @@ -1764,6 +1835,69 @@ export async function handleChatCore({ } } + // Check for empty content response (fake success) - trigger fallback + if (isEmptyContentResponse(responseBody)) { + appendRequestLog({ + model, + provider, + connectionId, + status: `FAILED ${HTTP_STATUS.BAD_GATEWAY}`, + }).catch(() => {}); + const emptyContentMessage = "Provider returned empty content"; + persistAttemptLogs({ + status: HTTP_STATUS.BAD_GATEWAY, + error: emptyContentMessage, + providerRequest: finalBody || translatedBody, + providerResponse: normalizedProviderPayload, + clientResponse: buildErrorBody(HTTP_STATUS.BAD_GATEWAY, emptyContentMessage), + }); + persistFailureUsage(HTTP_STATUS.BAD_GATEWAY, "empty_content"); + + // Trigger fallback for empty content + const nextModel = getNextFamilyFallback(currentModel, triedModels); + if (nextModel) { + triedModels.add(nextModel); + currentModel = nextModel; + translatedBody.model = nextModel; + log?.info?.( + "EMPTY_CONTENT_FALLBACK", + `${model} returned empty content → trying ${nextModel}` + ); + try { + const fallbackResult = await executeProviderRequest(nextModel, false); + if (fallbackResult.response.ok) { + providerResponse = fallbackResult.response; + providerUrl = fallbackResult.url; + providerHeaders = fallbackResult.headers; + finalBody = fallbackResult.transformedBody; + reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody); + log?.info("EMPTY_CONTENT_FALLBACK", `Serving ${nextModel} as fallback for ${model}`); + // Continue with the fallback response by re-processing + return handleChatCore({ + body, + modelInfo: { provider, model: nextModel, extendedContext }, + credentials, + log, + onCredentialsRefreshed, + onRequestSuccess, + onDisconnect, + clientRawRequest, + connectionId, + apiKeyInfo, + userAgent, + comboName, + comboStrategy, + isCombo, + }); + } + } catch { + // Fallback failed, continue to return error + } + } + + return createErrorResult(HTTP_STATUS.BAD_GATEWAY, emptyContentMessage); + } + if (sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE) { responseBody = restoreClaudePassthroughToolNames(responseBody, toolNameMap); } diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index d0c09e845e3..8e8f75236c2 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -36,6 +36,15 @@ export const CREDITS_EXHAUSTED_SIGNALS = [ "payment required", ]; +// T11: Signals that indicate OAuth token is invalid/expired (not permanent deactivation) +export const OAUTH_INVALID_TOKEN_SIGNALS = [ + "invalid authentication credentials", + "oauth 2", + "login cookie", + "valid authentication credential", + "invalid credentials", +]; + /** * T06: Returns true if response body indicates the account is permanently deactivated. */ @@ -52,6 +61,15 @@ export function isCreditsExhausted(errorText: string): boolean { return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig)); } +/** + * T11: Returns true if response body indicates OAuth token is invalid/expired. + * This is different from permanent account deactivation - token refresh can recover. + */ +export function isOAuthInvalidToken(errorText: string): boolean { + const lower = String(errorText || "").toLowerCase(); + return OAUTH_INVALID_TOKEN_SIGNALS.some((sig) => lower.includes(sig)); +} + // ─── Provider Profile Helper ──────────────────────────────────────────────── /** diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index e6ba23712be..7f86b8e9f42 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -17,6 +17,7 @@ import { selectProvider as selectAutoProvider } from "./autoCombo/engine.ts"; import { selectWithStrategy } from "./autoCombo/routerStrategy.ts"; import { DEFAULT_WEIGHTS, scorePool } from "./autoCombo/scoring.ts"; import { supportsToolCalling } from "./modelCapabilities.ts"; +import { getModelContextLimit } from "../../src/lib/modelsDevSync"; // Status codes that should mark semaphore + record circuit breaker failures const TRANSIENT_FOR_BREAKER = [429, 502, 503, 504]; @@ -309,6 +310,24 @@ function sortModelsByUsage(models, comboName) { return withUsage.map((e) => e.modelStr); } +/** + * Sort models by context window size (largest first) for context-optimized strategy. + * Uses models.dev synced capabilities to get context limits. + * @param {Array} models - Model strings in "provider/model" format + * @returns {Array} Sorted model strings (largest context first) + */ +function sortModelsByContextSize(models) { + const withContext = models.map((modelStr) => { + const parsed = parseModel(modelStr); + const provider = parsed.provider || parsed.providerAlias || "unknown"; + const model = parsed.model || modelStr; + const limit = getModelContextLimit(provider, model); + return { modelStr, context: limit ?? 0 }; + }); + withContext.sort((a, b) => b.context - a.context); + return withContext.map((e) => e.modelStr); +} + function toTextContent(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; @@ -908,6 +927,9 @@ export async function handleComboChat({ } else if (strategy === "cost-optimized") { orderedModels = await sortModelsByCost(orderedModels); log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedModels[0]})`); + } else if (strategy === "context-optimized") { + orderedModels = sortModelsByContextSize(orderedModels); + log.info("COMBO", `Context-optimized ordering: largest first (${orderedModels[0]})`); } let lastError = null; diff --git a/open-sse/services/contextManager.ts b/open-sse/services/contextManager.ts index 194a252091e..a899a141b50 100644 --- a/open-sse/services/contextManager.ts +++ b/open-sse/services/contextManager.ts @@ -6,6 +6,7 @@ */ import { REGISTRY } from "../config/providerRegistry.ts"; +import { getModelContextLimit } from "../../src/lib/modelsDevSync"; // Default token limits per provider (fallbacks when not in registry) const DEFAULT_LIMITS: Record = { @@ -47,20 +48,26 @@ export function estimateTokens(text) { /** * Get token limit for a provider/model combination - * Priority: Env override > Registry defaultContextLength > DEFAULT_LIMITS + * Priority: Env override > models.dev DB > Registry defaultContextLength > DEFAULT_LIMITS */ export function getTokenLimit(provider, model = null) { // 1. Check environment variable override first const envOverride = getEnvOverride(provider); if (envOverride) return envOverride; - // 2. Check registry for provider default + // 2. Check models.dev synced DB for per-model context limit + if (model) { + const dbLimit = getModelContextLimit(provider, model); + if (dbLimit && dbLimit > 0) return dbLimit; + } + + // 3. Check registry for provider default const registryEntry = REGISTRY[provider]; if (registryEntry?.defaultContextLength) { return registryEntry.defaultContextLength; } - // 3. Check if model name hints at a known limit + // 4. Check if model name hints at a known limit if (model) { const lower = model.toLowerCase(); if (lower.includes("claude")) return DEFAULT_LIMITS.claude; @@ -75,7 +82,7 @@ export function getTokenLimit(provider, model = null) { return DEFAULT_LIMITS.codex; } - // 4. Fallback to DEFAULT_LIMITS or default + // 5. Fallback to DEFAULT_LIMITS or default return DEFAULT_LIMITS[provider] || DEFAULT_LIMITS.default; } diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index 4d6444c91c0..8f14ecba9f7 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -1,15 +1,86 @@ -import { isAccountDeactivated, isCreditsExhausted } from "./accountFallback.ts"; +import { + isAccountDeactivated, + isCreditsExhausted, + isOAuthInvalidToken, +} from "./accountFallback.ts"; + +export function isEmptyContentResponse(responseBody: unknown): boolean { + if (!responseBody || typeof responseBody !== "object") return false; + + const body = responseBody as Record; + + if (Array.isArray(body.choices)) { + const firstChoice = body.choices[0] as Record | undefined; + if (!firstChoice) return true; + + const message = firstChoice.message as Record | undefined; + const delta = firstChoice.delta as Record | undefined; + + const content = message?.content ?? delta?.content; + const hasToolCalls = Array.isArray(firstChoice.tool_calls) && firstChoice.tool_calls.length > 0; + const hasDeltaToolCalls = + Array.isArray((delta as Record)?.tool_calls) && + ((delta as Record)?.tool_calls as unknown[])?.length > 0; + + const hasContent = content !== null && content !== undefined && content !== ""; + return !hasContent && !hasToolCalls && !hasDeltaToolCalls; + } + + if (Array.isArray(body.content)) { + return body.content.length === 0; + } + + if (typeof body.text === "string") { + return body.text.trim() === ""; + } + + if ("content" in body) { + const content = body.content; + return content === null || content === undefined || content === ""; + } + + return false; +} export const PROVIDER_ERROR_TYPES = { - RATE_LIMITED: "rate_limited", // 429 — transient, retry with backoff - UNAUTHORIZED: "unauthorized", // 401 — token expired, refresh - ACCOUNT_DEACTIVATED: "account_deactivated", // 401 + deactivation signal - FORBIDDEN: "forbidden", // 403 — account banned/revoked, disable node - SERVER_ERROR: "server_error", // 500/502/503 — retry limited - QUOTA_EXHAUSTED: "quota_exhausted", // 402/429/400 + billing signals - PROJECT_ROUTE_ERROR: "project_route_error", // 403 + stale project — transient, not a ban + RATE_LIMITED: "rate_limited", + UNAUTHORIZED: "unauthorized", + ACCOUNT_DEACTIVATED: "account_deactivated", + FORBIDDEN: "forbidden", + SERVER_ERROR: "server_error", + QUOTA_EXHAUSTED: "quota_exhausted", + PROJECT_ROUTE_ERROR: "project_route_error", + CONTEXT_OVERFLOW: "context_overflow", + OAUTH_INVALID_TOKEN: "oauth_invalid_token", + EMPTY_CONTENT: "empty_content", }; +const CONTEXT_OVERFLOW_SIGNALS = [ + "context overflow", + "prompt too large", + "context window", + "maximum context", + "exceeds context", + "input too long", + "token limit", + "too many tokens", + "context length", + "exceed.*context", + "messages exceed", +]; + +export function isContextOverflow(errorText: string): boolean { + const lower = String(errorText || "").toLowerCase(); + return CONTEXT_OVERFLOW_SIGNALS.some((sig) => { + try { + const regex = new RegExp(sig, "i"); + return regex.test(lower); + } catch { + return lower.includes(sig.toLowerCase()); + } + }); +} + function responseBodyToString(responseBody: unknown): string { if (typeof responseBody === "string") return responseBody; if (responseBody !== null && typeof responseBody === "object") { @@ -26,8 +97,8 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) const bodyStr = responseBodyToString(responseBody); const creditsExhausted = isCreditsExhausted(bodyStr); const accountDeactivated = isAccountDeactivated(bodyStr); + const oauthInvalid = isOAuthInvalidToken(bodyStr); - // T10: credits exhausted is terminal and can appear as 400/402/429 depending on provider. if ( creditsExhausted && (statusCode === 400 || statusCode === 402 || statusCode === 429 || statusCode === 403) @@ -39,8 +110,10 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) return PROVIDER_ERROR_TYPES.RATE_LIMITED; } - // T06: only deactivation-like 401s should be treated as permanent account expiry. if (statusCode === 401) { + if (oauthInvalid) { + return PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN; + } return accountDeactivated ? PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED : PROVIDER_ERROR_TYPES.UNAUTHORIZED; @@ -51,9 +124,6 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) return PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED; } if (statusCode === 403) { - // Cloud Code API returns 403 with "has not been used in project X" when the project - // field is wrong or stale. This is a routing/config error, not an account ban. - // Classify as project_route_error so the account stays active but the error is tracked. if (bodyStr.includes("has not been used in project")) { return PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR; } @@ -61,5 +131,9 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) } if (statusCode >= 500) return PROVIDER_ERROR_TYPES.SERVER_ERROR; + if (statusCode === 400 && isContextOverflow(bodyStr)) { + return PROVIDER_ERROR_TYPES.CONTEXT_OVERFLOW; + } + return null; } diff --git a/open-sse/services/modelFamilyFallback.ts b/open-sse/services/modelFamilyFallback.ts index dc8a2bfbc2a..658d6a749c4 100644 --- a/open-sse/services/modelFamilyFallback.ts +++ b/open-sse/services/modelFamilyFallback.ts @@ -11,6 +11,9 @@ * (commit 6cea566, Mar 8 2026). */ +import { getModelContextLimit } from "../../src/lib/modelsDevSync"; +import { parseModel } from "./model.ts"; + // ── Model Family Definitions ───────────────────────────────────────────────── /** @@ -116,6 +119,33 @@ export function isModelUnavailableError(status: number, errorMessage: string): b return MODEL_UNAVAILABLE_FRAGMENTS.some((fragment) => msg.includes(fragment)); } +const CONTEXT_OVERFLOW_FRAGMENTS = [ + "context overflow", + "prompt too large", + "context window", + "maximum context", + "exceeds context", + "input too long", + "token limit", + "too many tokens", + "context length", + "exceed.*context", + "messages exceed", +]; + +export function isContextOverflowError(status: number, errorMessage: string): boolean { + if (status !== 400) return false; + const msg = errorMessage.toLowerCase(); + return CONTEXT_OVERFLOW_FRAGMENTS.some((fragment) => { + try { + const regex = new RegExp(fragment, "i"); + return regex.test(msg); + } catch { + return msg.includes(fragment.toLowerCase()); + } + }); +} + // ── Fallback Resolution ────────────────────────────────────────────────────── /** @@ -156,3 +186,35 @@ export function getModelFamily(model: string): string[] { if (!family) return [model]; return [model, ...family]; } + +/** + * Find a model with larger context window from a list of candidate models. + * Uses models.dev synced capabilities to compare context limits. + */ +export function findLargerContextModel( + currentModel: string, + availableModels: string[] +): string | null { + const currentParsed = parseModel(currentModel); + const currentProvider = currentParsed.provider || currentParsed.providerAlias || "unknown"; + const currentModelId = currentParsed.model || currentModel; + const currentLimit = getModelContextLimit(currentProvider, currentModelId) ?? 0; + + let bestModel: string | null = null; + let bestLimit = currentLimit; + + for (const candidate of availableModels) { + if (candidate === currentModel) continue; + const parsed = parseModel(candidate); + const provider = parsed.provider || parsed.providerAlias || "unknown"; + const modelId = parsed.model || candidate; + const limit = getModelContextLimit(provider, modelId) ?? 0; + + if (limit > bestLimit) { + bestLimit = limit; + bestModel = candidate; + } + } + + return bestModel; +} diff --git a/src/lib/modelsDevSync.ts b/src/lib/modelsDevSync.ts index 320ce047b59..858baced6b2 100644 --- a/src/lib/modelsDevSync.ts +++ b/src/lib/modelsDevSync.ts @@ -692,3 +692,12 @@ export async function initModelsDevSync(): Promise { const interval = settings.modelsDevSyncInterval as number | undefined; startPeriodicSync(interval); } + +/** + * Get context window limit for a specific model from synced capabilities. + * Returns null if not available. + */ +export function getModelContextLimit(provider: string, modelId: string): number | null { + const caps = getSyncedCapabilities(provider, modelId); + return caps[provider]?.[modelId]?.limit_context ?? null; +} diff --git a/tests/unit/context-manager.test.mjs b/tests/unit/context-manager.test.mjs index f4d5d5f292c..1ce7125681f 100644 --- a/tests/unit/context-manager.test.mjs +++ b/tests/unit/context-manager.test.mjs @@ -1,11 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { - compressContext, - estimateTokens, - getTokenLimit, -} = await import("../../open-sse/services/contextManager.ts"); +const { compressContext, estimateTokens, getTokenLimit } = + await import("../../open-sse/services/contextManager.ts"); // ─── estimateTokens ───────────────────────────────────────────────────────── @@ -26,7 +23,7 @@ test("getTokenLimit: detects claude", () => { }); test("getTokenLimit: detects gemini", () => { - assert.equal(getTokenLimit("gemini", "gemini-2.5-pro"), 1000000); + assert.equal(getTokenLimit("gemini", "gemini-2.5-pro"), 1048576); }); test("getTokenLimit: default fallback", () => {