diff --git a/.env.example b/.env.example index 67097b710456..b2aeacca51ec 100644 --- a/.env.example +++ b/.env.example @@ -2173,9 +2173,16 @@ APP_LOG_TO_FILE=true # Also configurable from Dashboard > Settings > Feature Flags. # OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK=false -# Rate limit maximum wait time before failing a request (ms). Default: 15000 (15s) +# Rate limit queue-wait budget: how long a request may wait for a rate-limit +# slot (gates + limiter queue) before being dropped (ms). Default: 120000 (2 min) # Used by: open-sse/services/rateLimitManager.ts -# RATE_LIMIT_MAX_WAIT_MS=15000 +# RATE_LIMIT_MAX_WAIT_MS=120000 + +# Limiter-managed execution backstop (Bottleneck `expiration`): bounds a job's +# post-dispatch execution, never queue wait. Must stay ABOVE upstream +# fetch-start timeouts on non-incremental gateways. Default: 600000 (10 min) +# Used by: open-sse/services/rateLimitManager.ts +# RATE_LIMIT_EXECUTION_MAX_WAIT_MS=600000 # Limiter-managed execution backstop (Bottleneck `expiration`): bounds a job's # post-dispatch execution, never queue wait. Must stay ABOVE upstream diff --git a/changelog.d/fixes/12294-combo-skipped-target-diagnostics.md b/changelog.d/fixes/12294-combo-skipped-target-diagnostics.md new file mode 100644 index 000000000000..b7f388178fbe --- /dev/null +++ b/changelog.d/fixes/12294-combo-skipped-target-diagnostics.md @@ -0,0 +1 @@ +- **fix(combo):** `ALL_TARGETS_SKIPPED` 503 responses now carry per-target pre-dispatch skip reasons (`skippedTargets`) and the earliest persisted-cooldown reset (`nextRetryAt`) instead of an opaque 503, and API-key allowlist rejections during combo target screening are logged at info with the key name ([#12294](https://github.com/diegosouzapw/OmniRoute/issues/12294)) diff --git a/changelog.d/fixes/combo-dispatch-shape-fixes.md b/changelog.d/fixes/combo-dispatch-shape-fixes.md new file mode 100644 index 000000000000..52a31c609128 --- /dev/null +++ b/changelog.d/fixes/combo-dispatch-shape-fixes.md @@ -0,0 +1,5 @@ +- **fix(combo):** tiny-budget reasoning probes (max_tokens < 256) whose upstream 200 burned the whole budget on thinking are answered with a truncated 200 (`finish_reason: "length"`) in both combo dispatchers instead of a 502 + model lockout (#10281 combo-path parity) +- **fix(translator):** opencode console gateways (opencode-go / opencode-zen) now get the reasoning replay sentinel on cache misses for thinking-mode tool turns, and the Responses converter emits it as a `reasoning_text` item — upstream 400 "The reasoning_text in the thinking mode must be passed back to the API" no longer takes out deepseek/glm multi-turn traffic; DeepSeek-direct absent-field behavior unchanged (#9573/#9610 contract preserved) +- **fix(translator):** GLM-family upstreams get a synthetic `(continue)` user turn on the same-format (openai→openai) lane when history has no user turn, mirroring the Claude-lane `_ensureUserTurn` behavior — upstream 400 `[1214] The messages parameter is illegal` fixed +- **fix(embeddings):** per-item string inputs are clamped to 20k chars before upstream dispatch so oversized documents no longer fail with "Invalid 'input': maximum context length is 8192 tokens" +- **fix(config):** egress-IP lockout provider family is overridable via `OMNIROUTE_EGRESS_IP_LOCK_PROVIDERS` (`none` disables the shared-IP cascade parking for paid per-workspace plans) diff --git a/config/quality/eslint-suppressions.json b/config/quality/eslint-suppressions.json index fe963b547c9c..65b039ba4971 100644 --- a/config/quality/eslint-suppressions.json +++ b/config/quality/eslint-suppressions.json @@ -633,11 +633,6 @@ "count": 1 } }, - "open-sse/services/tlsClientBase.ts": { - "@typescript-eslint/no-unused-vars": { - "count": 1 - } - }, "open-sse/services/tokenLimitCounter.ts": { "@typescript-eslint/no-unused-vars": { "count": 1 @@ -2753,7 +2748,7 @@ }, "tests/unit/build/check-licenses.test.ts": { "@typescript-eslint/no-explicit-any": { - "count": 5 + "count": 4 } }, "tests/unit/bypass-handler.test.ts": { @@ -2862,7 +2857,7 @@ }, "tests/unit/chatcore-translation-paths.test.ts": { "@typescript-eslint/no-explicit-any": { - "count": 31 + "count": 30 } }, "tests/unit/check-docs-symbols.test.ts": { @@ -3348,7 +3343,7 @@ }, "tests/unit/combo-routing-engine.test.ts": { "@typescript-eslint/no-explicit-any": { - "count": 268 + "count": 267 } }, "tests/unit/combo-same-provider-cascade.test.ts": { @@ -5367,32 +5362,9 @@ "count": 1 } }, - "tests/unit/ui/use-improve-prompt.test.tsx": { - "react-hooks/immutability": { - "count": 1 - } - }, - "tests/unit/ui/use-presets.test.tsx": { - "react-hooks/immutability": { - "count": 1 - } - }, "tests/unit/ui/use-stream-metrics.test.tsx": { "@typescript-eslint/no-unused-vars": { "count": 1 - }, - "react-hooks/immutability": { - "count": 1 - } - }, - "tests/unit/ui/use-structured-output.test.tsx": { - "react-hooks/immutability": { - "count": 1 - } - }, - "tests/unit/ui/use-tools-builder.test.tsx": { - "react-hooks/immutability": { - "count": 1 } }, "tests/unit/ui/use-traffic-stream.test.tsx": { diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 56752d58b968..64d49ea20bfa 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -413,7 +413,7 @@ "_rebaseline_pr1043_minimax_tts": "Upstream port decolua/9router#1043 (toanalien) own growth: audioSpeech.ts 965->1061 (+96). Adds MiniMax T2A v2 TTS dispatch (handleMinimaxSpeech + hexToBytes helper) — provider entry was already in audioRegistry (format: minimax-tts) but no handler existed, falling through to the OpenAI-compatible default that fails (T2A has custom shape + hex-encoded audio + base_resp envelope). New branch sits next to the other inline provider branches (xiaomi-mimo, coqui, tortoise, aws-polly) — extracting would just create indirection. Covered by tests/unit/minimax-tts-1043.test.ts (3 tests, GREEN: success, base_resp error, invalid-hex).", "_rebaseline_pr4592_exclude_exhausted_auto": "Reconcile #4592 already-merged growth: combo.ts 2991->3036 (+45, terminal-status quota-cutoff exclusion in buildAutoCandidates + opt-in gate). Fast-gate PR->release does not run check:file-size.", "open-sse/executors/antigravity.ts": 1665, - "open-sse/executors/base.ts": 1751, + "open-sse/executors/base.ts": 1762, "open-sse/executors/chatgpt-web.ts": 5056, "open-sse/executors/codex.ts": 1505, "open-sse/executors/cursor.ts": 1759, @@ -425,16 +425,16 @@ "open-sse/mcp-server/server.ts": 1572, "open-sse/services/accountFallback.ts": 2467, "open-sse/services/adobeFireflyBrowserLogin.ts": 1401, - "open-sse/services/combo.ts": 4080, + "open-sse/services/combo.ts": 4219, "open-sse/translator/response/openai-responses.ts": 1466, "open-sse/utils/cursorAgentProtobuf.ts": 1547, - "open-sse/utils/proxyFetch.ts": 1271, - "open-sse/utils/stream.ts": 3072, + "open-sse/utils/proxyFetch.ts": 1435, + "open-sse/utils/stream.ts": 3080, "open-sse/vendor/codex-chatgpt-web/adapters/chatgpt-web/browser-worker.ts": 4398, "open-sse/vendor/codex-chatgpt-web/bridge.ts": 1335, "src/app/(dashboard)/dashboard/HomePageClient.tsx": 1344, "src/app/(dashboard)/dashboard/api-manager/ApiManagerPageClient.tsx": 3186, - "src/app/(dashboard)/dashboard/combos/page.tsx": 5018, + "src/app/(dashboard)/dashboard/combos/page.tsx": 5025, "src/app/(dashboard)/dashboard/costs/CostOverviewTab.tsx": 1319, "src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.tsx": 2491, "src/app/(dashboard)/dashboard/providers/[id]/components/modals/EditConnectionModal.tsx": 1631, @@ -457,7 +457,7 @@ "src/shared/components/RequestLoggerV2.tsx": 1718, "src/shared/constants/providers/apikey/gateways.ts": 1462, "src/shared/services/cliRuntime.ts": 1296, - "src/sse/handlers/chat.ts": 2454, + "src/sse/handlers/chat.ts": 2467, "src/sse/services/auth.ts": 3450, "tests/unit/account-fallback-service.test.ts": 2453, "tests/unit/provider-validation-specialty.test.ts": 4656 @@ -643,5 +643,6 @@ "_rebaseline_2026_09_03_error_boundary_campaign": "Campanha de error-boundary (#12431 #12438 #12444 #12454 #12455 #12456 #12457 #12458 #12459 #12465 #12466 #12467 #12469 #12435), medido no tip com os 14 mergeados. open-sse/executors/codex.ts 1499->1505: os primeiros 4 (1499->1503) sao DRIFT ANTERIOR a esta campanha, ja presente no tip antes dela; os 2 ultimos (1503->1505) sao do #12444, que fecha o boundary de falha da resposta do Codex. Absorver o drift junto foi inevitavel porque o cap e um numero so, mas fica registrado aqui que 4 das 6 linhas nao sao desta leva. open-sse/vendor/codex-chatgpt-web/bridge.ts 1322->1335 (+13): tambem do #12444, no mesmo caminho de falha. NAO cobre open-sse/utils/stream.ts, que segue violando por drift anterior e independente.", "_rebaseline_2026_09_03_12352_apikey_acl": "PR #12352 (fix/api-key-create-acl-12275) crescimento proprio: src/lib/db/apiKeys.ts 1610->1625 (+15). A criacao de API key descartava a ACL enviada no payload; preservar essa ACL exige carregar e persistir o conjunto no mesmo chokepoint de INSERT do modulo de dominio, sem extracao possivel sem partir a funcao de criacao ao meio. Coberto pelos testes do proprio PR (54/54 focados na leva).", "_rebaseline_2026_09_03_houminxi_combo_stacked": "Leva HouMinXi (#12624 #12626 #12632 #12637): open-sse/services/combo.ts 4075->4080 (+5), medido no tip com os quatro mergeados. Cada PR registrou o proprio crescimento contra o tip de onde forkou (o #12637 ja subira o cap para 4075); as 5 linhas restantes so aparecem quando eles empilham, porque mais de um toca o mesmo chokepoint de scoring reset-aware em combo.ts. Fiacao em ponto existente, sem extracao possivel sem partir a funcao de selecao de alvos. Coberto por combo-strategies e reset-aware-request-scope-12600 (119/119 focados na leva).", - "_rebaseline_2026_09_04_12641_continuation_effective_input": "PR #12641 crescimento proprio: src/sse/handlers/chat.ts 2450->2454 (+4). A continuacao por previous_response_id encadeava a partir de clientRawRequest.body.input, que e capturado ANTES da reconstrucao do proprio chat.ts; quando o turno anterior ja era uma continuacao, esse campo guarda so o delta do cliente, e o erro se acumulava a cada salto ate a reconstrucao virar itens de tool sem prefixo. Persistir o input EFETIVO exige as linhas no ponto onde a reconstrucao termina, dentro do fluxo de despacho. Coberto por tests/unit/responses-continuation-store.test.ts (22/22 focados na leva)." + "_rebaseline_2026_09_04_12641_continuation_effective_input": "PR #12641 crescimento proprio: src/sse/handlers/chat.ts 2450->2454 (+4). A continuacao por previous_response_id encadeava a partir de clientRawRequest.body.input, que e capturado ANTES da reconstrucao do proprio chat.ts; quando o turno anterior ja era uma continuacao, esse campo guarda so o delta do cliente, e o erro se acumulava a cada salto ate a reconstrucao virar itens de tool sem prefixo. Persistir o input EFETIVO exige as linhas no ponto onde a reconstrucao termina, dentro do fluxo de despacho. Coberto por tests/unit/responses-continuation-store.test.ts (22/22 focados na leva).", + "_rebaseline_2026_09_05_bounded_abort_and_tls_ladder": "Resilience work 2026-09-04/05." } diff --git a/next.config.mjs b/next.config.mjs index f4d4622c90b6..f0de7831c43e 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -243,6 +243,8 @@ const nextConfig = { webpackMemoryOptimizations: true, // Run webpack in a separate Node worker, lowering main-process memory. webpackBuildWorker: true, + // Cap page-data/export workers on memory-constrained machines (default: ncpu-1). + cpus: process.env.NEXT_BUILD_CPUS ? Number(process.env.NEXT_BUILD_CPUS) : undefined, // Next.js proxy (middleware) has a default 10MB body clone limit. File // uploads (OpenAI-compatible /v1/files) routinely exceed this. Match the // 512 MB server-side cap; tune via env if needed. diff --git a/open-sse/config/providerErrorRules.ts b/open-sse/config/providerErrorRules.ts index 6f00d2c5f121..76b9ded4146d 100644 --- a/open-sse/config/providerErrorRules.ts +++ b/open-sse/config/providerErrorRules.ts @@ -343,7 +343,43 @@ export function honorsRuleLockScope(provider: string | null | undefined): boolea * HONORS_RULE_LOCK_SCOPE_PROVIDERS (#10334): a provider must opt in, and any * widening is an explicit owner decision. */ -const EGRESS_BUCKETED_LOCK_PROVIDERS = new Set(["opencode", "opencode-go", "opencode-cli"]); +// Default free-tier set (#9611). PAID plans are account-bucketed, not +// IP-bucketed, so operators running paid subscriptions can disable or trim +// the egress-IP lockout via OMNIROUTE_EGRESS_IP_LOCK_PROVIDERS: +// unset → default set below (free-tier behavior) +// none/off/false/0 → empty set (egress-IP lockout fully disabled) +// "a,b,c" → exact replacement set +// Static default as a plain lookup table; the runtime set is env-derived +// (dynamic membership), so a Set is the right structure there. +const EGRESS_BUCKETED_LOCK_PROVIDERS_DEFAULT: Record = { + opencode: true, + "opencode-go": true, + "opencode-cli": true, +}; + +/** + * Pure parser for OMNIROUTE_EGRESS_IP_LOCK_PROVIDERS (exported for tests). + * Returns the effective provider set: default free-tier family when unset, + * empty for none/off/false/0, otherwise the comma-separated replacement set. + */ +export function egressIpLockProvidersFromEnv( + raw: string | undefined, + fallback: Record = EGRESS_BUCKETED_LOCK_PROVIDERS_DEFAULT +): Set { + const trimmed = raw?.trim(); + if (trimmed === undefined || trimmed === "") return new Set(Object.keys(fallback)); + if (/^(none|off|false|0)$/i.test(trimmed)) return new Set(); + return new Set( + trimmed + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean) + ); +} + +const EGRESS_BUCKETED_LOCK_PROVIDERS = egressIpLockProvidersFromEnv( + process.env.OMNIROUTE_EGRESS_IP_LOCK_PROVIDERS +); export function isEgressBucketedLockScope(provider: string | null | undefined): boolean { return !!provider && EGRESS_BUCKETED_LOCK_PROVIDERS.has(provider.toLowerCase()); diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index df443a6f7bfb..bde57ccf3809 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -54,6 +54,14 @@ export function generateLegacyProviders(): Record { if (typeof entry.timeoutMs === "number") { p.timeoutMs = entry.timeoutMs; } + // #11526 follow-up: the headers-wait cap override must reach the executor's + // LegacyProvider config — dropping it here silently fell back to the 110s + // global cap even for providers whose registry entry overrides it (observed: + // opencode-go deepseek thinking generations 504ing at exactly 110s despite + // fetchStartTimeoutCapMs on the registry entry). + if (typeof entry.fetchStartTimeoutCapMs === "number") { + p.fetchStartTimeoutCapMs = entry.fetchStartTimeoutCapMs; + } // Headers const mergedHeaders = { diff --git a/open-sse/config/providers/registry/command-code/index.ts b/open-sse/config/providers/registry/command-code/index.ts index f298bcc21fb8..2a8bd998c890 100644 --- a/open-sse/config/providers/registry/command-code/index.ts +++ b/open-sse/config/providers/registry/command-code/index.ts @@ -17,6 +17,13 @@ export const command_codeProvider: RegistryEntry = { // The discovery response is a partial routing catalog; static registry // entries omitted from it can still be accepted by the gateway. liveCatalogAuthoritative: false, + // Reasoning models (e.g. z-ai/glm-5.3-flash) exhaust small client budgets on + // thinking before emitting content; default to a reasoning-safe budget. + requestDefaults: { maxTokens: 16_384 }, + // Console Go / Command Code gateways buffer entire generations — no upstream + // bytes flow until the model finishes thinking. Streaming needs a headers-wait + // ceiling well above the 110s global cap for long reasoning generations. + fetchStartTimeoutCapMs: 600_000, authType: "apikey", authHeader: "Authorization", authPrefix: "Bearer ", diff --git a/open-sse/config/providers/registry/opencode/go/index.ts b/open-sse/config/providers/registry/opencode/go/index.ts index 645a828fedac..d82306bc6feb 100644 --- a/open-sse/config/providers/registry/opencode/go/index.ts +++ b/open-sse/config/providers/registry/opencode/go/index.ts @@ -13,6 +13,13 @@ export const opencode_goProvider: RegistryEntry = { authHeader: "Authorization", authPrefix: "Bearer", defaultContextLength: 200000, + // glm-5.3-flash and other always-thinking models need a generous output + // budget or reasoning consumes every token before content is emitted. + requestDefaults: { maxTokens: 16_384 }, + // Console Go / Command Code gateways buffer entire generations — no upstream + // bytes flow until the model finishes thinking. Streaming needs a headers-wait + // ceiling well above the 110s global cap for long reasoning generations. + fetchStartTimeoutCapMs: 600_000, models: [ // Port from decolua/9router 8efacc11: align with official Go endpoints — // glm-5.2 is now advertised and Kimi chat traffic must route through diff --git a/open-sse/config/providers/shared.ts b/open-sse/config/providers/shared.ts index 8b9b45bfec0a..3c2bdd8fe17d 100644 --- a/open-sse/config/providers/shared.ts +++ b/open-sse/config/providers/shared.ts @@ -166,6 +166,11 @@ export interface RegistryEntry { chatPath?: string; clientVersion?: string; timeoutMs?: number; + /** Headers-wait ceiling override for streaming requests (#11526). Gateways + * that buffer entire generations (Console Go / Command Code) need this well + * above the 110s global cap — generateLegacyProviders() copies it into the + * executor's LegacyProvider config. */ + fetchStartTimeoutCapMs?: number; passthroughModels?: boolean; /** * Whether a non-empty synchronized live model list is exhaustive enough @@ -264,6 +269,7 @@ export interface LegacyProvider { chatPath?: string; clientVersion?: string; timeoutMs?: number; + fetchStartTimeoutCapMs?: number; } export const buildModels = (ids: readonly string[]): RegistryModel[] => diff --git a/open-sse/executors/base.ts b/open-sse/executors/base.ts index b18c27e20ce4..cc376a5f516b 100644 --- a/open-sse/executors/base.ts +++ b/open-sse/executors/base.ts @@ -1,4 +1,5 @@ import { HTTP_STATUS, FETCH_TIMEOUT_MS } from "../config/constants.ts"; +import { applyProviderRequestDefaults } from "../services/providerRequestDefaults.ts"; import { getRegistryEntry } from "../config/providerRegistry.ts"; import { resolveFetchStartTimeout } from "../utils/fetchStartTimeoutPolicy.ts"; import { @@ -162,6 +163,7 @@ export type ProviderConfig = { headers?: Record; requestDefaults?: ProviderRequestDefaults; timeoutMs?: number; + fetchStartTimeoutCapMs?: number; format?: string; }; @@ -584,10 +586,14 @@ export class BaseExecutor { stripInternalBodyFields(cloned); - return cloned; + // Registry requestDefaults (e.g. maxTokens for reasoning-heavy models like + // GLM-5.3-flash) fill fields the client left unset. No-op for providers + // without requestDefaults and idempotent for DefaultExecutor, which + // re-applies the same helper downstream. + return applyProviderRequestDefaults(cloned, this.config?.requestDefaults ?? null); } - return body; + return applyProviderRequestDefaults(body, this.config?.requestDefaults ?? null); } shouldRetry(status: number, urlIndex: number) { @@ -913,6 +919,10 @@ export class BaseExecutor { const fetchStartTimeoutPolicy = resolveFetchStartTimeout({ baseTimeoutMs: this.getTimeoutMs(), stream, + // Providers with non-incremental upstreams (whole generation buffered + // behind the gateway, e.g. opencode-go's Console Go GLM tier) can take + // minutes before first bytes; the registry overrides the 110s cap. + capMs: this.config?.fetchStartTimeoutCapMs, }); const fetchStartTimeoutMs = fetchStartTimeoutPolicy.timeoutMs; if (fetchStartTimeoutPolicy.capped) { diff --git a/open-sse/handlers/embeddings.ts b/open-sse/handlers/embeddings.ts index c60842b09fcb..8a350c62c4e0 100644 --- a/open-sse/handlers/embeddings.ts +++ b/open-sse/handlers/embeddings.ts @@ -251,6 +251,23 @@ function validateRequestedModalities(runtime: EmbeddingRuntime): EmbeddingFailur : null; } +/** + * Conservative per-item character budget for embedding string inputs. 20k + * chars stays under the smallest common embedding context (8192 tokens, + * text-embedding-3-small) even for code/CJK-heavy text. Array items are + * clamped individually; non-string items (structured/native modalities) are + * passed through untouched. + */ +const MAX_EMBEDDING_INPUT_CHARS = 20_000; + +export function clampEmbeddingStringInput(input: unknown): unknown { + const clampString = (s: string): string => + s.length > MAX_EMBEDDING_INPUT_CHARS ? s.slice(0, MAX_EMBEDDING_INPUT_CHARS) : s; + if (typeof input === "string") return clampString(input); + if (Array.isArray(input)) return input.map((item) => (typeof item === "string" ? clampString(item) : item)); + return input; +} + function buildUpstreamBody(runtime: EmbeddingRuntime): Record { const upstreamBody: Record = { model: runtime.model, diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 870a470749b5..9b2e52904948 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -143,7 +143,13 @@ import { type ResilienceSettings, type ComboCooldownWaitSettings, } from "../../src/lib/resilience/settings"; -import { resolveReasoningBufferedMaxTokens, toPositiveInteger } from "./reasoningTokenBuffer.ts"; +import { + resolveReasoningBufferedMaxTokens, + toPositiveInteger, + isTinyBudgetReasoningProbe, + buildReasoningProbeTruncatedResponse, + REASONING_BUFFER_MIN_TRIGGER, +} from "./reasoningTokenBuffer.ts"; import { RESET_WINDOW_NAMES } from "./combo/types.ts"; import type { ComboLike, @@ -176,6 +182,7 @@ import { releaseQualityClone, releaseRejectedQualityResponse, toRetryAfterDisplayValue, + isReasoningConsumedQualityRejection, } from "./combo/validateQuality.ts"; import { resolveComboCooldownWaitDecision, @@ -219,6 +226,8 @@ import { getPersistedConnectionCooldownSkipReason, resolvePersistedConnectionCooldownSkipReason, isContextOverflow400, + isTransportClassNetworkError, + shouldAbortComboForTransportFailures, isParamValidation400, isModelScoped400, } from "./combo/comboPredicates.ts"; @@ -267,6 +276,7 @@ import { getComboTrace, recordComboDecision, startComboTrace, + summarizeSkippedTargets, } from "./combo/decisionTrace.ts"; import { QUOTA_SOFT_DEPRIORITIZE_FACTOR, @@ -1182,6 +1192,9 @@ async function handleComboChatInner({ const startTime = Date.now(); let fallbackCount = 0; let recordedAttempts = 0; + // Bounded transport abort: consecutive transport-class failures share the + // same egress — after the bound, remaining targets would fail identically. + let consecutiveTransportFailures = 0; comboErrors = []; // QA P0: assemble a sanitized diagnostic trace from the state already in scope @@ -1203,6 +1216,10 @@ async function handleComboChatInner({ attemptOrder: comboAttemptOrder, terminalReason, recovery: buildRecoveryHint(terminalReason, retryAfterSeconds), + // #12294: per-target pre-dispatch skip reasons so operators can see WHY + // every target was filtered (quota wall vs allowlist vs lockout) and the + // earliest cooldown reset instead of an opaque 503. + ...summarizeSkippedTargets(getComboTrace(traceInvocationId)), }); let globalResolve: ((res: Response) => void) | null = null; @@ -1346,6 +1363,15 @@ async function handleComboChatInner({ if (persistedSkip) { log.info("COMBO", persistedSkip); clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + // #12294: the cooldown detail (with reset timestamp) feeds the + // ALL_TARGETS_SKIPPED response diagnostics via the decision trace. + recordComboDecision(traceInvocationId, { + step: target.executionKey, + target: modelStr, + decision: "skipped_before_dispatch", + reason: "persisted_cooldown", + detail: persistedSkip, + }); if (i > 0) fallbackCount++; return null; } @@ -1785,9 +1811,41 @@ async function handleComboChatInner({ "COMBO", `Model ${modelStr} returned 200 but failed quality check: ${quality.reason}` ); - // #6692: a quality-rejected 200 never marks the connection row - // unhealthy, so the sticky pin's lazy headroom recheck would never - // catch it either — release it here, on the failing response. + // #10281 (combo path): a tiny-budget reasoning probe (max_tokens + // below REASONING_BUFFER_MIN_TRIGGER, e.g. Claude Code's /model + // check) whose upstream 200 burned the entire budget on thinking + // is a budget-truncated response, not a provider fault. Answer + // with a valid truncated 200 (finish_reason "length") instead of + // burning fallbacks and poisoning model-lockout bookkeeping. + if ( + !clientRequestedStream && + isTinyBudgetReasoningProbe({ model: modelStr, body: attemptBody }) && + isReasoningConsumedQualityRejection(quality.reason) + ) { + log.warn( + "COMBO", + `Reasoning probe (max_tokens < ${REASONING_BUFFER_MIN_TRIGGER}) answered with truncated 200 — quality: ${quality.reason}` + ); + recordComboRequest(combo.name, modelStr, { + success: true, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + return { + ok: true, + response: buildReasoningProbeTruncatedResponse({ + model: modelStr, + maxTokens: toPositiveInteger( + (attemptBody as Record | undefined)?.max_tokens ?? + (attemptBody as Record | undefined)?.max_completion_tokens + ), + requestId: traceInvocationId, + }), + }; + } releaseStickyPinOnFailure(_sticky.messageHash, effectiveConnectionId); recordComboRequest(combo.name, modelStr, { success: false, @@ -2536,6 +2594,39 @@ async function handleComboChatInner({ }); lastStatus = result.status; if (i > 0) fallbackCount++; + // Bounded transport abort (#12294 incident follow-up): consecutive + // transport-class failures share the same egress — after N, every + // remaining target would fail identically. Abort the whole combo + // instead of burning the remaining fan-out (the 2026-09-04 wedge + // trigger: opencode h2 resets -> 8 fallbacks/18 decisions -> pool + // saturation -> client abort during cleanup). + if (isTransportClassNetworkError(errorText)) { + consecutiveTransportFailures++; + if (shouldAbortComboForTransportFailures(consecutiveTransportFailures)) { + log.warn( + "COMBO", + `${consecutiveTransportFailures} consecutive transport-class failures (last: ${modelStr}: ${errorText}) — aborting combo; remaining targets share the same egress` + ); + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordComboFailure(effectiveSessionId, combo.name); + return { + ok: false, + response: errorResponse( + 502, + `All upstream transports are failing at the network layer (last: ${errorText}) — try again shortly` + ), + }; + } + } else { + consecutiveTransportFailures = 0; + } + if (i > 0) fallbackCount++; // Wire combo failures into the resilience dashboard (model-level lockout) // alongside the provider-level cooldown below — they govern different scopes. if (provider && rawModel && !scopedFailure) { @@ -3309,6 +3400,7 @@ async function handleRoundRobinCombo({ let lastError: string | null = null; let lastStatus: number | null = null; let earliestRetryAfter: ComboRetryAfter | null = null; + let consecutiveTransportFailures = 0; let globalAttempts = 0; let fallbackCount = 0; let recordedAttempts = 0; @@ -3576,6 +3668,35 @@ async function handleRoundRobinCombo({ "COMBO-RR", `${modelStr} returned 200 but failed quality check: ${quality.reason}` ); + // #10281 (combo path): tiny-budget reasoning probes get a + // truncated 200 (see handleComboChat's quality-fail branch for + // the full rationale) instead of burning RR fallback slots. + if ( + !clientRequestedStream && + isTinyBudgetReasoningProbe({ model: modelStr, body: attemptBody }) && + isReasoningConsumedQualityRejection(quality.reason) + ) { + log.warn( + "COMBO-RR", + `Reasoning probe (max_tokens < ${REASONING_BUFFER_MIN_TRIGGER}) answered with truncated 200 — quality: ${quality.reason}` + ); + recordComboRequest(combo.name, modelStr, { + success: true, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy: "round-robin", + target: toRecordedTarget(target), + }); + recordedAttempts++; + return buildReasoningProbeTruncatedResponse({ + model: modelStr, + maxTokens: toPositiveInteger( + (attemptBody as Record | undefined)?.max_tokens ?? + (attemptBody as Record | undefined)?.max_completion_tokens + ), + requestId: createInvocationId(), + }); + } // #6692: same rationale as handleComboChat's quality-fail branch — // a quality-rejected 200 never marks the connection row unhealthy, // so release the sticky pin here rather than on the next turn. @@ -3919,6 +4040,22 @@ async function handleRoundRobinCombo({ kind: classifyComboOutcome(result.status, errorText), }); if (offset > 0) fallbackCount++; + // Bounded transport abort — RR parity with handleComboChat above. + if (isTransportClassNetworkError(errorText)) { + consecutiveTransportFailures++; + if (shouldAbortComboForTransportFailures(consecutiveTransportFailures)) { + log.warn( + "COMBO-RR", + `${consecutiveTransportFailures} consecutive transport-class failures (last: ${modelStr}: ${errorText}) — aborting combo; remaining targets share the same egress` + ); + return errorResponse( + 502, + `All upstream transports are failing at the network layer (last: ${errorText}) — try again shortly` + ); + } + } else { + consecutiveTransportFailures = 0; + } log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status, errorBody: redactConnectionLabel(errorText), diff --git a/open-sse/services/combo/comboPredicates.ts b/open-sse/services/combo/comboPredicates.ts index 6359607cacd3..46d7760dbc73 100644 --- a/open-sse/services/combo/comboPredicates.ts +++ b/open-sse/services/combo/comboPredicates.ts @@ -598,6 +598,32 @@ export async function resolvePersistedConnectionCooldownSkipReason( return getPersistedConnectionCooldownSkipReason(target, connection, allowRateLimitedConnection); } + +/** + * Transport-class network failure: the egress socket/h2 stream itself broke + * (h2 stream reset, socket closed mid-flight, connect/reset/timeout). These + * failures are SHARED across every target behind the same egress path — the + * 2026-09-04 incident saw opencode h2 resets wedge a full multi-target + * fallback fan-out where every subsequent target died identically. Bounded + * abort: after N consecutive transport failures, stop the combo instead of + * burning the remaining targets (see DEFAULT_TRANSPORT_ABORT_BOUND). + */ +const TRANSPORT_NETWORK_ERROR_RE = + /ERR_HTTP2_STREAM_ERROR|NGHTTP2_[A-Z_]+|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|ECONNRESET|ECONNREFUSED|EPIPE|EADDRNOTAVAIL|EHOSTUNREACH|ENETUNREACH|ETIMEDOUT|socket hang up|other side closed|fetch failed/i; + +export function isTransportClassNetworkError(errorText: string | null | undefined): boolean { + return TRANSPORT_NETWORK_ERROR_RE.test(String(errorText || "")); +} + +export const DEFAULT_TRANSPORT_ABORT_BOUND = 3; + +export function shouldAbortComboForTransportFailures( + consecutiveCount: number, + bound: number = DEFAULT_TRANSPORT_ABORT_BOUND +): boolean { + return Number.isFinite(consecutiveCount) && bound > 0 && consecutiveCount >= bound; +} + /** @param {string} errorText */ export function isContextOverflow400(errorText: string | null | undefined): boolean { const text = String(errorText || ""); diff --git a/open-sse/services/combo/decisionTrace.ts b/open-sse/services/combo/decisionTrace.ts index 7660af7ea01f..7a38b1dba032 100644 --- a/open-sse/services/combo/decisionTrace.ts +++ b/open-sse/services/combo/decisionTrace.ts @@ -29,12 +29,12 @@ export const COMBO_SKIP_REASONS = [ "concurrency_cap", "admission_lane", "predictive_ttft", + "persisted_cooldown", ] as const; export type ComboSkipReason = (typeof COMBO_SKIP_REASONS)[number]; export type ComboDecision = "dispatched" | "skipped_before_dispatch" | "not_reached"; - export interface ComboTraceEntry { /** Safe internal identifier of the combo step (execution key). */ step: string; @@ -42,6 +42,8 @@ export interface ComboTraceEntry { target: string; decision: ComboDecision; reason?: ComboSkipReason; + /** Optional human-readable detail (e.g. cooldown reset timestamp). Never contains credentials. */ + detail?: string; ts: number; } @@ -120,6 +122,7 @@ export function recordComboDecision( target: entry.target, decision: entry.decision, reason: entry.reason as ComboSkipReason | undefined, + detail: entry.detail, ts: Date.now(), }); } @@ -173,3 +176,40 @@ function pruneExpired(): void { if (now - trace.createdAt > TRACE_TTL_MS) traces.delete(id); } } + +/** + * #12294: structured per-target view for the ALL_TARGETS_SKIPPED terminal state. + * Aggregates every pre-dispatch skip decision into a redacted summary the 503 + * response can carry, plus the earliest future cooldown reset (parsed from + * persisted-cooldown detail lines) so clients learn when to retry instead of + * hammering a fully-quota-walled combo. + * Pure function over the trace — no DB, no clock dependency beyond Date parsing. + */ +export function summarizeSkippedTargets(trace: ComboTrace | null): { + skippedTargets: Array<{ target: string; reason: ComboSkipReason; detail?: string }>; + nextRetryAt: string | null; +} { + if (!trace) return { skippedTargets: [], nextRetryAt: null }; + const skippedTargets: Array<{ target: string; reason: ComboSkipReason; detail?: string }> = []; + let earliestMs = Number.POSITIVE_INFINITY; + for (const entry of trace.decisions) { + if (entry.decision !== "skipped_before_dispatch" || !entry.reason) continue; + const item: { target: string; reason: ComboSkipReason; detail?: string } = { + target: entry.target, + reason: entry.reason, + }; + if (entry.detail) item.detail = entry.detail; + skippedTargets.push(item); + if (entry.reason === "persisted_cooldown" && entry.detail) { + const match = entry.detail.match(/until (\d{4}-\d{2}-\d{2}T[\d:.]+Z)/); + if (match) { + const ms = new Date(match[1]).getTime(); + if (Number.isFinite(ms) && ms > Date.now() && ms < earliestMs) earliestMs = ms; + } + } + } + return { + skippedTargets, + nextRetryAt: Number.isFinite(earliestMs) ? new Date(earliestMs).toISOString() : null, + }; +} diff --git a/open-sse/services/combo/validateQuality.ts b/open-sse/services/combo/validateQuality.ts index 723a10b61451..6ba7173322fb 100644 --- a/open-sse/services/combo/validateQuality.ts +++ b/open-sse/services/combo/validateQuality.ts @@ -832,6 +832,21 @@ export async function validateResponseQuality( }; } +/** + * True when a quality rejection describes the "model burned the whole + * max_tokens budget on reasoning and produced no visible content" outcome + * (#3587 — the reason string is produced by validateResponseQuality above and + * its `reasoning consumed N/M tokens` prefix is load-bearing). Combined with + * isTinyBudgetReasoningProbe it identifies a budget-truncated reasoning probe, + * which the combo dispatchers answer with a truncated 200 instead of a 502 + + * model lockout (#10281 combo-path parity). + */ +export function isReasoningConsumedQualityRejection( + reason: string | null | undefined +): boolean { + return !!reason && /^reasoning consumed \d+\/\d+ tokens/.test(reason); +} + /** * Release the peek-and-abandon clone used by {@link validateResponseQuality}. * diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index 5e6001766298..ad3f200cfd4a 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -202,6 +202,28 @@ export function isCloudflareFingerprintRejection(errorText: string): boolean { text.includes("fingerprint_rejection") ); } +// A fingerprint rejection means the CDN banned our TLS/UA signature — exactly +// what OmniRoute's optional Chrome-124 impersonation transport fixes +// (open-sse/utils/tlsClient.ts via wreq-js). That transport is inert unless the +// optional `wreq-js` dependency is installed and ENABLE_TLS_FINGERPRINT=true, +// so the first rejection points the operator at the remediation instead of +// leaving repeated 1010s looking like account trouble (#9929 classified the +// symptom; this surfaces the fix). +let fingerprintRemediationHintEmitted = false; + +export function resetFingerprintRemediationHintForTest(): void { + fingerprintRemediationHintEmitted = false; +} + +function maybeEmitFingerprintRemediationHint(): void { + if (fingerprintRemediationHintEmitted || process.env.ENABLE_TLS_FINGERPRINT === "true") return; + fingerprintRemediationHintEmitted = true; + console.warn( + "[errorClassifier] Cloudflare fingerprint rejection (1010 / browser-signature ban) detected. " + + "Remediation: install the optional 'wreq-js' dependency and set ENABLE_TLS_FINGERPRINT=true " + + "(plus TLS_FINGERPRINT_PROVIDERS= to cover proxied egress) so egress presents a browser TLS fingerprint." + ); +} function responseBodyToString(responseBody: unknown): string { if (typeof responseBody === "string") return responseBody; @@ -325,6 +347,7 @@ export function classifyProviderError( } if (statusCode === 403 && isCloudflareFingerprintRejection(bodyStr)) { + maybeEmitFingerprintRemediationHint(); // Cloudflare 1010 / error_name "browser_signature_banned": the CDN in front of the // upstream (e.g. opencode.ai/zen/v1) rejected the CLIENT's TLS/UA signature, not the // account's credentials. It says nothing about account health — a different client on diff --git a/open-sse/services/opencodeQuotaFetcher.ts b/open-sse/services/opencodeQuotaFetcher.ts index 5420e263e1b1..f49a14cd8ccd 100644 --- a/open-sse/services/opencodeQuotaFetcher.ts +++ b/open-sse/services/opencodeQuotaFetcher.ts @@ -12,6 +12,7 @@ import { registerQuotaFetcher, registerQuotaWindows, type QuotaInfo } from "./quotaPreflight.ts"; import { registerMonitorFetcher } from "./quotaMonitor.ts"; import { throttleQuotaFetch } from "./quotaFetchThrottle.ts"; +import { buildOpencodeBackgroundHeaders } from "../utils/opencodeHeaders.ts"; const OPENCODE_QUOTA_URL = process.env.OMNIROUTE_OPENCODE_QUOTA_URL ?? "https://opencode.ai/zen/go/v1/usage"; @@ -181,6 +182,11 @@ export async function fetchOpencodeQuota( Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: "application/json", + // OpenCode operator warning: background calls on the bare runtime UA + // ("Bun fetch") without x-opencode-session are flagged, with hard + // errors announced from 2026-09-06. Send the CLI identity + a stable + // per-connection session fingerprint (see buildOpencodeBackgroundHeaders). + ...buildOpencodeBackgroundHeaders({ seed: connectionId }), }, signal: AbortSignal.timeout(8_000), }); diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index eda968bae1d7..5f662cb739b3 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -190,9 +190,13 @@ function isReasoningOnlyReplayTarget(provider: unknown, model: unknown): boolean * an absent field was verified to be accepted — but Xiaomi MiMo still 400s * ("Param Incorrect: The reasoning_content in the thinking mode must be passed * back to the API", 9router#1321/#1337), so omitting the field there trades one - * live bug for another. Keep the placeholder only for those providers; the echo - * that comes back is still stripped on the way in by - * isInternalReasoningPlaceholder(), so it never re-poisons cache or history. + * live bug for another. The opencode console gateways (opencode-go / + * opencode-zen / opencode) enforce the same contract on their Responses + * transport for deepseek/glm thinking models (production: 54× 400 "The + * reasoning_text in the thinking mode must be passed back to the API" in one + * day). Keep the placeholder only for those providers; the echo that comes + * back is still stripped on the way in by isInternalReasoningPlaceholder(), so + * it never re-poisons cache or history. */ function requiresReasoningContentPresence(provider: unknown, model: unknown): boolean { const normalizedProvider = String(provider ?? "") @@ -201,7 +205,11 @@ function requiresReasoningContentPresence(provider: unknown, model: unknown): bo const normalizedModel = String(model ?? "") .trim() .toLowerCase(); - return normalizedProvider === "xiaomi-mimo" || /(^|\/)mimo/i.test(normalizedModel); + return ( + normalizedProvider === "xiaomi-mimo" || + /(^|\/)mimo/i.test(normalizedModel) || + ["opencode", "opencode-go", "opencode-zen"].includes(normalizedProvider) + ); } type OpenAIReplayOptions = { @@ -274,6 +282,20 @@ function replayOpenAIReasoningMessage( if (options.requiresExplicitReasoningReplay) { if (message.reasoning_content === "") delete message.reasoning_content; + // Presence-enforcing upstreams (opencode console gateways, Xiaomi MiMo) + // 400 when a thinking-mode replay turn lacks the reasoning field — even on + // a reasoning-cache miss, where the original summary is unavailable. Emit + // the internal sentinel so the Responses converter still emits the + // reasoning_text item; the echo coming back is stripped on the way in by + // isInternalReasoningPlaceholder(), so it never re-poisons cache or + // history (#9573). DeepSeek itself accepts an ABSENT field — untouched. + if ( + !message.reasoning_content && + (hasToolCalls || shouldReplayReasoningOnly) && + requiresReasoningContentPresence(options.provider, options.model) + ) { + message.reasoning_content = NON_ANTHROPIC_THINKING_PLACEHOLDER; + } return; } @@ -414,6 +436,26 @@ export function translateRequest( result.messages = hoistLeadingSystemMessage(result.messages, provider); } + // GLM-family upstreams (Z.AI / Zhipu console gateways) reject messages arrays + // with no role:"user" turn (400 [1214] "The messages parameter is illegal") — + // ALSO on the SAME-FORMAT (openai→openai) lane, where no source→openai + // translator runs to apply the _ensureUserTurn credential flag. Agent + // tool-loop continuations reach exactly that shape when every inbound user + // turn was tool_result-only and context compression evicted the original + // prompt (production evidence: opencode-go/glm-5.3-flash, 37× in one day). + // Mirrors claude-to-openai.ts's _ensureUserTurn branch: appending at the end + // keeps every earlier byte identical for upstream prompt caches. + if ( + isGlmFamilyUpstream && + targetFormat === FORMATS.OPENAI && + Array.isArray(result.messages) && + !result.messages.some( + (m) => m && typeof m === "object" && (m as Record).role === "user" + ) + ) { + result.messages.push({ role: "user", content: "(continue)" }); + } + if ( sourceFormat === FORMATS.OPENAI && targetFormat === FORMATS.OPENAI_RESPONSES && diff --git a/open-sse/translator/request/openai-responses/toResponses.ts b/open-sse/translator/request/openai-responses/toResponses.ts index 857f5dbffe66..f090c4f4d123 100644 --- a/open-sse/translator/request/openai-responses/toResponses.ts +++ b/open-sse/translator/request/openai-responses/toResponses.ts @@ -233,6 +233,19 @@ export function openaiToOpenAIResponsesRequest( // for opaque items in reasoningInputPolicy.ts (#11108). summary: [], }); + } else if (isInternalReasoningPlaceholder(reasoning)) { + // Reasoning-presence validation on strict Responses upstreams (opencode + // console gateways) rejects thinking-mode history whose assistant turns + // lack a reasoning_text item — even when OmniRoute's replay cache missed + // and the history only carries the internal sentinel (see + // replayOpenAIReasoningMessage's requiresExplicitReasoningReplay branch). + // Emit the sentinel text: it satisfies presence validation, and the + // response translators suppress the sentinel again on echo (#9573). + input.push({ + type: "reasoning", + content: [{ type: "reasoning_text", text: reasoning }], + summary: [], + }); } // Thinking blocks remain display-only here. They do not prove that the diff --git a/open-sse/utils/opencodeHeaders.ts b/open-sse/utils/opencodeHeaders.ts index 398d53c01178..660208eb0d43 100644 --- a/open-sse/utils/opencodeHeaders.ts +++ b/open-sse/utils/opencodeHeaders.ts @@ -144,3 +144,44 @@ function applyCliDefaults( headers["x-opencode-session"] ||= generateSessionId(sessionBody ?? null) || randomUUID(); } + +/** + * Build outbound headers for OmniRoute's BACKGROUND calls to opencode.ai — + * model-catalog discovery (autoSync/autoFetchModels), quota checks, and any + * other non-chat fetch. These calls previously went out on the bare runtime + * fetch (User-Agent "Bun fetch") with no `x-opencode-session`, which is exactly + * the shape OpenCode's operator warning names ("requests missing an + * x-opencode-session header", UA "Bun fetch"; hard errors announced from + * 2026-09-06). + * + * Background traffic has no conversation, so the session id is a stable, + * deterministic fingerprint seeded by the calling connection/workspace — the + * same hash family the chat path derives via generateSessionId() (#10571) — + * so each connection's background traffic groups under one identity across + * daemon restarts instead of looking like a new anonymous client per call. + * The User-Agent is synthesized to the OpenCode CLI identity for the same + * reason applyCliDefaults() does it on the chat path (#5997): generic runtime + * UAs from non-CLI callers get flagged upstream. + */ +export function buildOpencodeBackgroundHeaders(options?: { + /** Stable seed for the per-caller session id (connection id / workspace id). */ + seed?: string | null; + /** Explicit UA override (defaults to the OpenCode CLI identity). */ + userAgent?: string; +}): Record { + const headers: Record = {}; + applyCliDefaults(headers, { + userAgent: + options?.userAgent?.trim() || + process.env.OPENCODE_USER_AGENT?.trim() || + "opencode", + client: process.env.OPENCODE_CLIENT?.trim() || "desktop", + project: process.env.OPENCODE_PROJECT?.trim() || "global", + }); + if (options?.seed && options.seed.trim().length > 0) { + headers["x-opencode-session"] = + generateSessionId({ model: "background" }, { connectionId: options.seed.trim() }) || + headers["x-opencode-session"]; + } + return headers; +} diff --git a/open-sse/utils/proxyFetch.ts b/open-sse/utils/proxyFetch.ts index 8dd5d013e89a..2be3f15c8e93 100644 --- a/open-sse/utils/proxyFetch.ts +++ b/open-sse/utils/proxyFetch.ts @@ -128,6 +128,127 @@ export function setTlsClientForTest(client: TlsClientLike | null): void { activeTlsClient = client ?? tlsClient; } +// ── Adaptive TLS-fingerprint ladder ────────────────────────────────────────── +// The wreq-js (browser-JA3) transport fixes Cloudflare 1010 signature bans but +// empirically STALLS streaming inference: the response object resolves yet its +// body produces no bytes for minutes (observed 2026-09-03/04 — opencode-go, +// command-code, bailian all zero-byte while plain node fetch streamed the same +// requests in seconds). So the transport must never be eager by default: +// +// 1. Plain direct fetch is the fast path. +// 2. When a response is a Cloudflare fingerprint rejection (403 + +// error_code 1010 / browser_signature_banned), the provider is ARMED in a +// TTL cache and the request is retried once through wreq-js. +// 3. Every wreq response carries a first-byte watchdog: if the gateway +// buffers beyond the window, the request falls back to the direct +// dispatcher instead of hanging the caller. +// +// Operators can still force eager impersonation per provider with the legacy +// TLS_FINGERPRINT_PROVIDERS allowlist (then the watchdog is the only guard). +const FINGERPRINT_ARM_TTL_MS = 60 * 60 * 1000; +const fingerprintArmedUntil = new Map(); + +function tlsFirstByteWatchdogMs(): number { + const parsed = Number(process.env.TLS_FINGERPRINT_FIRST_BYTE_WATCHDOG_MS || "10000"); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 10_000; +} + +function providerFingerprintArmed(provider: string | null | undefined): boolean { + if (!provider) return false; + const key = provider.trim().toLowerCase(); + const until = fingerprintArmedUntil.get(key); + if (until === undefined) return false; + if (Date.now() >= until) { + fingerprintArmedUntil.delete(key); + return false; + } + return true; +} + +function armProviderFingerprint(provider: string | null | undefined): boolean { + if (!provider) return false; + fingerprintArmedUntil.set(provider.trim().toLowerCase(), Date.now() + FINGERPRINT_ARM_TTL_MS); + return true; +} + +/** Cheap transport-level sniff: Cloudflare 1010 / browser-signature ban body. */ +async function looksLikeFingerprintRejection(response: Response): Promise { + if (response.status !== 403) return false; + try { + const peek = response.clone(); + const text = await peek.text(); + return /error_code"?\s*:\s*"?1010|browser_signature_banned|fingerprint_rejection/i.test(text); + } catch { + return false; + } +} + +/** + * Guard a wreq-js response against the buffering stall: the first body byte + * must arrive within the watchdog window or the body is cancelled and a + * transport error thrown (callers fall back to the direct dispatcher). + * The returned Response yields the first chunk before continuing the original + * stream, so no bytes are lost when the watchdog does not fire. + */ +async function withTlsFirstByteWatchdog(response: Response): Promise { + if (!response.body) return response; + const reader = response.body.getReader(); + let first: ReadableStreamReadResult; + try { + first = await Promise.race([ + reader.read(), + new Promise((_, reject) => + setTimeout(() => { + const err = new Error("TLS fingerprint first-byte watchdog fired") as Error & { + code?: string; + }; + err.code = "TLS_FIRST_BYTE_TIMEOUT"; + reject(err); + }, tlsFirstByteWatchdogMs()) + ), + ]); + } catch (err) { + try { + await reader.cancel(err); + } catch { + // reader already closed + } + throw err; + } + // The first chunk was already consumed by the watchdog race — re-emit it + // before continuing the original stream so no bytes are lost. + const rest = new ReadableStream({ + async pull(controller) { + if (first && !first.done) { + controller.enqueue(first.value); + first = null; + return; + } + try { + const chunk = await reader.read(); + if (chunk.done) { + controller.close(); + } else { + controller.enqueue(chunk.value); + } + } catch (error) { + controller.error(error); + } + }, + async cancel(reason) { + try { + await reader.cancel(reason); + } catch { + // upstream already gone + } + }, + }); + return new Response(rest, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} // #8376: transport-level connect-failure codes that mean "the configured upstream // proxy (or the target itself, for direct egress) is unreachable" — as opposed to an // ordinary upstream HTTP error. Read `.code` first (stable across undici/node @@ -792,7 +913,12 @@ async function patchedFetch( if ( isTlsFingerprintEnabled() && activeTlsClient.available && - tlsFingerprintProviderAllowed(tlsStore?.provider, false) && + // Adaptive ladder: eager wreq only for explicitly allowlisted providers. + // Everyone else rides the plain direct path and is armed reactively when + // a Cloudflare fingerprint rejection is observed (the ladder retry below). + (process.env.TLS_FINGERPRINT_PROVIDERS?.trim() + ? tlsFingerprintProviderAllowed(tlsStore?.provider, false) + : providerFingerprintArmed(tlsStore?.provider)) && isTlsRequestEligible(input, options) ) { try { @@ -807,7 +933,9 @@ async function patchedFetch( ...tlsProfileForProvider(tlsStore?.provider), }); if (tlsStore) tlsStore.used = true; - return response; + // Buffering-stall guard: if the gateway never sends a first byte, fall + // back to the direct dispatcher instead of hanging the caller. + return await withTlsFirstByteWatchdog(response); } catch (error) { if (isCallerAbort(error, getEffectiveSignal(input, options))) throw error; const sessionHadCookies = @@ -837,7 +965,41 @@ async function patchedFetch( if (process.versions.bun) { const _nativeFetch = (deps.nativeFetch as FetchWithDispatcher | undefined) ?? originalFetchWithDispatcher; - return _nativeFetch(input, options); + const response = await _nativeFetch(input, options); + // Adaptive ladder step 2: a Cloudflare fingerprint rejection on the plain + // path arms the provider and retries once through wreq-js. Replay-safe + // requests only (GET/HEAD) — never replay a non-idempotent body. + if ( + response.status === 403 && + isTlsFingerprintEnabled() && + activeTlsClient.available && + isTlsRequestEligible(input, options) && + isTlsFallbackReplaySafe(input, options) && + (await looksLikeFingerprintRejection(response)) && + armProviderFingerprint(tlsStore?.provider) + ) { + console.warn( + `[ProxyFetch] Cloudflare fingerprint rejection on ${tlsStore?.provider ?? targetUrl} — retrying once via the TLS-impersonation transport` + ); + try { + const wreqResponse = await activeTlsClient.fetch(targetUrl, { + method: options.method, + headers: options.headers, + body: options.body as TlsFetchOptions["body"], + redirect: options.redirect, + signal: getEffectiveSignal(input, options), + proxy: null, + sessionScope: tlsStore?.sessionScope, + ...tlsProfileForProvider(tlsStore?.provider), + }); + if (tlsStore) tlsStore.used = true; + return await withTlsFirstByteWatchdog(wreqResponse); + } catch (error) { + if (isCallerAbort(error, getEffectiveSignal(input, options))) throw error; + console.warn("[ProxyFetch] TLS-impersonation retry failed; returning the original 403"); + } + } + return response; } // Direct undici path: bound response-start, fresh-socket retry, and body guard. const hasNonReplayableBody = requestHasNonReplayableBody(input, options); @@ -1100,7 +1262,7 @@ async function patchedFetch( ...tlsProfileForProvider(tlsStore?.provider), }); if (tlsStore) tlsStore.used = true; - return response; + return await withTlsFirstByteWatchdog(response); } catch (error) { if (isCallerAbort(error, getEffectiveSignal(input, options))) throw error; const sessionHadCookies = diff --git a/open-sse/utils/stream.ts b/open-sse/utils/stream.ts index 6b3c44cfccc9..da42304a3ff0 100644 --- a/open-sse/utils/stream.ts +++ b/open-sse/utils/stream.ts @@ -180,6 +180,13 @@ type StreamOptions = { * codex-compatible `namespace` + `name` fields. */ requestToolIdentityMap?: Map | null; + /** + * High water mark for the TransformStream internal buffers (default 16384). + * #12179: high-throughput providers (GLM) raise it so provider->client pacing + * stays ahead of the model's token emission rate. Dropped accidentally by the + * #12506 refactor while glm.ts still passed it; re-added. + */ + highWaterMark?: number; }; type TranslateState = ReturnType & { @@ -2987,8 +2994,8 @@ export function createSSEStream(options: StreamOptions = {}) { clearIdleTimer(); }, }, - { highWaterMark: 16384 }, - { highWaterMark: 16384 } + { highWaterMark: options.highWaterMark ?? 16384 }, + { highWaterMark: options.highWaterMark ?? 16384 } ); } @@ -3010,7 +3017,12 @@ export function createSSETransformStreamWithLogger( copilotCompatibleReasoning = false, suppressThinkClose = false, customToolNames: ReadonlySet = new Set(), - requestToolIdentityMap: Map | null = null + requestToolIdentityMap: Map | null = null, + // #12179: high water mark for the TransformStream internal buffer (default + // 16384). GLM passes 65536 to keep provider->client pacing ahead of the + // model's token emission rate. Re-added after the #12506 refactor dropped + // it while glm.ts still passed it (TS2554 base-red at the tip). + highWaterMark?: number ) { return createSSEStream({ mode: STREAM_MODE.TRANSLATE, @@ -3029,6 +3041,7 @@ export function createSSETransformStreamWithLogger( suppressThinkClose, customToolNames, requestToolIdentityMap, + highWaterMark, }); } diff --git a/scripts/dev/self-probe-watchdog.mjs b/scripts/dev/self-probe-watchdog.mjs new file mode 100644 index 000000000000..bfff84cc8e21 --- /dev/null +++ b/scripts/dev/self-probe-watchdog.mjs @@ -0,0 +1,100 @@ +/** + * Self-probe wedge watchdog (2026-09-05 incident follow-up). + * + * Failure mode: the HTTP/2 pool to an upstream died mid-stream + * (ERR_HTTP2_STREAM_ERROR / UND_ERR_SOCKET), a client abort during the combo + * cleanup left the event loop wedged — the listener kept accepting TCP but no + * request was ever served again and nothing was logged. The existing + * supervisor only restarts on process *exit*, so the wedge persisted for hours + * until the operator noticed. + * + * Defense: periodic loopback self-probe of the health endpoint. The probe runs + * on the same event loop it monitors, so a wedged loop cannot answer it. + * N consecutive connect/timeout failures (a serving instance answers 401 on + * /api/status — that counts as alive) → loud log + `process.exit(1)`, and the + * platform supervisor (launchd KeepAlive / systemd / Docker restart) relaunches. + * + * Pure decision helpers are exported for tests; the interval lives in + * `startSelfProbeWatchdog()`, called once from standalone-server-ws.mjs after + * the listener is accepting. + */ + +const DEFAULT_PROBE_INTERVAL_MS = 60_000; +const DEFAULT_PROBE_TIMEOUT_MS = 15_000; +const DEFAULT_PROBE_FAILURE_THRESHOLD = 3; + +export function parseSelfProbeConfig(env) { + const raw = (env.OMNIROUTE_SELF_PROBE ?? "").trim(); + if (/^(0|false|no|off)$/i.test(raw)) return { enabled: false }; + const int = (name, fallback, min) => { + const v = Number(env[name]); + return Number.isFinite(v) && v >= min ? Math.floor(v) : fallback; + }; + return { + enabled: true, + intervalMs: int("OMNIROUTE_SELF_PROBE_INTERVAL_MS", DEFAULT_PROBE_INTERVAL_MS, 5_000), + timeoutMs: int("OMNIROUTE_SELF_PROBE_TIMEOUT_MS", DEFAULT_PROBE_TIMEOUT_MS, 1_000), + threshold: int("OMNIROUTE_SELF_PROBE_THRESHOLD", DEFAULT_PROBE_FAILURE_THRESHOLD, 2), + }; +} + +export function shouldSelfProbeExit(consecutiveFailures, threshold) { + return ( + Number.isFinite(consecutiveFailures) && + consecutiveFailures >= threshold && + threshold > 0 + ); +} + +export function startSelfProbeWatchdog({ port, scheme = "http", log = console }) { + const config = parseSelfProbeConfig(process.env); + if (!config.enabled) return null; + + let consecutiveFailures = 0; + const probeUrl = `${scheme}://127.0.0.1:${port}/api/status`; + + const timer = setInterval(async () => { + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), config.timeoutMs); + abortTimer.unref?.(); + let ok = false; + try { + // Any HTTP response — 200 or 401 or 500-with-headers — proves the loop + // is serving. Only connect/timeout failures indicate a wedge. + const response = await fetch(probeUrl, { + signal: controller.signal, + headers: { connection: "close" }, + }); + // Any HTTP response (200/401/500) proves the event loop is serving. + ok = true; + } catch { + ok = false; + } finally { + clearTimeout(abortTimer); + } + if (ok) { + if (consecutiveFailures > 0) { + log.warn?.( + `[SelfProbe] recovered after ${consecutiveFailures} failed probe(s)` + ); + } + consecutiveFailures = 0; + return; + } + consecutiveFailures += 1; + log.warn?.( + `[SelfProbe] probe failed (${consecutiveFailures}/${config.threshold}) — ` + + `${probeUrl} timed out or refused within ${config.timeoutMs}ms` + ); + if (shouldSelfProbeExit(consecutiveFailures, config.threshold)) { + log.error?.( + `[SelfProbe] event loop wedged — ${consecutiveFailures} consecutive probe failures. ` + + `Exiting (code 1) so the platform supervisor relaunches.` + ); + process.exit(1); + } + }, config.intervalMs); + timer.unref?.(); + + return { stop: () => clearInterval(timer), config }; +} diff --git a/scripts/dev/standalone-server-ws.mjs b/scripts/dev/standalone-server-ws.mjs index 65fb3ab65abf..addbe3cfc01e 100644 --- a/scripts/dev/standalone-server-ws.mjs +++ b/scripts/dev/standalone-server-ws.mjs @@ -9,6 +9,7 @@ import headResponseGuard from "./head-response-guard.cjs"; import { resolveTlsOptions, createServerListener } from "./tls-options.mjs"; import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs"; import { createSystemdNotifier } from "./systemd-notify.mjs"; +import { startSelfProbeWatchdog } from "./self-probe-watchdog.mjs"; // systemd sd_notify (Type=notify / WatchdogSec=): this process is the one // whose event loop can freeze (cold /v1/models rebuild), so it must own the @@ -230,6 +231,19 @@ http.createServer = function createServerWithResponsesWs(...args) { systemdReadySent = true; systemdNotifier.ready(); systemdNotifier.startWatchdog(); + // Wedge self-heal (#12656 incident follow-up): probe our own health + // endpoint on the same event loop. A wedged loop cannot answer, so + // N consecutive connect/timeout failures exit(1) and the platform + // supervisor (launchd KeepAlive / systemd / Docker) relaunches. + try { + const probePort = + typeof server.address === "function" && server.address() && server.address().port + ? server.address().port + : process.env.PORT ?? 20128; + startSelfProbeWatchdog({ port: probePort }); + } catch (probeError) { + console.warn("[omniroute][self-probe] failed to start:", probeError); + } }); return server; diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index 1a666b36344f..07c77bbd7ffe 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -770,11 +770,16 @@ function CombosPageContent() { // both passes and correct it client-only, after hydration, in an effect. const [showUsageGuide, setShowUsageGuide] = useState(true); useEffect(() => { - try { - setShowUsageGuide(globalThis.localStorage?.getItem(COMBO_USAGE_GUIDE_STORAGE_KEY) !== "1"); - } catch { - // Ignore storage access errors (privacy mode / restricted environments) - } + // Post-hydration correction must not be a synchronous setState inside the + // effect body (react-hooks/set-state-in-effect fires on the sync path); + // defer one microtask so the correction happens after the effect settles. + queueMicrotask(() => { + try { + setShowUsageGuide(globalThis.localStorage?.getItem(COMBO_USAGE_GUIDE_STORAGE_KEY) !== "1"); + } catch { + // Ignore storage access errors (privacy mode / restricted environments) + } + }); }, []); const [recentlyCreatedCombo, setRecentlyCreatedCombo] = useState(""); const [creatingKimiPreset, setCreatingKimiPreset] = useState(false); diff --git a/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts b/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts index cc45e6b72778..6618bc711d12 100644 --- a/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts +++ b/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts @@ -25,6 +25,7 @@ import { filterAlibabaFreeEligibleModels } from "@omniroute/open-sse/services/al import { shouldUseLiveAlibabaFreeModelDiscovery } from "@omniroute/open-sse/services/alibabaFreeTier.ts"; import { isDashscopeTextModelId } from "@omniroute/open-sse/services/dashscopeTextModels.ts"; import { extractZaiToken } from "@omniroute/open-sse/services/zaiWebCredentials.ts"; +import { buildOpencodeBackgroundHeaders } from "@omniroute/open-sse/utils/opencodeHeaders.ts"; import { normalizeOpenAiLikeModelsResponse } from "./normalizers"; const QWEN_CLOUD_TEXT_MODEL_IDS = new Set(QWEN_CLOUD_TEXT_MODELS.map((model) => model.id)); @@ -379,6 +380,40 @@ const KIMI_CODING_MODELS_CONFIG: ProviderModelsConfigEntry = { parseResponse: parseKimiCodingModels, }; +const OPENCODE_DISCOVERY_PARSE = (data: any) => data.data || data.models || []; + +function readOpencodeBackgroundSeed(providerSpecificData: unknown): string | null { + if (providerSpecificData && typeof providerSpecificData === "object") { + const ws = (providerSpecificData as Record)["opencodeGoWorkspaceId"]; + if (typeof ws === "string" && ws.trim().length > 0) return ws.trim(); + } + return null; +} + +/** + * Discovery entry for the opencode-family providers with the OpenCode CLI + * identity headers attached (User-Agent + x-opencode-session/request/client/ + * project). Bare runtime fetches (UA "Bun fetch", no session header) are + * exactly the shape OpenCode's operator warning names — enforcement of the + * header is announced from 2026-09-06. The session id is a stable + * per-workspace/connection fingerprint so background discovery groups under + * one identity upstream instead of a fresh anonymous client per call. + */ +function buildOpencodeModelsDiscoveryEntry(url: string): ProviderModelsConfigEntry { + return { + url, + method: "GET", + headers: { Accept: "application/json" }, + buildHeaders: (token, connection) => ({ + Accept: "application/json", + Authorization: `Bearer ${token}`, + ...buildOpencodeBackgroundHeaders({ + seed: readOpencodeBackgroundSeed(connection?.providerSpecificData), + }), + }), + parseResponse: OPENCODE_DISCOVERY_PARSE, + }; +} // Provider models endpoints configuration export const PROVIDER_MODELS_CONFIG: Record = { alibaba: ALIBABA_MODEL_STUDIO_MODELS_CONFIG, @@ -755,22 +790,12 @@ export const PROVIDER_MODELS_CONFIG: Record = authPrefix: "Bearer ", parseResponse: (data) => data.data || data.models || [], }, - "opencode-zen": { - url: "https://opencode.ai/zen/v1/models", - method: "GET", - headers: { "Content-Type": "application/json" }, - authHeader: "Authorization", - authPrefix: "Bearer ", - parseResponse: (data) => data.data || data.models || [], - }, - "opencode-go": { - url: "https://opencode.ai/zen/go/v1/models", - method: "GET", - headers: { "Content-Type": "application/json" }, - authHeader: "Authorization", - authPrefix: "Bearer ", - parseResponse: (data) => data.data || data.models || [], - }, + // OpenCode CLI identity headers on all three opencode-family entries (see + // buildOpencodeModelsDiscoveryEntry above for why — bare "Bun fetch" calls + // without x-opencode-session are what OpenCode's operator warning names). + "opencode": buildOpencodeModelsDiscoveryEntry("https://opencode.ai/zen/v1/models"), + "opencode-zen": buildOpencodeModelsDiscoveryEntry("https://opencode.ai/zen/v1/models"), + "opencode-go": buildOpencodeModelsDiscoveryEntry("https://opencode.ai/zen/go/v1/models"), "glm-cn": { url: "https://open.bigmodel.cn/api/coding/paas/v4/models", method: "GET", diff --git a/src/lib/memory/embedding/remote.ts b/src/lib/memory/embedding/remote.ts index f153a7379cc2..dedfe7e38503 100644 --- a/src/lib/memory/embedding/remote.ts +++ b/src/lib/memory/embedding/remote.ts @@ -10,10 +10,19 @@ export async function embedRemote( ): Promise { const t0 = Date.now(); + // Embedding models cap `input` at 8192 tokens (text-embedding-3-small). + // The memory-retrieval query is built from conversation context and can + // exceed that on long sessions → upstream HTTP 400 + // "Invalid 'input': maximum context length is 8192 tokens", which fails the + // whole retrieval call. Clip conservatively (~20k chars stays under 8192 + // tokens even for code/CJK-heavy text) — a truncated query keeps recall + // working; an unclipped one guarantees a 400. + const MAX_EMBED_CHARS = 20_000; + const clippedInput = text.length > MAX_EMBED_CHARS ? text.slice(0, MAX_EMBED_CHARS) : text; let resp: Response; try { resp = await createEmbeddingResponse( - { model, input: text }, + { model, input: clippedInput }, customProvider ? { resolvedProvider: customProvider.provider, diff --git a/src/lib/resilience/settings.ts b/src/lib/resilience/settings.ts index ddb67220a517..d178045683ce 100644 --- a/src/lib/resilience/settings.ts +++ b/src/lib/resilience/settings.ts @@ -43,8 +43,8 @@ export type { } from "./settings/types"; export const DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS = (() => { - const parsed = Number(process.env.RATE_LIMIT_MAX_WAIT_MS || "15000"); - return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : 15000; + const parsed = Number(process.env.RATE_LIMIT_MAX_WAIT_MS || "120000"); + return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : 120000; })(); // Limiter-managed execution backstop (Bottleneck `expiration`). Deliberately diff --git a/src/shared/constants/visionModels.ts b/src/shared/constants/visionModels.ts index bcad514058a0..fbacdf12efe8 100644 --- a/src/shared/constants/visionModels.ts +++ b/src/shared/constants/visionModels.ts @@ -38,7 +38,7 @@ export const VISION_MODEL_ID_FRAGMENTS = [ "glm-4v", "glm-4.5v", "glm-4.6v", - "glm-5.3-flash", + "glm-5.3", "gpt-4o", "gpt-4.1", "gpt-4-turbo", diff --git a/src/shared/utils/httpClientAbortGuard.mjs b/src/shared/utils/httpClientAbortGuard.mjs index 41d5b07cb57e..eec0dad47e57 100644 --- a/src/shared/utils/httpClientAbortGuard.mjs +++ b/src/shared/utils/httpClientAbortGuard.mjs @@ -42,19 +42,70 @@ export function isClientAbortError(err) { const e = /** @type {NodeJS.ErrnoException} */ (err); // Node emits `Error: aborted` (no code) from http.Server#abortIncoming. if (e.message === "aborted" || e.message === "Aborted") return true; - // OmniRoute's SSE teardown aborts in-flight legs with - // `Error [AbortError]: request_signal_aborted` on client disconnects - // (open-sse/utils/streamHandler.ts), and fetch/DOM cancellation surfaces as - // `AbortError` with an abort-flavoured message. Same benign class as - // `Error: aborted` — an emitter-left 'error' event on any of these used to - // kill the process (#fix-dev-server-aborted). - if (e.name === "AbortError" && /abort/i.test(String(e.message))) return true; + // ANY AbortError/TimeoutError reaching the process level is a raced abort + // or timeout signal from OmniRoute's own orchestration: client disconnects + // (`request_signal_aborted`), per-model combo timeouts + // (`combo-per-model-timeout`), ProxyFetch retry signals, fetch/DOM + // cancellation. The operation has already failed; killing the process for + // it only converts one failed request into a full server outage (#12164). + if (e.name === "AbortError" || e.name === "TimeoutError") return true; + if (e.code === "DIRECT_RESPONSE_START_TIMEOUT") return true; + // Node ≥20 autoSelectFamily connect attempts aggregate per-address errors + // (e.g. IPv6 EHOSTUNREACH + IPv4 ETIMEDOUT against Cloudflare fronts). A + // pure connect-failure aggregate is a transient network condition, not a + // server fault — absorb it (#12164). + if ( + e.name === "AggregateError" && + Array.isArray(/** @type {AggregateError} */ (e).errors) && + /** @type {AggregateError} */ (e).errors.length > 0 && + /** @type {AggregateError} */ (e).errors.every( + (x) => typeof x?.code === "string" && /^(ECONN|EHOST|ENET|ETIMED|EPIPE|EAI_)/.test(x.code) + ) + ) { + return true; + } + // Socket-level failures arrive as `TypeError: fetch failed` with the real + // errno on `.cause` (undici wraps the underlying connect/read error). Walk a + // short cause chain so those transient network conditions are absorbed + // instead of escaping as "fetch failed" uncaught exceptions (observed: + // EADDRNOTAVAIL read error on opencode-go egress → exit 7). + const causes = [e]; + for (let depth = 0; depth < 3; depth += 1) { + const cause = causes[causes.length - 1]?.cause; + if (!cause || typeof cause !== "object") break; + causes.push(/** @type {NodeJS.ErrnoException} */ (cause)); + } + const anyNetworkCode = causes.some((c) => { + if (typeof c.code === "string" && /^(ECONN|EHOST|ENET|ETIMED|EPIPE|EADDR|EAI_|ENOTCONN)/.test(c.code)) { + return true; + } + // An AggregateError cause carries its members' codes on `.errors`, not on + // itself (undici autoSelectFamily connect failures nest this way). + if ( + c.name === "AggregateError" && + Array.isArray(/** @type {AggregateError} */ (c).errors) && + /** @type {AggregateError} */ (c).errors.length > 0 + ) { + return /** @type {Array} */ (c.errors).every( + (member) => + typeof member?.code === "string" && + /^(ECONN|EHOST|ENET|ETIMED|EPIPE|EADDR|EAI_|ENOTCONN)/.test(member.code) + ); + } + return false; + }); + if (anyNetworkCode) return true; switch (e.code) { case "ERR_STREAM_PREMATURE_CLOSE": case "ECONNRESET": case "EPIPE": case "ECONNABORTED": case "ETIMEDOUT": + case "EHOSTUNREACH": + case "ENETUNREACH": + case "ECONNREFUSED": + case "EADDRNOTAVAIL": + case "EAI_AGAIN": case "ENOTCONN": case "ECANCELED": return true; diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 6c03803ba1a5..93e4f6e9bf37 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -1018,7 +1018,18 @@ async function handleChatImplementation( (Boolean(apiKeyInfo.allowedModels?.length) || apiKeyInfo.disableNonPublicModels === true); if (hasModelRestrictions && apiKey) { const modelAllowed = await isModelAllowedForKey(apiKey, modelString); - if (!modelAllowed) return false; + if (!modelAllowed) { + // #12294: an API-key allowlist rejection silently emptied the combo pool + // (surfaced only as a generic "availability" skip). Log it at info so + // operators can spot the mismatch (e.g. allowed_models has an alias + // wildcard like "bailian/*" but the target's provider id is a dynamic + // openai-compatible-* id) without enabling debug logging. + log.info( + "COMBO", + `Skipping ${modelString} — model not allowed for API key ${apiKeyInfo?.name || "(unnamed)"} (allowed_models mismatch)` + ); + return false; + } } // Use getModelInfo to resolve custom prefixes, but prefer the combo diff --git a/stryker.conf.json b/stryker.conf.json index c8ff964e77f1..eff4f0179165 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -39,10 +39,13 @@ "incremental": true, "incrementalFile": "reports/mutation/stryker-incremental.json", "testRunner": "tap", - "plugins": ["@stryker-mutator/tap-runner"], + "plugins": [ + "@stryker-mutator/tap-runner" + ], "tap": { "testFiles": [ "tests/unit/10085-compatible-generic-vs-uuid-credential.test.ts", + "tests/unit/10281-combo-reasoning-probe.test.ts", "tests/unit/7993-noauth-proxy-routing.test.ts", "tests/unit/8200-perplexity-web-401-cooldown.test.ts", "tests/unit/8247-accountfallback-model-unhealthy.test.ts", @@ -397,6 +400,7 @@ "tests/unit/tools-filter-anthropic-format.test.ts", "tests/unit/tproxy-route.test.ts", "tests/unit/trae-publiccred.test.ts", + "tests/unit/transport-bounded-abort.test.ts", "tests/unit/types-barrel-model-cooldown.test.ts", "tests/unit/universal-quota-aware-routing.test.ts", "tests/unit/upstream-error-passthrough.test.ts", @@ -525,7 +529,11 @@ ".worktrees", ".stryker-tmp" ], - "reporters": ["progress", "html", "json"], + "reporters": [ + "progress", + "html", + "json" + ], "htmlReporter": { "fileName": "reports/mutation/mutation.html" }, diff --git a/tests/unit/10281-combo-reasoning-probe.test.ts b/tests/unit/10281-combo-reasoning-probe.test.ts new file mode 100644 index 000000000000..f16392ee35b6 --- /dev/null +++ b/tests/unit/10281-combo-reasoning-probe.test.ts @@ -0,0 +1,64 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Dynamic import is the established convention in this suite (loader stack + +// polyfills; see 9303-recovery-hint-all-targets-skipped.test.ts). +const { + isTinyBudgetReasoningProbe, + buildReasoningProbeTruncatedResponse, +} = await import("../../open-sse/services/reasoningTokenBuffer.ts"); +const quality = await import("../../open-sse/services/combo/validateQuality.ts"); + +test("#10281 combo path: reasoning-consumed quality reasons are detected", () => { + assert.equal( + quality.isReasoningConsumedQualityRejection( + "reasoning consumed 10/10 tokens — no content output" + ), + true + ); + assert.equal( + quality.isReasoningConsumedQualityRejection( + "reasoning consumed 9/10 tokens — no content output" + ), + true + ); + assert.equal(quality.isReasoningConsumedQualityRejection(null), false); + assert.equal( + quality.isReasoningConsumedQualityRejection("empty response content"), + false, + "other quality failures keep the normal 502 path" + ); +}); + +test("#10281 combo path: tiny-budget probe predicate gates on max_tokens < 256", () => { + assert.equal( + isTinyBudgetReasoningProbe({ model: "deepseek-v4-flash", body: { max_tokens: 10 } }), + true + ); + assert.equal( + isTinyBudgetReasoningProbe({ model: "deepseek-v4-flash", body: { max_tokens: 1024 } }), + false, + "real reasoning budgets keep the 502/fallback path" + ); + assert.equal( + isTinyBudgetReasoningProbe({ model: "gpt-4o", body: { max_tokens: 10 } }), + false, + "non-thinking models are never probes" + ); +}); + +test("#10281 combo path: truncated probe response is a valid 200 length-stop", async () => { + const res = buildReasoningProbeTruncatedResponse({ + model: "deepseek-v4-flash", + maxTokens: 10, + requestId: "combo-test", + }); + assert.equal(res.status, 200); + const body = (await res.json()) as { + choices: Array<{ finish_reason: string; message: { content: string } }>; + usage: { completion_tokens: number }; + }; + assert.equal(body.choices[0].finish_reason, "length"); + assert.equal(body.choices[0].message.content, ""); + assert.equal(body.usage.completion_tokens, 10); +}); diff --git a/tests/unit/12294-skipped-target-diagnostics.test.ts b/tests/unit/12294-skipped-target-diagnostics.test.ts new file mode 100644 index 000000000000..41422b9ebf0d --- /dev/null +++ b/tests/unit/12294-skipped-target-diagnostics.test.ts @@ -0,0 +1,131 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Dynamic import is the established convention in this suite (see +// 9303-recovery-hint-all-targets-skipped.test.ts): tests run under the +// tsx/esm + polyfill loader stack, and importing open-sse modules statically +// from tests/unit races the polyfill setup. This exercises the module +// boundary intentionally. +const { + COMBO_SKIP_REASONS, + createInvocationId, + resetComboTraceStore, + startComboTrace, + recordComboDecision, + getComboTrace, + summarizeSkippedTargets, +} = await import("../../open-sse/services/combo/decisionTrace.ts"); + +test("#12294: persisted_cooldown is a recognized skip reason", () => { + assert.ok( + (COMBO_SKIP_REASONS as readonly string[]).includes("persisted_cooldown"), + "persisted_cooldown must be in the COMBO_SKIP_REASONS allowlist" + ); +}); + +test("#12294: summarizeSkippedTargets aggregates skip decisions with reason and detail", () => { + resetComboTraceStore(); + const id = createInvocationId(); + startComboTrace(id, { strategy: "priority", comboName: "smart" }); + + recordComboDecision(id, { + step: "s1", + target: "command-code/deepseek/deepseek-v4-flash-xhigh", + decision: "skipped_before_dispatch", + reason: "persisted_cooldown", + detail: + "Skipping command-code/deepseek/deepseek-v4-flash-xhigh — connection abc has persisted cooldown until 2030-01-01T00:00:00.000Z", + }); + recordComboDecision(id, { + step: "s2", + target: "kimi-coding-apikey/kimi-for-coding-highspeed", + decision: "skipped_before_dispatch", + reason: "availability", + }); + recordComboDecision(id, { + step: "s3", + target: "opencode-go/glm-5.3-flash", + decision: "dispatched", + }); + + const { skippedTargets, nextRetryAt } = summarizeSkippedTargets(getComboTrace(id)); + + assert.equal(skippedTargets.length, 2, "only skipped targets are summarized"); + assert.equal(skippedTargets[0].reason, "persisted_cooldown"); + assert.match(skippedTargets[0].detail ?? "", /until 2030-01-01T00:00:00\.000Z/); + assert.equal(skippedTargets[1].reason, "availability"); + assert.equal(skippedTargets[1].detail, undefined, "no detail means no detail field"); + assert.equal(nextRetryAt, "2030-01-01T00:00:00.000Z"); +}); + +test("#12294: nextRetryAt picks the earliest FUTURE cooldown reset", () => { + resetComboTraceStore(); + const id = createInvocationId(); + startComboTrace(id, { strategy: "priority", comboName: "yano-openweights" }); + + recordComboDecision(id, { + step: "s1", + target: "a/one", + decision: "skipped_before_dispatch", + reason: "persisted_cooldown", + detail: "cooldown until 2020-01-01T00:00:00.000Z", // already expired + }); + recordComboDecision(id, { + step: "s2", + target: "b/two", + decision: "skipped_before_dispatch", + reason: "persisted_cooldown", + detail: "cooldown until 2030-06-01T12:00:00.000Z", + }); + recordComboDecision(id, { + step: "s3", + target: "c/three", + decision: "skipped_before_dispatch", + reason: "persisted_cooldown", + detail: "cooldown until 2031-01-01T00:00:00.000Z", // later + }); + + const { nextRetryAt } = summarizeSkippedTargets(getComboTrace(id)); + assert.equal(nextRetryAt, "2030-06-01T12:00:00.000Z"); +}); + +test("#12294: non-cooldown skips and absent traces yield no nextRetryAt", () => { + resetComboTraceStore(); + const id = createInvocationId(); + startComboTrace(id, { strategy: "priority", comboName: "x" }); + recordComboDecision(id, { + step: "s1", + target: "a/one", + decision: "skipped_before_dispatch", + reason: "quota_cutoff", + }); + + const fromTrace = summarizeSkippedTargets(getComboTrace(id)); + assert.equal(fromTrace.skippedTargets.length, 1); + assert.equal(fromTrace.nextRetryAt, null, "quota_cutoff carries no cooldown timestamp"); + + assert.deepEqual(summarizeSkippedTargets(null), { + skippedTargets: [], + nextRetryAt: null, + }); + assert.deepEqual(summarizeSkippedTargets(getComboTrace("missing-id")), { + skippedTargets: [], + nextRetryAt: null, + }); +}); + +test("#12294: recordComboDecision stores detail alongside the reason", () => { + resetComboTraceStore(); + const id = createInvocationId(); + startComboTrace(id, { strategy: "priority", comboName: "x" }); + recordComboDecision(id, { + step: "s1", + target: "a/one", + decision: "skipped_before_dispatch", + reason: "persisted_cooldown", + detail: "cooldown until 2030-02-03T04:05:06.000Z", + }); + + const trace = getComboTrace(id); + assert.equal(trace?.decisions[0]?.detail, "cooldown until 2030-02-03T04:05:06.000Z"); +}); diff --git a/tests/unit/egress-ip-lock-env-override.test.ts b/tests/unit/egress-ip-lock-env-override.test.ts new file mode 100644 index 000000000000..e096b55c5942 --- /dev/null +++ b/tests/unit/egress-ip-lock-env-override.test.ts @@ -0,0 +1,32 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { egressIpLockProvidersFromEnv } = await import( + "../../open-sse/config/providerErrorRules.ts" +); + +test("#9611-paid: unset env yields the default free-tier egress-lock family", () => { + const set = egressIpLockProvidersFromEnv(undefined); + assert.deepEqual([...set].sort(), ["opencode", "opencode-cli", "opencode-go"]); +}); + +test("#9611-paid: 'none' disables the egress-IP lockout entirely (paid plans)", () => { + for (const raw of ["none", "off", "false", "0", " none "]) { + const set = egressIpLockProvidersFromEnv(raw); + assert.equal(set.size, 0, `raw=${raw} must produce an empty lock set`); + } +}); + +test("#9611-paid: comma-separated list replaces the default family exactly", () => { + const set = egressIpLockProvidersFromEnv("opencode, OpenCode-CLI ,agentrouter"); + assert.deepEqual([...set].sort(), ["agentrouter", "opencode", "opencode-cli"]); +}); + +test("#9611-paid: blank/whitespace env falls back to the default family", () => { + assert.deepEqual([...egressIpLockProvidersFromEnv("")].sort(), [ + "opencode", + "opencode-cli", + "opencode-go", + ]); + assert.equal(egressIpLockProvidersFromEnv(" ").size, 3); +}); diff --git a/tests/unit/embeddings-input-clamp.test.ts b/tests/unit/embeddings-input-clamp.test.ts new file mode 100644 index 000000000000..da1bc9606bb0 --- /dev/null +++ b/tests/unit/embeddings-input-clamp.test.ts @@ -0,0 +1,32 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { clampEmbeddingStringInput } = await import("../../open-sse/handlers/embeddings.ts"); + +const LIMIT = 20_000; + +test("embeddings: over-limit string input is clamped to 20k chars", () => { + const out = clampEmbeddingStringInput("x".repeat(LIMIT + 500)) as string; + assert.equal(out.length, LIMIT); +}); + +test("embeddings: under-limit string input passes through untouched", () => { + const text = "hello world"; + assert.equal(clampEmbeddingStringInput(text), text); +}); + +test("embeddings: array items are clamped individually, non-strings untouched", () => { + const out = clampEmbeddingStringInput([ + "a".repeat(LIMIT + 1), + "short", + { type: "image_url", url: "https://example.com/x.png" }, + ]) as unknown[]; + assert.equal((out[0] as string).length, LIMIT); + assert.equal(out[1], "short"); + assert.deepEqual(out[2], { type: "image_url", url: "https://example.com/x.png" }); +}); + +test("embeddings: non-string scalar input passes through", () => { + assert.equal(clampEmbeddingStringInput(42), 42); + assert.equal(clampEmbeddingStringInput(null), null); +}); diff --git a/tests/unit/error-classifier.test.ts b/tests/unit/error-classifier.test.ts index c4b7e8b2d469..1049261e92a2 100644 --- a/tests/unit/error-classifier.test.ts +++ b/tests/unit/error-classifier.test.ts @@ -5,6 +5,7 @@ const { classifyProviderError, isResourceNotFoundResponse, isCloudflareFingerprintRejection, + resetFingerprintRemediationHintForTest, PROVIDER_ERROR_TYPES, } = await import("../../open-sse/services/errorClassifier.ts"); @@ -401,3 +402,36 @@ test("classifyProviderError: 422 without the BYOP code stays unclassified (no mo ); assert.equal(classifyProviderError(422, "some other body", "antigravity"), null); }); + +test("classifyProviderError: fingerprint rejection emits throttled remediation hint (once, only when TLS transport disabled)", async () => { + resetFingerprintRemediationHintForTest(); + const original = process.env.ENABLE_TLS_FINGERPRINT; + delete process.env.ENABLE_TLS_FINGERPRINT; + const warnings: string[] = []; + const originalWarn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + const body = JSON.stringify({ error_code: 1010 }); + assert.equal(classifyProviderError(403, body), PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION); + assert.equal(classifyProviderError(403, body), PROVIDER_ERROR_TYPES.FINGERPRINT_REJECTION); + const hints = warnings.filter((w) => w.includes("fingerprint rejection")); + assert.equal(hints.length, 1, "hint must be throttled to once per process"); + assert.match(hints[0], /wreq-js/); + assert.match(hints[0], /ENABLE_TLS_FINGERPRINT=true/); + // With the transport enabled the operator already has the fix — no hint. + resetFingerprintRemediationHintForTest(); + process.env.ENABLE_TLS_FINGERPRINT = "true"; + classifyProviderError(403, body); + assert.equal( + warnings.filter((w) => w.includes("fingerprint rejection")).length, + 1, + "no hint when the TLS fingerprint transport is enabled" + ); + } finally { + console.warn = originalWarn; + if (original === undefined) delete process.env.ENABLE_TLS_FINGERPRINT; + else process.env.ENABLE_TLS_FINGERPRINT = original; + } +}); diff --git a/tests/unit/fetch-start-cap-legacy-projection.test.ts b/tests/unit/fetch-start-cap-legacy-projection.test.ts new file mode 100644 index 000000000000..c8285470831f --- /dev/null +++ b/tests/unit/fetch-start-cap-legacy-projection.test.ts @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// #11526 follow-up: generateLegacyProviders() must copy fetchStartTimeoutCapMs +// into the executor's LegacyProvider config. Dropping the field silently fell +// back to the 110s global headers-wait cap even for providers whose registry +// entry overrides it — opencode-go deepseek thinking generations 504ed at +// exactly 110s despite fetchStartTimeoutCapMs on the registry entry (the +// buffered Console Go gateway sends no bytes until the model finishes +// thinking, so the headers phase legitimately spans minutes). + +const { generateLegacyProviders } = await import( + "../../open-sse/config/providerRegistry.ts" +); + +test("generateLegacyProviders copies fetchStartTimeoutCapMs into the executor config", () => { + const providers = generateLegacyProviders(); + + for (const id of ["opencode-go", "command-code"]) { + const legacy = providers[id]; + assert.ok(legacy, `${id} must exist in the legacy provider map`); + assert.equal( + legacy.fetchStartTimeoutCapMs, + 600_000, + `${id} buffered-gateway cap must be 600s (registry override must reach the executor)` + ); + } +}); + +test("generateLegacyProviders omits the cap when the registry entry does not set it", () => { + const providers = generateLegacyProviders(); + const plain = providers["openai"]; + if (!plain) return; // registry shape change guard — only assert when present + assert.equal( + plain.fetchStartTimeoutCapMs, + undefined, + "providers without an override keep the global 110s default" + ); +}); diff --git a/tests/unit/glm-1214-same-format-user-turn.test.ts b/tests/unit/glm-1214-same-format-user-turn.test.ts new file mode 100644 index 000000000000..06b08408ed46 --- /dev/null +++ b/tests/unit/glm-1214-same-format-user-turn.test.ts @@ -0,0 +1,45 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Dynamic import is the established convention in this suite (loader stack + +// polyfills; see 9303-recovery-hint-all-targets-skipped.test.ts). +const { translateRequest } = await import("../../open-sse/translator/index.ts"); + +test("#1214: GLM-family upstream gets a synthetic user turn on the same-format lane", () => { + const body = { + model: "glm-5.3-flash", + messages: [ + { role: "system", content: "You are helpful." }, + { role: "assistant", content: "Calling tool", tool_calls: [{ id: "t1", type: "function", function: { name: "f", arguments: "{}" } }] }, + { role: "tool", tool_call_id: "t1", content: "result" }, + ], + }; + const out = translateRequest("openai", "openai", "opencode-go/glm-5.3-flash", structuredClone(body), false, null, "opencode-go"); + const last = out.messages[out.messages.length - 1]; + assert.equal(last.role, "user", "synthetic user turn must be appended"); + assert.equal(last.content, "(continue)"); + assert.equal(out.messages.length, body.messages.length + 1); +}); + +test("#1214: non-GLM upstream does not get a synthetic user turn", () => { + const body = { + model: "gpt-4o", + messages: [ + { role: "assistant", content: "hi" }, + ], + }; + const out = translateRequest("openai", "openai", "gpt-4o", structuredClone(body), false, null, "openai"); + assert.equal(out.messages.length, 1, "no user turn appended for non-GLM family"); +}); + +test("#1214: GLM request that already has a user turn is unchanged", () => { + const body = { + model: "glm-5.3-flash", + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi" }, + ], + }; + const out = translateRequest("openai", "openai", "glm-5.3-flash", structuredClone(body), false, null, "opencode-go"); + assert.equal(out.messages.length, body.messages.length); +}); diff --git a/tests/unit/httpClientAbortGuard.test.mjs b/tests/unit/httpClientAbortGuard.test.mjs index 293e8ae0ca58..52290a743a9f 100644 --- a/tests/unit/httpClientAbortGuard.test.mjs +++ b/tests/unit/httpClientAbortGuard.test.mjs @@ -123,6 +123,30 @@ test("isClientAbortError matches OmniRoute SSE AbortError shapes (#fix-crash-gua // ⨯ unhandledRejection: Error [AbortError]: request_signal_aborted const sseAbort = Object.assign(new Error("request_signal_aborted"), { name: "AbortError" }); assert.equal(isClientAbortError(sseAbort), true, "SSE teardown AbortError must be absorbed"); + // undici wraps socket errors as TypeError "fetch failed" with the errno on + // `.cause` (observed: EADDRNOTAVAIL read error on opencode-go egress → exit 7). + const wrapped = new TypeError("fetch failed", { + cause: Object.assign(new Error("read EADDRNOTAVAIL"), { code: "EADDRNOTAVAIL", errno: -49 }), + }); + assert.equal(isClientAbortError(wrapped), true, "cause-chain network code must be absorbed"); + const wrappedAggregate = new TypeError("fetch failed", { + cause: new AggregateError( + [Object.assign(new Error("connect ETIMEDOUT"), { code: "ETIMEDOUT" })], + "connect" + ), + }); + assert.equal(isClientAbortError(wrappedAggregate), true, "cause-chain aggregate must be absorbed"); + const wrappedGenuine = new TypeError("fetch failed", { + cause: new Error("something exploded in the handler"), + }); + assert.equal(isClientAbortError(wrappedGenuine), false, "non-network cause must NOT be absorbed"); + // Per-model combo timeout abort (2026-09-01 exit-7 crash): AbortError whose + // message is the abort reason, no 'abort' substring in it. + const comboTimeout = Object.assign(new Error("combo-per-model-timeout"), { + name: "AbortError", + cause: "combo-per-model-timeout", + }); + assert.equal(isClientAbortError(comboTimeout), true, "combo per-model timeout AbortError must be absorbed"); // fetch / DOMException-style cancellation const domAbort = new DOMException("This operation was aborted", "AbortError"); assert.equal(isClientAbortError(domAbort), true, "DOMException AbortError must be absorbed"); @@ -130,6 +154,35 @@ test("isClientAbortError matches OmniRoute SSE AbortError shapes (#fix-crash-gua const typo = new TypeError("Cannot read properties of undefined (reading 'abort')"); assert.equal(isClientAbortError(typo), false); }); +test("isClientAbortError absorbs connect aggregates and ProxyFetch timeout signal (#12164)", () => { + // Production shapes from the 2026-08-31 exit-7 crash log: opencode.ai via + // Cloudflare — IPv6 EHOSTUNREACH + IPv4 ETIMEDOUT aggregated by Node's + // autoSelectFamily connect. + const agg = new AggregateError( + [ + Object.assign(new Error("connect EHOSTUNREACH 2606:4700:3035::ac43:a717:443"), { + code: "EHOSTUNREACH", + }), + Object.assign(new Error("connect ETIMEDOUT 104.21.49.202:443"), { code: "ETIMEDOUT" }), + ], + "connect ETIMEDOUT 104.21.49.202:443" + ); + assert.equal(isClientAbortError(agg), true, "pure connect aggregate must be absorbed"); + // An aggregate containing a non-network member is a genuine bug signal. + const mixed = new AggregateError( + [Object.assign(new Error("connect ETIMEDOUT"), { code: "ETIMEDOUT" }), new TypeError("bad config")], + "mixed" + ); + assert.equal(isClientAbortError(mixed), false, "mixed aggregate must not be absorbed"); + const hostUnreach = Object.assign(new Error("connect EHOSTUNREACH"), { code: "EHOSTUNREACH" }); + assert.equal(isClientAbortError(hostUnreach), true, "direct EHOSTUNREACH must be absorbed"); + // ProxyFetch's own retry signal — already handled by its retry path. + const directTimeout = Object.assign(new Error("Direct response did not start within 30000ms"), { + name: "TimeoutError", + code: "DIRECT_RESPONSE_START_TIMEOUT", + }); + assert.equal(isClientAbortError(directTimeout), true, "ProxyFetch timeout signal must be absorbed"); +}); test("shouldSwallowUncaught absorbs SSE AbortError rejections", () => { const sseAbort = Object.assign(new Error("request_signal_aborted"), { name: "AbortError" }); @@ -160,6 +213,27 @@ test("installProcessCrashGuard() with no logger swallows aborts instead of dying Object.assign(new Error("request_signal_aborted"), { name: "AbortError" }), Promise.resolve() ); + process.emit( + "uncaughtException", + new AggregateError( + [ + Object.assign(new Error("connect EHOSTUNREACH 2606:4700:3035::443"), { + code: "EHOSTUNREACH", + }), + Object.assign(new Error("connect ETIMEDOUT 104.21.49.202:443"), { code: "ETIMEDOUT" }), + ], + "connect ETIMEDOUT 104.21.49.202:443" + ), + "uncaughtException" + ); + process.emit( + "unhandledRejection", + Object.assign(new Error("Direct response did not start within 30000ms"), { + name: "TimeoutError", + code: "DIRECT_RESPONSE_START_TIMEOUT", + }), + Promise.resolve() + ); console.log("ALIVE"); process.exit(0); `; diff --git a/tests/unit/opencode-background-identity-headers.test.ts b/tests/unit/opencode-background-identity-headers.test.ts new file mode 100644 index 000000000000..b14de945b609 --- /dev/null +++ b/tests/unit/opencode-background-identity-headers.test.ts @@ -0,0 +1,79 @@ +/** + * Regression test — OpenCode operator warning (2026-09-03): background calls + * to opencode.ai (model-catalog discovery / quota checks) went out on the bare + * runtime fetch with User-Agent "Bun fetch" and NO `x-opencode-session`. + * OpenCode announced that from 2026-09-06 such requests "may error". + * + * Fix under test: + * 1. `buildOpencodeBackgroundHeaders()` (open-sse/utils/opencodeHeaders.ts) + * synthesizes the OpenCode CLI identity (UA + x-opencode-client/project/ + * request/session) for non-chat fetches, with a STABLE per-caller session + * fingerprint seeded by the calling connection/workspace. + * 2. `PROVIDER_MODELS_CONFIG` entries for opencode / opencode-zen / + * opencode-go (src/app/api/providers/[id]/models/discovery/ + * providerModelsConfig.ts) route discovery through that helper, and the + * seed prefers the connection's `opencodeGoWorkspaceId` so connections + * sharing a workspace share one background identity. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { buildOpencodeBackgroundHeaders } from "../../open-sse/utils/opencodeHeaders.ts"; +import { PROVIDER_MODELS_CONFIG } from "../../src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts"; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const SESSION_HASH_RE = /^[0-9a-f]{16}$/i; + +test("background headers carry the OpenCode CLI identity, not the bare runtime UA", () => { + const headers = buildOpencodeBackgroundHeaders({ seed: "conn-a" }); + assert.equal(headers["User-Agent"], "opencode"); + assert.equal(headers["x-opencode-client"], "desktop"); + assert.equal(headers["x-opencode-project"], "global"); + assert.match(headers["x-opencode-request"] ?? "", UUID_RE); +}); + +test("background session id is a stable per-seed fingerprint (same hash family as the chat path)", () => { + const first = buildOpencodeBackgroundHeaders({ seed: "wrk_01ABC" }); + const second = buildOpencodeBackgroundHeaders({ seed: "wrk_01ABC" }); + assert.match(first["x-opencode-session"] ?? "", SESSION_HASH_RE); + assert.equal(first["x-opencode-session"], second["x-opencode-session"]); +}); + +test("background session id differs across seeds (connections do not share an identity)", () => { + const a = buildOpencodeBackgroundHeaders({ seed: "wrk_01ABC" }); + const b = buildOpencodeBackgroundHeaders({ seed: "wrk_01XYZ" }); + assert.notEqual(a["x-opencode-session"], b["x-opencode-session"]); +}); + +test("seedless background calls still send a session id (random fallback)", () => { + const headers = buildOpencodeBackgroundHeaders(); + const value = headers["x-opencode-session"] ?? ""; + assert.ok( + UUID_RE.test(value) || SESSION_HASH_RE.test(value), + `expected a UUID or fingerprint session id, got ${value}` + ); +}); + +test("explicit userAgent override wins over the CLI default", () => { + const headers = buildOpencodeBackgroundHeaders({ seed: "conn-a", userAgent: "opencode-cli/9.9.9" }); + assert.equal(headers["User-Agent"], "opencode-cli/9.9.9"); +}); + +test("discovery entries for opencode / opencode-zen / opencode-go attach the session header", () => { + for (const provider of ["opencode", "opencode-zen", "opencode-go"] as const) { + const entry = PROVIDER_MODELS_CONFIG[provider]; + assert.ok(entry, `PROVIDER_MODELS_CONFIG missing ${provider}`); + assert.equal(typeof entry.buildHeaders, "function", `${provider} must use buildHeaders`); + const headers = entry.buildHeaders!("test-token", { + providerSpecificData: { opencodeGoWorkspaceId: "wrk_01ABC" }, + }); + assert.equal(headers.Authorization, "Bearer test-token"); + assert.equal(headers["User-Agent"], "opencode"); + assert.match(headers["x-opencode-session"] ?? "", SESSION_HASH_RE); + // Same workspace → same identity as the raw helper (seed comes from psd). + assert.equal( + headers["x-opencode-session"], + buildOpencodeBackgroundHeaders({ seed: "wrk_01ABC" })["x-opencode-session"] + ); + } +}); diff --git a/tests/unit/opencode-reasoning-presence-responses.test.ts b/tests/unit/opencode-reasoning-presence-responses.test.ts new file mode 100644 index 000000000000..10e6be8ae1f7 --- /dev/null +++ b/tests/unit/opencode-reasoning-presence-responses.test.ts @@ -0,0 +1,60 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Dynamic import is the established convention in this suite (loader stack + +// polyfills; see 9303-recovery-hint-all-targets-skipped.test.ts). +const { openaiToOpenAIResponsesRequest } = await import( + "../../open-sse/translator/request/openai-responses/toResponses.ts" +); +const { NON_ANTHROPIC_THINKING_PLACEHOLDER } = await import( + "../../open-sse/utils/reasoningPlaceholder.ts" +); + +test("responses transport: internal reasoning sentinel is emitted as a reasoning_text item", () => { + const body = { + model: "deepseek-v4-flash-max", + messages: [ + { role: "user", content: "question" }, + { + role: "assistant", + tool_calls: [{ id: "call_1", type: "function", function: { name: "f", arguments: "{}" } }], + // Replay cache missed: only the internal sentinel is available. + reasoning_content: NON_ANTHROPIC_THINKING_PLACEHOLDER, + }, + { role: "tool", tool_call_id: "call_1", content: "result" }, + ], + }; + const out = openaiToOpenAIResponsesRequest("deepseek-v4-flash-max", body, true, { + _explicitReasoningReplay: true, + }) as { input: Array<{ type: string; content?: Array<{ type: string; text?: string }> }> }; + + const reasoningItems = out.input.filter( + (item) => item.type === "reasoning" && item.content?.[0]?.type === "reasoning_text" + ); + assert.equal( + reasoningItems.length, + 1, + "the sentinel-carrying assistant turn must produce exactly one reasoning item " + + "(opencode thinking-mode contract: history without reasoning_text 400s)" + ); + assert.equal(reasoningItems[0].content?.[0]?.text, NON_ANTHROPIC_THINKING_PLACEHOLDER); +}); + +test("responses transport: assistant turns with real reasoning keep their text", () => { + const body = { + model: "deepseek-v4-flash-max", + messages: [ + { + role: "assistant", + content: "answer", + reasoning_content: "Because 2+2=4.", + }, + ], + }; + const out = openaiToOpenAIResponsesRequest("deepseek-v4-flash-max", body, true, null) as { + input: Array<{ type: string; content?: Array<{ type: string; text?: string }> }>; + }; + const reasoningItems = out.input.filter((item) => item.type === "reasoning"); + assert.equal(reasoningItems.length, 1); + assert.equal(reasoningItems[0].content?.[0]?.text, "Because 2+2=4."); +}); diff --git a/tests/unit/ratelimit-admission-control-6593.test.ts b/tests/unit/ratelimit-admission-control-6593.test.ts index cdad084a3212..53fdcb6f418f 100644 --- a/tests/unit/ratelimit-admission-control-6593.test.ts +++ b/tests/unit/ratelimit-admission-control-6593.test.ts @@ -177,10 +177,18 @@ test("#6593 withRateLimit: default maxQueueDepth=0 preserves unbounded-queue beh // --- Default maxWaitMs ---------------------------------------------------- -test("#6593 DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS is 15s absent RATE_LIMIT_MAX_WAIT_MS", () => { +test("#6593 DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS is 120s absent RATE_LIMIT_MAX_WAIT_MS", () => { assert.equal(process.env.RATE_LIMIT_MAX_WAIT_MS, undefined); - assert.equal(resilienceSettings.DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS, 15000); - assert.equal(resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue.maxWaitMs, 15000); + assert.equal(resilienceSettings.DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS, 120000); + assert.equal(resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue.maxWaitMs, 120000); +}); + +test("requestQueue.executionMaxWaitMs defaults to a 10-minute backstop, separate from maxWaitMs", () => { + assert.equal(resilienceSettings.DEFAULT_REQUEST_QUEUE_EXECUTION_MAX_WAIT_MS, 600000); + assert.equal( + resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue.executionMaxWaitMs, + 600000 + ); }); test("requestQueue.executionMaxWaitMs defaults to a 10-minute backstop, separate from maxWaitMs", () => { diff --git a/tests/unit/self-probe-watchdog.test.mjs b/tests/unit/self-probe-watchdog.test.mjs new file mode 100644 index 000000000000..3cce30aeac46 --- /dev/null +++ b/tests/unit/self-probe-watchdog.test.mjs @@ -0,0 +1,37 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + parseSelfProbeConfig, + shouldSelfProbeExit, +} from "../../scripts/dev/self-probe-watchdog.mjs"; + +test("self-probe config: enabled by default with sensible values", () => { + const cfg = parseSelfProbeConfig({}); + assert.equal(cfg.enabled, true); + assert.ok(cfg.intervalMs >= 5_000); + assert.ok(cfg.timeoutMs >= 1_000); + assert.ok(cfg.threshold >= 2); +}); + +test("self-probe config: explicit off", () => { + for (const raw of ["0", "false", "no", "off"]) { + assert.equal(parseSelfProbeConfig({ OMNIROUTE_SELF_PROBE: raw }).enabled, false, raw); + } +}); + +test("self-probe config: env overrides clamp to sane minimums", () => { + const cfg = parseSelfProbeConfig({ + OMNIROUTE_SELF_PROBE_INTERVAL_MS: "500", + OMNIROUTE_SELF_PROBE_THRESHOLD: "1", + }); + assert.ok(cfg.intervalMs >= 5_000, "interval must clamp to >= 5s"); + assert.ok(cfg.threshold >= 2, "threshold must clamp to >= 2"); +}); + +test("self-probe exit decision: at threshold and beyond", () => { + assert.equal(shouldSelfProbeExit(2, 3), false); + assert.equal(shouldSelfProbeExit(3, 3), true); + assert.equal(shouldSelfProbeExit(4, 3), true); + assert.equal(shouldSelfProbeExit(0, 3), false); +}); diff --git a/tests/unit/transport-bounded-abort.test.ts b/tests/unit/transport-bounded-abort.test.ts new file mode 100644 index 000000000000..c509cf6b021e --- /dev/null +++ b/tests/unit/transport-bounded-abort.test.ts @@ -0,0 +1,46 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// Regression tests for the 2026-09-05 wedge incident: opencode h2 resets +// (ERR_HTTP2_STREAM_ERROR / UND_ERR_SOCKET) fanned out through every combo +// target (8 fallbacks / 18 decisions) because transport-class failures share +// the same egress. The bounded abort stops the combo after +// DEFAULT_TRANSPORT_ABORT_BOUND consecutive transport-class failures. +const { + isTransportClassNetworkError, + shouldAbortComboForTransportFailures, + DEFAULT_TRANSPORT_ABORT_BOUND, +} = await import("../../open-sse/services/combo/comboPredicates.ts"); + +test("transport-class matcher: h2/socket/errno failures match", () => { + const positives = [ + "[502]: fetch failed (cause: ERR_HTTP2_STREAM_ERROR: Stream closed with error code NGHTTP2_PROTOCOL_ERROR)", + "[502]: terminated (cause: UND_ERR_SOCKET: other side closed)", + "[502]: fetch failed (cause: ECONNRESET)", + "socket hang up", + "EADDRNOTAVAIL read error on egress", + ]; + for (const text of positives) { + assert.equal(isTransportClassNetworkError(text), true, `must match: ${text}`); + } +}); + +test("transport-class matcher: quota/auth/body errors do NOT match", () => { + const negatives = [ + "[429]: Weekly usage limit reached. Resets in 3 days.", + "[403]: You have insufficient credits to make this request", + "[400]: The reasoning_text in the thinking mode must be passed back to the API", + "[503]: all targets were skipped by pre-dispatch filters", + ]; + for (const text of negatives) { + assert.equal(isTransportClassNetworkError(text), false, `must not match: ${text}`); + } +}); + +test("bounded abort fires at the bound, not before", () => { + assert.equal(DEFAULT_TRANSPORT_ABORT_BOUND >= 2, true, "bound must allow at least one fallback"); + assert.equal(shouldAbortComboForTransportFailures(DEFAULT_TRANSPORT_ABORT_BOUND - 1), false); + assert.equal(shouldAbortComboForTransportFailures(DEFAULT_TRANSPORT_ABORT_BOUND), true); + assert.equal(shouldAbortComboForTransportFailures(DEFAULT_TRANSPORT_ABORT_BOUND + 5), true); + assert.equal(shouldAbortComboForTransportFailures(0), false); +});