From 5979ea01859c5cdb7740a7502ae33ea9883ca93c Mon Sep 17 00:00:00 2001 From: vinhali Date: Wed, 9 Sep 2026 18:06:52 -0300 Subject: [PATCH] feat(routing): add explicit target overrides --- config/quality/file-size-baseline.json | 3 +- open-sse/services/combo.ts | 3219 +++++++++-------- open-sse/services/combo/comboStructure.ts | 6 +- open-sse/services/combo/decisionTrace.ts | 30 +- open-sse/services/combo/dispatchPrelude.ts | 16 +- .../services/combo/explicitTargetDetector.ts | 213 ++ .../services/combo/explicitTargetResolver.ts | 204 ++ open-sse/services/combo/routeSlashCommand.ts | 83 + .../combo/tryExplicitTargetDispatch.ts | 299 ++ open-sse/services/combo/types.ts | 11 + .../combo-explicit-target-detector.test.ts | 224 ++ .../combo-explicit-target-dispatch.test.ts | 930 +++++ .../combo-explicit-target-resolver.test.ts | 184 + .../combo/combo-route-slash-command.test.ts | 172 + 14 files changed, 4012 insertions(+), 1582 deletions(-) create mode 100644 open-sse/services/combo/explicitTargetDetector.ts create mode 100644 open-sse/services/combo/explicitTargetResolver.ts create mode 100644 open-sse/services/combo/routeSlashCommand.ts create mode 100644 open-sse/services/combo/tryExplicitTargetDispatch.ts create mode 100644 tests/unit/combo/combo-explicit-target-detector.test.ts create mode 100644 tests/unit/combo/combo-explicit-target-dispatch.test.ts create mode 100644 tests/unit/combo/combo-explicit-target-resolver.test.ts create mode 100644 tests/unit/combo/combo-route-slash-command.test.ts diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 754816df0a7c..065b19534e9c 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -1,4 +1,5 @@ { + "_rebaseline_2026_09_08_explicit_target_routing_v1": "explicit-target-routing-v1 own growth: open-sse/services/combo.ts 4080->4148 (checker-loc; corrected 2026-09-08, an earlier pass here understated this at 4111 before lint-staged's Prettier reflow had run against the final reconciled diff). Two components: (1) wires the new tryExplicitTargetDispatch() dispatch branch (combo/tryExplicitTargetDispatch.ts, new file) into handleComboChatInner ahead of the existing dispatch prelude, passing ~23 already-in-scope local variables — same individual-named-params call shape already used by the sibling tryFusionDispatch/tryPinnedModelDispatch calls a few lines below, so this is call-site wiring at the existing dispatch chokepoint, not a new pattern; (2) same class as _rebaseline_2026_09_07_ci_baseline_repair_prettier_reflow — lint-staged's Prettier reflow of several pre-existing >100-col lines elsewhere in this same function (log.info/log.warn calls, lockout/quota-affordability condition wraps, the final Response(...) at handleRoundRobinCombo) that this PR's own touch of the file now forces into wrapped form; verified via `git diff -w` between the reconciliation-fixup commit and its parent showing zero non-whitespace change outside the new dispatch block and one comment edit. Not extractable into a smaller call without either inventing a bespoke options-object convention inconsistent with those siblings, or moving unrelated pre-existing code out of this file, both worse than the frozen-cap increase. Covered by tests/unit/combo/combo-explicit-target-detector.test.ts, combo-explicit-target-dispatch.test.ts, combo-explicit-target-resolver.test.ts.", "_rebaseline_2026_09_07_ci_baseline_repair_prettier_reflow": "fix/release-v3.8.51-ci-baseline own growth, both cases the lint-staged Prettier reflow of a pre-existing >100-col line that any touch of the file now forces into wrapped form (same class as _rebaseline_2026_09_03_12649_free_tier_reaudit_gateways): src/sse/handlers/chat.ts 2454->2457 (+3 = the registerMoonshotQuotaFetcher/registerMoonshotFetchersForNodes import wraps; reproduces on pure release/v3.8.51 via `prettier --check`, unrelated to this PR's own one-word string fix in the image-model rejection message a few hundred lines below); open-sse/utils/stream.ts 3072->3077 (+5 = two pre-existing filterUsageForFormat/costUsd call-site wraps, also reproducing on pure base; this PR's own highWaterMark parameter restore -- a faithful restore of a previously-reverted commit's plumbing -- is a net +0 once its JSDoc is condensed to a trailing comment). Covered by tests/unit/chat-rejects-image-only-model.test.ts and the highwatermark benchmark/executor suites.", "_rebaseline_2026_09_03_reset_aware_model_family": "Own growth: open-sse/services/combo.ts 4036->4041 (+5). buildAutoCandidates now keys the reset-aware quota cache by getQuotaFetchScope and spreads requestedModel onto the connection so Gemini windows stay off a Claude-empty Antigravity account. Irreducible wiring at the existing fetchResetAwareQuotaWithCache call site; the family helper itself lives in antigravityQuotaFamily.ts. Covered by tests/unit/reset-aware-request-scope-12600.test.ts.", "_rebaseline_2026_09_03_overloaded_not_provider_breaker": "fix/overloaded-not-provider-breaker own growth: open-sse/services/combo.ts 4036->4075 (check-file-size split-newline, +39). Circuit-open pre-skip now records the breaker retryAfter and, when every target was skipped that way, waits the short reset via resolveCircuitOpenWaitDecision (new leaf in comboCooldownRetry.ts) instead of crystallizing ALL_TARGETS_SKIPPED in ~43ms. skippedForCircuitOpen / earliestCircuitOpenRetryMs reset each setTry so a later iteration cannot inherit a stale retryAfter. Irreducible at the existing ALL_TARGETS_SKIPPED chokepoint (same pattern as #7301/#8213 cooldown-wait). Predicate itself lives in circuitBreaker.ts / comboPredicates.ts / chatPredicates.ts, all under cap. Covered by tests/unit/overloaded-not-provider-breaker.test.ts + combo-cooldown-retry.test.ts.", @@ -426,7 +427,7 @@ "open-sse/mcp-server/server.ts": 1572, "open-sse/services/accountFallback.ts": 2467, "open-sse/services/adobeFireflyBrowserLogin.ts": 1401, - "open-sse/services/combo.ts": 4080, + "open-sse/services/combo.ts": 4148, "open-sse/translator/response/openai-responses.ts": 1466, "open-sse/utils/cursorAgentProtobuf.ts": 1547, "open-sse/utils/proxyFetch.ts": 1271, diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 870a470749b5..b40ae5c5896d 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -295,6 +295,7 @@ import { recordQuotaExhaustionClassification, withQuotaExhaustionClassification, } from "./combo/quotaExhaustion.ts"; +import { tryExplicitTargetDispatch } from "./combo/tryExplicitTargetDispatch.ts"; export { RESET_WINDOW_NAMES, QUOTA_SOFT_DEPRIORITIZE_FACTOR, setCandidateQuotaSoftPenalty }; export { scoreAutoTargets, expandAutoComboCandidatePool }; @@ -618,7 +619,10 @@ export async function buildAutoCandidates( const quota = await quotaPromises.get(quotaKey)!; resetWindowAffinity = calculateResetWindowAffinity(quota, resetWindowConfig); if (!quotaCutoffBlocked) { - quotaRemaining = quotaRemainingPercentFromQuota(quota, { provider, requestedModel: modelStr }); + quotaRemaining = quotaRemainingPercentFromQuota(quota, { + provider, + requestedModel: modelStr, + }); } if (!quotaCutoffBlocked && quotaCutoffEnabled) { const cutoffDecision = evaluateQuotaCutoff( @@ -771,6 +775,7 @@ async function handleComboChatInner({ endpointPath = null, requestHeaders = null, invocationId, + explicitTargetResolved = false, }: HandleComboChatOptions): Promise { const comboCtx = createComboContext({ body, combo, settings, relayOptions, log }); const { @@ -805,6 +810,34 @@ async function handleComboChatInner({ log, }); + // Explicit targets take precedence over pins and every normal combo strategy. + const explicitTargetDispatch = await tryExplicitTargetDispatch({ + body, + combo, + handleSingleModel, + isModelAvailable, + log, + settings, + allCombos, + relayOptions, + signal, + apiKeyAllowedConnections, + hiddenModelsByProvider, + invocationId: traceInvocationId, + clientManagedResponsesContext, + perTargetAdmission, + deferContextOverflowWhenCompressible, + compressionExclusions, + sourceFormat, + endpointPath, + requestHeaders, + handleSingleModelWithTimeout, + maxComboDepth: config.maxComboDepth, + runCombo: handleComboChat, + explicitTargetResolved, + }); + if (explicitTargetDispatch) return explicitTargetDispatch; + // Dispatch prelude: context-cache pin → fusion → chaos → pipeline → nested // combo-ref execute mode → round-robin. Each branch either owns the request or // falls through to the target iteration loop below. Implementations live in @@ -1153,864 +1186,832 @@ async function handleComboChatInner({ try { for (let setTry = 0; setTry <= maxSetRetries; setTry++) { - // #1731: Per-set-iteration set of providers whose quota is fully exhausted. - // Reset each retry so providers excluded in a previous attempt get another chance. - const exhaustedProviders = new Set(); - const exhaustedConnections = new Set(); - const transientRateLimitedProviders = new Set(); - skippedForCircuitOpen = false; - earliestCircuitOpenRetryMs = 0; - if (setTry > 0) { - log.info("COMBO", `All targets failed — retrying set (${setTry}/${maxSetRetries})`); - await new Promise((resolve) => { - const timer = setTimeout(resolve, setRetryDelayMs); - signal?.addEventListener( - "abort", - () => { - clearTimeout(timer); - resolve(undefined); - }, - { once: true } - ); - }); - if (signal?.aborted) { - log.info("COMBO", "Client disconnected during set retry delay — aborting"); - return errorResponse(499, "Client disconnected"); - } - } - - const startTime = Date.now(); - let fallbackCount = 0; - let recordedAttempts = 0; - comboErrors = []; - - // QA P0: assemble a sanitized diagnostic trace from the state already in scope - // (pool size + this set-try's exhausted providers/connections + attempt order + - // a terminal-reason code). Never touches keys/tokens — provider/model ids only. - // Silent-stop fix: include a `recovery` hint (action verb + human next-step) so the - // OC plugin + non-header-aware clients can render an actionable error instead of an - // opaque 5xx. The optional `retryAfterSeconds` carries the upstream Retry-After hint. - const buildComboDiag = ( - terminalReason: string, - retryAfterSeconds?: number - ): ComboDiagnostics => ({ - poolSize: orderedTargets.length, - attempted: recordedAttempts, - excluded: [ - ...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })), - ...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))), - ], - attemptOrder: comboAttemptOrder, - terminalReason, - recovery: buildRecoveryHint(terminalReason, retryAfterSeconds), - }); - - let globalResolve: ((res: Response) => void) | null = null; - const globalPromise = new Promise((res) => { - globalResolve = res; - }); - - // G1 (silent-stop fix): the speculative loop's `Promise.race` waits on - // `globalPromise`, which is ONLY resolved from inside a task (success or - // fatal error). If a target hangs — e.g. the operator disabled the per-model - // timeout (`targetTimeoutMs: 0`) and the upstream never settles — the race - // never resolves and the request hangs forever with no response. This safety - // promise force-resolves after the combo budget (comboTimeoutMs when set, - // otherwise a hard ceiling) so the request ALWAYS terminates with an - // actionable 504 instead of dying silently. `comboExpired` is flipped so the - // target loop stops launching new work; the existing comboExpired branch - // returns the aggregated 504. - const loopSafetyMs = comboTimeoutMs > 0 ? comboTimeoutMs : COMBO_LOOP_SAFETY_TIMEOUT_MS; - let loopSafetyFired = false; - let loopSafetyTimer: ReturnType | null = null; - const loopSafetyPromise = new Promise((resolve) => { - loopSafetyTimer = setTimeout(() => { - loopSafetyFired = true; - log.warn( - "COMBO", - `Combo loop safety timeout (${loopSafetyMs}ms) reached without a terminal response — force-terminating` - ); - resolve( - errorResponseWithComboDiagnostics( - 504, - `Combo global timeout (${loopSafetyMs}ms) without a terminal response`, - buildComboDiag("combo_timeout"), - { code: "COMBO_TIMEOUT", type: "server_error" } - ) - ); - }, loopSafetyMs); - loopSafetyTimer.unref?.(); - activeLoopSafetyTimer = loopSafetyTimer; - }); - const runningTasks = new Set>(); - let anySuccess = false; - // #10681: steps already recorded as dispatched (so per-target retries do not - // duplicate the decision). - const dispatchedTargets = new Set(); - // G1: flip comboExpired as soon as the safety timer fires so the next loop - // iteration breaks instead of launching more targets after the budget, and - // abort every in-flight target so a hung upstream actually gets cancelled - // (not just "response stops"). - const markLoopExpiredIfSafetyFired = () => { - if (loopSafetyFired) { - comboExpired = true; - for (const [, ac] of abortControllers.entries()) ac.abort(); - } - }; - const abortControllers = new Map(); - const zeroLatencyOptimizationsEnabled = config.zeroLatencyOptimizationsEnabled === true; - const hasProtectedPriorityTarget = - strategy === "priority" && - orderedTargets.some((target) => target.fallbackOnlyOnQuotaExhaustion === true); - - const executeTarget = async ( - i: number - ): Promise<{ ok: boolean; response?: Response } | null> => { - const target = orderedTargets[i]; - const modelStr = target.modelStr; - const rawModel = parseModel(modelStr).model || modelStr; - const provider = target.provider; - const protectedPriorityTarget = - strategy === "priority" && target.fallbackOnlyOnQuotaExhaustion === true; - const stopProtectedPriorityTarget = (message: string) => { - observeFailure(false, target.executionKey); - clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); - return protectedPriorityTarget - ? { ok: false, response: errorResponse(503, message) } - : null; - }; - - const cb = getCircuitBreaker(provider); - const cbStatus = cb.getStatus(); - if (cbStatus.state === "OPEN") { - skippedForCircuitOpen = true; - if ( - cbStatus.retryAfterMs > 0 && - (earliestCircuitOpenRetryMs === 0 || cbStatus.retryAfterMs < earliestCircuitOpenRetryMs) - ) { - earliestCircuitOpenRetryMs = cbStatus.retryAfterMs; - } - log.info("COMBO", `Skipping ${modelStr} — circuit breaker OPEN for ${provider}`); - recordComboDecision(traceInvocationId, { - step: target.executionKey, - target: modelStr, - decision: "skipped_before_dispatch", - reason: "circuit_open", - }); - if (i > 0) fallbackCount++; - return stopProtectedPriorityTarget(`Provider ${provider} circuit breaker is open`); - } - - if ( - resilienceSettings.providerCooldown.enabled && - Boolean(provider && provider !== "unknown") && - (isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings) || - isProviderInCooldown(provider, undefined, resilienceSettings)) - ) { - log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`); - recordComboDecision(traceInvocationId, { - step: target.executionKey, - target: modelStr, - decision: "skipped_before_dispatch", - reason: "provider_cooldown", + // #1731: Per-set-iteration set of providers whose quota is fully exhausted. + // Reset each retry so providers excluded in a previous attempt get another chance. + const exhaustedProviders = new Set(); + const exhaustedConnections = new Set(); + const transientRateLimitedProviders = new Set(); + skippedForCircuitOpen = false; + earliestCircuitOpenRetryMs = 0; + if (setTry > 0) { + log.info("COMBO", `All targets failed — retrying set (${setTry}/${maxSetRetries})`); + await new Promise((resolve) => { + const timer = setTimeout(resolve, setRetryDelayMs); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(undefined); + }, + { once: true } + ); }); - if (i > 0) fallbackCount++; - return stopProtectedPriorityTarget(`Provider ${provider} is in cooldown`); - } - - // Use pre-screened profile if available, otherwise fetch on demand - const preScreenEntry = preScreenMap.get(target.executionKey); - const profile = preScreenEntry?.profile ?? (await getRuntimeProviderProfile(provider)); - - const allowRateLimitedConnection = - Boolean(provider && provider !== "unknown") && - transientRateLimitedProviders.has(provider); - const targetForAttempt = allowRateLimitedConnection - ? { - ...target, - allowRateLimitedConnection: true, - modelAbortSignal: abortControllers.get(i)!.signal, - } - : { ...target, modelAbortSignal: abortControllers.get(i)!.signal }; - - // Persist the connection cooldown before dispatch. AUTH only learns - // unavailable during credential lookup, so a burst would otherwise - // burn max_concurrent slots on real upstream calls against a row - // SQLite already locked until the reset. - if (target.connectionId && !allowRateLimitedConnection) { - const persistedSkip = await resolvePersistedConnectionCooldownSkipReason( - target, - (id) => readConnectionForCooldownGate(id, false), - allowRateLimitedConnection - ); - if (persistedSkip) { - log.info("COMBO", persistedSkip); - clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); - if (i > 0) fallbackCount++; - return null; + if (signal?.aborted) { + log.info("COMBO", "Client disconnected during set retry delay — aborting"); + return errorResponse(499, "Client disconnected"); } } - // #1731 / #1731v2: skip targets already known-exhausted this request (shared predicate). - const exhaustedSkip = getExhaustedTargetSkipReason( - target, - exhaustedProviders, - exhaustedConnections - ); - if (exhaustedSkip) { - log.info("COMBO", exhaustedSkip); - recordComboDecision(traceInvocationId, { - step: target.executionKey, - target: modelStr, - decision: "skipped_before_dispatch", - reason: "request_exhaustion", - }); - if (i > 0) fallbackCount++; - return stopProtectedPriorityTarget(`Target ${modelStr} is unavailable`); - } + const startTime = Date.now(); + let fallbackCount = 0; + let recordedAttempts = 0; + comboErrors = []; + + // QA P0: assemble a sanitized diagnostic trace from the state already in scope + // (pool size + this set-try's exhausted providers/connections + attempt order + + // a terminal-reason code). Never touches keys/tokens — provider/model ids only. + // Silent-stop fix: include a `recovery` hint (action verb + human next-step) so the + // OC plugin + non-header-aware clients can render an actionable error instead of an + // opaque 5xx. The optional `retryAfterSeconds` carries the upstream Retry-After hint. + const buildComboDiag = ( + terminalReason: string, + retryAfterSeconds?: number + ): ComboDiagnostics => ({ + poolSize: orderedTargets.length, + attempted: recordedAttempts, + excluded: [ + ...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })), + ...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))), + ], + attemptOrder: comboAttemptOrder, + terminalReason, + recovery: buildRecoveryHint(terminalReason, retryAfterSeconds), + }); - // Pre-check: skip models locked by the resilience system (model-level lockout) - if (provider && rawModel && isModelLocked(provider, target.connectionId || "", rawModel)) { - log.info("COMBO", `Skipping ${modelStr} — model locked by resilience (cooldown active)`); - recordComboDecision(traceInvocationId, { - step: target.executionKey, - target: modelStr, - decision: "skipped_before_dispatch", - reason: "model_lockout", - }); - if (i > 0) fallbackCount++; - return stopProtectedPriorityTarget(`Model ${modelStr} is locked`); - } + let globalResolve: ((res: Response) => void) | null = null; + const globalPromise = new Promise((res) => { + globalResolve = res; + }); - // #5923 (Finding #4) — honor the same opt-in quota-exhaustion cutoff the - // "auto" strategy already applies (buildAutoCandidates), for every other - // strategy (priority, weighted, etc.). Strictly scoped per (provider, - // connectionId): a 0%-remaining connection is skipped here, but sibling - // connections/models on the same provider are untouched — the provider - // circuit breaker is never touched by this check. The "auto" strategy is - // excluded to avoid a redundant duplicate fetch — it already filtered its - // candidate pool via `routableCandidates` before reaching this loop. - if (strategy !== "auto" && provider && target.connectionId) { - const quotaCutoff = await resolveQuotaExhaustionCutoffForTarget( - provider, - target.connectionId, - resilienceSettings, - quotaCutoffResetWindowConfig, - combo.name, - log, - modelStr - ); - if (quotaCutoff.blocked) { - log.info( + // G1 (silent-stop fix): the speculative loop's `Promise.race` waits on + // `globalPromise`, which is ONLY resolved from inside a task (success or + // fatal error). If a target hangs — e.g. the operator disabled the per-model + // timeout (`targetTimeoutMs: 0`) and the upstream never settles — the race + // never resolves and the request hangs forever with no response. This safety + // promise force-resolves after the combo budget (comboTimeoutMs when set, + // otherwise a hard ceiling) so the request ALWAYS terminates with an + // actionable 504 instead of dying silently. `comboExpired` is flipped so the + // target loop stops launching new work; the existing comboExpired branch + // returns the aggregated 504. + const loopSafetyMs = comboTimeoutMs > 0 ? comboTimeoutMs : COMBO_LOOP_SAFETY_TIMEOUT_MS; + let loopSafetyFired = false; + let loopSafetyTimer: ReturnType | null = null; + const loopSafetyPromise = new Promise((resolve) => { + loopSafetyTimer = setTimeout(() => { + loopSafetyFired = true; + log.warn( "COMBO", - `Skipping ${modelStr} — quota exhaustion cutoff (${quotaCutoff.reason || "quota_exhausted"})` + `Combo loop safety timeout (${loopSafetyMs}ms) reached without a terminal response — force-terminating` + ); + resolve( + errorResponseWithComboDiagnostics( + 504, + `Combo global timeout (${loopSafetyMs}ms) without a terminal response`, + buildComboDiag("combo_timeout"), + { code: "COMBO_TIMEOUT", type: "server_error" } + ) ); + }, loopSafetyMs); + loopSafetyTimer.unref?.(); + activeLoopSafetyTimer = loopSafetyTimer; + }); + const runningTasks = new Set>(); + let anySuccess = false; + // #10681: steps already recorded as dispatched (so per-target retries do not + // duplicate the decision). + const dispatchedTargets = new Set(); + // G1: flip comboExpired as soon as the safety timer fires so the next loop + // iteration breaks instead of launching more targets after the budget, and + // abort every in-flight target so a hung upstream actually gets cancelled + // (not just "response stops"). + const markLoopExpiredIfSafetyFired = () => { + if (loopSafetyFired) { + comboExpired = true; + for (const [, ac] of abortControllers.entries()) ac.abort(); + } + }; + const abortControllers = new Map(); + const zeroLatencyOptimizationsEnabled = config.zeroLatencyOptimizationsEnabled === true; + const hasProtectedPriorityTarget = + strategy === "priority" && + orderedTargets.some((target) => target.fallbackOnlyOnQuotaExhaustion === true); + + const executeTarget = async ( + i: number + ): Promise<{ ok: boolean; response?: Response } | null> => { + const target = orderedTargets[i]; + const modelStr = target.modelStr; + const rawModel = parseModel(modelStr).model || modelStr; + const provider = target.provider; + const protectedPriorityTarget = + strategy === "priority" && target.fallbackOnlyOnQuotaExhaustion === true; + const stopProtectedPriorityTarget = (message: string) => { + observeFailure(false, target.executionKey); clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + return protectedPriorityTarget + ? { ok: false, response: errorResponse(503, message) } + : null; + }; + + const cb = getCircuitBreaker(provider); + const cbStatus = cb.getStatus(); + if (cbStatus.state === "OPEN") { + skippedForCircuitOpen = true; + if ( + cbStatus.retryAfterMs > 0 && + (earliestCircuitOpenRetryMs === 0 || + cbStatus.retryAfterMs < earliestCircuitOpenRetryMs) + ) { + earliestCircuitOpenRetryMs = cbStatus.retryAfterMs; + } + log.info("COMBO", `Skipping ${modelStr} — circuit breaker OPEN for ${provider}`); recordComboDecision(traceInvocationId, { step: target.executionKey, target: modelStr, decision: "skipped_before_dispatch", - reason: "quota_cutoff", + reason: "circuit_open", }); if (i > 0) fallbackCount++; - observeFailure(true, target.executionKey); - if (protectedPriorityTarget) { - const protectedTargetTrust = targetFailureTrust.get(target.executionKey); - if (!protectedTargetTrust?.allObservedFailuresQuota) { - return { - ok: false, - response: errorResponse(503, `Target ${modelStr} is unavailable`), - }; - } - } - return null; - } - } - - // Quota-aware scheduling (opt-in, OMNIROUTE_QUOTA_AWARE_ROUTING=1): - // when a per-connection token budget is configured (provider_quota_state), - // skip targets whose remaining budget cannot afford this request — - // BEFORE dispatching — instead of waiting for a 429. Fails open: when - // no budget is configured the decision is always affordable. - if (process.env.OMNIROUTE_QUOTA_AWARE_ROUTING === "1" && provider && target.connectionId) { - const quotaDecision = canAffordRequest( - target.connectionId, - modelStr, - body as Record | null | undefined - ); - if (!quotaDecision.affordable) { - log.info( - "COMBO", - `Skipping ${modelStr} — quota budget ${quotaDecision.reason} (remaining ${quotaDecision.tokensRemaining ?? 0}, cost ${quotaDecision.estimatedCost ?? 0})` - ); - clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); - if (i > 0) fallbackCount++; - return null; + return stopProtectedPriorityTarget(`Provider ${provider} circuit breaker is open`); } - } - // Pre-screen snapshot is NOT used as a permanent skip — availability - // is always re-checked via isModelAvailable below because connection - // cooldowns can expire between setTry retries, making a previously - // unavailable target available again. Circuit-breaker-OPEN providers - // are already caught by the dedicated breaker check above. - if (isModelAvailable) { - const available = await isModelAvailable(modelStr, targetForAttempt); - if (!available) { - log.debug?.( - "COMBO", - `Skipping ${modelStr} — no credentials available or model excluded` - ); - clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + if ( + resilienceSettings.providerCooldown.enabled && + Boolean(provider && provider !== "unknown") && + (isProviderInCooldown(provider, target.connectionId ?? undefined, resilienceSettings) || + isProviderInCooldown(provider, undefined, resilienceSettings)) + ) { + log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`); recordComboDecision(traceInvocationId, { step: target.executionKey, target: modelStr, decision: "skipped_before_dispatch", - reason: "availability", + reason: "provider_cooldown", }); if (i > 0) fallbackCount++; - return stopProtectedPriorityTarget(`Model ${modelStr} is unavailable`); + return stopProtectedPriorityTarget(`Provider ${provider} is in cooldown`); + } + + // Use pre-screened profile if available, otherwise fetch on demand + const preScreenEntry = preScreenMap.get(target.executionKey); + const profile = preScreenEntry?.profile ?? (await getRuntimeProviderProfile(provider)); + + const allowRateLimitedConnection = + Boolean(provider && provider !== "unknown") && + transientRateLimitedProviders.has(provider); + const targetForAttempt = allowRateLimitedConnection + ? { + ...target, + allowRateLimitedConnection: true, + modelAbortSignal: abortControllers.get(i)!.signal, + } + : { ...target, modelAbortSignal: abortControllers.get(i)!.signal }; + + // Persist the connection cooldown before dispatch. AUTH only learns + // unavailable during credential lookup, so a burst would otherwise + // burn max_concurrent slots on real upstream calls against a row + // SQLite already locked until the reset. + if (target.connectionId && !allowRateLimitedConnection) { + const persistedSkip = await resolvePersistedConnectionCooldownSkipReason( + target, + (id) => readConnectionForCooldownGate(id, false), + allowRateLimitedConnection + ); + if (persistedSkip) { + log.info("COMBO", persistedSkip); + clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + if (i > 0) fallbackCount++; + return null; + } } - } - // Credential gate: skip targets with known-bad credentials (fail-fast) - const connectionId = target.connectionId as string | undefined; - if (connectionId) { - const gateResult = checkCredentialGate(connectionId, provider, modelStr); - if (gateResult.allowed === false) { - logCredentialSkip(log, modelStr, gateResult.reason || "Credential gate blocked"); + // #1731 / #1731v2: skip targets already known-exhausted this request (shared predicate). + const exhaustedSkip = getExhaustedTargetSkipReason( + target, + exhaustedProviders, + exhaustedConnections + ); + if (exhaustedSkip) { + log.info("COMBO", exhaustedSkip); recordComboDecision(traceInvocationId, { step: target.executionKey, target: modelStr, decision: "skipped_before_dispatch", - reason: "credential_gate", + reason: "request_exhaustion", }); if (i > 0) fallbackCount++; - return stopProtectedPriorityTarget(`Credential gate blocked ${modelStr}`); + return stopProtectedPriorityTarget(`Target ${modelStr} is unavailable`); } - // Concurrency gate: fail-fast skip when connection is at max_concurrent capacity (e.g. Featherless 1/1) - const maxConcurrentCap = await lookupPositiveCap(connectionId); + // Pre-check: skip models locked by the resilience system (model-level lockout) if ( - maxConcurrentCap && - isAccountSemaphoreFull(provider, connectionId, maxConcurrentCap) + provider && + rawModel && + isModelLocked(provider, target.connectionId || "", rawModel) ) { log.info( "COMBO", - `Skipping ${modelStr} — connection ${connectionId} is at max concurrency cap (${maxConcurrentCap})` + `Skipping ${modelStr} — model locked by resilience (cooldown active)` ); recordComboDecision(traceInvocationId, { step: target.executionKey, target: modelStr, decision: "skipped_before_dispatch", - reason: "concurrency_cap", + reason: "model_lockout", }); if (i > 0) fallbackCount++; - return stopProtectedPriorityTarget(`Connection capacity reached for ${modelStr}`); + return stopProtectedPriorityTarget(`Model ${modelStr} is locked`); } - } - - // #9654 Wave 2: per-target lane-aware admission probe. With virtual - // lanes on, a tenant whose lane queue is full should skip extra - // fan-out targets instead of piling more queued work onto the lane. - // Strictly non-blocking (maxWaitMs 0) and a no-op when lanes are off — - // see createPerTargetAdmissionHook for the full contract. - if ( - perTargetAdmission && - !(await perTargetAdmission({ modelStr, executionKey: target.executionKey, body })) - ) { - log.info("COMBO", `Skipping ${modelStr} — admission lane full (#9654)`); - recordComboDecision(traceInvocationId, { - step: target.executionKey, - target: modelStr, - decision: "skipped_before_dispatch", - reason: "admission_lane", - }); - if (i > 0) fallbackCount++; - return null; - } - // Retry loop for transient errors - for (let retry = 0; retry <= maxRetries; retry++) { - // Fix #1681: Bail out immediately if the client has disconnected - if (signal?.aborted) { - log.info("COMBO", `Client disconnected — aborting combo loop before model ${modelStr}`); - return { ok: false, response: errorResponse(499, "Client disconnected") }; - } - globalAttempts++; - if (globalAttempts > maxGlobalAttempts) { - log.warn( - "COMBO", - `Maximum combo attempts (${maxGlobalAttempts}) exceeded across all targets and fallbacks. Terminating loop to prevent runaway background requests.` + // #5923 (Finding #4) — honor the same opt-in quota-exhaustion cutoff the + // "auto" strategy already applies (buildAutoCandidates), for every other + // strategy (priority, weighted, etc.). Strictly scoped per (provider, + // connectionId): a 0%-remaining connection is skipped here, but sibling + // connections/models on the same provider are untouched — the provider + // circuit breaker is never touched by this check. The "auto" strategy is + // excluded to avoid a redundant duplicate fetch — it already filtered its + // candidate pool via `routableCandidates` before reaching this loop. + if (strategy !== "auto" && provider && target.connectionId) { + const quotaCutoff = await resolveQuotaExhaustionCutoffForTarget( + provider, + target.connectionId, + resilienceSettings, + quotaCutoffResetWindowConfig, + combo.name, + log, + modelStr ); - // Actionable failure instead of an opaque 503 when every candidate - // failed the same recoverable way. If the dominant cause was reasoning - // models exhausting a too-small max_tokens budget (no content output), - // retrying other models can't help — tell the caller to raise max_tokens. - // Silent-stop fix: bump the consecutive-failure counter for this session-combo pair - // so the pin gets cleared on the 3rd attempt (recovery.next_step tells the client). - const reasoningExhausted = /reasoning consumed \d+\/\d+ tokens/.test(lastError || ""); - const failureReason = reasoningExhausted - ? "reasoning_budget_exhausted" - : "max_attempts_exceeded"; - recordComboFailure(effectiveSessionId, combo.name); - return { - ok: false, - response: errorResponseWithComboDiagnostics( - 503, - reasoningExhausted - ? "All combo candidates exhausted their token budget on reasoning without producing content. Increase max_tokens — reasoning models need a larger budget to emit content." - : "Maximum combo retry limit reached", - buildComboDiag(failureReason) - ), - }; - } - // Predictive TTFT Circuit Breaker (skip slow models) - if ( - zeroLatencyOptimizationsEnabled && - config.predictiveTtftMs && - config.predictiveTtftMs > 0 && - retry === 0 - ) { - const cMetrics = getComboMetrics(combo.name); - if (cMetrics) { - const targetKey = orderedTargets[i].executionKey || modelStr; - const m = cMetrics.byTarget[targetKey] || cMetrics.byModel[modelStr]; - if (shouldSkipForPredictedTtft(m, config.predictiveTtftMs)) { - log.warn( - "COMBO", - `Predictive TTFT Circuit Breaker: skipping ${modelStr} (avg ${m.avgLatencyMs}ms > max ${config.predictiveTtftMs}ms)` - ); - recordComboDecision(traceInvocationId, { - step: target.executionKey, - target: modelStr, - decision: "skipped_before_dispatch", - reason: "predictive_ttft", - }); - return stopProtectedPriorityTarget(`Predictive latency check rejected ${modelStr}`); + if (quotaCutoff.blocked) { + log.info( + "COMBO", + `Skipping ${modelStr} — quota exhaustion cutoff (${quotaCutoff.reason || "quota_exhausted"})` + ); + clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + recordComboDecision(traceInvocationId, { + step: target.executionKey, + target: modelStr, + decision: "skipped_before_dispatch", + reason: "quota_cutoff", + }); + if (i > 0) fallbackCount++; + observeFailure(true, target.executionKey); + if (protectedPriorityTarget) { + const protectedTargetTrust = targetFailureTrust.get(target.executionKey); + if (!protectedTargetTrust?.allObservedFailuresQuota) { + return { + ok: false, + response: errorResponse(503, `Target ${modelStr} is unavailable`), + }; + } } + return null; } } - if (retry > 0) { - log.info( - "COMBO", - `Retrying ${modelStr} in ${retryDelayMs}ms (attempt ${retry + 1}/${maxRetries + 1})` + // Quota-aware scheduling (opt-in, OMNIROUTE_QUOTA_AWARE_ROUTING=1): + // when a per-connection token budget is configured (provider_quota_state), + // skip targets whose remaining budget cannot afford this request — + // BEFORE dispatching — instead of waiting for a 429. Fails open: when + // no budget is configured the decision is always affordable. + if ( + process.env.OMNIROUTE_QUOTA_AWARE_ROUTING === "1" && + provider && + target.connectionId + ) { + const quotaDecision = canAffordRequest( + target.connectionId, + modelStr, + body as Record | null | undefined ); - await new Promise((resolve) => { - const timer = setTimeout(resolve, retryDelayMs); - signal?.addEventListener( - "abort", - () => { - clearTimeout(timer); - resolve(undefined); - }, - { once: true } + if (!quotaDecision.affordable) { + log.info( + "COMBO", + `Skipping ${modelStr} — quota budget ${quotaDecision.reason} (remaining ${quotaDecision.tokensRemaining ?? 0}, cost ${quotaDecision.estimatedCost ?? 0})` ); - }); - if (signal?.aborted) { - log.info("COMBO", `Client disconnected during retry delay — aborting`); - return { ok: false, response: errorResponse(499, "Client disconnected") }; - } - - // Retry re-check: a sibling attempt (or attempt 1) may have persisted - // a quota cooldown while this attempt was sleeping out its retry delay - // ("Trying model 1/7: zai/glm-5.3 (retry 1)" after "already marked - // unavailable until …"). Reads fresh, not cached: see readConnectionForCooldownGate. - const persistedRetrySkip = await resolvePersistedConnectionCooldownSkipReason( - target, - (id) => readConnectionForCooldownGate(id, true), - allowRateLimitedConnection - ); - if (persistedRetrySkip) { - log.info("COMBO", persistedRetrySkip); + clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); if (i > 0) fallbackCount++; return null; } } - log.info( - "COMBO", - `Trying model ${i + 1}/${orderedTargets.length}: ${modelStr}${retry > 0 ? ` (retry ${retry})` : ""}` - ); - emit("combo.target.attempt", { - comboName: combo.name, - targetIndex: i, - provider, - model: modelStr, - timestamp: Date.now(), - strategy, - }); - // QA P0 diagnostics: capture the attempt order (provider/model ids only). - comboAttemptOrder.push({ provider: provider ?? "unknown", model: modelStr }); - - // Copy-on-write, not a deep clone (#7847 — 9.53 MiB at 3 targets). Writes here are - // top-level scalars. Invariant: tests/unit/combo-attempt-body-isolation-7847.test.ts. - let attemptBody = { ...(body as Record) } as typeof body; - - // Proactive Context Compression for fallbacks (Zero-Latency optimization) - if ( - zeroLatencyOptimizationsEnabled && - i > 0 && - config.fallbackCompressionMode && - config.fallbackCompressionMode !== "off" - ) { - const { estimateTokens } = await import("./contextManager.ts"); - // #7847: object, not JSON.stringify — the string branch mis-counts inline images. - const estimatedTokens = estimateTokens(attemptBody); - if (estimatedTokens > (config.fallbackCompressionThreshold ?? 1000)) { - const { applyCompression } = await import("./compression/strategySelector.ts"); - const compressionResult = applyCompression( - attemptBody, - config.fallbackCompressionMode as CompressionMode, - // Opt into the TV1 bail-out so a throwing fallback engine is SKIPPED rather than - // propagating out of executeTarget and being swallowed as a "Speculative task - // error" (which silently drops this combo target). minGainPercent:0 keeps the - // advance behavior identical to the default path — this only adds skip-on-throw. - { model: modelStr, bailout: { enabled: true, minGainPercent: 0 } } + // Pre-screen snapshot is NOT used as a permanent skip — availability + // is always re-checked via isModelAvailable below because connection + // cooldowns can expire between setTry retries, making a previously + // unavailable target available again. Circuit-breaker-OPEN providers + // are already caught by the dedicated breaker check above. + if (isModelAvailable) { + const available = await isModelAvailable(modelStr, targetForAttempt); + if (!available) { + log.debug?.( + "COMBO", + `Skipping ${modelStr} — no credentials available or model excluded` ); - if (compressionResult.compressed) { - log.info( - "COMBO", - `Proactive fallback compression applied (${config.fallbackCompressionMode}): ${estimatedTokens} -> ${compressionResult.stats?.compressedTokens} tokens` - ); - attemptBody = compressionResult.body; - } + clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + recordComboDecision(traceInvocationId, { + step: target.executionKey, + target: modelStr, + decision: "skipped_before_dispatch", + reason: "availability", + }); + if (i > 0) fallbackCount++; + return stopProtectedPriorityTarget(`Model ${modelStr} is unavailable`); } } - // Universal handoff: inject existing handoff if model changed. i === 0 - // only: a fallback target (i > 0) serves the SAME client request the - // failed primary target would have served, with the original messages - // already intact -- there's nothing to hand off, since the client never - // saw the earlier target fail. Injecting a handoff note there replaces - // real context with a context-free note, which weaker fallback models - // have been observed treating as license to fabricate content instead - // of just answering the actual request (#12227 follow-up). - if ( - i === 0 && - universalHandoffConfig.enabled && - relayOptions?.sessionId && - !(body as Record)?.[SKIP_UNIVERSAL_HANDOFF_FLAG] - ) { - const lastModel = getLastSessionModel(relayOptions.sessionId, combo.name); - if (lastModel && lastModel !== modelStr) { - const existingHandoff = getHandoff(relayOptions.sessionId, combo.name); - attemptBody = injectUniversalHandoffBody( - attemptBody, // Use the cloned body to maintain isolation - lastModel, - modelStr, - `Model routing: ${lastModel} → ${modelStr}`, - existingHandoff, - universalHandoffConfig.relayMode - ); + // Credential gate: skip targets with known-bad credentials (fail-fast) + const connectionId = target.connectionId as string | undefined; + if (connectionId) { + const gateResult = checkCredentialGate(connectionId, provider, modelStr); + if (gateResult.allowed === false) { + logCredentialSkip(log, modelStr, gateResult.reason || "Credential gate blocked"); + recordComboDecision(traceInvocationId, { + step: target.executionKey, + target: modelStr, + decision: "skipped_before_dispatch", + reason: "credential_gate", + }); + if (i > 0) fallbackCount++; + return stopProtectedPriorityTarget(`Credential gate blocked ${modelStr}`); } - } - // Issue #3587: Reasoning models can spend the whole output budget on - // reasoning. Only add headroom when the complete buffer fits inside the - // model's known output cap; otherwise preserve the client's explicit limit. - { - const bodyRecord = attemptBody as Record; - const currentMaxTokens = toPositiveInteger(bodyRecord.max_tokens); - const bufferedMaxTokens = resolveReasoningBufferedMaxTokens( - modelStr, - bodyRecord.max_tokens, - { enabled: reasoningTokenBufferEnabled } - ); - if (currentMaxTokens !== null && bufferedMaxTokens !== null) { - bodyRecord.max_tokens = bufferedMaxTokens; - if (bufferedMaxTokens !== currentMaxTokens) { - log.info( - "COMBO", - `Reasoning model ${modelStr}: adjusted max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` - ); - } + // Concurrency gate: fail-fast skip when connection is at max_concurrent capacity (e.g. Featherless 1/1) + const maxConcurrentCap = await lookupPositiveCap(connectionId); + if ( + maxConcurrentCap && + isAccountSemaphoreFull(provider, connectionId, maxConcurrentCap) + ) { + log.info( + "COMBO", + `Skipping ${modelStr} — connection ${connectionId} is at max concurrency cap (${maxConcurrentCap})` + ); + recordComboDecision(traceInvocationId, { + step: target.executionKey, + target: modelStr, + decision: "skipped_before_dispatch", + reason: "concurrency_cap", + }); + if (i > 0) fallbackCount++; + return stopProtectedPriorityTarget(`Connection capacity reached for ${modelStr}`); } } - // #5501: server-side template expansion for the combo system_message — - // resolved per-target, scoped to combo-injected content only (never - // client-owned system messages). Gate: a non-empty combo system_message. - attemptBody = expandComboSystemPromptIfPresent(attemptBody, combo, { - modelId: modelStr, - providerId: provider !== "unknown" ? provider : "", - account: - typeof target.label === "string" && target.label.trim().length > 0 - ? target.label.trim() - : "", - fingerprint: resolveTargetFingerprint(target) ?? "", - }); - // #10681: record dispatch once per target (retries keep the first decision). - if (!dispatchedTargets.has(target.executionKey)) { - dispatchedTargets.add(target.executionKey); + + // #9654 Wave 2: per-target lane-aware admission probe. With virtual + // lanes on, a tenant whose lane queue is full should skip extra + // fan-out targets instead of piling more queued work onto the lane. + // Strictly non-blocking (maxWaitMs 0) and a no-op when lanes are off — + // see createPerTargetAdmissionHook for the full contract. + if ( + perTargetAdmission && + !(await perTargetAdmission({ modelStr, executionKey: target.executionKey, body })) + ) { + log.info("COMBO", `Skipping ${modelStr} — admission lane full (#9654)`); recordComboDecision(traceInvocationId, { step: target.executionKey, target: modelStr, - decision: "dispatched", + decision: "skipped_before_dispatch", + reason: "admission_lane", }); + if (i > 0) fallbackCount++; + return null; } - const result = await handleSingleModelWithTimeout(attemptBody, modelStr, { - ...targetForAttempt, - effectiveComboStrategy: strategy, - failoverBeforeRetry: config.failoverBeforeRetry, - }); - - // Success — validate response quality before returning - if (result.ok) { - const selectedConnectionId = - result.headers?.get("X-OmniRoute-Selected-Connection-Id") || - result.headers?.get("x-omniroute-selected-connection-id") || - undefined; - const effectiveConnectionId = selectedConnectionId || target.connectionId || ""; - // Clone BEFORE quality check — validateResponseQuality reads the body - // via getReader() which locks the stream. The clone's body is consumed - // by the quality check; the original stays unlocked for piping. - let qualityClone: Response; - try { - qualityClone = result.clone(); - } catch { - qualityClone = result; + // Retry loop for transient errors + for (let retry = 0; retry <= maxRetries; retry++) { + // Fix #1681: Bail out immediately if the client has disconnected + if (signal?.aborted) { + log.info( + "COMBO", + `Client disconnected — aborting combo loop before model ${modelStr}` + ); + return { ok: false, response: errorResponse(499, "Client disconnected") }; } - const quality = await validateResponseQuality( - qualityClone, - clientRequestedStream, - log, - config.responseValidation - ); - releaseQualityClone(qualityClone, result, quality); - if (!quality.valid) { - releaseRejectedQualityResponse(qualityClone, result); + globalAttempts++; + if (globalAttempts > maxGlobalAttempts) { log.warn( "COMBO", - `Model ${modelStr} returned 200 but failed quality check: ${quality.reason}` + `Maximum combo attempts (${maxGlobalAttempts}) exceeded across all targets and fallbacks. Terminating loop to prevent runaway background requests.` ); - // #6692: a quality-rejected 200 never marks the connection row - // unhealthy, so the sticky pin's lazy headroom recheck would never - // catch it either — release it here, on the failing response. - releaseStickyPinOnFailure(_sticky.messageHash, effectiveConnectionId); - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy, - target: toRecordedTarget(target), - }); - recordedAttempts++; - // Fix #1707: Set terminal state so the fallback doesn't emit - // misleading ALL_ACCOUNTS_INACTIVE when the real issue is quality. - lastError = `Upstream response failed quality validation: ${quality.reason}`; - lastStatus = 502; - // #10314: record quality failures as a FIRST-CLASS per-target outcome - // so a quality reason is never silently dropped from the aggregated - // terminal message when a later sibling overwrites lastError. - comboErrors.push({ - model: modelStr, - status: 502, - error: quality.reason || "upstream response failed quality validation", - kind: "quality", - }); - if (i > 0) fallbackCount++; - if (provider && rawModel) { - const mlSettings = resolveModelLockoutSettings(settings); - if (mlSettings.enabled && mlSettings.errorCodes.includes(502)) { - recordModelLockoutFailure( - provider, - target.connectionId || "", - rawModel, - "quality_failure", - 502, - mlSettings.baseCooldownMs, - profile, - { - exactCooldownMs: mlSettings.useExponentialBackoff - ? 0 - : mlSettings.baseCooldownMs, - maxCooldownMs: mlSettings.maxCooldownMs, - } + // Actionable failure instead of an opaque 503 when every candidate + // failed the same recoverable way. If the dominant cause was reasoning + // models exhausting a too-small max_tokens budget (no content output), + // retrying other models can't help — tell the caller to raise max_tokens. + // Silent-stop fix: bump the consecutive-failure counter for this session-combo pair + // so the pin gets cleared on the 3rd attempt (recovery.next_step tells the client). + const reasoningExhausted = /reasoning consumed \d+\/\d+ tokens/.test(lastError || ""); + const failureReason = reasoningExhausted + ? "reasoning_budget_exhausted" + : "max_attempts_exceeded"; + recordComboFailure(effectiveSessionId, combo.name); + return { + ok: false, + response: errorResponseWithComboDiagnostics( + 503, + reasoningExhausted + ? "All combo candidates exhausted their token budget on reasoning without producing content. Increase max_tokens — reasoning models need a larger budget to emit content." + : "Maximum combo retry limit reached", + buildComboDiag(failureReason) + ), + }; + } + // Predictive TTFT Circuit Breaker (skip slow models) + if ( + zeroLatencyOptimizationsEnabled && + config.predictiveTtftMs && + config.predictiveTtftMs > 0 && + retry === 0 + ) { + const cMetrics = getComboMetrics(combo.name); + if (cMetrics) { + const targetKey = orderedTargets[i].executionKey || modelStr; + const m = cMetrics.byTarget[targetKey] || cMetrics.byModel[modelStr]; + if (shouldSkipForPredictedTtft(m, config.predictiveTtftMs)) { + log.warn( + "COMBO", + `Predictive TTFT Circuit Breaker: skipping ${modelStr} (avg ${m.avgLatencyMs}ms > max ${config.predictiveTtftMs}ms)` + ); + recordComboDecision(traceInvocationId, { + step: target.executionKey, + target: modelStr, + decision: "skipped_before_dispatch", + reason: "predictive_ttft", + }); + return stopProtectedPriorityTarget( + `Predictive latency check rejected ${modelStr}` ); } } - emit("combo.target.failed", { - comboName: combo.name, - targetIndex: i, - provider, - model: modelStr, - error: `Quality: ${quality.reason}`, - latencyMs: Date.now() - startTime, - }); - observeFailure(false, target.executionKey); - return protectedPriorityTarget - ? { - ok: false, - response: errorResponse(502, "Upstream response failed quality validation"), - } - : null; } - if (clientManagedResponsesContext && effectiveConnectionId) { - pinNativeCodexTurn({ - body, - comboName: combo.name, - target, - connectionId: effectiveConnectionId, + if (retry > 0) { + log.info( + "COMBO", + `Retrying ${modelStr} in ${retryDelayMs}ms (attempt ${retry + 1}/${maxRetries + 1})` + ); + await new Promise((resolve) => { + const timer = setTimeout(resolve, retryDelayMs); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(undefined); + }, + { once: true } + ); }); - } + if (signal?.aborted) { + log.info("COMBO", `Client disconnected during retry delay — aborting`); + return { ok: false, response: errorResponse(499, "Client disconnected") }; + } - // Success decay: a healthy response walks the model's lockout failure - // count back down (and eventually clears an expired lockout entirely). - if (provider && rawModel) { - const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel); - if (dcResult.cleared) { - log.info("COMBO", `Model ${modelStr} fully recovered — lockout cleared`); - } else if (dcResult.newFailureCount > 0) { - log.debug( - "COMBO", - `Model ${modelStr} decayed to failureCount=${dcResult.newFailureCount}` - ); + // Retry re-check: a sibling attempt (or attempt 1) may have persisted + // a quota cooldown while this attempt was sleeping out its retry delay + // ("Trying model 1/7: zai/glm-5.3 (retry 1)" after "already marked + // unavailable until …"). Reads fresh, not cached: see readConnectionForCooldownGate. + const persistedRetrySkip = await resolvePersistedConnectionCooldownSkipReason( + target, + (id) => readConnectionForCooldownGate(id, true), + allowRateLimitedConnection + ); + if (persistedRetrySkip) { + log.info("COMBO", persistedRetrySkip); + if (i > 0) fallbackCount++; + return null; } } - const latencyMs = Date.now() - startTime; - emit("combo.target.succeeded", { + log.info( + "COMBO", + `Trying model ${i + 1}/${orderedTargets.length}: ${modelStr}${retry > 0 ? ` (retry ${retry})` : ""}` + ); + emit("combo.target.attempt", { comboName: combo.name, targetIndex: i, provider, model: modelStr, - latencyMs, - }); - log.info( - "COMBO", - `Model ${modelStr} succeeded (${latencyMs}ms, ${fallbackCount} fallbacks)` - ); - recordComboRequest(combo.name, modelStr, { - success: true, - latencyMs, - fallbackCount, + timestamp: Date.now(), strategy, - target: toRecordedTarget(target), }); - recordedAttempts++; + // QA P0 diagnostics: capture the attempt order (provider/model ids only). + comboAttemptOrder.push({ provider: provider ?? "unknown", model: modelStr }); - // Reset cooldown on success - if (provider && provider !== "unknown") { - recordProviderSuccess(provider, effectiveConnectionId || undefined); - } - if (strategy === "weighted" && stickyWeightedLimit > 1) { - const stickySuccessKey = getWeightedStepKeyForTarget(target); - if (stickySuccessKey) { - recordStickyWeightedSuccess(combo.name, stickySuccessKey, stickyWeightedLimit); - } - } - // Webhook fan-out: best-effort, never blocks the response stream. - notifyWebhookEvent("request.completed", { - combo: combo.name, - provider, - model: modelStr, - account: - typeof target.label === "string" && target.label.trim().length > 0 - ? target.label.trim() - : "", - accountId: effectiveConnectionId ?? "", - latencyMs, - fallbackCount, - }); + // Copy-on-write, not a deep clone (#7847 — 9.53 MiB at 3 targets). Writes here are + // top-level scalars. Invariant: tests/unit/combo-attempt-body-isolation-7847.test.ts. + let attemptBody = { ...(body as Record) } as typeof body; - // Silent-stop fix: reset the consecutive-failure counter for this session-combo pair - // on every successful dispatch so a transient recovery doesn't get "credited" against - // the threshold the user already paid through to clear the stale pin. - if (effectiveSessionId) { - clearComboFailureTracking(effectiveSessionId, combo.name); - } - // Context cache pinning: record model usage for session-based pinning - // (independent of universal handoff — always fires when context_cache_protection is on) - // #3825: write under the SAME effectiveSessionId used by the read site so a - // sessionless conversation re-pins to this model on its next turn. + // Proactive Context Compression for fallbacks (Zero-Latency optimization) if ( - combo.context_cache_protection && - effectiveSessionId && - !(body as Record)?.[SKIP_UNIVERSAL_HANDOFF_FLAG] + zeroLatencyOptimizationsEnabled && + i > 0 && + config.fallbackCompressionMode && + config.fallbackCompressionMode !== "off" ) { - recordSessionModelUsage( - effectiveSessionId, - combo.name, - modelStr, - provider, - target.connectionId ?? undefined - ); + const { estimateTokens } = await import("./contextManager.ts"); + // #7847: object, not JSON.stringify — the string branch mis-counts inline images. + const estimatedTokens = estimateTokens(attemptBody); + if (estimatedTokens > (config.fallbackCompressionThreshold ?? 1000)) { + const { applyCompression } = await import("./compression/strategySelector.ts"); + const compressionResult = applyCompression( + attemptBody, + config.fallbackCompressionMode as CompressionMode, + // Opt into the TV1 bail-out so a throwing fallback engine is SKIPPED rather than + // propagating out of executeTarget and being swallowed as a "Speculative task + // error" (which silently drops this combo target). minGainPercent:0 keeps the + // advance behavior identical to the default path — this only adds skip-on-throw. + { model: modelStr, bailout: { enabled: true, minGainPercent: 0 } } + ); + if (compressionResult.compressed) { + log.info( + "COMBO", + `Proactive fallback compression applied (${config.fallbackCompressionMode}): ${estimatedTokens} -> ${compressionResult.stats?.compressedTokens} tokens` + ); + attemptBody = compressionResult.body; + } + } } - // Universal handoff: record model usage for session + // Universal handoff: inject existing handoff if model changed. i === 0 + // only: a fallback target (i > 0) serves the SAME client request the + // failed primary target would have served, with the original messages + // already intact -- there's nothing to hand off, since the client never + // saw the earlier target fail. Injecting a handoff note there replaces + // real context with a context-free note, which weaker fallback models + // have been observed treating as license to fabricate content instead + // of just answering the actual request (#12227 follow-up). if ( + i === 0 && universalHandoffConfig.enabled && relayOptions?.sessionId && !(body as Record)?.[SKIP_UNIVERSAL_HANDOFF_FLAG] ) { - const prevModel = getLastSessionModel(relayOptions.sessionId, combo.name); - recordSessionModelUsage( - relayOptions.sessionId, - combo.name, + const lastModel = getLastSessionModel(relayOptions.sessionId, combo.name); + if (lastModel && lastModel !== modelStr) { + const existingHandoff = getHandoff(relayOptions.sessionId, combo.name); + attemptBody = injectUniversalHandoffBody( + attemptBody, // Use the cloned body to maintain isolation + lastModel, + modelStr, + `Model routing: ${lastModel} → ${modelStr}`, + existingHandoff, + universalHandoffConfig.relayMode + ); + } + } + + // Issue #3587: Reasoning models can spend the whole output budget on + // reasoning. Only add headroom when the complete buffer fits inside the + // model's known output cap; otherwise preserve the client's explicit limit. + { + const bodyRecord = attemptBody as Record; + const currentMaxTokens = toPositiveInteger(bodyRecord.max_tokens); + const bufferedMaxTokens = resolveReasoningBufferedMaxTokens( modelStr, - provider, - target.connectionId ?? undefined + bodyRecord.max_tokens, + { enabled: reasoningTokenBufferEnabled } + ); + if (currentMaxTokens !== null && bufferedMaxTokens !== null) { + bodyRecord.max_tokens = bufferedMaxTokens; + if (bufferedMaxTokens !== currentMaxTokens) { + log.info( + "COMBO", + `Reasoning model ${modelStr}: adjusted max_tokens ${currentMaxTokens} -> ${bufferedMaxTokens}` + ); + } + } + } + // #5501: server-side template expansion for the combo system_message — + // resolved per-target, scoped to combo-injected content only (never + // client-owned system messages). Gate: a non-empty combo system_message. + attemptBody = expandComboSystemPromptIfPresent(attemptBody, combo, { + modelId: modelStr, + providerId: provider !== "unknown" ? provider : "", + account: + typeof target.label === "string" && target.label.trim().length > 0 + ? target.label.trim() + : "", + fingerprint: resolveTargetFingerprint(target) ?? "", + }); + // #10681: record dispatch once per target (retries keep the first decision). + if (!dispatchedTargets.has(target.executionKey)) { + dispatchedTargets.add(target.executionKey); + recordComboDecision(traceInvocationId, { + step: target.executionKey, + target: modelStr, + decision: "dispatched", + }); + } + const result = await handleSingleModelWithTimeout(attemptBody, modelStr, { + ...targetForAttempt, + effectiveComboStrategy: strategy, + failoverBeforeRetry: config.failoverBeforeRetry, + }); + + // Success — validate response quality before returning + if (result.ok) { + const selectedConnectionId = + result.headers?.get("X-OmniRoute-Selected-Connection-Id") || + result.headers?.get("x-omniroute-selected-connection-id") || + undefined; + const effectiveConnectionId = selectedConnectionId || target.connectionId || ""; + + // Clone BEFORE quality check — validateResponseQuality reads the body + // via getReader() which locks the stream. The clone's body is consumed + // by the quality check; the original stays unlocked for piping. + let qualityClone: Response; + try { + qualityClone = result.clone(); + } catch { + qualityClone = result; + } + const quality = await validateResponseQuality( + qualityClone, + clientRequestedStream, + log, + config.responseValidation ); - // i === 0 only: a same-request fallback target (i > 0) never - // needs a summary generated for it -- see the injection-site - // comment above. recordSessionModelUsage above stays - // unconditional regardless of i: it must reflect whichever - // model actually served THIS response, since the next - // request's i === 0 comparison depends on that being - // accurate even when this response came from a fallback. - if (i === 0 && prevModel && prevModel !== modelStr) { - const handoffSourceMessages = - Array.isArray(body?.messages) && body.messages.length > 0 - ? body.messages - : Array.isArray(body?.input) - ? body.input - : []; - - maybeGenerateUniversalHandoff({ - sessionId: relayOptions.sessionId, + releaseQualityClone(qualityClone, result, quality); + if (!quality.valid) { + releaseRejectedQualityResponse(qualityClone, result); + log.warn( + "COMBO", + `Model ${modelStr} returned 200 but failed quality check: ${quality.reason}` + ); + // #6692: a quality-rejected 200 never marks the connection row + // unhealthy, so the sticky pin's lazy headroom recheck would never + // catch it either — release it here, on the failing response. + releaseStickyPinOnFailure(_sticky.messageHash, effectiveConnectionId); + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + // Fix #1707: Set terminal state so the fallback doesn't emit + // misleading ALL_ACCOUNTS_INACTIVE when the real issue is quality. + lastError = `Upstream response failed quality validation: ${quality.reason}`; + lastStatus = 502; + // #10314: record quality failures as a FIRST-CLASS per-target outcome + // so a quality reason is never silently dropped from the aggregated + // terminal message when a later sibling overwrites lastError. + comboErrors.push({ + model: modelStr, + status: 502, + error: quality.reason || "upstream response failed quality validation", + kind: "quality", + }); + if (i > 0) fallbackCount++; + if (provider && rawModel) { + const mlSettings = resolveModelLockoutSettings(settings); + if (mlSettings.enabled && mlSettings.errorCodes.includes(502)) { + recordModelLockoutFailure( + provider, + target.connectionId || "", + rawModel, + "quality_failure", + 502, + mlSettings.baseCooldownMs, + profile, + { + exactCooldownMs: mlSettings.useExponentialBackoff + ? 0 + : mlSettings.baseCooldownMs, + maxCooldownMs: mlSettings.maxCooldownMs, + } + ); + } + } + emit("combo.target.failed", { + comboName: combo.name, + targetIndex: i, + provider, + model: modelStr, + error: `Quality: ${quality.reason}`, + latencyMs: Date.now() - startTime, + }); + observeFailure(false, target.executionKey); + return protectedPriorityTarget + ? { + ok: false, + response: errorResponse(502, "Upstream response failed quality validation"), + } + : null; + } + + if (clientManagedResponsesContext && effectiveConnectionId) { + pinNativeCodexTurn({ + body, comboName: combo.name, - messages: handoffSourceMessages as MessageLike[], - prevModel, - currModel: modelStr, - universalConfig: universalHandoffConfig, - handleSingleModel: handleSingleModelWithTimeout, + target, + connectionId: effectiveConnectionId, }); } - recordSessionModelUsage( - relayOptions.sessionId, - combo.name, - modelStr, + // Success decay: a healthy response walks the model's lockout failure + // count back down (and eventually clears an expired lockout entirely). + if (provider && rawModel) { + const dcResult = decayModelFailureCount(provider, effectiveConnectionId, rawModel); + if (dcResult.cleared) { + log.info("COMBO", `Model ${modelStr} fully recovered — lockout cleared`); + } else if (dcResult.newFailureCount > 0) { + log.debug( + "COMBO", + `Model ${modelStr} decayed to failureCount=${dcResult.newFailureCount}` + ); + } + } + + const latencyMs = Date.now() - startTime; + emit("combo.target.succeeded", { + comboName: combo.name, + targetIndex: i, provider, - target.connectionId ?? undefined + model: modelStr, + latencyMs, + }); + log.info( + "COMBO", + `Model ${modelStr} succeeded (${latencyMs}ms, ${fallbackCount} fallbacks)` ); - } - // Context-relay intentionally splits responsibilities: - // combo.ts decides whether a successful turn should generate a handoff, - // while chat.ts injects the handoff after the real connectionId is resolved. - if ( - strategy === "context-relay" && - relayOptions?.sessionId && - relayConfig && - relayConfig.handoffProviders.includes(provider) && - provider === "codex" - ) { - const connectionId = getSessionConnection(relayOptions.sessionId); - if (connectionId) { - const quotaInfo = await fetchCodexQuota(connectionId).catch(() => null); - if (quotaInfo) { - const resetCandidates = [ - quotaInfo.windows?.session?.resetAt, - quotaInfo.windows?.weekly?.resetAt, - quotaInfo.resetAt, - ] - .filter( - (value): value is string => typeof value === "string" && value.length > 0 - ) - .sort((a, b) => a.localeCompare(b)); + recordComboRequest(combo.name, modelStr, { + success: true, + latencyMs, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + + // Reset cooldown on success + if (provider && provider !== "unknown") { + recordProviderSuccess(provider, effectiveConnectionId || undefined); + } + if (strategy === "weighted" && stickyWeightedLimit > 1) { + const stickySuccessKey = getWeightedStepKeyForTarget(target); + if (stickySuccessKey) { + recordStickyWeightedSuccess(combo.name, stickySuccessKey, stickyWeightedLimit); + } + } + // Webhook fan-out: best-effort, never blocks the response stream. + notifyWebhookEvent("request.completed", { + combo: combo.name, + provider, + model: modelStr, + account: + typeof target.label === "string" && target.label.trim().length > 0 + ? target.label.trim() + : "", + accountId: effectiveConnectionId ?? "", + latencyMs, + fallbackCount, + }); + + // Silent-stop fix: reset the consecutive-failure counter for this session-combo pair + // on every successful dispatch so a transient recovery doesn't get "credited" against + // the threshold the user already paid through to clear the stale pin. + if (effectiveSessionId) { + clearComboFailureTracking(effectiveSessionId, combo.name); + } + // Context cache pinning: record model usage for session-based pinning + // (independent of universal handoff — always fires when context_cache_protection is on) + // #3825: write under the SAME effectiveSessionId used by the read site so a + // sessionless conversation re-pins to this model on its next turn. + if ( + combo.context_cache_protection && + effectiveSessionId && + !(body as Record)?.[SKIP_UNIVERSAL_HANDOFF_FLAG] + ) { + recordSessionModelUsage( + effectiveSessionId, + combo.name, + modelStr, + provider, + target.connectionId ?? undefined + ); + } + + // Universal handoff: record model usage for session + if ( + universalHandoffConfig.enabled && + relayOptions?.sessionId && + !(body as Record)?.[SKIP_UNIVERSAL_HANDOFF_FLAG] + ) { + const prevModel = getLastSessionModel(relayOptions.sessionId, combo.name); + recordSessionModelUsage( + relayOptions.sessionId, + combo.name, + modelStr, + provider, + target.connectionId ?? undefined + ); + // i === 0 only: a same-request fallback target (i > 0) never + // needs a summary generated for it -- see the injection-site + // comment above. recordSessionModelUsage above stays + // unconditional regardless of i: it must reflect whichever + // model actually served THIS response, since the next + // request's i === 0 comparison depends on that being + // accurate even when this response came from a fallback. + if (i === 0 && prevModel && prevModel !== modelStr) { const handoffSourceMessages = Array.isArray(body?.messages) && body.messages.length > 0 ? body.messages @@ -2018,306 +2019,562 @@ async function handleComboChatInner({ ? body.input : []; - maybeGenerateHandoff({ + maybeGenerateUniversalHandoff({ sessionId: relayOptions.sessionId, comboName: combo.name, - connectionId, - percentUsed: quotaInfo.percentUsed, - messages: handoffSourceMessages, - model: modelStr, - expiresAt: resetCandidates[0] || null, - config: relayConfig, + messages: handoffSourceMessages as MessageLike[], + prevModel, + currModel: modelStr, + universalConfig: universalHandoffConfig, handleSingleModel: handleSingleModelWithTimeout, }); } + + recordSessionModelUsage( + relayOptions.sessionId, + combo.name, + modelStr, + provider, + target.connectionId ?? undefined + ); } - } - if (_sticky.messageHash && target.connectionId) - recordStickyBinding(_sticky.messageHash, target.connectionId); // LKGP (#919): - if (provider) { - const connId = effectiveConnectionId || undefined; - void (async () => { - try { - const { setLKGP } = await import("@/lib/db/settings"); - await Promise.all([ - setLKGP(combo.name, target.executionKey, provider, connId), - setLKGP(combo.name, combo.id || combo.name, provider, connId), - ]); - } catch (err) { - log.warn( - "COMBO", - "Failed to record Last Known Good Provider. This is non-fatal.", - { - err, - } - ); + // Context-relay intentionally splits responsibilities: + // combo.ts decides whether a successful turn should generate a handoff, + // while chat.ts injects the handoff after the real connectionId is resolved. + if ( + strategy === "context-relay" && + relayOptions?.sessionId && + relayConfig && + relayConfig.handoffProviders.includes(provider) && + provider === "codex" + ) { + const connectionId = getSessionConnection(relayOptions.sessionId); + if (connectionId) { + const quotaInfo = await fetchCodexQuota(connectionId).catch(() => null); + if (quotaInfo) { + const resetCandidates = [ + quotaInfo.windows?.session?.resetAt, + quotaInfo.windows?.weekly?.resetAt, + quotaInfo.resetAt, + ] + .filter( + (value): value is string => typeof value === "string" && value.length > 0 + ) + .sort((a, b) => a.localeCompare(b)); + const handoffSourceMessages = + Array.isArray(body?.messages) && body.messages.length > 0 + ? body.messages + : Array.isArray(body?.input) + ? body.input + : []; + + maybeGenerateHandoff({ + sessionId: relayOptions.sessionId, + comboName: combo.name, + connectionId, + percentUsed: quotaInfo.percentUsed, + messages: handoffSourceMessages, + model: modelStr, + expiresAt: resetCandidates[0] || null, + config: relayConfig, + handleSingleModel: handleSingleModelWithTimeout, + }); + } } - })(); - } + } + if (_sticky.messageHash && target.connectionId) + recordStickyBinding(_sticky.messageHash, target.connectionId); // LKGP (#919): + if (provider) { + const connId = effectiveConnectionId || undefined; + void (async () => { + try { + const { setLKGP } = await import("@/lib/db/settings"); + await Promise.all([ + setLKGP(combo.name, target.executionKey, provider, connId), + setLKGP(combo.name, combo.id || combo.name, provider, connId), + ]); + } catch (err) { + log.warn( + "COMBO", + "Failed to record Last Known Good Provider. This is non-fatal.", + { + err, + } + ); + } + })(); + } - return { ok: true, response: result }; - } + return { ok: true, response: result }; + } - // Extract error info from response - let errorText = result.statusText || ""; - let errorBody: ComboErrorBody = null; - let retryAfter: ComboRetryAfter | null = null; - try { - const cloned = result.clone(); + // Extract error info from response + let errorText = result.statusText || ""; + let errorBody: ComboErrorBody = null; + let retryAfter: ComboRetryAfter | null = null; try { - const text = await cloned.text(); - if (text) { - errorText = text.substring(0, 500); - errorBody = JSON.parse(text); - const parsedError = errorBody?.error; - errorText = - (typeof parsedError === "object" && parsedError?.message) || - (typeof parsedError === "string" ? parsedError : null) || - errorBody?.message || - errorText; - // Live incident (log id 1784457764961-73 follow-up): the pre-dispatch - // "all credentials cooling down" rejection (buildModelCooldownBody / - // handleNoCredentials in src/sse/handlers/chatHelpers.ts) nests its - // retry hint as error.retry_after (ISO string) / error.reset_seconds - // (seconds), not the top-level `retryAfter` every other 429 shape - // uses. Without this fallback, lastStatus gets recorded (fixed above) - // but earliestRetryAfter stays null, so the final check falls through - // to the generic "all combo models unavailable" error instead of ever - // reaching the cooldown-wait decision — same class of bug, different - // response shape. - const nestedRetryAfter = - typeof parsedError === "object" ? (parsedError?.retry_after ?? null) : null; - const nestedResetSeconds = - typeof parsedError === "object" ? (parsedError?.reset_seconds ?? null) : null; - retryAfter = - errorBody?.retryAfter || - nestedRetryAfter || - (typeof nestedResetSeconds === "number" && nestedResetSeconds > 0 - ? new Date(Date.now() + nestedResetSeconds * 1000).toISOString() - : null); + const cloned = result.clone(); + try { + const text = await cloned.text(); + if (text) { + errorText = text.substring(0, 500); + errorBody = JSON.parse(text); + const parsedError = errorBody?.error; + errorText = + (typeof parsedError === "object" && parsedError?.message) || + (typeof parsedError === "string" ? parsedError : null) || + errorBody?.message || + errorText; + // Live incident (log id 1784457764961-73 follow-up): the pre-dispatch + // "all credentials cooling down" rejection (buildModelCooldownBody / + // handleNoCredentials in src/sse/handlers/chatHelpers.ts) nests its + // retry hint as error.retry_after (ISO string) / error.reset_seconds + // (seconds), not the top-level `retryAfter` every other 429 shape + // uses. Without this fallback, lastStatus gets recorded (fixed above) + // but earliestRetryAfter stays null, so the final check falls through + // to the generic "all combo models unavailable" error instead of ever + // reaching the cooldown-wait decision — same class of bug, different + // response shape. + const nestedRetryAfter = + typeof parsedError === "object" ? (parsedError?.retry_after ?? null) : null; + const nestedResetSeconds = + typeof parsedError === "object" ? (parsedError?.reset_seconds ?? null) : null; + retryAfter = + errorBody?.retryAfter || + nestedRetryAfter || + (typeof nestedResetSeconds === "number" && nestedResetSeconds > 0 + ? new Date(Date.now() + nestedResetSeconds * 1000).toISOString() + : null); + } + } catch { + /* Clone parse failed */ } } catch { - /* Clone parse failed */ + /* Clone failed */ } - } catch { - /* Clone failed */ - } - - // Track earliest retryAfter - if ( - retryAfter && - (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter)) - ) { - earliestRetryAfter = retryAfter; - } - // Normalize error text - if (typeof errorText !== "string") { - try { - errorText = JSON.stringify(errorText); - } catch { - errorText = String(errorText); + // Track earliest retryAfter + if ( + retryAfter && + (!earliestRetryAfter || new Date(retryAfter) < new Date(earliestRetryAfter)) + ) { + earliestRetryAfter = retryAfter; } - } - const isStreamReadinessFailure = - (result.status === 502 || result.status === 504) && - isStreamReadinessFailureErrorBody(errorBody); - // An early EOF is an upstream failure, not a readiness probe — the breaker must - // see it even though the transient-retry path below treats both codes alike. - const isStreamEarlyEof = - (result.status === 502 || result.status === 504) && - isStreamEarlyEofErrorBody(errorBody); + // Normalize error text + if (typeof errorText !== "string") { + try { + errorText = JSON.stringify(errorText); + } catch { + errorText = String(errorText); + } + } - // FIX 5: a local per-API-key token-limit 429 must not cool shared accounts. - const isTokenLimitBreach = - result.status === 429 && isTokenLimitBreachErrorBody(errorBody); - const isLocalQueueCapacity = isLocalQueueCapacityErrorBody(errorBody); + const isStreamReadinessFailure = + (result.status === 502 || result.status === 504) && + isStreamReadinessFailureErrorBody(errorBody); + // An early EOF is an upstream failure, not a readiness probe — the breaker must + // see it even though the transient-retry path below treats both codes alike. + const isStreamEarlyEof = + (result.status === 502 || result.status === 504) && + isStreamEarlyEofErrorBody(errorBody); + + // FIX 5: a local per-API-key token-limit 429 must not cool shared accounts. + const isTokenLimitBreach = + result.status === 429 && isTokenLimitBreachErrorBody(errorBody); + const isLocalQueueCapacity = isLocalQueueCapacityErrorBody(errorBody); + + // Fix #1681: Status 499 means client disconnected — stop combo loop immediately. + // There is no point trying fallback models when nobody is listening. + if (result.status === 499) { + log.info( + "COMBO", + `Client disconnected (499) during ${modelStr} — stopping combo loop` + ); + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + // executeTarget must return the {ok,response} contract — a raw Response + // here makes the speculative loop's res.ok/res.response checks both miss, + // so the combo would wrongly fall through to the next model after a 499. + return { ok: false, response: result }; + } + if (isLocalQueueCapacity) { + log.info( + "COMBO", + `Local rate-limit queue capacity reached for ${modelStr} — returning without upstream fallback` + ); + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + if (i > 0) fallbackCount++; + return { ok: false, response: result }; + } - // Fix #1681: Status 499 means client disconnected — stop combo loop immediately. - // There is no point trying fallback models when nobody is listening. - if (result.status === 499) { - log.info("COMBO", `Client disconnected (499) during ${modelStr} — stopping combo loop`); - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy, - target: toRecordedTarget(target), - }); - recordedAttempts++; - // executeTarget must return the {ok,response} contract — a raw Response - // here makes the speculative loop's res.ok/res.response checks both miss, - // so the combo would wrongly fall through to the next model after a 499. - return { ok: false, response: result }; - } - if (isLocalQueueCapacity) { - log.info( - "COMBO", - `Local rate-limit queue capacity reached for ${modelStr} — returning without upstream fallback` + // Combo fallback is target-level orchestration: a non-ok target response is + // treated as local to that target and the combo continues to the next target. + // Error classification is retained only for retry/cooldown pacing; it must + // not decide whether fallback happens, including for generic 400 responses. + const rawError = errorBody?.error; + const structuredError = + rawError && typeof rawError === "object" + ? { + // Upstream JSON may carry a numeric `code`/`type` (e.g. {"code":40001}). + // Coerce to string if present instead of discarding, so downstream string + // ops (.toLowerCase, .startsWith) can run safely without type crashes. + code: + (rawError as Record).code !== undefined && + (rawError as Record).code !== null + ? String((rawError as Record).code) + : undefined, + type: + (rawError as Record).type !== undefined && + (rawError as Record).type !== null + ? String((rawError as Record).type) + : undefined, + } + : undefined; + const scopedFailure = isScopedFailure(result, errorText, structuredError); + + // #8375: input-bound request-scoped failures (context_length_exceeded) are + // deterministic for the same input — retrying on other accounts of the same + // model will fail identically. Short-circuit the combo immediately with the + // original error instead of burning MAX_GLOBAL_ATTEMPTS. + // Scoped to homogeneous remainders only: a heterogeneous combo (#6637) may + // have a later target with a different, larger context window that would + // NOT reject the same input — isContextOverflow400 below exists precisely to + // let that case fall through, so only short-circuit when every remaining + // target is the same model (the "retrying will fail identically" premise + // only holds within a homogeneous same-model pool). + const remainingTargets = orderedTargets.slice(i + 1); + const remainderIsHomogeneous = remainingTargets.every( + (nextInPool) => nextInPool.modelStr === modelStr ); - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy, - target: toRecordedTarget(target), + const isInputBoundFailure = + isInputBoundRequestFailure(structuredError) && remainderIsHomogeneous; + if (isInputBoundFailure) { + log.warn( + "COMBO", + `Input-bound request failure from ${modelStr} — aborting combo (same input will fail identically on every account)` + ); + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + if (i > 0) fallbackCount++; + return { ok: false, response: result }; + } + const fallbackResult = checkFallbackError( + result.status, + errorText, + 0, + protectedPriorityTarget ? rawModel : null, + provider, + result.headers, + profile, + structuredError + ); + const { cooldownMs } = fallbackResult; + // #6863: a parsed upstream quota reset (e.g. Antigravity "Resets in 92h27m28s") + // arrives in `quotaResetHintMs` — it bypasses the operator-gated + // `useUpstreamRetryHints` connection-cooldown setting. Mirror the + // single-model path (src/sse/services/auth.ts): when the retry hint was + // already honored, `cooldownMs` IS the upstream value; otherwise prefer the + // parsed quota reset — even when it is SHORTER than the fallback cooldown + // (e.g. subscription-quota 1h default vs a real "resets in 10m"). + // `selectLockoutCooldownMs` still ignores hints at/below the base cooldown, + // so absent/tiny hints keep the #1308 exponential-backoff behavior. + const lockoutHintMs = + fallbackResult.usedUpstreamRetryHint === true + ? cooldownMs + : (fallbackResult.quotaResetHintMs ?? 0); + // Only a transport header or google.rpc.RetryInfo is authoritative enough + // to bypass maxCooldownMs. Prose and generic JSON remain useful exact hints, + // but the operator cap still bounds them. + const lockoutHintVerified = retryHintBypassesMaxCooldownMs( + fallbackResult.retryHintSource + ); + const selectedConnectionId = + result.headers?.get("X-OmniRoute-Selected-Connection-Id") || + result.headers?.get("x-omniroute-selected-connection-id") || + undefined; + const targetWithConnection = selectedConnectionId + ? { ...target, connectionId: selectedConnectionId } + : target; + + // #1731 / #1731v2: classify the upstream error and update the exhaustion sets + // (shared with handleRoundRobinCombo). Returns whether the provider is fully exhausted. + const providerExhausted = applyComboTargetExhaustion(targetWithConnection, { + result, + fallbackResult, + errorText, + rawModel, + isTokenLimitBreach, + allAccountsRateLimited: false, + requestScopedFailure: scopedFailure, + sets: { exhaustedProviders, exhaustedConnections, transientRateLimitedProviders }, + log, + tag: "COMBO", + exhaustedLogLevel: "info", + structuredError, }); - recordedAttempts++; - if (i > 0) fallbackCount++; - return { ok: false, response: result }; - } + // #6692: this connection was just classified as provider/connection-level + // exhausted — if it's the currently sticky-bound one, release the pin now + // rather than waiting for the next turn's lazy headroom/status recheck. + releaseStickyPinOnFailure(_sticky.messageHash, targetWithConnection.connectionId); + if ( + providerExhausted || + exhaustedConnections.has(`${provider}:${targetWithConnection.connectionId}`) || + (provider && exhaustedProviders.has(provider)) + ) { + clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + } - // Combo fallback is target-level orchestration: a non-ok target response is - // treated as local to that target and the combo continues to the next target. - // Error classification is retained only for retry/cooldown pacing; it must - // not decide whether fallback happens, including for generic 400 responses. - const rawError = errorBody?.error; - const structuredError = - rawError && typeof rawError === "object" - ? { - // Upstream JSON may carry a numeric `code`/`type` (e.g. {"code":40001}). - // Coerce to string if present instead of discarding, so downstream string - // ops (.toLowerCase, .startsWith) can run safely without type crashes. - code: - (rawError as Record).code !== undefined && - (rawError as Record).code !== null - ? String((rawError as Record).code) - : undefined, - type: - (rawError as Record).type !== undefined && - (rawError as Record).type !== null - ? String((rawError as Record).type) - : undefined, - } - : undefined; - const scopedFailure = isScopedFailure(result, errorText, structuredError); + // #2101: Prevent infinite fallback loops with 400 Bad Request errors that are genuinely + // body-specific (malformed JSON, bad format, missing required fields). + // These should NOT stop the combo: + // - Context overflow: different models have different context windows + // - Max_tokens / param errors: different models have different output limits + // - Model access denied / "not supported": different providers serve different + // model sets — keep the model in the combo and try the next target (#5249). + // Wrapper words like "invalid" / "bad request" still stop only when the text is + // NOT model-scoped (e.g. "invalid message format"). + if ( + result.status === 400 && + fallbackResult.shouldFallback && + !isContextOverflow400(errorText) && + !isParamValidation400(errorText) && + !isModelScoped400(errorText) && + (errorText.toLowerCase().includes("context") || + errorText.toLowerCase().includes("prompt") || + errorText.toLowerCase().includes("token") || + errorText.toLowerCase().includes("malformed") || + errorText.toLowerCase().includes("invalid") || + errorText.toLowerCase().includes("bad request")) + ) { + log.warn( + "COMBO", + `400 Bad Request with body-specific error detected on ${modelStr} — skipping fallback to other targets to prevent infinite loop` + ); + // Record the failure and break to avoid trying other targets with the same bad request + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + lastError = errorText || String(result.status); + comboErrors.push({ + model: modelStr, + status: result.status, + error: errorText || String(result.status), + kind: classifyComboOutcome(result.status, errorText), + }); + lastStatus = result.status; + if (i > 0) fallbackCount++; + log.warn( + "COMBO", + `Model ${modelStr} failed with body-specific error, stopping combo` + ); + clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); + // #4279: surface the 400 via the {ok,response} contract so the OUTER + // target loop resolves the combo and stops. A bare `break` here only + // exits the inner retry loop; executeTarget then returns null, which + // the outer loop treats as "this target produced nothing" and advances + // to the next model — so the guard failed to stop fallback and a combo + // of N body-rejecting targets tried all N. Mirrors the 499 path above. + return { ok: false, response: result }; + } - // #8375: input-bound request-scoped failures (context_length_exceeded) are - // deterministic for the same input — retrying on other accounts of the same - // model will fail identically. Short-circuit the combo immediately with the - // original error instead of burning MAX_GLOBAL_ATTEMPTS. - // Scoped to homogeneous remainders only: a heterogeneous combo (#6637) may - // have a later target with a different, larger context window that would - // NOT reject the same input — isContextOverflow400 below exists precisely to - // let that case fall through, so only short-circuit when every remaining - // target is the same model (the "retrying will fail identically" premise - // only holds within a homogeneous same-model pool). - const remainingTargets = orderedTargets.slice(i + 1); - const remainderIsHomogeneous = remainingTargets.every( - (nextInPool) => nextInPool.modelStr === modelStr - ); - const isInputBoundFailure = - isInputBoundRequestFailure(structuredError) && remainderIsHomogeneous; - if (isInputBoundFailure) { - log.warn( - "COMBO", - `Input-bound request failure from ${modelStr} — aborting combo (same input will fail identically on every account)` - ); - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy, - target: toRecordedTarget(target), - }); - recordedAttempts++; - if (i > 0) fallbackCount++; - return { ok: false, response: result }; - } - const fallbackResult = checkFallbackError( - result.status, - errorText, - 0, - protectedPriorityTarget ? rawModel : null, - provider, - result.headers, - profile, - structuredError - ); - const { cooldownMs } = fallbackResult; - // #6863: a parsed upstream quota reset (e.g. Antigravity "Resets in 92h27m28s") - // arrives in `quotaResetHintMs` — it bypasses the operator-gated - // `useUpstreamRetryHints` connection-cooldown setting. Mirror the - // single-model path (src/sse/services/auth.ts): when the retry hint was - // already honored, `cooldownMs` IS the upstream value; otherwise prefer the - // parsed quota reset — even when it is SHORTER than the fallback cooldown - // (e.g. subscription-quota 1h default vs a real "resets in 10m"). - // `selectLockoutCooldownMs` still ignores hints at/below the base cooldown, - // so absent/tiny hints keep the #1308 exponential-backoff behavior. - const lockoutHintMs = - fallbackResult.usedUpstreamRetryHint === true - ? cooldownMs - : (fallbackResult.quotaResetHintMs ?? 0); - // Only a transport header or google.rpc.RetryInfo is authoritative enough - // to bypass maxCooldownMs. Prose and generic JSON remain useful exact hints, - // but the operator cap still bounds them. - const lockoutHintVerified = retryHintBypassesMaxCooldownMs( - fallbackResult.retryHintSource - ); - const selectedConnectionId = - result.headers?.get("X-OmniRoute-Selected-Connection-Id") || - result.headers?.get("x-omniroute-selected-connection-id") || - undefined; - const targetWithConnection = selectedConnectionId - ? { ...target, connectionId: selectedConnectionId } - : target; + // A model-scoped 400 ("The requested model is not supported" / "not + // available for integrator") is permanent for THIS connection — the + // account/integration will not gain support for the model mid-session. + // Combo still advances to the next target immediately (unchanged, + // preserves #5249's cross-provider fallback), but without a lockout + // here the SAME dead model gets retried on every future, separate + // request forever (observed: every auto-combo request wasted several + // upstream 400s on the same GitHub models, all day). isModelLocked() + // is checked before dispatch (see the pre-check above this loop), so + // this lockout is honored on the next request. + if (result.status === 400 && isModelScoped400(errorText) && provider && rawModel) { + lockModelIfPerModelQuota( + provider, + targetWithConnection.connectionId || "", + rawModel, + "model_capacity", + 60 * 60 * 1000 // 1h + ); + } - // #1731 / #1731v2: classify the upstream error and update the exhaustion sets - // (shared with handleRoundRobinCombo). Returns whether the provider is fully exhausted. - const providerExhausted = applyComboTargetExhaustion(targetWithConnection, { - result, - fallbackResult, - errorText, - rawModel, - isTokenLimitBreach, - allAccountsRateLimited: false, - requestScopedFailure: scopedFailure, - sets: { exhaustedProviders, exhaustedConnections, transientRateLimitedProviders }, - log, - tag: "COMBO", - exhaustedLogLevel: "info", - structuredError, - }); - // #6692: this connection was just classified as provider/connection-level - // exhausted — if it's the currently sticky-bound one, release the pin now - // rather than waiting for the next turn's lazy headroom/status recheck. - releaseStickyPinOnFailure(_sticky.messageHash, targetWithConnection.connectionId); - if ( - providerExhausted || - exhaustedConnections.has(`${provider}:${targetWithConnection.connectionId}`) || - (provider && exhaustedProviders.has(provider)) - ) { - clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); - } + // Trigger shared provider circuit breaker for 5xx errors and connection failures. If the + // next target is on the same provider, don't mark it failed (a different model may still + // succeed) — #8376: EXCEPT a proxy-unreachable failure, which poisons every model alike. + // G-02: when fallbackResult.skipProviderBreaker is set (embedded service supervisor outage + // signalled via X-Omni-Fallback-Hint: connection_cooldown) apply cooldown only — never trip. + const nextTarget = orderedTargets[i + 1]; + const sameProviderNext = + typeof nextTarget?.provider === "string" && nextTarget.provider === provider; + if ( + shouldRecordProviderBreakerFailure({ + isStreamReadinessFailure, + isStreamEarlyEof, + status: result.status, + sameProviderNext, + skipProviderBreaker: fallbackResult.skipProviderBreaker, + requestScopedFailure: scopedFailure, + error: errorText, + isProxyUnreachable: structuredError?.code === "proxy_unreachable", + }) + ) { + const isQueueTimeout = + errorText.includes("RATE_LIMIT_QUEUE_TIMEOUT") || + errorText.includes("RATE_LIMIT_QUEUE_WEDGED"); + recordProviderFailure(provider, log, targetWithConnection.connectionId, profile, { + isQueueTimeout, + isNetworkError: structuredError?.code === "proxy_unreachable", + }); + } + + const quotaExhausted = await isQuotaExhaustionResponse( + result, + provider, + rawModel, + profile + ); + recordQuotaExhaustionClassification(result, quotaExhausted); + observeFailure(quotaExhausted, target.executionKey); + + // Check if this is a transient error worth retrying on same model. + // A token-limit 429 is terminal for the client — never retry it. + const isTransient = + !isStreamReadinessFailure && + !isTokenLimitBreach && + !scopedFailure && + [408, 429, 500, 502, 503, 504].includes(result.status); + // failoverBeforeRetry means what it says: prefer the next sibling + // target over hammering this one again. Without this check, a + // transient error always re-hit the SAME model up to maxRetries + // times regardless of the setting — config.failoverBeforeRetry was + // threaded through to skipUpstreamRetry (a different, lower-level + // retry mechanism) but never consulted here, so a rate-limited + // model got maxRetries+1 back-to-back attempts on itself before + // this loop's own fallback-to-next-target ever ran (#2417). Only + // skip the same-model retry when `nextTarget` (computed above) + // actually gives us somewhere to fail over to — with no sibling + // left, skipping just burns the last attempt for nothing. + // + // #10217 round-4 fix: this guard reads `failoverBeforeRetryExplicit` + // (opt-in only), NOT `config.failoverBeforeRetry` — that field + // defaults to true for the separate skipUpstreamRetry mechanism + // (see DEFAULT_COMBO_CONFIG comment in comboConfig.ts) and reading + // it here would silently skip the same-model retry for every combo, + // not just ones that explicitly opted in. + if ( + retry < maxRetries && + isTransient && + !providerExhausted && + (!config.failoverBeforeRetryExplicit || !nextTarget) + ) { + if ( + !protectedPriorityTarget && + provider && + rawModel && + isModelLocked(provider, targetWithConnection.connectionId || "", rawModel) + ) { + log.info("COMBO", `Skipping retry for ${modelStr} — model lockout active`); + // Live incident (log id 1784457764961-73): earliestRetryAfter is already + // captured above from THIS dispatch's own response, but lastStatus was + // never recorded on this bail-out path — so once every target in the set + // hit an existing lockout, lastStatus stayed null and the final `if + // (!lastStatus)` check crystallized an immediate ALL_ACCOUNTS_INACTIVE 503 + // instead of ever reaching the `if (earliestRetryAfter)` cooldown-wait + // decision below, even though a real 429 with a short (~1min) retry-after + // was just observed. Recording it here mirrors the "done retrying" path. + lastError = errorText || String(result.status); + lastStatus = result.status; + if (i > 0) fallbackCount++; + return null; + } + // Record model lockout immediately on the first transient failure — + // once the model is cooling down, retrying it would waste an upstream + // call and extend the cooldown via exponential backoff. + let lockoutRecorded = false; + if ( + !protectedPriorityTarget && + provider && + rawModel && + retry === 0 && + !scopedFailure + ) { + const mlSettings = resolveModelLockoutSettings(settings); + if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) { + recordModelLockoutFailure( + provider, + targetWithConnection.connectionId || "", + rawModel, + classifyLockoutReason(result.status), + result.status, + mlSettings.baseCooldownMs, + profile, + { + // #1308/#6863: honor a long upstream reset (e.g. "Resets in 160h") over + // the short base cooldown / exponential backoff when present. #7940's + // maxCooldownMs cap only applies to synthetic values — a verified + // upstream reset (lockoutHintVerified) bypasses it. + exactCooldownMs: selectLockoutCooldownMs(lockoutHintMs, mlSettings), + maxCooldownMs: mlSettings.maxCooldownMs, + // Preserve authoritative structured/header resets; clamp body prose. + exactCooldownIsUpstreamReset: lockoutHintVerified, + } + ); + lockoutRecorded = true; + } + } + if (lockoutRecorded) { + log.info("COMBO", `Skipping retry for ${modelStr} — model lockout active`); + // Same fix as the already-locked branch above — this is the + // first-failure lockout path, so lastStatus needs recording here too. + lastError = errorText || String(result.status); + lastStatus = result.status; + if (i > 0) fallbackCount++; + return null; + } + continue; // Retry same model (transient error, no lockout recorded) + } - // #2101: Prevent infinite fallback loops with 400 Bad Request errors that are genuinely - // body-specific (malformed JSON, bad format, missing required fields). - // These should NOT stop the combo: - // - Context overflow: different models have different context windows - // - Max_tokens / param errors: different models have different output limits - // - Model access denied / "not supported": different providers serve different - // model sets — keep the model in the combo and try the next target (#5249). - // Wrapper words like "invalid" / "bad request" still stop only when the text is - // NOT model-scoped (e.g. "invalid message format"). - if ( - result.status === 400 && - fallbackResult.shouldFallback && - !isContextOverflow400(errorText) && - !isParamValidation400(errorText) && - !isModelScoped400(errorText) && - (errorText.toLowerCase().includes("context") || - errorText.toLowerCase().includes("prompt") || - errorText.toLowerCase().includes("token") || - errorText.toLowerCase().includes("malformed") || - errorText.toLowerCase().includes("invalid") || - errorText.toLowerCase().includes("bad request")) - ) { - log.warn( - "COMBO", - `400 Bad Request with body-specific error detected on ${modelStr} — skipping fallback to other targets to prevent infinite loop` - ); - // Record the failure and break to avoid trying other targets with the same bad request + // Done retrying this model + const protectedTargetTrust = targetFailureTrust.get(target.executionKey); + if ( + protectedPriorityTarget && + (!protectedTargetTrust?.observedFailure || + !protectedTargetTrust.allObservedFailuresQuota) + ) { + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; + return { ok: false, response: result }; + } recordComboRequest(combo.name, modelStr, { success: false, latencyMs: Date.now() - startTime, @@ -2325,6 +2582,12 @@ async function handleComboChatInner({ strategy, target: toRecordedTarget(target), }); + // LKGP (#919) mirror of the success-path set below: a just-failed target + // must not keep re-pinning itself as the "last known good" choice for the + // *next* separate request. Circuit breaker / model lockout deliberately + // don't react to request-scoped failure classes (see scopedFailure below), + // so nothing else clears this stale pin. + clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); recordedAttempts++; lastError = errorText || String(result.status); comboErrors.push({ @@ -2335,131 +2598,9 @@ async function handleComboChatInner({ }); lastStatus = result.status; if (i > 0) fallbackCount++; - log.warn("COMBO", `Model ${modelStr} failed with body-specific error, stopping combo`); - clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); - // #4279: surface the 400 via the {ok,response} contract so the OUTER - // target loop resolves the combo and stops. A bare `break` here only - // exits the inner retry loop; executeTarget then returns null, which - // the outer loop treats as "this target produced nothing" and advances - // to the next model — so the guard failed to stop fallback and a combo - // of N body-rejecting targets tried all N. Mirrors the 499 path above. - return { ok: false, response: result }; - } - - // A model-scoped 400 ("The requested model is not supported" / "not - // available for integrator") is permanent for THIS connection — the - // account/integration will not gain support for the model mid-session. - // Combo still advances to the next target immediately (unchanged, - // preserves #5249's cross-provider fallback), but without a lockout - // here the SAME dead model gets retried on every future, separate - // request forever (observed: every auto-combo request wasted several - // upstream 400s on the same GitHub models, all day). isModelLocked() - // is checked before dispatch (see the pre-check above this loop), so - // this lockout is honored on the next request. - if (result.status === 400 && isModelScoped400(errorText) && provider && rawModel) { - lockModelIfPerModelQuota( - provider, - targetWithConnection.connectionId || "", - rawModel, - "model_capacity", - 60 * 60 * 1000 // 1h - ); - } - - // Trigger shared provider circuit breaker for 5xx errors and connection failures. If the - // next target is on the same provider, don't mark it failed (a different model may still - // succeed) — #8376: EXCEPT a proxy-unreachable failure, which poisons every model alike. - // G-02: when fallbackResult.skipProviderBreaker is set (embedded service supervisor outage - // signalled via X-Omni-Fallback-Hint: connection_cooldown) apply cooldown only — never trip. - const nextTarget = orderedTargets[i + 1]; - const sameProviderNext = - typeof nextTarget?.provider === "string" && nextTarget.provider === provider; - if ( - shouldRecordProviderBreakerFailure({ - isStreamReadinessFailure, - isStreamEarlyEof, - status: result.status, - sameProviderNext, - skipProviderBreaker: fallbackResult.skipProviderBreaker, - requestScopedFailure: scopedFailure, - error: errorText, - isProxyUnreachable: structuredError?.code === "proxy_unreachable", - }) - ) { - const isQueueTimeout = - errorText.includes("RATE_LIMIT_QUEUE_TIMEOUT") || - errorText.includes("RATE_LIMIT_QUEUE_WEDGED"); - recordProviderFailure(provider, log, targetWithConnection.connectionId, profile, { - isQueueTimeout, - isNetworkError: structuredError?.code === "proxy_unreachable", - }); - } - - const quotaExhausted = await isQuotaExhaustionResponse( - result, - provider, - rawModel, - profile - ); - recordQuotaExhaustionClassification(result, quotaExhausted); - observeFailure(quotaExhausted, target.executionKey); - - // Check if this is a transient error worth retrying on same model. - // A token-limit 429 is terminal for the client — never retry it. - const isTransient = - !isStreamReadinessFailure && - !isTokenLimitBreach && - !scopedFailure && - [408, 429, 500, 502, 503, 504].includes(result.status); - // failoverBeforeRetry means what it says: prefer the next sibling - // target over hammering this one again. Without this check, a - // transient error always re-hit the SAME model up to maxRetries - // times regardless of the setting — config.failoverBeforeRetry was - // threaded through to skipUpstreamRetry (a different, lower-level - // retry mechanism) but never consulted here, so a rate-limited - // model got maxRetries+1 back-to-back attempts on itself before - // this loop's own fallback-to-next-target ever ran (#2417). Only - // skip the same-model retry when `nextTarget` (computed above) - // actually gives us somewhere to fail over to — with no sibling - // left, skipping just burns the last attempt for nothing. - // - // #10217 round-4 fix: this guard reads `failoverBeforeRetryExplicit` - // (opt-in only), NOT `config.failoverBeforeRetry` — that field - // defaults to true for the separate skipUpstreamRetry mechanism - // (see DEFAULT_COMBO_CONFIG comment in comboConfig.ts) and reading - // it here would silently skip the same-model retry for every combo, - // not just ones that explicitly opted in. - if ( - retry < maxRetries && - isTransient && - !providerExhausted && - (!config.failoverBeforeRetryExplicit || !nextTarget) - ) { - if ( - !protectedPriorityTarget && - provider && - rawModel && - isModelLocked(provider, targetWithConnection.connectionId || "", rawModel) - ) { - log.info("COMBO", `Skipping retry for ${modelStr} — model lockout active`); - // Live incident (log id 1784457764961-73): earliestRetryAfter is already - // captured above from THIS dispatch's own response, but lastStatus was - // never recorded on this bail-out path — so once every target in the set - // hit an existing lockout, lastStatus stayed null and the final `if - // (!lastStatus)` check crystallized an immediate ALL_ACCOUNTS_INACTIVE 503 - // instead of ever reaching the `if (earliestRetryAfter)` cooldown-wait - // decision below, even though a real 429 with a short (~1min) retry-after - // was just observed. Recording it here mirrors the "done retrying" path. - lastError = errorText || String(result.status); - lastStatus = result.status; - if (i > 0) fallbackCount++; - return null; - } - // Record model lockout immediately on the first transient failure — - // once the model is cooling down, retrying it would waste an upstream - // call and extend the cooldown via exponential backoff. - let lockoutRecorded = false; - if (!protectedPriorityTarget && provider && rawModel && retry === 0 && !scopedFailure) { + // Wire combo failures into the resilience dashboard (model-level lockout) + // alongside the provider-level cooldown below — they govern different scopes. + if (provider && rawModel && !scopedFailure) { const mlSettings = resolveModelLockoutSettings(settings); if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) { recordModelLockoutFailure( @@ -2471,9 +2612,8 @@ async function handleComboChatInner({ mlSettings.baseCooldownMs, profile, { - // #1308/#6863: honor a long upstream reset (e.g. "Resets in 160h") over - // the short base cooldown / exponential backoff when present. #7940's - // maxCooldownMs cap only applies to synthetic values — a verified + // #1308/#6863: honor a long upstream reset over base/exponential cooldown. + // #7940's maxCooldownMs cap only applies to synthetic values — a verified // upstream reset (lockoutHintVerified) bypasses it. exactCooldownMs: selectLockoutCooldownMs(lockoutHintMs, mlSettings), maxCooldownMs: mlSettings.maxCooldownMs, @@ -2481,270 +2621,216 @@ async function handleComboChatInner({ exactCooldownIsUpstreamReset: lockoutHintVerified, } ); - lockoutRecorded = true; } } - if (lockoutRecorded) { - log.info("COMBO", `Skipping retry for ${modelStr} — model lockout active`); - // Same fix as the already-locked branch above — this is the - // first-failure lockout path, so lastStatus needs recording here too. - lastError = errorText || String(result.status); - lastStatus = result.status; - if (i > 0) fallbackCount++; - return null; - } - continue; // Retry same model (transient error, no lockout recorded) - } - - // Done retrying this model - const protectedTargetTrust = targetFailureTrust.get(target.executionKey); - if ( - protectedPriorityTarget && - (!protectedTargetTrust?.observedFailure || - !protectedTargetTrust.allObservedFailuresQuota) - ) { - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy, - target: toRecordedTarget(target), + log.warn("COMBO", `Model ${modelStr} failed, trying next`, { + status: result.status, + errorBody: redactConnectionLabel(errorText), }); - recordedAttempts++; - return { ok: false, response: result }; - } - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy, - target: toRecordedTarget(target), - }); - // LKGP (#919) mirror of the success-path set below: a just-failed target - // must not keep re-pinning itself as the "last known good" choice for the - // *next* separate request. Circuit breaker / model lockout deliberately - // don't react to request-scoped failure classes (see scopedFailure below), - // so nothing else clears this stale pin. - clearStaleLKGP(combo.name, target.executionKey, combo.id, log, "COMBO"); - recordedAttempts++; - lastError = errorText || String(result.status); - comboErrors.push({ - model: modelStr, - status: result.status, - error: errorText || String(result.status), - kind: classifyComboOutcome(result.status, errorText), - }); - lastStatus = result.status; - if (i > 0) fallbackCount++; - // Wire combo failures into the resilience dashboard (model-level lockout) - // alongside the provider-level cooldown below — they govern different scopes. - if (provider && rawModel && !scopedFailure) { - const mlSettings = resolveModelLockoutSettings(settings); - if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) { - recordModelLockoutFailure( + + // #5976: per-model-quota providers (Gemini, GitHub, etc.) multiplex models + // behind one connection. A model-level 500 or 429 (RPM) must NOT cool down + // the entire provider — sibling models may still succeed. Skip cooldown + // recording for these providers on 500/429 errors so the next target can try. + if ( + resilienceSettings.providerCooldown.enabled && + provider && + provider !== "unknown" && + !scopedFailure && + !( + (result.status === 500 || result.status === 429) && + hasPerModelQuota(provider, rawModel) + ) + ) { + recordProviderCooldown( provider, - targetWithConnection.connectionId || "", - rawModel, - classifyLockoutReason(result.status), - result.status, - mlSettings.baseCooldownMs, - profile, - { - // #1308/#6863: honor a long upstream reset over base/exponential cooldown. - // #7940's maxCooldownMs cap only applies to synthetic values — a verified - // upstream reset (lockoutHintVerified) bypasses it. - exactCooldownMs: selectLockoutCooldownMs(lockoutHintMs, mlSettings), - maxCooldownMs: mlSettings.maxCooldownMs, - // Preserve authoritative structured/header resets; clamp body prose. - exactCooldownIsUpstreamReset: lockoutHintVerified, - } + targetWithConnection.connectionId ?? undefined, + resilienceSettings ); } - } - log.warn("COMBO", `Model ${modelStr} failed, trying next`, { - status: result.status, - errorBody: redactConnectionLabel(errorText), - }); - - // #5976: per-model-quota providers (Gemini, GitHub, etc.) multiplex models - // behind one connection. A model-level 500 or 429 (RPM) must NOT cool down - // the entire provider — sibling models may still succeed. Skip cooldown - // recording for these providers on 500/429 errors so the next target can try. - if ( - resilienceSettings.providerCooldown.enabled && - provider && - provider !== "unknown" && - !scopedFailure && - !( - (result.status === 500 || result.status === 429) && - hasPerModelQuota(provider, rawModel) - ) - ) { - recordProviderCooldown( - provider, - targetWithConnection.connectionId ?? undefined, - resilienceSettings - ); - } - const fallbackWaitMs = - fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS - ? Math.min(cooldownMs, fallbackDelayMs) - : 0; - if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) { - log.debug?.("COMBO", `Waiting ${fallbackWaitMs}ms before fallback to next model`); - await new Promise((resolve) => { - const timer = setTimeout(resolve, fallbackWaitMs); - signal?.addEventListener( - "abort", - () => { - clearTimeout(timer); - resolve(undefined); - }, - { once: true } - ); - }); - if (signal?.aborted) { - log.info("COMBO", `Client disconnected during fallback wait — aborting`); - return { ok: false, response: errorResponse(499, "Client disconnected") }; + const fallbackWaitMs = + fallbackDelayMs > 0 && cooldownMs > 0 && cooldownMs <= MAX_FALLBACK_WAIT_MS + ? Math.min(cooldownMs, fallbackDelayMs) + : 0; + if ([502, 503, 504].includes(result.status) && fallbackWaitMs > 0) { + log.debug?.("COMBO", `Waiting ${fallbackWaitMs}ms before fallback to next model`); + await new Promise((resolve) => { + const timer = setTimeout(resolve, fallbackWaitMs); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(undefined); + }, + { once: true } + ); + }); + if (signal?.aborted) { + log.info("COMBO", `Client disconnected during fallback wait — aborting`); + return { ok: false, response: errorResponse(499, "Client disconnected") }; + } } - } + return null; + } return null; - } - return null; - }; + }; - for (let i = 0; i < orderedTargets.length; i++) { - if (anySuccess || comboExpired) break; + for (let i = 0; i < orderedTargets.length; i++) { + if (anySuccess || comboExpired) break; - const abortController = new AbortController(); - abortControllers.set(i, abortController); - const onClientAbort = () => abortController.abort(); - signal?.addEventListener("abort", onClientAbort); + const abortController = new AbortController(); + abortControllers.set(i, abortController); + const onClientAbort = () => abortController.abort(); + signal?.addEventListener("abort", onClientAbort); - const task = (async () => { - try { - const res = await executeTarget(i); - if (res && !anySuccess) { - if (res.ok) { - anySuccess = true; - globalResolve!(res.response!); - for (const [idx, ac] of abortControllers.entries()) { - if (idx !== i) ac.abort(); + const task = (async () => { + try { + const res = await executeTarget(i); + if (res && !anySuccess) { + if (res.ok) { + anySuccess = true; + globalResolve!(res.response!); + for (const [idx, ac] of abortControllers.entries()) { + if (idx !== i) ac.abort(); + } + } else if (res.response) { + // Fatal error, abort combo + anySuccess = true; + globalResolve!(res.response); } - } else if (res.response) { - // Fatal error, abort combo - anySuccess = true; - globalResolve!(res.response); } + } finally { + signal?.removeEventListener("abort", onClientAbort); } - } finally { - signal?.removeEventListener("abort", onClientAbort); - } - })().catch((err) => { - const logError = log.error ?? log.warn; - logError("COMBO", `Speculative task error for target ${i}`, err); - // G2 (silent-stop fix): never leave the speculative loop waiting on an - // unresolved globalPromise. If a task throws unexpectedly (outside - // executeTarget's error handling) and no other task succeeds, the post-loop - // `Promise.race([globalPromise, ...])` would hang forever. Resolve with a - // 502 so the request terminates with an actionable error. - if (!anySuccess && globalResolve) { - anySuccess = true; - globalResolve(errorResponse(502, `Combo target ${i} failed with an unexpected error`)); + })().catch((err) => { + const logError = log.error ?? log.warn; + logError("COMBO", `Speculative task error for target ${i}`, err); + // G2 (silent-stop fix): never leave the speculative loop waiting on an + // unresolved globalPromise. If a task throws unexpectedly (outside + // executeTarget's error handling) and no other task succeeds, the post-loop + // `Promise.race([globalPromise, ...])` would hang forever. Resolve with a + // 502 so the request terminates with an actionable error. + if (!anySuccess && globalResolve) { + anySuccess = true; + globalResolve( + errorResponse(502, `Combo target ${i} failed with an unexpected error`) + ); + } + }); + + runningTasks.add(task); + task.finally(() => runningTasks.delete(task)); + + if ( + zeroLatencyOptimizationsEnabled && + config.hedging && + !hasProtectedPriorityTarget && + i + 1 < orderedTargets.length + ) { + const hedgeDelay = resolveDelayMs(config.hedgeDelayMs, 500); + let timeoutResolve: () => void; + const timeoutPromise = new Promise((r) => { + timeoutResolve = r; + setTimeout(r, hedgeDelay); + }); + await Promise.race([task, globalPromise, timeoutPromise, loopSafetyPromise]); + } else { + await Promise.race([task, globalPromise, loopSafetyPromise]); } - }); + markLoopExpiredIfSafetyFired(); - runningTasks.add(task); - task.finally(() => runningTasks.delete(task)); - - if ( - zeroLatencyOptimizationsEnabled && - config.hedging && - !hasProtectedPriorityTarget && - i + 1 < orderedTargets.length - ) { - const hedgeDelay = resolveDelayMs(config.hedgeDelayMs, 500); - let timeoutResolve: () => void; - const timeoutPromise = new Promise((r) => { - timeoutResolve = r; - setTimeout(r, hedgeDelay); - }); - await Promise.race([task, globalPromise, timeoutPromise, loopSafetyPromise]); - } else { - await Promise.race([task, globalPromise, loopSafetyPromise]); + // Global combo timeout check: after each target completes, stop trying + // further targets if the total elapsed time exceeds comboTimeoutMs. + if (!anySuccess && comboTimeoutMs > 0 && Date.now() - comboStartTime >= comboTimeoutMs) { + comboExpired = true; + log.info( + "COMBO", + `Combo global timeout (${comboTimeoutMs}ms) reached after ` + + `${i + 1}/${orderedTargets.length} targets (${recordedAttempts} attempted) — stopping` + ); + } } - markLoopExpiredIfSafetyFired(); - // Global combo timeout check: after each target completes, stop trying - // further targets if the total elapsed time exceeds comboTimeoutMs. - if (!anySuccess && comboTimeoutMs > 0 && Date.now() - comboStartTime >= comboTimeoutMs) { - comboExpired = true; - log.info( - "COMBO", - `Combo global timeout (${comboTimeoutMs}ms) reached after ` + - `${i + 1}/${orderedTargets.length} targets (${recordedAttempts} attempted) — stopping` - ); + if (!anySuccess && runningTasks.size > 0) { + // G1: include loopSafetyPromise so a hung last task (per-model timeout + // disabled) cannot freeze this post-loop race forever. + await Promise.race([globalPromise, Promise.all([...runningTasks]), loopSafetyPromise]); + markLoopExpiredIfSafetyFired(); } - } - if (!anySuccess && runningTasks.size > 0) { - // G1: include loopSafetyPromise so a hung last task (per-model timeout - // disabled) cannot freeze this post-loop race forever. - await Promise.race([globalPromise, Promise.all([...runningTasks]), loopSafetyPromise]); - markLoopExpiredIfSafetyFired(); - } + // G1: if the safety timer won the race (request would otherwise hang), give + // in-flight tasks a short drain window to land their per-model errors into + // comboErrors so the 504 carries the same "tried: a (500)" summary the + // regular comboExpired branch produces — then return the safety 504. + if (loopSafetyFired && !anySuccess) { + if (runningTasks.size > 0) { + await Promise.race([ + Promise.allSettled([...runningTasks]), + new Promise((resolve) => setTimeout(resolve, COMBO_SAFETY_DRAIN_MS)), + ]); + } + const summary = comboErrors + .slice(0, 5) + .map((e) => `${e.model} (${e.status})`) + .join(", "); + const msg = + `Combo global timeout (${loopSafetyMs}ms) after ${recordedAttempts}/${orderedTargets.length} targets` + + (comboErrors.length > 0 + ? ` | tried: ${summary}${comboErrors.length > 5 ? `... (+${comboErrors.length - 5})` : ""}` + : "") + + " without a terminal response"; + return errorResponseWithComboDiagnostics(504, msg, buildComboDiag("combo_timeout"), { + code: "COMBO_TIMEOUT", + type: "server_error", + }); + } - // G1: if the safety timer won the race (request would otherwise hang), give - // in-flight tasks a short drain window to land their per-model errors into - // comboErrors so the 504 carries the same "tried: a (500)" summary the - // regular comboExpired branch produces — then return the safety 504. - if (loopSafetyFired && !anySuccess) { - if (runningTasks.size > 0) { - await Promise.race([ - Promise.allSettled([...runningTasks]), - new Promise((resolve) => setTimeout(resolve, COMBO_SAFETY_DRAIN_MS)), - ]); + // #10681: finalize the decision trace (success). + finalizeComboTrace(traceInvocationId, orderedTargets); + finishComboTrace(traceInvocationId, { status: 200 }); + if (anySuccess) { + // G1: clear the safety timer on the happy path so a successful combo does + // not leave a 10-minute timer alive per request. + if (loopSafetyTimer) { + clearTimeout(loopSafetyTimer); + loopSafetyTimer = null; + } + return await globalPromise; } - const summary = comboErrors - .slice(0, 5) - .map((e) => `${e.model} (${e.status})`) - .join(", "); - const msg = - `Combo global timeout (${loopSafetyMs}ms) after ${recordedAttempts}/${orderedTargets.length} targets` + - (comboErrors.length > 0 - ? ` | tried: ${summary}${comboErrors.length > 5 ? `... (+${comboErrors.length - 5})` : ""}` - : "") + - " without a terminal response"; - return errorResponseWithComboDiagnostics(504, msg, buildComboDiag("combo_timeout"), { - code: "COMBO_TIMEOUT", - type: "server_error", - }); - } - // #10681: finalize the decision trace (success). - finalizeComboTrace(traceInvocationId, orderedTargets); - finishComboTrace(traceInvocationId, { status: 200 }); - if (anySuccess) { - // G1: clear the safety timer on the happy path so a successful combo does - // not leave a 10-minute timer alive per request. - if (loopSafetyTimer) { - clearTimeout(loopSafetyTimer); - loopSafetyTimer = null; + // #10681: finalize the decision trace (global timeout). + finalizeComboTrace(traceInvocationId, orderedTargets); + finishComboTrace(traceInvocationId, { status: 504 }); + // Global combo timeout: return aggregated error immediately, skipping set retries. + if (comboExpired) { + const summary = buildRedactedSummary(comboErrors); + const msg = + `Combo global timeout (${comboTimeoutMs}ms) after ${recordedAttempts}/${orderedTargets.length} targets` + + (comboErrors.length > 0 ? ` | tried: ${summary}` : ""); + const latencyMs = Date.now() - startTime; + if (recordedAttempts === 0) { + recordComboRequest(combo.name, null, { + success: false, + latencyMs, + fallbackCount, + strategy, + }); + } + notifyWebhookEvent("request.failed", { + combo: combo.name, + reason: "COMBO_TIMEOUT", + latencyMs, + fallbackCount, + }); + return errorResponseWithComboDiagnostics(504, msg, buildComboDiag("combo_timeout"), { + code: "COMBO_TIMEOUT", + type: "server_error", + }); } - return await globalPromise; - } - // #10681: finalize the decision trace (global timeout). - finalizeComboTrace(traceInvocationId, orderedTargets); - finishComboTrace(traceInvocationId, { status: 504 }); - // Global combo timeout: return aggregated error immediately, skipping set retries. - if (comboExpired) { - const summary = buildRedactedSummary(comboErrors); - const msg = - `Combo global timeout (${comboTimeoutMs}ms) after ${recordedAttempts}/${orderedTargets.length} targets` + - (comboErrors.length > 0 ? ` | tried: ${summary}` : ""); + // All models failed in this set try const latencyMs = Date.now() - startTime; if (recordedAttempts === 0) { recordComboRequest(combo.name, null, { @@ -2754,216 +2840,194 @@ async function handleComboChatInner({ strategy, }); } - notifyWebhookEvent("request.failed", { - combo: combo.name, - reason: "COMBO_TIMEOUT", - latencyMs, - fallbackCount, - }); - return errorResponseWithComboDiagnostics(504, msg, buildComboDiag("combo_timeout"), { - code: "COMBO_TIMEOUT", - type: "server_error", - }); - } - - // All models failed in this set try - const latencyMs = Date.now() - startTime; - if (recordedAttempts === 0) { - recordComboRequest(combo.name, null, { - success: false, - latencyMs, - fallbackCount, - strategy, - }); - } - // Retry the entire set if more attempts remain - if (setTry < maxSetRetries) continue; + // Retry the entire set if more attempts remain + if (setTry < maxSetRetries) continue; - if (!lastStatus && recordedAttempts === 0 && comboCooldownWaitEnabled) { - const circuitOpenWait = resolveCircuitOpenWaitDecision({ - skippedForCircuitOpen, - retryAfterMs: earliestCircuitOpenRetryMs, - attempt: comboCooldownAttempt, - budgetLeftMs: comboCooldownBudgetLeftMs, - settings: resilienceSettings.comboCooldownWait, - }); - if (circuitOpenWait.wait) { - log.info( - "COMBO", - `${strategy} circuit-open wait: waiting ${Math.ceil(circuitOpenWait.waitMs / 1000)}s (reason=${circuitOpenWait.reason ?? "circuit_open"}) then retrying (attempt ${comboCooldownAttempt + 1}/${resilienceSettings.comboCooldownWait.maxAttempts})` - ); - const completed = await waitForCooldownAwareRetry(circuitOpenWait.waitMs, signal); - if (!completed) { - return errorResponse(499, "Request aborted"); + if (!lastStatus && recordedAttempts === 0 && comboCooldownWaitEnabled) { + const circuitOpenWait = resolveCircuitOpenWaitDecision({ + skippedForCircuitOpen, + retryAfterMs: earliestCircuitOpenRetryMs, + attempt: comboCooldownAttempt, + budgetLeftMs: comboCooldownBudgetLeftMs, + settings: resilienceSettings.comboCooldownWait, + }); + if (circuitOpenWait.wait) { + log.info( + "COMBO", + `${strategy} circuit-open wait: waiting ${Math.ceil(circuitOpenWait.waitMs / 1000)}s (reason=${circuitOpenWait.reason ?? "circuit_open"}) then retrying (attempt ${comboCooldownAttempt + 1}/${resilienceSettings.comboCooldownWait.maxAttempts})` + ); + const completed = await waitForCooldownAwareRetry(circuitOpenWait.waitMs, signal); + if (!completed) { + return errorResponse(499, "Request aborted"); + } + comboCooldownAttempt += 1; + comboCooldownBudgetLeftMs = Math.max( + 0, + comboCooldownBudgetLeftMs - circuitOpenWait.waitMs + ); + return dispatchWithCooldownRetry(); } - comboCooldownAttempt += 1; - comboCooldownBudgetLeftMs = Math.max( - 0, - comboCooldownBudgetLeftMs - circuitOpenWait.waitMs - ); - return dispatchWithCooldownRetry(); } - } - // All set retries exhausted — return the final error - // #10681: finalize the decision trace (all targets failed or skipped). - finalizeComboTrace(traceInvocationId, orderedTargets); - finishComboTrace(traceInvocationId, { status: 503 }); - if (!lastStatus) { - if (recordedAttempts === 0) { + // All set retries exhausted — return the final error + // #10681: finalize the decision trace (all targets failed or skipped). + finalizeComboTrace(traceInvocationId, orderedTargets); + finishComboTrace(traceInvocationId, { status: 503 }); + if (!lastStatus) { + if (recordedAttempts === 0) { + notifyWebhookEvent("request.failed", { + combo: combo.name, + reason: "ALL_TARGETS_SKIPPED", + latencyMs, + fallbackCount, + }); + return withQuotaExhaustionClassification( + errorResponseWithComboDiagnostics( + 503, + "Service temporarily unavailable: all targets were skipped by pre-dispatch filters", + buildComboDiag("all_targets_skipped"), + { code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" } + ), + observedFailure ? allObservedFailuresQuota : null + ); + } notifyWebhookEvent("request.failed", { combo: combo.name, - reason: "ALL_TARGETS_SKIPPED", + reason: "ALL_ACCOUNTS_INACTIVE", latencyMs, fallbackCount, }); + recordComboFailure(effectiveSessionId, combo.name); + return errorResponseWithComboDiagnostics( + 503, + "Service temporarily unavailable: all upstream accounts are inactive", + buildComboDiag("all_accounts_inactive"), + { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" } + ); + } + + // #10501: derive the terminal HTTP status from the structured per-target + // outcomes instead of `lastStatus` (whichever target happened to fail + // LAST). A 4xx is preserved only when the request itself is genuinely + // invalid across every eligible target; a heterogeneous mix of failure + // classes (e.g. a quality failure + a sibling's 401) normalizes to a + // 5xx-class status reflecting an infra/provider problem, not a client + // error. See comboErrorAggregation.ts::resolveComboTerminalStatus. + const status = resolveComboTerminalStatus(comboErrors, lastStatus); + // #10314: build the terminal message from the structured per-target + // outcomes (each distinct class+reason listed separately) instead of + // mashing a single lastError with raw `[model (status)]` markers. Connection + // identifiers are redacted. Falls back to lastError when no target recorded + // a structured outcome. + const msg = formatComboOutcomes(comboErrors) || lastError || "All combo models unavailable"; + + // Cooldown-aware retry: instead of crystallizing a transient failure, wait + // out a SHORT cooldown and re-run the whole set loop. Guarded by the helper + // (quota_exhausted/auth/not-found excluded, ceiling, attempts, budget). + // MAX_GLOBAL_ATTEMPTS still bounds total dispatches. Available to ALL combo + // strategies when enabled — entry is driven by earliestRetryAfter + the + // real model-lockout reason, NOT by whichever target last overwrote + // `status` (a later 403 must not skip the allow-list check for an earlier + // 429's retry-after hint). SECURITY (see comboCooldownRetry.ts header): the + // allow-list is the PRIMARY barrier and `maxWaitMs` only the SECOND one. + // Hardcoding reason:"rate_limit" would drop the primary barrier and leave + // only the ceiling — which does NOT cover a quota_exhausted lock carrying a + // SHORT upstream retry-after. Model lockouts are recorded for all strategies, + // so the real reason is always available. + if (comboCooldownWaitEnabled && earliestRetryAfter) { + const decision: ResolveComboCooldownDecisionResult = resolveComboCooldownWaitDecision({ + targets: orderedTargets, + earliestRetryAfter, + attempt: comboCooldownAttempt, + budgetLeftMs: comboCooldownBudgetLeftMs, + settings: resilienceSettings.comboCooldownWait, + // Key each lookup on the TARGET's own model: quota-share combos are + // single-model/multi-account (so this is identical to the previous + // orderedTargets[0] behavior), but heterogeneous combos carry a + // different model per target. + lookupLock: (provider, connectionId, target) => { + const rawModel = parseModel(target?.modelStr ?? "").model || ""; + if (!rawModel) return null; + return getModelLockoutInfo(provider, connectionId, rawModel); + }, + computeWaitMs: (retryAfter) => computeClosestRetryAfter(retryAfter).waitMs, + }); + + if (decision.wait) { + log.info( + "COMBO", + `${strategy} cooldown wait: ${msg} — waiting ${Math.ceil( + decision.waitMs / 1000 + )}s (reason=${decision.reason ?? "?"}) then retrying (attempt ${ + comboCooldownAttempt + 1 + }/${resilienceSettings.comboCooldownWait.maxAttempts})` + ); + const completed = await waitForCooldownAwareRetry(decision.waitMs, signal); + if (!completed) { + log.info("COMBO", `${strategy} cooldown wait aborted by client disconnect`); + return errorResponse(499, "Request aborted"); + } + comboCooldownAttempt += 1; + comboCooldownBudgetLeftMs = Math.max(0, comboCooldownBudgetLeftMs - decision.waitMs); + return dispatchWithCooldownRetry(); + } + } + + // #10681: finalize the decision trace with the aggregated terminal status. + finalizeComboTrace(traceInvocationId, orderedTargets); + finishComboTrace(traceInvocationId, { status }); + // Retry-after decoration is separate from the wait decision above: only + // rate-limit-class final statuses may carry a `(reset after ...)` suffix + // (see unavailableRetryGate.ts — do not stitch a peer target's window onto + // a config-class status like 403/422). + if (earliestRetryAfter && isRetryAfterEligibleStatus(status)) { + const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(earliestRetryAfter)); + log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`); return withQuotaExhaustionClassification( - errorResponseWithComboDiagnostics( - 503, - "Service temporarily unavailable: all targets were skipped by pre-dispatch filters", - buildComboDiag("all_targets_skipped"), - { code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" } - ), + unavailableResponse(status, msg, earliestRetryAfter, retryHuman), observedFailure ? allObservedFailuresQuota : null ); } - notifyWebhookEvent("request.failed", { - combo: combo.name, - reason: "ALL_ACCOUNTS_INACTIVE", - latencyMs, - fallbackCount, - }); - recordComboFailure(effectiveSessionId, combo.name); - return errorResponseWithComboDiagnostics( - 503, - "Service temporarily unavailable: all upstream accounts are inactive", - buildComboDiag("all_accounts_inactive"), - { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" } - ); - } - - // #10501: derive the terminal HTTP status from the structured per-target - // outcomes instead of `lastStatus` (whichever target happened to fail - // LAST). A 4xx is preserved only when the request itself is genuinely - // invalid across every eligible target; a heterogeneous mix of failure - // classes (e.g. a quality failure + a sibling's 401) normalizes to a - // 5xx-class status reflecting an infra/provider problem, not a client - // error. See comboErrorAggregation.ts::resolveComboTerminalStatus. - const status = resolveComboTerminalStatus(comboErrors, lastStatus); - // #10314: build the terminal message from the structured per-target - // outcomes (each distinct class+reason listed separately) instead of - // mashing a single lastError with raw `[model (status)]` markers. Connection - // identifiers are redacted. Falls back to lastError when no target recorded - // a structured outcome. - const msg = formatComboOutcomes(comboErrors) || lastError || "All combo models unavailable"; - - // Cooldown-aware retry: instead of crystallizing a transient failure, wait - // out a SHORT cooldown and re-run the whole set loop. Guarded by the helper - // (quota_exhausted/auth/not-found excluded, ceiling, attempts, budget). - // MAX_GLOBAL_ATTEMPTS still bounds total dispatches. Available to ALL combo - // strategies when enabled — entry is driven by earliestRetryAfter + the - // real model-lockout reason, NOT by whichever target last overwrote - // `status` (a later 403 must not skip the allow-list check for an earlier - // 429's retry-after hint). SECURITY (see comboCooldownRetry.ts header): the - // allow-list is the PRIMARY barrier and `maxWaitMs` only the SECOND one. - // Hardcoding reason:"rate_limit" would drop the primary barrier and leave - // only the ceiling — which does NOT cover a quota_exhausted lock carrying a - // SHORT upstream retry-after. Model lockouts are recorded for all strategies, - // so the real reason is always available. - if (comboCooldownWaitEnabled && earliestRetryAfter) { - const decision: ResolveComboCooldownDecisionResult = resolveComboCooldownWaitDecision({ - targets: orderedTargets, - earliestRetryAfter, - attempt: comboCooldownAttempt, - budgetLeftMs: comboCooldownBudgetLeftMs, - settings: resilienceSettings.comboCooldownWait, - // Key each lookup on the TARGET's own model: quota-share combos are - // single-model/multi-account (so this is identical to the previous - // orderedTargets[0] behavior), but heterogeneous combos carry a - // different model per target. - lookupLock: (provider, connectionId, target) => { - const rawModel = parseModel(target?.modelStr ?? "").model || ""; - if (!rawModel) return null; - return getModelLockoutInfo(provider, connectionId, rawModel); - }, - computeWaitMs: (retryAfter) => computeClosestRetryAfter(retryAfter).waitMs, - }); - if (decision.wait) { + // Silent-stop fix: bump the failure counter (pin clears on 3rd consecutive) and emit + // `try-auto` recovery action via buildRecoveryHint so the OC plugin can show "→ Try + // model: auto" instead of an opaque 5xx. We pass the upstream retry-after seconds to + // the hint so the client can render a precise "wait Ns and retry" message. + log.warn("COMBO", `All models failed | ${msg}`); + const { pinClearedNow } = recordComboFailure(effectiveSessionId, combo.name); + if (pinClearedNow) { log.info( "COMBO", - `${strategy} cooldown wait: ${msg} — waiting ${Math.ceil( - decision.waitMs / 1000 - )}s (reason=${decision.reason ?? "?"}) then retrying (attempt ${ - comboCooldownAttempt + 1 - }/${resilienceSettings.comboCooldownWait.maxAttempts})` + `Auto-cleared session_model_history pin for combo "${combo.name}" after ${COMBO_FAILURE_THRESHOLD} consecutive failures to break the silent-stop loop` ); - const completed = await waitForCooldownAwareRetry(decision.waitMs, signal); - if (!completed) { - log.info("COMBO", `${strategy} cooldown wait aborted by client disconnect`); - return errorResponse(499, "Request aborted"); - } - comboCooldownAttempt += 1; - comboCooldownBudgetLeftMs = Math.max(0, comboCooldownBudgetLeftMs - decision.waitMs); - return dispatchWithCooldownRetry(); } - } - - // #10681: finalize the decision trace with the aggregated terminal status. - finalizeComboTrace(traceInvocationId, orderedTargets); - finishComboTrace(traceInvocationId, { status }); - // Retry-after decoration is separate from the wait decision above: only - // rate-limit-class final statuses may carry a `(reset after ...)` suffix - // (see unavailableRetryGate.ts — do not stitch a peer target's window onto - // a config-class status like 403/422). - if (earliestRetryAfter && isRetryAfterEligibleStatus(status)) { - const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(earliestRetryAfter)); - log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`); + const retryAfterSeconds = undefined; + // #10966: when every observed failure was independently classified as quota/ + // balance exhaustion (isQuotaExhaustionResponse, tracked via observeFailure's + // allObservedFailuresQuota accumulator), stamp a stable `quota_exhausted` + // terminalReason instead of forwarding the raw upstream error string. The raw + // string falls through buildRecoveryHint's default branch ("retry" / "failed + // transiently"), which is actively misleading for a durable wallet/quota + // exhaustion — retrying the same combo will never refill it. + const terminalReason = + observedFailure && allObservedFailuresQuota + ? "quota_exhausted" + : (lastError ?? "all_models_failed"); return withQuotaExhaustionClassification( - unavailableResponse(status, msg, earliestRetryAfter, retryHuman), + errorResponseWithComboDiagnostics( + status, + msg, + buildComboDiag(terminalReason, retryAfterSeconds) + ), observedFailure ? allObservedFailuresQuota : null ); } - // Silent-stop fix: bump the failure counter (pin clears on 3rd consecutive) and emit - // `try-auto` recovery action via buildRecoveryHint so the OC plugin can show "→ Try - // model: auto" instead of an opaque 5xx. We pass the upstream retry-after seconds to - // the hint so the client can render a precise "wait Ns and retry" message. - log.warn("COMBO", `All models failed | ${msg}`); - const { pinClearedNow } = recordComboFailure(effectiveSessionId, combo.name); - if (pinClearedNow) { - log.info( - "COMBO", - `Auto-cleared session_model_history pin for combo "${combo.name}" after ${COMBO_FAILURE_THRESHOLD} consecutive failures to break the silent-stop loop` - ); - } - const retryAfterSeconds = undefined; - // #10966: when every observed failure was independently classified as quota/ - // balance exhaustion (isQuotaExhaustionResponse, tracked via observeFailure's - // allObservedFailuresQuota accumulator), stamp a stable `quota_exhausted` - // terminalReason instead of forwarding the raw upstream error string. The raw - // string falls through buildRecoveryHint's default branch ("retry" / "failed - // transiently"), which is actively misleading for a durable wallet/quota - // exhaustion — retrying the same combo will never refill it. - const terminalReason = - observedFailure && allObservedFailuresQuota - ? "quota_exhausted" - : (lastError ?? "all_models_failed"); - return withQuotaExhaustionClassification( - errorResponseWithComboDiagnostics( - status, - msg, - buildComboDiag(terminalReason, retryAfterSeconds) - ), - observedFailure ? allObservedFailuresQuota : null - ); - } - - // Final fallback — when the dispatch returned without crystallizing a status (rare). - // Surface the recovery hint with a generic retry recommendation so the client at least - // gets a non-opaque message instead of "Combo routing completed without an upstream response". - recordComboFailure(effectiveSessionId, combo.name); + // Final fallback — when the dispatch returned without crystallizing a status (rare). + // Surface the recovery hint with a generic retry recommendation so the client at least + // gets a non-opaque message instead of "Combo routing completed without an upstream response". + recordComboFailure(effectiveSessionId, combo.name); return errorResponseWithComboDiagnostics( 503, "Combo routing completed without an upstream response", @@ -4075,5 +4139,8 @@ async function handleRoundRobinCombo({ } log.warn("COMBO-RR", `All models failed | ${msg}`); - return new Response(JSON.stringify({ error: { message: msg } }), { status, headers: { "Content-Type": "application/json" } }); + return new Response(JSON.stringify({ error: { message: msg } }), { + status, + headers: { "Content-Type": "application/json" }, + }); } diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts index 9065e123b2bb..4b0cfee5d9f8 100644 --- a/open-sse/services/combo/comboStructure.ts +++ b/open-sse/services/combo/comboStructure.ts @@ -79,7 +79,11 @@ function toComboLike(combo: ComboInput): ComboLike { }; } -function getCombosArray(allCombos: ComboCollectionLike): ComboLike[] { +/** Normalize the two ComboCollectionLike shapes (bare array or {combos}) into + * a flat, type-normalized ComboLike[]. Exported for explicitTargetResolver.ts + * (O360 explicit-target routing), which needs the same normalization to scan + * every combo's name/labels — reused rather than reimplemented. */ +export function getCombosArray(allCombos: ComboCollectionLike): ComboLike[] { const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; return combos.map((combo) => toComboLike(combo)); } diff --git a/open-sse/services/combo/decisionTrace.ts b/open-sse/services/combo/decisionTrace.ts index 7660af7ea01f..6ef3d02edd57 100644 --- a/open-sse/services/combo/decisionTrace.ts +++ b/open-sse/services/combo/decisionTrace.ts @@ -33,7 +33,35 @@ export const COMBO_SKIP_REASONS = [ export type ComboSkipReason = (typeof COMBO_SKIP_REASONS)[number]; -export type ComboDecision = "dispatched" | "skipped_before_dispatch" | "not_reached"; +/** + * The explicit_*, weak_*, and missing_task values are O360 explicit-target- + * routing V1 additions: the prelude branch that resolves either a deterministic + * "/route " command (routeSlashCommand.ts) or a natural-language + * "use " command (explicitTargetDetector.ts) records one of these + * instead of "dispatched"/"skipped_before_dispatch"/"not_reached", which + * describe the normal per-target attempt loop and don't fit an + * override-or-fail-closed decision made before that loop even starts. + * + * "weak_unresolved_fallthrough" is natural-language-only: an unresolved + * candidate there ALWAYS falls through to ordinary chat instead of failing + * closed (natural language is deliberately conservative — only "/route" is + * the deterministic, always-fail-closed-on-unknown control API). See each + * detector module's own doc comment for the full rationale. + * + * "missing_task" is "/route"-only: the command was the entire content of + * the user's turn, with no task text before or after it to forward to the + * resolved target. + */ +export type ComboDecision = + | "dispatched" + | "skipped_before_dispatch" + | "not_reached" + | "explicit_override" + | "unknown_target" + | "ambiguous_target" + | "target_unavailable" + | "weak_unresolved_fallthrough" + | "missing_task"; export interface ComboTraceEntry { /** Safe internal identifier of the combo step (execution key). */ diff --git a/open-sse/services/combo/dispatchPrelude.ts b/open-sse/services/combo/dispatchPrelude.ts index b39f592b51a1..f022fcf2f733 100644 --- a/open-sse/services/combo/dispatchPrelude.ts +++ b/open-sse/services/combo/dispatchPrelude.ts @@ -71,8 +71,12 @@ type RunCombo = (options: HandleComboChatOptions) => Promise; /** * The subset of handleComboChat's own arguments that a recursing branch has to * hand back to it when it dispatches a nested combo-ref. + * + * Exported for tryExplicitTargetDispatch.ts (O360 explicit-target routing), + * which needs the exact same option-bag rebuild for its combo-alias dispatch + * path — reused rather than duplicated. */ -type PreludeBaseOptionArgs = { +export type PreludeBaseOptionArgs = { invocationId?: string; body: Record; combo: ComboLike; @@ -99,7 +103,7 @@ type PreludeBaseOptionArgs = { }; /** Rebuild handleComboChat's option bag verbatim for a recursive dispatch. */ -function buildBaseOptions(a: PreludeBaseOptionArgs): HandleComboChatOptions { +export function buildBaseOptions(a: PreludeBaseOptionArgs): HandleComboChatOptions { return { body: a.body, combo: a.combo, @@ -169,8 +173,14 @@ export function pinIsDurablyUnhealthy( * Async wrapper: resolve the pinned model's provider, read its circuit state and * active connections, and decide via {@link pinIsDurablyUnhealthy}. Fail-open * (return false) on any error so a lookup bug never drops a healthy pin. + * + * Exported for tryExplicitTargetDispatch.ts (O360 explicit-target routing), + * which needs the identical health gate for a resolved explicit target — with + * the opposite failure policy at the call site: an explicit target that is + * durably unhealthy must fail closed (TARGET_UNAVAILABLE), not silently fall + * through to strategy the way a stale context-cache pin does. */ -async function isPinnedModelDurablyUnhealthy(pinnedModel: string): Promise { +export async function isPinnedModelDurablyUnhealthy(pinnedModel: string): Promise { try { const provider = parseModel(pinnedModel).provider; if (!provider) return false; diff --git a/open-sse/services/combo/explicitTargetDetector.ts b/open-sse/services/combo/explicitTargetDetector.ts new file mode 100644 index 000000000000..6757d63dca5e --- /dev/null +++ b/open-sse/services/combo/explicitTargetDetector.ts @@ -0,0 +1,213 @@ +/** + * Explicit target detector — natural-language "use " style commands in + * the caller's own last turn (O360 explicit-target-routing V1). + * + * This is the CONVENIENCE path, not the deterministic control API — that is + * routeSlashCommand.ts's "/route " (checked first, higher priority, + * by tryExplicitTargetDispatch.ts). Natural language is intentionally + * conservative: it never fails a request closed on its own account. An + * unresolved natural-language candidate simply falls through to ordinary + * chat (see tryExplicitTargetDispatch.ts) — only an unresolved "/route" + * command, or a candidate that resolves but is ambiguous/unavailable, fails + * closed. This keeps the regex here small on purpose: it only has to decide + * WHETHER the last user-role message asks to route to a named target, not + * how confidently, and it never resolves the alias to a real model/combo + * (see explicitTargetResolver.ts). + * + * Scope is intentionally narrow beyond that too: this module never looks at + * anything but the last role="user" message — never system messages, + * assistant history, tool results, or earlier user turns. Control text from + * anywhere else in the conversation is not something the caller typed to + * OmniRoute just now, and is not a safe place to look for a routing command. + */ + +/** One content part in an OpenAI/Anthropic-shaped message.content array. */ +type MessageContentPart = { type?: unknown; text?: unknown } | string; + +type MessageLike = { + role?: unknown; + content?: string | MessageContentPart[] | unknown; +}; + +/** + * Flatten an OpenAI/Anthropic-shaped content-parts array to plain text — only + * "text"/"input_text" parts contribute — image and tool-shaped parts are + * ignored, mirroring the part-type filter already used by + * extractMessageContents() in src/shared/utils/inputSanitizer.ts. Split out of + * extractLastUserMessageText() purely to keep that function's own branching + * shallow; no behavior change. + */ +function extractTextFromContentParts(content: MessageContentPart[]): string | null { + const parts: string[] = []; + for (const part of content) { + if (typeof part === "string") { + parts.push(part); + continue; + } + if (!part || typeof part !== "object") continue; + const p = part as { type?: unknown; text?: unknown }; + if ((p.type === "text" || p.type === "input_text") && typeof p.text === "string") { + parts.push(p.text); + } + } + const joined = parts.join("\n").trim(); + return joined.length > 0 ? joined : null; +} + +/** + * Extract the plain text of the LAST role="user" message in the request body, + * handling both a plain string `content` and an OpenAI/Anthropic-shaped + * content-parts array (only "text"/"input_text" parts contribute — image and + * tool-shaped parts are ignored, mirroring the part-type filter already used + * by extractMessageContents() in src/shared/utils/inputSanitizer.ts). + * + * Returns null when there is no user message or its content is empty/unusable. + * + * Shared with routeSlashCommand.ts — the deterministic "/route" path scans + * the same last-user-turn text, just with different (simpler) matching + * rules. + */ +export function extractLastUserMessageText(body: Record): string | null { + const raw = body?.messages; + const messages = Array.isArray(raw) ? raw : []; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i] as MessageLike; + if (!msg || typeof msg !== "object" || msg.role !== "user") continue; + const { content } = msg; + if (typeof content === "string") { + return content.length > 0 ? content : null; + } + if (Array.isArray(content)) { + return extractTextFromContentParts(content); + } + return null; + } + return null; +} + +// Reuse the same literal markers flattenToolHistory.ts already uses to embed +// tool calls/results into plain text for providers with no native tool +// messages — recognizing them here means a message that went through that +// flattening doesn't get re-scanned as if it were the user's own words. +const TOOL_CALL_PREFIX = "[Called tools: "; +const TOOL_RESULT_PREFIX = "[Tool result: "; + +const FENCED_CODE_BLOCK = /```[\s\S]*?```/g; +const INLINE_CODE_SPAN = /`[^`\n]*`/g; + +/** + * Strip fenced code blocks, inline code spans, and recognizable flattened + * tool-call/tool-result segments from a block of text, leaving only the + * natural-language remainder to run explicit-command patterns against. + * + * Deliberately does not attempt to recognize arbitrary/unfenced tool payloads + * pasted as plain prose — that is not a solvable text-shape problem without a + * real parser, and is out of scope for V1 (see the patch's OPEN_ISSUES). + */ +export function stripControlNoise(text: string): string { + let out = text.replace(FENCED_CODE_BLOCK, " ").replace(INLINE_CODE_SPAN, " "); + out = out + .split("\n") + .filter((line) => { + const trimmed = line.trimStart(); + return !trimmed.startsWith(TOOL_CALL_PREFIX) && !trimmed.startsWith(TOOL_RESULT_PREFIX); + }) + .join("\n"); + return out; +} + +export type ExplicitTargetMatch = { alias: string }; + +/** + * Command verbs that introduce an explicit-target phrase. "usar" (infinitive) + * is deliberately EXCLUDED: in Portuguese it is the form used in + * reported/descriptive speech ("para usar Astra", "recomendo usar o Claude") + * — not a direct command. The imperative forms "use"/"usa" (2nd person) are + * what a real command uses ("use Astra", "usa o Claude"), so only those are + * treated as command verbs. This is the concrete fix for the required + * negative case: "documentação diz para usar Astra" must never match. + * + * First-letter character classes (not the `i` flag) cover sentence-initial + * capitalization ("Use Astra") — see CANDIDATE below for why `i` is avoided + * entirely in this file. + */ +const COMMAND_VERBS = "(?:[Uu]se|[Uu]sa|[Pp]ede|[Pp]eça|[Pp]ergunta|[Pp]ergunte|[Mm]anda|[Mm]ande)"; + +/** + * Optional Portuguese connector between the verb and the candidate ("pede + * PRO Astra", "manda O Claude"). + */ +const CONNECTOR = "(?:pro|pra|para|ao|à|o|a)"; + +/** + * The candidate token itself. Aliases, combo names, and literal model ids + * are NOT proper nouns as a matter of system design — "Astra"/"Claude" are + * just examples, real combo names are routinely lowercase-hyphenated + * ("o360-cheap-worker") and literal model ids carry `/` and `.` + * ("cx/gpt-5.6-sol-high", per explicitTargetResolver.ts's own tier-4 + * literal-model-id lookup). Capitalization must never gate whether a + * candidate is even extracted — only whether it turns out to resolve to + * something real (see explicitTargetResolver.ts) decides that. No `i` flag + * is used anywhere in this file's patterns: combined with `u`, `i` case-folds + * Unicode property escapes like `\p{Lu}` in V8/JS (confirmed empirically, + * moot here since this class no longer needs `\p{Lu}`, but avoided + * throughout so that trap is never reintroduced later). + */ +const CANDIDATE = "([\\p{L}][\\p{L}0-9_\\-./]{1,79})"; + +const VERB_FIRST_PATTERN = new RegExp( + `\\b${COMMAND_VERBS}\\b(?:\\s+${CONNECTOR}\\b)?\\s+${CANDIDATE}`, + "u" +); + +/** + * "com o/a " alone is NOT enough signal: it matches plain descriptive/ + * reported-speech sentences just as easily as a command — + * "ontem falei com o Astra sobre isso", "já conversei com o Astra", + * "isso foi revisado com o Astra" all contain the exact same "com o " + * shape with zero routing intent. The one required-positive example, + * "com o Astra, procure falhas", has a real structural signal the negatives + * don't: an imperative task verb immediately follows the alias (across an + * optional comma). Requiring that verb (via a lookahead, so it isn't + * consumed/captured) is what separates a real command from a mention. + */ +const TASK_IMPERATIVE_VERBS = + "procure|verifique|analise|revise|critique|corrija|avalie|teste|confirme|valide|" + + "aponte|liste|resuma|explique|mostre|diga|ache|encontre|chame|pergunte|pe[çc]a|" + + "execute|rode|gere|crie|escreva|refatore|otimize"; + +const COM_O_PATTERN = new RegExp( + `\\b[Cc]om\\s+(?:o|a)\\s+${CANDIDATE}\\s*,?\\s*(?=\\b(?:${TASK_IMPERATIVE_VERBS})\\b)`, + "u" +); + +/** + * Detect an explicit-target command in already-sanitized natural-language + * text (see stripControlNoise). Returns the raw alias token as typed, or + * null when no command pattern matches — a bare mention of an alias name + * with no leading command verb ("Astra", "a documentação fala do Astra") + * never matches, by construction. + */ +export function detectExplicitTargetPhrase(sanitizedText: string): ExplicitTargetMatch | null { + const verbMatch = VERB_FIRST_PATTERN.exec(sanitizedText); + if (verbMatch?.[1]) return { alias: verbMatch[1] }; + + const comMatch = COM_O_PATTERN.exec(sanitizedText); + if (comMatch?.[1]) return { alias: comMatch[1] }; + + return null; +} + +/** + * Full pipeline: last user message → strip code/tool noise → detect command. + * tryExplicitTargetDispatch.ts calls this only after routeSlashCommand.ts's + * deterministic "/route" check has already come up empty. + */ +export function detectExplicitTargetInBody( + body: Record +): ExplicitTargetMatch | null { + const text = extractLastUserMessageText(body); + if (!text) return null; + const sanitized = stripControlNoise(text); + return detectExplicitTargetPhrase(sanitized); +} diff --git a/open-sse/services/combo/explicitTargetResolver.ts b/open-sse/services/combo/explicitTargetResolver.ts new file mode 100644 index 000000000000..614930728b2d --- /dev/null +++ b/open-sse/services/combo/explicitTargetResolver.ts @@ -0,0 +1,204 @@ +/** + * Explicit target alias resolution — O360 explicit-target-routing V1. + * + * Resolves a human-typed alias ("Astra", "cheap-worker", ...) to a LOGICAL + * destination — a combo name or a bare model/provider string — never directly + * to a connection_id. The destination is handed back to OmniRoute's normal + * dispatch (handleSingleModelWithTimeout for a model, handleComboChat for a + * combo), which keeps choosing a healthy connection/account exactly as it + * would for any other request. A future "pin one specific account" feature is + * out of scope for V1 and would be a connection_id-based extension of this + * same resolver, not a change to its precedence rules. + * + * Precedence (first tier with any match wins; a tier with more than one match + * is ambiguous and fails closed — never first-match/last-match/silent order): + * 1. explicit synonym (operator setting first, then built-in O360 friendly aliases) + * 2. exact combo name + * 3. exact ResolvedComboTarget/ResolvedComboRefTarget label (combo config) + * 4. exact provider/model id (validated via the caller's isModelAvailable) + * 5. UNKNOWN_TARGET + * + * Tiers 1-3 match the human-typed alias case-insensitively (that's the point + * of a human alias). Tier 4 is a literal model-id lookup and is intentionally + * case-sensitive, since model ids are case-sensitive tokens elsewhere in this + * codebase (e.g. "cx/gpt-5.6-sol-high") — a human is not expected to reach + * tier 4 by typing a bare alias; it exists for a power-user request that + * already names a real model id. + */ +import { getSettings } from "../../../src/lib/db/settings.ts"; +import { clampComboDepth } from "./comboPredicates.ts"; +import { getCombosArray, resolveComboTargets } from "./comboStructure.ts"; +import type { + ComboCollectionLike, + ComboLike, + HiddenModelsByProvider, + IsModelAvailable, +} from "./types.ts"; + +export type ExplicitTargetDestination = + { kind: "model"; modelStr: string } | { kind: "combo"; comboName: string }; + +export type ExplicitTargetResolution = + | ({ + status: "resolved"; + matchedVia: "synonym" | "comboName" | "label" | "modelId"; + } & ExplicitTargetDestination) + | { status: "unknown" } + | { status: "ambiguous"; tier: "synonym" | "comboName" | "label"; candidates: string[] }; + +/** Stable O360 V1 aliases. They keep the approved command surface working on + * a fresh deployment without requiring a settings write or schema change. + * An operator-provided synonym with the same key still wins. */ +const DEFAULT_EXPLICIT_TARGET_ALIASES: Readonly> = Object.freeze({ + astra: "cx/gpt-5.6-sol-high", + claude: "cc/claude-sonnet-5", +}); + +/** "combo/" ⇄ bare model string — the same convention already used for + * a combo request's own `model` field elsewhere in OmniRoute. Reusing it here + * means the synonym map and label lookups need no new {kind, value} shape. */ +function parseDestinationString(value: string): ExplicitTargetDestination | null { + const trimmed = typeof value === "string" ? value.trim() : ""; + if (!trimmed) return null; + if (trimmed.startsWith("combo/")) { + const comboName = trimmed.slice("combo/".length).trim(); + return comboName ? { kind: "combo", comboName } : null; + } + return { kind: "model", modelStr: trimmed }; +} + +function destinationKey(d: ExplicitTargetDestination): string { + return d.kind === "combo" ? `combo/${d.comboName}` : d.modelStr; +} + +/** Dedupe candidates that resolve to the exact same destination before an + * ambiguity check, so two routes to the SAME place are never flagged as + * ambiguous — only genuinely different destinations are. */ +function dedupeDestinations(items: ExplicitTargetDestination[]): ExplicitTargetDestination[] { + const seen = new Map(); + for (const item of items) { + seen.set(destinationKey(item), item); + } + return [...seen.values()]; +} + +function resolveConfiguredSynonyms(alias: string, map: unknown): ExplicitTargetDestination[] { + if (!map || typeof map !== "object") return []; + const lowerAlias = alias.toLowerCase(); + const matches: ExplicitTargetDestination[] = []; + for (const [key, value] of Object.entries(map as Record)) { + if (typeof key !== "string" || key.toLowerCase() !== lowerAlias) continue; + if (typeof value !== "string") continue; + const dest = parseDestinationString(value); + if (dest) matches.push(dest); + } + return dedupeDestinations(matches); +} + +async function resolveSynonymTier(alias: string): Promise { + const settings = await getSettings(); + const configuredMatches = resolveConfiguredSynonyms(alias, settings?.explicitTargetAliases); + if (configuredMatches.length > 0) return configuredMatches; + + const defaultDestination = DEFAULT_EXPLICIT_TARGET_ALIASES[alias.toLowerCase()]; + const parsedDefault = defaultDestination ? parseDestinationString(defaultDestination) : null; + return parsedDefault ? [parsedDefault] : []; +} + +function resolveComboNameTier( + alias: string, + allCombos: ComboCollectionLike +): ExplicitTargetDestination[] { + const lowerAlias = alias.toLowerCase(); + const combos = getCombosArray(allCombos); + const matches = combos.filter((c) => c.name.toLowerCase() === lowerAlias); + return dedupeDestinations(matches.map((c) => ({ kind: "combo" as const, comboName: c.name }))); +} + +/** + * resolveComboTargets() flattens combo-ref steps into the referenced combo's + * own model targets (see resolveNestedComboTargets) — it never returns a + * ResolvedComboRefTarget, so every element here is a real model target with + * its own label. A label set on the combo-ref STEP itself (rather than on a + * model target inside the referenced combo) is not reachable through this + * tier as a result — see OPEN_ISSUES in the patch notes. That is the one + * known gap in an otherwise complete reuse of the existing resolution code. + */ +function resolveLabelTier( + alias: string, + allCombos: ComboCollectionLike, + maxComboDepth: number, + hiddenModelsByProvider: HiddenModelsByProvider | undefined +): ExplicitTargetDestination[] { + const lowerAlias = alias.toLowerCase(); + const combos = getCombosArray(allCombos); + const matches: ExplicitTargetDestination[] = []; + for (const combo of combos) { + const targets = resolveComboTargets(combo, allCombos, maxComboDepth, hiddenModelsByProvider); + for (const target of targets) { + const label = typeof target.label === "string" ? target.label.trim() : ""; + if (!label || label.toLowerCase() !== lowerAlias) continue; + matches.push({ kind: "model", modelStr: target.modelStr }); + } + } + return dedupeDestinations(matches); +} + +/** + * Resolve a human-typed alias against the full precedence chain. Never falls + * back to a default combo/model — an empty or ambiguous tier either advances + * to the next tier (empty) or fails closed immediately (ambiguous). + */ +export async function resolveExplicitTarget(args: { + alias: string; + combo: ComboLike; + allCombos: ComboCollectionLike; + isModelAvailable?: IsModelAvailable; + maxComboDepth?: unknown; + hiddenModelsByProvider?: HiddenModelsByProvider; +}): Promise { + const { alias, allCombos, isModelAvailable, hiddenModelsByProvider } = args; + const maxComboDepth = clampComboDepth(args.maxComboDepth); + + const synonymMatches = await resolveSynonymTier(alias); + if (synonymMatches.length > 1) { + return { status: "ambiguous", tier: "synonym", candidates: synonymMatches.map(destinationKey) }; + } + if (synonymMatches.length === 1) { + return { status: "resolved", matchedVia: "synonym", ...synonymMatches[0] }; + } + + const comboNameMatches = resolveComboNameTier(alias, allCombos); + if (comboNameMatches.length > 1) { + return { + status: "ambiguous", + tier: "comboName", + candidates: comboNameMatches.map(destinationKey), + }; + } + if (comboNameMatches.length === 1) { + return { status: "resolved", matchedVia: "comboName", ...comboNameMatches[0] }; + } + + const labelMatches = resolveLabelTier(alias, allCombos, maxComboDepth, hiddenModelsByProvider); + if (labelMatches.length > 1) { + return { status: "ambiguous", tier: "label", candidates: labelMatches.map(destinationKey) }; + } + if (labelMatches.length === 1) { + return { status: "resolved", matchedVia: "label", ...labelMatches[0] }; + } + + if (isModelAvailable) { + try { + const available = await isModelAvailable(alias); + if (available) { + return { status: "resolved", matchedVia: "modelId", kind: "model", modelStr: alias }; + } + } catch { + // Fail through to UNKNOWN_TARGET — a broken availability check must + // never be treated as a match. + } + } + + return { status: "unknown" }; +} diff --git a/open-sse/services/combo/routeSlashCommand.ts b/open-sse/services/combo/routeSlashCommand.ts new file mode 100644 index 000000000000..97ad77af45de --- /dev/null +++ b/open-sse/services/combo/routeSlashCommand.ts @@ -0,0 +1,83 @@ +/** + * Deterministic "/route " slash command — O360 explicit-target- + * routing V1 hybrid contract. Highest routing priority: checked before the + * natural-language detector (explicitTargetDetector.ts) by + * tryExplicitTargetDispatch.ts. Unlike natural language, an unresolved + * "/route" target ALWAYS fails closed — the syntax itself is deterministic + * and unambiguous, so there is no "was this really meant as a command?" + * question left to hedge on. + * + * Recognized ONLY as the first non-blank line of the last role="user" + * message's plain text (see extractLastUserMessageText — shared with the + * natural-language detector, so this also never looks at system/assistant + * messages or earlier user turns). Deliberately does NOT reuse + * stripControlNoise's fenced/inline-code stripping: a real "/route" command + * is never itself wrapped in a code fence or inline-code span in any of the + * required examples, and — more importantly — a fence/span's own opening + * marker (an entire "```" line, or a leading backtick) already occupies the + * first-line position instead of "/route", so a plain first-line check + * already satisfies "must not be recognized inside fenced/inline code" + * without needing to touch (and risk mangling) the rest of the message. + */ +import { extractLastUserMessageText } from "./explicitTargetDetector.ts"; + +export type RouteSlashCommandMatch = { + /** The raw target token/string as typed, trimmed. Never case-normalized + * or otherwise interpreted here — resolution is explicitTargetResolver.ts's + * job. */ + target: string; + /** The last user message's text with the "/route " line removed, + * trimmed. Empty when the command was the ONLY content of the turn — the + * caller (tryExplicitTargetDispatch.ts) fails that closed with + * MISSING_TASK rather than forwarding an empty prompt. */ + strippedTaskContent: string; +}; + +const ROUTE_LINE_PATTERN = /^[ \t]*\/route[ \t]+(\S+)(?:[ \t]+(.*?))?[ \t]*$/u; + +function firstNonBlankLineIndex(lines: string[]): number { + let i = 0; + while (i < lines.length && lines[i].trim().length === 0) i++; + return i; +} + +/** + * Detect a "/route " directive in already-extracted plain text (see + * extractLastUserMessageText). The target is the first token after `/route`. + * Remaining text on that line, plus all following lines, is task content. + * Targets are aliases, combo names, or model ids and cannot contain spaces. + */ +export function detectRouteSlashCommandInText(text: string): RouteSlashCommandMatch | null { + const lines = text.split(/\r?\n/); + const idx = firstNonBlankLineIndex(lines); + if (idx >= lines.length) return null; + + const match = ROUTE_LINE_PATTERN.exec(lines[idx]); + if (!match?.[1]) return null; + + const target = match[1].trim(); + if (!target) return null; + + const sameLineTask = match[2]?.trim(); + const remainingLines = [ + ...lines.slice(0, idx), + ...(sameLineTask ? [sameLineTask] : []), + ...lines.slice(idx + 1), + ]; + const strippedTaskContent = remainingLines.join("\n").trim(); + + return { target, strippedTaskContent }; +} + +/** + * Full pipeline: last user message → detect "/route" directive. The single + * entry point tryExplicitTargetDispatch.ts should call, before falling back + * to the natural-language detector. + */ +export function detectRouteSlashCommandInBody( + body: Record +): RouteSlashCommandMatch | null { + const text = extractLastUserMessageText(body); + if (!text) return null; + return detectRouteSlashCommandInText(text); +} diff --git a/open-sse/services/combo/tryExplicitTargetDispatch.ts b/open-sse/services/combo/tryExplicitTargetDispatch.ts new file mode 100644 index 000000000000..082495d3f7ec --- /dev/null +++ b/open-sse/services/combo/tryExplicitTargetDispatch.ts @@ -0,0 +1,299 @@ +/** + * Explicit target dispatch — O360 explicit-target-routing V1 (hybrid + * contract). + * + * Two ways a human can name a target, checked in this order — the first to + * match wins completely, the second is never even consulted: + * + * 1. "/route " (routeSlashCommand.ts) — the DETERMINISTIC control + * API. An unresolved "/route" target ALWAYS fails closed + * (UNKNOWN_TARGET): the syntax itself is unambiguous, so there is no + * "was this really meant as a command?" question left to hedge on. + * Also the only path that mutates the forwarded task content — see + * routeSlashCommand.ts's strippedTaskContent. + * 2. natural language ("use ", explicitTargetDetector.ts) — the + * CONVENIENCE path. Deliberately conservative: an unresolved candidate + * here falls through to ordinary chat instead of failing closed, + * because ordinary prose constantly takes the exact same shape + * ("use combo fallback", "manda a mensagem") with zero routing intent, + * and real targets are routinely lowercase (combo names, model ids) so + * capitalization can't be used to tell those apart either. Full + * message-history passthrough, unmodified — natural language never + * rewrites the request body. + * + * The prelude branch handleComboChatInner calls immediately after + * phaseComboSetup() and BEFORE it acts on a context-cache/session pin. A + * resolved explicit target always wins over that pin: the pin was already + * computed (and may have rewritten ctx.body.model) by the time this runs, but + * that value is simply never read on this path — returning a Response here + * means the pinned-model branch, and everything after it, never executes. + * + * Two destination shapes, two dispatch mechanisms — both already provided by + * the caller, neither reimplemented here: + * - a bare model/provider id → handleSingleModelWithTimeout (same primitive + * tryPinnedModelDispatch uses), with the SAME durable-health gate + * (isPinnedModelDurablyUnhealthy) — but fail-closed on failure, never a + * silent fall-through to strategy. + * - a combo name → the same RunCombo callback (handleComboChat) + * already threaded through the dispatch prelude for nested combo-refs, + * called with the SAME full, unmodified body (full message history + * passthrough) and the internal explicitTargetResolved recursion guard set. + * + * Every branch records exactly one decision on the request's existing combo + * trace (recordComboDecision) — no new audit subsystem. + */ +import { errorResponse } from "../../utils/error.ts"; +import { detectExplicitTargetInBody } from "./explicitTargetDetector.ts"; +import { detectRouteSlashCommandInBody } from "./routeSlashCommand.ts"; +import { resolveExplicitTarget } from "./explicitTargetResolver.ts"; +import { + isPinnedModelDurablyUnhealthy, + buildBaseOptions, + type PreludeBaseOptionArgs, +} from "./dispatchPrelude.ts"; +import { getComboFromData } from "./comboStructure.ts"; +import { recordComboDecision } from "./decisionTrace.ts"; +import type { + ComboCollectionLike, + HandleComboChatOptions, + HandleSingleModel, + IsModelAvailable, + SingleModelTarget, +} from "./types.ts"; + +type RunCombo = (options: HandleComboChatOptions) => Promise; + +/** Where a candidate target came from — recorded on the trace and also what + * decides fail-closed-on-unknown vs. fall-through (see module doc comment). */ +type ExplicitTargetSource = "slash_command" | "natural_language"; + +/** + * Replace the LAST role="user" message's content with a new plain string, + * leaving every earlier message (and the message array's own identity for + * anything but that one entry) untouched. Used only for a "/route" match — + * natural language never rewrites the body (see module doc comment). + */ +function replaceLastUserMessageContent( + body: Record, + newContent: string +): Record { + const raw = body?.messages; + const messages = Array.isArray(raw) ? (raw as Record[]) : []; + const idx = messages.map((m) => m?.role).lastIndexOf("user"); + if (idx === -1) return body; + const newMessages = messages.slice(); + newMessages[idx] = { ...messages[idx], content: newContent }; + return { ...body, messages: newMessages }; +} + +export async function tryExplicitTargetDispatch( + args: PreludeBaseOptionArgs & { + isModelAvailable?: IsModelAvailable; + handleSingleModelWithTimeout: HandleSingleModel; + allCombos?: ComboCollectionLike; + maxComboDepth?: unknown; + runCombo: RunCombo; + explicitTargetResolved?: boolean; + } +): Promise { + // Recursion guard: this invocation is itself the result of an explicit + // override (a combo-typed target dispatched via runCombo). Never detect a + // second time on the same request, however many times "/route"/"use + // " still appears in the (unmodified, on purpose) message history. + if (args.explicitTargetResolved) return null; + + const traceInvocationId = args.invocationId; + const record = ( + decision: Parameters[1]["decision"], + target: string + ) => { + if (!traceInvocationId) return; + recordComboDecision(traceInvocationId, { step: "explicit_target", target, decision }); + }; + + const slashMatch = detectRouteSlashCommandInBody(args.body); + if (slashMatch) { + if (!slashMatch.strippedTaskContent) { + record("missing_task", `source=slash_command alias=${slashMatch.target}`); + return errorResponse( + 400, + `MISSING_TASK: "/route ${slashMatch.target}" has no task content — add task text on the same line after the target, or on the following line` + ); + } + const forwardBody = replaceLastUserMessageContent(args.body, slashMatch.strippedTaskContent); + return dispatchResolvedTarget({ + args, + forwardBody, + alias: slashMatch.target, + source: "slash_command", + // Deterministic syntax: an unresolved "/route" target ALWAYS fails + // closed. There is no lower-confidence fallback tier for this path. + failClosedOnUnknown: true, + // The user explicitly typed this exact string as a target — tier 4 + // (literal model id, resolveExplicitTarget's isModelAvailable check) + // is exactly what "/route cx/gpt-5.6-sol-high" needs. + allowLiteralModelIdLookup: true, + record, + }); + } + + const nlMatch = detectExplicitTargetInBody(args.body); + if (!nlMatch) return null; + return dispatchResolvedTarget({ + args, + forwardBody: args.body, + alias: nlMatch.alias, + source: "natural_language", + // Convenience syntax: an unresolved natural-language candidate falls + // through to ordinary chat instead of failing closed — see module doc + // comment for why capitalization/strength heuristics were dropped. + failClosedOnUnknown: false, + // Tier 4 (literal model id) must NEVER be reached for a natural-language + // candidate. The production isModelAvailable check (checkModelAvailable, + // src/sse/handlers/chat.ts) is written for a string the CALLER already + // believes might be a real model id — fed an arbitrary single word + // extracted from ordinary prose ("Use combo fallback" → candidate + // "combo"), it does not reliably return false for nonsense, and a false + // "yes" here would dispatch straight to a bogus literal model id instead + // of falling through. Restricting NL to tiers 1-3 (synonym/combo-name/ + // label — all local, deterministic lookups) is what makes the + // fall-through guarantee actually safe. + allowLiteralModelIdLookup: false, + record, + }); +} + +/** + * Gate tier 4 (literal model id, resolveExplicitTarget's isModelAvailable + * check) on the source's own confidence — see the "natural_language" call + * site's doc comment above for why NL must never reach it. Split out purely + * to keep dispatchResolvedTarget's own branching flat; no behavior change. + */ +function selectIsModelAvailable( + isModelAvailable: IsModelAvailable | undefined, + allowLiteralModelIdLookup: boolean +): IsModelAvailable | undefined { + return allowLiteralModelIdLookup ? isModelAvailable : undefined; +} + +async function dispatchResolvedTarget(opts: { + args: PreludeBaseOptionArgs & { + isModelAvailable?: IsModelAvailable; + handleSingleModelWithTimeout: HandleSingleModel; + allCombos?: ComboCollectionLike; + maxComboDepth?: unknown; + runCombo: RunCombo; + }; + forwardBody: Record; + alias: string; + source: ExplicitTargetSource; + failClosedOnUnknown: boolean; + allowLiteralModelIdLookup: boolean; + record: (decision: Parameters[1]["decision"], target: string) => void; +}): Promise { + const { + args, + forwardBody, + alias, + source, + failClosedOnUnknown, + allowLiteralModelIdLookup, + record, + } = opts; + + const resolution = await resolveExplicitTarget({ + alias, + combo: args.combo, + allCombos: args.allCombos, + isModelAvailable: selectIsModelAvailable(args.isModelAvailable, allowLiteralModelIdLookup), + maxComboDepth: args.maxComboDepth, + hiddenModelsByProvider: args.hiddenModelsByProvider, + }); + + if (resolution.status === "unknown") { + if (!failClosedOnUnknown) { + record("weak_unresolved_fallthrough", `source=${source} alias=${alias}`); + return null; + } + record("unknown_target", `source=${source} alias=${alias}`); + return errorResponse( + 400, + `UNKNOWN_TARGET: "${alias}" does not match any configured alias, combo, target label, or model id` + ); + } + + if (resolution.status === "ambiguous") { + record( + "ambiguous_target", + `source=${source} alias=${alias} tier=${resolution.tier} candidates=${resolution.candidates.join(",")}` + ); + return errorResponse( + 409, + `AMBIGUOUS_TARGET: "${alias}" matches more than one ${resolution.tier} candidate (${resolution.candidates.join(", ")}) — refusing to guess` + ); + } + + if (resolution.kind === "model") { + return dispatchToModel(args, forwardBody, alias, source, resolution.modelStr, record); + } + return dispatchToCombo(args, forwardBody, alias, source, resolution.comboName, record); +} + +type DispatchRecord = ( + decision: Parameters[1]["decision"], + target: string +) => void; + +async function dispatchToModel( + args: { handleSingleModelWithTimeout: HandleSingleModel }, + forwardBody: Record, + alias: string, + source: ExplicitTargetSource, + modelStr: string, + record: DispatchRecord +): Promise { + const unhealthy = await isPinnedModelDurablyUnhealthy(modelStr); + if (unhealthy) { + record("target_unavailable", `source=${source} alias=${alias}->${modelStr}`); + return errorResponse( + 503, + `TARGET_UNAVAILABLE: "${alias}" resolved to ${modelStr}, which is currently durably unhealthy` + ); + } + const response = await args.handleSingleModelWithTimeout(forwardBody, modelStr, { + modelPinned: false, + } as SingleModelTarget); + record("explicit_override", `source=${source} alias=${alias}->${modelStr}`); + return response; +} + +async function dispatchToCombo( + args: PreludeBaseOptionArgs & { allCombos?: ComboCollectionLike; runCombo: RunCombo }, + forwardBody: Record, + alias: string, + source: ExplicitTargetSource, + comboName: string, + record: DispatchRecord +): Promise { + const targetCombo = getComboFromData(comboName, args.allCombos); + if (!targetCombo) { + // Defensive only: resolution already validated the combo exists at + // resolve time. A combo deleted in the window between resolution and + // dispatch is a genuine (if vanishingly unlikely) race — fail closed + // rather than silently falling through to the CALLER's own combo. + record("target_unavailable", `source=${source} alias=${alias}->combo/${comboName}`); + return errorResponse( + 503, + `TARGET_UNAVAILABLE: "${alias}" resolved to combo/${comboName}, which no longer exists` + ); + } + + const nestedOptions: HandleComboChatOptions = { + ...buildBaseOptions({ ...args, body: forwardBody }), + combo: targetCombo, + explicitTargetResolved: true, + }; + const response = await args.runCombo(nestedOptions); + record("explicit_override", `source=${source} alias=${alias}->combo/${comboName}`); + return response; +} diff --git a/open-sse/services/combo/types.ts b/open-sse/services/combo/types.ts index 15800b25b5db..4a07e73bdd12 100644 --- a/open-sse/services/combo/types.ts +++ b/open-sse/services/combo/types.ts @@ -102,6 +102,17 @@ export type HiddenModelsByProvider = ReadonlyMap>; export type HandleComboChatOptions = { /** #10681: optional opaque parent invocation id for the decision trace. */ invocationId?: string; + /** + * O360 explicit-target-routing V1 recursion guard. Set ONLY by + * tryExplicitTargetDispatch.ts on its own recursive handleComboChat() call + * when an explicit target resolves to a combo — never read from client JSON + * (this lives on the internal options bag, not on `body`, specifically so a + * client cannot forge it in a request payload). When true, explicit-target + * detection is skipped on that invocation so a resolved combo whose own + * messages still contain the original "use " text cannot re-trigger + * a second override and recurse. + */ + explicitTargetResolved?: boolean; body: Record; combo: ComboLike; handleSingleModel: HandleSingleModel; diff --git a/tests/unit/combo/combo-explicit-target-detector.test.ts b/tests/unit/combo/combo-explicit-target-detector.test.ts new file mode 100644 index 000000000000..cacceb07ce4d --- /dev/null +++ b/tests/unit/combo/combo-explicit-target-detector.test.ts @@ -0,0 +1,224 @@ +// O360 explicit-target-routing V1 — natural-language detector unit tests. +// The deterministic "/route " command has its own test file: +// combo-route-slash-command.test.ts. +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + detectExplicitTargetInBody, + detectExplicitTargetPhrase, + extractLastUserMessageText, + stripControlNoise, +} from "../../../open-sse/services/combo/explicitTargetDetector.ts"; + +function bodyWithUser(text: string, extraMessages: Record[] = []) { + return { messages: [...extraMessages, { role: "user", content: text }] }; +} + +// ── extractLastUserMessageText ─────────────────────────────────────────────── + +test("extractLastUserMessageText: plain string content", () => { + assert.equal(extractLastUserMessageText(bodyWithUser("use Astra")), "use Astra"); +}); + +test("extractLastUserMessageText: content-parts array, text parts only", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "use Astra" }, + { type: "image_url", image_url: { url: "data:..." } }, + { type: "input_text", text: "para revisar" }, + ], + }, + ], + }; + assert.equal(extractLastUserMessageText(body), "use Astra\npara revisar"); +}); + +test("extractLastUserMessageText: only the LAST user message counts, not assistant/system/earlier user turns", () => { + const body = { + messages: [ + { role: "system", content: "use Astra" }, + { role: "user", content: "primeiro turno, ignorar" }, + { role: "assistant", content: "use Astra (assistant said this, must not count)" }, + { role: "user", content: "agora use Astra para criticar" }, + ], + }; + assert.equal(extractLastUserMessageText(body), "agora use Astra para criticar"); +}); + +test("extractLastUserMessageText: no user message returns null", () => { + assert.equal(extractLastUserMessageText({ messages: [{ role: "system", content: "x" }] }), null); +}); + +// ── stripControlNoise ───────────────────────────────────────────────────────── + +test("stripControlNoise: removes fenced code blocks but keeps surrounding prose", () => { + const text = "use Astra para revisar:\n```python\nuse_astra = True\n```\nobrigado"; + const out = stripControlNoise(text); + assert.ok(!out.includes("use_astra")); + assert.ok(out.includes("use Astra para revisar")); +}); + +test("stripControlNoise: removes inline code spans", () => { + const out = stripControlNoise("veja a variável `use_astra_flag` e use Astra para revisar"); + assert.ok(!out.includes("use_astra_flag")); + assert.ok(out.includes("use Astra para revisar")); +}); + +test("stripControlNoise: drops flattened tool-call/tool-result lines", () => { + const text = [ + "use Astra para revisar", + '[Called tools: grep(pattern="use Astra")]', + "[Tool result: use Astra found in 3 files]", + ].join("\n"); + const out = stripControlNoise(text); + assert.ok(out.includes("use Astra para revisar")); + assert.ok(!out.includes("Called tools")); + assert.ok(!out.includes("Tool result")); +}); + +// ── detectExplicitTargetPhrase — positive cases ─────────────────────────────── +// Capitalization is NOT a routing signal — every one of these must extract +// identically whether the alias is typed uppercase or lowercase. Whether the +// candidate actually resolves to something real is the resolver's job +// (explicitTargetResolver.ts), and whether an unresolved candidate fails +// closed or falls through is tryExplicitTargetDispatch.ts's job — both out of +// scope for this module's own tests. + +for (const phrase of [ + "use Astra para criticar isso", + "pede pro Astra revisar", + "pergunta pro Claude", + "manda o Astra analisar", + "manda Claude criticar isso", + "manda claude criticar isso", + "com o Astra, procure falhas", +]) { + test(`detectExplicitTargetPhrase: positive — "${phrase}"`, () => { + const result = detectExplicitTargetPhrase(phrase); + assert.ok(result, `expected a match for: ${phrase}`); + }); +} + +test('detectExplicitTargetPhrase: "use Astra para criticar isso" extracts alias "Astra"', () => { + assert.deepEqual(detectExplicitTargetPhrase("use Astra para criticar isso"), { alias: "Astra" }); +}); + +test('detectExplicitTargetPhrase: "manda o Astra analisar" extracts alias "Astra" (article swallowed)', () => { + assert.deepEqual(detectExplicitTargetPhrase("manda o Astra analisar"), { alias: "Astra" }); +}); + +test('detectExplicitTargetPhrase: "com o Astra, procure falhas" extracts alias "Astra"', () => { + assert.deepEqual(detectExplicitTargetPhrase("com o Astra, procure falhas"), { alias: "Astra" }); +}); + +// ── lowercase aliases — capitalization must never gate extraction ──────────── + +for (const [phrase, alias] of [ + ["use astra", "astra"], + ["usa claude", "claude"], + ["pede pro astra revisar", "astra"], + ["manda claude criticar isso", "claude"], + ["com o astra, procure falhas", "astra"], +] as const) { + test(`detectExplicitTargetPhrase: lowercase alias — "${phrase}" extracts "${alias}"`, () => { + assert.deepEqual(detectExplicitTargetPhrase(phrase), { alias }); + }); +} + +// ── lowercase-hyphenated combo names and literal model ids (with `/`/`.`) ──── +// Real targets are routinely NOT proper nouns: combo names are +// lowercase-hyphenated, and literal model ids (explicitTargetResolver.ts +// tier 4) carry `/` and `.`. Both must be extractable — whether they resolve +// is the resolver's concern, tested at dispatch level. + +test('detectExplicitTargetPhrase: "use o360-cheap-worker" extracts the full lowercase-hyphenated combo name', () => { + assert.deepEqual(detectExplicitTargetPhrase("use o360-cheap-worker"), { + alias: "o360-cheap-worker", + }); +}); + +test('detectExplicitTargetPhrase: "use cx/gpt-5.6-sol-high" extracts the full literal model id including "/" and "."', () => { + assert.deepEqual(detectExplicitTargetPhrase("use cx/gpt-5.6-sol-high"), { + alias: "cx/gpt-5.6-sol-high", + }); +}); + +// ── detectExplicitTargetPhrase — required negative cases (no match at all) ─── + +test('detectExplicitTargetPhrase: bare mention "Astra" alone does not match', () => { + assert.equal(detectExplicitTargetPhrase("Astra"), null); +}); + +test('detectExplicitTargetPhrase: "documentação diz para usar Astra" does not match (infinitive "usar", not a command)', () => { + assert.equal(detectExplicitTargetPhrase("documentação diz para usar Astra"), null); +}); + +// ── ordinary sentences — extraction is fine; "must not trigger routing" is +// proven at the dispatch level (combo-explicit-target-dispatch.test.ts), +// because natural language is deliberately conservative: an unresolved +// candidate always falls through instead of failing closed. Capitalization +// must never be the gate here — "combo"/"tool" are exactly as valid a +// candidate shape as "Astra" is; what actually saves these from becoming +// UNKNOWN_TARGET 400s is that natural language NEVER fails closed on its +// own account (only "/route" does — see routeSlashCommand.ts). + +for (const [phrase, alias] of [ + ["Use combo fallback", "combo"], + ["Use combo fallback but force a throw", "combo"], + ["Use a tool to look something up", "tool"], + ["manda a mensagem", "mensagem"], +] as const) { + test(`detectExplicitTargetPhrase: ordinary sentence — "${phrase}" extracts candidate "${alias}" (fallthrough proven at dispatch level)`, () => { + assert.deepEqual(detectExplicitTargetPhrase(phrase), { alias }); + }); +} + +// ── "com o " — required false-positive fix ──────────────────────────── + +for (const phrase of [ + "ontem falei com o Astra sobre isso", + "já conversei com o Astra", + "isso foi revisado com o Astra", + "com o Astra tudo certo", +]) { + test(`detectExplicitTargetPhrase: "com o" false-positive — "${phrase}" does not match`, () => { + assert.equal(detectExplicitTargetPhrase(phrase), null); + }); +} + +for (const phrase of [ + "com o Astra critique o código", + "com o Claude, corrija os testes", + "com a Astra verifique isso", +]) { + test(`detectExplicitTargetPhrase: "com o" + imperative verb — positive — "${phrase}"`, () => { + const result = detectExplicitTargetPhrase(phrase); + assert.ok(result, `expected a match for: ${phrase}`); + }); +} + +test('detectExplicitTargetPhrase: "a documentação recomenda usar o Claude" does not match', () => { + assert.equal(detectExplicitTargetPhrase("a documentação recomenda usar o Claude"), null); +}); + +// ── full pipeline: natural command + fenced code in the SAME message ───────── + +test("detectExplicitTargetInBody: a real command survives even with a fenced code block in the same message", () => { + const body = bodyWithUser( + 'use Astra para revisar:\n```python\ndef f():\n return "use Astra"\n```' + ); + assert.deepEqual(detectExplicitTargetInBody(body), { alias: "Astra" }); +}); + +test("detectExplicitTargetInBody: code-only content (fenced) with no natural command does not match", () => { + const body = bodyWithUser('```python\n# use Astra\nprint("use Astra")\n```'); + assert.equal(detectExplicitTargetInBody(body), null); +}); + +test("detectExplicitTargetInBody: no explicit-target body returns null", () => { + assert.equal(detectExplicitTargetInBody(bodyWithUser("corrija os pontos que encontrou")), null); +}); diff --git a/tests/unit/combo/combo-explicit-target-dispatch.test.ts b/tests/unit/combo/combo-explicit-target-dispatch.test.ts new file mode 100644 index 000000000000..ed23a1ead476 --- /dev/null +++ b/tests/unit/combo/combo-explicit-target-dispatch.test.ts @@ -0,0 +1,930 @@ +/** + * O360 explicit-target-routing V1 — orchestration tests via the PUBLIC + * handleComboChat entry point (same convention as combo-decision-trace.test.ts + * and combo-target-timeout-standards.test.ts: real combo dispatch, injected + * fake handleSingleModel — no private helpers, no real HTTP server, no real + * provider credentials). + * + * Covers BOTH routing paths, checked in priority order by + * tryExplicitTargetDispatch.ts: + * 1. deterministic "/route " (routeSlashCommand.ts) — always + * fails closed on an unresolved target. + * 2. natural-language "use " (explicitTargetDetector.ts) — + * convenience path, falls through to ordinary chat when unresolved. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-explicit-target-dispatch-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-target-dispatch-secret"; + +const { handleComboChat } = await import("../../../open-sse/services/combo.ts"); +const { updateSettings } = await import("../../../src/lib/db/settings.ts"); +const { recordSessionModelUsage } = await import("../../../src/lib/db/contextHandoffs.ts"); +const { createProviderConnection } = await import("../../../src/lib/db/providers.ts"); +const { resetComboTraceStore, getComboTrace, createInvocationId } = + await import("../../../open-sse/services/combo/decisionTrace.ts"); + +const noop = () => {}; +const log = { info: noop, warn: noop, debug: noop, error: noop }; + +function okResponse(content: string) { + return new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +// tryExplicitTargetDispatch reuses isPinnedModelDurablyUnhealthy (same health +// gate tryPinnedModelDispatch already applies to a context-cache pin) — a +// provider with ZERO active connections is durably unhealthy by design (no +// account can possibly serve it). "provider-b" is the resolved target for +// most explicit-alias tests below, so it needs one seeded active connection +// to be a realistic HEALTHY target, exactly like a real deployment would have +// before anyone could route to it. "cx" is seeded separately for the literal +// model-id tests (mirrors explicitTargetResolver.test.ts's own +// "cx/gpt-5.6-sol-high" example). +let seededProviderB = false; +async function ensureHealthyProviderB() { + if (seededProviderB) return; + await createProviderConnection({ + provider: "provider-b", + authType: "apikey", + name: "explicit-target-test-conn-b", + apiKey: "sk-test", + isActive: true, + testStatus: "active", + }); + seededProviderB = true; +} + +// "cx" in "cx/gpt-5.6-sol-high" is a registered ALIAS for the real "codex" +// provider (resolveProviderAlias, open-sse/services/model.ts) — parseModel() +// resolves it before the health gate ever runs, so the seeded connection has +// to be for "codex", not the literal "cx" prefix (confirmed empirically: a +// "cx"-only seed left the health gate reporting durably unhealthy). +let seededProviderCodex = false; +async function ensureHealthyProviderCodex() { + if (seededProviderCodex) return; + await createProviderConnection({ + provider: "codex", + authType: "apikey", + name: "explicit-target-test-conn-codex", + apiKey: "sk-test", + isActive: true, + testStatus: "active", + }); + seededProviderCodex = true; +} + +let seededProviderClaude = false; +async function ensureHealthyProviderClaude() { + if (seededProviderClaude) return; + await createProviderConnection({ + provider: "claude", + authType: "apikey", + name: "explicit-target-test-conn-claude", + apiKey: "sk-test", + isActive: true, + testStatus: "active", + }); + seededProviderClaude = true; +} + +// Fallthrough tests dispatch to the COMBO's OWN normal-strategy candidate +// ("provider-a/model-a") once an unresolved explicit-target candidate falls +// through — that candidate selection also needs a real healthy connection, +// same as provider-b/codex above. +let seededProviderA = false; +async function ensureHealthyProviderA() { + if (seededProviderA) return; + await createProviderConnection({ + provider: "provider-a", + authType: "apikey", + name: "explicit-target-test-conn-a", + apiKey: "sk-test", + isActive: true, + testStatus: "active", + }); + seededProviderA = true; +} + +test.beforeEach(async () => { + resetComboTraceStore(); + await updateSettings({ explicitTargetAliases: {} }); + await ensureHealthyProviderA(); + await ensureHealthyProviderB(); + await ensureHealthyProviderCodex(); + await ensureHealthyProviderClaude(); +}); + +// ═══════════════════════════════════════════════════════════════════════════ +// Natural language — bare-model target, full messages passthrough +// ═══════════════════════════════════════════════════════════════════════════ + +test("bare-model explicit target: dispatches directly, full message history preserved", async () => { + await updateSettings({ explicitTargetAliases: { astra: "provider-b/model-b" } }); + const calls: { modelStr: string; messages: unknown }[] = []; + const messages = [ + { role: "user", content: "primeiro turno" }, + { role: "assistant", content: "resposta anterior" }, + { role: "user", content: "agora use Astra para criticar o que você fez" }, + ]; + + const res = await handleComboChat({ + body: { messages }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (b: Record, modelStr: string) => { + calls.push({ modelStr, messages: b.messages }); + return okResponse("critique from astra"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + + assert.equal(res.status, 200); + assert.equal(calls.length, 1); + assert.equal(calls[0].modelStr, "provider-b/model-b"); + assert.deepEqual( + calls[0].messages, + messages, + "full conversation history must reach the resolved target unmodified" + ); +}); + +test("natural language: lowercase alias resolves identically to the capitalized form", async () => { + await updateSettings({ explicitTargetAliases: { astra: "provider-b/model-b" } }); + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "use astra para revisar" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["provider-b/model-b"]); +}); + +test('natural language: "use Claude" resolves the real configured Claude target', async () => { + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "use Claude" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["cc/claude-sonnet-5"]); +}); + +// ── combo target: full passthrough + exactly-one recursion guard ──────────── + +test("combo explicit target: recurses exactly once, full passthrough, no infinite loop", async () => { + await updateSettings({ explicitTargetAliases: { astra: "combo/critic-combo" } }); + const calls: { modelStr: string; messages: unknown }[] = []; + const messages = [ + { role: "user", content: "primeiro turno" }, + { role: "assistant", content: "resposta anterior" }, + { role: "user", content: "agora use Astra para criticar o que você fez" }, + ]; + const combos = [ + { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + { name: "critic-combo", strategy: "priority", models: ["provider-c/model-c"] }, + ]; + + const res = await handleComboChat({ + body: { messages }, + combo: combos[0], + handleSingleModel: async (b: Record, modelStr: string) => { + calls.push({ modelStr, messages: b.messages }); + return okResponse("critique from astra"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: combos, + }); + + assert.equal(res.status, 200); + // Exactly one dispatch, to the critic combo's model — never main-combo's own + // model, and never dispatched twice (which is what an unguarded recursive + // re-detection of "use Astra" in the same message would cause). + assert.equal(calls.length, 1, "explicit target must recurse exactly once, not loop"); + assert.equal(calls[0].modelStr, "provider-c/model-c"); + assert.deepEqual(calls[0].messages, messages); +}); + +// ═══════════════════════════════════════════════════════════════════════════ +// Natural language — conservative: unresolved candidates fall through +// ═══════════════════════════════════════════════════════════════════════════ + +for (const phrase of [ + "Use combo fallback", + "Use a tool to look something up", + "manda a mensagem", + "use FooModel para revisar", +]) { + test(`natural language: unresolved candidate in "${phrase}" falls through to normal combo dispatch, never UNKNOWN_TARGET`, async () => { + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: phrase }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("normal combo response"); + }, + // false for any explicit-target candidate (proving it never resolves), + // but true for the combo's OWN real model — this same callback also + // gates the combo's normal-strategy candidate filtering, not just + // explicit-target resolution. + isModelAvailable: async (m: string) => m === "provider-a/model-a", + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200, `"${phrase}" must not hard-fail the request`); + assert.deepEqual( + calls, + ["provider-a/model-a"], + `"${phrase}" must fall through to the combo's own strategy` + ); + }); +} + +test("natural language: never reaches tier-4 (literal model id) resolution, even when isModelAvailable would say yes", async () => { + // Regression guard: a real production isModelAvailable (checkModelAvailable, + // src/sse/handlers/chat.ts) does not reliably return false for a nonsense + // string — it is written for a model id the CALLER already believes might + // be real, not an arbitrary word extracted from ordinary prose. Feeding an + // NL candidate into tier 4 let "Use combo fallback" resolve "combo" as a + // literal (bogus) model id and dispatch to it — caught via a direct + // production stack trace, not just this stub. Simulating that "yes" here + // (isModelAvailable always true) proves NL still correctly ignores tier 4 + // and falls through, rather than relying on the stub happening to say no. + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "Use combo fallback" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("normal combo response"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual( + calls, + ["provider-a/model-a"], + "natural language must never dispatch to a tier-4-resolved literal model id" + ); +}); + +// ═══════════════════════════════════════════════════════════════════════════ +// Deterministic "/route " — the ONLY path that fails closed on an +// unresolved target, and the only path that strips the routing directive +// from the forwarded task text. +// ═══════════════════════════════════════════════════════════════════════════ + +for (const target of ["claude", "Claude", "CLAUDE", "astra", "ASTRA"]) { + test(`/route: "${target}" resolves case-insensitively to its approved real model`, async () => { + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: `/route ${target}\nreview this` }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, [ + target.toLowerCase() === "claude" ? "cc/claude-sonnet-5" : "cx/gpt-5.6-sol-high", + ]); + }); +} + +test("/route: literal model id with '/' and '.' routes to the exact model", async () => { + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "/route cx/gpt-5.6-sol-high\nreview this" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async (modelStr: string) => modelStr === "cx/gpt-5.6-sol-high", + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["cx/gpt-5.6-sol-high"]); +}); + +test("/route: lowercase-hyphenated combo name routes to that combo, case-insensitively", async () => { + const calls: string[] = []; + const combos = [ + { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + { name: "o360-cheap-worker", strategy: "priority", models: ["provider-c/model-c"] }, + ]; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "/route O360-CHEAP-WORKER\nreview this" }] }, + combo: combos[0], + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: combos, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["provider-c/model-c"]); +}); + +test('/route: "o360-coding" resolves to the configured combo', async () => { + const calls: string[] = []; + const combos = [ + { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + { name: "o360-coding", strategy: "priority", models: ["provider-b/model-b"] }, + ]; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "/route o360-coding\nreview this" }] }, + combo: combos[0], + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: combos, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["provider-b/model-b"]); +}); + +test("/route: unknown target ALWAYS fails closed (deterministic syntax, no lower-confidence fallback)", async () => { + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "/route TotallyUnknown\nreview this" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("should not happen"); + }, + isModelAvailable: async () => false, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 400); + assert.deepEqual(calls, [], "an unresolved /route target must never fall through"); + const body = await res.json(); + assert.match(JSON.stringify(body), /UNKNOWN_TARGET/); +}); + +test("/route: the control directive is stripped from the forwarded task text", async () => { + await updateSettings({ explicitTargetAliases: { claude: "provider-b/model-b" } }); + let forwardedContent: unknown; + const res = await handleComboChat({ + body: { + messages: [ + { role: "user", content: "primeiro turno" }, + { role: "assistant", content: "ok" }, + { role: "user", content: "/route claude Review this implementation for race conditions." }, + ], + }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (b: Record) => { + const messages = b.messages as { role: string; content: unknown }[]; + forwardedContent = messages[messages.length - 1].content; + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.equal( + forwardedContent, + "Review this implementation for race conditions.", + "the model must receive only the task text, not the raw /route control line" + ); +}); + +test("/route: earlier turns are never mutated, only the current turn's control line is removed", async () => { + await updateSettings({ explicitTargetAliases: { claude: "provider-b/model-b" } }); + const earlierTurns = [ + { role: "user", content: "primeiro turno" }, + { role: "assistant", content: "resposta anterior" }, + ]; + let forwardedMessages: unknown; + const res = await handleComboChat({ + body: { + messages: [...earlierTurns, { role: "user", content: "/route claude\nreview this" }], + }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (b: Record) => { + forwardedMessages = b.messages; + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + const messages = forwardedMessages as { role: string; content: unknown }[]; + assert.equal(messages.length, 3); + assert.deepEqual(messages[0], earlierTurns[0]); + assert.deepEqual(messages[1], earlierTurns[1]); + assert.equal(messages[2].content, "review this"); +}); + +test("/route: the command alone with no task content fails closed with MISSING_TASK", async () => { + await updateSettings({ explicitTargetAliases: { claude: "provider-b/model-b" } }); + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "/route claude" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("should not happen"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 400); + assert.deepEqual(calls, []); + const body = await res.json(); + assert.match(JSON.stringify(body), /MISSING_TASK/); +}); + +test("/route: wrapped in a fenced code block does not route (falls through to normal combo dispatch)", async () => { + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "```\n/route claude\n```" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async (m: string) => m === "provider-a/model-a", + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["provider-a/model-a"]); +}); + +test("/route: wrapped in inline code does not route (falls through to normal combo dispatch)", async () => { + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "`/route claude`" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async (m: string) => m === "provider-a/model-a", + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["provider-a/model-a"]); +}); + +test("/route: in an earlier turn (not the last user message) does not route", async () => { + await updateSettings({ explicitTargetAliases: { claude: "provider-b/model-b" } }); + const calls: string[] = []; + const res = await handleComboChat({ + body: { + messages: [ + { role: "user", content: "/route claude\nold instruction" }, + { role: "assistant", content: "ok" }, + { role: "user", content: "corrija os pontos que encontrou" }, + ], + }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("ok"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["provider-a/model-a"]); +}); + +// ═══════════════════════════════════════════════════════════════════════════ +// Fail-closed paths shared by both routing sources +// ═══════════════════════════════════════════════════════════════════════════ + +test("ambiguous label across two combos: AMBIGUOUS_TARGET (409), no dispatch at all", async () => { + const calls: string[] = []; + const combos = [ + { + name: "combo-a", + strategy: "priority", + models: [{ id: "s1", model: "provider-a/model-a", label: "Critic" }], + }, + { + name: "combo-b", + strategy: "priority", + models: [{ id: "s1", model: "provider-b/model-b", label: "Critic" }], + }, + ]; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "use Critic para revisar" }] }, + combo: combos[0], + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("should not happen"); + }, + isModelAvailable: async () => false, + log, + settings: null, + allCombos: combos, + }); + assert.equal(res.status, 409); + assert.deepEqual(calls, []); + const body = await res.json(); + assert.match(JSON.stringify(body), /AMBIGUOUS_TARGET/); +}); + +test("natural language: resolved-but-durably-unhealthy target: TARGET_UNAVAILABLE (503), no fallback to combo strategy", async () => { + await updateSettings({ + explicitTargetAliases: { brokenalias: "totally-fake-provider-xyz/model-1" }, + }); + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "use BrokenAlias para revisar" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("should not happen"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 503); + assert.deepEqual( + calls, + [], + "a durably unhealthy explicit target must fail closed, never fall through to the combo's own healthy candidate" + ); + const body = await res.json(); + assert.match(JSON.stringify(body), /TARGET_UNAVAILABLE/); +}); + +test("/route: resolved-but-durably-unhealthy target: TARGET_UNAVAILABLE (503), no fallback to combo strategy", async () => { + await updateSettings({ + explicitTargetAliases: { brokenalias: "totally-fake-provider-xyz/model-1" }, + }); + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "/route BrokenAlias\nreview this" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("should not happen"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 503); + assert.deepEqual(calls, []); + const body = await res.json(); + assert.match(JSON.stringify(body), /TARGET_UNAVAILABLE/); +}); + +// ── explicit target beats an active context-cache/session pin ─────────────── + +test("natural language: explicit target overrides an active context-cache pin for the same session", async () => { + await updateSettings({ explicitTargetAliases: { astra: "provider-b/model-b" } }); + recordSessionModelUsage("sess-pin-test", "main-combo", "provider-a/pinned-model", "provider-a"); + + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "agora use Astra para criticar" }] }, + combo: { + name: "main-combo", + strategy: "priority", + models: ["provider-a/pinned-model", "provider-a/model-a"], + context_cache_protection: true, + }, + relayOptions: { sessionId: "sess-pin-test" }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("critique"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + + assert.equal(res.status, 200); + assert.deepEqual( + calls, + ["provider-b/model-b"], + "explicit target must win over the session's context-cache pin" + ); +}); + +test("/route: overrides an active context-cache pin for the same session", async () => { + await updateSettings({ explicitTargetAliases: { claude: "provider-b/model-b" } }); + recordSessionModelUsage( + "sess-pin-test-slash", + "main-combo", + "provider-a/pinned-model", + "provider-a" + ); + + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "/route claude\ncritique this" }] }, + combo: { + name: "main-combo", + strategy: "priority", + models: ["provider-a/pinned-model", "provider-a/model-a"], + context_cache_protection: true, + }, + relayOptions: { sessionId: "sess-pin-test-slash" }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("critique"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + + assert.equal(res.status, 200); + assert.deepEqual(calls, ["provider-b/model-b"]); +}); + +test("explicit COMBO target beats an existing pin, and the old pin cannot reappear during the recursion", async () => { + await updateSettings({ explicitTargetAliases: { astra: "combo/critic-combo" } }); + // A prior turn pinned main-combo to provider-a/model-a for this session. + recordSessionModelUsage("sess-combo-pin-test", "main-combo", "provider-a/model-a", "provider-a"); + // critic-combo ALSO has context-cache protection on (the harder case): if + // the recursion guard or session threading were wrong, this combo's own + // phaseComboSetup could try to honor a pin too, and something might + // resolve back to the old provider-a/model-a — it must not, both because + // no pin was ever recorded for "critic-combo" specifically (pin history is + // keyed by (session_id, combo_name)) and because explicit-target detection + // is skipped on the recursed-into call (explicitTargetResolved=true), so it + // can never re-resolve "Astra" and re-recurse either. + const combos = [ + { + name: "main-combo", + strategy: "priority", + models: ["provider-a/model-a"], + context_cache_protection: true, + }, + { + name: "critic-combo", + strategy: "priority", + models: ["provider-c/model-c"], + context_cache_protection: true, + }, + ]; + + const calls: string[] = []; + const res = await handleComboChat({ + body: { messages: [{ role: "user", content: "use Astra para criticar" }] }, + combo: combos[0], + relayOptions: { sessionId: "sess-combo-pin-test" }, + handleSingleModel: async (_b: Record, modelStr: string) => { + calls.push(modelStr); + return okResponse("critique"); + }, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: combos, + }); + + assert.equal(res.status, 200); + assert.deepEqual( + calls, + ["provider-c/model-c"], + "critic-combo must dispatch to its OWN target — the old main-combo pin must never reappear during the recursion" + ); +}); + +// ── audit trail ─────────────────────────────────────────────────────────────── + +test("explicit override is recorded on the request's existing combo trace (no new audit subsystem)", async () => { + await updateSettings({ explicitTargetAliases: { astra: "provider-b/model-b" } }); + const invocationId = createInvocationId(); + const res = await handleComboChat({ + invocationId, + body: { messages: [{ role: "user", content: "use Astra para revisar" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async () => okResponse("ok"), + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + const trace = getComboTrace(invocationId)!; + const explicitDecisions = trace.decisions.filter((d) => d.step === "explicit_target"); + assert.equal(explicitDecisions.length, 1); + assert.equal(explicitDecisions[0].decision, "explicit_override"); + assert.match(explicitDecisions[0].target, /astra/i); + assert.match(explicitDecisions[0].target, /source=natural_language/); +}); + +test("/route: explicit override is recorded on the trace with source=slash_command", async () => { + await updateSettings({ explicitTargetAliases: { claude: "provider-b/model-b" } }); + const invocationId = createInvocationId(); + const res = await handleComboChat({ + invocationId, + body: { messages: [{ role: "user", content: "/route claude\nreview this" }] }, + combo: { name: "main-combo", strategy: "priority", models: ["provider-a/model-a"] }, + handleSingleModel: async () => okResponse("ok"), + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res.status, 200); + const trace = getComboTrace(invocationId)!; + const explicitDecisions = trace.decisions.filter((d) => d.step === "explicit_target"); + assert.equal(explicitDecisions.length, 1); + assert.equal(explicitDecisions[0].decision, "explicit_override"); + assert.match(explicitDecisions[0].target, /source=slash_command/); + assert.match(explicitDecisions[0].target, /claude/i); +}); + +// ── three-turn acceptance scenario ─────────────────────────────────────────── + +test("three-turn scenario: normal → explicit /route claude with full history → back to normal", async () => { + await updateSettings({ explicitTargetAliases: { claude: "provider-b/model-b" } }); + const dispatched: { modelStr: string; messageCount: number }[] = []; + const handleSingleModel = async (b: Record, modelStr: string) => { + const messages = (b.messages as unknown[]) ?? []; + dispatched.push({ modelStr, messageCount: messages.length }); + if (modelStr === "provider-b/model-b") return okResponse("critique: found 2 issues"); + return okResponse(`implementation from ${modelStr}`); + }; + const combo = { name: "o360-coding-sim", strategy: "priority", models: ["provider-a/codex"] }; + + // TURN 1 — normal coding request, no explicit target. + let messages: Record[] = [{ role: "user", content: "implemente a função X" }]; + const res1 = await handleComboChat({ + body: { messages }, + combo, + handleSingleModel, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res1.status, 200); + assert.equal(dispatched[0].modelStr, "provider-a/codex"); + messages = [...messages, { role: "assistant", content: "implementação da função X pronta" }]; + + // TURN 2 — "/route claude\nreview the previous answer"; full prior history + // must reach the resolved target, with only the current turn's control + // line stripped. + messages = [...messages, { role: "user", content: "/route claude\nreview the previous answer" }]; + const res2 = await handleComboChat({ + body: { messages }, + combo, + handleSingleModel, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res2.status, 200); + assert.equal(dispatched[1].modelStr, "provider-b/model-b"); + assert.equal( + dispatched[1].messageCount, + messages.length, + "turn 2 must carry the full prior history" + ); + messages = [...messages, { role: "assistant", content: "critique: found 2 issues" }]; + + // TURN 3 — back to normal, no explicit target: routing returns to the + // combo's own candidate, not stuck on claude. + messages = [...messages, { role: "user", content: "now improve it" }]; + const res3 = await handleComboChat({ + body: { messages }, + combo, + handleSingleModel, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res3.status, 200); + assert.equal( + dispatched[2].modelStr, + "provider-a/codex", + "turn 3 must route back to normal combo selection" + ); + assert.equal(dispatched.length, 3); +}); + +test("three-turn scenario (natural language): normal → explicit Astra with full history → back to normal", async () => { + await updateSettings({ explicitTargetAliases: { astra: "provider-b/model-b" } }); + const dispatched: { modelStr: string; messageCount: number }[] = []; + const handleSingleModel = async (b: Record, modelStr: string) => { + const messages = (b.messages as unknown[]) ?? []; + dispatched.push({ modelStr, messageCount: messages.length }); + if (modelStr === "provider-b/model-b") return okResponse("critique: found 2 issues"); + return okResponse(`implementation from ${modelStr}`); + }; + const combo = { name: "o360-coding-sim", strategy: "priority", models: ["provider-a/codex"] }; + + let messages: Record[] = [{ role: "user", content: "implemente a função X" }]; + const res1 = await handleComboChat({ + body: { messages }, + combo, + handleSingleModel, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res1.status, 200); + assert.equal(dispatched[0].modelStr, "provider-a/codex"); + messages = [...messages, { role: "assistant", content: "implementação da função X pronta" }]; + + messages = [ + ...messages, + { role: "user", content: "agora use Astra para criticar o que você fez" }, + ]; + const res2 = await handleComboChat({ + body: { messages }, + combo, + handleSingleModel, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res2.status, 200); + assert.equal(dispatched[1].modelStr, "provider-b/model-b"); + assert.equal(dispatched[1].messageCount, messages.length); + messages = [...messages, { role: "assistant", content: "critique: found 2 issues" }]; + + messages = [...messages, { role: "user", content: "agora corrija os pontos que encontrou" }]; + const res3 = await handleComboChat({ + body: { messages }, + combo, + handleSingleModel, + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + assert.equal(res3.status, 200); + assert.equal(dispatched[2].modelStr, "provider-a/codex"); + assert.equal(dispatched.length, 3); +}); diff --git a/tests/unit/combo/combo-explicit-target-resolver.test.ts b/tests/unit/combo/combo-explicit-target-resolver.test.ts new file mode 100644 index 000000000000..3eb9e911f668 --- /dev/null +++ b/tests/unit/combo/combo-explicit-target-resolver.test.ts @@ -0,0 +1,184 @@ +// O360 explicit-target-routing V1 — alias resolution precedence unit tests. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-explicit-target-resolver-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-target-resolver-secret"; + +const { resolveExplicitTarget } = + await import("../../../open-sse/services/combo/explicitTargetResolver.ts"); +const { updateSettings } = await import("../../../src/lib/db/settings.ts"); + +const currentCombo = { name: "o360-coding", strategy: "priority", models: ["cx/codex-a"] }; + +test("tier 1 — built-in Claude alias resolves to the approved real model without settings", async () => { + await updateSettings({ explicitTargetAliases: {} }); + const result = await resolveExplicitTarget({ + alias: "Claude", + combo: currentCombo, + allCombos: [currentCombo], + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "synonym", + kind: "model", + modelStr: "cc/claude-sonnet-5", + }); +}); + +test("tier 1 — built-in Astra alias is case-insensitive and resolves to the approved model", async () => { + await updateSettings({ explicitTargetAliases: {} }); + const result = await resolveExplicitTarget({ + alias: "ASTRA", + combo: currentCombo, + allCombos: [currentCombo], + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "synonym", + kind: "model", + modelStr: "cx/gpt-5.6-sol-high", + }); +}); + +test("tier 1 — settings synonym wins, case-insensitively", async () => { + await updateSettings({ explicitTargetAliases: { glm: "combo/o360-cheap-worker" } }); + const result = await resolveExplicitTarget({ + alias: "GLM", + combo: currentCombo, + allCombos: [currentCombo, { name: "o360-cheap-worker", models: ["zai/glm-5.1"] }], + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "synonym", + kind: "combo", + comboName: "o360-cheap-worker", + }); +}); + +test("tier 1 — configured synonym overrides a built-in friendly alias", async () => { + await updateSettings({ explicitTargetAliases: { astra: "provider-b/model-b" } }); + const result = await resolveExplicitTarget({ + alias: "astra", + combo: currentCombo, + allCombos: [currentCombo], + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "synonym", + kind: "model", + modelStr: "provider-b/model-b", + }); +}); + +test("tier 2 — exact combo name, case-insensitive, when no synonym matches", async () => { + await updateSettings({ explicitTargetAliases: {} }); + const allCombos = [currentCombo, { name: "o360-cheap-worker", models: ["zai/glm-5.1"] }]; + const result = await resolveExplicitTarget({ + alias: "O360-CHEAP-WORKER", + combo: currentCombo, + allCombos, + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "comboName", + kind: "combo", + comboName: "o360-cheap-worker", + }); +}); + +test("tier 3 — exact target label, case-insensitive, when no synonym/combo-name matches", async () => { + await updateSettings({ explicitTargetAliases: {} }); + const allCombos = [ + { + name: "o360-coding", + strategy: "priority", + models: [{ id: "s1", model: "cx/gpt-5.6-sol-high", label: "Reviewer" }], + }, + ]; + const result = await resolveExplicitTarget({ + alias: "reviewer", + combo: allCombos[0], + allCombos, + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "label", + kind: "model", + modelStr: "cx/gpt-5.6-sol-high", + }); +}); + +test("tier 3 ambiguity — the same label on two different targets fails closed", async () => { + await updateSettings({ explicitTargetAliases: {} }); + const allCombos = [ + { + name: "combo-a", + strategy: "priority", + models: [{ id: "s1", model: "cx/model-a", label: "Critic" }], + }, + { + name: "combo-b", + strategy: "priority", + models: [{ id: "s1", model: "cx/model-b", label: "Critic" }], + }, + ]; + const result = await resolveExplicitTarget({ + alias: "critic", + combo: allCombos[0], + allCombos, + }); + assert.equal(result.status, "ambiguous"); + assert.equal((result as { tier: string }).tier, "label"); +}); + +test("tier 4 — literal model id validated via isModelAvailable", async () => { + await updateSettings({ explicitTargetAliases: {} }); + const result = await resolveExplicitTarget({ + alias: "cx/gpt-5.6-sol-high", + combo: currentCombo, + allCombos: [currentCombo], + isModelAvailable: async (modelStr: string) => modelStr === "cx/gpt-5.6-sol-high", + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "modelId", + kind: "model", + modelStr: "cx/gpt-5.6-sol-high", + }); +}); + +test("no tier matches → UNKNOWN_TARGET (unknown status), never a silent default", async () => { + await updateSettings({ explicitTargetAliases: {} }); + const result = await resolveExplicitTarget({ + alias: "foo-model", + combo: currentCombo, + allCombos: [currentCombo], + isModelAvailable: async () => false, + }); + assert.deepEqual(result, { status: "unknown" }); +}); + +test("precedence — a synonym match short-circuits before combo-name/label tiers are even consulted", async () => { + // "o360-cheap-worker" is BOTH a real combo name AND overridden by a synonym + // pointing somewhere else — the synonym (tier 1) must win. + await updateSettings({ + explicitTargetAliases: { "o360-cheap-worker": "cx/gpt-5.6-sol-high" }, + }); + const allCombos = [currentCombo, { name: "o360-cheap-worker", models: ["zai/glm-5.1"] }]; + const result = await resolveExplicitTarget({ + alias: "o360-cheap-worker", + combo: currentCombo, + allCombos, + }); + assert.deepEqual(result, { + status: "resolved", + matchedVia: "synonym", + kind: "model", + modelStr: "cx/gpt-5.6-sol-high", + }); +}); diff --git a/tests/unit/combo/combo-route-slash-command.test.ts b/tests/unit/combo/combo-route-slash-command.test.ts new file mode 100644 index 000000000000..0b8c2cf16492 --- /dev/null +++ b/tests/unit/combo/combo-route-slash-command.test.ts @@ -0,0 +1,172 @@ +// O360 explicit-target-routing V1 — deterministic "/route " slash +// command detector unit tests. Natural language has its own test file: +// combo-explicit-target-detector.test.ts. +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + detectRouteSlashCommandInBody, + detectRouteSlashCommandInText, +} from "../../../open-sse/services/combo/routeSlashCommand.ts"; + +function bodyWithUser(text: string, extraMessages: Record[] = []) { + return { messages: [...extraMessages, { role: "user", content: text }] }; +} + +// ── positive cases — case must never matter for the target token ──────────── + +for (const target of ["claude", "Claude", "CLAUDE", "astra", "o360-coding", "o360-cheap-worker"]) { + test(`detectRouteSlashCommandInText: "/route ${target}" extracts target "${target}" with no task`, () => { + assert.deepEqual(detectRouteSlashCommandInText(`/route ${target}`), { + target, + strippedTaskContent: "", + }); + }); +} + +test('detectRouteSlashCommandInText: "/route cx/gpt-5.6-sol-high" extracts the full literal model id including "/" and "."', () => { + assert.deepEqual(detectRouteSlashCommandInText("/route cx/gpt-5.6-sol-high"), { + target: "cx/gpt-5.6-sol-high", + strippedTaskContent: "", + }); +}); + +test('detectRouteSlashCommandInText: "/route TotallyUnknown" extracts the target regardless of whether it resolves (resolution is the resolver\'s job)', () => { + assert.deepEqual(detectRouteSlashCommandInText("/route TotallyUnknown"), { + target: "TotallyUnknown", + strippedTaskContent: "", + }); +}); + +// ── task content on following line(s) is stripped from the control line +// only — the actual task text must survive completely, formatting and all. + +test("detectRouteSlashCommandInText: task text on the following line is preserved as strippedTaskContent", () => { + const result = detectRouteSlashCommandInText( + "/route claude\nReview this implementation for race conditions." + ); + assert.deepEqual(result, { + target: "claude", + strippedTaskContent: "Review this implementation for race conditions.", + }); +}); + +test("detectRouteSlashCommandInText: task text on the directive line is preserved", () => { + assert.deepEqual(detectRouteSlashCommandInText("/route claude Explain this code"), { + target: "claude", + strippedTaskContent: "Explain this code", + }); +}); + +test("detectRouteSlashCommandInText: same-line and following-line task text are both preserved", () => { + assert.deepEqual(detectRouteSlashCommandInText("/route astra Review this\nand explain why"), { + target: "astra", + strippedTaskContent: "Review this\nand explain why", + }); +}); + +test("detectRouteSlashCommandInText: multi-line task content (including blank lines) survives intact", () => { + const result = detectRouteSlashCommandInText( + "/route claude\n\nReview this.\n\nMore context below." + ); + assert.deepEqual(result, { + target: "claude", + strippedTaskContent: "Review this.\n\nMore context below.", + }); +}); + +test("detectRouteSlashCommandInText: a fenced code block AFTER the command in real task text is preserved untouched", () => { + const result = detectRouteSlashCommandInText( + "/route claude\nReview this:\n```js\nfunction foo() { return 1; }\n```" + ); + assert.deepEqual(result, { + target: "claude", + strippedTaskContent: "Review this:\n```js\nfunction foo() { return 1; }\n```", + }); +}); + +// ── missing task — the command was the ONLY content of the turn ───────────── + +test('detectRouteSlashCommandInText: "/route claude" alone has empty strippedTaskContent', () => { + const result = detectRouteSlashCommandInText("/route claude"); + assert.equal(result?.strippedTaskContent, ""); +}); + +// ── must NOT be recognized inside fenced code, inline code, or if not the +// first line ────────────────────────────────────────────────────────────── + +test("detectRouteSlashCommandInText: /route wrapped in a fenced code block does not route", () => { + assert.equal(detectRouteSlashCommandInText("```\n/route claude\n```"), null); +}); + +test("detectRouteSlashCommandInText: /route wrapped in inline code does not route", () => { + assert.equal(detectRouteSlashCommandInText("`/route claude`"), null); +}); + +test("detectRouteSlashCommandInText: /route not on the first non-blank line does not route", () => { + assert.equal(detectRouteSlashCommandInText("some preamble text\n/route claude"), null); +}); + +test("detectRouteSlashCommandInText: leading blank lines before the command are tolerated", () => { + assert.deepEqual(detectRouteSlashCommandInText("\n\n/route claude\ntask"), { + target: "claude", + strippedTaskContent: "task", + }); +}); + +// ── malformed / absent command ──────────────────────────────────────────── + +test("detectRouteSlashCommandInText: ordinary prose without a slash command does not match", () => { + assert.equal(detectRouteSlashCommandInText("use Astra para revisar"), null); +}); + +test('detectRouteSlashCommandInText: "/routeclaude" (no space) does not match', () => { + assert.equal(detectRouteSlashCommandInText("/routeclaude"), null); +}); + +test('detectRouteSlashCommandInText: "/route" with no target at all does not match', () => { + assert.equal(detectRouteSlashCommandInText("/route"), null); +}); + +test('detectRouteSlashCommandInText: "/route" followed by only whitespace does not match', () => { + assert.equal(detectRouteSlashCommandInText("/route "), null); +}); + +// ── full pipeline: only the LAST user message counts ──────────────────────── + +test("detectRouteSlashCommandInBody: only the last role=user message is scanned, not assistant/system/earlier turns", () => { + const body = { + messages: [ + { role: "system", content: "/route claude" }, + { role: "user", content: "primeiro turno" }, + { role: "assistant", content: "/route claude (assistant said this, must not count)" }, + { role: "user", content: "/route astra\nreview this" }, + ], + }; + assert.deepEqual(detectRouteSlashCommandInBody(body), { + target: "astra", + strippedTaskContent: "review this", + }); +}); + +test("detectRouteSlashCommandInBody: no slash command in the last user message returns null", () => { + assert.equal(detectRouteSlashCommandInBody(bodyWithUser("just a normal message")), null); +}); + +test("detectRouteSlashCommandInBody: content-parts array input is scanned the same way", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "/route claude" }, + { type: "input_text", text: "review this" }, + ], + }, + ], + }; + assert.deepEqual(detectRouteSlashCommandInBody(body), { + target: "claude", + strippedTaskContent: "review this", + }); +});