From 9f474111fa36cf7b359529bdda3154646e0e5bc4 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Wed, 24 Jun 2026 16:59:35 -0300 Subject: [PATCH] feat(quota): combo quota-share espera cooldown curto e re-despacha (Variante A) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hoje um combo quota-share (qtSd/) de 1 conexão propaga 429 model_cooldown na hora quando o provider devolve um 429 transitório curto (reset em poucos segundos). Esta feature faz o combo ESPERAR o cooldown curto e RE-DESPACHAR, em vez de cristalizar o 429 — somente para combos quota-share, com guardas e configurável. O que muda: - Helper PURO e testável open-sse/services/combo/comboCooldownRetry.ts: shouldWaitForComboCooldown + resolveComboCooldownWaitDecision. Decide esperar SOMENTE se: feature ligada E reason transitório (rate_limit/...) E reason NAO em {quota_exhausted, auth_error, not_found, not_found_local} E waitMs em (0, maxWaitMs] E attempt < maxAttempts E budget suficiente. - Wrap em handleComboChat (combo.ts) gated em strategy==="quota-share" && comboCooldownWait.enabled. Para NAO re-indentar ~1600 linhas, o laço de set (setTry) fica na indentação original: ele é içado para uma closure recursiva dispatchWithCooldownRetry, e esperar+re-despachar é um tail `return dispatchWithCooldownRetry()` — re-roda SÓ o laço de set (mesma semântica do antigo continue-ao-topo-do-set-loop), sem refazer seleção/shadow-routing/setup. No ponto onde cristalizaria 429 (lastStatus===429 + earliestRetryAfter), resolve o reason via getModelLockoutInfo, decide via o helper, e se esperar: await waitForCooldownAwareRetry(waitMs, signal) (499 no abort), abate o budget e recursa. globalAttempts persiste → MAX_GLOBAL_ATTEMPTS continua limitando. A espera ocorre fora de qualquer seção de semáforo (o caminho quota-share não usa semaphore.acquire). `git diff` == `git diff -w` em combo.ts (sem ruído de re-indentação). - Settings comboCooldownWait { enabled, maxWaitMs, maxAttempts, budgetMs } em src/lib/resilience/settings.ts (defaults conservadores: 5s / 2 / 8s), com parsing/validação/clamp, exposto na API /api/resilience (GET+PATCH), no schema Zod e num card mínimo em Settings > Resilience. Segurança: isRetryableModelLockoutReason (auth.ts:533) trata quota_exhausted como retryável (lock até meia-noite); o helper o EXCLUI explicitamente, e o teto pequeno de maxWaitMs é a 2a barreira. TDD: combo-cooldown-retry.test.ts (helper puro, todos os ramos + resolveComboCooldownWaitDecision), combo-quota-share-cooldown-wait.test.ts (integração via handleComboChat: rate_limit espera+recupera 200; quota_exhausted sem espera; abort=499; priority/disabled inalterados), resilience-settings-combo-cooldown-wait.test.ts (round-trip/clamp). file-size rebaselinado ao valor real do gate (combo.ts 3225->3293, ResilienceTab 983->1098). --- config/quality/file-size-baseline.json | 5 +- open-sse/services/combo.ts | 69 ++++- open-sse/services/combo/comboCooldownRetry.ts | 237 +++++++++++++++ .../settings/components/ResilienceTab.tsx | 131 +++++++++ src/app/api/resilience/route.ts | 11 +- src/lib/resilience/settings.ts | 56 ++++ src/shared/validation/schemas/settings.ts | 13 + tests/unit/combo-cooldown-retry.test.ts | 271 ++++++++++++++++++ .../combo-quota-share-cooldown-wait.test.ts | 234 +++++++++++++++ ...ience-settings-combo-cooldown-wait.test.ts | 102 +++++++ 10 files changed, 1124 insertions(+), 5 deletions(-) create mode 100644 open-sse/services/combo/comboCooldownRetry.ts create mode 100644 tests/unit/combo-cooldown-retry.test.ts create mode 100644 tests/unit/combo-quota-share-cooldown-wait.test.ts create mode 100644 tests/unit/resilience-settings-combo-cooldown-wait.test.ts diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 3760adaac99d..50dab3ac5d11 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -1,6 +1,7 @@ { "_comment": "Catraca de tamanho (check-file-size.mjs). frozen so pode encolher; arquivos novos <= cap. --update ratcheta.", "_rebaseline_2026_06_24_per_connection_max_concurrent": "Feature (per-connection max_concurrent enforcement, issue #9 follow-up) own growth: open-sse/services/combo.ts 3225->3238 (+13 net). Provider connections already carry a maxConcurrent ceiling (provider_connections.max_concurrent, migration 029, set in the UI/API/DB) but routing ignored it, so low-concurrency subscription accounts (GLM/MiniMax ~1) got flooded -> 429 + cooldown, worse under quota-share where keys share one account. The new wiring lives at the two combo dispatch chokepoints: (1) the strategy===\"quota-share\" branch resolves each target connection's cap and passes it into selectQuotaShareTarget so an at-cap connection is deprioritized (fail-open, never hard-blocked); (2) handleRoundRobinCombo resolves the target connection's cap and uses it as the semaphore maxConcurrency (fallback = combo-level concurrency). ALL extractable logic was moved into the new pure leaf open-sse/services/combo/concurrencyCaps.ts (resolveMaxConcurrentByConnection + makeConnectionConcurrencyResolver, DB reads deduped per connectionId, fail-open) and the pure effectiveMaxConcurrency resolver into combo/comboPredicates.ts; combo.ts keeps only the irreducible chokepoint wiring (a 5-line import + the two call-site bindings). Cohesive at the existing dispatch boundaries, not further extractable. Structural shrink of combo.ts tracked in #3501. Covered by tests/unit/quota-share-strategy.test.ts (per-connection maxConcurrent gating cases) + tests/unit/combo/effective-max-concurrency.test.ts.", + "_rebaseline_2026_06_24_combo_cooldown_wait_quota_share": "Feature quota-share combo cooldown-aware retry (Variante A) own growth: open-sse/services/combo.ts 3225->3293 (+68 = the cooldown-wait wrap inside handleComboChat's quota-share path. The existing setTry loop body is LEFT at its original indentation: instead of an outer `while (true)` (which would re-indent ~1600 lines and bloat the review), the setTry loop is hoisted into a small recursive closure `dispatchWithCooldownRetry`, and a wait+redispatch is a tail `return dispatchWithCooldownRetry()` — re-running ONLY the set loop (exactly the prior continue-to-top-of-set-loop semantics) while selection/shadow-routing/setup above stay untouched. At the 429 crystallization point the lock reason is resolved via getModelLockoutInfo, the decision via the new pure resolveComboCooldownWaitDecision, then await waitForCooldownAwareRetry (499 on abort), decrement the budget, recurse. Gated to strategy==='quota-share' && comboCooldownWait.enabled. `git diff` == `git diff -w` for combo.ts (zero re-indentation noise). The gating policy + reason resolution are extracted to the new pure leaf open-sse/services/combo/comboCooldownRetry.ts (1098 (+115 = the ComboCooldownWaitCard exposing enabled/maxWaitMs/maxAttempts/budgetMs in Settings > Resilience, mirroring WaitForCooldownCard; wired through GET/PATCH in src/app/api/resilience/route.ts + comboCooldownWaitSettingsSchema in src/shared/validation/schemas/settings.ts; new UI labels use the t(key)||English-fallback pattern, en-only). Structural shrink of combo.ts + ResilienceTab tracked in #3501.", "_rebaseline_2026_06_23_4774_combo_legacy_strip": "PR #4774 (KooshaPari, #4382 round-trip) own growth: src/app/(dashboard)/dashboard/combos/page.tsx 4434->4456 (+22 = the client-side LEGACY_COMBO_RESILIENCE_KEYS Set gains queueTimeoutMs + the 12 v3.8.31-era removed keys (queueDepth/fallbackDelayMs/handoffProviders/maxComboDepth/manifestRouting/complexityAwareRouting/pipeline_enabled/pipelineConcurrency/shadowRouting/evalRouting/resetAwareEnabled/resetAwareWindow) with explanatory comments, mirroring the server-side strip list in src/app/api/combos/[id]/route.ts so the modal never re-introduces removed keys on Save). Functional strip list, not a movable block; combos/page.tsx structural shrink tracked in #3501. Covered by tests/unit/combo-config.test.ts (auto-promote + passthrough + legacy-key round-trip).", "_rebaseline_2026_06_21_4421_node_lookup": "Issue #4421 own growth: src/lib/db/providers.ts 1050->1063 (+13 = resolveProviderNodeForConnection at the existing provider-node lookup — resolves a connection node by exact id OR the bare derived type when unambiguous, + import). Pure selection logic in the new src/lib/db/providerNodeSelect.ts (2289 (+10): #4530 só fiou maxCooldownMs nos 3 sites de combo.ts; os 4 sites de markAccountUnavailable (per-model quota, grok-web 403, per-model 403, local 404) nunca passavam o cap → resolvo mlSettings uma vez e passo maxCooldownMs em todos. tests/unit/db-core-init.test.ts 864->867 (+3): comentário explicando o cap intencional busy_timeout 5s->2s do v3.8.32. Wiring necessário ao chokepoint de lockout; não extraível. Coberto por model-lockout-max-cooldown.test.ts + db-core-init.test.ts. (Demais base-reds — áudio mp3 #912/#913 dedup de handler em geminiHelper.ts, e closure #3578 src/models/ no package.json files — não cresceram arquivo congelado.)", @@ -144,7 +145,7 @@ "_rebaseline_2026_06_24_headroom_strategy": "Headroom-aware connection selection (dario technique): combo.ts 3168->3180 (+12 = a new `else if (strategy === \"headroom\")` dispatch branch in handleComboChat that delegates to orderTargetsByHeadroom + its log line, plus the import). The actual logic lives OUT of the god-file: the pure ranker rankByHeadroom/computeHeadroom is the new leaf open-sse/services/combo/headroomRanking.ts (91 LOC, 3190 (+10 = one new `else if (strategy === \"quota-share\")` dispatch branch in handleComboChat that delegates 100% to selectQuotaShareTarget + its log line, plus the import). All the new logic lives OUT of the god-file in two new leaves under open-sse/services/combo/: quotaShareInflight.ts (in-flight counter with TTL/lease, ~150 LOC 3225 (+35) = one new `else if (strategy === \"task-aware\")` dispatch branch delegating 100% to selectTaskAwareTarget + its imports/log lines. All scoring/classification logic lives OUT of the god-file in the new leaf open-sse/services/taskAwareRouting.ts (553 LOC => { for (let setTry = 0; setTry <= maxSetRetries; setTry++) { // #1731: Per-set-iteration set of providers whose quota is fully exhausted. // Reset each retry so providers excluded in a previous attempt get another chance. @@ -2565,6 +2591,43 @@ export async function handleComboChat({ const msg = lastError || "All combo models unavailable"; if (earliestRetryAfter) { + // Quota-share cooldown-aware retry: instead of crystallizing the 429, + // wait out a SHORT transient cooldown and re-run the whole set loop. + // Guarded by the helper (quota_exhausted/auth/not-found excluded, + // ceiling, attempts, budget). MAX_GLOBAL_ATTEMPTS still bounds total + // dispatches. + if (comboCooldownWaitEnabled && status === 429) { + const decision = resolveComboCooldownWaitDecision({ + targets: orderedTargets, + earliestRetryAfter, + attempt: comboCooldownAttempt, + budgetLeftMs: comboCooldownBudgetLeftMs, + settings: resilienceSettings.comboCooldownWait, + lookupLock: (provider, connectionId) => { + const rawModel = parseModel(orderedTargets[0]?.modelStr ?? "").model || ""; + return getModelLockoutInfo(provider, connectionId, rawModel); + }, + computeWaitMs: (retryAfter) => computeClosestRetryAfter(retryAfter).waitMs, + }); + if (decision.wait) { + log.info( + "COMBO", + `Quota-share cooldown wait: ${msg} — waiting ${Math.ceil( + decision.waitMs / 1000 + )}s (reason=${decision.reason ?? "?"}) then retrying (attempt ${ + comboCooldownAttempt + 1 + }/${resilienceSettings.comboCooldownWait.maxAttempts})` + ); + const completed = await waitForCooldownAwareRetry(decision.waitMs, signal); + if (!completed) { + log.info("COMBO", "Quota-share cooldown wait aborted by client disconnect"); + return errorResponse(499, "Request aborted"); + } + comboCooldownAttempt += 1; + comboCooldownBudgetLeftMs = Math.max(0, comboCooldownBudgetLeftMs - decision.waitMs); + return dispatchWithCooldownRetry(); + } + } const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(earliestRetryAfter)); log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`); return unavailableResponse(status, msg, earliestRetryAfter, retryHuman); @@ -2578,6 +2641,10 @@ export async function handleComboChat({ } return errorResponse(503, "Combo routing completed without an upstream response"); + }; + + try { + return await dispatchWithCooldownRetry(); } finally { // G2: Clean up candidate registry to prevent unbounded memory growth. _unregisterExecutionCandidates(_registeredExecutionKeys); diff --git a/open-sse/services/combo/comboCooldownRetry.ts b/open-sse/services/combo/comboCooldownRetry.ts new file mode 100644 index 000000000000..775bae62caae --- /dev/null +++ b/open-sse/services/combo/comboCooldownRetry.ts @@ -0,0 +1,237 @@ +/** + * comboCooldownRetry.ts — pure decision helper for the quota-share combo + * cooldown-aware retry (Variante A). + * + * Problem it solves + * ----------------- + * In a single-connection quota-share combo (`qtSd/…`), when the upstream + * returns a SHORT transient 429 (reset in a few seconds), the combo loop + * treats the only target as locked-by-resilience, skips it, and crystallizes a + * 429 `model_cooldown` to the client immediately — even though waiting a couple + * of seconds would let the same connection succeed. This helper decides whether + * `handleComboChat` should WAIT for that short cooldown and re-dispatch instead + * of propagating the 429. + * + * Why a pure helper + * ----------------- + * The actual wait (`waitForCooldownAwareRetry`) and the lock lookup + * (`getModelLockoutInfo`) live in `handleComboChat`; only the *gating policy* + * lives here. Keeping the policy pure (no I/O, no clock) means every branch is + * unit-testable deterministically and the orchestration in combo.ts stays thin. + * + * SECURITY — `quota_exhausted` must be excluded + * --------------------------------------------- + * `isRetryableModelLockoutReason` (src/sse/services/auth.ts:533) considers + * `quota_exhausted` RETRYABLE — its NON_RETRYABLE set only holds + * `not_found`/`not_found_local`. But `recordModelLockoutFailure` + * (accountFallback.ts) locks a quota-exhausted model UNTIL MIDNIGHT. If this + * helper trusted that classifier, a combo would wait until midnight. So this + * helper uses its OWN allow-list semantics: a reason qualifies for a wait only + * when it is a known short/transient reason AND is not in the explicit + * non-retryable set below. The small `maxWaitMs` ceiling is the second barrier. + */ + +/** + * Reasons that must NEVER trigger a wait, regardless of the upstream-provided + * retry hint. `quota_exhausted` is the critical one (locked until midnight); + * the auth/not-found reasons can never be cured by waiting a few seconds. + */ +export const COMBO_COOLDOWN_NON_RETRYABLE_REASONS: ReadonlySet = new Set([ + "quota_exhausted", + "auth_error", + "not_found", + "not_found_local", +]); + +/** + * Reasons recognised as short/transient and therefore eligible for a wait. We + * use an allow-list (rather than "anything not in the deny-list") so an unknown + * or empty reason fails closed — only an explicit transient reason qualifies. + * These match the model-lockout reasons produced by classifyLockoutReason / + * recordModelLockoutFailure for rate-limit-class failures. + */ +export const COMBO_COOLDOWN_RETRYABLE_REASONS: ReadonlySet = new Set([ + "rate_limit", + "rate_limited", + "transient", + "overloaded", + "server_error", +]); + +export interface ComboCooldownWaitSettings { + /** Master switch — when false the helper always returns wait=false. */ + enabled: boolean; + /** Hard ceiling (ms) on a single wait. Inclusive upper bound. */ + maxWaitMs: number; + /** Maximum number of wait+redispatch attempts for one request. */ + maxAttempts: number; + /** Total wait budget (ms) for the request — combo.ts decrements it. */ + budgetMs: number; +} + +export interface ShouldWaitForComboCooldownInput { + /** Lock reason from getModelLockoutInfo (may be null/unknown). */ + reason: unknown; + /** Computed wait derived from the upstream retry-after hint (ms). */ + waitMs: unknown; + /** Zero-based count of waits already performed for this request. */ + attempt: number; + /** Remaining wait budget (ms) for this request. */ + budgetLeftMs: number; + settings: ComboCooldownWaitSettings; +} + +export interface ShouldWaitForComboCooldownResult { + wait: boolean; + /** + * The wait duration the caller should pass to waitForCooldownAwareRetry. It + * is surfaced even when `wait` is false (clamped to a finite >= 0 value) so + * the caller can log "would have waited Xms" without re-deriving it. + */ + waitMs: number; +} + +function toFiniteWaitMs(value: unknown): number { + if (typeof value === "number" && Number.isFinite(value) && value > 0) { + return value; + } + return 0; +} + +function isRetryableComboCooldownReason(reason: unknown): boolean { + if (typeof reason !== "string" || reason.length === 0) return false; + if (COMBO_COOLDOWN_NON_RETRYABLE_REASONS.has(reason)) return false; + return COMBO_COOLDOWN_RETRYABLE_REASONS.has(reason); +} + +/** + * Decide whether a quota-share combo should wait for a short cooldown and + * re-dispatch. Pure — no I/O, no clock. All inputs are explicit. + * + * Returns wait=true only when ALL hold: + * - settings.enabled + * - reason is a recognised transient reason AND not in the non-retryable set + * (quota_exhausted / auth_error / not_found / not_found_local) + * - waitMs is finite and 0 < waitMs <= settings.maxWaitMs + * - attempt < settings.maxAttempts + * - budgetLeftMs >= waitMs + */ +export function shouldWaitForComboCooldown( + input: ShouldWaitForComboCooldownInput +): ShouldWaitForComboCooldownResult { + const { reason, attempt, budgetLeftMs, settings } = input; + const waitMs = toFiniteWaitMs(input.waitMs); + + const wait = + settings.enabled === true && + isRetryableComboCooldownReason(reason) && + waitMs > 0 && + waitMs <= settings.maxWaitMs && + attempt < settings.maxAttempts && + budgetLeftMs >= waitMs; + + return { wait, waitMs }; +} + +/** Minimal lock shape resolved from getModelLockoutInfo. */ +export interface ComboCooldownLockInfo { + reason: unknown; + remainingMs: number; +} + +export interface ResolveComboCooldownDecisionInput { + /** Combo targets to inspect for an active short lock. */ + targets: ReadonlyArray<{ provider?: string | null; connectionId?: string | null }>; + /** Earliest retry-after the loop crystallized (string | number | Date | null). */ + earliestRetryAfter: unknown; + attempt: number; + budgetLeftMs: number; + settings: ComboCooldownWaitSettings; + /** Per-target lock lookup (getModelLockoutInfo bound to the request model). */ + lookupLock: (provider: string, connectionId: string) => ComboCooldownLockInfo | null; + /** Derives the wait (ms) from the retry-after hint (computeClosestRetryAfter). */ + computeWaitMs: (retryAfter: unknown) => number | null; +} + +export interface ResolveComboCooldownDecisionResult extends ShouldWaitForComboCooldownResult { + /** Reason that drove the decision (for logging); null when none resolved. */ + reason: string | null; +} + +/** + * Small safety margin (ms) added on top of the wait so the model lock is + * reliably EXPIRED when the loop re-checks `isModelLocked` after the wait (real + * clocks drift a few ms between locking and re-evaluation; under-waiting would + * make the 2nd pass skip the target again and crystallize a 503/429). + */ +export const COMBO_COOLDOWN_WAIT_MARGIN_MS = 50; + +/** + * Resolve, from the combo targets and the crystallized retry-after, whether the + * quota-share combo should wait and re-dispatch. Picks the locked target with + * the SMALLEST positive remaining cooldown (the soonest to recover) and uses its + * reason. The wait must last long enough to actually clear that lock, so it is + * `max(lock remainingMs, upstream retry-after hint) + margin` — the lock's own + * remaining time is authoritative (it already folds in a longer upstream hint + * via selectLockoutCooldownMs), and we honor an even longer hint if present. The + * ceiling check in shouldWaitForComboCooldown then gates on that real duration, + * so only genuinely SHORT cooldowns are waited. Pure given the injected + * `lookupLock`/`computeWaitMs` — no I/O, no clock — so it is unit-testable. + */ +export function resolveComboCooldownWaitDecision( + input: ResolveComboCooldownDecisionInput +): ResolveComboCooldownDecisionResult { + const { + targets, + earliestRetryAfter, + attempt, + budgetLeftMs, + settings, + lookupLock, + computeWaitMs, + } = input; + + // Short-circuit before any lookup when the feature is off. + if (!settings.enabled) return { wait: false, waitMs: 0, reason: null }; + + let best: { reason: unknown; remainingMs: number } | null = null; + for (const target of targets) { + const provider = typeof target.provider === "string" ? target.provider : ""; + if (!provider) continue; + const connectionId = typeof target.connectionId === "string" ? target.connectionId : ""; + const info = lookupLock(provider, connectionId); + if (!info) continue; + const remainingMs = + typeof info.remainingMs === "number" && Number.isFinite(info.remainingMs) + ? info.remainingMs + : 0; + if (remainingMs <= 0) continue; + if (!best || remainingMs < best.remainingMs) { + best = { reason: info.reason, remainingMs }; + } + } + + if (!best) return { wait: false, waitMs: 0, reason: null }; + + // Wait long enough to actually clear the lock: the larger of the lock's own + // remaining time and the upstream retry-after hint, plus a small margin. + const rawHintWaitMs = computeWaitMs(earliestRetryAfter); + const hintWaitMs = + typeof rawHintWaitMs === "number" && Number.isFinite(rawHintWaitMs) && rawHintWaitMs > 0 + ? rawHintWaitMs + : 0; + const waitMs = Math.max(best.remainingMs, hintWaitMs) + COMBO_COOLDOWN_WAIT_MARGIN_MS; + + const decision = shouldWaitForComboCooldown({ + reason: best.reason, + waitMs, + attempt, + budgetLeftMs, + settings, + }); + + return { + ...decision, + reason: typeof best.reason === "string" ? best.reason : null, + }; +} diff --git a/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx index fda5ce43bf98..6a66ffdcea7d 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ResilienceTab.tsx @@ -36,6 +36,13 @@ type WaitForCooldownSettings = { maxRetryWaitSec: number; }; +type ComboCooldownWaitSettings = { + enabled: boolean; + maxWaitMs: number; + maxAttempts: number; + budgetMs: number; +}; + type ProviderCooldownSettings = { enabled: boolean; minRetryCooldownMs: number; @@ -53,6 +60,7 @@ type ResilienceResponse = { apikey: ProviderBreakerProfileSettings; }; waitForCooldown: WaitForCooldownSettings; + comboCooldownWait: ComboCooldownWaitSettings; providerCooldown: ProviderCooldownSettings; }; @@ -722,6 +730,124 @@ function WaitForCooldownCard({ ); } +function ComboCooldownWaitCard({ + value, + onSave, + saving, +}: { + value: ComboCooldownWaitSettings; + onSave: (next: ComboCooldownWaitSettings) => Promise; + saving: boolean; +}) { + const t = useTranslations("settings"); + const [editing, setEditing] = useState(false); + const [draft, setDraft] = useState(value); + + useEffect(() => { + setDraft(value); + }, [value]); + + const title = t("resilienceComboCooldownWaitTitle") || "Quota-share combo cooldown wait"; + const desc = + t("resilienceComboCooldownWaitDesc") || + "For quota-share combos only: wait out a short transient cooldown and re-dispatch instead of returning a 429 immediately. Never waits on quota_exhausted."; + + return ( + +
+
+ timer +

{title}

+
+ setEditing(true)} + onCancel={() => { + setDraft(value); + setEditing(false); + }} + onSave={async () => { + await onSave(draft); + setEditing(false); + }} + /> +
+ +

{desc}

+ +
+ {editing ? ( + <> + setDraft((prev) => ({ ...prev, enabled }))} + /> + setDraft((prev) => ({ ...prev, maxWaitMs }))} + /> + setDraft((prev) => ({ ...prev, maxAttempts }))} + /> + setDraft((prev) => ({ ...prev, budgetMs }))} + /> + + ) : ( + <> +
+
+ {t("resilienceEnableServerWait") || "Enabled"} +
+
+ {value.enabled ? t("statusEnabled") : t("statusDisabled")} +
+
+
+
+ {t("resilienceComboCooldownMaxWaitMs") || "Max wait per attempt"} +
+
+ {formatMs(value.maxWaitMs)} +
+
+
+
+ {t("resilienceMaxAttempts") || "Max attempts"} +
+
{value.maxAttempts}
+
+
+
+ {t("resilienceComboCooldownBudgetMs") || "Total wait budget"} +
+
+ {formatMs(value.budgetMs)} +
+
+ + )} +
+ + ); +} + function ProviderCooldownCard({ value, onSave, @@ -955,6 +1081,11 @@ export default function ResilienceTab() { saving={savingSection === "waitForCooldown"} onSave={(waitForCooldown) => savePatch("waitForCooldown", { waitForCooldown })} /> + savePatch("comboCooldownWait", { comboCooldownWait })} + /> ; providerBreaker: Record; waitForCooldown: WaitForCooldownSettings; + comboCooldownWait: ComboCooldownWaitSettings; providerCooldown: ProviderCooldownSettings; quotaPreflight: QuotaPreflightSettings; streamRecovery: StreamRecoverySettings; @@ -135,6 +151,7 @@ export interface ResilienceSettingsPatch { connectionCooldown?: Partial>>; providerBreaker?: Partial>>; waitForCooldown?: Partial; + comboCooldownWait?: Partial; providerCooldown?: Partial; quotaPreflight?: Partial; streamRecovery?: Partial; @@ -246,6 +263,15 @@ export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = { maxRetryWaitSec: 30, maxRetryWaitMs: 30000, }, + // Conservative defaults: wait at most 5s for a single short transient + // cooldown, at most 2 redispatch cycles, never more than 8s total. Active only + // for quota-share combos and only for transient (non quota_exhausted) reasons. + comboCooldownWait: { + enabled: true, + maxWaitMs: 5000, + maxAttempts: 2, + budgetMs: 8000, + }, providerCooldown: { minRetryCooldownMs: Number(process.env.PROVIDER_COOLDOWN_MIN_MS || "5000"), maxRetryCooldownMs: Number(process.env.PROVIDER_COOLDOWN_MAX_MS || "300000"), @@ -503,6 +529,27 @@ function normalizeWaitForCooldownSettings( }; } +function normalizeComboCooldownWaitSettings( + next: unknown, + fallback: ComboCooldownWaitSettings +): ComboCooldownWaitSettings { + const record = asRecord(next); + // Hard ceiling of 30s on a single wait — this layer only ever exists for + // SHORT transient cooldowns; anything longer should fall through to the + // existing 429 crystallization (and the cross-request cooldown layers). + const maxWaitMs = toInteger(record.maxWaitMs, fallback.maxWaitMs, { min: 0, max: 30000 }); + const maxAttempts = toInteger(record.maxAttempts, fallback.maxAttempts, { min: 0, max: 10 }); + // Budget can never be smaller than a single wait, otherwise no wait could + // ever fire; floor it at maxWaitMs. + const budgetMs = toInteger(record.budgetMs, fallback.budgetMs, { + min: maxWaitMs, + max: 5 * 60 * 1000, + }); + const enabled = toBoolean(record.enabled, fallback.enabled) && maxWaitMs > 0 && maxAttempts > 0; + + return { enabled, maxWaitMs, maxAttempts, budgetMs }; +} + function normalizeProviderCooldownSettings( next: unknown, fallback: ProviderCooldownSettings @@ -617,6 +664,7 @@ function buildLegacyFallback(settings: JsonRecord): ResilienceSettings { maxRetryWaitSec: waitMaxRetrySec, maxRetryWaitMs: waitMaxRetrySec * 1000, }, + comboCooldownWait: DEFAULT_RESILIENCE_SETTINGS.comboCooldownWait, providerCooldown: DEFAULT_RESILIENCE_SETTINGS.providerCooldown, quotaPreflight: DEFAULT_RESILIENCE_SETTINGS.quotaPreflight, streamRecovery: streamRecoveryDefaults, @@ -656,6 +704,10 @@ export function resolveResilienceSettings( current.waitForCooldown, fallback.waitForCooldown ), + comboCooldownWait: normalizeComboCooldownWaitSettings( + current.comboCooldownWait, + fallback.comboCooldownWait + ), providerCooldown: normalizeProviderCooldownSettings( current.providerCooldown, fallback.providerCooldown @@ -701,6 +753,10 @@ export function mergeResilienceSettings( updates.waitForCooldown, current.waitForCooldown ), + comboCooldownWait: normalizeComboCooldownWaitSettings( + updates.comboCooldownWait, + current.comboCooldownWait + ), providerCooldown: normalizeProviderCooldownSettings( updates.providerCooldown, current.providerCooldown diff --git a/src/shared/validation/schemas/settings.ts b/src/shared/validation/schemas/settings.ts index 75e813c93b36..04c6097b05a3 100644 --- a/src/shared/validation/schemas/settings.ts +++ b/src/shared/validation/schemas/settings.ts @@ -95,6 +95,17 @@ export const waitForCooldownSettingsSchema = z }) .strict(); +// Quota-share combo cooldown-aware retry (Variante A). Bounds mirror +// normalizeComboCooldownWaitSettings: a single wait <= 30s, <= 10 attempts. +export const comboCooldownWaitSettingsSchema = z + .object({ + enabled: z.boolean().optional(), + maxWaitMs: z.number().int().min(0).max(30000).optional(), + maxAttempts: z.number().int().min(0).max(10).optional(), + budgetMs: z.number().int().min(0).max(300000).optional(), + }) + .strict(); + export const providerCooldownSettingsSchema = z .object({ enabled: z.boolean().optional(), @@ -134,6 +145,7 @@ export const updateResilienceSchema = z .strict() .optional(), waitForCooldown: waitForCooldownSettingsSchema.optional(), + comboCooldownWait: comboCooldownWaitSettingsSchema.optional(), providerCooldown: providerCooldownSettingsSchema.optional(), profiles: z .object({ @@ -151,6 +163,7 @@ export const updateResilienceSchema = z !value.connectionCooldown && !value.providerBreaker && !value.waitForCooldown && + !value.comboCooldownWait && !value.providerCooldown && !value.profiles && !value.defaults diff --git a/tests/unit/combo-cooldown-retry.test.ts b/tests/unit/combo-cooldown-retry.test.ts new file mode 100644 index 000000000000..91d0339b1b2b --- /dev/null +++ b/tests/unit/combo-cooldown-retry.test.ts @@ -0,0 +1,271 @@ +/** + * TDD — shouldWaitForComboCooldown (pure decision helper for the quota-share + * combo cooldown-aware retry, Variante A). + * + * The helper decides whether a quota-share (qtSd/) combo that would otherwise + * crystallize a 429 `model_cooldown` should instead WAIT for a short transient + * cooldown and re-dispatch. It is pure (no I/O, no clock) so every branch is + * exhaustively covered here: + * - happy path (short transient cooldown within budget → wait) + * - reason === "quota_exhausted" EXCLUDED (would otherwise wait until midnight) + * - reason in the extra non-retryable set (auth_error/not_found/...) EXCLUDED + * - waitMs above the configured ceiling → no wait + * - waitMs <= 0 → no wait + * - attempt >= maxAttempts → no wait + * - budget insufficient (budgetLeftMs < waitMs) → no wait + * - settings.enabled === false → no wait + * + * Security note (auth.ts:533 isRetryableModelLockoutReason): `quota_exhausted` + * is classified RETRYABLE there, so the helper MUST exclude it explicitly or a + * combo would wait until midnight. This file pins that exclusion. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { shouldWaitForComboCooldown, resolveComboCooldownWaitDecision, COMBO_COOLDOWN_WAIT_MARGIN_MS } = + await import("../../open-sse/services/combo/comboCooldownRetry.ts"); + +function baseSettings(overrides: Partial> = {}) { + return { + enabled: true, + maxWaitMs: 5000, + maxAttempts: 2, + budgetMs: 8000, + ...overrides, + }; +} + +function baseInput(overrides: Record = {}) { + return { + reason: "rate_limit", + waitMs: 3000, + attempt: 0, + budgetLeftMs: 8000, + settings: baseSettings(), + ...overrides, + }; +} + +test("happy path: short transient cooldown within budget → wait=true", () => { + const r = shouldWaitForComboCooldown(baseInput() as never); + assert.equal(r.wait, true); + assert.equal(r.waitMs, 3000); +}); + +test("reason quota_exhausted is EXCLUDED even though it is otherwise 'retryable'", () => { + const r = shouldWaitForComboCooldown(baseInput({ reason: "quota_exhausted" }) as never); + assert.equal(r.wait, false); +}); + +test("reason auth_error is EXCLUDED (non-retryable safety set)", () => { + const r = shouldWaitForComboCooldown(baseInput({ reason: "auth_error" }) as never); + assert.equal(r.wait, false); +}); + +test("reason not_found is EXCLUDED", () => { + const r = shouldWaitForComboCooldown(baseInput({ reason: "not_found" }) as never); + assert.equal(r.wait, false); +}); + +test("reason not_found_local is EXCLUDED", () => { + const r = shouldWaitForComboCooldown(baseInput({ reason: "not_found_local" }) as never); + assert.equal(r.wait, false); +}); + +test("missing/unknown reason (null) → no wait (only an explicit transient reason qualifies)", () => { + const r = shouldWaitForComboCooldown(baseInput({ reason: null }) as never); + assert.equal(r.wait, false); +}); + +test("waitMs above the configured ceiling → no wait", () => { + const r = shouldWaitForComboCooldown( + baseInput({ waitMs: 5001, settings: baseSettings({ maxWaitMs: 5000 }) }) as never + ); + assert.equal(r.wait, false); + // waitMs is still surfaced for logging even when we decline to wait. + assert.equal(r.waitMs, 5001); +}); + +test("waitMs exactly at the ceiling → wait (inclusive bound)", () => { + const r = shouldWaitForComboCooldown( + baseInput({ waitMs: 5000, settings: baseSettings({ maxWaitMs: 5000 }) }) as never + ); + assert.equal(r.wait, true); +}); + +test("waitMs <= 0 → no wait", () => { + assert.equal(shouldWaitForComboCooldown(baseInput({ waitMs: 0 }) as never).wait, false); + assert.equal(shouldWaitForComboCooldown(baseInput({ waitMs: -10 }) as never).wait, false); +}); + +test("attempt >= maxAttempts → no wait", () => { + const r = shouldWaitForComboCooldown( + baseInput({ attempt: 2, settings: baseSettings({ maxAttempts: 2 }) }) as never + ); + assert.equal(r.wait, false); +}); + +test("attempt below maxAttempts (last allowed) → wait", () => { + const r = shouldWaitForComboCooldown( + baseInput({ attempt: 1, settings: baseSettings({ maxAttempts: 2 }) }) as never + ); + assert.equal(r.wait, true); +}); + +test("budget insufficient (budgetLeftMs < waitMs) → no wait", () => { + const r = shouldWaitForComboCooldown(baseInput({ waitMs: 3000, budgetLeftMs: 2999 }) as never); + assert.equal(r.wait, false); +}); + +test("budget exactly equal to waitMs → wait (inclusive bound)", () => { + const r = shouldWaitForComboCooldown(baseInput({ waitMs: 3000, budgetLeftMs: 3000 }) as never); + assert.equal(r.wait, true); +}); + +test("settings.enabled === false → no wait", () => { + const r = shouldWaitForComboCooldown( + baseInput({ settings: baseSettings({ enabled: false }) }) as never + ); + assert.equal(r.wait, false); +}); + +test("non-finite / garbage waitMs → no wait (defensive)", () => { + assert.equal(shouldWaitForComboCooldown(baseInput({ waitMs: Number.NaN }) as never).wait, false); + assert.equal( + shouldWaitForComboCooldown(baseInput({ waitMs: Number.POSITIVE_INFINITY }) as never).wait, + false + ); +}); + +test("returned waitMs is clamped to a finite number (0 when input invalid)", () => { + const r = shouldWaitForComboCooldown(baseInput({ waitMs: Number.NaN }) as never); + assert.equal(Number.isFinite(r.waitMs), true); + assert.equal(r.waitMs, 0); +}); + +// ── resolveComboCooldownWaitDecision (target resolution + hint/fallback) ────── + +const M = COMBO_COOLDOWN_WAIT_MARGIN_MS; + +function decisionInput(overrides: Record = {}) { + return { + targets: [{ provider: "openai", connectionId: "conn-1" }], + earliestRetryAfter: new Date(Date.now() + 2000).toISOString(), + attempt: 0, + budgetLeftMs: 8000, + settings: baseSettings(), + lookupLock: (_p: string, _c: string) => ({ reason: "rate_limit", remainingMs: 2000 }), + computeWaitMs: (_r: unknown) => 2000, + ...overrides, + }; +} + +test("resolve: locked target with transient reason within budget → wait", () => { + const r = resolveComboCooldownWaitDecision(decisionInput() as never); + assert.equal(r.wait, true); + // wait = max(lock remaining, hint) + margin so the lock reliably clears. + assert.equal(r.waitMs, 2000 + M); + assert.equal(r.reason, "rate_limit"); +}); + +test("resolve: quota_exhausted lock → no wait (the critical security guard)", () => { + const r = resolveComboCooldownWaitDecision( + decisionInput({ + lookupLock: () => ({ reason: "quota_exhausted", remainingMs: 3000 }), + }) as never + ); + assert.equal(r.wait, false); +}); + +test("resolve: settings.enabled false short-circuits before any lookup", () => { + let lookups = 0; + const r = resolveComboCooldownWaitDecision( + decisionInput({ + settings: baseSettings({ enabled: false }), + lookupLock: () => { + lookups += 1; + return { reason: "rate_limit", remainingMs: 3000 }; + }, + }) as never + ); + assert.equal(r.wait, false); + assert.equal(lookups, 0); +}); + +test("resolve: no locked target → no wait", () => { + const r = resolveComboCooldownWaitDecision(decisionInput({ lookupLock: () => null }) as never); + assert.equal(r.wait, false); + assert.equal(r.reason, null); +}); + +test("resolve: expired lock (remainingMs <= 0) is ignored → no wait", () => { + const r = resolveComboCooldownWaitDecision( + decisionInput({ lookupLock: () => ({ reason: "rate_limit", remainingMs: 0 }) }) as never + ); + assert.equal(r.wait, false); +}); + +test("resolve: picks the soonest-to-recover locked target across many", () => { + const r = resolveComboCooldownWaitDecision( + decisionInput({ + targets: [ + { provider: "openai", connectionId: "a" }, + { provider: "openai", connectionId: "b" }, + ], + // 'a' recovers in 3s, 'b' in 2s → 'b' wins; both rate_limit so 'b' (2s) + // drives the wait. No usable hint → wait = lock remaining + margin. + lookupLock: (_p: string, c: string) => + c === "b" + ? { reason: "rate_limit", remainingMs: 2000 } + : { reason: "rate_limit", remainingMs: 3000 }, + computeWaitMs: () => null, + }) as never + ); + assert.equal(r.wait, true); + assert.equal(r.waitMs, 2000 + M); +}); + +test("resolve: uses lock remainingMs (+margin) when the retry-after hint yields nothing", () => { + const r = resolveComboCooldownWaitDecision( + decisionInput({ + computeWaitMs: () => null, + lookupLock: () => ({ reason: "rate_limit", remainingMs: 1500 }), + }) as never + ); + assert.equal(r.wait, true); + assert.equal(r.waitMs, 1500 + M); +}); + +test("resolve: honors a LONGER upstream hint over the lock remaining (still capped)", () => { + // lock says 1s left but upstream hint says 3s → wait the longer 3s (+margin). + const r = resolveComboCooldownWaitDecision( + decisionInput({ + computeWaitMs: () => 3000, + lookupLock: () => ({ reason: "rate_limit", remainingMs: 1000 }), + }) as never + ); + assert.equal(r.wait, true); + assert.equal(r.waitMs, 3000 + M); +}); + +test("resolve: hint above the ceiling → no wait even if lock remaining is short", () => { + const r = resolveComboCooldownWaitDecision( + decisionInput({ + settings: baseSettings({ maxWaitMs: 5000 }), + computeWaitMs: () => 6000, + lookupLock: () => ({ reason: "rate_limit", remainingMs: 1000 }), + }) as never + ); + assert.equal(r.wait, false); +}); + +test("resolve: lock remaining above the ceiling → no wait (not a SHORT cooldown)", () => { + const r = resolveComboCooldownWaitDecision( + decisionInput({ + settings: baseSettings({ maxWaitMs: 5000 }), + computeWaitMs: () => null, + lookupLock: () => ({ reason: "rate_limit", remainingMs: 6000 }), + }) as never + ); + assert.equal(r.wait, false); +}); diff --git a/tests/unit/combo-quota-share-cooldown-wait.test.ts b/tests/unit/combo-quota-share-cooldown-wait.test.ts new file mode 100644 index 000000000000..f699759a6e4d --- /dev/null +++ b/tests/unit/combo-quota-share-cooldown-wait.test.ts @@ -0,0 +1,234 @@ +/** + * TDD (integration) — quota-share combo cooldown-aware retry (Variante A), + * exercised through handleComboChat. + * + * Scenarios (modelled on model-lockout-max-cooldown.test.ts): + * 1. strategy="quota-share", single connection, model hits a 429 with a SHORT + * retry-after → the combo WAITS out the cooldown and re-dispatches; the 2nd + * pass (lock expired) returns 200 instead of propagating the 429. + * 2. A 403 (quota_exhausted, locked until midnight) → NO wait, the 403/429 is + * propagated immediately (the helper's critical exclusion). + * 3. Client abort DURING the wait → 499 "Request aborted". + * 4. strategy="priority" (non quota-share) → unchanged: the 429 is propagated + * immediately with NO wait. + * 5. comboCooldownWait disabled in settings → unchanged: 429 propagated, no wait. + * + * The waits use a real (short) cooldown so the real setTimeout in + * waitForCooldownAwareRetry elapses fast and the model lock expires naturally. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omr-combo-cooldown-wait-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = "test-combo-cooldown-wait-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { clearAllModelLockouts } = await import("../../open-sse/services/accountFallback.ts"); + +function createLog() { + return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }; +} + +// A short transient cooldown so the real wait is fast but the lock genuinely +// expires between passes. The 429 carries a retry-after hint slightly LONGER +// than baseCooldownMs so waiting it out guarantees the lock has cleared. +const BASE_COOLDOWN_MS = 150; +const RETRY_AFTER_MS = 250; + +function shortModelLockoutSettings() { + return { + modelLockout: { + enabled: true, + errorCodes: [403, 429], + baseCooldownMs: BASE_COOLDOWN_MS, + maxCooldownMs: 5000, + maxBackoffSteps: 0, + useExponentialBackoff: false, + }, + }; +} + +function jsonResponse(status: number, body: Record) { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function okResponse() { + return jsonResponse(200, { id: "ok", choices: [{ message: { content: "recovered" } }] }); +} + +function rateLimitResponse(status: number) { + return jsonResponse(status, { + error: { message: `rate limited (${status})` }, + // string ISO retry-after → computeClosestRetryAfter yields ~RETRY_AFTER_MS + retryAfter: new Date(Date.now() + RETRY_AFTER_MS).toISOString(), + }); +} + +function comboOf(strategy: string) { + return { + name: `qtSd/${strategy}-${Math.random().toString(16).slice(2, 8)}`, + strategy, + models: ["openai/gpt-4"], + config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, maxSetRetries: 0 }, + }; +} + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(async () => { + clearAllModelLockouts(); + await resetStorage(); +}); + +test.after(async () => { + clearAllModelLockouts(); + try { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + } catch { + /* best effort */ + } +}); + +test("quota-share: short 429 cooldown → waits and re-dispatches (2nd pass 200)", async () => { + let calls = 0; + const handleSingleModel = async () => { + calls += 1; + // 1st dispatch: transient 429 (records a rate_limit lock). 2nd dispatch + // (after the wait, lock expired): success. + return calls === 1 ? rateLimitResponse(429) : okResponse(); + }; + + const startedAt = Date.now(); + const res = await handleComboChat({ + body: { model: "openai/gpt-4" }, + combo: comboOf("quota-share"), + handleSingleModel, + isModelAvailable: async () => true, + log: createLog() as never, + settings: shortModelLockoutSettings(), + allCombos: null, + }); + const elapsed = Date.now() - startedAt; + + assert.equal(res.status, 200, "expected the retried dispatch to succeed with 200"); + assert.equal(calls, 2, "expected exactly one wait+redispatch (2 upstream calls)"); + assert.ok( + elapsed >= BASE_COOLDOWN_MS, + `expected to have waited out the cooldown, only ${elapsed}ms elapsed` + ); +}); + +test("quota-share: 403 quota_exhausted → NO wait, error propagated immediately", async () => { + let calls = 0; + const handleSingleModel = async () => { + calls += 1; + return rateLimitResponse(403); + }; + + const startedAt = Date.now(); + const res = await handleComboChat({ + body: { model: "openai/gpt-4" }, + combo: comboOf("quota-share"), + handleSingleModel, + isModelAvailable: async () => true, + log: createLog() as never, + settings: shortModelLockoutSettings(), + allCombos: null, + }); + const elapsed = Date.now() - startedAt; + + assert.notEqual(res.status, 200, "quota_exhausted must not be retried into a success"); + // The real signal that the cooldown wait did NOT fire: a single upstream + // dispatch (no redispatch). The 403 lock cooldown is multi-second, so the + // wait — had it fired — would dominate the elapsed time; assert we stayed far + // below that (loose bound; the first combo dispatch pays DB/import overhead). + assert.equal(calls, 1, "quota_exhausted must NOT trigger a wait+redispatch"); + assert.ok(elapsed < 1500, `quota_exhausted must not wait out a cooldown, but ${elapsed}ms elapsed`); +}); + +test("quota-share: client abort during the wait → 499", async () => { + const controller = new AbortController(); + let calls = 0; + const handleSingleModel = async () => { + calls += 1; + // Always 429 so the loop reaches the wait; abort fires during the wait. + return rateLimitResponse(429); + }; + + // Abort shortly after the request starts — within the cooldown wait window. + setTimeout(() => controller.abort(), 50); + + const res = await handleComboChat({ + body: { model: "openai/gpt-4" }, + combo: comboOf("quota-share"), + handleSingleModel, + isModelAvailable: async () => true, + log: createLog() as never, + settings: shortModelLockoutSettings(), + allCombos: null, + signal: controller.signal, + }); + + assert.equal(res.status, 499, "abort during the cooldown wait must return 499"); +}); + +test("non quota-share (priority): 429 propagated immediately, NO wait", async () => { + let calls = 0; + const handleSingleModel = async () => { + calls += 1; + return rateLimitResponse(429); + }; + + const startedAt = Date.now(); + const res = await handleComboChat({ + body: { model: "openai/gpt-4" }, + combo: { ...comboOf("priority"), name: "priority-combo" }, + handleSingleModel, + isModelAvailable: async () => true, + log: createLog() as never, + settings: shortModelLockoutSettings(), + allCombos: null, + }); + const elapsed = Date.now() - startedAt; + + assert.equal(res.status, 429, "priority combo must propagate the 429 unchanged"); + assert.equal(calls, 1, "priority combo must NOT wait+redispatch"); + assert.ok(elapsed < 1500, `priority combo must not wait out a cooldown, but ${elapsed}ms elapsed`); +}); + +test("quota-share with comboCooldownWait disabled → 429 propagated, NO wait", async () => { + let calls = 0; + const handleSingleModel = async () => { + calls += 1; + return rateLimitResponse(429); + }; + + const res = await handleComboChat({ + body: { model: "openai/gpt-4" }, + combo: comboOf("quota-share"), + handleSingleModel, + isModelAvailable: async () => true, + log: createLog() as never, + settings: { + ...shortModelLockoutSettings(), + resilienceSettings: { comboCooldownWait: { enabled: false } }, + }, + allCombos: null, + }); + + assert.equal(res.status, 429, "disabled feature must propagate the 429 unchanged"); + assert.equal(calls, 1, "disabled feature must NOT wait+redispatch"); +}); diff --git a/tests/unit/resilience-settings-combo-cooldown-wait.test.ts b/tests/unit/resilience-settings-combo-cooldown-wait.test.ts new file mode 100644 index 000000000000..63bfab127c9b --- /dev/null +++ b/tests/unit/resilience-settings-combo-cooldown-wait.test.ts @@ -0,0 +1,102 @@ +/** + * TDD — ResilienceSettings.comboCooldownWait (quota-share combo cooldown-aware + * retry, Variante A). Locks the defaults, the resolve/merge round-trip and the + * clamping/validation contract of normalizeComboCooldownWaitSettings. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + DEFAULT_RESILIENCE_SETTINGS, + mergeResilienceSettings, + resolveResilienceSettings, + type ResilienceSettings, +} from "../../src/lib/resilience/settings.ts"; + +function cloneDefaults(): ResilienceSettings { + return structuredClone(DEFAULT_RESILIENCE_SETTINGS); +} + +test("default comboCooldownWait is conservative (on, 5s ceiling, 2 attempts, 8s budget)", () => { + const s = cloneDefaults().comboCooldownWait; + assert.equal(s.enabled, true); + assert.equal(s.maxWaitMs, 5000); + assert.equal(s.maxAttempts, 2); + assert.equal(s.budgetMs, 8000); +}); + +test("resolveResilienceSettings returns the default block when nothing is stored", () => { + const resolved = resolveResilienceSettings({}); + assert.deepEqual(resolved.comboCooldownWait, DEFAULT_RESILIENCE_SETTINGS.comboCooldownWait); +}); + +test("resolveResilienceSettings reads stored comboCooldownWait overrides", () => { + const resolved = resolveResilienceSettings({ + resilienceSettings: { + comboCooldownWait: { enabled: true, maxWaitMs: 3000, maxAttempts: 1, budgetMs: 4000 }, + }, + }); + assert.deepEqual(resolved.comboCooldownWait, { + enabled: true, + maxWaitMs: 3000, + maxAttempts: 1, + budgetMs: 4000, + }); +}); + +test("mergeResilienceSettings round-trips a partial patch", () => { + const merged = mergeResilienceSettings(cloneDefaults(), { + comboCooldownWait: { maxWaitMs: 2000 }, + }); + assert.equal(merged.comboCooldownWait.maxWaitMs, 2000); + // Unspecified fields keep the current value. + assert.equal(merged.comboCooldownWait.maxAttempts, 2); + assert.equal(merged.comboCooldownWait.budgetMs, 8000); + assert.equal(merged.comboCooldownWait.enabled, true); +}); + +test("enabled is forced false when maxWaitMs or maxAttempts is zero", () => { + const a = mergeResilienceSettings(cloneDefaults(), { + comboCooldownWait: { enabled: true, maxWaitMs: 0 }, + }); + assert.equal(a.comboCooldownWait.enabled, false); + + const b = mergeResilienceSettings(cloneDefaults(), { + comboCooldownWait: { enabled: true, maxAttempts: 0 }, + }); + assert.equal(b.comboCooldownWait.enabled, false); +}); + +test("maxWaitMs is clamped to the 30s hard ceiling", () => { + const merged = mergeResilienceSettings(cloneDefaults(), { + comboCooldownWait: { maxWaitMs: 999_999 }, + }); + assert.equal(merged.comboCooldownWait.maxWaitMs, 30000); +}); + +test("budgetMs can never drop below a single maxWaitMs", () => { + const merged = mergeResilienceSettings(cloneDefaults(), { + comboCooldownWait: { maxWaitMs: 5000, budgetMs: 100 }, + }); + // budget floored at maxWaitMs so at least one wait can fire. + assert.equal(merged.comboCooldownWait.budgetMs, 5000); +}); + +test("explicit disable is honored", () => { + const merged = mergeResilienceSettings(cloneDefaults(), { + comboCooldownWait: { enabled: false }, + }); + assert.equal(merged.comboCooldownWait.enabled, false); +}); + +test("garbage values fall back to the current numbers", () => { + const merged = mergeResilienceSettings(cloneDefaults(), { + comboCooldownWait: { + maxWaitMs: "not-a-number" as never, + maxAttempts: null as never, + budgetMs: undefined, + }, + }); + assert.equal(merged.comboCooldownWait.maxWaitMs, 5000); + assert.equal(merged.comboCooldownWait.maxAttempts, 2); + assert.equal(merged.comboCooldownWait.budgetMs, 8000); +});