From b2ac2b141da697323b2434f7ef682f759af49f13 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sat, 3 Oct 2026 23:23:03 -0400 Subject: [PATCH 1/2] fix(combo): lock a model after a local target timeout A combo_target_timeout is request-scoped so it does not trip the provider breaker. The lockout checks used that same flag, so the timed-out model was tried again on the next request. Record the model lockout for that code only. Other request-scoped failures stay unlocked. Related to #15414. Signed-off-by: Minxi Hou --- open-sse/services/combo/comboPredicates.ts | 23 +++++++++++++ .../services/combo/executeTargetAttempt.ts | 10 ++++-- .../combo-target-timeout-standards.test.ts | 34 +++++++++++++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/open-sse/services/combo/comboPredicates.ts b/open-sse/services/combo/comboPredicates.ts index b7bec951f36d..5d855cec8f10 100644 --- a/open-sse/services/combo/comboPredicates.ts +++ b/open-sse/services/combo/comboPredicates.ts @@ -323,6 +323,29 @@ export function isRequestScopedUpstreamFailure(error?: { ); } +export function isComboTargetTimeoutFailure(error?: { + code?: string | null; + type?: string | null; +}): boolean { + const code = typeof error?.code === "string" ? error.code.toLowerCase() : ""; + const type = typeof error?.type === "string" ? error.type.toLowerCase() : ""; + return code === "combo_target_timeout" || type === "combo_target_timeout"; +} + +/** + * Model lockout is per model, not per provider. A local target timeout must + * lock that model so the next request does not spend another gate wait on it. + * Other request-scoped failures (context length, local queue) stay unlocked. + * The provider breaker still uses the request-scoped flag and does not see this. + */ +export function shouldRecordModelLockoutForComboFailure( + requestScopedFailure: boolean, + error?: { code?: string | null; type?: string | null } +): boolean { + if (!requestScopedFailure) return true; + return isComboTargetTimeoutFailure(error); +} + /** Request-scoped classification that also has access to the HTTP body. */ export function isComboRequestScopedFailure( response: Response, diff --git a/open-sse/services/combo/executeTargetAttempt.ts b/open-sse/services/combo/executeTargetAttempt.ts index 91e73052280b..b752f5faf064 100644 --- a/open-sse/services/combo/executeTargetAttempt.ts +++ b/open-sse/services/combo/executeTargetAttempt.ts @@ -60,6 +60,7 @@ import { shouldSkipForPredictedTtft, shouldRecordProviderBreakerFailure, isComboRequestScopedFailure as isScopedFailure, + shouldRecordModelLockoutForComboFailure, isStreamReadinessFailureErrorBody, isStreamEarlyEofErrorBody, isLocalKeyPolicyBreachErrorBody, @@ -1077,7 +1078,7 @@ export async function executeTargetAttempt(opts: { provider && rawModel && retry === 0 && - !scopedFailure && + shouldRecordModelLockoutForComboFailure(scopedFailure, structuredError) && !isConnectionScopedClaudeQuota ) { const mlSettings = resolveModelLockoutSettings(deps.settings); @@ -1167,7 +1168,12 @@ export async function executeTargetAttempt(opts: { if (i > 0) state.fallbackCount++; // Wire combo failures into the resilience dashboard (model-level lockout) // alongside the provider-level cooldown below — they govern different scopes. - if (provider && rawModel && !scopedFailure && !isConnectionScopedClaudeQuota) { + if ( + provider && + rawModel && + shouldRecordModelLockoutForComboFailure(scopedFailure, structuredError) && + !isConnectionScopedClaudeQuota + ) { const mlSettings = resolveModelLockoutSettings(deps.settings); if (mlSettings.enabled && mlSettings.errorCodes.includes(result.status)) { recordModelLockoutFailure( diff --git a/tests/unit/combo/combo-target-timeout-standards.test.ts b/tests/unit/combo/combo-target-timeout-standards.test.ts index 52c832251994..95ae07a0e7d0 100644 --- a/tests/unit/combo/combo-target-timeout-standards.test.ts +++ b/tests/unit/combo/combo-target-timeout-standards.test.ts @@ -20,8 +20,11 @@ process.env.DATA_DIR = TEST_DATA_DIR; process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "combo-target-timeout-std-secret"; const { handleComboChat } = await import("../../../open-sse/services/combo.ts"); -const { isComboRequestScopedFailure, shouldRecordProviderBreakerFailure } = - await import("../../../open-sse/services/combo/comboPredicates.ts"); +const { + isComboRequestScopedFailure, + shouldRecordProviderBreakerFailure, + shouldRecordModelLockoutForComboFailure, +} = await import("../../../open-sse/services/combo/comboPredicates.ts"); const { applyComboTargetExhaustion } = await import("../../../open-sse/services/combo/targetExhaustion.ts"); const { getProviderBreakerState } = await import("../../../open-sse/services/accountFallback.ts"); @@ -97,6 +100,33 @@ test.beforeEach(() => { // ── Decision seam: breaker + request-scoped classification ────────────────── +test("decision seam: typed combo_target_timeout still records a model lockout", () => { + const scoped = isComboRequestScopedFailure( + new Response("timed out", { status: 504 }), + "Model openai/slow timed out", + { code: "combo_target_timeout", type: "combo_target_timeout" } + ); + assert.equal(scoped, true, "timeout stays request-scoped for the provider breaker"); + assert.equal( + shouldRecordModelLockoutForComboFailure(scoped, { + code: "combo_target_timeout", + type: "combo_target_timeout", + }), + true, + "a locally timed-out model must be locked so the next request does not wait on it again" + ); +}); + +test("decision seam: other request-scoped failures still skip model lockout", () => { + assert.equal( + shouldRecordModelLockoutForComboFailure(true, { + code: "context_length_exceeded", + type: "invalid_request_error", + }), + false + ); +}); + test("decision seam: typed combo_target_timeout 504 is request-scoped and does not record breaker failure", () => { const decision = decideProviderBreakerRecord({ status: 504, From 96a041b3ca53ff0425bc23ddbeed19557a041dda Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Sat, 3 Oct 2026 23:38:17 -0400 Subject: [PATCH 2/2] docs(changelog): add the fragment for the target-timeout lockout The lockout change needs its own changelog fragment so release aggregation picks it up. Signed-off-by: Minxi Hou --- changelog.d/fixes/15494-combo-target-timeout-lockout.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fixes/15494-combo-target-timeout-lockout.md diff --git a/changelog.d/fixes/15494-combo-target-timeout-lockout.md b/changelog.d/fixes/15494-combo-target-timeout-lockout.md new file mode 100644 index 000000000000..92cbead5cc9f --- /dev/null +++ b/changelog.d/fixes/15494-combo-target-timeout-lockout.md @@ -0,0 +1 @@ +- **fix(combo):** a local per-target timeout records a model lockout without tripping the provider breaker ([#15494](https://github.com/diegosouzapw/OmniRoute/pull/15494) — thanks @HouMinXi)