From 5f3656a09c53200754beece254dea5e740479748 Mon Sep 17 00:00:00 2001 From: serenity-X <295272363+serenity-X@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:42:15 +0330 Subject: [PATCH] fix(sse): persist terminal account failures during combo fallback Keep exhausted account-scoped connections out of routing even when the combo owns transient 429 state. Preserve model-scoped passthrough and transient cooldown behavior. Refs #18633 --- ...8649-credit-exhaustion-parks-connection.md | 1 + src/sse/services/auth.ts | 3 +- ...credit-exhaustion-parks-connection.test.ts | 68 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixes/18649-credit-exhaustion-parks-connection.md create mode 100644 tests/unit/18633-credit-exhaustion-parks-connection.test.ts diff --git a/changelog.d/fixes/18649-credit-exhaustion-parks-connection.md b/changelog.d/fixes/18649-credit-exhaustion-parks-connection.md new file mode 100644 index 000000000000..869840072966 --- /dev/null +++ b/changelog.d/fixes/18649-credit-exhaustion-parks-connection.md @@ -0,0 +1 @@ +- **fix(sse):** a connection whose account runs out of credit is now parked as `credits_exhausted` even when the failure happens inside combo fallback — previously the combo path dropped the terminal status and the dead account kept receiving traffic (#18633) diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index b97c66fc7311..be62dc64ffe6 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -3152,7 +3152,8 @@ export async function markAccountUnavailable( }; const persistUnavailableState = options.persistUnavailableState !== false; - if (!persistUnavailableState) { + if (!persistUnavailableState && !terminalStatus) { + // Terminal account failures must persist even when a combo owns transient cooldowns. // Combo-managed transient failure (e.g. 429): keep the connection clean in // the DB, but record an in-memory model lockout so credential selection // skips this exact provider+connection+model while it cools down — other diff --git a/tests/unit/18633-credit-exhaustion-parks-connection.test.ts b/tests/unit/18633-credit-exhaustion-parks-connection.test.ts new file mode 100644 index 000000000000..c81c435b6203 --- /dev/null +++ b/tests/unit/18633-credit-exhaustion-parks-connection.test.ts @@ -0,0 +1,68 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-18633-credit-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); +const accountFallback = await import("../../open-sse/services/accountFallback.ts"); +const { classify429 } = await import("../../src/shared/utils/classify429.ts"); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("combo 429 insufficient account balance parks the zai connection across models", async () => { + const empty = await providersDb.createProviderConnection({ + provider: "zai", + authType: "apikey", + apiKey: "test-empty-account", + isActive: true, + testStatus: "active", + }); + const live = await providersDb.createProviderConnection({ + provider: "zai", + authType: "apikey", + apiKey: "test-live-account", + isActive: true, + testStatus: "active", + }); + const errorText = JSON.stringify({ + error: { code: "1113", message: "Insufficient account balance" }, + }); + const failureKind = classify429({ status: 429, body: errorText }); + + // Mirror chat.ts: combos do not persist transient 429 connection cooldowns. + const result = await auth.markAccountUnavailable( + empty.id, + 429, + errorText, + "zai", + "glm-4.7", + null, + { + isCombo: true, + persistUnavailableState: !(failureKind === "rate_limit" || failureKind === "transient"), + } + ); + + assert.equal(result.shouldFallback, true); + const after = await providersDb.getProviderConnectionById(empty.id); + assert.equal(after.testStatus, "credits_exhausted"); + assert.equal( + after.rateLimitedUntil == null, + true, + "terminal exhaustion must not expire as a cooldown" + ); + assert.equal(accountFallback.getModelLockoutInfo("zai", empty.id, "glm-4.7"), null); + + const selected = await auth.getProviderCredentials("zai", null, null, "glm-5.1"); + assert.ok(selected); + assert.equal(selected.connectionId, live.id, "sibling models must skip the empty account"); +});