From 036ae39fc71137019ca0109974d5013eb3f5d847 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 7 Sep 2026 12:52:19 -0400 Subject: [PATCH 1/8] fix(sse): export shared expired-reprobe blocklist The health-check path already skips real deactivations. The explicit-inactive probe will reuse the same Set rather than copying its members. Signed-off-by: Minxi Hou --- src/lib/quota/connectionRecovery.ts | 2 +- src/sse/services/explicitInactiveProbe.ts | 2 ++ tests/unit/explicit-inactive-probe-12874.test.ts | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 src/sse/services/explicitInactiveProbe.ts create mode 100644 tests/unit/explicit-inactive-probe-12874.test.ts diff --git a/src/lib/quota/connectionRecovery.ts b/src/lib/quota/connectionRecovery.ts index 26952a262dfb..ec23d8175da7 100644 --- a/src/lib/quota/connectionRecovery.ts +++ b/src/lib/quota/connectionRecovery.ts @@ -160,7 +160,7 @@ export function isRecoverableCooldownConnection( * Pure — `nowMs` and `reprobeMs` are injected so callers/tests control the clock. */ -const EXPIRED_REPROBE_BLOCKLIST = new Set([ +export const EXPIRED_REPROBE_BLOCKLIST = new Set([ "account_deactivated", "invalid_grant", "unrecoverable_refresh_error", diff --git a/src/sse/services/explicitInactiveProbe.ts b/src/sse/services/explicitInactiveProbe.ts new file mode 100644 index 000000000000..9aad084ee97a --- /dev/null +++ b/src/sse/services/explicitInactiveProbe.ts @@ -0,0 +1,2 @@ +import { EXPIRED_REPROBE_BLOCKLIST } from "@/lib/quota/connectionRecovery"; +export const EXPLICIT_PROBE_BLOCKLIST = EXPIRED_REPROBE_BLOCKLIST; diff --git a/tests/unit/explicit-inactive-probe-12874.test.ts b/tests/unit/explicit-inactive-probe-12874.test.ts new file mode 100644 index 000000000000..d0021c2d4470 --- /dev/null +++ b/tests/unit/explicit-inactive-probe-12874.test.ts @@ -0,0 +1,8 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { EXPIRED_REPROBE_BLOCKLIST } from "../../src/lib/quota/connectionRecovery.ts"; +import { EXPLICIT_PROBE_BLOCKLIST } from "../../src/sse/services/explicitInactiveProbe.ts"; + +test("P-14 EXPLICIT_PROBE_BLOCKLIST is the same object as EXPIRED_REPROBE_BLOCKLIST", () => { + assert.equal(EXPLICIT_PROBE_BLOCKLIST, EXPIRED_REPROBE_BLOCKLIST); +}); From 94a658faa7dadfe38079ebdc038fc57fef148be8 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 7 Sep 2026 20:45:16 -0400 Subject: [PATCH 2/8] fix(auth): classify explicit pin probes for inactive connections A pinned request may need to treat a quota-disabled row as recoverable. The classifier is a pure function so the pin path can decide probe vs skip without touching the live pool. Signed-off-by: Minxi Hou --- src/sse/services/explicitInactiveProbe.ts | 73 ++++++++++++++++ .../explicit-inactive-probe-12874.test.ts | 86 ++++++++++++++++++- 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/src/sse/services/explicitInactiveProbe.ts b/src/sse/services/explicitInactiveProbe.ts index 9aad084ee97a..3d5249d22ffe 100644 --- a/src/sse/services/explicitInactiveProbe.ts +++ b/src/sse/services/explicitInactiveProbe.ts @@ -1,2 +1,75 @@ import { EXPIRED_REPROBE_BLOCKLIST } from "@/lib/quota/connectionRecovery"; + export const EXPLICIT_PROBE_BLOCKLIST = EXPIRED_REPROBE_BLOCKLIST; + +export const RECOVERABLE_INACTIVE_TEST_STATUSES = new Set([ + "active", + "success", + "credits_exhausted", + "unavailable", + "error", + "", +]); + +export function isRecoverableInactiveConnection( + conn: { + isActive?: boolean; + testStatus?: string | null; + lastErrorType?: string | null; + rateLimitedUntil?: string | null; + }, + nowMs: number = Date.now() +): boolean { + if (conn.isActive !== false) return false; + const status = (conn.testStatus || "").trim().toLowerCase(); + if (status === "banned") return false; + const err = (conn.lastErrorType || "").trim().toLowerCase(); + if (EXPLICIT_PROBE_BLOCKLIST.has(err)) return false; + if (status === "expired") return true; + if (status === "unavailable") { + const until = conn.rateLimitedUntil; + if (until) { + const ms = Date.parse(until); + if (Number.isFinite(ms) && ms > nowMs) return false; + } + } + return RECOVERABLE_INACTIVE_TEST_STATUSES.has(status); +} + +export function selectExplicitInactiveProbe(params: { + forcedConnectionId: string | null; + activeConnections: { id: string }[]; + pinnedRow: { + id: string; + provider?: string | null; + isActive?: boolean; + testStatus?: string | null; + lastErrorType?: string | null; + rateLimitedUntil?: string | null; + } | null; + providersToSearch: string[]; + allowedConnectionIds: string[] | null; + nowMs: number; + lastProbeAtMs: number | null; + intervalMs: number; +}): { kind: "probe" } | { kind: "suppressed" } | { kind: "skip" } { + const id = params.forcedConnectionId; + if (!id) return { kind: "skip" }; + if (params.activeConnections.some((c) => c.id === id)) return { kind: "skip" }; + const row = params.pinnedRow; + if (!row || row.id !== id) return { kind: "skip" }; + if ( + params.allowedConnectionIds && + params.allowedConnectionIds.length > 0 && + !params.allowedConnectionIds.includes(id) + ) { + return { kind: "skip" }; + } + const prov = (row.provider || "").trim(); + if (prov && !params.providersToSearch.includes(prov)) return { kind: "skip" }; + if (!isRecoverableInactiveConnection(row, params.nowMs)) return { kind: "skip" }; + if (params.lastProbeAtMs != null && params.nowMs - params.lastProbeAtMs < params.intervalMs) { + return { kind: "suppressed" }; + } + return { kind: "probe" }; +} diff --git a/tests/unit/explicit-inactive-probe-12874.test.ts b/tests/unit/explicit-inactive-probe-12874.test.ts index d0021c2d4470..4c04cbe4978f 100644 --- a/tests/unit/explicit-inactive-probe-12874.test.ts +++ b/tests/unit/explicit-inactive-probe-12874.test.ts @@ -1,8 +1,92 @@ import test from "node:test"; import assert from "node:assert/strict"; import { EXPIRED_REPROBE_BLOCKLIST } from "../../src/lib/quota/connectionRecovery.ts"; -import { EXPLICIT_PROBE_BLOCKLIST } from "../../src/sse/services/explicitInactiveProbe.ts"; +import { + EXPLICIT_PROBE_BLOCKLIST, + selectExplicitInactiveProbe, +} from "../../src/sse/services/explicitInactiveProbe.ts"; + +const pin = { + id: "c1", + provider: "siliconflow", + isActive: false, + testStatus: "active", + lastErrorType: null, + rateLimitedUntil: null, +}; + +function select(overrides: Partial[0]> = {}) { + return selectExplicitInactiveProbe({ + forcedConnectionId: "c1", + activeConnections: [], + pinnedRow: pin, + providersToSearch: ["siliconflow"], + allowedConnectionIds: null, + nowMs: 1_000_000, + lastProbeAtMs: null, + intervalMs: 60_000, + ...overrides, + }); +} test("P-14 EXPLICIT_PROBE_BLOCKLIST is the same object as EXPIRED_REPROBE_BLOCKLIST", () => { assert.equal(EXPLICIT_PROBE_BLOCKLIST, EXPIRED_REPROBE_BLOCKLIST); }); + +test("P-1 pin + inactive active -> probe", () => { + assert.equal(select().kind, "probe"); +}); +test("P-2 credits_exhausted -> probe", () => { + assert.equal(select({ pinnedRow: { ...pin, testStatus: "credits_exhausted" } }).kind, "probe"); +}); +test("P-3 no pin -> skip", () => { + assert.equal(select({ forcedConnectionId: null }).kind, "skip"); +}); +test("P-4 live pool already has id -> skip", () => { + assert.equal(select({ activeConnections: [{ id: "c1" }] }).kind, "skip"); +}); +test("P-5 banned -> skip", () => { + assert.equal(select({ pinnedRow: { ...pin, testStatus: "banned" } }).kind, "skip"); +}); +test("P-6 expired + no_refresh_token -> skip", () => { + assert.equal( + select({ pinnedRow: { ...pin, testStatus: "expired", lastErrorType: "no_refresh_token" } }).kind, + "skip" + ); +}); +test("P-7 expired + empty lastErrorType -> probe", () => { + assert.equal(select({ pinnedRow: { ...pin, testStatus: "expired", lastErrorType: "" } }).kind, "probe"); +}); +test("P-8 error + unrecoverable_refresh_error -> skip", () => { + assert.equal( + select({ pinnedRow: { ...pin, testStatus: "error", lastErrorType: "unrecoverable_refresh_error" } }).kind, + "skip" + ); +}); +test("P-9 last probe within 60s -> suppressed", () => { + assert.equal(select({ lastProbeAtMs: 1_000_000 - 10_000 }).kind, "suppressed"); +}); +test("P-10 pin not in allowedConnectionIds -> skip", () => { + assert.equal(select({ allowedConnectionIds: ["other"] }).kind, "skip"); +}); +test("P-11 provider not in providersToSearch -> skip", () => { + assert.equal(select({ providersToSearch: ["openai"] }).kind, "skip"); +}); +test("P-12 unavailable + future cooldown -> skip", () => { + assert.equal( + select({ + pinnedRow: { ...pin, testStatus: "unavailable", rateLimitedUntil: new Date(2_000_000_000).toISOString() }, + nowMs: 1_000_000, + }).kind, + "skip" + ); +}); +test("P-13 unavailable + elapsed cooldown -> probe", () => { + assert.equal( + select({ + pinnedRow: { ...pin, testStatus: "unavailable", rateLimitedUntil: new Date(500_000).toISOString() }, + nowMs: 1_000_000, + }).kind, + "probe" + ); +}); From 8b394e4d20d9aa0a0e5a8ef6a80a63e91fd9a939 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 7 Sep 2026 21:06:11 -0400 Subject: [PATCH 3/8] fix(auth): add explicit-probe storm gate and reactivation helper Signed-off-by: Minxi Hou --- src/sse/services/explicitInactiveProbe.ts | 51 ++++++++ .../explicit-inactive-probe-12874.test.ts | 123 ++++++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/src/sse/services/explicitInactiveProbe.ts b/src/sse/services/explicitInactiveProbe.ts index 3d5249d22ffe..c6b787df8e50 100644 --- a/src/sse/services/explicitInactiveProbe.ts +++ b/src/sse/services/explicitInactiveProbe.ts @@ -1,3 +1,4 @@ +import { updateProviderConnection } from "@/lib/db/providers"; import { EXPIRED_REPROBE_BLOCKLIST } from "@/lib/quota/connectionRecovery"; export const EXPLICIT_PROBE_BLOCKLIST = EXPIRED_REPROBE_BLOCKLIST; @@ -73,3 +74,53 @@ export function selectExplicitInactiveProbe(params: { } return { kind: "probe" }; } + +export const EXPLICIT_INACTIVE_PROBE_INTERVAL_MS = 60_000; +const MAX_PROBE_MAP = 4096; +const lastExplicitProbeAtMs = new Map(); + +export function noteExplicitProbe(id: string, nowMs: number): void { + lastExplicitProbeAtMs.set(id, nowMs); + if (lastExplicitProbeAtMs.size > MAX_PROBE_MAP) { + const oldest = lastExplicitProbeAtMs.keys().next().value; + if (oldest !== undefined) lastExplicitProbeAtMs.delete(oldest); + } +} + +export function lastExplicitProbeTime(id: string): number | null { + return lastExplicitProbeAtMs.get(id) ?? null; +} + +export function resetExplicitProbeMapForTests(): void { + lastExplicitProbeAtMs.clear(); +} + +export async function reactivateRecoveredConnection(connectionId: string): Promise { + await updateProviderConnection(connectionId, { isActive: true }); +} + +export async function maybeReactivateAfterExplicitProbe( + input: { + connectionId: string; + reactivatedFromInactive?: boolean; + explicitProbeSuppressed?: boolean; + isShadowTraffic?: boolean; + allowSuppressedConnections?: boolean; + requestedModel?: string | null; + provider?: string | null; + }, + reactivate: (connectionId: string) => Promise = reactivateRecoveredConnection +): Promise { + if (!input.reactivatedFromInactive) return; + if (input.explicitProbeSuppressed) return; + if (input.isShadowTraffic) return; + if (input.allowSuppressedConnections) return; + if ( + input.provider === "openrouter" && + typeof input.requestedModel === "string" && + input.requestedModel.includes(":free") + ) { + return; + } + await reactivate(input.connectionId); +} diff --git a/tests/unit/explicit-inactive-probe-12874.test.ts b/tests/unit/explicit-inactive-probe-12874.test.ts index 4c04cbe4978f..89baeb94f15b 100644 --- a/tests/unit/explicit-inactive-probe-12874.test.ts +++ b/tests/unit/explicit-inactive-probe-12874.test.ts @@ -2,7 +2,12 @@ import test from "node:test"; import assert from "node:assert/strict"; import { EXPIRED_REPROBE_BLOCKLIST } from "../../src/lib/quota/connectionRecovery.ts"; import { + EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, EXPLICIT_PROBE_BLOCKLIST, + lastExplicitProbeTime, + maybeReactivateAfterExplicitProbe, + noteExplicitProbe, + resetExplicitProbeMapForTests, selectExplicitInactiveProbe, } from "../../src/sse/services/explicitInactiveProbe.ts"; @@ -90,3 +95,121 @@ test("P-13 unavailable + elapsed cooldown -> probe", () => { "probe" ); }); + +test("storm Map note then select within interval is suppressed", () => { + resetExplicitProbeMapForTests(); + noteExplicitProbe("c1", 1e6); + assert.equal(lastExplicitProbeTime("c1"), 1e6); + assert.equal( + select({ lastProbeAtMs: lastExplicitProbeTime("c1"), nowMs: 1e6 + 10_000 }).kind, + "suppressed" + ); +}); + +test("storm Map interval constant is 60s", () => { + assert.equal(EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, 60_000); +}); + +test("storm Map reset clears last probe time", () => { + resetExplicitProbeMapForTests(); + noteExplicitProbe("c1", 1e6); + resetExplicitProbeMapForTests(); + assert.equal(lastExplicitProbeTime("c1"), null); +}); + +test("storm Map evicts oldest when over 4096 entries", () => { + resetExplicitProbeMapForTests(); + for (let i = 0; i < 4096; i++) { + noteExplicitProbe(`id-${i}`, i); + } + noteExplicitProbe("overflow", 4096); + assert.equal(lastExplicitProbeTime("id-0"), null); + assert.equal(lastExplicitProbeTime("id-1"), 1); + assert.equal(lastExplicitProbeTime("overflow"), 4096); +}); + +test("W-6 openrouter :free does not reactivate", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + provider: "openrouter", + requestedModel: "openrouter/foo:free", + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe calls reactivate on recovered pin", async () => { + let called = 0; + let seenId = ""; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + }, + async (id) => { + called += 1; + seenId = id; + } + ); + assert.equal(called, 1); + assert.equal(seenId, "c1"); +}); + +test("maybeReactivateAfterExplicitProbe no-ops without reactivatedFromInactive", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe({ connectionId: "c1" }, async () => { + called += 1; + }); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe no-ops when explicitProbeSuppressed", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + explicitProbeSuppressed: true, + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe no-ops on shadow traffic", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + isShadowTraffic: true, + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe no-ops when allowSuppressedConnections", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + allowSuppressedConnections: true, + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); From f47ad14e558f4fc8be3d4028158428ecf5a693e4 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 7 Sep 2026 21:53:34 -0400 Subject: [PATCH 4/8] fix(auth): probe a pinned inactive connection once Signed-off-by: Minxi Hou --- src/sse/services/auth.ts | 40 +++++++++++++++++-- .../explicit-inactive-probe-12874.test.ts | 34 ++++++++++++++++ tests/unit/forced-connection-fallback.test.ts | 4 ++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 79fc31dfecc8..66d95631b7e9 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -7,6 +7,7 @@ import { getCachedRawProviderConnections, getCachedProviderNodes, getCachedSettings, + getCachedProviderConnectionById, } from "@/lib/db/readCache"; import { getProviderConnections, @@ -149,6 +150,12 @@ import { planSessionAffinityConnection, syncSessionAffinityRuntimeFields, } from "./sessionAffinityPin"; +import { + EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, + lastExplicitProbeTime, + noteExplicitProbe, + selectExplicitInactiveProbe, +} from "./explicitInactiveProbe"; import { isAnonymousFallbackDisabledBySettings, isNoAuthProviderBlockedBySettings, @@ -1061,7 +1068,10 @@ async function hydrateAccountProxyReferences( async function materializeConnection( connection: ProviderConnectionView, options: CredentialSelectionOptions, - extra: DeferredLeaseSelection & { exclusiveLease?: ExclusiveConnectionLease } = {} + extra: DeferredLeaseSelection & { + exclusiveLease?: ExclusiveConnectionLease; + reactivatedFromInactive?: boolean; + } = {} ) { const providerSpecificData = await hydrateAccountProxyReferences(connection.providerSpecificData); const apiKeyHealth = providerSpecificData.apiKeyHealth as Record | undefined; @@ -1260,6 +1270,27 @@ export async function getProviderCredentials( if (allowedConnections && allowedConnections.length > 0) { connections = connections.filter((conn) => allowedConnections.includes(conn.id)); } + let explicitProbeKind: "probe" | "suppressed" | "skip" = "skip"; + if (forcedConnectionId && !connections.some((c) => c.id === forcedConnectionId)) { + const pinnedRaw = await getCachedProviderConnectionById(forcedConnectionId); + const pinnedRow = pinnedRaw ? toProviderConnection(pinnedRaw) : null; + const nowMs = Date.now(); + const decision = selectExplicitInactiveProbe({ + forcedConnectionId, + activeConnections: connections, + pinnedRow, + providersToSearch, + allowedConnectionIds: allowedConnections ?? null, + nowMs, + lastProbeAtMs: lastExplicitProbeTime(forcedConnectionId), + intervalMs: EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, + }); + explicitProbeKind = decision.kind; + if (decision.kind === "probe" && pinnedRow) { + noteExplicitProbe(forcedConnectionId, nowMs); + connections = [pinnedRow]; + } + } const forcedConnectionEligible = connections.some((conn) => conn.id === forcedConnectionId); if (options.lease && forcedConnectionId && !forcedConnectionEligible) return null; if (options.lease?.mode === "request" && forcedConnectionId) { @@ -1486,7 +1517,7 @@ export async function getProviderCredentials( connectionFilterStatus.set(c.id, "modelNotAdvertised"); return false; } - if (!allowSuppressedConnections) { + if (!allowSuppressedConnections && explicitProbeKind !== "probe") { if (!allowRateLimitedConnections && isAccountUnavailable(c.rateLimitedUntil)) { connectionFilterStatus.set(c.id, "rateLimited"); return false; @@ -2149,7 +2180,10 @@ export async function getProviderCredentials( ); } - return materializeConnection(connection, options, { exclusiveLease }); + return materializeConnection(connection, options, { + exclusiveLease, + ...(explicitProbeKind === "probe" ? { reactivatedFromInactive: true } : {}), + }); } finally { selectionLock?.release(); } diff --git a/tests/unit/explicit-inactive-probe-12874.test.ts b/tests/unit/explicit-inactive-probe-12874.test.ts index 89baeb94f15b..2b76f8ddbf0f 100644 --- a/tests/unit/explicit-inactive-probe-12874.test.ts +++ b/tests/unit/explicit-inactive-probe-12874.test.ts @@ -213,3 +213,37 @@ test("maybeReactivateAfterExplicitProbe no-ops when allowSuppressedConnections", ); assert.equal(called, 0); }); + +test("W-2 pin + inactive returns credentials after wiring (was null)", async () => { + const fs = await import("node:fs"); + const os = await import("node:os"); + const path = await import("node:path"); + const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-explicit-inactive-w2-")); + process.env.DATA_DIR = TEST_DATA_DIR; + process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-inactive-w2-test-secret"; + + resetExplicitProbeMapForTests(); + const core = await import("../../src/lib/db/core.ts"); + const providersDb = await import("../../src/lib/db/providers.ts"); + const auth = await import("../../src/sse/services/auth.ts"); + + try { + const row = await providersDb.createProviderConnection({ + provider: "siliconflow", + authType: "apikey", + name: "sf-inactive", + apiKey: "sf-inactive-test-key", + isActive: false, + testStatus: "active", + }); + const creds = await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", { + forcedConnectionId: row.id, + }); + assert.ok(creds); + assert.equal(creds.connectionId, row.id); + assert.equal(creds.reactivatedFromInactive, true); + } finally { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); diff --git a/tests/unit/forced-connection-fallback.test.ts b/tests/unit/forced-connection-fallback.test.ts index 33add5791734..935d8b52e295 100644 --- a/tests/unit/forced-connection-fallback.test.ts +++ b/tests/unit/forced-connection-fallback.test.ts @@ -112,6 +112,10 @@ test("BUG CASE: forced connection deactivated (missing from active pool) is dete ); }); +test("recoverable inactive pin is not missing-from-pool once in connections", () => { + assert.equal(isForcedConnectionMissingFromPool("c1", new Set(), [{ id: "c1" }]), false); +}); + test("EXISTING BEHAVIOR: forced connection already excluded after a failed attempt is NOT missing-from-pool", () => { // The account is still present in the (active) pool — it 429'd and the retry loop // added it to excludedConnectionIds. This must keep going through From 01f5448a994e30f756e9888c738b4dfa71b5b01d Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 7 Sep 2026 22:26:30 -0400 Subject: [PATCH 5/8] fix(auth): stamp probe flag on every credential return A leased probe returned credentials without reactivatedFromInactive, so Task 5 would leave the pin inactive. One probeStamp const now rides every credential-shaped return. The SQLite W-2 cases live in their own file so DATA_DIR is bound before core loads, and credits_exhausted covers the suppressed-filter skip that testStatus active never hits. Signed-off-by: Minxi Hou --- src/sse/services/auth.ts | 12 ++- .../explicit-inactive-probe-12874.test.ts | 34 --------- tests/unit/explicit-inactive-probe-w2.test.ts | 73 +++++++++++++++++++ 3 files changed, 83 insertions(+), 36 deletions(-) create mode 100644 tests/unit/explicit-inactive-probe-w2.test.ts diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 66d95631b7e9..c1674eaea982 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -1291,6 +1291,8 @@ export async function getProviderCredentials( connections = [pinnedRow]; } } + const probeStamp = + explicitProbeKind === "probe" ? { reactivatedFromInactive: true as const } : {}; const forcedConnectionEligible = connections.some((conn) => conn.id === forcedConnectionId); if (options.lease && forcedConnectionId && !forcedConnectionEligible) return null; if (options.lease?.mode === "request" && forcedConnectionId) { @@ -2150,6 +2152,7 @@ export async function getProviderCredentials( return materializeConnection(connection, options, { commitSelectionSideEffects, selectNextLeaseCandidate, + ...probeStamp, }); } let claim = mutateExclusiveConnectionLease( @@ -2169,7 +2172,12 @@ export async function getProviderCredentials( exclusiveLease = claim.lease; await commitSelectionSideEffects?.(); if (options.materializeCredentials === false) { - return { exclusiveLease, connectionId: connection.id, provider: connection.provider }; + return { + exclusiveLease, + connectionId: connection.id, + provider: connection.provider, + ...probeStamp, + }; } } @@ -2182,7 +2190,7 @@ export async function getProviderCredentials( return materializeConnection(connection, options, { exclusiveLease, - ...(explicitProbeKind === "probe" ? { reactivatedFromInactive: true } : {}), + ...probeStamp, }); } finally { selectionLock?.release(); diff --git a/tests/unit/explicit-inactive-probe-12874.test.ts b/tests/unit/explicit-inactive-probe-12874.test.ts index 2b76f8ddbf0f..89baeb94f15b 100644 --- a/tests/unit/explicit-inactive-probe-12874.test.ts +++ b/tests/unit/explicit-inactive-probe-12874.test.ts @@ -213,37 +213,3 @@ test("maybeReactivateAfterExplicitProbe no-ops when allowSuppressedConnections", ); assert.equal(called, 0); }); - -test("W-2 pin + inactive returns credentials after wiring (was null)", async () => { - const fs = await import("node:fs"); - const os = await import("node:os"); - const path = await import("node:path"); - const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-explicit-inactive-w2-")); - process.env.DATA_DIR = TEST_DATA_DIR; - process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-inactive-w2-test-secret"; - - resetExplicitProbeMapForTests(); - const core = await import("../../src/lib/db/core.ts"); - const providersDb = await import("../../src/lib/db/providers.ts"); - const auth = await import("../../src/sse/services/auth.ts"); - - try { - const row = await providersDb.createProviderConnection({ - provider: "siliconflow", - authType: "apikey", - name: "sf-inactive", - apiKey: "sf-inactive-test-key", - isActive: false, - testStatus: "active", - }); - const creds = await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", { - forcedConnectionId: row.id, - }); - assert.ok(creds); - assert.equal(creds.connectionId, row.id); - assert.equal(creds.reactivatedFromInactive, true); - } finally { - core.resetDbInstance(); - fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); - } -}); diff --git a/tests/unit/explicit-inactive-probe-w2.test.ts b/tests/unit/explicit-inactive-probe-w2.test.ts new file mode 100644 index 000000000000..2d8f07deee0d --- /dev/null +++ b/tests/unit/explicit-inactive-probe-w2.test.ts @@ -0,0 +1,73 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-explicit-inactive-w2-")); + +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-inactive-w2-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); +const { resetExplicitProbeMapForTests } = await import( + "../../src/sse/services/explicitInactiveProbe.ts" +); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function seedInactiveSiliconflow(testStatus: "active" | "credits_exhausted") { + const row = await providersDb.createProviderConnection({ + provider: "siliconflow", + authType: "apikey", + name: "sf-inactive", + apiKey: "sf-inactive-test-key", + isActive: false, + testStatus, + }); + assert.ok(typeof row?.id === "string" && row.id.length > 0); + return { id: row.id }; +} + +function asPinnedCreds(creds: unknown) { + assert.ok(creds); + return creds as { connectionId?: string; reactivatedFromInactive?: boolean }; +} + +test.beforeEach(async () => { + resetExplicitProbeMapForTests(); + await resetStorage(); +}); + +test.after(async () => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("W-2 pin + inactive returns credentials after wiring (was null)", async () => { + const row = await seedInactiveSiliconflow("active"); + const creds = asPinnedCreds( + await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", { + forcedConnectionId: row.id, + }) + ); + assert.equal(creds.connectionId, row.id); + assert.equal(creds.reactivatedFromInactive, true); +}); + +test("W-2 pin + credits_exhausted returns credentials after wiring", async () => { + const row = await seedInactiveSiliconflow("credits_exhausted"); + const creds = asPinnedCreds( + await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", { + forcedConnectionId: row.id, + }) + ); + assert.equal(creds.connectionId, row.id); + assert.equal(creds.reactivatedFromInactive, true); +}); From 3177d312f084d67a634cdecd07beba36979152e7 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 7 Sep 2026 22:54:43 -0400 Subject: [PATCH 6/8] fix(auth): re-enable a pinned connection after a successful probe A successful probe used to clear error fields and leave isActive false, so the pin stayed disabled. onRequestSuccess now reactivates that row after the error clear. Signed-off-by: Minxi Hou --- .../fixes/12874-explicit-inactive-probe.md | 1 + src/sse/handlers/chatHelpers.ts | 8 +++++ .../explicit-inactive-probe-12874.test.ts | 34 +++++++++++++++++++ tests/unit/explicit-inactive-probe-w2.test.ts | 14 +++++++- 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixes/12874-explicit-inactive-probe.md diff --git a/changelog.d/fixes/12874-explicit-inactive-probe.md b/changelog.d/fixes/12874-explicit-inactive-probe.md new file mode 100644 index 000000000000..a347b6d3c97b --- /dev/null +++ b/changelog.d/fixes/12874-explicit-inactive-probe.md @@ -0,0 +1 @@ +- **fix(auth):** an explicit connection pin may probe a quota-disabled row once and re-enable it on success ([#12874](https://github.com/diegosouzapw/OmniRoute/issues/12874)) diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index b98ee96542f8..734f7663b2a6 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -4,6 +4,7 @@ import { getModelInfoOrRetirementResponse, } from "../services/model"; import { clearAccountError, markAccountUnavailable } from "../services/auth"; +import { maybeReactivateAfterExplicitProbe } from "../services/explicitInactiveProbe"; import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts"; import { createBuiltinAutoCombo } from "@omniroute/open-sse/services/autoCombo/builtinCatalog.ts"; import * as log from "../utils/logger"; @@ -521,6 +522,13 @@ export async function executeChatWithBreaker({ onRequestSuccess: async () => { if (isShadowTraffic) return; await clearAccountError(credentials.connectionId, credentials); + await maybeReactivateAfterExplicitProbe({ + connectionId: credentials.connectionId, + reactivatedFromInactive: credentials.reactivatedFromInactive, + isShadowTraffic, + requestedModel: model, + provider, + }); }, onStreamFailure: async (failure: any) => { if (isShadowTraffic) return; diff --git a/tests/unit/explicit-inactive-probe-12874.test.ts b/tests/unit/explicit-inactive-probe-12874.test.ts index 89baeb94f15b..f4926258d59a 100644 --- a/tests/unit/explicit-inactive-probe-12874.test.ts +++ b/tests/unit/explicit-inactive-probe-12874.test.ts @@ -1,5 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; +import fs from "node:fs"; import { EXPIRED_REPROBE_BLOCKLIST } from "../../src/lib/quota/connectionRecovery.ts"; import { EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, @@ -213,3 +214,36 @@ test("maybeReactivateAfterExplicitProbe no-ops when allowSuppressedConnections", ); assert.equal(called, 0); }); + +test("W-7 chatHelpers onRequestSuccess calls maybeReactivateAfterExplicitProbe", async () => { + const src = fs.readFileSync(new URL("../../src/sse/handlers/chatHelpers.ts", import.meta.url), "utf8"); + assert.match(src, /await maybeReactivateAfterExplicitProbe\(/); + assert.match(src, /clearAccountError/); +}); + +test("W-3 clearAccountError update payload has no isActive key", () => { + const src = fs.readFileSync(new URL("../../src/sse/services/auth.ts", import.meta.url), "utf8"); + const start = src.indexOf("export async function clearAccountError"); + assert.ok(start >= 0, "clearAccountError must exist"); + const next = src.indexOf("\nexport ", start + 1); + const body = next >= 0 ? src.slice(start, next) : src.slice(start); + const updateStart = body.indexOf("await updateProviderConnection("); + assert.ok(updateStart >= 0, "clearAccountError must call updateProviderConnection"); + const brace = body.indexOf("{", updateStart); + assert.ok(brace >= 0); + let depth = 0; + let end = -1; + for (let i = brace; i < body.length; i++) { + if (body[i] === "{") depth += 1; + else if (body[i] === "}") { + depth -= 1; + if (depth === 0) { + end = i; + break; + } + } + } + assert.ok(end > brace); + const literal = body.slice(brace, end + 1); + assert.equal(/\bisActive\b/.test(literal), false); +}); diff --git a/tests/unit/explicit-inactive-probe-w2.test.ts b/tests/unit/explicit-inactive-probe-w2.test.ts index 2d8f07deee0d..eec2fe9c3ec9 100644 --- a/tests/unit/explicit-inactive-probe-w2.test.ts +++ b/tests/unit/explicit-inactive-probe-w2.test.ts @@ -12,7 +12,7 @@ process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-inactive-w2 const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const auth = await import("../../src/sse/services/auth.ts"); -const { resetExplicitProbeMapForTests } = await import( +const { maybeReactivateAfterExplicitProbe, resetExplicitProbeMapForTests } = await import( "../../src/sse/services/explicitInactiveProbe.ts" ); @@ -71,3 +71,15 @@ test("W-2 pin + credits_exhausted returns credentials after wiring", async () => assert.equal(creds.connectionId, row.id); assert.equal(creds.reactivatedFromInactive, true); }); + +test("W-1 successful probe re-enables inactive pin in SQLite", async () => { + const row = await seedInactiveSiliconflow("active"); + const before = await providersDb.getProviderConnectionById(row.id); + assert.equal(before?.isActive, false); + await maybeReactivateAfterExplicitProbe({ + reactivatedFromInactive: true, + connectionId: row.id, + }); + const after = await providersDb.getProviderConnectionById(row.id); + assert.equal(after?.isActive, true); +}); From 8776c1dcf20311ffd4d3cdbf7fc80194838ef335 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 7 Sep 2026 23:10:54 -0400 Subject: [PATCH 7/8] fix(auth): list the inactive-probe sqlite test in mutation coverage The new pin-probe test imports auth.ts, which is already in the mutate set. Without the tap.testFiles entry the coverage gate treats that file as uncovered. Signed-off-by: Minxi Hou --- stryker.conf.json | 1 + 1 file changed, 1 insertion(+) diff --git a/stryker.conf.json b/stryker.conf.json index e09883155f32..7992aaa6012e 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -258,6 +258,7 @@ "tests/unit/executor-contract-violation-terminal.test.ts", "tests/unit/executor-devin-cli-agentic-acp.test.ts", "tests/unit/executor-web-cookie-sweep.test.ts", + "tests/unit/explicit-inactive-probe-w2.test.ts", "tests/unit/false-terminal-401-quota.test.ts", "tests/unit/follow-up-transcript.test.ts", "tests/unit/format-provider-error-cause.test.ts", From a62f032b6afc578016e3efac2305c023d1f63f95 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Tue, 8 Sep 2026 00:41:21 -0400 Subject: [PATCH 8/8] docs(changelog): attach #13017 to the inactive-probe fragment Signed-off-by: Minxi Hou --- ...licit-inactive-probe.md => 13017-explicit-inactive-probe.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename changelog.d/fixes/{12874-explicit-inactive-probe.md => 13017-explicit-inactive-probe.md} (53%) diff --git a/changelog.d/fixes/12874-explicit-inactive-probe.md b/changelog.d/fixes/13017-explicit-inactive-probe.md similarity index 53% rename from changelog.d/fixes/12874-explicit-inactive-probe.md rename to changelog.d/fixes/13017-explicit-inactive-probe.md index a347b6d3c97b..f3179991bdb1 100644 --- a/changelog.d/fixes/12874-explicit-inactive-probe.md +++ b/changelog.d/fixes/13017-explicit-inactive-probe.md @@ -1 +1 @@ -- **fix(auth):** an explicit connection pin may probe a quota-disabled row once and re-enable it on success ([#12874](https://github.com/diegosouzapw/OmniRoute/issues/12874)) +- **fix(auth):** an explicit connection pin may probe a quota-disabled row once and re-enable it on success ([#12874](https://github.com/diegosouzapw/OmniRoute/issues/12874)) ([#13017](https://github.com/diegosouzapw/OmniRoute/pull/13017))