From 61395b3f29ad198b82a7355d502b114781ab38b7 Mon Sep 17 00:00:00 2001 From: KaspaPulse Date: Wed, 26 Aug 2026 10:39:19 +0000 Subject: [PATCH] fix(sse): scope credential health to exact key --- open-sse/handlers/chatCore.ts | 181 ++++++++++-------- open-sse/handlers/chatCore/keyHealth.ts | 41 +++- open-sse/services/apiKeyRotator.ts | 28 ++- src/app/api/providers/[id]/test/route.ts | 21 +- tests/unit/chatcore-key-health.test.ts | 61 ++++++ ...rovider-connection-test-key-health.test.ts | 151 +++++++++++++++ 6 files changed, 385 insertions(+), 98 deletions(-) create mode 100644 tests/unit/provider-connection-test-key-health.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 4d8abc4f51f..495f25d486a 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -704,8 +704,9 @@ export async function handleChatCore({ const recordKeyHealthStatus = ( status: number, creds: Record | null | undefined, - transport?: string - ): void => recordKeyHealthStatusFor(status, creds, log, transport); + transport?: string, + failureDetail?: string + ): void => recordKeyHealthStatusFor(status, creds, log, transport, failureDetail); // ── Phase 9.2: Idempotency check ── // Resolve the idempotency key once here and reuse it at the Phase 9.2 save site below, // rather than re-deriving it. (#3821-review LEDGER-6) @@ -3174,11 +3175,20 @@ export async function handleChatCore({ }); if ( - res.response.status === 401 && + stream && + (res.response.ok || + res.response.status === HTTP_STATUS.UNAUTHORIZED || + res.response.status === HTTP_STATUS.FORBIDDEN) && executionConnectionId && !(await shouldIsolateProbeFailures()) ) { - recordKeyHealthStatus(401, execCreds); + const failureDetail = res.response.ok + ? "" + : await res.response + .clone() + .text() + .catch(() => ""); + recordKeyHealthStatus(res.response.status, execCreds, res.transport, failureDetail); } if (isModelScope() && res.response.status === 429 && attempts < maxAttempts - 1) { @@ -3529,13 +3539,6 @@ export async function handleChatCore({ // Non-stream: release semaphore immediately after reading full response body. const status = rawResult.response.status; - // Use execution credentials captured during request processing - if ( - rawResult._executionCredentials?.connectionId && - rawResult._executionCredentials?.apiKey - ) { - recordKeyHealthStatus(status, rawResult._executionCredentials, rawResult.transport); - } releaseRawResultAccountSemaphore = typeof rawResult._accountSemaphoreRelease === "function" ? rawResult._accountSemaphoreRelease @@ -3561,6 +3564,19 @@ export async function handleChatCore({ contentType, upstreamStream ); + // Use the exact execution credential selected for this request. Model capability + // failures stay in routing telemetry; authoritative success only recovers this key. + if ( + rawResult._executionCredentials?.connectionId && + (rawResult._executionCredentials.apiKey || rawResult._executionCredentials.accessToken) + ) { + recordKeyHealthStatus( + status, + rawResult._executionCredentials, + rawResult.transport, + status >= 400 ? payload : "" + ); + } releaseRawResultAccountSemaphore(); releaseRawResultAccountSemaphore = () => {}; @@ -4241,79 +4257,84 @@ export async function handleChatCore({ `[provider] Node ${errorConnectionId} probe ${errorType} (${statusCode}) — connection stays active` ); } else { - // Kimi's 403 says "billing cycle" for both an exhausted subscription and a - // temporary request window. Read its official usage endpoint before making - // the connection terminal: a non-zero Weekly quota plus an empty Ratelimit - // window must recover automatically at the reported reset time. - let kimiRateLimitResetAt: string | null = null; - if (provider === "kimi-coding") { - try { - const { fetchAndPersistProviderLimits } = - await import("@/lib/usage/providerLimits"); - const { usage } = await fetchAndPersistProviderLimits(errorConnectionId, "manual"); - kimiRateLimitResetAt = getKimiTemporaryRateLimitResetAt(usage); - } catch { - // Preserve the existing quota handling when Kimi's usage endpoint is unavailable. + // Kimi's 403 says "billing cycle" for both an exhausted subscription and a + // temporary request window. Read its official usage endpoint before making + // the connection terminal: a non-zero Weekly quota plus an empty Ratelimit + // window must recover automatically at the reported reset time. + let kimiRateLimitResetAt: string | null = null; + if (provider === "kimi-coding") { + try { + const { fetchAndPersistProviderLimits } = + await import("@/lib/usage/providerLimits"); + const { usage } = await fetchAndPersistProviderLimits( + errorConnectionId, + "manual" + ); + kimiRateLimitResetAt = getKimiTemporaryRateLimitResetAt(usage); + } catch { + // Preserve the existing quota handling when Kimi's usage endpoint is unavailable. + } } - } - // Providers with per-model quotas — lock the model only, not the connection - const quotaCooldownMs = kimiRateLimitResetAt - ? Math.max(new Date(kimiRateLimitResetAt).getTime() - Date.now(), 0) - : retryAfterMs || COOLDOWN_MS.rateLimit; - const accountSemaphoreKey = resolveAccountSemaphoreKey({ - provider, - model: currentModel, - connectionId: errorConnectionId, - credentials, - }); - if (accountSemaphoreKey) { - markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); - } - if (kimiRateLimitResetAt) { - await updateProviderConnection(errorConnectionId, { - testStatus: "unavailable", - rateLimitedUntil: kimiRateLimitResetAt, - backoffLevel: 0, - lastErrorType: PROVIDER_ERROR_TYPES.RATE_LIMITED, - lastError: message, - errorCode: statusCode, - }); - console.warn( - `[provider] Node ${errorConnectionId} Kimi request window exhausted (${statusCode}) — retrying after ${kimiRateLimitResetAt}` - ); - } else if (isModelScope() && errorConnectionId) { - const lockFn = provider === "antigravity" ? lockExactModel : lockModel; - lockFn(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); - console.warn( - `[provider] Node ${errorConnectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` - ); - } else if ( - lockModelIfPerModelQuota( + // Providers with per-model quotas — lock the model only, not the connection + const quotaCooldownMs = kimiRateLimitResetAt + ? Math.max(new Date(kimiRateLimitResetAt).getTime() - Date.now(), 0) + : retryAfterMs || COOLDOWN_MS.rateLimit; + const accountSemaphoreKey = resolveAccountSemaphoreKey({ provider, - errorConnectionId, - model, - "quota_exhausted", - quotaCooldownMs - ) - ) { - const quotaScope = getQuotaScopeLabelForProvider(provider, model); - console.warn( - `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)` - ); - } else { - await writeTerminalStatus( - errorConnectionId, - { - testStatus: "credits_exhausted", + model: currentModel, + connectionId: errorConnectionId, + credentials, + }); + if (accountSemaphoreKey) { + markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); + } + if (kimiRateLimitResetAt) { + await updateProviderConnection(errorConnectionId, { + testStatus: "unavailable", + rateLimitedUntil: kimiRateLimitResetAt, + backoffLevel: 0, + lastErrorType: PROVIDER_ERROR_TYPES.RATE_LIMITED, lastError: message, - lastErrorType: errorType, - errorCode: String(statusCode), - }, - "production" - ); - console.warn(`[provider] Node ${errorConnectionId} exhausted quota (${statusCode})`); - } + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${errorConnectionId} Kimi request window exhausted (${statusCode}) — retrying after ${kimiRateLimitResetAt}` + ); + } else if (isModelScope() && errorConnectionId) { + const lockFn = provider === "antigravity" ? lockExactModel : lockModel; + lockFn(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); + console.warn( + `[provider] Node ${errorConnectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` + ); + } else if ( + lockModelIfPerModelQuota( + provider, + errorConnectionId, + model, + "quota_exhausted", + quotaCooldownMs + ) + ) { + const quotaScope = getQuotaScopeLabelForProvider(provider, model); + console.warn( + `[provider] Node ${errorConnectionId} ${quotaScope}-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (cooldown_scope=${quotaScope}, ttl_source=${retryAfterMs ? "upstream" : "inferred"}, connection stays active)` + ); + } else { + await writeTerminalStatus( + errorConnectionId, + { + testStatus: "credits_exhausted", + lastError: message, + lastErrorType: errorType, + errorCode: String(statusCode), + }, + "production" + ); + console.warn( + `[provider] Node ${errorConnectionId} exhausted quota (${statusCode})` + ); + } } // close probeIsolated3 else } } else if (errorType === PROVIDER_ERROR_TYPES.UNAUTHORIZED) { diff --git a/open-sse/handlers/chatCore/keyHealth.ts b/open-sse/handlers/chatCore/keyHealth.ts index 1a65233e7cc..6c19a8346bd 100644 --- a/open-sse/handlers/chatCore/keyHealth.ts +++ b/open-sse/handlers/chatCore/keyHealth.ts @@ -6,12 +6,13 @@ * handleChatCore. Translates an upstream HTTP status into the in-memory key-health state * (apiKeyRotator) for the connection's currently-selected key, and persists the change to the * provider connection so it survives process restarts: - * - 401 → record a failure (warning, then invalid at the threshold), always persisted. + * - genuine 401/403 credential rejection → record a failure (warning, then invalid at the + * threshold), always persisted. * - 402 → terminal (insufficient balance); mark the current key invalid immediately (#5239), * persisted on the active→invalid transition. * - 2xx → record a success, persisted only when recovering from a warning/invalid state. - * Any other status only refreshes the tracked extra-key set. The handler binds its `log` once and - * delegates here, keeping the existing call sites unchanged. + * Model availability failures remain model/routing telemetry even when an upstream reports them + * with 401/403. Any other status only refreshes the tracked extra-key set. */ import { @@ -21,6 +22,7 @@ import { trackConnectionExtraKeys, type KeyHealth, } from "../../services/apiKeyRotator.ts"; +import { isModelUnavailableError } from "../../services/modelFamilyFallback.ts"; import { updateProviderConnection } from "@/lib/db/providers"; type KeyHealthLog = { @@ -28,11 +30,38 @@ type KeyHealthLog = { error?: (tag: string, message: string) => void; } | null; +const CREDENTIAL_FAILURE_PATTERNS = [ + /\b(?:invalid|incorrect|expired|missing|revoked)\s+api[\s_-]?key\b/i, + /\bapi[\s_-]?key\s+(?:is\s+)?(?:invalid|incorrect|expired|missing|revoked|not\s+valid)\b/i, + /\bauthentication[\s_-]+(?:failed|error|required)\b/i, + /\b(?:invalid|expired|missing|revoked)\s+(?:token|credentials?|bearer)\b/i, + /\bunauthorized\b/i, + /\bnot\s+authenticated\b/i, + /\bforbidden\b/i, + /\baccess\s+denied\b/i, +]; + +function isModelCapabilityFailure(status: number, failureDetail: string): boolean { + if (!failureDetail) return false; + const normalizedDetail = failureDetail.replace(/[_-]+/g, " "); + // Model-family fallback already owns these phrases. Use a model-capable status for + // classification because some aggregators misreport the same model rejection as 401. + return isModelUnavailableError(status === 401 ? 403 : status, normalizedDetail); +} + +function isCredentialFailure(status: number, failureDetail: string): boolean { + if (status !== 401 && status !== 403) return false; + if (isModelCapabilityFailure(status, failureDetail)) return false; + if (status === 401) return true; + return CREDENTIAL_FAILURE_PATTERNS.some((pattern) => pattern.test(failureDetail)); +} + export function recordKeyHealthStatus( status: number, creds: Record | null | undefined, log?: KeyHealthLog, - transport?: string + transport?: string, + failureDetail = "" ): void { // CLIProxyAPI owns a shared external credential pool. Its auth failures cannot be // attributed to the native OmniRoute connection selected before proxy dispatch. @@ -55,11 +84,11 @@ export function recordKeyHealthStatus( trackConnectionExtraKeys(connId, extraKeys); - if (status === 401) { + if (isCredentialFailure(status, failureDetail)) { const updatedHealth = recordKeyFailure(connId, currentKeyId); log?.warn?.( "AUTH", - `401 on connection ${connId.slice(0, 8)} - key marked as failed (failure #${updatedHealth.failures})` + `${status} on connection ${connId.slice(0, 8)} - key marked as failed (failure #${updatedHealth.failures})` ); // Persist health status to DB on every failure (not just invalid transitions) diff --git a/open-sse/services/apiKeyRotator.ts b/open-sse/services/apiKeyRotator.ts index f63972b8dca..ed5fc25056f 100644 --- a/open-sse/services/apiKeyRotator.ts +++ b/open-sse/services/apiKeyRotator.ts @@ -31,7 +31,10 @@ const MAX_CONNECTION_EXTRA_KEYS = 500; */ export function trackConnectionExtraKeys(connectionId: string, extraKeys: string[]): void { const validExtras = extraKeys.filter((k) => typeof k === "string" && k.trim().length > 0); - if (!_connectionExtraKeys.has(connectionId) && _connectionExtraKeys.size >= MAX_CONNECTION_EXTRA_KEYS) { + if ( + !_connectionExtraKeys.has(connectionId) && + _connectionExtraKeys.size >= MAX_CONNECTION_EXTRA_KEYS + ) { const oldest = _connectionExtraKeys.keys().next().value; if (oldest !== undefined) _connectionExtraKeys.delete(oldest); } @@ -308,6 +311,29 @@ export function syncHealthFromDB(connectionId: string, health?: Record | undefined { + const data = + providerSpecificData && typeof providerSpecificData === "object" + ? (providerSpecificData as Record) + : {}; + const health = data.apiKeyHealth as Record | undefined; + const currentHealth = health?.[keyId]; + if (!currentHealth || (currentHealth.status === "active" && currentHealth.failures === 0)) { + return undefined; + } + + syncHealthFromDB(connectionId, health); + return { + ...data, + apiKeyHealth: { ...health, [keyId]: recordKeySuccess(connectionId, keyId) }, + }; +} + /** * Reset the rotation index for a connection. * Call this when a key fails (401/403) to skip the bad key next time. diff --git a/src/app/api/providers/[id]/test/route.ts b/src/app/api/providers/[id]/test/route.ts index fcffe0a34a6..5f0a9aeca6b 100644 --- a/src/app/api/providers/[id]/test/route.ts +++ b/src/app/api/providers/[id]/test/route.ts @@ -29,7 +29,7 @@ import { import { providerAllowsOptionalApiKey } from "@/shared/constants/providers"; import { shouldUseApiKeyConnectionTest } from "./webSessionTestDispatch"; import { testCodexAppServerConnection, makeDiagnosis } from "./codexAppServerHealth"; -import { removeConnectionHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts"; +import { recoverKeyHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts"; import { shouldClearErrorStateOnValidProbe } from "@/lib/usage/providerLimits"; import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation"; import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth"; @@ -1125,7 +1125,11 @@ export async function testSingleConnection(connectionId: string, validationModel lastError: clearErrorState ? null : result.valid ? connection.lastError : result.error, lastErrorAt: clearErrorState ? null : result.valid ? connection.lastErrorAt : now, lastTested: now, - lastErrorType: clearErrorState ? null : result.valid ? connection.lastErrorType : diagnosis.type, + lastErrorType: clearErrorState + ? null + : result.valid + ? connection.lastErrorType + : diagnosis.type, lastErrorSource: clearErrorState ? null : result.valid @@ -1147,16 +1151,11 @@ export async function testSingleConnection(connectionId: string, validationModel if (clearErrorState) { updateData.backoffLevel = 0; + } - const psd = connection?.providerSpecificData as Record | undefined; - updateData.providerSpecificData = { - ...(psd || {}), - apiKeyHealth: {}, - }; - - try { - removeConnectionHealth(connectionId); - } catch {} + if (result.valid && (connection.apiKey || connection.accessToken)) { + const recovered = recoverKeyHealth(connectionId, "primary", connection.providerSpecificData); + if (recovered) updateData.providerSpecificData = recovered; } // If token was refreshed, update tokens in DB diff --git a/tests/unit/chatcore-key-health.test.ts b/tests/unit/chatcore-key-health.test.ts index 4d39eed5686..302765a29fc 100644 --- a/tests/unit/chatcore-key-health.test.ts +++ b/tests/unit/chatcore-key-health.test.ts @@ -49,6 +49,55 @@ test("401 reaches invalid at the failure threshold (2 consecutive)", () => { assert.equal(h?.status, "invalid"); }); +test("genuine 403 credential failures warn then invalidate only the rejected key", () => { + const conn = "kh-403-credential"; + const failure = JSON.stringify({ + error: { code: "invalid_api_key", message: "Invalid API key" }, + }); + + recordKeyHealthStatus( + 403, + creds(conn, { selectedKeyId: "extra_0" }), + noopLog, + undefined, + failure + ); + let all = getAllKeyHealth(); + assert.equal(all[`${conn}:extra_0`]?.status, "warning"); + assert.equal(all[`${conn}:extra_0`]?.failures, 1); + assert.equal(all[`${conn}:primary`], undefined); + + recordKeyHealthStatus( + 403, + creds(conn, { selectedKeyId: "extra_0" }), + noopLog, + undefined, + failure + ); + all = getAllKeyHealth(); + assert.equal(all[`${conn}:extra_0`]?.status, "invalid"); + assert.equal(all[`${conn}:extra_0`]?.failures, 2); + assert.equal(all[`${conn}:primary`], undefined); +}); + +test("model capability failures and model-sync conflicts do not touch credential health", () => { + const cases = [ + [400, JSON.stringify({ error: { code: "model_not_found", message: "Model not found" } })], + [403, JSON.stringify({ error: { code: "unsupported_model", message: "Unsupported model" } })], + [409, JSON.stringify({ error: "Model discovery deferred" })], + ] as const; + + for (const [status, failure] of cases) { + const conn = `kh-non-credential-${status}`; + recordKeyHealthStatus(status, creds(conn), noopLog, undefined, failure); + assert.equal( + getAllKeyHealth()[`${conn}:primary`], + undefined, + `HTTP ${status} must remain outside API-key credential health` + ); + } +}); + test("2xx after a failure resets the key to active with 0 failures", () => { const conn = "kh-2xx-recover"; recordKeyHealthStatus(401, creds(conn), noopLog); @@ -66,6 +115,18 @@ test("honors selectedKeyId — scopes the update to the active extra key, not pr assert.equal(all[`${conn}:primary`], undefined); }); +test("success on another key cannot clear the affected key", () => { + const conn = "kh-cross-key-isolation"; + recordKeyHealthStatus(401, creds(conn), noopLog); + recordKeyHealthStatus(204, creds(conn, { selectedKeyId: "extra_0" }), noopLog); + + const all = getAllKeyHealth(); + assert.equal(all[`${conn}:primary`]?.status, "warning"); + assert.equal(all[`${conn}:primary`]?.failures, 1); + assert.equal(all[`${conn}:extra_0`]?.status, "active"); + assert.equal(all[`${conn}:extra_0`]?.failures, 0); +}); + test("non-401 / non-2xx status does not touch key health", () => { const conn = "kh-5xx-noop"; recordKeyHealthStatus(500, creds(conn), noopLog); diff --git a/tests/unit/provider-connection-test-key-health.test.ts b/tests/unit/provider-connection-test-key-health.test.ts new file mode 100644 index 00000000000..170b72f2bcb --- /dev/null +++ b/tests/unit/provider-connection-test-key-health.test.ts @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-exact-key-health-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; +process.env.OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK = "true"; + +const originalFetch = globalThis.fetch; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const { testSingleConnection } = await import("../../src/app/api/providers/[id]/test/route.ts"); +const { selfFetchWithRetry } = + await import("../../src/app/api/providers/[id]/sync-models/route.ts"); +const { getAllKeyHealth, recordKeyFailure, removeConnectionHealth } = + await import("../../open-sse/services/apiKeyRotator.ts"); + +type StoredKeyHealth = { + status: "active" | "warning" | "invalid"; + failures: number; + lastFailure: string | null; + lastSuccess: string | null; + totalRequests: number; + totalFailures: number; +}; + +const WARNING_HEALTH: StoredKeyHealth = { + status: "warning", + failures: 1, + lastFailure: "2026-08-26T01:00:00.000Z", + lastSuccess: null, + totalRequests: 1, + totalFailures: 1, +}; + +async function resetStorage(): Promise { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function createConnection(args: { + name: string; + rateLimitedUntil?: string | null; + primary?: StoredKeyHealth; + extra?: StoredKeyHealth; +}) { + const created = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: args.name, + apiKey: `test-key-${args.name}`, + testStatus: args.rateLimitedUntil ? "unavailable" : "active", + rateLimitedUntil: args.rateLimitedUntil ?? null, + providerSpecificData: { + extraApiKeys: [`test-extra-${args.name}`], + apiKeyHealth: { + ...(args.primary ? { primary: args.primary } : {}), + ...(args.extra ? { extra_0: args.extra } : {}), + }, + }, + }); + assert.ok(created?.id); + return created as { id: string }; +} + +test.beforeEach(async () => { + await resetStorage(); + globalThis.fetch = (async () => + new Response(JSON.stringify({ data: [{ id: "available-model" }] }), { + status: 200, + headers: { "content-type": "application/json" }, + })) as typeof fetch; +}); + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("authoritative validation clears only the exact primary credential during a quota cooldown", async () => { + const cooldown = new Date(Date.now() + 60_000).toISOString(); + const connection = await createConnection({ + name: "primary-recovery", + rateLimitedUntil: cooldown, + primary: WARNING_HEALTH, + extra: WARNING_HEALTH, + }); + + const result = await testSingleConnection(connection.id); + assert.equal(result.valid, true); + + const updated = await providersDb.getProviderConnectionById(connection.id); + const health = updated.providerSpecificData?.apiKeyHealth as Record; + + assert.equal(health.primary.status, "active"); + assert.equal(health.primary.failures, 0); + assert.equal(health.extra_0.status, "warning"); + assert.equal(health.extra_0.failures, 1); + assert.equal( + updated.rateLimitedUntil, + cooldown, + "credential success must preserve quota cooldown" + ); + + const inMemory = getAllKeyHealth(); + assert.equal(inMemory[`${connection.id}:primary`]?.status, "active"); + assert.equal(inMemory[`${connection.id}:extra_0`]?.status, "warning"); + removeConnectionHealth(connection.id); +}); + +test("validating the primary key cannot clear a warning on another key", async () => { + const connection = await createConnection({ + name: "other-key-isolation", + extra: WARNING_HEALTH, + }); + + const result = await testSingleConnection(connection.id); + assert.equal(result.valid, true); + + const updated = await providersDb.getProviderConnectionById(connection.id); + const health = updated.providerSpecificData?.apiKeyHealth as Record; + assert.equal(health.primary, undefined); + assert.equal(health.extra_0.status, "warning"); + assert.equal(health.extra_0.failures, 1); + removeConnectionHealth(connection.id); +}); + +test("model-sync 409 remains model telemetry and cannot mutate API-key health", async () => { + const connectionId = "model-sync-conflict"; + const before = recordKeyFailure(connectionId, "primary"); + + const response = await selfFetchWithRetry("http://127.0.0.1/models", { + fetch: async () => + new Response(JSON.stringify({ error: "Model discovery deferred" }), { status: 409 }), + maxRetries: 1, + skipReadinessGate: true, + }); + + assert.equal(response.status, 409); + assert.deepEqual(getAllKeyHealth()[`${connectionId}:primary`], before); + removeConnectionHealth(connectionId); +});