From 37b2c9adf14506b03716e016e0a36f09007b0de2 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 16 Oct 2026 18:25:50 +0900 Subject: [PATCH 1/6] test: pin API-key cooldown to the credential actually sent Co-authored-by: GSL-R <174104911+GSL-R@users.noreply.github.com> --- .../messages-native-key-selection.test.ts | 23 +++++++++++++ tests/responses/compact-key-selection.test.ts | 16 +++++++++ .../server-chat-native-key-selection.test.ts | 33 +++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/tests/claude-integration/messages-native-key-selection.test.ts b/tests/claude-integration/messages-native-key-selection.test.ts index 6881ce0ac04..91bb4f4353b 100644 --- a/tests/claude-integration/messages-native-key-selection.test.ts +++ b/tests/claude-integration/messages-native-key-selection.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../src/config"; +import { registerUpstreamRewriter } from "../../src/plugins/upstream-hooks"; import { apiKeyAccountLogLabel } from "../../src/codex/account-label"; import { configureSharedSpendLedger, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; import { captureProviderApiKeySelection } from "../../src/providers/api-key-selection"; @@ -321,6 +322,28 @@ for (const statusCode of [401, 429]) for (const overridden of [false, true]) tes unchanged(config); noLeaks(config); }); +for (const statusCode of [401, 429]) for (const transport of ["bearer", "x-api-key"] as const) { + for (const rewrite of ["credential", "destination"] as const) test(`Messages physical ${transport} ${statusCode} ownership after ${rewrite} rewrite`, async () => { + const config = fixture(2, false, true, [statusCode, 200]); + provider(config).apiKeyTransport = transport; + saveConfig(config); + const off = registerUpstreamRewriter("fixture-key-owner", target => { + if (rewrite === "credential") target.headers.set(transport === "bearer" ? "authorization" : "x-api-key", + transport === "bearer" ? "Bearer fixture-plugin-key" : "fixture-plugin-key"); + if (rewrite === "destination") target.url += "?physical-owner=other"; + }); + try { + const result = await start(config, new AbortController()); + expect(result.response.status).toBe(statusCode); + expect(seen).toHaveLength(1); + expect(seen[0]!.key).toBe(rewrite === "credential" ? "fixture-plugin-key" : KEYS[0]); + expect(getKeyCooldownUntil(NAME, "k1")).toBeNull(); + expect(getKeyCooldownUntil(NAME, "k2")).toBeNull(); + unchanged(config); noLeaks(config); + } finally { off(); } + }); +} + for (const cancel of [false, true]) test(`active manual key switch readmits the selected key (cancel=${cancel})`, async () => { const config = fixture(); config.providers[NAME]!.requestPacing = { enabled: true, keys: { maxConcurrentRequests: 1 } }; diff --git a/tests/responses/compact-key-selection.test.ts b/tests/responses/compact-key-selection.test.ts index 8b5b3518d0b..2a74ec11f17 100644 --- a/tests/responses/compact-key-selection.test.ts +++ b/tests/responses/compact-key-selection.test.ts @@ -247,6 +247,22 @@ test("a synthetic non-replayable compact 429 never cools a distributed key", asy unchanged(config); noLeaks(config); }); +for (const statusCode of [401, 429]) test(`compact static Authorization ${statusCode} retains pool availability`, async () => { + const config = fixture(); + config.providers[NAME]!.headers = { Authorization: "Bearer fixture-compact-override" }; + globalThis.fetch = (async (input, init) => { + seen.push({ key: new Headers(init?.headers).get("authorization"), url: String(input), body: JSON.parse(String(init?.body)), headers: new Headers(init?.headers) }); + return Response.json({ error: { message: "fixture refusal" } }, { status: statusCode }); + }) as typeof fetch; + for (let request = 0; request < 4; request++) { + expect((await start(config, new AbortController())).response.status).toBe(statusCode); + expect(seen).toHaveLength(request + 1); + for (const id of ["k1", "k2", "k3"]) expect(getKeyCooldownUntil(NAME, id)).toBeNull(); + } + expect(seen.map(row => row.key)).toEqual(Array(4).fill("Bearer fixture-compact-override")); + unchanged(config); noLeaks(config); +}); + for (const cancel of [false, true]) test(`active manual key switch readmits the selected key (cancel=${cancel})`, async () => { const config = fixture(); config.providers[NAME]!.requestPacing = { enabled: true, keys: { maxConcurrentRequests: 1 } }; diff --git a/tests/server/server-chat-native-key-selection.test.ts b/tests/server/server-chat-native-key-selection.test.ts index 4f36ea91044..60017b9aadf 100644 --- a/tests/server/server-chat-native-key-selection.test.ts +++ b/tests/server/server-chat-native-key-selection.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../src/config"; +import { registerUpstreamRewriter } from "../../src/plugins/upstream-hooks"; import { apiKeyAccountLogLabel } from "../../src/codex/account-label"; import { captureProviderApiKeySelection } from "../../src/providers/api-key-selection"; import { clearKeyCooldowns, forgetApiKeyRotationCursor, getKeyCooldownUntil, recordDistributedKeyFailure } from "../../src/providers/key-failover"; @@ -228,6 +229,38 @@ test("chat owned 401 rotates the rejected distributed credential before output", unchanged(config); noLeaks(config); }); +for (const statusCode of [401, 429]) for (const rewrite of ["credential", "destination", "none"] as const) { + test(`chat physical ${statusCode} ownership after ${rewrite} rewrite`, async () => { + const config = fixture(); + const off = registerUpstreamRewriter("fixture-key-owner", target => { + if (rewrite === "credential") target.headers.set("authorization", "Bearer fixture-plugin-key"); + if (rewrite === "destination") target.url = "https://replacement.example/v1/chat/completions"; + }); + const reply = globalThis.fetch; + globalThis.fetch = (async (input, init) => { + if (seen.length) return reply(input, init); + seen.push({ key: new Headers(init?.headers).get("authorization"), url: String(input), body: JSON.parse(String(init?.body)) }); + return Response.json({ error: { type: "upstream_error", message: "fixture refusal" } }, { status: statusCode }); + }) as typeof fetch; + try { + const result = await start(config, new AbortController()); + if (rewrite === "none") { + expect(result.response.status).toBe(200); + expect(seen.map(row => row.key)).toEqual([`Bearer ${KEYS[0]}`, `Bearer ${KEYS[1]}`]); + expect(getKeyCooldownUntil(NAME, "k1")).not.toBeNull(); + } else { + expect(result.response.status).toBe(statusCode); + expect(seen).toHaveLength(1); + expect(seen[0]!.key).toBe(rewrite === "credential" ? "Bearer fixture-plugin-key" : `Bearer ${KEYS[0]}`); + expect(seen[0]!.url).toBe(rewrite === "destination" ? "https://replacement.example/v1/chat/completions" : "https://gateway.example/v1/chat/completions"); + expect(getKeyCooldownUntil(NAME, "k1")).toBeNull(); + } + for (const id of ["k2", "k3"]) expect(getKeyCooldownUntil(NAME, id)).toBeNull(); + unchanged(config); noLeaks(config); + } finally { off(); } + }); +} + for (const cancel of [false, true]) test(`active manual key switch readmits the selected key (cancel=${cancel})`, async () => { const config = fixture(); config.providers[NAME]!.requestPacing = { enabled: true, keys: { maxConcurrentRequests: 1 } }; From 295cb3e283cc148dc35e4bc0c93ab8defde9e88e Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 16 Oct 2026 18:33:46 +0900 Subject: [PATCH 2/6] test: exercise compact credential ownership on native transport Co-authored-by: GSL-R <174104911+GSL-R@users.noreply.github.com> --- tests/responses/compact-key-selection.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/responses/compact-key-selection.test.ts b/tests/responses/compact-key-selection.test.ts index 2a74ec11f17..289e0689872 100644 --- a/tests/responses/compact-key-selection.test.ts +++ b/tests/responses/compact-key-selection.test.ts @@ -249,7 +249,8 @@ test("a synthetic non-replayable compact 429 never cools a distributed key", asy for (const statusCode of [401, 429]) test(`compact static Authorization ${statusCode} retains pool availability`, async () => { const config = fixture(); - config.providers[NAME]!.headers = { Authorization: "Bearer fixture-compact-override" }; + config.providers[NAME]!.supportsNativeCompactEndpoint = true; + config.providers[NAME]!.headers = { Authorization: "Bearer fixture-override" }; globalThis.fetch = (async (input, init) => { seen.push({ key: new Headers(init?.headers).get("authorization"), url: String(input), body: JSON.parse(String(init?.body)), headers: new Headers(init?.headers) }); return Response.json({ error: { message: "fixture refusal" } }, { status: statusCode }); @@ -259,7 +260,7 @@ for (const statusCode of [401, 429]) test(`compact static Authorization ${status expect(seen).toHaveLength(request + 1); for (const id of ["k1", "k2", "k3"]) expect(getKeyCooldownUntil(NAME, id)).toBeNull(); } - expect(seen.map(row => row.key)).toEqual(Array(4).fill("Bearer fixture-compact-override")); + expect(seen.map(row => row.key)).toEqual(Array(4).fill("Bearer fixture-override")); unchanged(config); noLeaks(config); }); From aed974b40ce38752a1e4d2adb444f351ec6e1d20 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 16 Oct 2026 18:34:16 +0900 Subject: [PATCH 3/6] fix: charge key cooldown to the credential sent Co-authored-by: GSL-R <174104911+GSL-R@users.noreply.github.com> --- src/providers/api-key-response-ownership.ts | 26 +++++++++++++++------ src/server/chat-native.ts | 9 +++---- src/server/messages-native.ts | 15 ++++++------ src/server/responses/compact.ts | 10 ++++---- structure/providers-and-adapters.md | 2 +- 5 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/providers/api-key-response-ownership.ts b/src/providers/api-key-response-ownership.ts index 7011b56a924..01b17c8c7a5 100644 --- a/src/providers/api-key-response-ownership.ts +++ b/src/providers/api-key-response-ownership.ts @@ -15,15 +15,19 @@ registerProviderResponseProvenance(owners); /** Capture the credential after connection-policy rewrites, immediately before fetch. */ export function distributedKeyPhysicalFetch( physicalFetch: typeof globalThis.fetch, providerName: string, provider: OcxProviderConfig, target: string, + authHeader: "authorization" | "x-api-key" = "authorization", ): typeof globalThis.fetch { if (!provider._apiKeyAttempt?.distributed) return physicalFetch; const wrapped = (async (input, init) => { const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); const destination = input instanceof Request ? input.url : String(input); - const ownsCredential = destination === target && headers.get("authorization") === `Bearer ${provider.apiKey}` - && !headers.has("x-api-key") && !headers.has("api-key"); + const ownsCredential = destination === target && !headers.has("api-key") + && (authHeader === "authorization" + ? headers.get("authorization") === `Bearer ${provider.apiKey}` && !headers.has("x-api-key") + : headers.get("x-api-key") === provider.apiKey && !headers.has("authorization")); const owner: KeyResponseOwner | null = provider._apiKeyAttempt?.distributed && ownsCredential - ? { providerName, provider: { ...provider, _apiKeyAttempt: { ...provider._apiKeyAttempt } } } : null; + ? Object.freeze({ providerName, provider: Object.freeze({ ...provider, + _apiKeyAttempt: Object.freeze({ ...provider._apiKeyAttempt }) }) }) : null; const response = await physicalFetch(input, init); // Reused response identities cannot safely describe two physical sends. owners.set(response, owners.has(response) ? null : owner); @@ -32,14 +36,22 @@ export function distributedKeyPhysicalFetch( return isEgressTransparentExecutor(physicalFetch) ? markEgressTransparentExecutor(wrapped) : wrapped; } +/** Return only a still-configured credential captured at the physical send. */ +export function distributedKeyResponseOwner( + response: Response, config: OcxConfig, providerName: string, +): OcxProviderConfig | null { + const owner = owners.get(response); + return owner?.providerName === providerName + && providerApiKeySelectionOwnsFailure(config, providerName, owner.provider) ? owner.provider : null; +} + /** Settle only the terminal refusal; configured same-key 429 retries retain their key. */ export function recordDistributedResponseFailure( response: Response, config: OcxConfig, providerName: string, signal?: AbortSignal, ): void { if (signal?.aborted || isNonReplayableResponse(response) || (response.status !== 401 && response.status !== 429)) return; - const owner = owners.get(response); - if (!owner || owner.providerName !== providerName - || !providerApiKeySelectionOwnsFailure(config, providerName, owner.provider)) return; - recordDistributedKeyFailure(config, providerName, owner.provider._apiKeyAttempt, + const owner = distributedKeyResponseOwner(response, config, providerName); + if (!owner) return; + recordDistributedKeyFailure(config, providerName, owner._apiKeyAttempt, response.status, Date.now(), response.headers.get("retry-after")); } diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 56c5a41e6e9..96546c29595 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -62,6 +62,7 @@ import { import { fastPolicyForModel } from "../providers/service-tier"; import { RequestPacingQueueOverloadError } from "../providers/request-pacing"; import { stampApiKeyAccountLabel } from "../providers/label"; +import { distributedKeyPhysicalFetch, distributedKeyResponseOwner } from "../providers/api-key-response-ownership"; import { providerApiKeySelectionIsCurrent, resolveCurrentProviderApiKeyTransport } from "../providers/api-key-selection"; import { enrichOpenCodeZenFreeTierMessage } from "../providers/opencode-zen-rate-limit"; import type { OcxProviderTransport } from "../providers/xai-transport"; @@ -515,8 +516,6 @@ export async function runNativeChatAttempt( const spendReport = createPhysicalSendReporter(physicalBudget, () => ({ poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, }), execution.comboDispatchPermit); - const ownsDistributedKey = !activeProvider._apiKeyAttempt?.distributed - || headers.get("authorization") === `Bearer ${activeProvider.apiKey}`; let dispatched: Response; try { // Capture this request's spend policy at its first physical boundary, including @@ -534,7 +533,8 @@ export async function runNativeChatAttempt( // A reselected provider transport is still a physical send: the connection policy // and manual-redirect ownership wrap the selected implementation (#4992). dispatched = await sendWithConnectionPolicy( - (activeProvider as OcxProviderTransport).fetch ?? execute, + distributedKeyPhysicalFetch((activeProvider as OcxProviderTransport).fetch ?? execute, + route.providerName, activeProvider, request.url), request.url, applyUpstreamRecoveryInit({ ...init, method: request.method, headers, body: request.body, @@ -549,7 +549,8 @@ export async function runNativeChatAttempt( try { if (physicalBudget.spendEnforced) spendReport(1); } finally { spendReport.close?.(); } } - responseOwnsDistributedKey = ownsDistributedKey; + responseOwnsDistributedKey = !activeProvider._apiKeyAttempt?.distributed + || distributedKeyResponseOwner(dispatched, config, route.providerName) !== null; if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal); return dispatched; }, diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index d61fdf56954..e67683f2578 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -61,13 +61,13 @@ import { type UpstreamSendRecovery, UPSTREAM_RESET_REPLAY_REFUSED_CODE, } from "../lib/upstream-retry"; +import { distributedKeyPhysicalFetch, distributedKeyResponseOwner, recordDistributedResponseFailure } from "../providers/api-key-response-ownership"; import { providerApiKeySelectionIsCurrent, resolveCurrentProviderApiKeyTransport } from "../providers/api-key-selection"; import { RequestPacingQueueOverloadError } from "../providers/request-pacing"; import { hasKeyPoolFailover, rateLimitRetryDelayMs, rateLimitRetryPolicyFor, - recordDistributedKeyFailure, rotateProviderTransportOn401, rotateProviderTransportOn429, selectProactiveApiKeyTransport, @@ -616,10 +616,6 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) const releaseFamily = snapshot && oauthBinding ? anthropicModelQuotaFor(oauthBinding.instance).claimAnthropicFamilyRevalidation(snapshot.accountId, route.modelId) : () => {}; if (!releaseFamily) throw new AnthropicAccountCooldownError(1); const selection = activeProvider._apiKeyAttempt; - const ownsDistributedKey = !selection?.distributed - || (activeProvider.apiKeyTransport === "bearer" - ? headers.get("authorization") === `Bearer ${activeProvider.apiKey}` && !headers.has("x-api-key") - : headers.get("x-api-key") === activeProvider.apiKey && !headers.has("authorization")); let dispatched: Response; const spendReport = createPhysicalSendReporter(physicalBudget, () => ({ poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, @@ -640,7 +636,9 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) if (snapshot && !sendOwner) throw new NativeOAuthSelectionChangedError(); physicalSendsUsed++; dispatched = await sendWithConnectionPolicy( - anthropicBillingPhysicalFetch((activeProvider as OcxProviderTransport).fetch ?? execute, route.providerName, activeProvider), + distributedKeyPhysicalFetch( + anthropicBillingPhysicalFetch((activeProvider as OcxProviderTransport).fetch ?? execute, route.providerName, activeProvider), + route.providerName, activeProvider, wire.url, activeProvider.apiKeyTransport === "bearer" ? "authorization" : "x-api-key"), wire.url, applyUpstreamRecoveryInit({ ...init, method: "POST", headers, body: wire.body }, transportRecovery), { providerName: route.providerName, provider: activeProvider }, @@ -664,12 +662,13 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) } } catch { /* Passive observation must not fail the response. */ } } + const ownsDistributedKey = !selection?.distributed + || distributedKeyResponseOwner(dispatched, config, route.providerName) !== null; distributedResponseOwnsKey = ownsDistributedKey; const failureStatus = dispatched.status; pendingDistributedFailure = selection?.distributed && ownsDistributedKey && (failureStatus === 401 || failureStatus === 429) && !isNonReplayableResponse(dispatched) - ? () => { recordDistributedKeyFailure(config, route.providerName, selection, failureStatus, - Date.now(), dispatched.headers.get("retry-after")); } + ? () => { recordDistributedResponseFailure(dispatched, config, route.providerName, req.signal); } : undefined; if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal); return dispatched; diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index cf581085fa6..644a73b38af 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -164,8 +164,9 @@ import { UnsupportedContentEncodingError, } from "../request-decompress"; import { resolveAdapter, resolveWireProtocolOverride } from "../adapter-resolve"; -import { hasKeyPoolFailover, recordDistributedKeyFailure, rotateProviderTransportOn429, selectProactiveApiKeyTransport, transientRetryPolicyFor } from "../../providers/key-failover"; +import { hasKeyPoolFailover, rotateProviderTransportOn429, selectProactiveApiKeyTransport, transientRetryPolicyFor } from "../../providers/key-failover"; import { transientSendCapFor } from "./request-send-budget"; +import { distributedKeyPhysicalFetch, recordDistributedResponseFailure } from "../../providers/api-key-response-ownership"; import { providerApiKeySelectionIsCurrent, resolveCurrentProviderApiKeyTransport } from "../../providers/api-key-selection"; import { stampApiKeyAccountLabel } from "../../providers/label"; import { shouldAttemptImageTierRetry } from "../image-retry"; @@ -1203,7 +1204,8 @@ export async function handleResponsesCompact( if (!rebindPhysicalSend(sendBudget, { poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel })) throw new SendBudgetExhaustedError(safeHostLabel(compactUrl)); noteProviderAttemptSend(logCtx, route.providerName, sendProvider, logCtx.usageLogInputTokens, upstreamRecovery); - return sendWithConnectionPolicy((sendProvider as { fetch?: typeof fetch }).fetch ?? execute, compactUrl, + return sendWithConnectionPolicy( + distributedKeyPhysicalFetch((sendProvider as { fetch?: typeof fetch }).fetch ?? execute, route.providerName, sendProvider, compactUrl), compactUrl, { ...init, headers: sendHeaders, body: JSON.stringify({ ...buildCompactSendBody() as typeof compactBody, model: route.modelId }) }, { providerName: route.providerName, provider: sendProvider }); } : undefined, @@ -1283,9 +1285,7 @@ export async function handleResponsesCompact( return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); } - if ((upstream.status === 401 || upstream.status === 429) && !isNonReplayableResponse(upstream)) { - recordDistributedKeyFailure(config, route.providerName, compactProvider._apiKeyAttempt, upstream.status, Date.now(), upstream.headers.get("retry-after")); - } + recordDistributedResponseFailure(upstream, config, route.providerName, req.signal); if ( upstream.status === 401 && (authCtx.kind === "main-pool" || authCtx.kind === "pool") diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 80adbf549b4..e4f670bafbd 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -281,7 +281,7 @@ Cursor, one lease spans the whole turn: RunSSE and BidiAppend may overlap inside turns wait at the cap. Follow-up sends still obey start intervals. The turn owner returns its lease after `runTurn` settles, so RunSSE body completion cannot admit another turn early. A capped canonical Codex WebSocket turn uses HTTP/SSE because the socket has no response-body lifecycle. -Capacity waits use the same bounded queue and retryable queue-overload errors as interval waits. Optional `requestPacing.keys` adds a shared cross-model limit for each credential reference; duplicate references share capacity, but distinct references resolving to one upstream project do not imply separate quota. `keySelection: round-robin` assigns request-local healthy keys without mutating persisted selection, skips known cooldowns, and refuses exhausted pools with retryable 429. Provider, model and key gates reserve atomically; key counters and the allocation cursor are process-local, not token or billing balances. OAuth and forwarded credentials retain their existing selection. Cooldown state is shared through the dependency-light `src/providers/api-key-cooldowns.ts` (retaining the longest outstanding expiry until explicit manual clearing) owner. Distributed selections are checked again at physical dispatch; a cooled queued key releases its old permit, selects a healthy sibling, and reacquires pacing before sending. Native Messages follows the same readmission rule. Native Responses settles terminal distributed refusals from immutable physical-send metadata; successful same-key retry and unowned refusals do not cool keys. Failure ownership remains valid for a still-configured cooling key, so late in-flight rejections retain the longest cooldown without admitting another send. CLI-backed adapters check `IncomingMeta.beforePhysicalSend` after asynchronous setup and before spawn. An established distributed turn refuses later sends after its key cools instead of changing credentials mid-turn. +Capacity waits use the same bounded queue and retryable queue-overload errors as interval waits. Optional `requestPacing.keys` adds a shared cross-model limit for each credential reference; duplicate references share capacity, but distinct references resolving to one upstream project do not imply separate quota. `keySelection: round-robin` assigns request-local healthy keys without mutating persisted selection, skips known cooldowns, and refuses exhausted pools with retryable 429. Provider, model and key gates reserve atomically; key counters and the allocation cursor are process-local, not token or billing balances. OAuth and forwarded credentials retain their existing selection. Cooldown state is shared through the dependency-light `src/providers/api-key-cooldowns.ts` (retaining the longest outstanding expiry until explicit manual clearing) owner. Distributed selections are checked again at physical dispatch; a cooled queued key releases its old permit, selects a healthy sibling, and reacquires pacing before sending. Native Messages follows the same readmission rule. Native Responses, compact, Chat and Messages settle distributed refusals from immutable credential and destination metadata captured by `src/providers/api-key-response-ownership.ts` at physical fetch; successful same-key retry and unowned refusals do not cool keys. Failure ownership remains valid for a still-configured cooling key, so late in-flight rejections retain the longest cooldown without admitting another send. CLI-backed adapters check `IncomingMeta.beforePhysicalSend` after asynchronous setup and before spawn. An established distributed turn refuses later sends after its key cools instead of changing credentials mid-turn. Command Code effort defaults in `src/providers/command-code-efforts.ts` combine public-profile facts with the live API measurements from #5096. Both presets share the exact per-model rows; From 779a08e415b4f5b882769d6160af306c844b5f45 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 16 Oct 2026 18:39:47 +0900 Subject: [PATCH 4/6] test: cover physical key ownership with configured retries Co-authored-by: GSL-R <174104911+GSL-R@users.noreply.github.com> --- tests/server/server-chat-native-key-selection.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/server/server-chat-native-key-selection.test.ts b/tests/server/server-chat-native-key-selection.test.ts index 60017b9aadf..70e568bff9b 100644 --- a/tests/server/server-chat-native-key-selection.test.ts +++ b/tests/server/server-chat-native-key-selection.test.ts @@ -229,9 +229,10 @@ test("chat owned 401 rotates the rejected distributed credential before output", unchanged(config); noLeaks(config); }); -for (const statusCode of [401, 429]) for (const rewrite of ["credential", "destination", "none"] as const) { - test(`chat physical ${statusCode} ownership after ${rewrite} rewrite`, async () => { +for (const statusCode of [401, 429]) for (const rewrite of ["credential", "destination", "none"] as const) for (const retry of [false, true]) { + test(`chat physical ${statusCode} ownership after ${rewrite} rewrite (retry=${retry})`, async () => { const config = fixture(); + if (retry) config.providers[NAME]!.retryOn429 = { attempts: 1, intervalMs: 0, respectRetryAfter: false }; const off = registerUpstreamRewriter("fixture-key-owner", target => { if (rewrite === "credential") target.headers.set("authorization", "Bearer fixture-plugin-key"); if (rewrite === "destination") target.url = "https://replacement.example/v1/chat/completions"; @@ -246,8 +247,10 @@ for (const statusCode of [401, 429]) for (const rewrite of ["credential", "desti const result = await start(config, new AbortController()); if (rewrite === "none") { expect(result.response.status).toBe(200); - expect(seen.map(row => row.key)).toEqual([`Bearer ${KEYS[0]}`, `Bearer ${KEYS[1]}`]); - expect(getKeyCooldownUntil(NAME, "k1")).not.toBeNull(); + const sameKeyRetry = retry && statusCode === 429; + expect(seen.map(row => row.key)).toEqual([`Bearer ${KEYS[0]}`, `Bearer ${KEYS[sameKeyRetry ? 0 : 1]}`]); + if (sameKeyRetry) expect(getKeyCooldownUntil(NAME, "k1")).toBeNull(); + else expect(getKeyCooldownUntil(NAME, "k1")).not.toBeNull(); } else { expect(result.response.status).toBe(statusCode); expect(seen).toHaveLength(1); From 00203226dbc3d993605eb3065d8dc738912dba09 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 16 Oct 2026 18:40:08 +0900 Subject: [PATCH 5/6] fix: retain physical key ownership through Chat retries Co-authored-by: GSL-R <174104911+GSL-R@users.noreply.github.com> --- src/server/chat-native.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 96546c29595..04c83c60f12 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -587,6 +587,7 @@ export async function runNativeChatAttempt( let retries = 0; while ( response.status === 429 + && responseOwnsDistributedKey // A 429 this proxy synthesized for a refused reset replay is not a provider rate // limit: waiting and re-sending here is exactly the duplicate inference the refusal // exists to stop. It kept the same shape under the old 502 only because 502 never From a24843b972177e9ae16d588611a899a2e1398768 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 16 Oct 2026 18:44:47 +0900 Subject: [PATCH 6/6] fix: preserve configured same-key Chat retries Co-authored-by: GSL-R <174104911+GSL-R@users.noreply.github.com> --- src/server/chat-native.ts | 1 - tests/server/server-chat-native-key-selection.test.ts | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 04c83c60f12..96546c29595 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -587,7 +587,6 @@ export async function runNativeChatAttempt( let retries = 0; while ( response.status === 429 - && responseOwnsDistributedKey // A 429 this proxy synthesized for a refused reset replay is not a provider rate // limit: waiting and re-sending here is exactly the duplicate inference the refusal // exists to stop. It kept the same shape under the old 502 only because 502 never diff --git a/tests/server/server-chat-native-key-selection.test.ts b/tests/server/server-chat-native-key-selection.test.ts index 70e568bff9b..e0320ea9ffc 100644 --- a/tests/server/server-chat-native-key-selection.test.ts +++ b/tests/server/server-chat-native-key-selection.test.ts @@ -252,10 +252,13 @@ for (const statusCode of [401, 429]) for (const rewrite of ["credential", "desti if (sameKeyRetry) expect(getKeyCooldownUntil(NAME, "k1")).toBeNull(); else expect(getKeyCooldownUntil(NAME, "k1")).not.toBeNull(); } else { - expect(result.response.status).toBe(statusCode); - expect(seen).toHaveLength(1); - expect(seen[0]!.key).toBe(rewrite === "credential" ? "Bearer fixture-plugin-key" : `Bearer ${KEYS[0]}`); - expect(seen[0]!.url).toBe(rewrite === "destination" ? "https://replacement.example/v1/chat/completions" : "https://gateway.example/v1/chat/completions"); + const sameKeyRetry = retry && statusCode === 429; + expect(result.response.status).toBe(sameKeyRetry ? 200 : statusCode); + expect(seen).toHaveLength(sameKeyRetry ? 2 : 1); + for (const row of seen) { + expect(row.key).toBe(rewrite === "credential" ? "Bearer fixture-plugin-key" : `Bearer ${KEYS[0]}`); + expect(row.url).toBe(rewrite === "destination" ? "https://replacement.example/v1/chat/completions" : "https://gateway.example/v1/chat/completions"); + } expect(getKeyCooldownUntil(NAME, "k1")).toBeNull(); } for (const id of ["k2", "k3"]) expect(getKeyCooldownUntil(NAME, id)).toBeNull();