From b94839264ef6585f5efbb40099aeb8d2314133e2 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 1 Oct 2026 11:22:10 +0900 Subject: [PATCH] fix(anthropic): fail over on proven pre-output account 403 refusals --- .../src/content/docs/fr/guides/claude-code.md | 3 +- .../fr/reference/configuration/providers.md | 3 +- .../src/content/docs/guides/claude-code.md | 11 +- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../src/content/docs/tr/guides/claude-code.md | 3 +- .../tr/reference/configuration/providers.md | 3 +- .../reference/configuration/providers.md | 2 +- .../content/docs/zh-tw/guides/claude-code.md | 2 +- .../reference/configuration/providers.md | 2 +- src/images/loop.ts | 2 + src/oauth/anthropic-account-refusal.ts | 76 +++++++ src/oauth/anthropic-routing.ts | 48 ++++- src/server/responses/adapter-continuation.ts | 34 ++- src/server/responses/adapter-delivery.ts | 4 +- src/server/responses/adapter-dispatch.ts | 31 +-- src/server/responses/request-transport.ts | 2 + src/server/responses/sidecar-execution.ts | 23 +- src/web-search/loop.ts | 9 +- structure/providers-and-adapters.md | 2 +- structure/providers/anthropic-account-pool.md | 24 +++ structure/transports/inventory.md | 2 +- structure/transports/responses-failover.md | 3 +- .../anthropic-quota-dispatch.test.ts | 204 +++++++++++++++++- 26 files changed, 410 insertions(+), 91 deletions(-) create mode 100644 src/oauth/anthropic-account-refusal.ts diff --git a/docs-site/src/content/docs/fr/guides/claude-code.md b/docs-site/src/content/docs/fr/guides/claude-code.md index cdc60bf5320..e2f9b6ece52 100644 --- a/docs-site/src/content/docs/fr/guides/claude-code.md +++ b/docs-site/src/content/docs/fr/guides/claude-code.md @@ -28,8 +28,7 @@ Comportement lorsque cette option est activée : - Un **429** en amont place le compte en temporisation selon `Retry-After` lorsqu'il est présent, ou selon un délai de repli, efface ses affinités et peut faire basculer la requête vers un autre compte admissible, dans les limites prévues. - L'affinité est **locale au processus** et disparaît au redémarrage du proxy. -- Les erreurs d'identification **401/403** mettent le compte en quarantaine (`needsReauth`) afin de l'exclure de la - sélection jusqu'à sa réauthentification. +- Les erreurs de renouvellement du jeton conservent la règle `needsReauth`. Un 403 confirmé lié à un abonnement ou à la facturation du compte peut déclencher un basculement avant la sortie, avec une temporisation selon `Retry-After` ou de dix minutes. Un refus d’autorisation ordinaire reste terminal. Voir la [reprise des comptes](/guides/claude-code/). - Si chaque compte éligible est en temporisation, le proxy renvoie **429** (et non 401) avec `Retry-After` lorsqu'il est connu. - La récupération, y compris le basculement 429, utilise `quotaWindow` pour classer les comptes de diff --git a/docs-site/src/content/docs/fr/reference/configuration/providers.md b/docs-site/src/content/docs/fr/reference/configuration/providers.md index 9549a6fbc94..7b285f3d9bd 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/fr/reference/configuration/providers.md @@ -246,8 +246,7 @@ rotation automatique peut déclencher des restrictions du fournisseur. | `anthropicAccountPool.stickyLimit?` | `number` | `1` | Liaisons de nouvelle session réussies conservées sur une sélection à tour de rôle. Portée 1–100. | Lorsque cette option est activée, un 429 enregistre une temporisation bornée à partir de `Retry-After` ou d'un délai de repli, puis peut -faire basculer la requête vers un autre compte. L'affinité est locale au processus et de taille bornée. Un 401/403 lié aux identifiants marque le compte -comme devant être réauthentifié. Si tous les comptes admissibles sont en temporisation, les clients reçoivent un 429 accompagné de +faire basculer la requête vers un autre compte. L'affinité est locale au processus et de taille bornée. Les erreurs de renouvellement du jeton suivent les règles de réauthentification existantes. Un 403 confirmé lié à un abonnement ou à la facturation du compte permet un basculement avant la sortie, avec une temporisation selon `Retry-After` ou de dix minutes par défaut. Un refus d’autorisation ordinaire ne déclenche pas de basculement. Si tous les comptes admissibles sont en temporisation, les clients reçoivent un 429 accompagné de `Retry-After` lorsqu'il est connu, et non une erreur d'authentification. :::caution[Expérimental] diff --git a/docs-site/src/content/docs/guides/claude-code.md b/docs-site/src/content/docs/guides/claude-code.md index 8ba8e33f281..88eae2b4083 100644 --- a/docs-site/src/content/docs/guides/claude-code.md +++ b/docs-site/src/content/docs/guides/claude-code.md @@ -61,8 +61,15 @@ Operational contract when enabled: whose known reset time has passed are discarded as unknown, including retained model-specific windows. Values without a known reset are preserved; missing data is never reported as zero usage. - Affinity is **process-local** (lost on proxy restart). -- **401/403** credential failures quarantine the account (`needsReauth`) so it is excluded from - selection until re-authenticated. +- A complete, structured **403** account-entitlement or billing refusal can rotate before + output. Recognized cases include no Claude Code access, an expired/inactive subscription, + and an insufficient Anthropic credit balance. The refused account loses its affinities and + cools for `Retry-After`, or ten minutes without a deadline. Generic permission, model/resource + access, policy and unrecognized errors stay terminal. Recovery respects model routes and + send limits; if no replacement is eligible, the original 403 is returned. This also works + with proactive pooling off. A 403 after assistant output starts never switches accounts. +- Token-refresh credential failures retain the existing `needsReauth` policy. Subscription + renewal does not require reauthentication, but the account waits for its cooldown to expire. - If every eligible account is cooling, the proxy returns **429** (not 401) with `Retry-After` when known. - Recovery, including 429 failover, uses `quotaWindow` to rank eligible replacements without diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 115a801138f..18b16e9331e 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -222,7 +222,7 @@ affinity を維持します。これらの戦略は provider enforcement を回 | `anthropicAccountPool.quotaWindow?` | `"five-hour" \| "weekly" \| "max-utilization"` | `"five-hour"` |使用量ベースのアカウント選択で使う、プロバイダー報告のキャッシュ済み使用率です。`five-hour` は従来の動作を維持します。`weekly` は週次使用量を使い、他に対象アカウントが残る間だけ 5 時間使用量が上限に達したアカウントを除外し、残らない場合はそれらへフォールバックします。`max-utilization` は判明している値のうち最も高いものを使うため、週次使用量が未取得でも 5 時間使用量を利用できます。どちらも不明なら unknown の順位付けに従います。既知の使用量は unknown より先ですが、対象がすべて unknown でも対象順の先頭を選択します。記載した 5 時間使用量による同点判定後も完全に同点なら、対象順を維持します。正常な affinity セッションを先回りして再配置することはありません。新規セッションの割り当てと、対象となる 429 代替後のルーティング復旧では、`quota` はこの期間で対象候補を直接順位付けし、`fill-first` はこの期間のしきい値と上限到達ルールを使って安定順に進み、`round-robin` はこの設定を無視します。クールダウン、フェイルオーバー上限、再認証の適格性は別のローカル状態です。アカウント別の週次使用量は、ダッシュボードのプロバイダーページで取得した後にのみ利用できます。 | | `anthropicAccountPool.stickyLimit?` | `number` | `1` |成功した新しいセッションのバインドは 1 つのラウンドロビン選択で保持されます。範囲は 1 ～ 100。 | -有効にすると、429 レコードは `Retry-After` またはデフォルトのバックオフからの制限されたクールダウンを記録し、リクエスト内でローテーションする可能性があります。アフィニティはプロセスローカルであり、サイズ制限があります。資格情報 401/403 は、アカウントに再認証が必要であることをマークします。すべての対象となるアカウントが冷却されている場合、クライアントは、既知の場合、認証エラーではなく、`Retry-After` を含む 429 を受け取ります。 +有効にすると、429 レコードは `Retry-After` またはデフォルトのバックオフからの制限されたクールダウンを記録し、リクエスト内でローテーションする可能性があります。アフィニティはプロセスローカルであり、サイズ制限があります。トークン更新の失敗は既存の再認証ルールに従います。確認済みの契約・アカウント請求の 403 は出力前に別アカウントへ切り替え、`Retry-After` または既定の10分間のクールダウンを記録します。一般的な権限拒否では切り替えません。すべての対象となるアカウントが冷却されている場合、クライアントは、既知の場合、認証エラーではなく、`Retry-After` を含む 429 を受け取ります。 :::caution[実験的] Anthropic アカウント ポリシーのリスクを理解していない限り、これは無効のままにしてください。不明な場合は、`ocx account use anthropic ` を手動で切り替えることをお勧めします。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 9d3c91587a8..3a003a48b5a 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -224,7 +224,7 @@ affinity 초기화 뒤의 기존 작업도 포함될 수 있습니다. 출력 | `anthropicAccountPool.quotaWindow?` | `"five-hour" \| "weekly" \| "max-utilization"` | `"five-hour"` | 사용량 기반 계정 선택에 사용하는, 공급자가 보고한 캐시 사용률 막대입니다. `five-hour`는 기존 동작을 유지합니다. `weekly`는 주간 막대를 사용하며 다른 사용 가능한 계정이 남아 있을 때만 5시간 막대가 소진된 계정을 건너뛰고, 아무 계정도 남지 않으면 해당 계정으로 폴백합니다. `max-utilization`은 알려진 값 중 가장 높은 값을 사용하므로 주간 사용량을 알기 전에도 5시간 사용량을 쓸 수 있고, 둘 다 모르면 unknown 순서를 따릅니다. 알려진 사용량은 unknown보다 앞서지만, 사용 가능한 계정이 모두 unknown이어도 사용 가능한 순서의 계정을 선택합니다. 앞서 설명한 5시간 사용량 동점 판정 뒤에도 완전히 같으면 사용 가능한 순서를 유지합니다. 정상 affinity 세션을 선제적으로 재배치하지 않습니다. 새 세션 배정과 가능한 429 대체 이후 라우팅 복구에서 `quota`는 이 창으로 사용 가능한 후보의 순위를 직접 매기고, `fill-first`는 이 창의 임계값과 소진 규칙에 따라 안정 순서로 이동하며, `round-robin`은 이 설정을 무시합니다. 쿨다운, failover 한도, 재인증 가능 여부는 별도의 로컬 상태로 유지됩니다. 계정별 주간 막대는 대시보드의 프로바이더 페이지에서 조회한 뒤에만 알 수 있습니다. | | `anthropicAccountPool.stickyLimit?` | `number` | `1` | 성공한 새 세션 결속이 한 번의 라운드로빈 선택에 유지되는 횟수입니다. 범위는 1–100입니다. | -활성화되면 429 레코드가 `Retry-After` 또는 기본 backoff에서 제한된 쿨다운을 기록하고, 요청 안에서 회전할 수 있습니다. 결속은 프로세스 로컬이며 크기가 제한됩니다. 자격 증명 401/403은 해당 계정이 재인증이 필요함을 표시합니다. 적격한 계정이 모두 쿨다운 중이면, 클라이언트는 인증 오류가 아니라 알려진 경우 `Retry-After`가 포함된 429를 받습니다. +활성화되면 429 레코드가 `Retry-After` 또는 기본 backoff에서 제한된 쿨다운을 기록하고, 요청 안에서 회전할 수 있습니다. 결속은 프로세스 로컬이며 크기가 제한됩니다. 토큰 갱신 실패는 기존 재인증 규칙을 따릅니다. 확인된 구독 또는 계정 결제 403은 출력 전에 다른 계정으로 전환하며 `Retry-After` 또는 기본 10분 동안 냉각합니다. 일반 권한 거부는 전환하지 않습니다. 적격한 계정이 모두 쿨다운 중이면, 클라이언트는 인증 오류가 아니라 알려진 경우 `Retry-After`가 포함된 429를 받습니다. :::caution[실험적 기능] Anthropic 계정 정책 위험을 이해하지 못한다면 이 기능은 꺼두십시오. 확신이 없으면 수동 `ocx account use anthropic ` 전환을 우선하십시오. diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 7a15ef01c81..33371d1104f 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -880,7 +880,7 @@ from a usable `Retry-After`, otherwise from the latest valid reset time among ra Anthropic reports as `rejected`, including weekly windows. Valid upstream deadlines are not shortened to a fixed cooldown ceiling; non-finite or unrepresentable deadlines are ignored. A refusal with no usable deadline falls back to a 60-second default backoff. Affinity is process-local -and size-bounded. Credential 401/403 marks the account as needing reauthentication. If all eligible accounts are cooling, clients receive 429 with +and size-bounded. Token-refresh credential failures retain the existing reauthentication policy. Classified pre-output account-entitlement/billing 403s clear affinity and cool the account for `Retry-After`, or ten minutes by default, before trying an eligible replacement. Generic or request-level 403s remain terminal; see [Claude account recovery](/guides/claude-code/). If all eligible accounts are cooling, clients receive 429 with `Retry-After` when known, not an authentication error. Anthropic responses also report the serving account's 5-hour and weekly utilization, and whichever diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index da501fe9a03..8c1f6e22c5d 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -258,7 +258,7 @@ reauth или эффективного порога исчерпания это Если функция включена, 429 записывает ограниченный cooldown из `Retry-After` или из default backoff и может переключить аккаунт уже внутри текущего запроса. Affinity локальна для процесса и -ограничена по размеру. Credential 401/403 помечает аккаунт как нуждающийся в переавторизации. +ограничена по размеру. Ошибки обновления токена сохраняют существующие правила повторной аутентификации. Подтверждённый 403 из-за подписки или оплаты аккаунта допускает переключение до вывода и cooldown по `Retry-After` либо на десять минут. Обычный отказ в доступе не переключает аккаунт. Если все eligible-аккаунты в cooldown, клиент получает 429 с `Retry-After`, если он известен, а не authentication error. diff --git a/docs-site/src/content/docs/tr/guides/claude-code.md b/docs-site/src/content/docs/tr/guides/claude-code.md index e88cd7f0ac7..54c702c2665 100644 --- a/docs-site/src/content/docs/tr/guides/claude-code.md +++ b/docs-site/src/content/docs/tr/guides/claude-code.md @@ -34,8 +34,7 @@ Etkinleştirildiğinde operasyonel sözleşme: içinde uygun başka bir hesaba dönebilir (sınırlı). - Bağlılık **işleme özeldir (process-local)** (proxy yeniden başlatıldığında kaybolur). -- **401/403** kimlik bilgisi hataları hesabı karantinaya alır (`needsReauth`), - böylece yeniden kimlik doğrulanana kadar seçimden hariç tutulur. +- Token yenileme hataları mevcut `needsReauth` kuralını korur. Doğrulanmış abonelik veya hesap ödeme 403 hatası çıktı başlamadan hesap değiştirebilir; soğuma `Retry-After` veya varsayılan on dakikadır. Genel izin reddi terminal kalır. Bkz. [hesap kurtarma](/guides/claude-code/). - Uygun tüm hesaplar soğutuluyorsa, proxy bilindiğinde `Retry-After` ile birlikte **429** (401 değil) döndürür. - 429 yük devretmesi dahil kurtarma, mevcut soğuma ve yük devretme sınırlarını diff --git a/docs-site/src/content/docs/tr/reference/configuration/providers.md b/docs-site/src/content/docs/tr/reference/configuration/providers.md index f9316a269f8..989ec7af701 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/tr/reference/configuration/providers.md @@ -277,8 +277,7 @@ ve otomatik rotasyon sağlayıcı kısıtlamalarını tetikleyebilir. Etkinleştirildiğinde 429, `Retry-After`'dan veya varsayılan bir geri çekilmeden sınırlı soğuma kaydeder ve istek içinde dönebilir. Bağlılık işleme özeldir ve -boyut sınırlıdır. Kimlik bilgisi 401/403, hesabı yeniden kimlik doğrulama -gerektiriyor olarak işaretler. Uygun tüm hesaplar soğuyorsa istemciler bir +boyut sınırlıdır. Token yenileme hataları mevcut yeniden kimlik doğrulama kurallarını korur. Doğrulanmış abonelik veya hesap ödeme 403 hatası çıktı başlamadan hesap değiştirebilir ve `Retry-After` veya varsayılan on dakika soğuma uygular. Genel izin reddi hesap değiştirmez. Uygun tüm hesaplar soğuyorsa istemciler bir kimlik doğrulama hatası değil, bilindiğinde `Retry-After` ile 429 alır. :::caution[Deneysel] diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 6f76d17d511..92225c1226d 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -248,7 +248,7 @@ affinity。这些策略不能规避 provider enforcement。 | `anthropicAccountPool.quotaWindow?` | `"five-hour" \| "weekly" \| "max-utilization"` | `"five-hour"` | 基于用量选择账户时使用的、由提供商报告并缓存的用量条。`five-hour` 保持原有行为。`weekly` 使用每周用量条，并在仍有其他可用账户时跳过 5 小时用量已耗尽的账户；若没有其他账户，则回退使用这些账户。`max-utilization` 使用已知值中的最高值，因此每周用量尚不可用时仍可使用 5 小时用量；两者都未知时，账户遵循 unknown 用量排序。已知用量排在 unknown 之前，但如果所有可用账户都未知，仍会按可用顺序选择一个账户。在前述较低 5 小时用量的同分判定之后，完全相同时也保留可用顺序。不会主动重新平衡健康且已建立亲和性的会话。在新会话分配和符合条件的 429 替代后的路由恢复中，`quota` 直接按此窗口对可用候选账户排序；`fill-first` 按此窗口的阈值和耗尽规则以稳定顺序前进；`round-robin` 忽略此设置。冷却状态、故障转移上限和重新认证资格仍是独立的本地状态。各账户的每周用量只有在控制面板的提供商页面完成查询后才可用。 | | `anthropicAccountPool.stickyLimit?` | `number` | `1` | 在一次轮询选择中保留的成功新会话绑定次数。范围 1–100。 | -启用后，429 会根据 `Retry-After` 记录有界冷却，或者使用默认退避，并且可能在同一请求内轮换。亲和性是进程本地的，并且有大小上限。凭据 401/403 会将账户标记为需要重新认证。如果所有合格账户都在冷却，客户端会在已知时收到带 `Retry-After` 的 429，而不是身份验证错误。 +启用后，429 会根据 `Retry-After` 记录有界冷却，或者使用默认退避，并且可能在同一请求内轮换。亲和性是进程本地的，并且有大小上限。令牌刷新失败保留原有重新认证规则。明确的订阅或账户计费 403 可在输出前切换账户，并按 `Retry-After` 或默认十分钟冷却；普通权限拒绝不会切换。如果所有合格账户都在冷却，客户端会在已知时收到带 `Retry-After` 的 429，而不是身份验证错误。 :::caution[Experimental] 除非你理解 Anthropic 账户策略风险，否则请保持关闭。若不确定，优先手动使用 `ocx account use anthropic ` 切换。 diff --git a/docs-site/src/content/docs/zh-tw/guides/claude-code.md b/docs-site/src/content/docs/zh-tw/guides/claude-code.md index 3d8e58d7f55..47ec6f2e7ca 100644 --- a/docs-site/src/content/docs/zh-tw/guides/claude-code.md +++ b/docs-site/src/content/docs/zh-tw/guides/claude-code.md @@ -26,7 +26,7 @@ sticky session affinity 與依用量的新工作階段選擇。它**不**控制 - 上游 **429** 會讓該帳號冷卻（有 `Retry-After` 時使用它，否則用預設 backoff）、清除其 affinity， 並可能在同一個請求內輪換到另一個合格帳號（有上限）。 - Affinity 是**程序本機**的（proxy 重啟後就會遺失）。 -- **401/403** 憑證失敗會隔離該帳號（`needsReauth`），直到重新認證前都不會參與選擇。 +- Token 更新失敗保留既有 `needsReauth` 規則。明確的訂閱或帳號計費 403 可在輸出前切換帳號，冷卻遵循 `Retry-After` 或預設十分鐘；一般權限拒絕不切換。詳見[帳號復原](/guides/claude-code/)。 - 如果每個合格帳號都在冷卻，proxy 會回傳 **429**（不是 401），並在已知時附上 `Retry-After`。 - 復原（包括 429 容錯移轉）會使用 `quotaWindow` 為合格的替代帳號排序，且不改變現有的冷卻或 容錯移轉上限；`round-robin` 會忽略 `quotaWindow`。 diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md index 3771ec64272..d583396414c 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md @@ -182,7 +182,7 @@ API-key 供應商可持有字面值金鑰或環境參考。OAuth 供應商使用 | `anthropicAccountPool.quotaWindow?` | `"five-hour" \| "weekly" \| "max-utilization"` | `"five-hour"` | 使用量型帳號選擇所採用、由供應商回報並快取的用量列。`five-hour` 保留原有行為。`weekly` 使用每週用量列，並在仍有其他可用帳號時略過 5 小時用量已用盡的帳號；若沒有其他帳號，則退回使用這些帳號。`max-utilization` 使用已知值中的最高值，因此每週用量尚未取得時仍可使用 5 小時用量；兩者都未知時，帳號遵循 unknown 用量排序。已知用量排在 unknown 之前，但若所有可用帳號都是 unknown，仍會依可用順序選出一個。完成前述較低 5 小時用量的同分判定後，完全相同時也保留可用順序。不會主動重新平衡健康且已有 affinity 的 session。在分配新 session 與符合條件的 429 替代後進行路由復原時，`quota` 直接依此視窗排序可用候選帳號；`fill-first` 依此視窗的門檻與用盡規則按穩定順序前進；`round-robin` 忽略此設定。冷卻狀態、容錯移轉上限與重新驗證資格仍是獨立的本機狀態。每個帳號的每週用量只有在 dashboard 的供應商頁面完成查詢後才可得知。 | | `anthropicAccountPool.stickyLimit?` | `number` | `1` | 在一次 round-robin 選擇上保留的成功新 session 綁定。範圍 1–100。 | -啟用時，429 記錄來自 `Retry-After` 或預設 backoff 的有界冷卻，並可能在請求內輪換。親和性為行程本地且有界。憑證 401/403 將帳號標記為需要重新認證。若所有合格帳號都在冷卻，客戶端收到附帶已知 `Retry-After` 的 429，而非認證錯誤。 +啟用時，429 記錄來自 `Retry-After` 或預設 backoff 的有界冷卻，並可能在請求內輪換。親和性為行程本地且有界。Token 更新失敗保留原有重新認證規則。已分類的訂閱或帳號計費 403 可在輸出前切換帳號，並按 `Retry-After` 或預設十分鐘冷卻；一般權限拒絕不切換。若所有合格帳號都在冷卻，客戶端收到附帶已知 `Retry-After` 的 429，而非認證錯誤。 :::caution[實驗性] 除非你了解 Anthropic 帳號政策風險，否則保持停用。不確定時偏好手動 `ocx account use anthropic ` 切換。 diff --git a/src/images/loop.ts b/src/images/loop.ts index ecdbcc3cf73..e15730d1229 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -335,6 +335,7 @@ export interface ImageBridgeDeps { retryAfterHeader: string | null, responseHeaders?: Headers, retryParsed?: OcxParsedRequest, + // A 403 requires this complete response; status-only callbacks retain 429 semantics. originalResponse?: Response, ) => | { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } @@ -687,6 +688,7 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise>(); +const verdicts = new WeakMap>(); + +/** Called only when the outgoing headers prove ownership of the selected stored bearer. */ +export function bindAnthropicRefusalCredential(response: Response, snapshot: OAuthAccessSnapshot): void { + responseCredentials.set(response, { accountId: snapshot.accountId, generation: snapshot.generation }); +} + +async function isAccountRefusal(response: Response, signal?: AbortSignal): Promise { + try { + const body = await readBoundedResponseBody(response.clone(), { signal }); + if (!body.displaySafe || body.truncated) return false; + const payload: unknown = JSON.parse(body.text); + if (!payload || typeof payload !== "object" || Array.isArray(payload) || !("error" in payload)) return false; + const error = payload.error; + if (!error || typeof error !== "object" || Array.isArray(error) || !("type" in error) || !("message" in error) + || typeof error.message !== "string") return false; + // Whole-message patterns: request-policy/model/resource refusals and quoted diagnostics + // must not acquire account authority merely by containing entitlement keywords. + if ("code" in error && error.code != null + && (typeof error.code !== "string" || !["subscription_required", "insufficient_quota", "permission_denied"].includes(error.code))) return false; + const message = error.message.trim(); + if (error.type === "permission_error" && /^(?:Your account does not have access to Claude Code|Your (?:Claude )?subscription (?:has expired|is (?:expired|inactive))|Your account does not have an active subscription)[.!]?$/i.test(message)) return true; + return (error.type === "billing_error" || error.type === "permission_error") + && /^Your credit balance is too low to access the Anthropic API\.(?: Please go to Plans & Billing to upgrade or purchase credits\.)?$/i.test(message); + } catch { + // Malformed, over-limit, interrupted and unreadable error bodies are never authority. + return false; + } +} + +export async function rotateAnthropicAccountOnResponse( + response: Response, + options: { + config: OcxConfig; + accountId: string; + sessionKey?: string | null; + decision?: AnthropicRouteDecision | null; + signal?: AbortSignal; + canRetry: boolean; + allowAccountRefusal?: boolean; + }, +): Promise { + if (options.signal?.aborted || isNonReplayableResponse(response)) return null; + if (response.status !== 429) { + if (response.status !== 403 || options.allowAccountRefusal === false) return null; + const sent = responseCredentials.get(response); + if (!sent || sent.accountId !== options.accountId) return null; + let verdict = verdicts.get(response); + if (!verdict) { + verdict = isAccountRefusal(response, options.signal); + verdicts.set(response, verdict); + } + if (!await verdict || options.signal?.aborted) return null; + // A late refusal must not cool a new credential stored while its body was being read. + const current = getAccountCredentialWithStatus("anthropic", sent.accountId); + if (!current || current.paused || current.needsReauth || credentialGeneration(current.credential) !== sent.generation) return null; + } + const status = response.status === 429 ? 429 : 403; + if (!options.canRetry) { + recordAnthropicAccountRefusal(options.config, options.accountId, status, + response.headers.get("retry-after"), Date.now(), response.headers); + return null; + } + return rotateAnthropicAccountOnRefusal(options.config, options.accountId, status, + response.headers.get("retry-after"), options.sessionKey, Date.now(), response.headers, options.decision); +} diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index a65e0adc30f..3e324178f20 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -3,7 +3,7 @@ * * Default OFF. When enabled: * - Sticky session affinity across requests that share a session key - * - 429 cools the failed account and fails over to another eligible account + * - 429 and classified pre-output account 403s cool the failed account and fail over * - New sessions use `strategy` (default quota): lowest known fiveHour usage (#493), * round-robin, or fill-first — affinity still wins for bound sessions * @@ -12,8 +12,8 @@ * * Affinity is process-local (lost on restart). Cooldown uses Retry-After when present, else * the reset time of whichever rate-limit window upstream reports as rejected, else a default - * backoff. 401/403 credential failures should set needsReauth on the store (existing OAuth - * path) so the account is excluded from eligibility. + * backoff. Classified entitlement/billing 403s use a ten-minute default cooldown. + * Token refresh failures retain the existing store needsReauth policy. */ import { createHash } from "node:crypto"; import { captureOAuthAccountSelection, commitOAuthAccountSelection, credentialGeneration, getAccountSet, getAccountCredential, getAccountCredentialWithStatus } from "./store"; @@ -53,6 +53,7 @@ export type AnthropicRateLimitHeaders = Pick; const PROVIDER = "anthropic"; /** Backoff only when upstream supplies no usable deadline. */ const DEFAULT_COOLDOWN_MS = 60_000; +const ACCOUNT_REFUSAL_COOLDOWN_MS = 10 * 60_000; const AFFINITY_IDLE_TTL_MS = 24 * 60 * 60_000; const MAX_AFFINITY_ENTRIES = 2_000; const MAX_AFFINITY_COMPONENT_BYTES = 512; @@ -879,7 +880,21 @@ export function rotateAnthropicAccountOn429( rateLimitHeaders?: AnthropicRateLimitHeaders | null, decision: AnthropicRouteDecision | null = null, ): string | null { - if (!recordAnthropicAccount429(config, failedAccountId, retryAfterHeader, now, rateLimitHeaders)) return null; + return rotateAnthropicAccountOnRefusal(config, failedAccountId, 429, retryAfterHeader, sessionKey, now, rateLimitHeaders, decision); +} + +/** The caller proves that a 403 is an account refusal before entering this pool policy. */ +export function rotateAnthropicAccountOnRefusal( + config: OcxConfig, + failedAccountId: string, + status: 429 | 403, + retryAfterHeader: string | null | undefined, + sessionKey?: string | null, + now = Date.now(), + rateLimitHeaders?: AnthropicRateLimitHeaders | null, + decision: AnthropicRouteDecision | null = null, +): string | null { + if (!recordAnthropicAccountRefusal(config, failedAccountId, status, retryAfterHeader, now, rateLimitHeaders)) return null; // The pool's strategy is a PROACTIVE policy. When the pool is disabled, reactive // presence-only recovery must not silently reactivate round-robin/fill-first merely @@ -889,13 +904,14 @@ export function rotateAnthropicAccountOn429( ? pickAlternateAnthropicAccount(config, failedAccountId, now, decision) : pickLowestUsage(config, failedAccountId, now); if (!next) { - console.warn(`[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning 429`); + console.warn(`[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning ${status}`); return null; } - console.warn( - `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`, - ); + console.warn(status === 403 + ? `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}account unavailable (403); failing over` + : `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`); + return next; } @@ -906,6 +922,18 @@ export function recordAnthropicAccount429( retryAfterHeader: string | null | undefined, now = Date.now(), rateLimitHeaders?: AnthropicRateLimitHeaders | null, +): boolean { + return recordAnthropicAccountRefusal(config, failedAccountId, 429, retryAfterHeader, now, rateLimitHeaders); +} + +/** Account entitlement refusals use a finite backoff; renewing a plan does not require login. */ +export function recordAnthropicAccountRefusal( + config: OcxConfig, + failedAccountId: string, + status: 429 | 403, + retryAfterHeader: string | null | undefined, + now = Date.now(), + rateLimitHeaders?: AnthropicRateLimitHeaders | null, ): boolean { // Reactive 429 failover is NOT gated on the pool flag. That flag buys PROACTIVE routing -- // session affinity, quota-ranked new-session selection, autoSwitchThreshold, strategy -- all @@ -923,8 +951,8 @@ export function recordAnthropicAccount429( // without that fallback such a refusal cools for the 60s default and the exhausted // account is back in the rotation a minute later. const parsedRetry = parseRetryAfterMs(retryAfterHeader, now); - const resetDerived = parsedRetry === undefined ? parseRateLimitReset(rateLimitHeaders, now) : undefined; - const cooldownMs = parsedRetry ?? resetDerived?.delayMs ?? DEFAULT_COOLDOWN_MS; + const resetDerived = status === 429 && parsedRetry === undefined ? parseRateLimitReset(rateLimitHeaders, now) : undefined; + const cooldownMs = parsedRetry ?? resetDerived?.delayMs ?? (status === 403 ? ACCOUNT_REFUSAL_COOLDOWN_MS : DEFAULT_COOLDOWN_MS); const cooldownGeneration = noteAnthropicCooldownMutation(failedAccountId); upstreamHealth.set(failedAccountId, { cooldownUntil: now + cooldownMs, diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index d111cff8577..2d3aa39e5e7 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -1,3 +1,4 @@ +import { rotateAnthropicAccountOnResponse } from "../../oauth/anthropic-account-refusal"; import type { ResponsesRequestContext } from "./core-options"; import type { PreparedResponsesRequest } from "./request-prepare"; import type { ResponsesTransport } from "./request-transport"; @@ -34,8 +35,6 @@ import { resolveWireProtocolOverride } from "../adapter-resolve"; import { bindRouteReasoningReplayScope } from "./core-replay"; import { ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, - rotateAnthropicAccountOn429, - recordAnthropicAccount429, getAnthropicPoolAccessSnapshot, formatAnthropicProviderForLog, } from "../../oauth/anthropic-routing"; @@ -158,6 +157,8 @@ export function createAdapterContinuations( const fetchTerminalGuardContinuation = async function* ( nextParsed: OcxParsedRequest, initialRecoveryKind?: AttemptRecoveryKind, + // Ordinary terminal continuations have already delivered output; empty retries opt in. + allowAccountRefusal = false, ): AsyncGenerator { let response: Response | undefined; let kiroRefusalPendingReplay: Response | undefined; @@ -389,25 +390,21 @@ export function createAdapterContinuations( } } if ( - response.status === 429 + (response.status === 429 || response.status === 403) && transportState.anthropicPoolAccountId && !isNonReplayableResponse(response) - && transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST ) { - const nextAccountId = rotateAnthropicAccountOn429( - config, - transportState.anthropicPoolAccountId, - response.headers.get("retry-after"), - anthropicSessionKey, - Date.now(), - response.headers, - transportState.anthropicRouteDecision, - ); + const nextAccountId = await rotateAnthropicAccountOnResponse(response, { + config, accountId: transportState.anthropicPoolAccountId, sessionKey: anthropicSessionKey, + decision: transportState.anthropicRouteDecision, signal: upstream.signal, + canRetry: transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, + allowAccountRefusal, + }); if (nextAccountId) { - try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } try { const admitted = await commitResolvedOAuthSelection(await getAnthropicPoolAccessSnapshot(nextAccountId)); if (!admitted) throw new Error("OAuth selection changed during recovery"); + try { void response.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } transportState.anthropicPoolAccountId = admitted.accountId; transportState.anthropicPoolFailovers += 1; route.provider = { ...route.provider, apiKey: admitted.accessToken }; @@ -426,11 +423,6 @@ export function createAdapterContinuations( } } } - if (response.status === 429 && transportState.anthropicPoolAccountId - && transportState.anthropicPoolFailovers >= ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST) { - recordAnthropicAccount429(config, transportState.anthropicPoolAccountId, - response.headers.get("retry-after"), Date.now(), response.headers); - } // Generic OAuth rotation for the continuation loop. The streaming loop grew this arm with // #2568 and this one did not, so an xAI/Cursor/Kimi/Copilot/Antigravity/Nous continuation // 429 stayed terminal even with failover fully active -- the same class of divergence the @@ -704,14 +696,14 @@ export function createAdapterContinuations( }; const fetchGuardedEmptyCompletionRetry = (): AsyncIterable => { - const retryEvents = fetchTerminalGuardContinuation(parsed, "empty-completion"); + const retryEvents = fetchTerminalGuardContinuation(parsed, "empty-completion", true); return terminalGuardEnabled ? guardTerminalEventStream({ parsed, firstEvents: retryEvents, adapterName: transportState.activeAdapter.name, maxAutoContinuations: 1, - continuation: fetchTerminalGuardContinuation, + continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream), }) : retryEvents; }; diff --git a/src/server/responses/adapter-delivery.ts b/src/server/responses/adapter-delivery.ts index 48c30dd87ed..4223dc0d4a1 100644 --- a/src/server/responses/adapter-delivery.ts +++ b/src/server/responses/adapter-delivery.ts @@ -110,7 +110,7 @@ export async function deliverAdapterResponse( firstEvents: initialEventStream, adapterName: transportState.activeAdapter.name, maxAutoContinuations: 1, - continuation: fetchTerminalGuardContinuation, + continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream), }) : initialEventStream; // The empty-completion guard sits OUTSIDE the terminal guard: a completed @@ -235,7 +235,7 @@ export async function deliverAdapterResponse( firstEvents: (async function* () { yield* initialEvents; })(), adapterName: transportState.activeAdapter.name, maxAutoContinuations: 1, - continuation: fetchTerminalGuardContinuation, + continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream), })) guardedEvents.push(event); } else { guardedEvents = initialEvents; diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index ba7f509eba5..b06c7692ea4 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -1,3 +1,4 @@ +import { rotateAnthropicAccountOnResponse } from "../../oauth/anthropic-account-refusal"; import { isNonReplayableResponse } from "../../lib/upstream-retry"; import { isLocalUpstream } from "../../lib/local-upstream"; import type { ResponsesRequestContext, ResponsesAdmissionState } from "./core-options"; @@ -57,8 +58,6 @@ import { bindRouteReasoningReplayScope } from "./core-replay"; import { AnthropicAccountCooldownError, ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, - rotateAnthropicAccountOn429, - recordAnthropicAccount429, getAnthropicPoolAccessSnapshot, formatAnthropicProviderForLog, } from "../../oauth/anthropic-routing"; @@ -986,27 +985,22 @@ export async function prepareAdapterExchange( if (isNonReplayableResponse(upstreamResponse)) continue recovery; } - // Opt-in Anthropic OAuth account pool (#294): cool the failed account and retry - // with another eligible OAuth account (bounded per request). Disabled by default. + // Anthropic OAuth: recover a rate limit or proven account entitlement refusal + // before output, within the shared request and account rotation limits. while ( - upstreamResponse.status === 429 + (upstreamResponse.status === 429 || upstreamResponse.status === 403) && transportState.anthropicPoolAccountId - && transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST ) { - const nextAccountId = rotateAnthropicAccountOn429( - config, - transportState.anthropicPoolAccountId, - upstreamResponse.headers.get("retry-after"), - anthropicSessionKey, - Date.now(), - upstreamResponse.headers, - transportState.anthropicRouteDecision, - ); + const nextAccountId = await rotateAnthropicAccountOnResponse(upstreamResponse, { + config, accountId: transportState.anthropicPoolAccountId, sessionKey: anthropicSessionKey, + decision: transportState.anthropicRouteDecision, signal: upstream.signal, + canRetry: transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, + }); if (!nextAccountId) break; - try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } try { const admitted = await commitResolvedOAuthSelection(await getAnthropicPoolAccessSnapshot(nextAccountId)); if (!admitted) throw new Error("OAuth selection changed during recovery"); + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } transportState.anthropicPoolAccountId = admitted.accountId; transportState.anthropicPoolFailovers += 1; route.provider = { ...route.provider, apiKey: admitted.accessToken }; @@ -1026,11 +1020,6 @@ export async function prepareAdapterExchange( break; } } - if (upstreamResponse.status === 429 && transportState.anthropicPoolAccountId - && transportState.anthropicPoolFailovers >= ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST) { - recordAnthropicAccount429(config, transportState.anthropicPoolAccountId, - upstreamResponse.headers.get("retry-after"), Date.now(), upstreamResponse.headers); - } // Generic OAuth account failover (#2568) rotates reactively after a refusal when // two accounts are stored. Kiro additionally classifies bounded 400/403 refusals; // all other providers retain the original 429 loop below. diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index ebabd087625..7d1bf0feec0 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -1,3 +1,4 @@ +import { bindAnthropicRefusalCredential } from "../../oauth/anthropic-account-refusal"; import type { ResponsesRequestContext, ResponsesAdmissionState } from "./core-options"; import type { PreparedResponsesRequest } from "./request-prepare"; import type { OAuthAccessSnapshot } from "../../oauth"; @@ -521,6 +522,7 @@ export async function prepareResponsesTransport( try { const current = getAccountCredentialWithStatus("anthropic", snapshot.accountId); if (current && !current.needsReauth && credentialGeneration(current.credential) === snapshot.generation) { + bindAnthropicRefusalCredential(response, snapshot); recordAnthropicAccountQuotaFromHeaders(snapshot.accountId, response.headers, writerGeneration); } } catch { /* best-effort observation cannot fail the response */ } diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 771fb915e77..db7cff14a2f 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -1,3 +1,4 @@ +import { rotateAnthropicAccountOnResponse } from "../../oauth/anthropic-account-refusal"; import type { ResponsesRequestContext } from "./core-options"; import type { PreparedResponsesRequest } from "./request-prepare"; import type { ResponsesTransport } from "./request-transport"; @@ -32,7 +33,6 @@ import { persistKiroAccountState } from "../../providers/kiro-account-state-disk import { readDisplaySafeErrorText } from "./core-errors"; import { ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, - rotateAnthropicAccountOn429, getAnthropicPoolAccessSnapshot, formatAnthropicProviderForLog, } from "../../oauth/anthropic-routing"; @@ -183,7 +183,7 @@ export async function executeResponsesSidecars( retryParsed?: OcxParsedRequest, originalResponse?: Response, ): Promise<{ adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } | null> => { - if (route.providerName !== "kiro" && originalResponse && originalResponse.status !== 429) return null; + if (route.providerName !== "kiro" && !(route.providerName === "anthropic" && originalResponse?.status === 403) && originalResponse && originalResponse.status !== 429) return null; const refusal = route.providerName === "kiro" && originalResponse ? classifyKiroRefusal(originalResponse.status, await readDisplaySafeErrorText(originalResponse.clone(), options.abortSignal ?? new AbortController().signal, "")).kind @@ -200,7 +200,7 @@ export async function executeResponsesSidecars( // sidecar loops used to flatten all three to `key-429`, so an account rotation read as a key // rotation in the attempt row and in the Logs UI. let recoveryKind: AttemptRecoveryKind = "key-429"; - const rotated = route.providerName !== "kiro" || originalResponse?.status === 429 + const rotated = !originalResponse || originalResponse.status === 429 ? rotateProviderTransportOn429(config, route.providerName, route.provider, { retryAfter, now: Date.now(), @@ -260,7 +260,6 @@ export async function executeResponsesSidecars( // web-search or image-bridge turn was terminal even with the pool fully enabled -- while // the very same 429 on the main response path rotated. transportState.anthropicPoolAccountId - && transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST ) { // Same intersection for the Anthropic roster: its own per-request bound still applies, // and the shared budget decides whether this request may spend another send at all. @@ -268,16 +267,12 @@ export async function executeResponsesSidecars( "auth-recovery", `${route.providerName}|${route.modelId}|sidecar-anthropic-429`, ); - if (!hop.allowed) return null; - const nextAccountId = rotateAnthropicAccountOn429( - config, - transportState.anthropicPoolAccountId, - retryAfter, - anthropicSessionKey, - Date.now(), - responseHeaders, - transportState.anthropicRouteDecision, - ); + const nextAccountId = await rotateAnthropicAccountOnResponse( + originalResponse ?? new Response(null, { status: 429, headers: responseHeaders ?? (retryAfter ? { "retry-after": retryAfter } : undefined) }), { + config, accountId: transportState.anthropicPoolAccountId, sessionKey: anthropicSessionKey, + decision: transportState.anthropicRouteDecision, signal: options.abortSignal, + canRetry: hop.allowed && transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, + }); if (!nextAccountId) { hop.permit?.release(); return null; diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index e7a544654b1..03061366e1c 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -325,7 +325,7 @@ export interface WebSearchLoopDeps { /** Called before each routed-model dispatch in the loop, for attempt telemetry. Same-target 429 replays pass the `rate-limit-429` recovery kind. */ onAttemptSend?: (recovery?: AttemptRecoveryKind) => void; /** - * 429 failover hook: rotate the provider's active credential and return a rebuilt adapter, + * Account/key failover hook (429, plus provider-classified pre-output refusals): rotate the provider's active credential and return a rebuilt adapter, * or null when the pool is exhausted. Async hooks support OAuth refresh; existing synchronous * key-pool hooks remain valid. * @@ -346,6 +346,7 @@ export interface WebSearchLoopDeps { retryAfterHeader: string | null, responseHeaders?: Headers, retryParsed?: OcxParsedRequest, + // A 403 requires this complete response; status-only callbacks retain 429 semantics. originalResponse?: Response, ) => | { adapter: ProviderAdapter; recoveryKind: AttemptRecoveryKind } @@ -405,6 +406,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise` position, never its name. + `src/server/responses/request-transport.ts` resolves the final Anthropic model ID once for each enabled-pool request and holds its model route through admission. The three account-refusal retry sites in `adapter-dispatch.ts`, `adapter-continuation.ts` and `sidecar-execution.ts` use the same route, keep the original refusal if no replacement exists inside it, and preserve existing send and output replay limits. A local cooldown returns 429 with the earliest known Retry-After among accounts the route can use when they recover; explicit fallback includes usable ordinary-pool accounts even when the saved route IDs have been removed, while a strict route stays route-scoped. Selection, refusal and 429-rotation logs use the rule’s 1-based `route:#` position, never its name. +Classified account-entitlement 403 recovery follows the [Anthropic account-pool contract](../providers/anthropic-account-pool.md#account-entitlement-refusal-recovery). `src/server/responses/compaction-recovery-policy.ts` is a pure eligibility policy, not a dispatcher. It requires explicit configuration and normalized attempt evidence, preserves ordinary requests, and refuses cancellation, committed semantic output, tool effects, protected failures, exhausted diff --git a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts index 3e641d93713..e16a1fe04ce 100644 --- a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +++ b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts @@ -4,7 +4,7 @@ import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { clearAnthropicAccountPoolState, forgetAnthropicFailoverQuorum } from "../../../src/oauth/anthropic-routing"; +import { clearAnthropicAccountPoolState, forgetAnthropicFailoverQuorum, getEligibleAnthropicAccounts, getAnthropicAccountHealthSnapshot } from "../../../src/oauth/anthropic-routing"; import { clearGenericFailoverHealth } from "../../../src/oauth/generic-account-failover"; import { getAccountSet, saveAccountCredential, saveCredential, setActiveAccount } from "../../../src/oauth/store"; import { clearAccountQuotaCache, getCachedProviderAccountQuota, resetProviderQuotaReconcileStateForTests } from "../../../src/providers/quota"; @@ -372,3 +372,205 @@ test("real image bridge routed loop records A429 and B200 through fetchForReques expectQuota(a!, 100, 61); expectQuota(b!, 23, 47); }); + +// BUG-R6340: account refusals recover only before output; generic 403s stay terminal. +function accountRefusal(message = "Your account does not have access to Claude Code", type = "permission_error") { + return Response.json({ type: "error", error: { type, message } }, { status: 403 }); +} + +test.each([false, true])("account entitlement 403 fails over and clears sticky eligibility (pool=%s)", async enabled => { + const [a, b] = await seed(); + const config = configFor(body => sent.length === 1 ? accountRefusal() : answer(body.stream === true)); + config.anthropicAccountPool = { enabled, stickyLimit: 1 }; + const response = await post(config, { prompt_cache_key: "same-session" }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expect(getEligibleAnthropicAccounts()).toEqual([b!]); + expect(getAnthropicAccountHealthSnapshot(a!)).not.toBeNull(); + const subsequent = await post(config, { prompt_cache_key: "same-session" }); + expect(subsequent.status).toBe(200); + await subsequent.text(); + expect(sent[2]!.authorization).toBe(`Bearer ${credential(1).access}`); +}); + +test.each([ + ["Your Claude subscription has expired", "permission_error"], + ["Your account does not have an active subscription", "permission_error"], + ["Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", "billing_error"], +])("credential 403 recovers: %s", async (message, type) => { + await seed(); + const response = await post(configFor(body => sent.length === 1 ? accountRefusal(message, type) : answer(body.stream === true)), { stream: true }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent).toHaveLength(2); +}); + +test.each([ + ["Request blocked by content policy", "permission_error"], + ["You do not have permission to access this resource", "permission_error"], + ["Your account does not have access to this model", "permission_error"], + ["Subscription required to access this content", "permission_error"], + ["Your account does not have access to Claude Code", "invalid_request_error"], + ["Request denied: Your account does not have access to Claude Code", "permission_error"], +])("request-level or ambiguous 403 stays terminal: %s (%s)", async (message, type) => { + const ids = await seed(); + const response = await post(configFor(() => accountRefusal(message, type))); + expect(response.status).toBe(403); + await response.text(); + expect(sent).toHaveLength(1); + expect(getEligibleAnthropicAccounts()).toEqual(ids); +}); + +test("all accounts refused retain the upstream 403 and cool each account", async () => { + const ids = await seed(); + const response = await post(configFor(() => accountRefusal())); + expect(response.status).toBe(403); + expect(await response.text()).toContain("Your account does not have access to Claude Code"); + expect(sent).toHaveLength(2); + for (const id of ids) expect(getAnthropicAccountHealthSnapshot(id)).not.toBeNull(); +}); + +test.each(["search", "image"])("real %s sidecar recovers account 403", async kind => { + const [a, b] = await seed(); + const config = configFor(() => sent.length === 1 ? accountRefusal() : answer(true)); + if (kind === "search") config.webSearchSidecar = { backend: "anthropic", enabled: true }; + else { + config.images = { bridgeEnabled: true }; + config.providers.xai = { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "key", apiKey: "synthetic-image-key" }; + } + const response = await post(config, { stream: true, tools: [{ type: kind === "search" ? "web_search" : "image_generation" }] }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent).toHaveLength(2); + expect(getEligibleAnthropicAccounts()).toEqual([b!]); + expect(getAnthropicAccountHealthSnapshot(a!)).not.toBeNull(); +}); + +test("empty pre-output continuation recovers account 403", async () => { + await seed(); + const config = configFor(body => { + if (sent.length === 1) return answer(body.stream === true, "0.1", "0.2", ""); + if (sent.length === 2) return accountRefusal(); + return answer(body.stream === true); + }); + config.emptyCompletionRetry = true; + const response = await post(config, { stream: true }); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); +}); + +test("403 continuation after assistant output never switches accounts", async () => { + const ids = await seed(); + const response = await post(configFor(body => sent.length === 1 + ? answer(body.stream === true, "0.1", "0.2", "I will modify the file now.") : accountRefusal()), { + stream: true, input: "Please modify the file now", + tools: [{ type: "function", name: "read_file", description: "read a file", parameters: { type: "object" } }], + }); + expect(await response.text()).toContain("I will modify the file now."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(0).access}`]); + expect(getEligibleAnthropicAccounts()).toEqual(ids); +}); + + +test.each(overriddenHeaders)("403 with $label cannot cool the selected stored credential", async ({ headers }) => { + const ids = await seed(); + const response = await post(configFor(() => accountRefusal(), headers)); + expect(response.status).toBe(403); + await response.text(); + expect(sent).toHaveLength(1); + expect(getEligibleAnthropicAccounts()).toEqual(ids); +}); + +test("late 403 from a replaced credential cannot cool its replacement", async () => { + const ids = await seed(); + const entered = deferred(); + const returned = deferred(); + const pending = post(configFor(() => { entered.resolve(); return returned.promise; })); + await entered.promise; + await saveAccountCredential("anthropic", ids[0]!, { + ...credential(0), access: "synthetic-renewed-access", refresh: "synthetic-renewed-refresh", + }); + returned.resolve(accountRefusal()); + const response = await pending; + expect(response.status).toBe(403); + await response.text(); + expect(sent).toHaveLength(1); + expect(getEligibleAnthropicAccounts()).toEqual(ids); +}); + +test.each([ + "not JSON", + JSON.stringify({ error: { message: "Your account does not have access to Claude Code" } }), + JSON.stringify({ error: { type: "permission_error", code: "invalid_prompt", message: "Your account does not have access to Claude Code" } }), + JSON.stringify({ error: { type: "permission_error", message: "Your account does not have access to Claude Code" }, padding: "x".repeat(70_000) }), +])("unproven 403 body leaves eligibility unchanged (%#)", async body => { + const ids = await seed(); + const response = await post(configFor(() => new Response(body, { status: 403 }))); + expect(response.status).toBe(403); + await response.text(); + expect(sent).toHaveLength(1); + expect(getEligibleAnthropicAccounts()).toEqual(ids); +}); + +test("account 403 cools the final sender even when the request's rotation cap is spent", async () => { + const ids = await seed(5); + const response = await post(configFor(() => accountRefusal())); + expect(response.status).toBe(403); + await response.text(); + expect(sent).toHaveLength(4); + for (let index = 0; index < 4; index++) expect(getAnthropicAccountHealthSnapshot(ids[index]!)).not.toBeNull(); + expect(getEligibleAnthropicAccounts()).toEqual([ids[4]!]); +}); + +test("strict model route does not widen on an account 403", async () => { + const ids = await seed(); + const config = configFor(() => accountRefusal()); + config.anthropicAccountPool = { enabled: true, routes: [{ name: "restricted", match: "claude-sonnet-4-5", accounts: [ids[0]!] }] }; + const response = await post(config); + expect(response.status).toBe(403); + await response.text(); + expect(sent).toHaveLength(1); + expect(getEligibleAnthropicAccounts()).toEqual([ids[1]!]); +}); + + +test("Claude Messages bridge fails over on account entitlement 403", async () => { + await seed(); + const config = configFor(body => sent.length === 1 ? accountRefusal() : answer(body.stream === true)); + config.anthropicAccountPool = { enabled: true }; + const { handleClaudeMessages } = await import("../../../src/server/claude-messages"); + const response = await handleClaudeMessages(new Request("http://localhost/v1/messages", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", messages: [{ role: "user", content: "Answer briefly" }], max_tokens: 64, stream: false }), + }), config, { model: "", provider: "" }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); +}); + + +test("a generic 403 on the replacement is terminal and does not cool that replacement", async () => { + const [a, b] = await seed(); + const response = await post(configFor(() => sent.length === 1 ? accountRefusal() : accountRefusal("Request blocked by content policy"))); + expect(response.status).toBe(403); + expect(await response.text()).toContain("Request blocked by content policy"); + expect(sent).toHaveLength(2); + expect(getAnthropicAccountHealthSnapshot(a!)).not.toBeNull(); + expect(getAnthropicAccountHealthSnapshot(b!)).toBeNull(); +}); + +test.each(["search", "image"])("real %s sidecar leaves request-level 403 terminal", async kind => { + const ids = await seed(); + const config = configFor(() => accountRefusal("Request blocked by content policy")); + if (kind === "search") config.webSearchSidecar = { backend: "anthropic", enabled: true }; + else { + config.images = { bridgeEnabled: true }; + config.providers.xai = { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "key", apiKey: "synthetic-image-key" }; + } + const response = await post(config, { stream: true, tools: [{ type: kind === "search" ? "web_search" : "image_generation" }] }); + expect(response.status).toBe(403); + await response.text(); + expect(sent).toHaveLength(1); + expect(getEligibleAnthropicAccounts()).toEqual(ids); +});