From 6d9b1b48de27f6485afb4a87ec3b7e19ab9bba01 Mon Sep 17 00:00:00 2001 From: 02356abc <198679067+02356abc@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:42:29 +0800 Subject: [PATCH] fix(agent): trigger fallback on quota-exhaustion 429s (e.g. Kimi monthly limit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-part fix for the same bug: 1. error_classifier.py: apply _BILLING_PATTERNS + _USAGE_LIMIT_PATTERNS disambiguation to the 429 branch, same as the existing 402/400/no-status paths. Kimi's "monthly usage limit" 429 now classifies as billing instead of rate_limit, avoiding 3 wasted retry attempts with backoff. 2. agent_runtime_helpers.py: replace the ad-hoc hardcoded strings in recover_with_credential_pool with the shared _USAGE_LIMIT_PATTERNS list so the fallback_model chain is actually triggered when quota is exhausted. Previously "monthly usage limit" was not matched and the error surfaced directly to the user instead of falling back. A plain 429 with no billing/quota phrase continues to classify as rate_limit (retryable=True) — no regression for normal throttling. Co-Authored-By: Claude Sonnet 4.6 --- agent/agent_runtime_helpers.py | 6 ++-- agent/error_classifier.py | 31 ++++++++++++++++- tests/agent/test_error_classifier.py | 50 ++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 73f3cba435df6..4aac12120a07a 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -35,7 +35,7 @@ from agent.tool_dispatch_helpers import _trajectory_normalize_msg, make_tool_result_message from agent.trajectory import convert_scratchpad_to_think from agent.credential_pool import STATUS_EXHAUSTED -from agent.error_classifier import FailoverReason +from agent.error_classifier import FailoverReason, _USAGE_LIMIT_PATTERNS from utils import base_url_host_matches, base_url_hostname, env_var_enabled, atomic_json_write logger = logging.getLogger(__name__) @@ -621,11 +621,11 @@ def recover_with_credential_pool( if error_context: context_reason = str(error_context.get("reason") or "").lower() context_message = str(error_context.get("message") or "").lower() + haystack = context_reason + " " + context_message usage_limit_reached = ( "usage_limit_reached" in context_reason or "gousagelimit" in context_reason - or "usage limit reached" in context_message - or "usage limit has been reached" in context_message + or any(p in haystack for p in _USAGE_LIMIT_PATTERNS) ) if not has_retried_429 and not usage_limit_reached: return False, True diff --git a/agent/error_classifier.py b/agent/error_classifier.py index e8a44866b28e1..7529f32d9911a 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -822,7 +822,36 @@ def _classify_by_status( ) if status_code == 429: - # Already checked long_context_tier above; this is a normal rate limit + # Already checked long_context_tier above. + # Some providers return monthly/cycle quota exhaustion as 429 (e.g. + # Kimi "monthly usage limit"). Disambiguate the same way as 402: + # if the message carries a billing or permanent-quota-exhaustion + # signal with no transient "try again / resets at / wait" qualifier, + # classify as billing (non-retryable) so we don't burn retries on a + # quota that won't recover until the next billing cycle. + if any(p in error_msg for p in _BILLING_PATTERNS): + return result_fn( + FailoverReason.billing, + retryable=False, + should_rotate_credential=True, + should_fallback=True, + ) + # Only check usage-limit patterns when the message does NOT already + # carry an explicit rate-limit signal (e.g. "rate limit exceeded"). + # _USAGE_LIMIT_PATTERNS includes broad tokens like "limit exceeded" + # that overlap with normal transient 429 messages; _RATE_LIMIT_PATTERNS + # takes priority so those stay retryable. + has_rate_limit_signal = any(p in error_msg for p in _RATE_LIMIT_PATTERNS) + has_usage_limit = not has_rate_limit_signal and any(p in error_msg for p in _USAGE_LIMIT_PATTERNS) + if has_usage_limit: + has_transient_signal = any(p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS) + if not has_transient_signal: + return result_fn( + FailoverReason.billing, + retryable=False, + should_rotate_credential=True, + should_fallback=True, + ) return result_fn( FailoverReason.rate_limit, retryable=True, diff --git a/tests/agent/test_error_classifier.py b/tests/agent/test_error_classifier.py index b98fbe5beb9f6..b9bb033a5b300 100644 --- a/tests/agent/test_error_classifier.py +++ b/tests/agent/test_error_classifier.py @@ -324,6 +324,56 @@ def test_alibaba_rate_increased_too_quickly(self): assert result.retryable is True assert result.should_rotate_credential is True + def test_429_kimi_monthly_usage_limit_is_billing(self): + """Kimi returns HTTP 429 for monthly quota exhaustion — must not retry. + + Real error: "You've reached kimi monthly usage limit for this billing + cycle. Your quota will be refreshed in the next cycle." + 'usage limit' matches _USAGE_LIMIT_PATTERNS; 'refreshed in the next + cycle' is NOT in _USAGE_LIMIT_TRANSIENT_SIGNALS → billing, non-retryable. + """ + msg = ( + "You've reached kimi monthly usage limit for this billing cycle. " + "Your quota will be refreshed in the next cycle. Upgrade to get more." + ) + e = MockAPIError(msg, status_code=429) + result = classify_api_error(e, provider="kimi") + assert result.reason == FailoverReason.billing + assert result.retryable is False + assert result.should_rotate_credential is True + assert result.should_fallback is True + + def test_429_with_transient_usage_limit_is_rate_limit(self): + """429 carrying 'usage limit' + 'try again' is a transient quota → rate_limit.""" + e = MockAPIError( + "usage limit exceeded, try again in 60 seconds", + status_code=429, + ) + result = classify_api_error(e) + assert result.reason == FailoverReason.rate_limit + assert result.retryable is True + + def test_429_with_billing_pattern_is_billing(self): + """429 carrying an explicit billing phrase (e.g. 'insufficient credits') → billing.""" + e = MockAPIError("insufficient credits", status_code=429) + result = classify_api_error(e) + assert result.reason == FailoverReason.billing + assert result.retryable is False + + def test_429_plain_no_message_is_rate_limit(self): + """Plain 429 with no billing/usage signal stays rate_limit (regression guard).""" + e = MockAPIError("Too Many Requests", status_code=429) + result = classify_api_error(e) + assert result.reason == FailoverReason.rate_limit + assert result.retryable is True + + def test_429_quota_with_reset_window_is_rate_limit(self): + """429 + 'quota' + 'resets at' → transient, stays rate_limit.""" + e = MockAPIError("quota exceeded, resets at midnight UTC", status_code=429) + result = classify_api_error(e) + assert result.reason == FailoverReason.rate_limit + assert result.retryable is True + # ── Server errors ── def test_500_server_error(self):