fix(agent): trigger fallback on quota-exhaustion 429s (e.g. Kimi monthly limit)

agent/agent_runtime_helpers.py

@@ -35,7 +35,7 @@
 from agent.tool_dispatch_helpers import _trajectory_normalize_msg, make_tool_result_message
 from agent.trajectory import convert_scratchpad_to_think
 from agent.credential_pool import STATUS_EXHAUSTED
-from agent.error_classifier import FailoverReason
+from agent.error_classifier import FailoverReason, _USAGE_LIMIT_PATTERNS
 from utils import base_url_host_matches, base_url_hostname, env_var_enabled, atomic_json_write
 
 logger = logging.getLogger(__name__)
@@ -621,11 +621,11 @@ def recover_with_credential_pool(
         if error_context:
             context_reason = str(error_context.get("reason") or "").lower()
             context_message = str(error_context.get("message") or "").lower()
+            haystack = context_reason + " " + context_message
             usage_limit_reached = (
                 "usage_limit_reached" in context_reason
                 or "gousagelimit" in context_reason
-                or "usage limit reached" in context_message
-                or "usage limit has been reached" in context_message
+                or any(p in haystack for p in _USAGE_LIMIT_PATTERNS)
             )
         if not has_retried_429 and not usage_limit_reached:
             return False, True

agent/error_classifier.py

@@ -822,7 +822,36 @@ def _classify_by_status(
         )
 
     if status_code == 429:
-        # Already checked long_context_tier above; this is a normal rate limit
+        # Already checked long_context_tier above.
+        # Some providers return monthly/cycle quota exhaustion as 429 (e.g.
+        # Kimi "monthly usage limit").  Disambiguate the same way as 402:
+        # if the message carries a billing or permanent-quota-exhaustion
+        # signal with no transient "try again / resets at / wait" qualifier,
+        # classify as billing (non-retryable) so we don't burn retries on a
+        # quota that won't recover until the next billing cycle.
+        if any(p in error_msg for p in _BILLING_PATTERNS):
+            return result_fn(
+                FailoverReason.billing,
+                retryable=False,
+                should_rotate_credential=True,
+                should_fallback=True,
+            )
+        # Only check usage-limit patterns when the message does NOT already
+        # carry an explicit rate-limit signal (e.g. "rate limit exceeded").
+        # _USAGE_LIMIT_PATTERNS includes broad tokens like "limit exceeded"
+        # that overlap with normal transient 429 messages; _RATE_LIMIT_PATTERNS
+        # takes priority so those stay retryable.
+        has_rate_limit_signal = any(p in error_msg for p in _RATE_LIMIT_PATTERNS)
+        has_usage_limit = not has_rate_limit_signal and any(p in error_msg for p in _USAGE_LIMIT_PATTERNS)
+        if has_usage_limit:
+            has_transient_signal = any(p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS)
+            if not has_transient_signal:
+                return result_fn(
+                    FailoverReason.billing,
+                    retryable=False,
+                    should_rotate_credential=True,
+                    should_fallback=True,
+                )
         return result_fn(
             FailoverReason.rate_limit,
             retryable=True,

tests/agent/test_error_classifier.py

@@ -324,6 +324,56 @@ def test_alibaba_rate_increased_too_quickly(self):
         assert result.retryable is True
         assert result.should_rotate_credential is True
 
+    def test_429_kimi_monthly_usage_limit_is_billing(self):
+        """Kimi returns HTTP 429 for monthly quota exhaustion — must not retry.
+
+        Real error: "You've reached kimi monthly usage limit for this billing
+        cycle. Your quota will be refreshed in the next cycle."
+        'usage limit' matches _USAGE_LIMIT_PATTERNS; 'refreshed in the next
+        cycle' is NOT in _USAGE_LIMIT_TRANSIENT_SIGNALS → billing, non-retryable.
+        """
+        msg = (
+            "You've reached kimi monthly usage limit for this billing cycle. "
+            "Your quota will be refreshed in the next cycle. Upgrade to get more."
+        )
+        e = MockAPIError(msg, status_code=429)
+        result = classify_api_error(e, provider="kimi")
+        assert result.reason == FailoverReason.billing
+        assert result.retryable is False
+        assert result.should_rotate_credential is True
+        assert result.should_fallback is True
+
+    def test_429_with_transient_usage_limit_is_rate_limit(self):
+        """429 carrying 'usage limit' + 'try again' is a transient quota → rate_limit."""
+        e = MockAPIError(
+            "usage limit exceeded, try again in 60 seconds",
+            status_code=429,
+        )
+        result = classify_api_error(e)
+        assert result.reason == FailoverReason.rate_limit
+        assert result.retryable is True
+
+    def test_429_with_billing_pattern_is_billing(self):
+        """429 carrying an explicit billing phrase (e.g. 'insufficient credits') → billing."""
+        e = MockAPIError("insufficient credits", status_code=429)
+        result = classify_api_error(e)
+        assert result.reason == FailoverReason.billing
+        assert result.retryable is False
+
+    def test_429_plain_no_message_is_rate_limit(self):
+        """Plain 429 with no billing/usage signal stays rate_limit (regression guard)."""
+        e = MockAPIError("Too Many Requests", status_code=429)
+        result = classify_api_error(e)
+        assert result.reason == FailoverReason.rate_limit
+        assert result.retryable is True
+
+    def test_429_quota_with_reset_window_is_rate_limit(self):
+        """429 + 'quota' + 'resets at' → transient, stays rate_limit."""
+        e = MockAPIError("quota exceeded, resets at midnight UTC", status_code=429)
+        result = classify_api_error(e)
+        assert result.reason == FailoverReason.rate_limit
+        assert result.retryable is True
+
     # ── Server errors ──
 
     def test_500_server_error(self):