fix(error_classifier): treat Z.AI 1305 overload as overloaded, not rate_limit

agent/auxiliary_client.py

@@ -4195,6 +4195,44 @@ def _recoverable_pool_provider(
     return None
 
 
+def _is_overloaded_error(exc: Exception) -> bool:
+    """Detect provider "temporarily overloaded" responses.
+
+    These reuse HTTP 429 (notably Z.AI / Zhipu error code 1305) but are
+    server-side capacity conditions, NOT per-credential rate limits: the key is
+    valid and the correct recovery is backoff / fallback, never marking the
+    credential exhausted. Shares the pattern and code tables with
+    ``error_classifier`` so both layers classify identically. (#14038)
+    """
+    try:
+        from agent.error_classifier import (
+            _OVERLOADED_PATTERNS,
+            _OVERLOAD_ERROR_CODES,
+        )
+    except Exception:
+        return False
+
+    text = str(exc).lower()
+    code = ""
+    body = getattr(exc, "body", None)
+    if isinstance(body, dict):
+        err_obj = body.get("error")
+        msg = ""
+        if isinstance(err_obj, dict):
+            code = str(err_obj.get("code") or "").strip()
+            msg = str(err_obj.get("message") or "")
+        if not code:
+            code = str(body.get("code") or "").strip()
+        if not msg:
+            msg = str(body.get("message") or "")
+        if msg:
+            text = f"{text} {msg.lower()}"
+
+    if code and code in _OVERLOAD_ERROR_CODES:
+        return True
+    return any(p in text for p in _OVERLOADED_PATTERNS)
+
+
 def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str = "") -> bool:
     """Try same-provider credential-pool recovery for auxiliary calls.
 
@@ -4216,6 +4254,20 @@ def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str
     error_context = _pool_error_context(exc)
     hint = failed_api_key or None
 
+    # A provider "temporarily overloaded" response (e.g. Z.AI / Zhipu HTTP 429
+    # code 1305) is not a credential failure -- the key is valid, the endpoint
+    # is merely busy. Marking it exhausted burns the pool while the endpoint is
+    # still overloaded and, for a single-key user, leaves nothing to rotate to
+    # (the exact failure #14038 describes). Leave the credential intact and let
+    # the caller retry / fall back.
+    if _is_overloaded_error(exc):
+        logger.info(
+            "Auxiliary client: %s reported a transient overload (not a "
+            "credential rate limit) -- leaving credential intact, no rotation",
+            normalized,
+        )
+        return False
+
     if _is_auth_error(exc):
         refreshed = pool.try_refresh_current()
         if refreshed is not None:

agent/error_classifier.py

@@ -192,8 +192,32 @@ def is_auth(self) -> bool:
     "currently overloaded",
     "at capacity",
     "over capacity",
+    # Localized (zh-CN) overload text. Z.AI / Zhipu return the code-1305
+    # overload message in Chinese depending on account locale, so none of the
+    # English patterns above match and a server-side overload still burned the
+    # credential pool -- the exact failure #14038 set out to fix.
+    # Deliberately EXCLUDES "请求过于频繁" (requests too frequent) and the
+    # generic "请稍后再试" (try again later), which also appear on genuine
+    # per-credential rate limits and must keep rotating. (#14038)
+    "访问量过大",          # "traffic volume too large" -- Z.AI 1305
+    "服务器繁忙",          # "server busy"
+    "系统繁忙",            # "system busy"
 ]
 
+# Provider error codes that mean "the server is temporarily overloaded", not
+# "this credential is rate-limited". Matching the numeric code is
+# locale-independent, so classification holds regardless of the account's
+# message language.
+#
+# Z.AI / Zhipu (all surfaced as HTTP 429):
+#   1305 -> "service may be temporarily overloaded"  => overload, retry same key
+#   1302 -> "rate limit reached for requests"        => genuine rate limit
+#   1313 -> fair-usage throttle                      => genuine rate limit
+#   1308 / 1310 / 1316-1321 -> usage & spend limits  => quota/billing
+# Only 1305 is a server-capacity condition; the rest must keep their existing
+# (rotating / billing) behavior. (#14038)
+_OVERLOAD_ERROR_CODES = {"1305"}
+
 # Usage-limit patterns that need disambiguation (could be billing OR rate_limit)
 _USAGE_LIMIT_PATTERNS = [
     "usage limit",
@@ -1089,6 +1113,17 @@ def _classify_by_status(
         # endpoint is still busy, and does nothing for a single-key user).
         # Disambiguate on the error body so an overload 429 takes the
         # transient-overload path instead of burning the pool. (#14038)
+        #
+        # Check the structured error code FIRST: it is locale-independent,
+        # whereas the message text is localized per account (Z.AI returns the
+        # 1305 overload message in Chinese for zh-CN accounts, which no
+        # English pattern matches -- leaving the original bug live for the
+        # very provider that motivated the fix).
+        if error_code.strip() in _OVERLOAD_ERROR_CODES:
+            return result_fn(
+                FailoverReason.overloaded,
+                retryable=True,
+            )
         if any(p in error_msg for p in _OVERLOADED_PATTERNS):
             return result_fn(
                 FailoverReason.overloaded,

tests/agent/test_error_classifier.py

@@ -1050,3 +1050,67 @@ def test_longer_than_context_length_still_overflow(self):
 
 
 
+
+
+class TestZaiLocalizedOverload:
+    """Z.AI returns HTTP 429 code 1305 for server-side overload, with the
+    message localized per account. The English-only _OVERLOADED_PATTERNS from
+    #53578 never matched the zh-CN text, so a transient provider overload was
+    still classified as rate_limit -- rotating and exhausting the pool, which
+    is fatal for a single-key user. (#14038)"""
+
+    def test_429_code_1305_chinese_message_is_overloaded(self):
+        """The real-world failure: code 1305 + Chinese body."""
+        e = MockAPIError(
+            "Error code: 429 - {'error': {'code': '1305', "
+            "'message': '该模型当前访问量过大，请您稍后再试'}}",
+            status_code=429,
+            body={"error": {"code": "1305",
+                            "message": "该模型当前访问量过大，请您稍后再试"}},
+        )
+        result = classify_api_error(e, provider="zai", model="glm-4.6v-flash")
+        assert result.reason == FailoverReason.overloaded
+        assert result.retryable is True
+        assert result.should_rotate_credential is False
+
+    def test_429_code_1305_classifies_by_code_even_without_known_text(self):
+        """Code-based match must hold even if the message is unrecognized."""
+        e = MockAPIError(
+            "Error code: 429",
+            status_code=429,
+            body={"error": {"code": "1305", "message": "\u2014"}},
+        )
+        result = classify_api_error(e, provider="zai")
+        assert result.reason == FailoverReason.overloaded
+        assert result.should_rotate_credential is False
+
+    def test_429_chinese_overload_text_without_code_is_overloaded(self):
+        """Text net: localized overload phrasing with no structured code."""
+        e = MockAPIError(
+            "该模型当前访问量过大，请您稍后再试",
+            status_code=429,
+        )
+        result = classify_api_error(e, provider="zai")
+        assert result.reason == FailoverReason.overloaded
+        assert result.should_rotate_credential is False
+
+    def test_429_code_1302_is_still_a_genuine_rate_limit(self):
+        """Guard: 1302 IS a real per-credential rate limit -- must still rotate."""
+        e = MockAPIError(
+            "Error code: 429 - {'error': {'code': '1302', "
+            "'message': 'Rate limit reached for requests'}}",
+            status_code=429,
+            body={"error": {"code": "1302",
+                            "message": "Rate limit reached for requests"}},
+        )
+        result = classify_api_error(e, provider="zai")
+        assert result.reason == FailoverReason.rate_limit
+        assert result.should_rotate_credential is True
+
+    def test_429_chinese_too_frequent_is_rate_limit_not_overload(self):
+        """Guard: "请求过于频繁" (too many requests) is a genuine rate
+        limit and must NOT be swallowed by the localized overload patterns."""
+        e = MockAPIError("请求过于频繁，请稍后再试", status_code=429)
+        result = classify_api_error(e, provider="zai")
+        assert result.reason == FailoverReason.rate_limit
+        assert result.should_rotate_credential is True