diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 22bf9c165cf0..524efdc31aec 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -4195,6 +4195,44 @@ def _recoverable_pool_provider( return None +def _is_overloaded_error(exc: Exception) -> bool: + """Detect provider "temporarily overloaded" responses. + + These reuse HTTP 429 (notably Z.AI / Zhipu error code 1305) but are + server-side capacity conditions, NOT per-credential rate limits: the key is + valid and the correct recovery is backoff / fallback, never marking the + credential exhausted. Shares the pattern and code tables with + ``error_classifier`` so both layers classify identically. (#14038) + """ + try: + from agent.error_classifier import ( + _OVERLOADED_PATTERNS, + _OVERLOAD_ERROR_CODES, + ) + except Exception: + return False + + text = str(exc).lower() + code = "" + body = getattr(exc, "body", None) + if isinstance(body, dict): + err_obj = body.get("error") + msg = "" + if isinstance(err_obj, dict): + code = str(err_obj.get("code") or "").strip() + msg = str(err_obj.get("message") or "") + if not code: + code = str(body.get("code") or "").strip() + if not msg: + msg = str(body.get("message") or "") + if msg: + text = f"{text} {msg.lower()}" + + if code and code in _OVERLOAD_ERROR_CODES: + return True + return any(p in text for p in _OVERLOADED_PATTERNS) + + def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str = "") -> bool: """Try same-provider credential-pool recovery for auxiliary calls. @@ -4216,6 +4254,20 @@ def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str error_context = _pool_error_context(exc) hint = failed_api_key or None + # A provider "temporarily overloaded" response (e.g. Z.AI / Zhipu HTTP 429 + # code 1305) is not a credential failure -- the key is valid, the endpoint + # is merely busy. Marking it exhausted burns the pool while the endpoint is + # still overloaded and, for a single-key user, leaves nothing to rotate to + # (the exact failure #14038 describes). Leave the credential intact and let + # the caller retry / fall back. + if _is_overloaded_error(exc): + logger.info( + "Auxiliary client: %s reported a transient overload (not a " + "credential rate limit) -- leaving credential intact, no rotation", + normalized, + ) + return False + if _is_auth_error(exc): refreshed = pool.try_refresh_current() if refreshed is not None: diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 8ac0b6c87234..cef45ae549f7 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -192,8 +192,32 @@ def is_auth(self) -> bool: "currently overloaded", "at capacity", "over capacity", + # Localized (zh-CN) overload text. Z.AI / Zhipu return the code-1305 + # overload message in Chinese depending on account locale, so none of the + # English patterns above match and a server-side overload still burned the + # credential pool -- the exact failure #14038 set out to fix. + # Deliberately EXCLUDES "请求过于频繁" (requests too frequent) and the + # generic "请稍后再试" (try again later), which also appear on genuine + # per-credential rate limits and must keep rotating. (#14038) + "访问量过大", # "traffic volume too large" -- Z.AI 1305 + "服务器繁忙", # "server busy" + "系统繁忙", # "system busy" ] +# Provider error codes that mean "the server is temporarily overloaded", not +# "this credential is rate-limited". Matching the numeric code is +# locale-independent, so classification holds regardless of the account's +# message language. +# +# Z.AI / Zhipu (all surfaced as HTTP 429): +# 1305 -> "service may be temporarily overloaded" => overload, retry same key +# 1302 -> "rate limit reached for requests" => genuine rate limit +# 1313 -> fair-usage throttle => genuine rate limit +# 1308 / 1310 / 1316-1321 -> usage & spend limits => quota/billing +# Only 1305 is a server-capacity condition; the rest must keep their existing +# (rotating / billing) behavior. (#14038) +_OVERLOAD_ERROR_CODES = {"1305"} + # Usage-limit patterns that need disambiguation (could be billing OR rate_limit) _USAGE_LIMIT_PATTERNS = [ "usage limit", @@ -1089,6 +1113,17 @@ def _classify_by_status( # endpoint is still busy, and does nothing for a single-key user). # Disambiguate on the error body so an overload 429 takes the # transient-overload path instead of burning the pool. (#14038) + # + # Check the structured error code FIRST: it is locale-independent, + # whereas the message text is localized per account (Z.AI returns the + # 1305 overload message in Chinese for zh-CN accounts, which no + # English pattern matches -- leaving the original bug live for the + # very provider that motivated the fix). + if error_code.strip() in _OVERLOAD_ERROR_CODES: + return result_fn( + FailoverReason.overloaded, + retryable=True, + ) if any(p in error_msg for p in _OVERLOADED_PATTERNS): return result_fn( FailoverReason.overloaded, diff --git a/tests/agent/test_error_classifier.py b/tests/agent/test_error_classifier.py index be162f5aa3a8..9c7085e4dac6 100644 --- a/tests/agent/test_error_classifier.py +++ b/tests/agent/test_error_classifier.py @@ -1050,3 +1050,67 @@ def test_longer_than_context_length_still_overflow(self): + + +class TestZaiLocalizedOverload: + """Z.AI returns HTTP 429 code 1305 for server-side overload, with the + message localized per account. The English-only _OVERLOADED_PATTERNS from + #53578 never matched the zh-CN text, so a transient provider overload was + still classified as rate_limit -- rotating and exhausting the pool, which + is fatal for a single-key user. (#14038)""" + + def test_429_code_1305_chinese_message_is_overloaded(self): + """The real-world failure: code 1305 + Chinese body.""" + e = MockAPIError( + "Error code: 429 - {'error': {'code': '1305', " + "'message': '该模型当前访问量过大，请您稍后再试'}}", + status_code=429, + body={"error": {"code": "1305", + "message": "该模型当前访问量过大，请您稍后再试"}}, + ) + result = classify_api_error(e, provider="zai", model="glm-4.6v-flash") + assert result.reason == FailoverReason.overloaded + assert result.retryable is True + assert result.should_rotate_credential is False + + def test_429_code_1305_classifies_by_code_even_without_known_text(self): + """Code-based match must hold even if the message is unrecognized.""" + e = MockAPIError( + "Error code: 429", + status_code=429, + body={"error": {"code": "1305", "message": "\u2014"}}, + ) + result = classify_api_error(e, provider="zai") + assert result.reason == FailoverReason.overloaded + assert result.should_rotate_credential is False + + def test_429_chinese_overload_text_without_code_is_overloaded(self): + """Text net: localized overload phrasing with no structured code.""" + e = MockAPIError( + "该模型当前访问量过大，请您稍后再试", + status_code=429, + ) + result = classify_api_error(e, provider="zai") + assert result.reason == FailoverReason.overloaded + assert result.should_rotate_credential is False + + def test_429_code_1302_is_still_a_genuine_rate_limit(self): + """Guard: 1302 IS a real per-credential rate limit -- must still rotate.""" + e = MockAPIError( + "Error code: 429 - {'error': {'code': '1302', " + "'message': 'Rate limit reached for requests'}}", + status_code=429, + body={"error": {"code": "1302", + "message": "Rate limit reached for requests"}}, + ) + result = classify_api_error(e, provider="zai") + assert result.reason == FailoverReason.rate_limit + assert result.should_rotate_credential is True + + def test_429_chinese_too_frequent_is_rate_limit_not_overload(self): + """Guard: "请求过于频繁" (too many requests) is a genuine rate + limit and must NOT be swallowed by the localized overload patterns.""" + e = MockAPIError("请求过于频繁，请稍后再试", status_code=429) + result = classify_api_error(e, provider="zai") + assert result.reason == FailoverReason.rate_limit + assert result.should_rotate_credential is True