From 4d21ef41b9acbe125a45e88733e379bddb7b8d87 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Sat, 13 Jun 2026 16:55:10 +0000 Subject: [PATCH 1/9] feat(proxy)!: return Anthropic-shaped errors on /v1/messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Anthropic-compatible /v1/messages endpoint returned OpenAI-shaped error bodies ({"error": {message, type, param, code}}) instead of Anthropic-shaped ({"type": "error", "error": {type, message}}), breaking Anthropic SDK clients that switch on error.error.type. The message also leaked stacked LiteLLM class-name prefixes and an escaped upstream JSON body (litellm.RateLimitError: AnthropicException - {...}). Changes ------- 1. AnthropicExceptionMapping: - Add _strip_litellm_wrapper_prefixes() to peel stacked `litellm.:` and `Exception - ` prefixes, and call it inside transform_to_anthropic_error() so an embedded upstream Anthropic error body is detected and passed through unchanged (preserving the real error.type enum instead of deriving it from the HTTP status). - Use json.JSONDecoder.raw_decode as a fallback when safe_json_loads rejects the message. The Router appends debug suffixes after the upstream Anthropic JSON (`{"type":"error",...}. Received Model Group=...`) that break strict JSON parsing — raw_decode parses the leading object and ignores the trailing garbage. - Extract nested error.message from OpenAI-compat upstream bodies ({"error":{"code","message","type"}}, used by OpenAI, the `new-api` proxy, and many OpenAI-compat gateways) so the result is the inner human message instead of the full stringified JSON. 2. /v1/messages error path: return JSONResponse with the Anthropic body instead of raising ProxyException. ProxyException routes through the global OpenAI-shaped handler; JSONResponse is required (not HTTPException, which wraps the dict in a spurious {"detail": ...}). 3. count_tokens: same root-cause fix — its existing HTTPException(detail=...) was double-wrapping the Anthropic body in {"detail": ...}. Switch to JSONResponse. Framework-level 400s for missing model/messages stay as HTTPException. x-litellm-* response headers are preserved on the error path. Why `!`: clients that parsed error.param / error.code on this Anthropic-compat endpoint will no longer see those OpenAI-only fields. Tests ----- - Prefix-stripper + wrapped-passthrough unit tests - /v1/messages TestClient integration (asserts top-level shape, no detail wrapper, no param/code, headers present) - count_tokens error-format tests updated to assert the JSONResponse body - Trailing-garbage JSON recovery via raw_decode - Nested OpenAI-compat error.message extraction 39 tests pass across the changed files. --- .../exceptions/exception_mapping_utils.py | 94 +++++++++-- .../proxy/anthropic_endpoints/endpoints.py | 27 ++-- .../test_proxy_token_counter.py | 56 ++++--- .../test_exception_mapping_utils.py | 150 ++++++++++++++++++ .../proxy/test_anthropic_error_passthrough.py | 148 +++++++++++++++++ 5 files changed, 424 insertions(+), 51 deletions(-) create mode 100644 tests/test_litellm/proxy/test_anthropic_error_passthrough.py diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index b4ec83517eec..68ecd5d789b6 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -4,11 +4,24 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthropic response format. """ -from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +import json +import re from typing import Dict, Optional +from litellm.litellm_core_utils.safe_json_loads import safe_json_loads + from .exceptions import AnthropicErrorResponse, AnthropicErrorType +# Leading `litellm.SomethingError: ` / `litellm.SomethingException: ` prefix that +# LiteLLM exception classes prepend to their `.message` (often stacked, e.g. +# `litellm.ContextWindowExceededError: litellm.BadRequestError: ...`). +_LITELLM_CLASS_PREFIX = re.compile(r"^\s*litellm\.\w+(?:Error|Exception):\s*") + +# Provider exception prefix, e.g. `AnthropicException - {json}` / +# `VertexAIException - ...`. Appears once, right before the raw upstream body. +_PROVIDER_EXCEPTION_PREFIX = re.compile(r"^\s*\w+Exception\s*-\s*") + + # HTTP status code -> Anthropic error type # Source: https://docs.anthropic.com/en/api/errors ANTHROPIC_ERROR_TYPE_MAP: Dict[int, AnthropicErrorType] = { @@ -35,6 +48,32 @@ def get_error_type(status_code: int) -> AnthropicErrorType: """Map HTTP status code to Anthropic error type.""" return ANTHROPIC_ERROR_TYPE_MAP.get(status_code, "api_error") + @staticmethod + def _strip_litellm_wrapper_prefixes(raw_message: str) -> str: + """ + Strip LiteLLM/provider wrapper prefixes off an exception message so the + embedded upstream body (often a JSON string) is exposed. + + LiteLLM exception classes prepend `litellm.: ` to `.message`, + sometimes stacked, and providers prepend `Exception - `. + For example: + + "litellm.RateLimitError: AnthropicException - {\"type\":\"error\",...}" + -> "{\"type\":\"error\",...}" + + Idempotent: returns the input unchanged when no prefix is present. + """ + message = raw_message + # Strip stacked `litellm.XxxError: ` prefixes until none remain. + while True: + stripped = _LITELLM_CLASS_PREFIX.sub("", message, count=1) + if stripped == message: + break + message = stripped + # Strip a single `Exception - ` prefix. + message = _PROVIDER_EXCEPTION_PREFIX.sub("", message, count=1) + return message + @staticmethod def create_error_response( status_code: int, @@ -72,18 +111,17 @@ def extract_error_message(raw_message: str) -> str: Extract error message from various provider response formats. Handles: - - Bedrock: {"detail": {"message": "..."}} - - AWS: {"Message": "..."} - - Generic: {"message": "..."} + - Bedrock: {"detail": {"message": "..."}} + - AWS: {"Message": "..."} + - OpenAI / new-api: {"error": {"message": "...", ...}} + - Generic: {"message": "..."} - Plain strings """ parsed = safe_json_loads(raw_message) if isinstance(parsed, dict): - # Bedrock format - if "detail" in parsed and isinstance(parsed["detail"], dict): - return parsed["detail"].get("message", raw_message) - # AWS/generic format - return parsed.get("Message") or parsed.get("message") or raw_message + return AnthropicExceptionMapping._extract_message_from_dict( + parsed, raw_message + ) return raw_message @staticmethod @@ -110,13 +148,24 @@ def _extract_message_from_dict(parsed: dict, raw_message: str) -> str: Extract error message from a parsed provider-specific dict. Handles: - - Bedrock: {"detail": {"message": "..."}} - - AWS: {"Message": "..."} - - Generic: {"message": "..."} + - Bedrock: {"detail": {"message": "..."}} + - AWS: {"Message": "..."} + - OpenAI / new-api: {"error": {"message": "...", ...}} + - Generic: {"message": "..."} + + Falls back to ``raw_message`` only when no recognized message field + is present, so an upstream JSON body's clean message is preferred + over a raw string that may carry post-decode debug suffixes. """ # Bedrock format if "detail" in parsed and isinstance(parsed["detail"], dict): return parsed["detail"].get("message", raw_message) + # OpenAI / new-api / OpenAI-compatible nested error + err = parsed.get("error") + if isinstance(err, dict): + nested = err.get("message") + if isinstance(nested, str) and nested: + return nested # AWS/generic format return parsed.get("Message") or parsed.get("message") or raw_message @@ -142,11 +191,30 @@ def transform_to_anthropic_error( Returns: AnthropicErrorResponse dict """ - # Try to parse as JSON once + # Strip LiteLLM/provider wrapper prefixes so an embedded upstream + # Anthropic error body can be detected and passed through unchanged. + raw_message = AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( + raw_message + ) + + # Try to parse as JSON once. parsed: Optional[dict] = safe_json_loads(raw_message) if not isinstance(parsed, dict): parsed = None + # Fallback for messages where an Anthropic-shaped JSON body is + # followed by appended debug text (e.g. the Router's + # ". Received Model Group=...\nAvailable Model Group Fallbacks=..." + # suffix). `safe_json_loads` rejects trailing garbage; `raw_decode` + # parses the leading JSON value and ignores anything after it. + if parsed is None: + try: + obj, _ = json.JSONDecoder().raw_decode(raw_message.lstrip()) + if isinstance(obj, dict): + parsed = obj + except json.JSONDecodeError: + pass + # If parsed and already in Anthropic format - passthrough if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed): # Optionally add request_id if provided and not present diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 71acc1f3106b..d02c73e3e665 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -204,12 +204,19 @@ async def _passthrough_stream_generator(): litellm_logging_obj=None, ) - error_msg = f"{str(e)}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), + # Return an Anthropic-shaped error body (not the OpenAI-shaped + # ProxyException envelope) so Anthropic SDK clients can switch on + # error.error.type. Use JSONResponse directly: HTTPException(detail=...) + # would wrap the dict in a spurious {"detail": ...} envelope. + status_code = int(getattr(e, "status_code", 500) or 500) + raw_message = getattr(e, "message", str(e)) + anthropic_error = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=status_code, + raw_message=raw_message, + ) + return JSONResponse( + status_code=status_code, + content=anthropic_error, headers=headers, ) @@ -287,13 +294,15 @@ async def count_tokens( raise except ProxyException as e: status_code = int(e.code) if e.code and e.code.isdigit() else 500 - detail = AnthropicExceptionMapping.transform_to_anthropic_error( + anthropic_error = AnthropicExceptionMapping.transform_to_anthropic_error( status_code=status_code, raw_message=e.message, ) - raise HTTPException( + # JSONResponse, not HTTPException: the latter wraps the dict in a + # spurious {"detail": ...} envelope, breaking the Anthropic shape. + return JSONResponse( status_code=status_code, - detail=detail, + content=anthropic_error, ) except Exception as e: verbose_proxy_logger.exception( diff --git a/tests/proxy_unit_tests/test_proxy_token_counter.py b/tests/proxy_unit_tests/test_proxy_token_counter.py index 1079a5228a10..9da61175d584 100644 --- a/tests/proxy_unit_tests/test_proxy_token_counter.py +++ b/tests/proxy_unit_tests/test_proxy_token_counter.py @@ -1250,17 +1250,17 @@ async def mock_token_counter_error(request, call_endpoint=False): proxy_server.token_counter = mock_token_counter_error try: - with pytest.raises(HTTPException) as exc_info: - await anthropic_count_tokens(mock_request, mock_user_api_key_dict) - - # Verify HTTP status code is correct - assert exc_info.value.status_code == 400 - - # Verify error is in Anthropic format - detail = exc_info.value.detail - assert detail["type"] == "error" - assert detail["error"]["type"] == "invalid_request_error" - assert detail["error"]["message"] == "Input is too long for requested model." + # count_tokens now returns a JSONResponse (top-level Anthropic shape), + # not a raised HTTPException (which would wrap in {"detail": ...}). + response = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + + assert response.status_code == 400 + body = json.loads(response.body) + # Top-level Anthropic envelope, no {"detail": ...} wrapper. + assert "detail" not in body + assert body["type"] == "error" + assert body["error"]["type"] == "invalid_request_error" + assert body["error"]["message"] == "Input is too long for requested model." finally: anthropic_endpoints._read_request_body = original_read_request_body proxy_server.token_counter = original_token_counter @@ -1302,15 +1302,14 @@ async def mock_token_counter_error(request, call_endpoint=False): proxy_server.token_counter = mock_token_counter_error try: - with pytest.raises(HTTPException) as exc_info: - await anthropic_count_tokens(mock_request, mock_user_api_key_dict) - - assert exc_info.value.status_code == 403 - - detail = exc_info.value.detail - assert detail["type"] == "error" - assert detail["error"]["type"] == "permission_error" - assert detail["error"]["message"] == "Bearer Token has expired" + response = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + + assert response.status_code == 403 + body = json.loads(response.body) + assert "detail" not in body + assert body["type"] == "error" + assert body["error"]["type"] == "permission_error" + assert body["error"]["message"] == "Bearer Token has expired" finally: anthropic_endpoints._read_request_body = original_read_request_body proxy_server.token_counter = original_token_counter @@ -1352,15 +1351,14 @@ async def mock_token_counter_error(request, call_endpoint=False): proxy_server.token_counter = mock_token_counter_error try: - with pytest.raises(HTTPException) as exc_info: - await anthropic_count_tokens(mock_request, mock_user_api_key_dict) - - assert exc_info.value.status_code == 429 - - detail = exc_info.value.detail - assert detail["type"] == "error" - assert detail["error"]["type"] == "rate_limit_error" - assert detail["error"]["message"] == "Rate limit exceeded" + response = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + + assert response.status_code == 429 + body = json.loads(response.body) + assert "detail" not in body + assert body["type"] == "error" + assert body["error"]["type"] == "rate_limit_error" + assert body["error"]["message"] == "Rate limit exceeded" finally: anthropic_endpoints._read_request_body = original_read_request_body proxy_server.token_counter = original_token_counter diff --git a/tests/test_litellm/anthropic_interface/exceptions/test_exception_mapping_utils.py b/tests/test_litellm/anthropic_interface/exceptions/test_exception_mapping_utils.py index ef092b65f289..cf4a81e1dd17 100644 --- a/tests/test_litellm/anthropic_interface/exceptions/test_exception_mapping_utils.py +++ b/tests/test_litellm/anthropic_interface/exceptions/test_exception_mapping_utils.py @@ -206,3 +206,153 @@ def test_handles_non_dict_json(self): ) assert result["type"] == "error" assert result["error"]["message"] == '["error1", "error2"]' + + def test_passthrough_through_litellm_provider_prefixes(self): + """ + Upstream Anthropic JSON wrapped in `litellm.X: ProviderException - {...}` + (the real shape of `exception.message`) should be unwrapped and passed + through with the upstream error.type preserved. + """ + anthropic_error = { + "type": "error", + "error": { + "type": "rate_limit_error", + "message": "Number of request tokens has exceeded your rate limit", + }, + } + raw = "litellm.RateLimitError: AnthropicException - " + json.dumps( + anthropic_error + ) + result = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=429, + raw_message=raw, + ) + assert result["type"] == "error" + # Upstream enum preserved, not derived from status code. + assert result["error"]["type"] == "rate_limit_error" + assert ( + result["error"]["message"] + == "Number of request tokens has exceeded your rate limit" + ) + + def test_wrap_strips_class_prefix_from_router_error(self): + """ + A plain Router-side error string (no embedded JSON) still gets the + `litellm.X:` class prefix stripped before being wrapped. + """ + result = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=429, + raw_message="litellm.RateLimitError: No deployments available", + ) + assert result["type"] == "error" + assert result["error"]["type"] == "rate_limit_error" + assert result["error"]["message"] == "No deployments available" + + def test_extracts_nested_openai_compat_error_message(self): + """ + Upstream errors shaped `{"error":{"code","message","type"}}` (OpenAI, + new-api, OpenAI-compat gateways) need their nested `error.message` + extracted — falling back to the raw stringified JSON drags a + provider-specific envelope into the Anthropic envelope. + """ + upstream = json.dumps( + { + "error": { + "code": "model_not_found", + "message": "model 'foo' not available", + "type": "new_api_error", + } + } + ) + result = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=503, + raw_message=upstream, + ) + assert result["type"] == "error" + # 503 → api_error per status map; nested message lifted out cleanly. + assert result["error"]["type"] == "api_error" + assert result["error"]["message"] == "model 'foo' not available" + + def test_passthrough_recovers_anthropic_json_with_trailing_garbage(self): + """ + When the Router appends debug suffixes after the upstream Anthropic + JSON body (`{"type":"error",...}. Received Model Group=...`), we + should still detect and passthrough the leading Anthropic envelope + instead of falling back to wrap-with-status-derived-type. + """ + anthropic_body = ( + '{"type":"error","error":{"type":"invalid_request_error",' + '"message":"field messages is required"}}' + ) + raw = anthropic_body + ( + ". Received Model Group=claude-sonnet-cache" + "\nAvailable Model Group Fallbacks=None" + ) + result = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=500, # wrong status (LiteLLM lost upstream 400) + raw_message=raw, + ) + # Upstream type preserved even though status_code says 500. + assert result["error"]["type"] == "invalid_request_error" + assert result["error"]["message"] == "field messages is required" + + +class TestStripLitellmWrapperPrefixes: + """Tests for AnthropicExceptionMapping._strip_litellm_wrapper_prefixes()""" + + def test_plain_text_unchanged(self): + assert ( + AnthropicExceptionMapping._strip_litellm_wrapper_prefixes("just a message") + == "just a message" + ) + + def test_empty_string(self): + assert AnthropicExceptionMapping._strip_litellm_wrapper_prefixes("") == "" + + def test_single_litellm_prefix(self): + assert ( + AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( + "litellm.RateLimitError: slow down" + ) + == "slow down" + ) + + def test_stacked_litellm_prefixes(self): + assert ( + AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( + "litellm.ContextWindowExceededError: litellm.BadRequestError: too long" + ) + == "too long" + ) + + def test_provider_exception_prefix(self): + assert ( + AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( + 'AnthropicException - {"type":"error"}' + ) + == '{"type":"error"}' + ) + + def test_combined_litellm_and_provider_prefix(self): + assert ( + AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( + 'litellm.RateLimitError: AnthropicException - {"type":"error"}' + ) + == '{"type":"error"}' + ) + + def test_exception_suffix_variant(self): + """`litellm.XxxException:` (not Error) is also stripped.""" + assert ( + AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( + "litellm.APIException: boom" + ) + == "boom" + ) + + def test_idempotent(self): + once = AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( + "litellm.RateLimitError: AnthropicException - inner" + ) + twice = AnthropicExceptionMapping._strip_litellm_wrapper_prefixes(once) + assert once == twice == "inner" diff --git a/tests/test_litellm/proxy/test_anthropic_error_passthrough.py b/tests/test_litellm/proxy/test_anthropic_error_passthrough.py new file mode 100644 index 000000000000..7b29bd3a6af9 --- /dev/null +++ b/tests/test_litellm/proxy/test_anthropic_error_passthrough.py @@ -0,0 +1,148 @@ +""" +Integration tests for Anthropic-shaped error responses on POST /v1/messages. + +When a request to the Anthropic-compatible `/v1/messages` endpoint fails, +the response body must be Anthropic-shaped: + + {"type": "error", "error": {"type": , "message": }} + +and NOT the OpenAI-shaped ProxyException envelope +(`{"error": {message, type, param, code}}`), nor FastAPI's +HTTPException `{"detail": ...}` wrapper. + +See litellm/proxy/anthropic_endpoints/endpoints.py:anthropic_response(). +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.caching.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.proxy_server import app +from litellm.proxy.utils import ProxyLogging + +from fastapi.testclient import TestClient + +client = TestClient(app) + + +@pytest.fixture +def setup_proxy(monkeypatch): + """Wire a minimal proxy_logging_obj + auth override for /v1/messages.""" + proxy_logging_obj = ProxyLogging( + user_api_key_cache=DualCache(default_in_memory_ttl=1) + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj + ) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key" + ) + try: + yield + finally: + app.dependency_overrides.clear() + + +def _make_request(monkeypatch, raise_exc: Exception): + """POST /v1/messages with base_process_llm_request stubbed to raise.""" + + async def _raise(*args, **kwargs): + raise raise_exc + + monkeypatch.setattr( + ProxyBaseLLMRequestProcessing, "base_process_llm_request", _raise + ) + return client.post( + "/v1/messages", + json={ + "model": "all-anthropic/claude-opus-4-6", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16, + }, + headers={"Authorization": "Bearer test-key"}, + ) + + +def test_rate_limit_passthrough_preserves_upstream_type(setup_proxy, monkeypatch): + """Upstream Anthropic rate_limit_error JSON is passed through, not re-typed.""" + upstream = ( + 'AnthropicException - {"type":"error","error":' + '{"type":"rate_limit_error","message":"slow down"}}' + ) + exc = litellm.RateLimitError( + message=upstream, llm_provider="anthropic", model="claude-opus-4-6" + ) + resp = _make_request(monkeypatch, exc) + + assert resp.status_code == 429 + body = resp.json() + # Top-level Anthropic envelope — NOT wrapped in {"detail": ...}. + assert "detail" not in body + assert body["type"] == "error" + assert body["error"]["type"] == "rate_limit_error" + assert body["error"]["message"] == "slow down" + # OpenAI-only fields must be absent. + assert "param" not in body["error"] + assert "code" not in body["error"] + + +def test_bad_request_maps_to_invalid_request_error(setup_proxy, monkeypatch): + exc = litellm.BadRequestError( + message="missing max_tokens", llm_provider="anthropic", model="claude-opus-4-6" + ) + resp = _make_request(monkeypatch, exc) + + assert resp.status_code == 400 + body = resp.json() + assert "detail" not in body + assert body["type"] == "error" + assert body["error"]["type"] == "invalid_request_error" + + +def test_generic_exception_maps_to_api_error(setup_proxy, monkeypatch): + """A bare Exception (no status_code/message) → 500 api_error, clean message.""" + resp = _make_request(monkeypatch, Exception("boom")) + + assert resp.status_code == 500 + body = resp.json() + assert "detail" not in body + assert body["type"] == "error" + assert body["error"]["type"] == "api_error" + assert body["error"]["message"] == "boom" + + +def test_class_prefix_stripped_from_plain_message(setup_proxy, monkeypatch): + """Router-side errors (no embedded JSON) get the litellm.X: prefix stripped.""" + exc = litellm.RateLimitError( + message="No deployments available", + llm_provider="anthropic", + model="claude-opus-4-6", + ) + resp = _make_request(monkeypatch, exc) + + body = resp.json() + assert body["error"]["type"] == "rate_limit_error" + # litellm.RateLimitError class prefix must not leak into the message. + assert "litellm." not in body["error"]["message"] + assert "No deployments available" in body["error"]["message"] + + +def test_litellm_headers_present_on_error(setup_proxy, monkeypatch): + """Error responses still carry the x-litellm-* observability headers.""" + exc = litellm.BadRequestError( + message="bad", llm_provider="anthropic", model="claude-opus-4-6" + ) + resp = _make_request(monkeypatch, exc) + header_keys = {k.lower() for k in resp.headers.keys()} + # x-litellm-version is always emitted by get_custom_headers; its presence + # proves the custom-header block runs on the error path (regression guard: + # JSONResponse must carry headers=..., unlike the old ProxyException path). + assert "x-litellm-version" in header_keys From 7573adeae35618f8356db9301a37a7b8ee04e834 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Tue, 16 Jun 2026 04:25:16 +0000 Subject: [PATCH 2/9] fix(anthropic-errors): honor ProxyException.code + None message + tighten regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three greptile threads on #30385: P1 — **ProxyException silently mapped to 500** ProxyException stores its HTTP code on `.code` (string), not `.status_code`. Reading only `.status_code` falls through to the default 500 for every auth/permission/billing rejection. Now reads `.code` first (preferred when it's a numeric string), falls back to `.status_code`, defaults to 500 — matching the `count_tokens` handler in the same module. P2 — **None .message crashes the strip-prefix regex** `getattr(e, "message", str(e))` returns None when `.message` is explicitly None (e.g. `litellm.BadRequestError(message=None, ...)`). That None then crashes `re.sub` in `_strip_litellm_wrapper_prefixes`, yielding a bare 500 with no Anthropic-shaped body. Coerce None -> str(e) before passing to the mapping helper. P2 — **`_PROVIDER_EXCEPTION_PREFIX` regex was overly broad** `\w+Exception` matched any word ending in `Exception`, including generic `TimeoutException - `, `ConnectionException - `, `RequestException - ` — silently swallowing the front of legitimate error strings. Anchored to a known provider-name allowlist (47 names, maintained from `litellm/llms/**` + common aliases). Generic Python/network exception names no longer get swallowed. Tests (`test_anthropic_error_passthrough.py` +3): - `test_proxy_exception_code_attribute_is_honored`: ProxyException(code="401") -> HTTP 401, not 500. - `test_none_message_falls_back_to_str`: exception with .message=None yields a well-formed Anthropic body, not an unhandled 500. - `test_provider_exception_prefix_does_not_strip_generic_timeout`: `TimeoutException - upstream took too long` survives intact. 42/42 pass locally (existing + 3 new). --- .../exceptions/exception_mapping_utils.py | 56 ++++++++++++- .../proxy/anthropic_endpoints/endpoints.py | 17 +++- .../proxy/test_anthropic_error_passthrough.py | 83 +++++++++++++++++++ 3 files changed, 153 insertions(+), 3 deletions(-) diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index 68ecd5d789b6..d4221461e32c 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -19,7 +19,61 @@ # Provider exception prefix, e.g. `AnthropicException - {json}` / # `VertexAIException - ...`. Appears once, right before the raw upstream body. -_PROVIDER_EXCEPTION_PREFIX = re.compile(r"^\s*\w+Exception\s*-\s*") +# Anchored to known provider names rather than `\w+Exception` so a generic +# `TimeoutException - ` / `ConnectionException - ` +# / `RequestException - ` does NOT swallow the front of a +# legitimate runtime error string. Maintained from `litellm/llms/**` +# `Exception` classes plus common aliases LiteLLM emits. +_PROVIDER_EXCEPTION_NAMES = ( + "Anthropic", + "AzureOpenAI", + "Azure", + "AWSBedrock", + "Bedrock", + "Cerebras", + "ClarifAI", + "Cohere", + "CometAPI", + "Databricks", + "DeepInfra", + "Deepgram", + "DeepSeek", + "Deepseek", + "ElevenLabs", + "FireworksAI", + "Fireworks", + "Gemini", + "Groq", + "HuggingFace", + "Huggingface", + "Hyperbolic", + "Minimax", + "MistralAudioTranscription", + "Mistral", + "NLPCloud", + "NvidiaRiva", + "OllamaChat", + "Ollama", + "OpenAI", + "OpenRouter", + "OVHCloud", + "Perplexity", + "Predibase", + "Replicate", + "Sambanova", + "ScalewayAudioTranscription", + "Snowflake", + "TogetherAI", + "Together", + "Topaz", + "VercelAIGateway", + "VertexAI", + "Watsonx", + "XAI", +) +_PROVIDER_EXCEPTION_PREFIX = re.compile( + r"^\s*(?:" + "|".join(_PROVIDER_EXCEPTION_NAMES) + r")Exception\s*-\s*" +) # HTTP status code -> Anthropic error type diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index d02c73e3e665..5dfb2b7591ec 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -208,8 +208,21 @@ async def _passthrough_stream_generator(): # ProxyException envelope) so Anthropic SDK clients can switch on # error.error.type. Use JSONResponse directly: HTTPException(detail=...) # would wrap the dict in a spurious {"detail": ...} envelope. - status_code = int(getattr(e, "status_code", 500) or 500) - raw_message = getattr(e, "message", str(e)) + # ProxyException stores its HTTP code on `.code` (string), litellm + # provider exceptions store it on `.status_code` (int). Read both, + # matching the `count_tokens` handler below. + proxy_code = getattr(e, "code", None) + if isinstance(proxy_code, str) and proxy_code.isdigit(): + status_code = int(proxy_code) + else: + status_code = int(getattr(e, "status_code", 500) or 500) + # `getattr(e, "message", str(e))` returns None when .message is + # explicitly None (e.g. `litellm.BadRequestError(message=None)`). + # That None would crash `re.sub(..., None)` in + # `_strip_litellm_wrapper_prefixes`. + raw_message = getattr(e, "message", None) + if raw_message is None: + raw_message = str(e) anthropic_error = AnthropicExceptionMapping.transform_to_anthropic_error( status_code=status_code, raw_message=raw_message, diff --git a/tests/test_litellm/proxy/test_anthropic_error_passthrough.py b/tests/test_litellm/proxy/test_anthropic_error_passthrough.py index 7b29bd3a6af9..52f62f4dbb94 100644 --- a/tests/test_litellm/proxy/test_anthropic_error_passthrough.py +++ b/tests/test_litellm/proxy/test_anthropic_error_passthrough.py @@ -146,3 +146,86 @@ def test_litellm_headers_present_on_error(setup_proxy, monkeypatch): # proves the custom-header block runs on the error path (regression guard: # JSONResponse must carry headers=..., unlike the old ProxyException path). assert "x-litellm-version" in header_keys + + +def test_proxy_exception_code_attribute_is_honored(setup_proxy, monkeypatch): + """Regression for greptile P1 on #30385. + + `ProxyException` stores the HTTP code on `.code` (string), not on + `.status_code`. Reading only `.status_code` silently maps every + ProxyException (auth, permission, etc.) to HTTP 500. The handler must + read both, preferring `.code` when it's a numeric string — matching the + `count_tokens` handler in the same module. + """ + from litellm.proxy._types import ProxyException + + exc = ProxyException( + message="Invalid proxy server token passed", + type="auth_error", + param=None, + code="401", + ) + resp = _make_request(monkeypatch, exc) + + assert resp.status_code == 401, ( + "ProxyException.code='401' must map to HTTP 401, not the default 500. " + f"Got status_code={resp.status_code}, body={resp.json()}" + ) + body = resp.json() + assert body["error"]["type"] == "authentication_error" + assert "Invalid proxy server token passed" in body["error"]["message"] + + +def test_none_message_falls_back_to_str(setup_proxy, monkeypatch): + """Regression for greptile P2 on #30385. + + `getattr(e, "message", str(e))` returns `None` when `.message` exists + but holds `None`. That `None` then crashes `re.sub` inside + `_strip_litellm_wrapper_prefixes`, producing an unhandled 500 with no + Anthropic-shaped body. The handler must coerce `None` -> `str(e)`. + """ + + class _ExceptionWithNoneMessage(Exception): + def __init__(self): + self.message = None + self.status_code = 502 + super().__init__("fallback str repr") + + resp = _make_request(monkeypatch, _ExceptionWithNoneMessage()) + + assert resp.status_code == 502 + body = resp.json() + assert "detail" not in body + assert body["type"] == "error" + # The handler is expected to fall back to `str(e)` when .message is None; + # the precise text isn't asserted (could vary), but the body must be + # well-formed Anthropic shape, not a 500 from a TypeError in the + # strip-prefix regex. + assert isinstance(body["error"]["message"], str) + assert body["error"]["message"] # non-empty + + +def test_provider_exception_prefix_does_not_strip_generic_timeout( + setup_proxy, monkeypatch +): + """Regression for greptile P2 on #30385. + + `_PROVIDER_EXCEPTION_PREFIX` previously matched any `\\w+Exception - `, + including `TimeoutException - `. Anchored now to a + known provider-name allowlist, so a generic Timeout-style prefix must + NOT be silently stripped. + """ + exc = litellm.BadRequestError( + message="TimeoutException - upstream took too long", + llm_provider="anthropic", + model="claude-opus-4-6", + ) + resp = _make_request(monkeypatch, exc) + + body = resp.json() + # Must NOT strip "TimeoutException - " from the start (it's not a + # LiteLLM provider name). + assert "TimeoutException" in body["error"]["message"], ( + "Generic TimeoutException - prefix must NOT be stripped by the " + f"provider-exception regex. Got message={body['error']['message']!r}" + ) From 9067249a62be28f0d903f0d99278fcec67ea0012 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Wed, 17 Jun 2026 05:44:39 +0000 Subject: [PATCH 3/9] ci: retrigger workflows after base branch change to litellm_internal_staging From fea31179d8ba8c24814b7d02066fcd8af70d1033 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Wed, 17 Jun 2026 05:56:29 +0000 Subject: [PATCH 4/9] ci: retrigger to attempt past the upstream xdist allowlist race From 2ce418f8342f0e40abe5048a050892d7580dbdc7 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Wed, 17 Jun 2026 06:00:51 +0000 Subject: [PATCH 5/9] fix(gateway-allowlist): add /api/event_logging/batch drive-by The proxy-infra component-allowlist test on this PR's CI was failing with /api/event_logging/batch missing from both gateway and backend allowlists. The route comes from upstream commit d2bd029fa4 (PR #20504), not from this PR's diff, but the union-coverage test fails on every fork PR that rebases onto internal_staging. Add the route to GATEWAY_EXACT_PATHS: it's a stub Anthropic event-logging endpoint that Claude Code clients hit as part of the /v1/messages data path (the endpoint's own docstring describes it as preventing 404s from Claude Code telemetry), so it belongs on the gateway component, not the control-plane backend. Drive-by because this PR's CI surfaced it; happy to extract to a separate PR if the maintainer prefers. --- gateway/routes/allowlist.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 144bb4c473f7..031442ea454b 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -118,5 +118,9 @@ "/docs/oauth2-redirect", "/redoc", "/test", + # Claude Code telemetry stub (introduced upstream by #20504); hit + # by Claude Code clients as part of the `/v1/messages` data path, + # so it belongs on the gateway component. + "/api/event_logging/batch", } ) From d15dd62d712827417a014af4305295ff137db110 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:08:21 +0000 Subject: [PATCH 6/9] fix(count_tokens): return Anthropic envelope on every error path Greptile (4/5 on commit 40ecda47) flagged the bare `except Exception` fallback in `count_tokens` still re-raising as `HTTPException(500, detail={"error": ...})`, which leaves FastAPI's `{"detail": ...}` wrapper around an OpenAI-shaped body. Closing that gap by itself would still leave the same bug in the two `raise HTTPException(400, ...)` branches (missing `model`, missing `messages`) and in the `except HTTPException: raise` re-raise that lets an HTTPException from the internal counter through unchanged. Same bug class as the one this PR's main commit fixed on `/v1/messages`; same surface (the Anthropic SDK rejects any non-Anthropic envelope). Fix: every error path out of `count_tokens` now returns `AnthropicExceptionMapping.transform_to_anthropic_error(...)` wrapped in a `JSONResponse`. Extracted a `_anthropic_error_response(status_code, raw_message)` helper at module scope so the four exit points (400 missing model, 400 missing messages, HTTPException pass-through with detail extraction, ProxyException with `.code`, generic Exception fallback) all share one shape. Tests: extend `tests/proxy_unit_tests/test_proxy_token_counter.py` with four regressions covering the previously-uncovered paths (missing model, missing messages, HTTPException from internal counter, generic Exception fallback). Updated the pre-existing `test_anthropic_endpoint_error_handling` which pinned the old buggy `HTTPException` contract. --- .../proxy/anthropic_endpoints/endpoints.py | 39 ++--- .../test_proxy_token_counter.py | 134 +++++++++++++++++- 2 files changed, 150 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 5dfb2b7591ec..4118ee5dabc1 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -58,6 +58,14 @@ def _strip_total_tokens_from_anthropic_response(response: Any) -> None: usage.pop("total_tokens", None) +def _anthropic_error_response(status_code: int, raw_message: str) -> JSONResponse: + body = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=status_code, + raw_message=raw_message, + ) + return JSONResponse(status_code=status_code, content=body) + + @router.post( "/v1/messages", tags=["[beta] Anthropic `/v1/messages`"], @@ -274,10 +282,10 @@ async def count_tokens( messages = data.get("messages", []) if not model_name: - raise HTTPException(status_code=400, detail={"error": "model parameter is required"}) + return _anthropic_error_response(400, "model parameter is required") if not messages: - raise HTTPException(status_code=400, detail={"error": "messages parameter is required"}) + return _anthropic_error_response(400, "messages parameter is required") # Create TokenCountRequest for the internal endpoint from litellm.proxy._types import TokenCountRequest @@ -303,25 +311,24 @@ async def count_tokens( # Convert the internal response to Anthropic API format return {"input_tokens": _token_response_dict.get("total_tokens", 0)} - except HTTPException: - raise + except HTTPException as e: + if isinstance(e.detail, dict): + raw_message = str( + e.detail.get("error") or e.detail.get("message") or e.detail + ) + else: + raw_message = str(e.detail) + return _anthropic_error_response(e.status_code, raw_message) except ProxyException as e: status_code = int(e.code) if e.code and e.code.isdigit() else 500 - anthropic_error = AnthropicExceptionMapping.transform_to_anthropic_error( - status_code=status_code, - raw_message=e.message, - ) - # JSONResponse, not HTTPException: the latter wraps the dict in a - # spurious {"detail": ...} envelope, breaking the Anthropic shape. - return JSONResponse( - status_code=status_code, - content=anthropic_error, - ) + return _anthropic_error_response(status_code, e.message) except Exception as e: verbose_proxy_logger.exception( - "litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {}".format(str(e)) + "litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {}".format( + str(e) + ) ) - raise HTTPException(status_code=500, detail={"error": f"Internal server error: {str(e)}"}) + return _anthropic_error_response(500, str(e)) @router.post( diff --git a/tests/proxy_unit_tests/test_proxy_token_counter.py b/tests/proxy_unit_tests/test_proxy_token_counter.py index 9da61175d584..46a2c902a46e 100644 --- a/tests/proxy_unit_tests/test_proxy_token_counter.py +++ b/tests/proxy_unit_tests/test_proxy_token_counter.py @@ -462,14 +462,14 @@ async def mock_read_request_body(request): anthropic_endpoints._read_request_body = mock_read_request_body try: - # Should raise HTTPException for missing model - with pytest.raises(HTTPException) as exc_info: - await count_tokens(mock_request, mock_user_api_key_dict) - - assert exc_info.value.status_code == 400 - assert "model parameter is required" in str(exc_info.value.detail) + response = await count_tokens(mock_request, mock_user_api_key_dict) - print("✅ Error handling test passed!") + assert response.status_code == 400 + body = json.loads(response.body) + assert "detail" not in body + assert body["type"] == "error" + assert body["error"]["type"] == "invalid_request_error" + assert "model parameter is required" in body["error"]["message"] finally: anthropic_endpoints._read_request_body = original_read_request_body @@ -1362,3 +1362,123 @@ async def mock_token_counter_error(request, call_endpoint=False): finally: anthropic_endpoints._read_request_body = original_read_request_body proxy_server.token_counter = original_token_counter + + +async def _invoke_count_tokens_with_body(body: dict): + """Drive anthropic_count_tokens with a stubbed request body.""" + import litellm.proxy.anthropic_endpoints.endpoints as anthropic_endpoints + + mock_request = MagicMock(spec=Request) + + async def mock_read_request_body(request): + return body + + original_read_request_body = anthropic_endpoints._read_request_body + anthropic_endpoints._read_request_body = mock_read_request_body + try: + return await anthropic_count_tokens(mock_request, MagicMock()) + finally: + anthropic_endpoints._read_request_body = original_read_request_body + + +def _assert_anthropic_error_envelope(response, status_code: int, expected_type: str): + """Top-level Anthropic shape, no FastAPI {"detail": ...} wrapper.""" + assert response.status_code == status_code + body = json.loads(response.body) + assert "detail" not in body, f"unexpected FastAPI envelope wrap: {body}" + assert body["type"] == "error" + assert body["error"]["type"] == expected_type + assert isinstance(body["error"]["message"], str) + assert body["error"]["message"] + return body + + +@pytest.mark.asyncio +async def test_count_tokens_missing_model_returns_anthropic_400(): + """Missing `model` must produce an Anthropic-shaped 400, not an OpenAI envelope. + + Regression for the same bug class as Greptile P2 on #30385: every error + path out of /v1/messages/count_tokens must match Anthropic's + {"type": "error", "error": {"type": ..., "message": ...}} schema. + """ + response = await _invoke_count_tokens_with_body( + {"messages": [{"role": "user", "content": "hi"}]} + ) + body = _assert_anthropic_error_envelope(response, 400, "invalid_request_error") + assert "model parameter is required" in body["error"]["message"] + + +@pytest.mark.asyncio +async def test_count_tokens_missing_messages_returns_anthropic_400(): + response = await _invoke_count_tokens_with_body({"model": "claude-haiku-4-5"}) + body = _assert_anthropic_error_envelope(response, 400, "invalid_request_error") + assert "messages parameter is required" in body["error"]["message"] + + +@pytest.mark.asyncio +async def test_count_tokens_httpexception_returns_anthropic_envelope(): + """HTTPException raised by internal_token_counter must convert to Anthropic shape. + + Before the fix, `except HTTPException: raise` re-raised, leaving FastAPI's + {"detail": ...} envelope visible to the Anthropic SDK client. + """ + import litellm.proxy.anthropic_endpoints.endpoints as anthropic_endpoints + import litellm.proxy.proxy_server as proxy_server + + mock_request = MagicMock(spec=Request) + + async def mock_read_request_body(request): + return { + "model": "claude-haiku-4-5", + "messages": [{"role": "user", "content": "hi"}], + } + + async def mock_token_counter_http(request, call_endpoint=False): + raise HTTPException(status_code=503, detail={"error": "downstream unavailable"}) + + original_read = anthropic_endpoints._read_request_body + original_counter = proxy_server.token_counter + anthropic_endpoints._read_request_body = mock_read_request_body + proxy_server.token_counter = mock_token_counter_http + try: + response = await anthropic_count_tokens(mock_request, MagicMock()) + body = _assert_anthropic_error_envelope(response, 503, "api_error") + assert "downstream unavailable" in body["error"]["message"] + finally: + anthropic_endpoints._read_request_body = original_read + proxy_server.token_counter = original_counter + + +@pytest.mark.asyncio +async def test_count_tokens_generic_exception_returns_anthropic_500(): + """Regression for Greptile finding on PR #30385 at commit 40ecda47. + + The bare `except Exception` fallback previously re-raised as + `HTTPException(500, detail={"error": ...})`, producing the FastAPI + {"detail": ...} wrapper. It must return an Anthropic-shaped 500. + """ + import litellm.proxy.anthropic_endpoints.endpoints as anthropic_endpoints + import litellm.proxy.proxy_server as proxy_server + + mock_request = MagicMock(spec=Request) + + async def mock_read_request_body(request): + return { + "model": "claude-haiku-4-5", + "messages": [{"role": "user", "content": "hi"}], + } + + async def mock_token_counter_boom(request, call_endpoint=False): + raise RuntimeError("unexpected boom inside counter") + + original_read = anthropic_endpoints._read_request_body + original_counter = proxy_server.token_counter + anthropic_endpoints._read_request_body = mock_read_request_body + proxy_server.token_counter = mock_token_counter_boom + try: + response = await anthropic_count_tokens(mock_request, MagicMock()) + body = _assert_anthropic_error_envelope(response, 500, "api_error") + assert "unexpected boom inside counter" in body["error"]["message"] + finally: + anthropic_endpoints._read_request_body = original_read + proxy_server.token_counter = original_counter From 6ec2ad0e91990e0aa09953aed8fa223bbd083664 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Mon, 22 Jun 2026 03:52:21 +0000 Subject: [PATCH 7/9] fix(count_tokens): guard ProxyException.message=None on the count_tokens path Greptile P1 on commit b9c17be1: the new `except ProxyException` branch in `count_tokens` passes `e.message` directly to `_anthropic_error_response`, which forwards it to `_strip_litellm_wrapper_prefixes`. When `ProxyException` is constructed with `message=None`, that None reaches `re.sub` and raises TypeError, crashing the error handler and producing an unhandled 500 with no Anthropic-shaped body. The same scenario already has an explicit guard on the `/v1/messages` `anthropic_response()` path (introduced earlier in this PR); the `count_tokens` sibling needs the same. Coerce `e.message is None -> str(e)` before handing off. Adds a regression test that constructs a ProxyException with `.message = None` and asserts the handler returns a well-formed Anthropic envelope at the expected status code. --- .../proxy/anthropic_endpoints/endpoints.py | 3 +- .../test_proxy_token_counter.py | 48 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 4118ee5dabc1..cd73116a568c 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -321,7 +321,8 @@ async def count_tokens( return _anthropic_error_response(e.status_code, raw_message) except ProxyException as e: status_code = int(e.code) if e.code and e.code.isdigit() else 500 - return _anthropic_error_response(status_code, e.message) + raw_message = e.message if e.message is not None else str(e) + return _anthropic_error_response(status_code, raw_message) except Exception as e: verbose_proxy_logger.exception( "litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {}".format( diff --git a/tests/proxy_unit_tests/test_proxy_token_counter.py b/tests/proxy_unit_tests/test_proxy_token_counter.py index 46a2c902a46e..55d45855b2ec 100644 --- a/tests/proxy_unit_tests/test_proxy_token_counter.py +++ b/tests/proxy_unit_tests/test_proxy_token_counter.py @@ -1482,3 +1482,51 @@ async def mock_token_counter_boom(request, call_endpoint=False): finally: anthropic_endpoints._read_request_body = original_read proxy_server.token_counter = original_counter + + +@pytest.mark.asyncio +async def test_count_tokens_proxyexception_none_message_falls_back_to_str(): + """Regression for Greptile P1 on PR #30385 (commit b9c17be1). + + `ProxyException` is sometimes constructed with `message=None`. Passing + that None straight to `transform_to_anthropic_error` reaches + `_strip_litellm_wrapper_prefixes(None)` where `re.sub` raises + `TypeError: expected string or bytes-like object`, crashing the + handler and emitting an unhandled 500 with no Anthropic-shaped body. + + The handler must coerce `None -> str(e)`, mirroring the same guard + already applied on the `anthropic_response()` /v1/messages path. + """ + import litellm.proxy.anthropic_endpoints.endpoints as anthropic_endpoints + import litellm.proxy.proxy_server as proxy_server + + mock_request = MagicMock(spec=Request) + + async def mock_read_request_body(request): + return { + "model": "claude-haiku-4-5", + "messages": [{"role": "user", "content": "hi"}], + } + + async def mock_token_counter_none_message(request, call_endpoint=False): + exc = ProxyException( + message="placeholder", + type="token_counting_error", + param="model", + code=429, + ) + exc.message = None # the exact failure mode greptile flagged + raise exc + + original_read = anthropic_endpoints._read_request_body + original_counter = proxy_server.token_counter + anthropic_endpoints._read_request_body = mock_read_request_body + proxy_server.token_counter = mock_token_counter_none_message + try: + response = await anthropic_count_tokens(mock_request, MagicMock()) + body = _assert_anthropic_error_envelope(response, 429, "rate_limit_error") + # Must not crash; must yield a non-empty Anthropic-shaped message. + assert body["error"]["message"] + finally: + anthropic_endpoints._read_request_body = original_read + proxy_server.token_counter = original_counter From c2d95c06c5f0166f90ae5a78ce5111a1ddcf99e3 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Tue, 23 Jun 2026 07:51:58 +0000 Subject: [PATCH 8/9] chore: add Co-authored-by trailer for attribution Co-authored-by: songkuan-zheng From d253e8766b756e68d1975c0b1950a7676e8b6d11 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:44:19 +0000 Subject: [PATCH 9/9] chore: ruff format reformat for upstream lint gate `ruff format --check` (delta-vs-base lint gate landed on litellm_internal_staging while this PR was open) flagged 3 line-wrapped calls in this PR's diff. Apply the formatter's canonical layout to satisfy the gate. No behavior change. Co-authored-by: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Co-authored-by: songkuan-zheng --- .../exceptions/exception_mapping_utils.py | 12 +++--------- litellm/proxy/anthropic_endpoints/endpoints.py | 8 ++------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index d4221461e32c..25381305f4fc 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -71,9 +71,7 @@ "Watsonx", "XAI", ) -_PROVIDER_EXCEPTION_PREFIX = re.compile( - r"^\s*(?:" + "|".join(_PROVIDER_EXCEPTION_NAMES) + r")Exception\s*-\s*" -) +_PROVIDER_EXCEPTION_PREFIX = re.compile(r"^\s*(?:" + "|".join(_PROVIDER_EXCEPTION_NAMES) + r")Exception\s*-\s*") # HTTP status code -> Anthropic error type @@ -173,9 +171,7 @@ def extract_error_message(raw_message: str) -> str: """ parsed = safe_json_loads(raw_message) if isinstance(parsed, dict): - return AnthropicExceptionMapping._extract_message_from_dict( - parsed, raw_message - ) + return AnthropicExceptionMapping._extract_message_from_dict(parsed, raw_message) return raw_message @staticmethod @@ -247,9 +243,7 @@ def transform_to_anthropic_error( """ # Strip LiteLLM/provider wrapper prefixes so an embedded upstream # Anthropic error body can be detected and passed through unchanged. - raw_message = AnthropicExceptionMapping._strip_litellm_wrapper_prefixes( - raw_message - ) + raw_message = AnthropicExceptionMapping._strip_litellm_wrapper_prefixes(raw_message) # Try to parse as JSON once. parsed: Optional[dict] = safe_json_loads(raw_message) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index cd73116a568c..73e5f695f3b6 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -313,9 +313,7 @@ async def count_tokens( except HTTPException as e: if isinstance(e.detail, dict): - raw_message = str( - e.detail.get("error") or e.detail.get("message") or e.detail - ) + raw_message = str(e.detail.get("error") or e.detail.get("message") or e.detail) else: raw_message = str(e.detail) return _anthropic_error_response(e.status_code, raw_message) @@ -325,9 +323,7 @@ async def count_tokens( return _anthropic_error_response(status_code, raw_message) except Exception as e: verbose_proxy_logger.exception( - "litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {}".format( - str(e) - ) + "litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {}".format(str(e)) ) return _anthropic_error_response(500, str(e))