From e5bbfca333381b061eed766f9a85d41143e0361b Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Sat, 13 Jun 2026 16:43:21 +0000 Subject: [PATCH 1/4] fix(anthropic): strip LiteLLM-injected total_tokens from /v1/messages response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-streaming /v1/messages response carries a LiteLLM-injected usage.total_tokens = input_tokens + output_tokens that is not part of the Anthropic API spec. This caused three problems: 1. Shape divergence with streaming on the same endpoint. message_delta.usage in the SSE path never carries total_tokens. Clients parsing both paths get two different schemas from one endpoint. 2. Shape divergence with upstream. Direct calls to https://api.anthropic.com/v1/messages return no total_tokens field, so clients using the official Anthropic SDK couldn't rely on it, and clients that did rely on the LiteLLM-injected one broke when bypassing the proxy. 3. Numerical misuse. total = input + output undercounts when cache_read_input_tokens and cache_creation_input_tokens are non-zero, because cache tokens are reported in their own fields. A 100k-token cached prompt with 1 non-cache input token + 200 output tokens reports total_tokens = 201, off by ~99.8% from any reasonable definition of "total." Fix: add _strip_total_tokens_from_anthropic_response in litellm/proxy/anthropic_endpoints/endpoints.py and invoke it in the success path of anthropic_response right before returning. Only mutates dict-shaped responses; streaming (which already lacks the field) is left untouched. spend_logs / Prometheus continue to compute total_tokens internally for billing — this fix only strips the field from the wire response. Scope: only the Anthropic passthrough endpoint /v1/messages. The OpenAI-shape /v1/chat/completions is unaffected. --- .../proxy/anthropic_endpoints/endpoints.py | 32 ++++++++++ .../anthropic_endpoints/test_endpoints.py | 59 +++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1995ff275c9d..cdcc4298f4db 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -23,6 +23,26 @@ router = APIRouter() +def _strip_total_tokens_from_anthropic_response(response: Any) -> None: + """Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM + injects into Anthropic /v1/messages responses. + + The Anthropic /v1/messages spec only defines: + input_tokens, output_tokens, cache_creation_input_tokens, + cache_read_input_tokens, cache_creation.{ephemeral_5m,ephemeral_1h} + The streaming SSE path (message_delta.usage) already does not include + total_tokens; this brings the non-streaming path into the same shape. + + Only mutates dict-shaped responses. Streaming (StreamingResponse, + AsyncIterator, etc.) is left untouched. + """ + if not isinstance(response, dict): + return + usage = response.get("usage") + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) + + @router.post( "/v1/messages", tags=["[beta] Anthropic `/v1/messages`"], @@ -72,6 +92,18 @@ async def anthropic_response( user_api_base=user_api_base, version=version, ) + # Strip the non-Anthropic `usage.total_tokens` field LiteLLM adds + # internally. Anthropic's official /v1/messages spec only defines + # input_tokens / output_tokens / cache_*_input_tokens; total_tokens + # is an OpenAI convention. Keeping it here causes: + # - Inconsistency with the streaming SSE path (message_delta.usage + # never carries total_tokens) + # - Inconsistency with direct gateway / Anthropic API responses + # - Numerical misuse: total = input + output undercounts when + # cache_read/creation tokens are present + # spend_logs / Prometheus still compute total internally — this only + # strips it from the wire response to clients. + _strip_total_tokens_from_anthropic_response(result) return result except ModifyResponseException as e: # Guardrail flagged content in passthrough mode - return 200 with violation message diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index f6189382d741..febe8a601bfb 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -86,3 +86,62 @@ def test_event_logging_batch_endpoint_exists(self): assert response.status_code == 200 assert response.json() == {"status": "ok"} + + +class TestStripTotalTokens(unittest.TestCase): + """Cover ``_strip_total_tokens_from_anthropic_response``. + + The Anthropic /v1/messages spec does not define ``usage.total_tokens``. + LiteLLM injects it internally; the helper must remove it from the wire + response so the non-streaming path matches the streaming SSE shape and + direct Anthropic API responses. + """ + + def test_strips_total_tokens_when_present(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = { + "id": "msg_123", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + }, + } + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response["usage"] + assert response["usage"]["input_tokens"] == 100 + assert response["usage"]["output_tokens"] == 50 + assert response["usage"]["cache_read_input_tokens"] == 0 + + def test_no_op_when_total_tokens_absent(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"usage": {"input_tokens": 100, "output_tokens": 50}} + _strip_total_tokens_from_anthropic_response(response) + assert response["usage"] == {"input_tokens": 100, "output_tokens": 50} + + def test_no_op_when_usage_missing(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"id": "msg_123"} + _strip_total_tokens_from_anthropic_response(response) + assert response == {"id": "msg_123"} + + def test_no_op_on_non_dict_response(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # Streaming responses (StreamingResponse, async iterators) are not dicts. + # The helper must not raise or attempt to mutate them. + for value in (None, "stream", 42, [{"usage": {"total_tokens": 1}}]): + _strip_total_tokens_from_anthropic_response(value) # no raise From fca9affcd0aa84a15f343c700bfc540c348d3b78 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Tue, 16 Jun 2026 04:29:31 +0000 Subject: [PATCH 2/4] fix(anthropic): gate total_tokens strip behind flag + handle Pydantic .usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two P1 greptile threads on #30382: P1 — **Backwards-incompatible removal without a feature flag** Stripping `usage.total_tokens` unconditionally breaks any client currently reading the LiteLLM-shaped non-streaming /v1/messages response. Per the codebase's policy (mirrors #30418), gate behind a new flag. - `litellm.strip_anthropic_total_tokens: bool = False` (default — backward-compat: clients keep seeing total_tokens). - Env override: `LITELLM_STRIP_ANTHROPIC_TOTAL_TOKENS=true`. - Docstring: planned to flip to True in a future major release; opt in early. P1 — **Silent no-op if `result` is a Pydantic model** `base_process_llm_request` may return a Pydantic-style object whose `.usage` is a plain dict (the most common shape — e.g. objects wrapping raw upstream JSON). The original `isinstance(response, dict)` guard skipped strip on those, so `total_tokens` would still hit the wire. Helper now also reads `getattr(response, "usage", None)` and strips when that's a dict. Strongly-typed Pydantic `Usage` sub-models with required `total_tokens` fields are still skipped — those impose type constraints the helper doesn't try to subvert. Tests: - `test_strips_total_tokens_on_pydantic_model_with_dict_usage` - `test_flag_defaults_off` 8/8 pass locally. --- litellm/__init__.py | 10 ++++ .../proxy/anthropic_endpoints/endpoints.py | 47 ++++++++++++------- .../anthropic_endpoints/test_endpoints.py | 34 ++++++++++++++ 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 0d6a788e368f..a803cebd54f7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -235,6 +235,16 @@ def _dev_env_hot_reload_enabled() -> bool: use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API +# When True, strip the OpenAI-flavored `usage.total_tokens` field that +# LiteLLM injects into non-streaming /v1/messages responses, bringing the +# wire response into line with the Anthropic spec (matches the streaming +# SSE path, which already omits total_tokens). Default False to preserve +# backward compatibility for clients that read the LiteLLM-shaped +# `usage.total_tokens` today. Planned to flip to True in a future major +# release; opt in early with `litellm.strip_anthropic_total_tokens = True`. +strip_anthropic_total_tokens: bool = ( + os.getenv("LITELLM_STRIP_ANTHROPIC_TOTAL_TOKENS", "false").lower() == "true" +) route_all_chat_openai_to_responses: bool = ( os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index cdcc4298f4db..856b788b54b9 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse +import litellm from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping from litellm.integrations.custom_guardrail import ModifyResponseException @@ -33,12 +34,26 @@ def _strip_total_tokens_from_anthropic_response(response: Any) -> None: The streaming SSE path (message_delta.usage) already does not include total_tokens; this brings the non-streaming path into the same shape. - Only mutates dict-shaped responses. Streaming (StreamingResponse, - AsyncIterator, etc.) is left untouched. + Handles both shapes returned by `base_process_llm_request`: + - plain `dict` (most common — `AnthropicMessagesResponse` is a TypedDict + and is `dict` at runtime) + - Pydantic model whose `usage` attribute is dict-shaped (e.g. a + BaseModel that holds raw Anthropic usage as a `dict[str, int]`) + + Streaming results (StreamingResponse, AsyncIterator, etc.) and Pydantic + models with strongly-typed Usage sub-models are left untouched — + those paths either have separate serialization handling or impose + type constraints the helper does not try to subvert. """ - if not isinstance(response, dict): + if response is None: + return + if isinstance(response, dict): + usage = response.get("usage") + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) return - usage = response.get("usage") + # Pydantic-model fallback: only mutate if `usage` is a dict. + usage = getattr(response, "usage", None) if isinstance(usage, dict) and "total_tokens" in usage: usage.pop("total_tokens", None) @@ -92,18 +107,18 @@ async def anthropic_response( user_api_base=user_api_base, version=version, ) - # Strip the non-Anthropic `usage.total_tokens` field LiteLLM adds - # internally. Anthropic's official /v1/messages spec only defines - # input_tokens / output_tokens / cache_*_input_tokens; total_tokens - # is an OpenAI convention. Keeping it here causes: - # - Inconsistency with the streaming SSE path (message_delta.usage - # never carries total_tokens) - # - Inconsistency with direct gateway / Anthropic API responses - # - Numerical misuse: total = input + output undercounts when - # cache_read/creation tokens are present - # spend_logs / Prometheus still compute total internally — this only - # strips it from the wire response to clients. - _strip_total_tokens_from_anthropic_response(result) + # Optionally strip the non-Anthropic `usage.total_tokens` field + # LiteLLM adds internally. Anthropic's official /v1/messages spec + # only defines input_tokens / output_tokens / cache_*_input_tokens; + # total_tokens is an OpenAI convention. Default off + # (`litellm.strip_anthropic_total_tokens = False`) to preserve + # backward compatibility for clients that currently read it; set + # to True to align the wire response with the spec (and with the + # streaming SSE path, which already omits total_tokens). + # spend_logs / Prometheus still compute total internally — this + # only affects the wire response. + if litellm.strip_anthropic_total_tokens: + _strip_total_tokens_from_anthropic_response(result) return result except ModifyResponseException as e: # Guardrail flagged content in passthrough mode - return 200 with violation message diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index febe8a601bfb..a4da4587b7f5 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -145,3 +145,37 @@ def test_no_op_on_non_dict_response(self): # The helper must not raise or attempt to mutate them. for value in (None, "stream", 42, [{"usage": {"total_tokens": 1}}]): _strip_total_tokens_from_anthropic_response(value) # no raise + + def test_strips_total_tokens_on_pydantic_model_with_dict_usage(self): + """Greptile P1 on #30382: helper must not silently no-op when the + response is a Pydantic-shaped object whose `usage` attribute is a + plain dict (the common case for objects wrapping raw upstream JSON). + """ + from types import SimpleNamespace + + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # SimpleNamespace mimics the .usage attribute access pattern; the + # helper's contract: if .usage is dict-shaped, strip total_tokens. + response = SimpleNamespace( + usage={"input_tokens": 100, "output_tokens": 50, "total_tokens": 150} + ) + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response.usage + assert response.usage == {"input_tokens": 100, "output_tokens": 50} + + +class TestStripTotalTokensFeatureFlag(unittest.TestCase): + """The strip is gated behind `litellm.strip_anthropic_total_tokens`. + + Default off (backward compat). Greptile P1 on #30382 required a + user-controlled flag so existing clients reading the LiteLLM-shaped + `usage.total_tokens` continue to work after this PR lands. + """ + + def test_flag_defaults_off(self): + import litellm + + assert litellm.strip_anthropic_total_tokens is False From ca8351f18db5b5032ca2c507dbfdf66cc5766df0 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Tue, 16 Jun 2026 04:46:29 +0000 Subject: [PATCH 3/4] fix(anthropic): drop env var for strip flag (docs CI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors #30418's pattern (`expose_router_debug_in_errors: bool = True`, no `os.getenv`). The `LITELLM_STRIP_ANTHROPIC_TOTAL_TOKENS` env var introduced in the prior commit was flagged by `tests/documentation_tests/test_env_keys.py` because the documentation file `docs/my-website/docs/proxy/config_settings.md` lives in `BerriAI/litellm-docs` (separate repo) and registering a new env key requires a parallel docs PR — a friction we avoid here by exposing the flag only as a Python attribute + `litellm_settings` config key, both of which load through the existing proxy config plumbing without needing the env-var registry to be updated. No semantic change: default still False, behavior identical when set via `litellm.strip_anthropic_total_tokens = True` or `litellm_settings.strip_anthropic_total_tokens: true` in config.yaml. Verified locally: env scan no longer surfaces the key; 8/8 tests pass. --- litellm/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index a803cebd54f7..957d25b3cb6f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -241,10 +241,11 @@ def _dev_env_hot_reload_enabled() -> bool: # SSE path, which already omits total_tokens). Default False to preserve # backward compatibility for clients that read the LiteLLM-shaped # `usage.total_tokens` today. Planned to flip to True in a future major -# release; opt in early with `litellm.strip_anthropic_total_tokens = True`. -strip_anthropic_total_tokens: bool = ( - os.getenv("LITELLM_STRIP_ANTHROPIC_TOTAL_TOKENS", "false").lower() == "true" -) +# release; opt in early via Python: +# `litellm.strip_anthropic_total_tokens = True` +# Or via `litellm_settings.strip_anthropic_total_tokens: true` in +# config.yaml. +strip_anthropic_total_tokens: bool = False route_all_chat_openai_to_responses: bool = ( os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge From 198f966da86795381dc3dc1b36c1e316cd684593 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Wed, 17 Jun 2026 05:44:31 +0000 Subject: [PATCH 4/4] ci: retrigger workflows after base branch change to litellm_internal_staging