diff --git a/litellm/__init__.py b/litellm/__init__.py index 0d6a788e368f..957d25b3cb6f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -235,6 +235,17 @@ def _dev_env_hot_reload_enabled() -> bool: use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API +# When True, strip the OpenAI-flavored `usage.total_tokens` field that +# LiteLLM injects into non-streaming /v1/messages responses, bringing the +# wire response into line with the Anthropic spec (matches the streaming +# SSE path, which already omits total_tokens). Default False to preserve +# backward compatibility for clients that read the LiteLLM-shaped +# `usage.total_tokens` today. Planned to flip to True in a future major +# release; opt in early via Python: +# `litellm.strip_anthropic_total_tokens = True` +# Or via `litellm_settings.strip_anthropic_total_tokens: true` in +# config.yaml. +strip_anthropic_total_tokens: bool = False route_all_chat_openai_to_responses: bool = ( os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1995ff275c9d..856b788b54b9 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse +import litellm from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping from litellm.integrations.custom_guardrail import ModifyResponseException @@ -23,6 +24,40 @@ router = APIRouter() +def _strip_total_tokens_from_anthropic_response(response: Any) -> None: + """Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM + injects into Anthropic /v1/messages responses. + + The Anthropic /v1/messages spec only defines: + input_tokens, output_tokens, cache_creation_input_tokens, + cache_read_input_tokens, cache_creation.{ephemeral_5m,ephemeral_1h} + The streaming SSE path (message_delta.usage) already does not include + total_tokens; this brings the non-streaming path into the same shape. + + Handles both shapes returned by `base_process_llm_request`: + - plain `dict` (most common — `AnthropicMessagesResponse` is a TypedDict + and is `dict` at runtime) + - Pydantic model whose `usage` attribute is dict-shaped (e.g. a + BaseModel that holds raw Anthropic usage as a `dict[str, int]`) + + Streaming results (StreamingResponse, AsyncIterator, etc.) and Pydantic + models with strongly-typed Usage sub-models are left untouched — + those paths either have separate serialization handling or impose + type constraints the helper does not try to subvert. + """ + if response is None: + return + if isinstance(response, dict): + usage = response.get("usage") + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) + return + # Pydantic-model fallback: only mutate if `usage` is a dict. + usage = getattr(response, "usage", None) + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) + + @router.post( "/v1/messages", tags=["[beta] Anthropic `/v1/messages`"], @@ -72,6 +107,18 @@ async def anthropic_response( user_api_base=user_api_base, version=version, ) + # Optionally strip the non-Anthropic `usage.total_tokens` field + # LiteLLM adds internally. Anthropic's official /v1/messages spec + # only defines input_tokens / output_tokens / cache_*_input_tokens; + # total_tokens is an OpenAI convention. Default off + # (`litellm.strip_anthropic_total_tokens = False`) to preserve + # backward compatibility for clients that currently read it; set + # to True to align the wire response with the spec (and with the + # streaming SSE path, which already omits total_tokens). + # spend_logs / Prometheus still compute total internally — this + # only affects the wire response. + if litellm.strip_anthropic_total_tokens: + _strip_total_tokens_from_anthropic_response(result) return result except ModifyResponseException as e: # Guardrail flagged content in passthrough mode - return 200 with violation message diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index f6189382d741..a4da4587b7f5 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -86,3 +86,96 @@ def test_event_logging_batch_endpoint_exists(self): assert response.status_code == 200 assert response.json() == {"status": "ok"} + + +class TestStripTotalTokens(unittest.TestCase): + """Cover ``_strip_total_tokens_from_anthropic_response``. + + The Anthropic /v1/messages spec does not define ``usage.total_tokens``. + LiteLLM injects it internally; the helper must remove it from the wire + response so the non-streaming path matches the streaming SSE shape and + direct Anthropic API responses. + """ + + def test_strips_total_tokens_when_present(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = { + "id": "msg_123", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + }, + } + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response["usage"] + assert response["usage"]["input_tokens"] == 100 + assert response["usage"]["output_tokens"] == 50 + assert response["usage"]["cache_read_input_tokens"] == 0 + + def test_no_op_when_total_tokens_absent(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"usage": {"input_tokens": 100, "output_tokens": 50}} + _strip_total_tokens_from_anthropic_response(response) + assert response["usage"] == {"input_tokens": 100, "output_tokens": 50} + + def test_no_op_when_usage_missing(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"id": "msg_123"} + _strip_total_tokens_from_anthropic_response(response) + assert response == {"id": "msg_123"} + + def test_no_op_on_non_dict_response(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # Streaming responses (StreamingResponse, async iterators) are not dicts. + # The helper must not raise or attempt to mutate them. + for value in (None, "stream", 42, [{"usage": {"total_tokens": 1}}]): + _strip_total_tokens_from_anthropic_response(value) # no raise + + def test_strips_total_tokens_on_pydantic_model_with_dict_usage(self): + """Greptile P1 on #30382: helper must not silently no-op when the + response is a Pydantic-shaped object whose `usage` attribute is a + plain dict (the common case for objects wrapping raw upstream JSON). + """ + from types import SimpleNamespace + + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # SimpleNamespace mimics the .usage attribute access pattern; the + # helper's contract: if .usage is dict-shaped, strip total_tokens. + response = SimpleNamespace( + usage={"input_tokens": 100, "output_tokens": 50, "total_tokens": 150} + ) + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response.usage + assert response.usage == {"input_tokens": 100, "output_tokens": 50} + + +class TestStripTotalTokensFeatureFlag(unittest.TestCase): + """The strip is gated behind `litellm.strip_anthropic_total_tokens`. + + Default off (backward compat). Greptile P1 on #30382 required a + user-controlled flag so existing clients reading the LiteLLM-shaped + `usage.total_tokens` continue to work after this PR lands. + """ + + def test_flag_defaults_off(self): + import litellm + + assert litellm.strip_anthropic_total_tokens is False